diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cd33e6a..ebef778f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,6 +77,16 @@ jobs: --health-timeout=5s --health-retries=5 + mongodb: + image: mongo:7 + ports: + - 27017:27017 + options: >- + --health-cmd="mongosh --eval 'db.runCommand({ ping: 1 })' || exit 1" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + env: REAL_POSTGRES_HOST: localhost REAL_POSTGRES_PORT: 5432 @@ -116,6 +126,12 @@ jobs: REAL_CLICKHOUSE_DATABASE: testdb REAL_CLICKHOUSE_SSL: "false" + REAL_MONGODB_HOST: localhost + REAL_MONGODB_PORT: 27017 + REAL_MONGODB_DATABASE: testdb + REAL_MONGODB_URI: mongodb://localhost:27017/testdb + MONGO_TEST_URI: mongodb://localhost:27017 + # SQLite — no env vars needed; tests use a temp file steps: @@ -154,6 +170,10 @@ jobs: run: | node bridge/scripts/seed-clickhouse.cjs + - name: Seed MongoDB database + run: | + node bridge/scripts/seed-mongodb.cjs + - name: Run tests run: | cd bridge diff --git a/FEATURES.md b/FEATURES.md index 888cc793..a36806cb 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -2,7 +2,7 @@ This document provides a comprehensive breakdown of all features and capabilities in RelWave. For installation and setup instructions, see the main [README](README.md). -**Tech Stack:** Tauri + React 19 + TypeScript + Tailwind CSS + shadcn/ui + React Query + ReactFlow + Recharts + CodeMirror + better-sqlite3 + DuckDB + ClickHouse +**Tech Stack:** Tauri + React 19 + TypeScript + Tailwind CSS + shadcn/ui + React Query + ReactFlow + Recharts + CodeMirror + better-sqlite3 + DuckDB + ClickHouse + MongoDB --- @@ -31,6 +31,19 @@ This document provides a comprehensive breakdown of all features and capabilitie | SQLite | Local file path | ✅ | ✅ | ✅ | ✅ | ✅ | | DuckDB | Local file path | ✅ | ✅ | ✅ | ✅ | ✅ | | ClickHouse | Host/port + SSL | ✅ | ✅ | ✅ | ✅ | ✅ | +| MongoDB | URI / Host/port | ✅ | ✅ | ✅ | ✅ | ✅ | + +### MongoDB-Specific Features + +- **Document & Analytical Engine** — powered by official `mongodb` Node driver (^6.21.0), verified on MongoDB 6.0 and 7.0 +- **Flexible Connection Topologies** — connect via Standalone (`mongodb://`), Replica Sets (`replicaSet=rs0`), and MongoDB Atlas (`mongodb+srv://`) +- **Secure Credentials & TLS** — OS-keyring protected authentication (SCRAM-SHA-1, SCRAM-SHA-256) with full TLS/SSL certificate support +- **MQL Workspace & Visual Aggregations** — interactive aggregation pipeline builder with stage reordering, stage previews, and live query cancellation +- **Strict BSON Fidelity** — canonical EJSON serialization preserving ObjectId, Date, Decimal128, Long (64-bit int), and Binary data +- **Schema Intelligence & Visual ERD** — probabilistic schema inference with stratified sampling, field type distribution charts, and automatic relationship detection (ObjectId refs, DBRefs, and embedded arrays) +- **Streaming Bulk Operations** — chunked mutations, document imports, and streaming exports (JSON, NDJSON, CSV) with automatic 2-hour orphan purge +- **Safe Schema & Document Migrations** — dry-run previews, mutex lock acquisition (`_relwave_migrations_lock`), and auto-generated rollback scripts +- **Production Safety Controls** — read-only connection enforcement and safe MQL operator blacklisting (`$where`, `$function`, `$accumulator`, `$out`, `$merge`) ### ClickHouse-Specific Features diff --git a/README.md b/README.md index 0646fc6a..b7050898 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ _RelWave brings the power of native Git versioning, visual ER diagrams, and seamless schema management into one blazingly fast desktop application. Built for developers who demand more._ -[![Version](https://img.shields.io/badge/version-1.2.0-0066ff?style=for-the-badge&logo=semver)](https://github.com/Relwave/relwave-app/releases) +[![Version](https://img.shields.io/badge/version-1.3.0-0066ff?style=for-the-badge&logo=semver)](https://github.com/Relwave/relwave-app/releases) [![License](https://img.shields.io/badge/license-MIT-00cc66?style=for-the-badge)](LICENSE) [![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux-1f2937?style=for-the-badge&logo=windows)](https://github.com/Relwave/relwave-app/releases) [![Tauri](https://img.shields.io/badge/built%20with-Tauri-FFC131?style=for-the-badge&logo=tauri)](https://tauri.app/) @@ -40,7 +40,7 @@ _RelWave brings the power of native Git versioning, visual ER diagrams, and seam Native Core

- Direct connections via native drivers (PostgreSQL, MySQL, SQLite, DuckDB, ClickHouse). Zero browser overhead. Pure speed. + Direct connections via native drivers (PostgreSQL, MySQL, SQLite, DuckDB, ClickHouse, MongoDB). Zero browser overhead. Pure speed.
@@ -84,7 +84,7 @@ RelWave is built using a modern, high-performance tech stack ensuring both a but

- +

Powered by Tauri, React 19, and a high-speed Node.js Bridge. @@ -133,7 +133,7 @@ RelWave uses a **three-layer bridge architecture**. React owns the user experien image -The request path is **React UI -> Tauri command -> Node.js JSON-RPC handler -> service -> database, Git, local storage, monitoring, or LLM provider**. Responses return through the same boundary in reverse. Long-running SQL queries additionally stream `query.result`, `query.progress`, and `query.done` notifications back to the React client. The Node.js bridge contains the database connectors (PostgreSQL, MySQL/MariaDB, SQLite, DuckDB, ClickHouse), Git service, AI services, monitoring services, project stores, credential access, and SSH tunnel handling. +The request path is **React UI -> Tauri command -> Node.js JSON-RPC handler -> service -> database, Git, local storage, monitoring, or LLM provider**. Responses return through the same boundary in reverse. Long-running SQL queries additionally stream `query.result`, `query.progress`, and `query.done` notifications back to the React client. The Node.js bridge contains the database connectors (PostgreSQL, MySQL/MariaDB, SQLite, DuckDB, ClickHouse, MongoDB), Git service, AI services, monitoring services, project stores, credential access, and SSH tunnel handling.
diff --git a/bridge/__tests__/connectors/mongoAggregation.test.ts b/bridge/__tests__/connectors/mongoAggregation.test.ts new file mode 100644 index 00000000..6e79573b --- /dev/null +++ b/bridge/__tests__/connectors/mongoAggregation.test.ts @@ -0,0 +1,482 @@ +import { describe, expect, test, jest } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { + assertReadOnlyPipeline, + assertSafeExpressions, + mongoConnector, +} from "../../src/connectors/mongo"; +import { MAX_AGG_STAGES, MAX_EXPRESSION_DEPTH } from "../../src/types/mongo"; +import type { EJSONDocument, MongoConfig } from "../../src/types/mongo"; + +describe("assertReadOnlyPipeline & assertSafeExpressions", () => { + describe("Empty and basic valid pipelines", () => { + test("accepts empty pipeline [] without error", () => { + expect(() => assertReadOnlyPipeline([])).not.toThrow(); + }); + + test("accepts standard read-only stages", () => { + const pipeline: EJSONDocument[] = [ + { $match: { status: "active", age: { $gt: 18 } } }, + { $project: { name: 1, email: 1, _id: 0 } }, + { $sort: { createdAt: -1 } }, + { $limit: 10 }, + { $skip: 5 }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).not.toThrow(); + }); + }); + + describe("Rejection of top-level write stages", () => { + test("rejects $out with MONGO_WRITE_STAGE_FORBIDDEN", () => { + expect(() => assertReadOnlyPipeline([{ $out: "target_coll" }])).toThrow( + expect.objectContaining({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + message: "Stage '$out' is forbidden in read-only aggregation queries", + }), + ); + }); + + test("rejects $merge with MONGO_WRITE_STAGE_FORBIDDEN", () => { + expect(() => assertReadOnlyPipeline([{ $merge: { into: "target_coll" } }])).toThrow( + expect.objectContaining({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + message: "Stage '$merge' is forbidden in read-only aggregation queries", + }), + ); + }); + }); + + describe("Rejection of nested write stages in sub-pipelines", () => { + test("rejects nested $out in $lookup.pipeline", () => { + const pipeline: EJSONDocument[] = [ + { + $lookup: { + from: "orders", + pipeline: [{ $out: "target" }], + as: "orders", + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + message: "Stage '$out' is forbidden in read-only aggregation queries", + }), + ); + }); + + test("rejects nested $merge in $facet sub-pipelines", () => { + const pipeline: EJSONDocument[] = [ + { + $facet: { + categorizedByTags: [{ $match: { tags: "sale" } }], + categorizedByPrice: [{ $merge: { into: "output_coll" } }], + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + message: "Stage '$merge' is forbidden in read-only aggregation queries", + }), + ); + }); + + test("rejects nested $out in $unionWith.pipeline", () => { + const pipeline: EJSONDocument[] = [ + { + $unionWith: { + coll: "archive", + pipeline: [{ $out: "archive_dump" }], + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + message: "Stage '$out' is forbidden in read-only aggregation queries", + }), + ); + }); + }); + + describe("Rejection of forbidden server-side JavaScript expression operators", () => { + test("rejects $function in expressions with MONGO_FORBIDDEN_OPERATOR", () => { + const pipeline: EJSONDocument[] = [ + { + $addFields: { + fn: { + $function: { + body: "function() { return 1; }", + args: [], + lang: "js", + }, + }, + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: + "Operator '$function' is forbidden (server-side JavaScript execution is not permitted)", + }), + ); + }); + + test("rejects $accumulator in $group stage with MONGO_FORBIDDEN_OPERATOR", () => { + const pipeline: EJSONDocument[] = [ + { + $group: { + _id: null, + custom: { + $accumulator: { + init: "function() { return 0; }", + accumulate: "function(s) { return s + 1; }", + }, + }, + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: + "Operator '$accumulator' is forbidden (server-side JavaScript execution is not permitted)", + }), + ); + }); + + test("rejects $where in $match stage with MONGO_FORBIDDEN_OPERATOR", () => { + const pipeline: EJSONDocument[] = [ + { + $match: { + $where: "this.credits === this.debits", + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: + "Operator '$where' is forbidden (server-side JavaScript execution is not permitted)", + }), + ); + }); + + test("rejects forbidden operators nested in $lookup sub-pipelines", () => { + const pipeline: EJSONDocument[] = [ + { + $lookup: { + from: "details", + pipeline: [ + { + $match: { + $where: "this.active === true", + }, + }, + ], + as: "matched", + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: + "Operator '$where' is forbidden (server-side JavaScript execution is not permitted)", + }), + ); + }); + }); + + describe("Validation of malformed stages and sub-pipelines", () => { + test("rejects non-array pipeline with MONGO_INVALID_PIPELINE", () => { + expect(() => assertReadOnlyPipeline("not an array" as any)).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "Pipeline must be an array", + }), + ); + }); + + test("rejects non-object stage with MONGO_INVALID_PIPELINE", () => { + expect(() => assertReadOnlyPipeline([null as any])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "Each pipeline stage must be a non-null object", + }), + ); + expect(() => assertReadOnlyPipeline([[1, 2] as any])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "Each pipeline stage must be a non-null object", + }), + ); + }); + + test("rejects stage with zero keys or multiple keys", () => { + expect(() => assertReadOnlyPipeline([{}])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "Each pipeline stage must have exactly one operator key", + }), + ); + expect(() => assertReadOnlyPipeline([{ $match: {}, $sort: {} }])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "Each pipeline stage must have exactly one operator key", + }), + ); + }); + + test("rejects stage key not starting with $", () => { + expect(() => assertReadOnlyPipeline([{ match: {} }])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "Invalid stage name 'match'; must begin with '$'", + }), + ); + }); + + test("rejects non-array $lookup.pipeline", () => { + const pipeline: EJSONDocument[] = [ + { + $lookup: { + from: "orders", + pipeline: "invalid_string" as any, + as: "orders", + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "$lookup.pipeline must be an array", + }), + ); + }); + + test("rejects non-array $unionWith.pipeline", () => { + const pipeline: EJSONDocument[] = [ + { + $unionWith: { + coll: "archive", + pipeline: { invalid: "obj" } as any, + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "$unionWith.pipeline must be an array", + }), + ); + }); + + test("rejects non-array $facet sub-pipeline", () => { + const pipeline: EJSONDocument[] = [ + { + $facet: { + invalidBranch: 12345 as any, + }, + }, + ]; + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "$facet pipeline for 'invalidBranch' must be an array", + }), + ); + }); + + test("rejects malformed $facet stage specifications", () => { + for (const badFacet of [[], "str", null]) { + expect(() => assertReadOnlyPipeline([{ $facet: badFacet as any }])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "$facet stage specification must be an object", + }), + ); + } + expect(() => assertReadOnlyPipeline([{ $facet: { f: "not-array" } as any }])).toThrow( + expect.objectContaining({ + code: "MONGO_INVALID_PIPELINE", + message: "$facet pipeline for 'f' must be an array", + }), + ); + }); + }); + + describe("Nesting depth and resource limits", () => { + test("rejects pipeline exceeding MAX_AGG_STAGES (64)", () => { + const pipeline = Array.from({ length: 65 }, () => ({ $match: { a: 1 } })); + expect(() => assertReadOnlyPipeline(pipeline)).toThrow( + expect.objectContaining({ + code: "MONGO_LIMIT_EXCEEDED", + message: `Pipeline exceeds ${MAX_AGG_STAGES} stages`, + }), + ); + }); + + test("rejects stage exceeding MAX_AGG_STAGE_SIZE_BYTES (16KB)", () => { + const largeStage = { + $match: { + data: "x".repeat(20 * 1024), + }, + }; + expect(() => assertReadOnlyPipeline([largeStage])).toThrow( + expect.objectContaining({ + code: "MONGO_LIMIT_EXCEEDED", + }), + ); + }); + + test("rejects expression nesting depth > 15 with MONGO_LIMIT_EXCEEDED", () => { + let nested: any = { value: 1 }; + for (let i = 0; i <= MAX_EXPRESSION_DEPTH + 1; i++) { + nested = { child: nested }; + } + expect(() => assertSafeExpressions(nested)).toThrow( + expect.objectContaining({ + code: "MONGO_LIMIT_EXCEEDED", + message: `Expression nesting exceeds maximum depth of ${MAX_EXPRESSION_DEPTH}`, + }), + ); + }); + + test("rejects deeply nested sub-pipelines with MONGO_LIMIT_EXCEEDED", () => { + let inner: any = [{ $match: { final: true } }]; + for (let i = 0; i < 16; i++) { + inner = [ + { + $lookup: { + from: "coll", + pipeline: inner, + as: `as_${i}`, + }, + }, + ]; + } + expect(() => assertReadOnlyPipeline(inner)).toThrow( + expect.objectContaining({ + code: "MONGO_LIMIT_EXCEEDED", + }), + ); + }); + }); + + describe("runAggregation bounded cursor iteration and option validation", () => { + const dummyConfig: MongoConfig = { + uri: "mongodb://localhost:27017/testdb", + database: "testdb", + }; + + test("rejects invalid maxTimeMS (0, negative, non-integer) with MONGO_VALIDATION_ERROR", async () => { + for (const badTimeout of [0, -100, 1.5, NaN, "fast" as any]) { + await expect( + mongoConnector.runAggregation(dummyConfig, { + collection: "orders", + pipeline: [], + options: { maxTimeMS: badTimeout }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "maxTimeMS must be a positive integer", + }); + } + }); + + test("clamps maxTimeMS between 1 and 60000", async () => { + let capturedOptions: any; + const mockCursor = { + [Symbol.asyncIterator]: async function* () {}, + close: jest.fn().mockImplementation(() => Promise.resolve()), + }; + const mockCollection = { + aggregate: jest.fn().mockImplementation((_stages, options) => { + capturedOptions = options; + return mockCursor; + }), + }; + jest.spyOn(MongoClient.prototype, "db").mockReturnValue({ + collection: () => mockCollection, + } as any); + + await mongoConnector.runAggregation(dummyConfig, { + collection: "orders", + pipeline: [], + options: { maxTimeMS: 999999 }, + }); + expect(capturedOptions.maxTimeMS).toBe(60000); + + await mongoConnector.runAggregation(dummyConfig, { + collection: "orders", + pipeline: [], + options: { maxTimeMS: 25000 }, + }); + expect(capturedOptions.maxTimeMS).toBe(25000); + }); + + test("terminates iteration at 1,001 documents, throws MONGO_LIMIT_EXCEEDED, and closes cursor", async () => { + let iteratedCount = 0; + const mockCursor = { + [Symbol.asyncIterator]: async function* () { + for (let i = 1; i <= 1050; i++) { + iteratedCount++; + yield { _id: i, name: `doc_${i}` }; + } + }, + close: jest.fn().mockImplementation(() => Promise.resolve()), + }; + const mockCollection = { + aggregate: jest.fn().mockReturnValue(mockCursor), + }; + jest.spyOn(MongoClient.prototype, "db").mockReturnValue({ + collection: () => mockCollection, + } as any); + + await expect( + mongoConnector.runAggregation(dummyConfig, { + collection: "orders", + pipeline: [{ $match: {} }], + }), + ).rejects.toMatchObject({ + code: "MONGO_LIMIT_EXCEEDED", + message: "aggregation returned more than 1000 documents (max 1000); add a $limit stage", + }); + + expect(iteratedCount).toBe(1001); + expect(mockCursor.close).toHaveBeenCalledTimes(1); + }); + + test("passes validated allowDiskUse and batchSize options to driver cursor", async () => { + let capturedOptions: any; + const mockCursor = { + [Symbol.asyncIterator]: async function* () { + yield { _id: 1 }; + }, + close: jest.fn().mockImplementation(() => Promise.resolve()), + }; + const mockCollection = { + aggregate: jest.fn().mockImplementation((_stages, options) => { + capturedOptions = options; + return mockCursor; + }), + }; + jest.spyOn(MongoClient.prototype, "db").mockReturnValue({ + collection: () => mockCollection, + } as any); + + const res = await mongoConnector.runAggregation(dummyConfig, { + collection: "orders", + pipeline: [], + options: { allowDiskUse: true, batchSize: 50 }, + }); + + expect(capturedOptions.allowDiskUse).toBe(true); + expect(capturedOptions.batchSize).toBe(50); + expect(res.documents).toHaveLength(1); + }); + }); +}); diff --git a/bridge/__tests__/connectors/mongoBsonRoundTrip.test.ts b/bridge/__tests__/connectors/mongoBsonRoundTrip.test.ts new file mode 100644 index 00000000..39ed2783 --- /dev/null +++ b/bridge/__tests__/connectors/mongoBsonRoundTrip.test.ts @@ -0,0 +1,284 @@ +import { describe, expect, test } from "@jest/globals"; +import { Binary } from "mongodb"; +import { + validateAndParseSubtype, + validateAndDecodeBase64, + mongoEJSONtoBSON, + serializeMongoValue, +} from "../../src/connectors/mongoBson"; + +describe("mongoBsonRoundTrip", () => { + describe("Supported Payload Shapes & Aliases", () => { + test("deserializes RelWave wire format { $binary: { base64, subtype } }", () => { + const ejson = { $binary: { base64: "aGVsbG8=", subtype: "00" } }; + const bin = mongoEJSONtoBSON(ejson) as unknown as Binary; + expect(bin).toBeInstanceOf(Binary); + expect(Buffer.from(bin.value()).toString("utf8")).toBe("hello"); + expect(bin.sub_type).toBe(0); + }); + + test("deserializes Canonical EJSON v2 format { $binary: { base64, subType } }", () => { + const ejson = { $binary: { base64: "aGVsbG8=", subType: "04" } }; + const bin = mongoEJSONtoBSON(ejson) as unknown as Binary; + expect(bin).toBeInstanceOf(Binary); + expect(Buffer.from(bin.value()).toString("utf8")).toBe("hello"); + expect(bin.sub_type).toBe(4); + }); + + test("deserializes MongoDB internal format { $binary: { $base64, $type } }", () => { + const ejson = { $binary: { $base64: "aGVsbG8=", $type: "05" } }; + const bin = mongoEJSONtoBSON(ejson) as unknown as Binary; + expect(bin).toBeInstanceOf(Binary); + expect(Buffer.from(bin.value()).toString("utf8")).toBe("hello"); + expect(bin.sub_type).toBe(5); + }); + + test("deserializes compatibility alias format { $binary: { base64, sub_type } }", () => { + const ejson = { $binary: { base64: "aGVsbG8=", sub_type: "80" } }; + const bin = mongoEJSONtoBSON(ejson) as unknown as Binary; + expect(bin).toBeInstanceOf(Binary); + expect(Buffer.from(bin.value()).toString("utf8")).toBe("hello"); + expect(bin.sub_type).toBe(128); + }); + + test("deserializes flat string shorthand { $binary: '' } with default subtype 0", () => { + const ejson = { $binary: "aGVsbG8=" }; + const bin = mongoEJSONtoBSON(ejson) as unknown as Binary; + expect(bin).toBeInstanceOf(Binary); + expect(Buffer.from(bin.value()).toString("utf8")).toBe("hello"); + expect(bin.sub_type).toBe(0); + }); + + test("accepts identical redundant aliases ({ base64: '...', $base64: '...' })", () => { + const ejson = { + $binary: { + base64: "aGVsbG8=", + $base64: "aGVsbG8=", + subtype: "00", + subType: "00", + $type: "00", + }, + }; + const bin = mongoEJSONtoBSON(ejson) as unknown as Binary; + expect(Buffer.from(bin.value()).toString("utf8")).toBe("hello"); + expect(bin.sub_type).toBe(0); + }); + + test("rejects conflicting base64 aliases ({ base64: '...', $base64: '...' }) with TypeError", () => { + const ejson = { + $binary: { + base64: "aGVsbG8=", + $base64: "d29ybGQ=", + }, + }; + expect(() => mongoEJSONtoBSON(ejson)).toThrow(TypeError); + expect(() => mongoEJSONtoBSON(ejson)).toThrow(/conflicting base64 payload properties/i); + }); + + test("rejects conflicting subtype aliases ({ subtype: '00', subType: '04' }) with TypeError", () => { + const ejson = { + $binary: { + base64: "aGVsbG8=", + subtype: "00", + subType: "04", + }, + }; + expect(() => mongoEJSONtoBSON(ejson)).toThrow(TypeError); + expect(() => mongoEJSONtoBSON(ejson)).toThrow(/conflicting subtype payload properties/i); + }); + }); + + describe("Strict Subtype Validation", () => { + test("parses valid numeric integer subtypes in range 0-255", () => { + expect(validateAndParseSubtype(0)).toBe(0); + expect(validateAndParseSubtype(4)).toBe(4); + expect(validateAndParseSubtype(128)).toBe(128); + expect(validateAndParseSubtype(255)).toBe(255); + }); + + test("parses valid hexadecimal string subtypes ('00', '04', '80', '0x04', 'a', 'FF')", () => { + expect(validateAndParseSubtype("00")).toBe(0); + expect(validateAndParseSubtype("04")).toBe(4); + expect(validateAndParseSubtype("0x04")).toBe(4); + expect(validateAndParseSubtype("0X04")).toBe(4); + expect(validateAndParseSubtype("80")).toBe(128); + expect(validateAndParseSubtype("a")).toBe(10); + expect(validateAndParseSubtype("FF")).toBe(255); + expect(validateAndParseSubtype("0xff")).toBe(255); + }); + + test("preserves full canonical subtype matrix (0x00-0x07, 0x80)", () => { + const matrix = [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80]; + for (const st of matrix) { + expect(validateAndParseSubtype(st)).toBe(st); + expect(validateAndParseSubtype(st.toString(16).padStart(2, "0"))).toBe(st); + } + }); + + test("rejects negative numeric subtype (-1) with TypeError", () => { + expect(() => validateAndParseSubtype(-1)).toThrow(TypeError); + }); + + test("rejects fractional numeric subtype (4.5) with TypeError", () => { + expect(() => validateAndParseSubtype(4.5)).toThrow(TypeError); + }); + + test("rejects overflow numeric subtype (> 255) with TypeError", () => { + expect(() => validateAndParseSubtype(256)).toThrow(TypeError); + expect(() => validateAndParseSubtype(1000)).toThrow(TypeError); + }); + + test("rejects NaN, Infinity, -Infinity subtypes with TypeError", () => { + expect(() => validateAndParseSubtype(Number.NaN)).toThrow(TypeError); + expect(() => validateAndParseSubtype(Number.POSITIVE_INFINITY)).toThrow(TypeError); + expect(() => validateAndParseSubtype(Number.NEGATIVE_INFINITY)).toThrow(TypeError); + }); + + test("rejects invalid hexadecimal string subtypes ('zz', '0x100', '-01') with TypeError", () => { + expect(() => validateAndParseSubtype("zz")).toThrow(TypeError); + expect(() => validateAndParseSubtype("0x100")).toThrow(TypeError); + expect(() => validateAndParseSubtype("-01")).toThrow(TypeError); + expect(() => validateAndParseSubtype("")).toThrow(TypeError); + }); + + test("never silently defaults an invalid subtype to zero", () => { + expect(() => validateAndParseSubtype(null)).toThrow(TypeError); + expect(() => validateAndParseSubtype({})).toThrow(TypeError); + expect(() => validateAndParseSubtype([])).toThrow(TypeError); + expect(() => validateAndParseSubtype(true)).toThrow(TypeError); + }); + }); + + describe("Structural & Canonical Base64 Validation", () => { + test("accepts valid standard padded base64 string ('aGVsbG8=')", () => { + const buf = validateAndDecodeBase64("aGVsbG8="); + expect(buf.toString("utf8")).toBe("hello"); + }); + + test("accepts valid unpadded base64 string ('aGVsbG8')", () => { + const buf = validateAndDecodeBase64("aGVsbG8"); + expect(buf.toString("utf8")).toBe("hello"); + }); + + test("accepts valid empty base64 string ('') and preserves zero-byte Binary", () => { + const buf = validateAndDecodeBase64(""); + expect(buf.length).toBe(0); + const bin = mongoEJSONtoBSON({ $binary: { base64: "", subtype: 0 } }) as unknown as Binary; + expect(bin.length()).toBe(0); + expect(bin.value().length).toBe(0); + }); + + test("rejects base64 with invalid characters ('hello!world', 'abc$', spaces) with TypeError before decoding", () => { + expect(() => validateAndDecodeBase64("hello!world")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("abc$")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("aGVs bG8=")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("aGVsbG8=\n")).toThrow(TypeError); + }); + + test("rejects base64 with misplaced padding in middle ('a=b=', 'aG=sbG8=') with TypeError before decoding", () => { + expect(() => validateAndDecodeBase64("a=b=")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("aG=sbG8=")).toThrow(TypeError); + }); + + test("rejects base64 with excess padding (3+ equals: 'a===', '====', 'abc===') with TypeError before decoding", () => { + expect(() => validateAndDecodeBase64("a===")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("====")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("abc===")).toThrow(TypeError); + }); + + test("rejects base64 with characters after padding ('aGVsbG8=A', 'aA== ') with TypeError before decoding", () => { + expect(() => validateAndDecodeBase64("aGVsbG8=A")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("aA== ")).toThrow(TypeError); + }); + + test("rejects base64 with invalid padded length ('aG=', 'aGV=') with TypeError before decoding", () => { + expect(() => validateAndDecodeBase64("aG=")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("aG==")).toThrow(TypeError); + }); + + test("rejects truncated base64 (length % 4 === 1: 'a', 'abcde') with TypeError before decoding", () => { + expect(() => validateAndDecodeBase64("a")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("abcde")).toThrow(TypeError); + }); + + test("rejects non-canonical base64 with non-zero pad bits ('aB==') with TypeError via re-encoding validation", () => { + expect(() => validateAndDecodeBase64("aB==")).toThrow(TypeError); + expect(() => validateAndDecodeBase64("aB==")).toThrow(/non-canonical base64 encoding/i); + }); + }); + + describe("Malformed Payload Safety", () => { + test("rejects null or non-record binary payload ({ $binary: null }, { $binary: 123 }) with TypeError", () => { + expect(() => mongoEJSONtoBSON({ $binary: null as any })).toThrow(TypeError); + expect(() => mongoEJSONtoBSON({ $binary: 123 as any })).toThrow(TypeError); + }); + + test("rejects empty dictionary payload ({ $binary: {} }) with TypeError", () => { + expect(() => mongoEJSONtoBSON({ $binary: {} })).toThrow(TypeError); + expect(() => mongoEJSONtoBSON({ $binary: {} })).toThrow(/missing base64 data/i); + }); + + test("never converts malformed input into an empty Binary", () => { + expect(() => mongoEJSONtoBSON({ $binary: { base64: "invalid!base64" } })).toThrow(TypeError); + expect(() => + mongoEJSONtoBSON({ $binary: { base64: "aGVsbG8=", subtype: "invalid" } }), + ).toThrow(TypeError); + }); + }); + + describe("Logical Payload Round-Trip Invariants", () => { + test("verifies logical payload bytes b.value(), length b.length(), and sub_type in forward direction", () => { + const raw = Buffer.from([0x01, 0x02, 0x03, 0x04, 0x05]); + const original = new Binary(raw, 2); + + const serialized = serializeMongoValue(original) as { + $binary: { base64: string; subtype: string }; + }; + expect(serialized.$binary.base64).toBe(raw.toString("base64")); + expect(serialized.$binary.subtype).toBe("02"); + + const deserialized = mongoEJSONtoBSON(serialized) as unknown as Binary; + expect(deserialized.sub_type).toBe(original.sub_type); + expect(deserialized.length()).toBe(original.length()); + expect(Buffer.from(deserialized.value()).equals(Buffer.from(original.value()))).toBe(true); + }); + + test("verifies logical payload bytes, length, and sub_type in reverse direction", () => { + const ejsonDoc = { $binary: { base64: "AQIDBA==", subType: "04" } }; + const bin = mongoEJSONtoBSON(ejsonDoc) as unknown as Binary; + + expect(bin.sub_type).toBe(4); + expect(bin.length()).toBe(4); + expect(Buffer.from(bin.value())).toEqual(Buffer.from([1, 2, 3, 4])); + + const reSerialized = serializeMongoValue(bin) as { + $binary: { base64: string; subtype: string }; + }; + expect(reSerialized.$binary.base64).toBe("AQIDBA=="); + expect(reSerialized.$binary.subtype).toBe("04"); + }); + + test("losslessly preserves 16-byte UUIDs (subtype 0x04) and MD5 hashes (subtype 0x05) across round-trips", () => { + const uuidBytes = Buffer.from( + "c3ca6eaa-a103-49fe-a436-0ddb0e4b1eb6".replace(/-/g, ""), + "hex", + ); + expect(uuidBytes.length).toBe(16); + const uuidBinary = new Binary(uuidBytes, 4); + + const wireUuid = serializeMongoValue(uuidBinary) as any; + const deserializedUuid = mongoEJSONtoBSON(wireUuid) as unknown as Binary; + expect(deserializedUuid.sub_type).toBe(4); + expect(Buffer.from(deserializedUuid.value()).equals(uuidBytes)).toBe(true); + + const md5Bytes = Buffer.from("098f6bcd4621d373cade4e832627b4f6", "hex"); + expect(md5Bytes.length).toBe(16); + const md5Binary = new Binary(md5Bytes, 5); + + const wireMd5 = serializeMongoValue(md5Binary) as any; + const deserializedMd5 = mongoEJSONtoBSON(wireMd5) as unknown as Binary; + expect(deserializedMd5.sub_type).toBe(5); + expect(Buffer.from(deserializedMd5.value()).equals(md5Bytes)).toBe(true); + }); + }); +}); diff --git a/bridge/__tests__/connectors/mongoDocuments.test.ts b/bridge/__tests__/connectors/mongoDocuments.test.ts new file mode 100644 index 00000000..82fe73c1 --- /dev/null +++ b/bridge/__tests__/connectors/mongoDocuments.test.ts @@ -0,0 +1,300 @@ +import { describe, expect, test } from "@jest/globals"; +import { ObjectId, Int32, Long } from "mongodb"; +import { + mongoEJSONtoBSON, + mongoBSONtoEJSON, + serializeMongoUpdateOperand, + serializeMongoPipelineStage, +} from "../../src/connectors/mongoBson"; +import { + mongoConnector, + mongodbCache, + MongoApiError, + mapMongoError, + withMongoTimeout, + assertReadOnlyPipeline, +} from "../../src/connectors/mongo"; +import type { + MongoConfig, + MongoFindRequest, + MongoSortSpec, + EJSONDocument, +} from "../../src/types/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; + +// --------------------------------------------------------------------------- +// M2 unit tests — driver-free. Covers the EJSON<->BSON converters, the request +// validation / limit guards (which reject before any driver call), the error +// mapper, the client-side timeout wrapper, and the collection-scoped cache. +// (docs/MONGODB_M2_SPEC.md §16.1) +// --------------------------------------------------------------------------- + +describe("mongoBson EJSON round-trip", () => { + const cases: unknown[] = [ + { $oid: "507f1f77bcf86cd799439011" }, + { $date: "2024-01-01T00:00:00.000Z" }, + { $numberInt: "123" }, + { $numberLong: "9007199254740993" }, + { $numberDecimal: "1.50" }, + { $binary: { $type: "00", $base64: "aGVsbG8=" } }, + { $regex: "ab+c", $options: "i" }, + { $timestamp: { $t: "1", $i: "2" } }, + { + name: "x", + tags: ["a", "b"], + nested: { + n: { $numberInt: "5" }, + id: { $oid: "507f1f77bcf86cd799439011" }, + }, + }, + "plain string", + 42, + 3.14, + true, + false, + null, + ]; + + test("mongoBSONtoEJSON(mongoEJSONtoBSON(x)) is structurally equal to x", () => { + for (const c of cases) { + expect(mongoBSONtoEJSON(mongoEJSONtoBSON(c as EJSONDocument))).toEqual(c); + } + }); + + test("a bare hex string stays a string; only $oid yields an ObjectId", () => { + const hex = "507f1f77bcf86cd799439011"; + expect(mongoEJSONtoBSON({ v: hex }).v).toBe(hex); + expect(mongoEJSONtoBSON({ v: { $oid: hex } }).v).toBeInstanceOf(ObjectId); + }); + + test("unknown $-wrappers pass through unchanged", () => { + const input: EJSONDocument = { $unknownThing: "x" }; + expect(mongoBSONtoEJSON(mongoEJSONtoBSON(input))).toEqual(input); + }); + + test("serializeMongoUpdateOperand preserves operator keys and resolves leaf values", () => { + const op = serializeMongoUpdateOperand({ + $set: { name: "a", age: { $numberInt: "5" } }, + $inc: { count: { $numberLong: "1" } }, + }); + expect(Object.keys(op).sort()).toEqual(["$inc", "$set"]); + const set = op.$set as Record; + const inc = op.$inc as Record; + expect(set.name).toBe("a"); + expect(set.age).toBeInstanceOf(Int32); + expect(inc.count).toBeInstanceOf(Long); + }); + + test("serializeMongoPipelineStage resolves EJSON but does not inject $literal", () => { + const stage = serializeMongoPipelineStage({ + $group: { _id: "$dept", avg: { $avg: "$salary" } }, + }); + expect(stage).toEqual({ $group: { _id: "$dept", avg: { $avg: "$salary" } } }); + }); +}); + +describe("M2 request validation (rejects before touching the driver)", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("findDocuments rejects a page beyond the maximum", async () => { + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", page: 99 }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("findDocuments rejects a non-object filter", async () => { + for (const bad of ["str", 123, true, [1, 2]]) { + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", filter: bad as any }), + ).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + message: "filter must be an object", + }); + } + }); + + test("findDocuments rejects an oversized filter", async () => { + const big: EJSONDocument = { big: "x".repeat(9000) }; + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", filter: big }), + ).rejects.toMatchObject({ code: "MONGO_LIMIT_EXCEEDED" }); + }); + + test("findDocuments rejects an invalid sort direction", async () => { + const req: MongoFindRequest = { + collection: "c", + sort: { a: "SIDEWAYS" } as unknown as MongoSortSpec, + }; + await expect(mongoConnector.findDocuments(dummy, req)).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + }); + }); + + test("findDocuments rejects a projection mixing inclusion and exclusion", async () => { + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", projection: { a: 1, b: 0 } }), + ).rejects.toMatchObject({ code: "MONGO_INVALID_FILTER" }); + }); + + test("updateDocument rejects an empty filter (bulk-op guard)", async () => { + await expect( + mongoConnector.updateDocument(dummy, { + collection: "c", + filter: {}, + update: { $set: { a: 1 } }, + }), + ).rejects.toMatchObject({ code: "MONGO_BULK_OP_FORBIDDEN" }); + }); + + test("updateDocument rejects a non-operator (replacement) document", async () => { + await expect( + mongoConnector.updateDocument(dummy, { + collection: "c", + filter: { x: 1 }, + update: { name: "a" }, + }), + ).rejects.toMatchObject({ code: "MONGO_INVALID_UPDATE_OPERAND" }); + }); + + test("deleteDocument rejects an empty filter (bulk-op guard)", async () => { + await expect( + mongoConnector.deleteDocument(dummy, { collection: "c", filter: {} }), + ).rejects.toMatchObject({ code: "MONGO_BULK_OP_FORBIDDEN" }); + }); + + test("runAggregation accepts an empty pipeline without error", () => { + expect(() => assertReadOnlyPipeline([])).not.toThrow(); + }); + + test("runAggregation rejects a write stage ($out)", async () => { + await expect( + mongoConnector.runAggregation(dummy, { + collection: "c", + pipeline: [{ $out: { coll: "x" } }], + }), + ).rejects.toMatchObject({ code: "MONGO_WRITE_STAGE_FORBIDDEN" }); + }); + + test("runAggregation rejects a pipeline over the stage cap", async () => { + const pipeline = Array.from({ length: 65 }, (): EJSONDocument => ({ $match: {} })); + await expect( + mongoConnector.runAggregation(dummy, { collection: "c", pipeline }), + ).rejects.toMatchObject({ code: "MONGO_LIMIT_EXCEEDED" }); + }); + + test("insertDocument rejects a non-object document", async () => { + await expect( + mongoConnector.insertDocument(dummy, { + collection: "c", + document: [1, 2, 3] as unknown as EJSONDocument, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); +}); + +describe("M2 error mapping", () => { + test("maps common MongoServerError shapes to stable codes", () => { + expect(mapMongoError({ name: "MongoServerError", codeName: "NamespaceNotFound" }).code).toBe( + "MONGO_NS_NOT_FOUND", + ); + expect(mapMongoError({ name: "MongoServerError", code: 11000 }).code).toBe( + "MONGO_DUPLICATE_KEY", + ); + expect(mapMongoError({ name: "MongoServerError", codeName: "AuthenticationFailed" }).code).toBe( + "MONGO_AUTH_ERROR", + ); + expect(mapMongoError({ name: "MongoServerError", codeName: "MaxTimeMSExpired" }).code).toBe( + "MONGO_OPERATION_TIMEOUT", + ); + }); + + test("maps a generic error to a connection error and passes through MongoApiError", () => { + expect(mapMongoError(new Error("boom")).code).toBe("MONGO_CONNECTION_ERROR"); + const api = new MongoApiError("MONGO_BULK_OP_FORBIDDEN", "nope"); + expect(mapMongoError(api)).toBe(api); + }); +}); + +describe("M2 client-side timeout wrapper", () => { + test("passes through a fast result", async () => { + await expect(withMongoTimeout(1000, async () => 42)).resolves.toBe(42); + }); + + test("propagates a real error from the operation", async () => { + await expect( + withMongoTimeout(1000, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + }); + + test("rejects with MONGO_OPERATION_TIMEOUT when the operation hangs", async () => { + const start = Date.now(); + await expect(withMongoTimeout(30, () => new Promise(() => {}))).rejects.toMatchObject({ + code: "MONGO_OPERATION_TIMEOUT", + name: "MongoApiError", + }); + expect(Date.now() - start).toBeLessThan(1000); + }); +}); + +describe("M2 collection-scoped cache", () => { + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/cachetest", database: "cachetest" }; + + test("stores and reads collection stats and document pages, then evicts on clearCollectionCache", () => { + mongodbCache.setCollectionStats(cfg, "c", { + collection: "c", + count: 5, + sizeBytes: 100, + avgObjSize: 20, + }); + expect(mongodbCache.getCollectionStats(cfg, "c")).toEqual({ + collection: "c", + count: 5, + sizeBytes: 100, + avgObjSize: 20, + }); + expect(mongodbCache.getCollectionStats(cfg, "other")).toBeNull(); + + mongodbCache.setDocumentPage(cfg, "key1", [{ a: 1 }]); + expect(mongodbCache.getDocumentPage(cfg, "key1")).toEqual([{ a: 1 }]); + + mongodbCache.clearCollectionCache(cfg, "c"); + expect(mongodbCache.getCollectionStats(cfg, "c")).toBeNull(); + expect(mongodbCache.getDocumentPage(cfg, "key1")).toBeNull(); + }); + + test("isolation: a different connection does not see another connection's pages", () => { + const other: MongoConfig = { uri: "mongodb://localhost:27017/other", database: "other" }; + mongodbCache.setDocumentPage(cfg, "shared", [{ a: 1 }]); + expect(mongodbCache.getDocumentPage(other, "shared")).toBeNull(); + expect(mongodbCache.getDocumentPage(cfg, "shared")).toEqual([{ a: 1 }]); + }); +}); + +describe("M2 executor document-op routing (spec §12)", () => { + const executor = new QueryExecutor(); + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects document operations on a non-MongoDB connection with MONGO_UNSUPPORTED_OPERATION", async () => { + await expect( + executor.findDocuments(cfg, DBType.POSTGRES, { collection: "c" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.insertDocument(cfg, DBType.MYSQL, { collection: "c", document: { a: 1 } }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.updateDocument(cfg, DBType.POSTGRES, { collection: "c", filter: {}, update: {} }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.deleteDocument(cfg, DBType.POSTGRES, { collection: "c", filter: {} }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.runAggregation(cfg, DBType.POSTGRES, { + collection: "c", + pipeline: [{ $match: {} }], + }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); +}); diff --git a/bridge/__tests__/connectors/mongoInferSchema.test.ts b/bridge/__tests__/connectors/mongoInferSchema.test.ts new file mode 100644 index 00000000..839286c4 --- /dev/null +++ b/bridge/__tests__/connectors/mongoInferSchema.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, test } from "@jest/globals"; +import { ObjectId, Int32, Long, Decimal128, Binary, Timestamp } from "mongodb"; +import { bsonTypeOf, bsonNumericKind } from "../../src/connectors/mongoBson"; +import { + FieldAccumulator, + clampInferSampleSize, + clampInferMaxDepth, + mongodbCache, +} from "../../src/connectors/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; +import type { MongoConfig } from "../../src/types/mongo"; + +describe("M3 pure BSON type inference (bsonTypeOf / bsonNumericKind)", () => { + test("bsonTypeOf detects primitives, BSON types, and composites correctly", () => { + expect(bsonTypeOf(null)).toBe("null"); + expect(bsonTypeOf(undefined)).toBe("undefined"); + expect(bsonTypeOf("test")).toBe("string"); + expect(bsonTypeOf(true)).toBe("bool"); + expect(bsonTypeOf(false)).toBe("bool"); + expect(bsonTypeOf(42)).toBe("number"); + expect(bsonTypeOf(3.14)).toBe("number"); + expect(bsonTypeOf(new Int32(100))).toBe("number"); + expect(bsonTypeOf(new Long(9007199254740993n))).toBe("long"); + expect(bsonTypeOf(new Decimal128("12.34"))).toBe("decimal"); + expect(bsonTypeOf(new Timestamp({ t: 1, i: 2 }))).toBe("timestamp"); + expect(bsonTypeOf(new Binary(Buffer.from("abc")))).toBe("binary"); + expect(bsonTypeOf(Buffer.from("abc"))).toBe("binary"); + expect(bsonTypeOf(new Uint8Array([1, 2, 3]))).toBe("binary"); + expect(bsonTypeOf(new Date("2026-01-01T00:00:00.000Z"))).toBe("date"); + expect(bsonTypeOf(new ObjectId("507f1f77bcf86cd799439011"))).toBe("objectId"); + expect(bsonTypeOf([1, 2, 3])).toBe("array"); + expect(bsonTypeOf({ a: 1, b: "hello" })).toBe("object"); + expect(bsonTypeOf({ _bsontype: "UnknownSubtype" })).toBe("object"); + }); + + test("bsonNumericKind identifies int and double accurately", () => { + expect(bsonNumericKind(new Int32(5))).toBe("int"); + expect(bsonNumericKind(42)).toBe("int"); + expect(bsonNumericKind(3.14)).toBe("double"); + expect(bsonNumericKind({ _bsontype: "Double", valueOf: () => 5.5 })).toBe("double"); + expect(bsonNumericKind("42")).toBeUndefined(); + expect(bsonNumericKind(null)).toBeUndefined(); + expect(bsonNumericKind(undefined)).toBeUndefined(); + expect(bsonNumericKind(new Long(100n))).toBeUndefined(); + }); +}); + +describe("M3 clamping helpers", () => { + test("clampInferSampleSize clamps to [1, 1000] and defaults to 100", () => { + expect(clampInferSampleSize(undefined)).toBe(100); + expect(clampInferSampleSize(NaN)).toBe(100); + expect(clampInferSampleSize(0)).toBe(1); + expect(clampInferSampleSize(-10)).toBe(1); + expect(clampInferSampleSize(50)).toBe(50); + expect(clampInferSampleSize(2000)).toBe(1000); + }); + + test("clampInferMaxDepth clamps to [1, 16] and defaults to 8", () => { + expect(clampInferMaxDepth(undefined)).toBe(8); + expect(clampInferMaxDepth(NaN)).toBe(8); + expect(clampInferMaxDepth(0)).toBe(1); + expect(clampInferMaxDepth(-5)).toBe(1); + expect(clampInferMaxDepth(4)).toBe(4); + expect(clampInferMaxDepth(25)).toBe(16); + }); +}); + +describe("FieldAccumulator (pure schema inference)", () => { + test("infers uniform flat documents correctly", () => { + const acc = new FieldAccumulator(2, 8); + acc.ingest({ _id: new ObjectId(), name: "Alice", age: 30 }); + acc.ingest({ _id: new ObjectId(), name: "Bob", age: 25 }); + const fields = acc.toFields(); + + expect(fields).toHaveLength(3); + const idField = fields.find((f) => f.path === "_id")!; + expect(idField.bsonType).toBe("objectId"); + expect(idField.presencePct).toBe(100); + expect(idField.nullable).toBe(false); + expect(idField.optional).toBe(false); + expect(idField.depth).toBe(0); + expect(idField.typeConflicts).toHaveLength(0); + + const nameField = fields.find((f) => f.path === "name")!; + expect(nameField.bsonType).toBe("string"); + expect(nameField.presencePct).toBe(100); + + const ageField = fields.find((f) => f.path === "age")!; + expect(ageField.bsonType).toBe("number"); + expect(ageField.numericKinds).toEqual(["int"]); + }); + + test("tracks nested paths, depths, parentPath, and maxChildDepth", () => { + const acc = new FieldAccumulator(1, 8); + acc.ingest({ + user: { + profile: { + bio: "Engineer", + }, + }, + }); + const fields = acc.toFields(); + + const user = fields.find((f) => f.path === "user")!; + expect(user.depth).toBe(0); + expect(user.isNested).toBe(true); + expect(user.parentPath).toBeUndefined(); + expect(user.maxChildDepth).toBe(2); + + const profile = fields.find((f) => f.path === "user.profile")!; + expect(profile.depth).toBe(1); + expect(profile.isNested).toBe(true); + expect(profile.parentPath).toBe("user"); + expect(profile.maxChildDepth).toBe(2); + + const bio = fields.find((f) => f.path === "user.profile.bio")!; + expect(bio.depth).toBe(2); + expect(bio.isNested).toBe(false); + expect(bio.parentPath).toBe("user.profile"); + expect(bio.bsonType).toBe("string"); + }); + + test("handles arrays and distinct arrayElementTypes without emitting indexed subpaths", () => { + const acc = new FieldAccumulator(2, 8); + acc.ingest({ tags: ["red", "green"] }); + acc.ingest({ tags: ["blue", 123] }); + const fields = acc.toFields(); + + expect(fields.some((f) => f.path.startsWith("tags."))).toBe(false); + + const tagsField = fields.find((f) => f.path === "tags")!; + expect(tagsField.isArray).toBe(true); + expect(tagsField.bsonType).toBe("array"); + expect(tagsField.arrayElementTypes?.sort()).toEqual(["number", "string"]); + }); + + test("tracks mixed type conflicts with sorted counts and nullable/optional flags", () => { + const acc = new FieldAccumulator(4, 8); + acc.ingest({ score: 100 }); + acc.ingest({ score: "N/A" }); + acc.ingest({ score: "exempt" }); + acc.ingest({ score: null }); + + const fields = acc.toFields(); + const score = fields.find((f) => f.path === "score")!; + + expect(score.nullable).toBe(true); + expect(score.nullCount).toBe(1); + expect(score.count).toBe(4); + expect(score.optional).toBe(false); + expect(score.presencePct).toBe(100); + + expect(score.bsonType).toBe("string"); + expect(score.typeConflicts.length).toBeGreaterThan(1); + expect(score.typeConflicts[0].bsonType).toBe("string"); + expect(score.typeConflicts[0].count).toBe(2); + }); + + test("respects maxDepth and marks objects as truncated", () => { + const acc = new FieldAccumulator(1, 1); + acc.ingest({ + level0: { + level1: { + level2: "deep", + }, + }, + }); + + const fields = acc.toFields(); + expect(fields.find((f) => f.path === "level0.level1.level2")).toBeUndefined(); + const l1 = fields.find((f) => f.path === "level0.level1")!; + expect(l1.depth).toBe(1); + expect(l1.truncated).toBe(true); + }); +}); + +describe("M3 Cache Manager inferred schema integration", () => { + const dummyCfg: MongoConfig = { + uri: "mongodb://127.0.0.1:27017/test_db", + database: "test_db", + }; + + test("stores, retrieves, and invalidates inferred schema", () => { + const schemaPayload: any = { + collection: "users", + sampleSize: 10, + totalDocs: 100, + inferredAt: new Date().toISOString(), + fieldCount: 1, + nestedFieldCount: 0, + arrayFieldCount: 0, + fields: [], + sampleDocuments: [], + }; + + mongodbCache.setInferredSchema(dummyCfg, "users", 100, 8, schemaPayload); + const cached = mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8); + expect(cached).toEqual(schemaPayload); + + // Eviction on clearCollectionCache + mongodbCache.clearCollectionCache(dummyCfg, "users"); + expect(mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8)).toBeNull(); + + // Re-set and clearForConnection + mongodbCache.setInferredSchema(dummyCfg, "users", 100, 8, schemaPayload); + mongodbCache.clearForConnection(dummyCfg); + expect(mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8)).toBeNull(); + + // Re-set and clearAll + mongodbCache.setInferredSchema(dummyCfg, "users", 100, 8, schemaPayload); + mongodbCache.clearAll(); + expect(mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8)).toBeNull(); + }); +}); + +describe("QueryExecutor M3 MongoDB guards", () => { + const executor = new QueryExecutor(); + + test("listCollections rejects non-MongoDB dbType", async () => { + await expect(executor.listCollections({} as any, DBType.POSTGRES)).rejects.toThrow( + "listCollections requires a MongoDB connection", + ); + }); + + test("inferCollectionSchema rejects non-MongoDB dbType", async () => { + await expect( + executor.inferCollectionSchema({} as any, DBType.MYSQL, { collection: "users" }), + ).rejects.toThrow("inferCollectionSchema requires a MongoDB connection"); + }); +}); diff --git a/bridge/__tests__/connectors/mongoMql.test.ts b/bridge/__tests__/connectors/mongoMql.test.ts new file mode 100644 index 00000000..35759dd6 --- /dev/null +++ b/bridge/__tests__/connectors/mongoMql.test.ts @@ -0,0 +1,769 @@ +import { describe, expect, test, beforeEach, jest } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { + assertCollectionName, + clampMqlTimeout, + executeMql, + explainMql, + mongoConnector, +} from "../../src/connectors/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; +import type { + MongoConfig, + MongoMqlExecuteRequest, + MongoExplainRequest, +} from "../../src/types/mongo"; + +describe("Mongo MQL Unit Tests (docs/MONGODB_M5_SPEC.md)", () => { + const dummyConfig: MongoConfig = { + uri: "mongodb://localhost:27017/testdb", + database: "testdb", + }; + + let mockCursorDocs: any[]; + let mockCursor: any; + let mockCollection: any; + + beforeEach(() => { + mockCursorDocs = []; + mockCursor = { + [Symbol.asyncIterator]: async function* () { + for (const doc of mockCursorDocs) { + yield doc; + } + }, + close: jest.fn().mockImplementation(() => Promise.resolve()), + toArray: jest.fn().mockImplementation(() => Promise.resolve(mockCursorDocs)), + explain: jest.fn().mockImplementation((verbosity: string) => + Promise.resolve({ + queryPlanner: { + plannerVersion: 1, + namespace: "testdb.users", + indexFilterSet: false, + winningPlan: { stage: "COLLSCAN" }, + }, + executionStats: + verbosity !== "queryPlanner" + ? { + executionSuccess: true, + nReturned: 5, + executionTimeMillis: 2, + totalKeysExamined: 0, + totalDocsExamined: 5, + executionStages: { stage: "COLLSCAN" }, + } + : undefined, + serverInfo: { + host: "localhost", + port: 27017, + version: "7.0.0", + }, + }), + ), + sort: jest.fn().mockReturnThis(), + project: jest.fn().mockReturnThis(), + skip: jest.fn().mockReturnThis(), + limit: jest.fn().mockReturnThis(), + }; + + mockCollection = { + find: jest.fn().mockReturnValue(mockCursor), + aggregate: jest.fn().mockReturnValue(mockCursor), + countDocuments: jest.fn().mockImplementation(() => Promise.resolve(42)), + estimatedDocumentCount: jest.fn().mockImplementation(() => Promise.resolve(100)), + distinct: jest.fn().mockImplementation(() => Promise.resolve(["apple", "banana"])), + }; + + jest.spyOn(MongoClient.prototype, "db").mockReturnValue({ + collection: () => mockCollection, + } as any); + }); + + describe("assertCollectionName", () => { + test("accepts valid collection names including dots, dashes, underscores, and numbers", () => { + expect(() => assertCollectionName("users")).not.toThrow(); + expect(() => assertCollectionName("audit.events")).not.toThrow(); + expect(() => assertCollectionName("logs.2026.10")).not.toThrow(); + expect(() => assertCollectionName("2026_events")).not.toThrow(); + expect(() => assertCollectionName("user-data")).not.toThrow(); + expect(() => assertCollectionName("collection with spaces")).not.toThrow(); + expect(() => assertCollectionName("a".repeat(120))).not.toThrow(); + }); + + test("rejects empty or non-string collection names", () => { + expect(() => assertCollectionName("")).toThrow( + expect.objectContaining({ code: "MONGO_VALIDATION_ERROR" }), + ); + expect(() => assertCollectionName(null)).toThrow( + expect.objectContaining({ code: "MONGO_VALIDATION_ERROR" }), + ); + expect(() => assertCollectionName(undefined)).toThrow( + expect.objectContaining({ code: "MONGO_VALIDATION_ERROR" }), + ); + expect(() => assertCollectionName(123)).toThrow( + expect.objectContaining({ code: "MONGO_VALIDATION_ERROR" }), + ); + }); + + test("rejects collection names exceeding 120 bytes", () => { + expect(() => assertCollectionName("a".repeat(121))).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "collection name must not exceed 120 bytes", + }), + ); + }); + + test("rejects collection names with null bytes or dollar signs", () => { + expect(() => assertCollectionName("audit\0events")).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "collection name must not contain '$' or null bytes", + }), + ); + expect(() => assertCollectionName("my$collection")).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "collection name must not contain '$' or null bytes", + }), + ); + }); + + test("rejects collection names starting with 'system.'", () => { + expect(() => assertCollectionName("system.views")).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "collection name must not start with 'system.'", + }), + ); + expect(() => assertCollectionName("system.profile")).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "collection name must not start with 'system.'", + }), + ); + }); + }); + + describe("clampMqlTimeout", () => { + test("returns default when undefined or null", () => { + expect(clampMqlTimeout(undefined, 5000)).toBe(5000); + expect(clampMqlTimeout(null as any, 10000)).toBe(10000); + }); + + test("throws MONGO_VALIDATION_ERROR for non-integers, negative, or NaN", () => { + expect(() => clampMqlTimeout(0, 5000)).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "maxTimeMS must be a positive integer", + }), + ); + expect(() => clampMqlTimeout(-10, 5000)).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "maxTimeMS must be a positive integer", + }), + ); + expect(() => clampMqlTimeout(NaN, 5000)).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "maxTimeMS must be a positive integer", + }), + ); + expect(() => clampMqlTimeout(1.5, 5000)).toThrow( + expect.objectContaining({ + code: "MONGO_VALIDATION_ERROR", + message: "maxTimeMS must be a positive integer", + }), + ); + }); + + test("clamps value between 1 and maxMs (default 60000)", () => { + expect(clampMqlTimeout(100, 5000)).toBe(100); + expect(clampMqlTimeout(50000, 5000)).toBe(50000); + expect(clampMqlTimeout(120000, 5000)).toBe(60000); + expect(clampMqlTimeout(80000, 5000, 30000)).toBe(30000); + }); + }); + + describe("executeMql: Mutating operations guard", () => { + const mutatingOps = [ + "insertOne", + "insertMany", + "updateOne", + "updateMany", + "replaceOne", + "deleteOne", + "deleteMany", + "drop", + "dropDatabase", + "rename", + "createIndex", + "dropIndex", + "bulkWrite", + "command", + "runCommand", + "eval", + ]; + + test.each(mutatingOps)("rejects mutating operation '%s'", async (op) => { + const req = { + collection: "users", + op: { operation: op, payload: {} }, + } as any; + + await expect(executeMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN_OPERATION", + message: `Operation '${op}' is forbidden in read-only query workspace. Use RelWave document mutation actions.`, + }); + }); + }); + + describe("executeMql: 'find' operation", () => { + test("rejects non-object filter with MONGO_INVALID_FILTER", async () => { + for (const badFilter of ["string", 123, true, [1, 2]]) { + await expect( + executeMql(dummyConfig, { + collection: "users", + op: { + operation: "find", + payload: { filter: badFilter as any }, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + message: "filter must be an object", + }); + } + }); + + test("validates safe expressions and rejects $where in filter", async () => { + const req: MongoMqlExecuteRequest = { + collection: "users", + op: { + operation: "find", + payload: { + filter: { $where: "this.age > 21" }, + }, + }, + }; + + await expect(executeMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: + "Operator '$where' is forbidden (server-side JavaScript execution is not permitted)", + }); + }); + + test("rejects negative skip", async () => { + const req: MongoMqlExecuteRequest = { + collection: "users", + op: { + operation: "find", + payload: { + filter: {}, + skip: -5, + }, + }, + }; + + await expect(executeMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "skip must be a non-negative integer", + }); + }); + + test("clamps limit between 1 and MAX_PAGE_SIZE (100) and defaults to 50", async () => { + // 1. Omitted limit -> defaults to 50 + await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "find", + payload: { filter: {} }, + }, + }); + expect(mockCollection.find).toHaveBeenCalledWith({}, expect.objectContaining({ limit: 50 })); + + // 2. Limit 200 -> clamped to 100 + await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "find", + payload: { filter: {}, limit: 200 }, + }, + }); + expect(mockCollection.find).toHaveBeenCalledWith({}, expect.objectContaining({ limit: 100 })); + + // 3. Limit 0 or negative -> defaults to 50 + await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "find", + payload: { filter: {}, limit: -10 }, + }, + }); + expect(mockCollection.find).toHaveBeenCalledWith({}, expect.objectContaining({ limit: 50 })); + }); + + test("executes find and converts results to EJSON", async () => { + mockCursor.toArray.mockResolvedValueOnce([{ _id: "123", name: "Alice", age: 30 }]); + + const res = await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "find", + payload: { + filter: { name: "Alice" }, + sort: { age: 1 }, + skip: 10, + limit: 25, + }, + }, + }); + + expect(res.operation).toBe("find"); + expect(res.collection).toBe("users"); + expect((res as any).totalReturned).toBe(1); + expect((res as any).documents).toEqual([{ _id: "123", name: "Alice", age: 30 }]); + expect(typeof res.executionTimeMs).toBe("number"); + }); + }); + + describe("executeMql: 'aggregate' operation", () => { + test("rejects $out in pipeline", async () => { + const req: MongoMqlExecuteRequest = { + collection: "users", + op: { + operation: "aggregate", + payload: { + pipeline: [{ $out: "target" }], + }, + }, + }; + + await expect(executeMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + }); + }); + + test("executes empty pipeline [] successfully", async () => { + mockCursorDocs = [{ _id: "doc1" }, { _id: "doc2" }]; + + const res = await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "aggregate", + payload: { + pipeline: [], + }, + }, + }); + + expect(res.operation).toBe("aggregate"); + expect((res as any).stageCount).toBe(0); + expect((res as any).documents).toHaveLength(2); + expect(mockCursor.close).toHaveBeenCalledTimes(1); + }); + + test("terminates iteration at 1,001 documents, throws MONGO_LIMIT_EXCEEDED, and closes cursor", async () => { + let yieldedCount = 0; + mockCursor[Symbol.asyncIterator] = async function* () { + for (let i = 1; i <= 1050; i++) { + yieldedCount++; + yield { _id: `doc_${i}` }; + } + }; + + await expect( + executeMql(dummyConfig, { + collection: "users", + op: { + operation: "aggregate", + payload: { + pipeline: [{ $match: {} }], + }, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_LIMIT_EXCEEDED", + message: "aggregation returned more than 1000 documents (max 1000); add a $limit stage", + }); + + expect(yieldedCount).toBe(1001); + expect(mockCursor.close).toHaveBeenCalledTimes(1); + }); + + test("passes allowDiskUse and batchSize options to aggregate", async () => { + mockCursorDocs = [{ _id: 1 }]; + + await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "aggregate", + payload: { + pipeline: [], + allowDiskUse: true, + batchSize: 100, + }, + }, + }); + + expect(mockCollection.aggregate).toHaveBeenCalledWith( + [], + expect.objectContaining({ + allowDiskUse: true, + batchSize: 100, + }), + ); + }); + }); + + describe("executeMql: 'countDocuments' operation", () => { + test("rejects non-object filter with MONGO_INVALID_FILTER", async () => { + for (const badFilter of ["string", 123, true, [1, 2]]) { + await expect( + executeMql(dummyConfig, { + collection: "users", + op: { + operation: "countDocuments", + payload: { filter: badFilter as any }, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + message: "filter must be an object", + }); + } + }); + + test("rejects $function in filter", async () => { + const req: MongoMqlExecuteRequest = { + collection: "users", + op: { + operation: "countDocuments", + payload: { + filter: { + $function: { body: "function() {}", args: [], lang: "js" }, + } as any, + }, + }, + }; + + await expect(executeMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN_OPERATOR", + }); + }); + + test("executes count with safe filter", async () => { + mockCollection.countDocuments.mockResolvedValueOnce(88); + + const res = await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "countDocuments", + payload: { + filter: { active: true }, + }, + }, + }); + + expect(res.operation).toBe("countDocuments"); + expect((res as any).count).toBe(88); + }); + }); + + describe("executeMql: 'estimatedDocumentCount' operation", () => { + test("accepts empty or undefined payload", async () => { + mockCollection.estimatedDocumentCount.mockResolvedValue(500); + + const res1 = await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "estimatedDocumentCount", + }, + }); + expect((res1 as any).count).toBe(500); + + const res2 = await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "estimatedDocumentCount", + payload: {}, + }, + }); + expect((res2 as any).count).toBe(500); + }); + + test("rejects non-empty filter or non-object payload with MONGO_VALIDATION_ERROR", async () => { + for (const badPayload of [null, 123, "string", true, [1], { filter: {} }]) { + const req: MongoMqlExecuteRequest = { + collection: "users", + op: { + operation: "estimatedDocumentCount", + payload: badPayload as any, + }, + }; + + await expect(executeMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: + "estimatedDocumentCount does not accept a filter; payload must be empty or undefined. Use countDocuments for filtered counts.", + }); + } + }); + + test("passes signal and timeout to estimatedDocumentCount", async () => { + mockCollection.estimatedDocumentCount.mockResolvedValueOnce(250); + await executeMql(dummyConfig, { + collection: "users", + op: { operation: "estimatedDocumentCount" }, + options: { maxTimeMS: 5000 }, + }); + expect(mockCollection.estimatedDocumentCount).toHaveBeenCalledWith( + expect.objectContaining({ + maxTimeMS: 5000, + signal: expect.any(Object), + }), + ); + }); + }); + + describe("executeMql: 'distinct' operation", () => { + test("rejects non-object filter with MONGO_INVALID_FILTER", async () => { + for (const badFilter of ["string", 123, true, [1, 2]]) { + await expect( + executeMql(dummyConfig, { + collection: "users", + op: { + operation: "distinct", + payload: { field: "category", filter: badFilter as any }, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + message: "filter must be an object", + }); + } + }); + + test("passes signal and timeout to distinct", async () => { + mockCollection.distinct.mockResolvedValueOnce(["cat1", "cat2"]); + await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "distinct", + payload: { field: "category" }, + }, + options: { maxTimeMS: 4000 }, + }); + expect(mockCollection.distinct).toHaveBeenCalledWith( + "category", + {}, + expect.objectContaining({ + maxTimeMS: 4000, + signal: expect.any(Object), + }), + ); + }); + + test("rejects missing or empty field", async () => { + await expect( + executeMql(dummyConfig, { + collection: "users", + op: { + operation: "distinct", + payload: { field: "" }, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "field must be a non-empty string", + }); + + await expect( + executeMql(dummyConfig, { + collection: "users", + op: { + operation: "distinct", + payload: { field: " " }, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "field must be a non-empty string", + }); + }); + + test("caps distinct results at MAX_DISTINCT_VALUES (10000)", async () => { + const bigValues = Array.from({ length: 10050 }, (_, i) => `val_${i}`); + mockCollection.distinct.mockResolvedValueOnce(bigValues); + + const res = await executeMql(dummyConfig, { + collection: "users", + op: { + operation: "distinct", + payload: { field: "category" }, + }, + }); + + expect(res.operation).toBe("distinct"); + expect((res as any).values).toHaveLength(10000); + expect((res as any).count).toBe(10000); + }); + }); + + describe("explainMql", () => { + test("rejects mutating target (e.g. insertOne)", async () => { + const req: MongoExplainRequest = { + collection: "users", + target: { operation: "insertOne" } as any, + }; + + await expect(explainMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN_OPERATION", + message: "Explain is only supported for 'find' and 'aggregate' operations", + }); + }); + + test("rejects unsafe aggregate target containing $out", async () => { + const req: MongoExplainRequest = { + collection: "users", + target: { + operation: "aggregate", + pipeline: [{ $out: "dump" }], + }, + }; + + await expect(explainMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_WRITE_STAGE_FORBIDDEN", + }); + }); + + test("rejects non-object filter for find target with MONGO_INVALID_FILTER", async () => { + for (const badFilter of ["string", 123, true, [1, 2]]) { + await expect( + explainMql(dummyConfig, { + collection: "users", + target: { + operation: "find", + filter: badFilter as any, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + message: "filter must be an object", + }); + } + }); + + test("rejects unsafe find target containing $where", async () => { + const req: MongoExplainRequest = { + collection: "users", + target: { + operation: "find", + filter: { $where: "this.active" }, + }, + }; + + await expect(explainMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN_OPERATOR", + }); + }); + + test("rejects invalid verbosity", async () => { + const req: MongoExplainRequest = { + collection: "users", + verbosity: "unsupported" as any, + target: { + operation: "find", + filter: {}, + }, + }; + + await expect(explainMql(dummyConfig, req)).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "Invalid explain verbosity 'unsupported'", + }); + }); + + test("explains find and aggregate across supported verbosities", async () => { + for (const verbosity of ["queryPlanner", "executionStats", "allPlansExecution"] as const) { + const res = await explainMql(dummyConfig, { + collection: "users", + verbosity, + target: { + operation: "find", + filter: { status: "active" }, + }, + }); + + expect(res.operation).toBe("find"); + expect(res.collection).toBe("users"); + expect(res.queryPlanner).toBeDefined(); + expect(res.rawExplain).toBeDefined(); + } + }); + }); + + describe("QueryExecutor MQL routing", () => { + const executor = new QueryExecutor(); + + test("rejects executeMql on Postgres with MONGO_UNSUPPORTED_OPERATION", async () => { + await expect( + executor.executeMql(dummyConfig, DBType.POSTGRES, { + collection: "users", + op: { operation: "find", payload: { filter: {} } }, + }), + ).rejects.toMatchObject({ + code: "MONGO_UNSUPPORTED_OPERATION", + }); + }); + + test("rejects explainMql on MySQL with MONGO_UNSUPPORTED_OPERATION", async () => { + await expect( + executor.explainMql(dummyConfig, DBType.MYSQL, { + collection: "users", + target: { operation: "find", filter: {} }, + }), + ).rejects.toMatchObject({ + code: "MONGO_UNSUPPORTED_OPERATION", + }); + }); + + test("routes executeMql and explainMql on DBType.MONGODB to mongoConnector", async () => { + const execSpy = jest.spyOn(mongoConnector, "executeMql").mockResolvedValueOnce({ + operation: "find", + collection: "users", + documents: [], + totalReturned: 0, + executionTimeMs: 1, + }); + + const explainSpy = jest.spyOn(mongoConnector, "explainMql").mockResolvedValueOnce({ + collection: "users", + operation: "find", + queryPlanner: { + plannerVersion: 1, + namespace: "testdb.users", + indexFilterSet: false, + winningPlan: {}, + }, + rawExplain: {}, + executionTimeMs: 1, + }); + + await executor.executeMql(dummyConfig, DBType.MONGODB, { + collection: "users", + op: { operation: "find", payload: { filter: {} } }, + }); + expect(execSpy).toHaveBeenCalled(); + + await executor.explainMql(dummyConfig, DBType.MONGODB, { + collection: "users", + target: { operation: "find", filter: {} }, + }); + expect(explainSpy).toHaveBeenCalled(); + }); + }); +}); diff --git a/bridge/__tests__/connectors/mongoMutations.test.ts b/bridge/__tests__/connectors/mongoMutations.test.ts new file mode 100644 index 00000000..f7024cb9 --- /dev/null +++ b/bridge/__tests__/connectors/mongoMutations.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, test } from "@jest/globals"; +import { + assertNewCollectionName, + assertNoOperatorKeys, + mongoConnector, + mongodbCache, + MongoApiError, +} from "../../src/connectors/mongo"; +import type { MongoConfig, EJSONDocument } from "../../src/types/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; + +describe("assertNewCollectionName", () => { + test("accepts valid collection names", () => { + expect(() => assertNewCollectionName("users")).not.toThrow(); + expect(() => assertNewCollectionName("user_accounts")).not.toThrow(); + expect(() => assertNewCollectionName("orders-2026")).not.toThrow(); + expect(() => assertNewCollectionName("_audit_log")).not.toThrow(); + expect(() => assertNewCollectionName("A")).not.toThrow(); + expect(() => assertNewCollectionName("a".repeat(120))).not.toThrow(); + }); + + test("rejects non-string or empty name", () => { + expect(() => assertNewCollectionName("")).toThrow(MongoApiError); + expect(() => assertNewCollectionName(null)).toThrow(MongoApiError); + expect(() => assertNewCollectionName(undefined)).toThrow(MongoApiError); + expect(() => assertNewCollectionName(123)).toThrow(MongoApiError); + }); + + test("rejects names longer than 120 chars", () => { + expect(() => assertNewCollectionName("a".repeat(121))).toThrow(MongoApiError); + }); + + test("rejects names with '$' or null bytes", () => { + expect(() => assertNewCollectionName("my$coll")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("coll\0name")).toThrow(MongoApiError); + }); + + test("rejects names starting with 'system.'", () => { + expect(() => assertNewCollectionName("system.views")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("system.profile")).toThrow(MongoApiError); + }); + + test("rejects names starting with numbers or containing invalid characters", () => { + expect(() => assertNewCollectionName("123users")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("-users")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("my.collection")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("my collection")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("coll@test")).toThrow(MongoApiError); + }); +}); + +describe("assertNoOperatorKeys", () => { + test("allows documents without top-level operator keys", () => { + expect(() => assertNoOperatorKeys({ name: "Alice", age: 30 })).not.toThrow(); + expect(() => assertNoOperatorKeys({ _id: { $oid: "507f1f77bcf86cd799439011" } })).not.toThrow(); + expect(() => assertNoOperatorKeys({})).not.toThrow(); + }); + + test("throws MONGO_VALIDATION_ERROR if any top-level key starts with '$'", () => { + expect(() => assertNoOperatorKeys({ $set: { name: "Bob" } })).toThrow( + "replacement document must not contain top-level operator keys", + ); + expect(() => assertNoOperatorKeys({ name: "Bob", $inc: { count: 1 } })).toThrow( + "replacement document must not contain top-level operator keys", + ); + }); +}); + +describe("M4 replaceDocument validation (rejects before driver)", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects empty filter", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: {}, + replacement: { name: "test" }, + }), + ).rejects.toMatchObject({ code: "MONGO_BULK_OP_FORBIDDEN" }); + }); + + test("rejects non-object replacement", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: "123" }, + replacement: [1, 2, 3] as unknown as EJSONDocument, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects replacement with operator keys", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: "123" }, + replacement: { $set: { a: 1 } }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects changing _id between filter and replacement", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: { $oid: "507f1f77bcf86cd799439011" } }, + replacement: { _id: { $oid: "507f1f77bcf86cd799439022" }, name: "New" }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "Cannot change _id of a document", + }); + }); + + test("rejects oversized replacement", async () => { + const huge = "x".repeat(17 * 1024 * 1024); + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: "123" }, + replacement: { data: huge }, + }), + ).rejects.toMatchObject({ code: "MONGO_LIMIT_EXCEEDED" }); + }); +}); + +describe("M4 createCollection validation", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects invalid collection name", async () => { + await expect( + mongoConnector.createCollection(dummy, { collection: "system.mycoll" }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { collection: "invalid.name" }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects capped collection without size or size < 4096", async () => { + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true } as any, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 2048 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 5000.5 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects capped collection with max <= 0 or non-integer", async () => { + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 8192, max: 0 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 8192, max: -5 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 8192, max: 10.5 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); +}); + +describe("M4 dropCollection validation", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects dropping system collection", async () => { + await expect( + mongoConnector.dropCollection(dummy, { collection: "system.views" }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: "Cannot drop system collection", + }); + + await expect( + mongoConnector.dropCollection(dummy, { collection: "system.profile" }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: "Cannot drop system collection", + }); + }); + + test("rejects invalid collection name", async () => { + await expect(mongoConnector.dropCollection(dummy, { collection: "" })).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + }); + }); +}); + +describe("M4 cache invalidation", () => { + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/cachedb", database: "cachedb" }; + + test("clearCollectionCache and flushConnection invalidate collection metadata and page cache", () => { + mongodbCache.setCollectionStats(cfg, "coll1", { + collection: "coll1", + count: 10, + sizeBytes: 200, + avgObjSize: 20, + }); + mongodbCache.setDocumentPage(cfg, "p1", [{ doc: 1 }]); + mongodbCache.setTableList(cfg, [{ name: "coll1", type: "TABLE", schema: "cachedb" }]); + + expect(mongodbCache.getCollectionStats(cfg, "coll1")).toBeDefined(); + expect(mongodbCache.getDocumentPage(cfg, "p1")).toBeDefined(); + expect(mongodbCache.getTableList(cfg)).toBeDefined(); + + mongodbCache.clearCollectionCache(cfg, "coll1"); + expect(mongodbCache.getCollectionStats(cfg, "coll1")).toBeNull(); + expect(mongodbCache.getDocumentPage(cfg, "p1")).toBeNull(); + expect(mongodbCache.getTableList(cfg)).toBeDefined(); + + mongodbCache.flushConnection(cfg); + expect(mongodbCache.getTableList(cfg)).toBeNull(); + }); +}); + +describe("M4 QueryExecutor routing", () => { + const executor = new QueryExecutor(); + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects replaceDocument on non-MongoDB connection", async () => { + await expect( + executor.replaceDocument(cfg, DBType.POSTGRES, { + collection: "c", + filter: { a: 1 }, + replacement: { a: 2 }, + }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); + + test("rejects createCollection on non-MongoDB connection", async () => { + await expect( + executor.createCollection(cfg, DBType.POSTGRES, { collection: "c" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); + + test("rejects dropCollection on non-MongoDB connection", async () => { + await expect( + executor.dropCollection(cfg, DBType.POSTGRES, { collection: "c" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); +}); diff --git a/bridge/__tests__/connectors/mongoSecurity.test.ts b/bridge/__tests__/connectors/mongoSecurity.test.ts new file mode 100644 index 00000000..386879d5 --- /dev/null +++ b/bridge/__tests__/connectors/mongoSecurity.test.ts @@ -0,0 +1,520 @@ +import { describe, expect, test, jest } from "@jest/globals"; +import { + assertSafeExpressions, + assertMongoNotReadOnly, + mongoConnector, + MongoApiError, +} from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; +import { MongoBulkEngine, assertNotReadOnly } from "../../src/services/mongoBulkEngine"; +import { MigrationHandlers } from "../../src/handlers/migrationHandlers"; +import { DBType } from "../../src/types"; + +describe("Mongo Security Test Suite", () => { + describe("assertSafeExpressions", () => { + test("throws on $where at top level", () => { + expect(() => { + assertSafeExpressions({ $where: "this.a > 3" }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$where' is forbidden"), + }), + ); + }); + + test("throws on $where nested in objects", () => { + expect(() => { + assertSafeExpressions({ + query: { + nested: { + $where: "function() { return true; }", + }, + }, + }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$where' is forbidden"), + }), + ); + }); + + test("throws on $where nested in arrays", () => { + expect(() => { + assertSafeExpressions({ + items: [{ field: "valid" }, { $where: "sleep(5000)" }], + }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$where' is forbidden"), + }), + ); + }); + + test("throws on $function at top level and nested", () => { + expect(() => { + assertSafeExpressions({ + $function: { + body: "function() { return 1; }", + args: [], + lang: "js", + }, + }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$function' is forbidden"), + }), + ); + + expect(() => { + assertSafeExpressions({ + $match: { + val: { + $function: { + body: "function(v) { return v > 10; }", + args: ["$val"], + lang: "js", + }, + }, + }, + }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$function' is forbidden"), + }), + ); + }); + + test("throws on $accumulator at top level and nested", () => { + expect(() => { + assertSafeExpressions({ + $accumulator: { + init: "function() { return 0; }", + accumulate: "function(state, val) { return state + val; }", + accumulateArgs: ["$val"], + merge: "function(s1, s2) { return s1 + s2; }", + lang: "js", + }, + }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$accumulator' is forbidden"), + }), + ); + + expect(() => { + assertSafeExpressions({ + $group: { + _id: "$dept", + total: { + $accumulator: { + init: "function() { return 0; }", + accumulate: "function(s, v) { return s + v; }", + accumulateArgs: ["$salary"], + merge: "function(s1, s2) { return s1 + s2; }", + lang: "js", + }, + }, + }, + }); + }).toThrow( + expect.objectContaining({ + code: "MONGO_FORBIDDEN_OPERATOR", + message: expect.stringContaining("Operator '$accumulator' is forbidden"), + }), + ); + }); + + test("permits standard operators and normal fields", () => { + expect(() => { + assertSafeExpressions({ + $set: { status: "active", count: 10 }, + $inc: { score: 5 }, + $match: { age: { $gte: 18, $lte: 65 } }, + $group: { _id: "$dept", total: { $sum: "$salary" } }, + }); + }).not.toThrow(); + + expect(() => { + assertSafeExpressions({ + name: "Alice", + email: "alice@example.com", + profile: { + bio: "Engineer", + tags: ["typescript", "mongodb"], + }, + scores: [95, 87, 92], + isActive: true, + count: null, + }); + }).not.toThrow(); + }); + + test("handles deep nesting (>20 levels without stack overflow)", () => { + let nested: any = { leaf: "value" }; + for (let i = 0; i < 30; i++) { + nested = { child: nested }; + } + + // Must terminate gracefully with MONGO_LIMIT_EXCEEDED instead of RangeError (Call stack exceeded) + expect(() => { + assertSafeExpressions(nested); + }).toThrow( + expect.objectContaining({ + code: "MONGO_LIMIT_EXCEEDED", + message: expect.stringContaining("Expression nesting exceeds maximum depth"), + }), + ); + + // Verify with nested arrays + let nestedArr: any = ["leaf"]; + for (let i = 0; i < 30; i++) { + nestedArr = [nestedArr]; + } + + expect(() => { + assertSafeExpressions(nestedArr); + }).toThrow( + expect.objectContaining({ + code: "MONGO_LIMIT_EXCEEDED", + }), + ); + }); + }); + + describe("assertMongoNotReadOnly", () => { + test("throws when conn.readonly: true", () => { + const conn: MongoConfig = { uri: "mongodb://localhost:27017/test", readonly: true }; + expect(() => { + assertMongoNotReadOnly(conn, "insertDocument"); + }).toThrow("Database is configured as read-only. 'insertDocument' is prohibited."); + + try { + assertMongoNotReadOnly(conn, "testOp"); + } catch (err: any) { + expect(err).toBeInstanceOf(MongoApiError); + expect(err.code).toBe("MONGO_READ_ONLY"); + } + }); + + test("throws when (conn as any).readOnly: true", () => { + const conn = { uri: "mongodb://localhost:27017/test", readOnly: true } as any; + expect(() => { + assertMongoNotReadOnly(conn, "dropIndex"); + }).toThrow("Database is configured as read-only. 'dropIndex' is prohibited."); + }); + + test("permits when conn.readonly is false or undefined", () => { + expect(() => { + assertMongoNotReadOnly( + { uri: "mongodb://localhost:27017/test", readonly: false }, + "insertDocument", + ); + }).not.toThrow(); + + expect(() => { + assertMongoNotReadOnly({ uri: "mongodb://localhost:27017/test" }, "insertDocument"); + }).not.toThrow(); + }); + + describe("rejects through connector mutating operations", () => { + const readOnlyConn: MongoConfig = { + uri: "mongodb://localhost:27017/test", + database: "test", + readonly: true, + }; + + test("insertDocument rejects", async () => { + await expect( + mongoConnector.insertDocument(readOnlyConn, { + collection: "users", + document: { name: "test" }, + }), + ).rejects.toThrow("Database is configured as read-only. 'insertDocument' is prohibited."); + }); + + test("updateDocument rejects", async () => { + await expect( + mongoConnector.updateDocument(readOnlyConn, { + collection: "users", + filter: { name: "test" }, + update: { $set: { age: 30 } }, + }), + ).rejects.toThrow("Database is configured as read-only. 'updateDocument' is prohibited."); + }); + + test("replaceDocument rejects", async () => { + await expect( + mongoConnector.replaceDocument(readOnlyConn, { + collection: "users", + filter: { name: "test" }, + replacement: { name: "test2", age: 31 }, + }), + ).rejects.toThrow("Database is configured as read-only. 'replaceDocument' is prohibited."); + }); + + test("deleteDocument rejects", async () => { + await expect( + mongoConnector.deleteDocument(readOnlyConn, { + collection: "users", + filter: { name: "test" }, + }), + ).rejects.toThrow("Database is configured as read-only. 'deleteDocument' is prohibited."); + }); + + test("createCollection rejects", async () => { + await expect( + mongoConnector.createCollection(readOnlyConn, { + collection: "new_users", + }), + ).rejects.toThrow("Database is configured as read-only. 'createCollection' is prohibited."); + }); + + test("dropCollection rejects", async () => { + await expect( + mongoConnector.dropCollection(readOnlyConn, { + collection: "users", + }), + ).rejects.toThrow("Database is configured as read-only. 'dropCollection' is prohibited."); + }); + + test("createIndexes rejects", async () => { + await expect( + mongoConnector.createIndexes(readOnlyConn, { + collection: "users", + key: { email: 1 }, + }), + ).rejects.toThrow("Database is configured as read-only. 'createIndexes' is prohibited."); + }); + + test("dropIndex rejects", async () => { + await expect( + mongoConnector.dropIndex(readOnlyConn, { + collection: "users", + indexName: "idx_email", + }), + ).rejects.toThrow("Database is configured as read-only. 'dropIndex' is prohibited."); + }); + + test("hideIndex rejects", async () => { + await expect( + mongoConnector.hideIndex(readOnlyConn, { + collection: "users", + indexName: "idx_email", + hidden: true, + }), + ).rejects.toThrow("Database is configured as read-only. 'hideIndex' is prohibited."); + }); + }); + }); + + describe("Bulk engine mutations read-only rejection", () => { + test("assertNotReadOnly directly throws MONGO_READ_ONLY", () => { + expect(() => { + assertNotReadOnly({ readonly: true }, {}); + }).toThrow("Database is in read-only mode"); + + expect(() => { + assertNotReadOnly({}, { readonly: true }); + }).toThrow("Database is in read-only mode"); + + expect(() => { + assertNotReadOnly({ readOnly: true }, {}); + }).toThrow("Database is in read-only mode"); + + expect(() => { + assertNotReadOnly({}, { readOnly: true }); + }).toThrow("Database is in read-only mode"); + + expect(() => { + assertNotReadOnly({ readonly: false }, { readonly: false }); + }).not.toThrow(); + }); + + test("bulkInsert rejects when conn.readonly: true", async () => { + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + }), + } as any; + const engine = new MongoBulkEngine(mockDbService); + + await expect( + engine.bulkInsert({ + dbId: "test_db", + collection: "items", + documents: [{ x: 1 }], + }), + ).rejects.toMatchObject({ + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + test("bulkUpdate rejects when conn.readonly: true", async () => { + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + }), + } as any; + const engine = new MongoBulkEngine(mockDbService); + + await expect( + engine.bulkUpdate({ + dbId: "test_db", + collection: "items", + filter: { x: 1 }, + update: { $set: { y: 2 } }, + }), + ).rejects.toMatchObject({ + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + test("bulkDelete rejects when conn.readonly: true", async () => { + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + }), + } as any; + const engine = new MongoBulkEngine(mockDbService); + + await expect( + engine.bulkDelete({ + dbId: "test_db", + collection: "items", + filter: { x: 1 }, + }), + ).rejects.toMatchObject({ + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + test("importChunk rejects when conn.readonly: true", async () => { + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + }), + } as any; + const engine = new MongoBulkEngine(mockDbService); + + // Create an active import operation in OpManager + const state = engine.opManager.createOperation("import", "test_db", "items", "json"); + + await expect( + engine.importChunk({ + operationId: state.operation.operationId, + chunkIndex: 0, + isLastChunk: true, + payload: JSON.stringify([{ a: 1 }]), + }), + ).rejects.toMatchObject({ + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + }); + + describe("Migration apply/rollback read-only rejection", () => { + test("applyMigration rejects when conn.readonly: true", async () => { + const mockRpc = { sendResponse: jest.fn(), sendError: jest.fn() }; + const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() } as any; + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + }), + } as any; + + const handlers = new MigrationHandlers(mockRpc as any, mockLogger, mockDbService, {} as any); + + await handlers.handleApplyMigration({ dbId: "test_db", version: 1 }, 101); + + expect(mockRpc.sendError).toHaveBeenCalledWith(101, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + expect(mockRpc.sendResponse).not.toHaveBeenCalled(); + }); + + test("applyMigration rejects when db.readonly: true", async () => { + const mockRpc = { sendResponse: jest.fn(), sendError: jest.fn() }; + const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() } as any; + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: true }, + conn: { uri: "mongodb://localhost:27017/test", readonly: false }, + }), + } as any; + + const handlers = new MigrationHandlers(mockRpc as any, mockLogger, mockDbService, {} as any); + + await handlers.handleApplyMigration({ dbId: "test_db", version: 1 }, 102); + + expect(mockRpc.sendError).toHaveBeenCalledWith(102, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + test("rollbackMigration rejects when conn.readonly: true", async () => { + const mockRpc = { sendResponse: jest.fn(), sendError: jest.fn() }; + const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() } as any; + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + }), + } as any; + + const handlers = new MigrationHandlers(mockRpc as any, mockLogger, mockDbService, {} as any); + + await handlers.handleRollbackMigration({ dbId: "test_db", version: 1 }, 201); + + expect(mockRpc.sendError).toHaveBeenCalledWith(201, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + expect(mockRpc.sendResponse).not.toHaveBeenCalled(); + }); + + test("rollbackMigration rejects when db.readonly: true", async () => { + const mockRpc = { sendResponse: jest.fn(), sendError: jest.fn() }; + const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() } as any; + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "test_db", readonly: true }, + conn: { uri: "mongodb://localhost:27017/test", readonly: false }, + }), + } as any; + + const handlers = new MigrationHandlers(mockRpc as any, mockLogger, mockDbService, {} as any); + + await handlers.handleRollbackMigration({ dbId: "test_db", version: 1 }, 202); + + expect(mockRpc.sendError).toHaveBeenCalledWith(202, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + }); +}); diff --git a/bridge/__tests__/connectors/mongodb.test.ts b/bridge/__tests__/connectors/mongodb.test.ts new file mode 100644 index 00000000..f2e97345 --- /dev/null +++ b/bridge/__tests__/connectors/mongodb.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, test, beforeAll, afterAll, jest } from "@jest/globals"; +import { MongoClient, ObjectId } from "mongodb"; +import * as mongoConnector from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const MONGO_HOST = process.env.REAL_MONGODB_HOST || "127.0.0.1"; +const MONGO_PORT = process.env.REAL_MONGODB_PORT || "27017"; +const MONGO_DB = process.env.REAL_MONGODB_DATABASE || "testdb"; + +// Credential-free URI: the test mongo service runs without auth. +const validConfig: MongoConfig = { + uri: process.env.REAL_MONGODB_URI || `mongodb://${MONGO_HOST}:${MONGO_PORT}/${MONGO_DB}`, + database: MONGO_DB, +}; + +// A closed local port with a short selection timeout → fast, deterministic failure. +const invalidConfig: MongoConfig = { + uri: "mongodb://127.0.0.1:1/testdb?serverSelectionTimeoutMS=5000", + database: "testdb", +}; + +let seedClient: MongoClient | null = null; + +async function seedFixtures(): Promise { + seedClient = new MongoClient(validConfig.uri as string, { + serverSelectionTimeoutMS: 5000, + connectTimeoutMS: 5000, + }); + await seedClient.connect(); + const db = seedClient.db(MONGO_DB); + + for (const name of ["users", "orders", "active_users_view"]) { + try { + await db.dropCollection(name); + } catch { + // may not exist + } + } + + const userIds: ObjectId[] = []; + const userInsert = await db.collection("users").insertMany([ + { name: "John Doe", email: "john@example.com", age: 30, active: true, createdAt: new Date() }, + { name: "Jane Smith", email: "jane@example.com", age: 27, active: true, createdAt: new Date() }, + { name: "Bob Lee", email: "bob@example.com", age: 45, active: false, createdAt: new Date() }, + ]); + for (const id of Object.values(userInsert.insertedIds)) userIds.push(id); + + await db.collection("orders").insertMany([ + { + userId: userIds[0], + items: [ + { sku: "ABC-1", qty: 2, price: 9.99 }, + { sku: "XYZ-9", qty: 1, price: 19.5 }, + ], + total: 39.48, + status: "placed", + orderedAt: new Date(), + }, + { + userId: userIds[1], + items: [{ sku: "ABC-1", qty: 5, price: 9.99 }], + total: 49.95, + status: "shipped", + orderedAt: new Date(), + }, + ]); + + await db.command({ + create: "active_users_view", + viewOn: "users", + pipeline: [{ $match: { active: true } }], + }); +} + +describe("MongoDB Connector", () => { + jest.setTimeout(30000); + + beforeAll(async () => { + await seedFixtures(); + }); + + afterAll(async () => { + await mongoConnector.closeAllMongoClients(); + if (seedClient) { + await seedClient.close(); + seedClient = null; + } + }); + + test("fails to connect with an invalid config", async () => { + const result = await mongoConnector.testConnection(invalidConfig); + expect(result.ok).toBe(false); + expect(result.status).toBe("disconnected"); + }); + + test("connects with a valid config", async () => { + const result = await mongoConnector.testConnection(validConfig); + expect(result).toStrictEqual({ + message: "Connection successful", + status: "connected", + ok: true, + }); + }); + + test("lists collections (including a view)", async () => { + const collections = await mongoConnector.listCollections(validConfig); + expect(Array.isArray(collections)).toBe(true); + const byName = new Map(collections.map((c) => [c.name, c.type])); + expect(byName.get("users")).toBe("collection"); + expect(byName.get("orders")).toBe("collection"); + expect(byName.get("active_users_view")).toBe("view"); + }); + + test("lists tables as the relational TableInfo adapter", async () => { + const tables = await mongoConnector.listTables(validConfig); + expect(Array.isArray(tables)).toBe(true); + const byName = new Map(tables.map((t) => [t.name, t.type])); + expect(byName.get("users")).toBe("BASE TABLE"); + expect(byName.get("active_users_view")).toBe("VIEW"); + expect(tables.every((t) => t.schema === MONGO_DB)).toBe(true); + }); + + test("returns database statistics", async () => { + const stats = await mongoConnector.getDBStats(validConfig); + expect(stats).toHaveProperty("total_tables"); + expect(stats).toHaveProperty("total_rows"); + expect(stats).toHaveProperty("total_db_size_mb"); + expect(stats.total_tables).toBeGreaterThanOrEqual(2); + expect(stats.total_rows).toBeGreaterThanOrEqual(5); + }); + + test("returns a single pseudo-schema name", async () => { + const names = await mongoConnector.listSchemaNames(validConfig); + expect(Array.isArray(names)).toBe(true); + expect(names).toContain(MONGO_DB); + }); + + test("connectToDatabase returns collections as pseudo-schemas with no migrations", async () => { + const result = await mongoConnector.connectToDatabase(validConfig, "mongo-integration-id"); + expect(result.baselined).toBe(false); + expect(result.schema.map((s) => s.name)).toContain("users"); + expect(result.migrations.local).toEqual([]); + expect(result.migrations.applied).toEqual([]); + }); +}); diff --git a/bridge/__tests__/databaseService.mongo.test.ts b/bridge/__tests__/databaseService.mongo.test.ts new file mode 100644 index 00000000..551e7a9e --- /dev/null +++ b/bridge/__tests__/databaseService.mongo.test.ts @@ -0,0 +1,89 @@ +import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; +import fs from "fs/promises"; +import fsSync from "fs"; +import os from "os"; +import path from "path"; + +// RELWAVE_HOME must be set BEFORE the databaseService (and its dbStore/config +// singletons) are loaded, so point it at an isolated temp directory. +const TEST_RELWAVE_HOME = path.join(os.tmpdir(), `mongo-db-service-test-${Date.now()}`); +process.env.RELWAVE_HOME = TEST_RELWAVE_HOME; + +const { DatabaseService } = require("../src/services/databaseService"); + +const { dbStoreInstance } = require("../src/services/dbStore"); + +const SECRET = "sup3r-s3cret-pw"; +const createdIds: string[] = []; + +describe("DatabaseService MongoDB credential handling", () => { + const dbService = new DatabaseService(); + + beforeAll(async () => { + if (!fsSync.existsSync(TEST_RELWAVE_HOME)) { + await fs.mkdir(TEST_RELWAVE_HOME, { recursive: true }); + } + }); + + afterAll(async () => { + for (const id of createdIds) { + try { + await dbService.deleteDatabase(id); + } catch { + // ignore + } + } + if (fsSync.existsSync(TEST_RELWAVE_HOME)) { + await fs.rm(TEST_RELWAVE_HOME, { recursive: true, force: true }); + } + }); + + test("strips the embedded password from the persisted URI and AES-stores it", async () => { + const url = `mongodb://testuser:${SECRET}@localhost:27017/testdb`; + const res = await dbService.addDatabase({ name: "MongoCredDB", type: "mongodb", url }); + createdIds.push(res.id); + + expect(res.id).toBeDefined(); + + const meta = await dbService.getDatabase(res.id); + expect(meta?.type).toBe("mongodb"); + // The persisted URI is sanitized — no plaintext password. + expect(meta?.url).toBe("mongodb://testuser@localhost:27017/testdb"); + expect(meta?.url).not.toContain(SECRET); + expect(meta?.credentialId).toBeDefined(); + + // The password is retrievable through the encrypted credential store. + const pwd = meta ? await dbStoreInstance.getPasswordFor(meta) : null; + expect(pwd).toBe(SECRET); + + // Byte-level scan of the persisted metadata finds no plaintext password. + const raw = await fs.readFile(path.join(TEST_RELWAVE_HOME, "databases.json"), "utf-8"); + expect(raw).not.toContain(SECRET); + }); + + test("reconstructs the authenticated URI in memory at runtime, never persisting it", async () => { + const url = `mongodb://alice:${SECRET}@localhost:27017/proddb`; + const res = await dbService.addDatabase({ name: "MongoRuntimeDB", type: "mongodb", url }); + createdIds.push(res.id); + + const { conn, dbType } = await dbService.getDatabaseConnection(res.id); + expect(dbType).toBe("mongodb"); + // Runtime config carries the password for the driver. + expect((conn as { uri?: string }).uri).toContain(SECRET); + expect((conn as { uri?: string }).uri).toContain("alice"); + + // But the persisted metadata keeps the sanitized URI. + const meta = await dbService.getDatabase(res.id); + expect(meta?.url).toBe("mongodb://alice@localhost:27017/proddb"); + expect(meta?.url).not.toContain(SECRET); + + const raw = await fs.readFile(path.join(TEST_RELWAVE_HOME, "databases.json"), "utf-8"); + expect(raw).not.toContain(SECRET); + }); + + test("requires name and type for a MongoDB connection", async () => { + await expect( + dbService.addDatabase({ type: "mongodb", url: "mongodb://localhost/db" }), + ).rejects.toThrow("Missing required field: name"); + }); +}); diff --git a/bridge/__tests__/handlers/migrationHandlers.mongo.test.ts b/bridge/__tests__/handlers/migrationHandlers.mongo.test.ts new file mode 100644 index 00000000..af2afa52 --- /dev/null +++ b/bridge/__tests__/handlers/migrationHandlers.mongo.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, test, beforeEach, jest } from "@jest/globals"; +import { MigrationHandlers } from "../../src/handlers/migrationHandlers"; +import { DBType } from "../../src/types"; +import { mongoMigrationEngine } from "../../src/services/mongoMigrationEngine"; + +describe("MigrationHandlers MongoDB Unit Tests (bridge/src/handlers/migrationHandlers.ts)", () => { + let mockRpc: any; + let mockLogger: any; + let mockDbService: any; + let mockQueryExecutor: any; + let handlers: MigrationHandlers; + + beforeEach(() => { + mockRpc = { + sendResponse: jest.fn(), + sendError: jest.fn(), + sendNotification: jest.fn(), + }; + mockLogger = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + }; + mockDbService = { + getDatabaseConnection: jest.fn().mockImplementation(() => + Promise.resolve({ + db: { id: "mongo-db-1" }, + conn: { uri: "mongodb://localhost:27017/testdb", database: "testdb" }, + dbType: DBType.MONGODB, + }), + ), + }; + mockQueryExecutor = { + listSchemas: jest.fn(), + }; + + handlers = new MigrationHandlers(mockRpc, mockLogger, mockDbService, mockQueryExecutor); + }); + + describe("handleValidateMigration", () => { + test("rejects missing manifest with BAD_REQUEST", async () => { + await handlers.handleValidateMigration({}, 1); + expect(mockRpc.sendError).toHaveBeenCalledWith(1, { + code: "BAD_REQUEST", + message: "Missing manifest", + }); + }); + + test("validates manifest for MongoDB database", async () => { + const manifest: any = { + version: "1.0", + id: "20261031120000", + name: "test_migration", + up: [{ type: "createCollection", collection: "test" }], + }; + + await handlers.handleValidateMigration({ dbId: "mongo-db-1", manifest }, 2); + expect(mockRpc.sendResponse).toHaveBeenCalledWith( + 2, + expect.objectContaining({ + ok: true, + data: expect.objectContaining({ valid: true }), + }), + ); + }); + }); + + describe("handleListMigrations", () => { + test("rejects missing dbId with BAD_REQUEST", async () => { + await handlers.handleListMigrations({}, 1); + expect(mockRpc.sendError).toHaveBeenCalledWith(1, { + code: "BAD_REQUEST", + message: "Missing dbId", + }); + }); + + test("delegates to mongoMigrationEngine.listMigrations for mongodb dbType", async () => { + const spy = jest + .spyOn(mongoMigrationEngine, "listMigrations") + .mockImplementation(async () => ({ + appliedMigrations: [], + pendingMigrations: [], + isLocked: false, + })); + + await handlers.handleListMigrations({ dbId: "mongo-db-1", projectRoot: "/fake/root" }, 2); + + expect(spy).toHaveBeenCalled(); + expect(mockRpc.sendResponse).toHaveBeenCalledWith( + 2, + expect.objectContaining({ + ok: true, + data: expect.objectContaining({ isLocked: false }), + }), + ); + spy.mockRestore(); + }); + }); + + describe("handleGetMigrationStatus", () => { + test("delegates to mongoMigrationEngine.getMigrationStatus", async () => { + const spy = jest + .spyOn(mongoMigrationEngine, "getMigrationStatus") + .mockImplementation(async () => ({ + lockStatus: { isLocked: false, isExpired: false }, + pendingCount: 0, + })); + + await handlers.handleGetMigrationStatus( + { dbId: "mongo-db-1", migrationId: "20261031120000" }, + 3, + ); + + expect(spy).toHaveBeenCalled(); + expect(mockRpc.sendResponse).toHaveBeenCalledWith( + 3, + expect.objectContaining({ + ok: true, + data: expect.objectContaining({ + lockStatus: { isLocked: false, isExpired: false }, + }), + }), + ); + spy.mockRestore(); + }); + }); + + describe("handleApplyMigration", () => { + test("rejects read-only MongoDB database", async () => { + mockDbService.getDatabaseConnection.mockImplementationOnce(() => + Promise.resolve({ + db: { id: "mongo-db-1", readOnly: true }, + conn: { uri: "mongodb://localhost:27017/testdb", database: "testdb" }, + dbType: DBType.MONGODB, + }), + ); + + await handlers.handleApplyMigration({ dbId: "mongo-db-1", version: "20261031120000" }, 4); + expect(mockRpc.sendError).toHaveBeenCalledWith(4, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + test("delegates apply to mongoMigrationEngine.applyMigration", async () => { + const spy = jest + .spyOn(mongoMigrationEngine, "applyMigration") + .mockImplementation(async () => ({ + success: true, + appliedCount: 1, + results: [], + })); + + await handlers.handleApplyMigration( + { dbId: "mongo-db-1", version: "20261031120000", projectRoot: "/fake/root" }, + 5, + ); + + expect(spy).toHaveBeenCalled(); + expect(mockRpc.sendResponse).toHaveBeenCalledWith( + 5, + expect.objectContaining({ + ok: true, + data: expect.objectContaining({ success: true, appliedCount: 1 }), + }), + ); + spy.mockRestore(); + }); + }); + + describe("handleRollbackMigration", () => { + test("rejects read-only MongoDB database", async () => { + mockDbService.getDatabaseConnection.mockImplementationOnce(() => + Promise.resolve({ + db: { id: "mongo-db-1", readOnly: true }, + conn: { uri: "mongodb://localhost:27017/testdb", database: "testdb" }, + dbType: DBType.MONGODB, + }), + ); + + await handlers.handleRollbackMigration({ dbId: "mongo-db-1", version: "20261031120000" }, 6); + expect(mockRpc.sendError).toHaveBeenCalledWith(6, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + test("delegates rollback to mongoMigrationEngine.rollbackMigration", async () => { + const spy = jest + .spyOn(mongoMigrationEngine, "rollbackMigration") + .mockImplementation(async () => ({ + success: true, + version: "20261031120000", + status: "ROLLED_BACK", + executionTimeMs: 10, + stepsCompensated: 1, + })); + + await handlers.handleRollbackMigration( + { dbId: "mongo-db-1", version: "20261031120000", projectRoot: "/fake/root" }, + 7, + ); + + expect(spy).toHaveBeenCalled(); + expect(mockRpc.sendResponse).toHaveBeenCalledWith( + 7, + expect.objectContaining({ + ok: true, + data: expect.objectContaining({ status: "ROLLED_BACK", stepsCompensated: 1 }), + }), + ); + spy.mockRestore(); + }); + }); +}); diff --git a/bridge/__tests__/handlers/mongoBulkHandlers.test.ts b/bridge/__tests__/handlers/mongoBulkHandlers.test.ts new file mode 100644 index 00000000..fee05941 --- /dev/null +++ b/bridge/__tests__/handlers/mongoBulkHandlers.test.ts @@ -0,0 +1,328 @@ +import { describe, expect, test, beforeEach, jest } from "@jest/globals"; +import { MongoBulkHandlers } from "../../src/handlers/mongoBulkHandlers"; +import { MongoApiError } from "../../src/connectors/mongo"; + +describe("MongoBulkHandlers JSON-RPC Dispatch & Parameter Validation Tests", () => { + let mockRpc: any; + let mockLogger: any; + let mockEngine: any; + let handlers: MongoBulkHandlers; + + const validUuid = "12345678-1234-4234-8234-123456789abc"; + + beforeEach(() => { + mockRpc = { + sendResponse: jest.fn(), + sendError: jest.fn(), + }; + mockLogger = { + error: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + }; + mockEngine = { + importStart: jest.fn().mockResolvedValue({ + operationId: validUuid, + status: "pending", + format: "json", + collection: "users", + }), + importChunk: jest.fn().mockResolvedValue({ + operationId: validUuid, + chunkIndex: 0, + status: "in_progress", + insertedCount: 10, + failedCount: 0, + isComplete: false, + totalInserted: 10, + totalFailed: 0, + }), + exportStart: jest.fn().mockResolvedValue({ + operationId: validUuid, + status: "pending", + collection: "orders", + format: "csv", + }), + getOperationStatus: jest.fn().mockResolvedValue({ + operationId: validUuid, + type: "import", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + }), + cancelOperation: jest.fn().mockResolvedValue({ + operationId: validUuid, + status: "cancelled", + message: "Operation cancelled", + }), + cleanupOperation: jest.fn().mockResolvedValue({ + operationId: validUuid, + success: true, + }), + exportSaveTo: jest.fn().mockResolvedValue({ + operationId: validUuid, + destinationPath: "/abs/path/export.json", + bytesWritten: 1234, + success: true, + }), + bulkInsert: jest.fn().mockResolvedValue({ + collection: "products", + insertedCount: 5, + failedCount: 0, + }), + bulkUpdate: jest.fn().mockResolvedValue({ + collection: "products", + matchedCount: 5, + modifiedCount: 5, + }), + bulkDelete: jest.fn().mockResolvedValue({ + collection: "products", + deletedCount: 2, + }), + }; + + handlers = new MongoBulkHandlers(mockRpc, mockLogger, mockEngine); + }); + + test("handleImportStart dispatches successfully and validates params", async () => { + await handlers.handleImportStart({ dbId: "db-1", collection: "users", format: "json" }, 1); + expect(mockEngine.importStart).toHaveBeenCalledWith({ + dbId: "db-1", + collection: "users", + format: "json", + options: undefined, + }); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(1, { + ok: true, + data: expect.objectContaining({ operationId: validUuid }), + }); + + // Invalid format + await handlers.handleImportStart({ dbId: "db-1", collection: "users", format: "xml" }, 2); + expect(mockRpc.sendError).toHaveBeenCalledWith(2, { + code: "BAD_REQUEST", + message: expect.stringContaining("format must be 'json', 'ndjson', or 'csv'"), + }); + }); + + test("handleImportChunk validates UUIDv4 and dispatches", async () => { + await handlers.handleImportChunk({ operationId: validUuid, chunkIndex: 0, payload: "[]" }, 3); + expect(mockEngine.importChunk).toHaveBeenCalledWith({ + operationId: validUuid, + chunkIndex: 0, + isLastChunk: undefined, + payload: "[]", + }); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(3, { + ok: true, + data: expect.objectContaining({ chunkIndex: 0 }), + }); + + // Invalid operationId (path traversal attempt) + await handlers.handleImportChunk( + { operationId: "../malicious", chunkIndex: 0, payload: "[]" }, + 4, + ); + expect(mockRpc.sendError).toHaveBeenCalledWith(4, { + code: "BAD_REQUEST", + message: expect.stringContaining("operationId must be a valid UUIDv4"), + }); + }); + + test("handleExportStart validates params and dispatches", async () => { + await handlers.handleExportStart({ dbId: "db-1", collection: "orders", format: "csv" }, 5); + expect(mockEngine.exportStart).toHaveBeenCalled(); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(5, { + ok: true, + data: expect.objectContaining({ format: "csv" }), + }); + }); + + test("handleGetOperationStatus, handleCancelOperation, and handleCleanupOperation validate operationId", async () => { + await handlers.handleGetOperationStatus({ operationId: validUuid }, 6); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(6, { + ok: true, + data: expect.objectContaining({ operationId: validUuid }), + }); + + await handlers.handleCancelOperation({ operationId: validUuid }, 7); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(7, { + ok: true, + data: expect.objectContaining({ status: "cancelled" }), + }); + + await handlers.handleCleanupOperation({ operationId: validUuid }, 8); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(8, { + ok: true, + data: { operationId: validUuid, success: true }, + }); + }); + + test("handleBulkInsert, handleBulkUpdate, handleBulkDelete validate mutations", async () => { + await handlers.handleBulkInsert( + { dbId: "db-1", collection: "products", documents: [{ sku: "A" }] }, + 9, + ); + expect(mockEngine.bulkInsert).toHaveBeenCalled(); + + await handlers.handleBulkUpdate( + { + dbId: "db-1", + collection: "products", + filter: { sku: "A" }, + update: { $set: { price: 10 } }, + }, + 10, + ); + expect(mockEngine.bulkUpdate).toHaveBeenCalled(); + + await handlers.handleBulkDelete( + { dbId: "db-1", collection: "products", filter: { sku: "A" } }, + 11, + ); + expect(mockEngine.bulkDelete).toHaveBeenCalled(); + }); + + test("propagates MongoApiError code and message correctly", async () => { + mockEngine.bulkInsert.mockRejectedValueOnce( + new MongoApiError("MONGO_READ_ONLY", "Database is in read-only mode"), + ); + + await handlers.handleBulkInsert( + { dbId: "db-1", collection: "products", documents: [{ sku: "A" }] }, + 12, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(12, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + }); + + describe("handleExportSaveTo", () => { + test("dispatches successfully with valid arguments", async () => { + await handlers.handleExportSaveTo( + { + operationId: validUuid, + destinationPath: "/abs/path/export.json", + options: { overwrite: true }, + }, + 13, + ); + + expect(mockEngine.exportSaveTo).toHaveBeenCalledWith({ + operationId: validUuid, + destinationPath: "/abs/path/export.json", + options: { overwrite: true }, + }); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(13, { + ok: true, + data: expect.objectContaining({ + operationId: validUuid, + destinationPath: "/abs/path/export.json", + bytesWritten: 1234, + success: true, + }), + }); + }); + + test("validates operationId is valid UUIDv4", async () => { + await handlers.handleExportSaveTo( + { + operationId: "invalid-uuid", + destinationPath: "/abs/path/export.json", + }, + 14, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(14, { + code: "BAD_REQUEST", + message: expect.stringContaining("operationId must be a valid UUIDv4"), + }); + expect(mockEngine.exportSaveTo).not.toHaveBeenCalled(); + }); + + test("validates destinationPath is non-empty string", async () => { + await handlers.handleExportSaveTo( + { + operationId: validUuid, + destinationPath: " ", + }, + 15, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(15, { + code: "BAD_REQUEST", + message: expect.stringContaining("destinationPath must be a non-empty string"), + }); + + await handlers.handleExportSaveTo( + { + operationId: validUuid, + destinationPath: null, + }, + 16, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(16, { + code: "BAD_REQUEST", + message: expect.stringContaining("destinationPath must be a non-empty string"), + }); + }); + + test("validates options must be object", async () => { + await handlers.handleExportSaveTo( + { + operationId: validUuid, + destinationPath: "/abs/path/export.json", + options: "not-an-object", + }, + 17, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(17, { + code: "BAD_REQUEST", + message: expect.stringContaining("options must be an object"), + }); + }); + + test("validates options.overwrite must be boolean", async () => { + await handlers.handleExportSaveTo( + { + operationId: validUuid, + destinationPath: "/abs/path/export.json", + options: { overwrite: "yes" as any }, + }, + 18, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(18, { + code: "BAD_REQUEST", + message: expect.stringContaining("options.overwrite must be a boolean"), + }); + }); + + test("propagates MongoApiError from engine", async () => { + mockEngine.exportSaveTo.mockRejectedValueOnce( + new MongoApiError("MONGO_FILE_EXISTS", "Destination file already exists"), + ); + + await handlers.handleExportSaveTo( + { + operationId: validUuid, + destinationPath: "/abs/path/export.json", + }, + 19, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(19, { + code: "MONGO_FILE_EXISTS", + message: "Destination file already exists", + }); + }); + }); +}); diff --git a/bridge/__tests__/handlers/mongoRelationshipHandlers.test.ts b/bridge/__tests__/handlers/mongoRelationshipHandlers.test.ts new file mode 100644 index 00000000..207e017f --- /dev/null +++ b/bridge/__tests__/handlers/mongoRelationshipHandlers.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, test, beforeEach, jest } from "@jest/globals"; +import { MongoRelationshipHandlers } from "../../src/handlers/mongoRelationshipHandlers"; + +describe("MongoRelationshipHandlers Dispatch & Validation Tests (M7 Phase 1 Remediation)", () => { + let mockRpc: any; + let mockLogger: any; + let mockEngine: any; + let handlers: MongoRelationshipHandlers; + + beforeEach(() => { + mockRpc = { + sendResponse: jest.fn(), + sendError: jest.fn(), + }; + mockLogger = { + error: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + }; + mockEngine = { + inferRelationships: jest.fn().mockResolvedValue({ + relationships: [], + insufficientEvidence: [], + inferredAt: "2026-11-02T12:00:00.000Z", + collectionsScanned: 5, + }), + listRelationships: jest.fn().mockResolvedValue({ + inferred: [], + confirmed: [], + rejectedIds: [], + manual: [], + insufficientEvidence: [], + }), + setRelationshipStatus: jest.fn().mockResolvedValue({ + relationshipId: "test-rel", + status: "confirmed", + }), + saveRelationship: jest.fn().mockResolvedValue({ + relationship: { id: "test-rel" }, + }), + deleteRelationship: jest.fn().mockResolvedValue({ + success: true, + relationshipId: "test-rel", + }), + }; + + handlers = new MongoRelationshipHandlers(mockRpc, mockLogger, mockEngine); + }); + + // ========================================================================= + // 1. handleInfer + // ========================================================================= + describe("handleInfer", () => { + test("dispatches infer successfully with valid params", async () => { + await handlers.handleInfer( + { + dbId: "mongo-1", + projectId: "proj-1", + collections: ["users", "orders"], + sampleSize: 50, + timeoutMs: 15000, + }, + 1, + ); + expect(mockEngine.inferRelationships).toHaveBeenCalledWith("mongo-1", { + projectId: "proj-1", + collections: ["users", "orders"], + sampleSize: 50, + timeoutMs: 15000, + maxCollections: undefined, + }); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(1, { + ok: true, + data: expect.objectContaining({ collectionsScanned: 5 }), + }); + }); + + test("rejects missing or empty dbId with BAD_REQUEST", async () => { + await handlers.handleInfer({}, 2); + expect(mockRpc.sendError).toHaveBeenCalledWith(2, { + code: "BAD_REQUEST", + message: expect.stringContaining("Missing or invalid required parameter: dbId"), + }); + + await handlers.handleInfer({ dbId: " " }, 3); + expect(mockRpc.sendError).toHaveBeenCalledWith(3, { + code: "BAD_REQUEST", + message: expect.stringContaining("Missing or invalid required parameter: dbId"), + }); + }); + + test("rejects invalid projectId with BAD_REQUEST", async () => { + await handlers.handleInfer({ dbId: "mongo-1", projectId: 123 }, 4); + expect(mockRpc.sendError).toHaveBeenCalledWith(4, { + code: "BAD_REQUEST", + message: expect.stringContaining("projectId must be a non-empty string"), + }); + + await handlers.handleInfer({ dbId: "mongo-1", projectId: " " }, 5); + expect(mockRpc.sendError).toHaveBeenCalledWith(5, { + code: "BAD_REQUEST", + message: expect.stringContaining("projectId must be a non-empty string"), + }); + }); + + test("rejects invalid collections with BAD_REQUEST", async () => { + await handlers.handleInfer({ dbId: "mongo-1", collections: "not-array" }, 6); + expect(mockRpc.sendError).toHaveBeenCalledWith(6, { + code: "BAD_REQUEST", + message: expect.stringContaining("collections must be an array of non-empty strings"), + }); + + await handlers.handleInfer({ dbId: "mongo-1", collections: ["valid", ""] }, 7); + expect(mockRpc.sendError).toHaveBeenCalledWith(7, { + code: "BAD_REQUEST", + message: expect.stringContaining("collections must be an array of non-empty strings"), + }); + }); + + test("rejects invalid sampleSize and timeoutMs", async () => { + await handlers.handleInfer({ dbId: "mongo-1", sampleSize: -10 }, 8); + expect(mockRpc.sendError).toHaveBeenCalledWith(8, { + code: "BAD_REQUEST", + message: expect.stringContaining("sampleSize must be a positive number"), + }); + + await handlers.handleInfer({ dbId: "mongo-1", timeoutMs: 0 }, 9); + expect(mockRpc.sendError).toHaveBeenCalledWith(9, { + code: "BAD_REQUEST", + message: expect.stringContaining("timeoutMs must be a positive number"), + }); + }); + }); + + // ========================================================================= + // 2. handleList + // ========================================================================= + describe("handleList", () => { + test("dispatches list successfully", async () => { + await handlers.handleList({ dbId: "mongo-1", projectId: "proj-1" }, 10); + expect(mockEngine.listRelationships).toHaveBeenCalledWith("mongo-1", "proj-1"); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(10, { + ok: true, + data: expect.objectContaining({ inferred: [] }), + }); + }); + + test("rejects missing dbId or invalid projectId", async () => { + await handlers.handleList({}, 11); + expect(mockRpc.sendError).toHaveBeenCalledWith(11, { + code: "BAD_REQUEST", + message: expect.stringContaining("Missing or invalid required parameter: dbId"), + }); + + await handlers.handleList({ dbId: "mongo-1", projectId: "" }, 12); + expect(mockRpc.sendError).toHaveBeenCalledWith(12, { + code: "BAD_REQUEST", + message: expect.stringContaining("projectId must be a non-empty string"), + }); + }); + }); + + // ========================================================================= + // 3. handleSetStatus + // ========================================================================= + describe("handleSetStatus", () => { + test("confirms relationship status successfully", async () => { + await handlers.handleSetStatus( + { dbId: "mongo-1", relationshipId: "rel-1", status: "confirmed" }, + 13, + ); + expect(mockEngine.setRelationshipStatus).toHaveBeenCalledWith( + "mongo-1", + "rel-1", + "confirmed", + undefined, + ); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(13, { + ok: true, + data: { relationshipId: "test-rel", status: "confirmed" }, + }); + }); + + test("rejects invalid status", async () => { + await handlers.handleSetStatus( + { dbId: "mongo-1", relationshipId: "rel-1", status: "invalid_status" }, + 14, + ); + expect(mockRpc.sendError).toHaveBeenCalledWith(14, { + code: "BAD_REQUEST", + message: 'Parameter status must be either "confirmed" or "rejected"', + }); + }); + + test("rejects missing relationshipId", async () => { + await handlers.handleSetStatus({ dbId: "mongo-1", status: "confirmed" }, 15); + expect(mockRpc.sendError).toHaveBeenCalledWith(15, { + code: "BAD_REQUEST", + message: expect.stringContaining("relationshipId"), + }); + }); + }); + + // ========================================================================= + // 4. handleSave + // ========================================================================= + describe("handleSave", () => { + test("saves relationship successfully with valid object", async () => { + const rel = { + id: "orders.user_id->users._id", + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + }; + + await handlers.handleSave({ dbId: "mongo-1", relationship: rel }, 16); + expect(mockEngine.saveRelationship).toHaveBeenCalledWith("mongo-1", rel, undefined); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(16, { + ok: true, + data: expect.objectContaining({ relationship: expect.any(Object) }), + }); + }); + + test("rejects invalid cardinality", async () => { + const rel = { + id: "orders.user_id->users._id", + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + cardinality: "not-a-valid-cardinality", + }; + + await handlers.handleSave({ dbId: "mongo-1", relationship: rel }, 17); + expect(mockRpc.sendError).toHaveBeenCalledWith(17, { + code: "BAD_REQUEST", + message: expect.stringContaining("Invalid cardinality"), + }); + }); + + test("rejects incomplete relationship object", async () => { + await handlers.handleSave( + { dbId: "mongo-1", relationship: { id: "rel-1", sourceCollection: "orders" } }, + 18, + ); + expect(mockRpc.sendError).toHaveBeenCalledWith(18, { + code: "BAD_REQUEST", + message: expect.stringContaining( + "Relationship object missing required non-empty string fields", + ), + }); + }); + }); + + // ========================================================================= + // 5. handleDelete + // ========================================================================= + describe("handleDelete", () => { + test("deletes relationship successfully", async () => { + await handlers.handleDelete({ dbId: "mongo-1", relationshipId: "rel-1" }, 19); + expect(mockEngine.deleteRelationship).toHaveBeenCalledWith("mongo-1", "rel-1", undefined); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(19, { + ok: true, + data: { success: true, relationshipId: "test-rel" }, + }); + }); + + test("rejects missing relationshipId", async () => { + await handlers.handleDelete({ dbId: "mongo-1" }, 20); + expect(mockRpc.sendError).toHaveBeenCalledWith(20, { + code: "BAD_REQUEST", + message: expect.stringContaining("relationshipId"), + }); + }); + }); +}); diff --git a/bridge/__tests__/handlers/queryHandlers.mql.test.ts b/bridge/__tests__/handlers/queryHandlers.mql.test.ts new file mode 100644 index 00000000..74ec919d --- /dev/null +++ b/bridge/__tests__/handlers/queryHandlers.mql.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, test, beforeEach, jest } from "@jest/globals"; +import { QueryHandlers } from "../../src/handlers/queryHandlers"; +import { MongoApiError } from "../../src/connectors/mongo"; +import { DBType } from "../../src/types"; + +describe("QueryHandlers MQL Unit Tests (bridge/src/handlers/queryHandlers.ts)", () => { + let mockRpc: any; + let mockLogger: any; + let mockSessions: any; + let mockDbService: any; + let mockQueryExecutor: any; + let handlers: QueryHandlers; + + beforeEach(() => { + mockRpc = { + sendResponse: jest.fn(), + sendError: jest.fn(), + sendNotification: jest.fn(), + }; + mockLogger = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + }; + mockSessions = { + registerCancel: jest.fn(), + }; + mockDbService = { + getDatabaseConnection: jest.fn().mockImplementation(() => + Promise.resolve({ + conn: { uri: "mongodb://localhost:27017/testdb", database: "testdb" }, + dbType: DBType.MONGODB, + }), + ), + }; + mockQueryExecutor = { + executeMql: jest.fn(), + explainMql: jest.fn(), + }; + + handlers = new QueryHandlers( + mockRpc, + mockLogger, + mockSessions, + mockDbService, + mockQueryExecutor, + ); + }); + + describe("handleExecuteMql", () => { + test("rejects missing dbId with MONGO_VALIDATION_ERROR", async () => { + await handlers.handleExecuteMql( + { collection: "users", op: { operation: "find", payload: { filter: {} } } }, + 1, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(1, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or op", + }); + expect(mockQueryExecutor.executeMql).not.toHaveBeenCalled(); + }); + + test("rejects missing collection with MONGO_VALIDATION_ERROR", async () => { + await handlers.handleExecuteMql( + { dbId: "conn-1", op: { operation: "find", payload: { filter: {} } } }, + 2, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(2, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or op", + }); + expect(mockQueryExecutor.executeMql).not.toHaveBeenCalled(); + }); + + test("rejects missing op with MONGO_VALIDATION_ERROR", async () => { + await handlers.handleExecuteMql({ dbId: "conn-1", collection: "users" }, 3); + + expect(mockRpc.sendError).toHaveBeenCalledWith(3, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or op", + }); + expect(mockQueryExecutor.executeMql).not.toHaveBeenCalled(); + }); + + test("handles database connection lookup failure via sendMongoError", async () => { + mockDbService.getDatabaseConnection.mockRejectedValueOnce(new Error("Connection not found")); + + await handlers.handleExecuteMql( + { + dbId: "invalid-id", + collection: "users", + op: { operation: "find", payload: { filter: {} } }, + }, + 4, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith( + 4, + expect.objectContaining({ + code: "MONGO_CONNECTION_ERROR", + }), + ); + expect(mockRpc.sendResponse).not.toHaveBeenCalled(); + }); + + test("successfully dispatches executeMql and sends response", async () => { + const mockResult = { + operation: "find", + collection: "users", + documents: [{ _id: "1", name: "Alice" }], + totalReturned: 1, + executionTimeMs: 5, + }; + mockQueryExecutor.executeMql.mockResolvedValueOnce(mockResult); + + const params = { + dbId: "mongo-1", + collection: "users", + op: { operation: "find", payload: { filter: { name: "Alice" } } }, + options: { maxTimeMS: 5000 }, + }; + + await handlers.handleExecuteMql(params, 5); + + expect(mockDbService.getDatabaseConnection).toHaveBeenCalledWith("mongo-1"); + expect(mockQueryExecutor.executeMql).toHaveBeenCalledWith( + expect.objectContaining({ database: "testdb" }), + DBType.MONGODB, + { + collection: "users", + op: params.op, + options: params.options, + }, + expect.any(Object), + ); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(5, { + ok: true, + data: mockResult, + }); + }); + + test("forwards MongoApiError from queryExecutor to rpc.sendError", async () => { + mockQueryExecutor.executeMql.mockRejectedValueOnce( + new MongoApiError( + "MONGO_FORBIDDEN_OPERATION", + "Operation 'drop' is forbidden in read-only query workspace. Use RelWave document mutation actions.", + ), + ); + + await handlers.handleExecuteMql( + { + dbId: "mongo-1", + collection: "users", + op: { operation: "drop" as any }, + }, + 6, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(6, { + code: "MONGO_FORBIDDEN_OPERATION", + message: + "Operation 'drop' is forbidden in read-only query workspace. Use RelWave document mutation actions.", + }); + expect(mockRpc.sendResponse).not.toHaveBeenCalled(); + }); + }); + + describe("handleExplainMql", () => { + test("rejects missing dbId with MONGO_VALIDATION_ERROR", async () => { + await handlers.handleExplainMql( + { collection: "users", target: { operation: "find", filter: {} } }, + 11, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(11, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or target", + }); + expect(mockQueryExecutor.explainMql).not.toHaveBeenCalled(); + }); + + test("rejects missing collection with MONGO_VALIDATION_ERROR", async () => { + await handlers.handleExplainMql( + { dbId: "conn-1", target: { operation: "find", filter: {} } }, + 12, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(12, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or target", + }); + expect(mockQueryExecutor.explainMql).not.toHaveBeenCalled(); + }); + + test("rejects missing target with MONGO_VALIDATION_ERROR", async () => { + await handlers.handleExplainMql({ dbId: "conn-1", collection: "users" }, 13); + + expect(mockRpc.sendError).toHaveBeenCalledWith(13, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or target", + }); + expect(mockQueryExecutor.explainMql).not.toHaveBeenCalled(); + }); + + test("handles database lookup failure via sendMongoError", async () => { + mockDbService.getDatabaseConnection.mockRejectedValueOnce(new Error("Lookup failed")); + + await handlers.handleExplainMql( + { + dbId: "invalid-id", + collection: "users", + target: { operation: "find", filter: {} }, + }, + 14, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith( + 14, + expect.objectContaining({ + code: "MONGO_CONNECTION_ERROR", + }), + ); + expect(mockRpc.sendResponse).not.toHaveBeenCalled(); + }); + + test("successfully dispatches explainMql and sends response", async () => { + const mockExplainResult = { + collection: "users", + operation: "find", + queryPlanner: { + plannerVersion: 1, + namespace: "testdb.users", + indexFilterSet: false, + winningPlan: { stage: "COLLSCAN" }, + }, + rawExplain: {}, + executionTimeMs: 3, + }; + mockQueryExecutor.explainMql.mockResolvedValueOnce(mockExplainResult); + + const params = { + dbId: "mongo-1", + collection: "users", + target: { operation: "find", filter: { status: "active" } } as const, + verbosity: "queryPlanner" as const, + options: { maxTimeMS: 5000 }, + }; + + await handlers.handleExplainMql(params, 15); + + expect(mockDbService.getDatabaseConnection).toHaveBeenCalledWith("mongo-1"); + expect(mockQueryExecutor.explainMql).toHaveBeenCalledWith( + expect.objectContaining({ database: "testdb" }), + DBType.MONGODB, + { + collection: "users", + target: params.target, + verbosity: params.verbosity, + options: params.options, + }, + expect.any(Object), + ); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(15, { + ok: true, + data: mockExplainResult, + }); + }); + + test("forwards MongoApiError from explainMql to rpc.sendError", async () => { + mockQueryExecutor.explainMql.mockRejectedValueOnce( + new MongoApiError( + "MONGO_WRITE_STAGE_FORBIDDEN", + "Stage '$out' is forbidden in read-only aggregation queries", + ), + ); + + await handlers.handleExplainMql( + { + dbId: "mongo-1", + collection: "users", + target: { operation: "aggregate", pipeline: [{ $out: "dump" }] }, + }, + 16, + ); + + expect(mockRpc.sendError).toHaveBeenCalledWith(16, { + code: "MONGO_WRITE_STAGE_FORBIDDEN", + message: "Stage '$out' is forbidden in read-only aggregation queries", + }); + expect(mockRpc.sendResponse).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/bridge/__tests__/integration/mongoDocuments.integration.test.ts b/bridge/__tests__/integration/mongoDocuments.integration.test.ts new file mode 100644 index 00000000..1a9017b8 --- /dev/null +++ b/bridge/__tests__/integration/mongoDocuments.integration.test.ts @@ -0,0 +1,349 @@ +import { describe, expect, test, beforeAll, afterAll, jest } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { mongoConnector, collectionExists, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +// --------------------------------------------------------------------------- +// M2 integration tests — require a live MongoDB. Uses a dedicated database so +// it never collides with the M1 `testdb` fixtures. Skips cleanly when the +// server is unreachable (spec §16.2). +// --------------------------------------------------------------------------- + +const MONGO_HOST = process.env.REAL_MONGODB_HOST || "127.0.0.1"; +const MONGO_PORT = process.env.REAL_MONGODB_PORT || "27017"; +const MONGO_DB = process.env.REAL_MONGODB_DATABASE_M2 || "relwave_m2_it"; + +const config: MongoConfig = { + uri: `mongodb://${MONGO_HOST}:${MONGO_PORT}/${MONGO_DB}`, + database: MONGO_DB, +}; + +let seedClient: MongoClient | null = null; +let connected = false; + +beforeAll(async () => { + jest.setTimeout(60000); + try { + seedClient = new MongoClient(config.uri as string); + await seedClient.connect(); + try { + await seedClient.db(MONGO_DB).dropDatabase(); + } catch { + // ignore — may lack dropDatabase privilege; collections are dropped per-test + } + connected = true; + } catch { + connected = false; + console.warn( + `[mongoDocuments.integration] MongoDB not reachable at ${MONGO_HOST}:${MONGO_PORT}; skipping M2 integration tests.`, + ); + } +}, 60000); + +afterAll(async () => { + if (seedClient) { + try { + await seedClient.close(); + } catch { + // ignore + } + seedClient = null; + } + await closeAllMongoClients(); +}); + +describe("M2 findDocuments", () => { + test("round-trips ObjectId / int32 / int64 / decimal / date / nested types", async () => { + if (!connected) return; + const oid = new (await import("mongodb")).ObjectId(); + const now = new Date("2024-01-01T00:00:00.000Z"); + const res = await mongoConnector.insertDocument(config, { + collection: "rt", + document: { + _id: { $oid: oid.toHexString() }, + name: "Alice", + age: { $numberInt: "30" }, + score: { $numberDecimal: "9.50" }, + big: { $numberLong: "9007199254740993" }, + createdAt: { $date: now.toISOString() }, + tags: ["a", "b"], + meta: { nested: true }, + }, + }); + expect(res.insertedId).toEqual({ $oid: oid.toHexString() }); + expect(res.document._id).toEqual({ $oid: oid.toHexString() }); + + const found = await mongoConnector.findDocuments(config, { + collection: "rt", + filter: { _id: { $oid: oid.toHexString() } }, + }); + expect(found.documents).toHaveLength(1); + const d = found.documents[0]; + expect(d.name).toBe("Alice"); + expect(d.age).toEqual({ $numberInt: "30" }); + expect(d.score).toEqual({ $numberDecimal: "9.50" }); + expect(d.big).toEqual({ $numberLong: "9007199254740993" }); + expect(d.createdAt).toEqual({ $date: now.toISOString() }); + expect(d.tags).toEqual(["a", "b"]); + expect(d.meta).toEqual({ nested: true }); + }); + + test("filters with $in, $gt (typed), $regex and _id $oid", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "filt", + document: { tag: "red", n: { $numberInt: "10" }, name: "Alice" }, + }); + await mongoConnector.insertDocument(config, { + collection: "filt", + document: { tag: "blue", n: { $numberInt: "20" }, name: "Bob" }, + }); + await mongoConnector.insertDocument(config, { + collection: "filt", + document: { tag: "red", n: { $numberInt: "30" }, name: "Carol" }, + }); + + const byIn = await mongoConnector.findDocuments(config, { + collection: "filt", + filter: { tag: { $in: ["red", "green"] } }, + }); + expect(byIn.total).toBe(2); + + const byGt = await mongoConnector.findDocuments(config, { + collection: "filt", + filter: { n: { $gt: { $numberInt: "15" } } }, + }); + expect(byGt.total).toBe(2); + + const byRegex = await mongoConnector.findDocuments(config, { + collection: "filt", + filter: { name: { $regex: "^A" } }, + }); + expect(byRegex.documents).toHaveLength(1); + expect(byRegex.documents[0].name).toBe("Alice"); + }); + + test("paginates with total / totalPages / hasMore", async () => { + if (!connected) return; + for (let i = 0; i < 25; i++) { + await mongoConnector.insertDocument(config, { + collection: "page", + document: { idx: { $numberInt: String(i) } }, + }); + } + const p1 = await mongoConnector.findDocuments(config, { + collection: "page", + pageSize: 10, + page: 1, + sort: { idx: 1 }, + }); + expect(p1.total).toBe(25); + expect(p1.documents).toHaveLength(10); + expect(p1.totalPages).toBe(3); + expect(p1.hasMore).toBe(true); + expect(p1.documents[0].idx).toEqual({ $numberInt: "0" }); + + const p3 = await mongoConnector.findDocuments(config, { + collection: "page", + pageSize: 10, + page: 3, + sort: { idx: 1 }, + }); + expect(p3.documents).toHaveLength(5); + expect(p3.hasMore).toBe(false); + }); + + test("applies sort and projection", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "sp", + document: { k: "a", secret: "s1", val: { $numberInt: "3" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "sp", + document: { k: "b", secret: "s2", val: { $numberInt: "1" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "sp", + document: { k: "c", secret: "s3", val: { $numberInt: "2" } }, + }); + + const res = await mongoConnector.findDocuments(config, { + collection: "sp", + sort: { val: 1 }, + projection: { k: 1, _id: 0 }, + }); + expect(res.documents.map((d) => d.k)).toEqual(["b", "c", "a"]); + expect(res.documents[0]).not.toHaveProperty("secret"); + expect(res.documents[0]).not.toHaveProperty("_id"); + }); +}); + +describe("M2 updateDocument", () => { + test("applies $set + $inc and returns the after-document", async () => { + if (!connected) return; + const ins = await mongoConnector.insertDocument(config, { + collection: "upd", + document: { name: "Bob", count: { $numberInt: "1" } }, + }); + const res = await mongoConnector.updateDocument(config, { + collection: "upd", + filter: { _id: ins.insertedId }, + update: { $set: { name: "Bobby" }, $inc: { count: { $numberInt: "5" } } }, + options: { returnDocument: "after" }, + }); + expect(res.matchedCount).toBe(1); + expect(res.modifiedCount).toBe(1); + expect(res.updatedDocument?.name).toBe("Bobby"); + expect(res.updatedDocument?.count).toEqual({ $numberInt: "6" }); + }); + + test("returnDocument=before returns the pre-update document and persists the change", async () => { + if (!connected) return; + const ins = await mongoConnector.insertDocument(config, { + collection: "updb", + document: { v: { $numberInt: "1" } }, + }); + const res = await mongoConnector.updateDocument(config, { + collection: "updb", + filter: { _id: ins.insertedId }, + update: { $set: { v: { $numberInt: "2" } } }, + options: { returnDocument: "before" }, + }); + expect(res.updatedDocument?.v).toEqual({ $numberInt: "1" }); + const found = await mongoConnector.findDocuments(config, { + collection: "updb", + filter: { _id: ins.insertedId }, + }); + expect(found.documents[0].v).toEqual({ $numberInt: "2" }); + }); + + test("upsert creates a document and reports upsertedId", async () => { + if (!connected) return; + const res = await mongoConnector.updateDocument(config, { + collection: "ups", + filter: { code: "XYZ" }, + update: { $set: { val: 1 } }, + options: { upsert: true }, + }); + expect(res.matchedCount).toBe(0); + expect(res.upsertedId).toBeDefined(); + const found = await mongoConnector.findDocuments(config, { + collection: "ups", + filter: { code: "XYZ" }, + }); + expect(found.documents).toHaveLength(1); + }); + + test("multi updates every match", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "multiu", + document: { g: "a", n: { $numberInt: "0" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "multiu", + document: { g: "a", n: { $numberInt: "0" } }, + }); + const res = await mongoConnector.updateDocument(config, { + collection: "multiu", + filter: { g: "a" }, + update: { $set: { n: { $numberInt: "9" } } }, + options: { multi: true }, + }); + expect(res.matchedCount).toBe(2); + expect(res.modifiedCount).toBe(2); + }); +}); + +describe("M2 deleteDocument", () => { + test("removes a single document and reports deletedCount", async () => { + if (!connected) return; + const ins = await mongoConnector.insertDocument(config, { + collection: "dels", + document: { x: 1 }, + }); + const res = await mongoConnector.deleteDocument(config, { + collection: "dels", + filter: { _id: ins.insertedId }, + }); + expect(res.deletedCount).toBe(1); + const found = await mongoConnector.findDocuments(config, { + collection: "dels", + filter: { _id: ins.insertedId }, + }); + expect(found.documents).toHaveLength(0); + }); + + test("multi removes every match", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { collection: "delm", document: { g: "z" } }); + await mongoConnector.insertDocument(config, { collection: "delm", document: { g: "z" } }); + const res = await mongoConnector.deleteDocument(config, { + collection: "delm", + filter: { g: "z" }, + options: { multi: true }, + }); + expect(res.deletedCount).toBe(2); + }); +}); + +describe("M2 runAggregation", () => { + test("groups by a field and computes an aggregate", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "agg", + document: { dept: "eng", salary: { $numberInt: "100" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "agg", + document: { dept: "eng", salary: { $numberInt: "200" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "agg", + document: { dept: "sales", salary: { $numberInt: "150" } }, + }); + + const res = await mongoConnector.runAggregation(config, { + collection: "agg", + pipeline: [{ $group: { _id: "$dept", avg: { $avg: "$salary" } } }, { $sort: { _id: 1 } }], + }); + expect(res.stageCount).toBe(2); + expect(res.documents).toHaveLength(2); + expect(res.documents[0]._id).toBe("eng"); + expect(Number(res.documents[0].avg)).toBe(150); + }); +}); + +describe("M2 cache + existence", () => { + test("document-page cache reflects a mutation after invalidation", async () => { + if (!connected) return; + const cacheConfig: MongoConfig = { ...config, cacheDocuments: true }; + const ins = await mongoConnector.insertDocument(cacheConfig, { + collection: "cache", + document: { k: "v1" }, + }); + const q = { collection: "cache", filter: { _id: ins.insertedId } }; + const first = await mongoConnector.findDocuments(cacheConfig, q); + expect(first.documents[0].k).toBe("v1"); + + // A mutation clears the collection cache, so the next read reflects v2. + await mongoConnector.updateDocument(cacheConfig, { + collection: "cache", + filter: { _id: ins.insertedId }, + update: { $set: { k: "v2" } }, + }); + const second = await mongoConnector.findDocuments(cacheConfig, q); + expect(second.documents[0].k).toBe("v2"); + }); + + test("collectionExists reports presence and absence", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "exists_check", + document: { a: 1 }, + }); + expect(await collectionExists(config, "exists_check")).toBe(true); + expect(await collectionExists(config, "nonexistent_xyz_123")).toBe(false); + }); +}); diff --git a/bridge/__tests__/integration/mongoInferSchema.integration.test.ts b/bridge/__tests__/integration/mongoInferSchema.integration.test.ts new file mode 100644 index 00000000..9b182f8e --- /dev/null +++ b/bridge/__tests__/integration/mongoInferSchema.integration.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, test, beforeAll, afterAll } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { mongoConnector, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const TEST_URI = process.env.MONGO_TEST_URI; +const itLive = TEST_URI ? test : test.skip; + +describe("M3 MongoDB Schema Inference (Integration)", () => { + let client: MongoClient | null = null; + const dbName = "relwave_m3_infer_test"; + let config: MongoConfig; + + beforeAll(async () => { + if (!TEST_URI) return; + client = new MongoClient(TEST_URI); + await client.connect(); + const db = client.db(dbName); + try { + await db.dropDatabase(); + } catch { + // ignore drop errors + } + + config = { + uri: TEST_URI, + database: dbName, + cacheDocuments: true, + }; + + // Seed flat collection + await db.collection("flat").insertMany([ + { name: "Alpha", active: true, score: 10 }, + { name: "Beta", active: false, score: 20 }, + ]); + + // Seed sparse and mixed collection + await db.collection("sparse_mixed").insertMany([ + { title: "Item 1", meta: { color: "blue" }, tags: ["a", "b"] }, + { title: "Item 2", meta: { color: "red", extra: 123 }, tags: ["c"], nullableField: null }, + { title: 3, tags: [true, 1] }, + ]); + + // Seed empty collection + await db.createCollection("empty_col"); + }, 30000); + + afterAll(async () => { + if (client) { + try { + await client.db(dbName).dropDatabase(); + await client.close(); + } catch { + // ignore + } + } + await closeAllMongoClients(); + }); + + itLive("listCollections returns seeded collections", async () => { + const collections = await mongoConnector.listCollections(config); + const names = collections.map((c) => c.name); + expect(names).toContain("flat"); + expect(names).toContain("sparse_mixed"); + expect(names).toContain("empty_col"); + }); + + itLive("inferCollectionSchema infers flat uniform collection accurately", async () => { + const schema = await mongoConnector.inferCollectionSchema(config, { + collection: "flat", + sampleSize: 100, + }); + + expect(schema.collection).toBe("flat"); + expect(schema.sampleSize).toBe(2); + expect(schema.totalDocs).toBe(2); + expect(schema.sampleDocuments.length).toBe(2); + + const nameField = schema.fields.find((f) => f.path === "name")!; + expect(nameField.bsonType).toBe("string"); + expect(nameField.presencePct).toBe(100); + expect(nameField.optional).toBe(false); + }); + + itLive("inferCollectionSchema infers sparse, mixed, and nested schemas", async () => { + const schema = await mongoConnector.inferCollectionSchema(config, { + collection: "sparse_mixed", + sampleSize: 100, + }); + + expect(schema.collection).toBe("sparse_mixed"); + expect(schema.sampleSize).toBe(3); + + const titleField = schema.fields.find((f) => f.path === "title")!; + expect(titleField.typeConflicts.length).toBeGreaterThan(0); + + const tagsField = schema.fields.find((f) => f.path === "tags")!; + expect(tagsField.isArray).toBe(true); + expect(tagsField.arrayElementTypes?.length).toBeGreaterThanOrEqual(2); + + const metaExtra = schema.fields.find((f) => f.path === "meta.extra")!; + expect(metaExtra.optional).toBe(true); + expect(metaExtra.depth).toBe(1); + expect(metaExtra.parentPath).toBe("meta"); + }); + + itLive("inferCollectionSchema succeeds on empty collection", async () => { + const schema = await mongoConnector.inferCollectionSchema(config, { + collection: "empty_col", + sampleSize: 50, + }); + + expect(schema.collection).toBe("empty_col"); + expect(schema.sampleSize).toBe(0); + expect(schema.totalDocs).toBe(0); + expect(schema.fields).toHaveLength(0); + expect(schema.sampleDocuments).toHaveLength(0); + }); +}); diff --git a/bridge/__tests__/integration/mongoMql.integration.test.ts b/bridge/__tests__/integration/mongoMql.integration.test.ts new file mode 100644 index 00000000..55c0adda --- /dev/null +++ b/bridge/__tests__/integration/mongoMql.integration.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, test, beforeAll, afterAll } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { executeMql, explainMql, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const mongoUri = process.env.MONGO_TEST_URI; +const describeIntegration = mongoUri ? describe : describe.skip; + +describeIntegration("MongoDB M5 MQL Integration Tests (Live Server)", () => { + const dbName = "relwave_m5_it"; + const config: MongoConfig = { + uri: mongoUri, + database: dbName, + }; + + let client: MongoClient | null = null; + const collectionName = "products"; + + beforeAll(async () => { + if (!mongoUri) return; + client = new MongoClient(mongoUri); + await client.connect(); + const db = client.db(dbName); + try { + await db.collection(collectionName).drop(); + } catch { + // ignore + } + + await db.collection(collectionName).insertMany([ + { sku: "ITEM-1", name: "Widget A", price: 10, category: "tools", stock: 100 }, + { sku: "ITEM-2", name: "Widget B", price: 25, category: "tools", stock: 50 }, + { sku: "ITEM-3", name: "Gadget C", price: 50, category: "electronics", stock: 20 }, + { sku: "ITEM-4", name: "Gadget D", price: 100, category: "electronics", stock: 5 }, + ]); + }); + + afterAll(async () => { + if (client) { + try { + await client.db(dbName).dropDatabase(); + await client.close(); + } catch { + // ignore + } + } + await closeAllMongoClients(); + }); + + test("executes find with projection, sort, and limit", async () => { + const res = await executeMql(config, { + collection: collectionName, + op: { + operation: "find", + payload: { + filter: { category: "tools" }, + projection: { sku: 1, name: 1, _id: 0 }, + sort: { price: -1 }, + limit: 10, + }, + }, + }); + + expect(res.operation).toBe("find"); + if (res.operation === "find") { + expect(res.documents).toHaveLength(2); + expect((res.documents[0] as any).sku).toBe("ITEM-2"); + expect((res.documents[1] as any).sku).toBe("ITEM-1"); + } + }); + + test("executes aggregate with multi-stage pipeline", async () => { + const res = await executeMql(config, { + collection: collectionName, + op: { + operation: "aggregate", + payload: { + pipeline: [ + { $group: { _id: "$category", totalStock: { $sum: "$stock" } } }, + { $sort: { totalStock: -1 } }, + ], + }, + }, + }); + + expect(res.operation).toBe("aggregate"); + if (res.operation === "aggregate") { + expect(res.documents).toHaveLength(2); + expect((res.documents[0] as any)._id).toBe("tools"); + expect((res.documents[0] as any).totalStock).toEqual({ $numberInt: "150" }); + } + }); + + test("executes aggregate with empty pipeline []", async () => { + const res = await executeMql(config, { + collection: collectionName, + op: { + operation: "aggregate", + payload: { + pipeline: [], + }, + }, + }); + + expect(res.operation).toBe("aggregate"); + if (res.operation === "aggregate") { + expect(res.documents).toHaveLength(4); + } + }); + + test("executes countDocuments and estimatedDocumentCount", async () => { + const filteredCountRes = await executeMql(config, { + collection: collectionName, + op: { + operation: "countDocuments", + payload: { filter: { category: "electronics" } }, + }, + }); + if (filteredCountRes.operation === "countDocuments") { + expect(filteredCountRes.count).toBe(2); + } + + const estCountRes = await executeMql(config, { + collection: collectionName, + op: { + operation: "estimatedDocumentCount", + }, + }); + if (estCountRes.operation === "estimatedDocumentCount") { + expect(estCountRes.count).toBeGreaterThanOrEqual(4); + } + }); + + test("executes distinct values", async () => { + const res = await executeMql(config, { + collection: collectionName, + op: { + operation: "distinct", + payload: { field: "category" }, + }, + }); + + expect(res.operation).toBe("distinct"); + if (res.operation === "distinct") { + expect(res.values.sort()).toEqual(["electronics", "tools"]); + expect(res.count).toBe(2); + } + }); + + test("explains find and aggregate queries", async () => { + const findExplain = await explainMql(config, { + collection: collectionName, + target: { + operation: "find", + filter: { sku: "ITEM-1" }, + }, + verbosity: "executionStats", + }); + + expect(findExplain.operation).toBe("find"); + expect(findExplain.queryPlanner).toBeDefined(); + expect(findExplain.executionStats).toBeDefined(); + + const aggExplain = await explainMql(config, { + collection: collectionName, + target: { + operation: "aggregate", + pipeline: [{ $match: { category: "tools" } }], + }, + verbosity: "queryPlanner", + }); + + expect(aggExplain.operation).toBe("aggregate"); + expect(aggExplain.queryPlanner).toBeDefined(); + }); +}); diff --git a/bridge/__tests__/integration/mongoMutations.integration.test.ts b/bridge/__tests__/integration/mongoMutations.integration.test.ts new file mode 100644 index 00000000..826eeaef --- /dev/null +++ b/bridge/__tests__/integration/mongoMutations.integration.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test, beforeAll, afterAll, jest } from "@jest/globals"; +import { MongoClient, ObjectId } from "mongodb"; +import { mongoConnector, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const TEST_URI = process.env.MONGO_TEST_URI; +const TEST_DB = "relwave_m4_it"; + +const config: MongoConfig = { + uri: TEST_URI ? `${TEST_URI.replace(/\/$/, "")}/${TEST_DB}` : undefined, + database: TEST_DB, +}; + +let client: MongoClient | null = null; +let connected = false; + +beforeAll(async () => { + jest.setTimeout(60000); + if (!TEST_URI) { + console.warn( + "[mongoMutations.integration] MONGO_TEST_URI not set; skipping M4 integration tests.", + ); + return; + } + try { + client = new MongoClient(config.uri as string); + await client.connect(); + try { + await client.db(TEST_DB).dropDatabase(); + } catch { + // ignore + } + connected = true; + } catch { + connected = false; + console.warn( + `[mongoMutations.integration] MongoDB not reachable at ${TEST_URI}; skipping M4 integration tests.`, + ); + } +}, 60000); + +afterAll(async () => { + if (client) { + try { + await client.db(TEST_DB).dropDatabase(); + await client.close(); + } catch { + // ignore + } + client = null; + } + await closeAllMongoClients(); +}); + +describe("M4 Mutations Lifecycle (Integration)", () => { + const collection = "test_m4_mutations"; + + test("creates a new collection explicitly", async () => { + if (!connected) return; + const res = await mongoConnector.createCollection(config, { collection }); + expect(res.ok).toBe(true); + expect(res.collection).toBe(collection); + + const list = await mongoConnector.listCollections(config); + expect(list.some((c) => c.name === collection)).toBe(true); + }); + + test("creates a capped collection with validated size", async () => { + if (!connected) return; + const cappedColl = "test_m4_capped"; + const res = await mongoConnector.createCollection(config, { + collection: cappedColl, + options: { capped: true, size: 8192, max: 100 }, + }); + expect(res.ok).toBe(true); + expect(res.collection).toBe(cappedColl); + + await mongoConnector.dropCollection(config, { collection: cappedColl }); + }); + + test("replaces a document and updates cache", async () => { + if (!connected) return; + const oid = new ObjectId(); + const originalDoc = { + _id: { $oid: oid.toHexString() }, + title: "Initial Title", + version: { $numberInt: "1" }, + tags: ["alpha", "beta"], + }; + + const inserted = await mongoConnector.insertDocument(config, { + collection, + document: originalDoc, + }); + expect(inserted.insertedId).toEqual({ $oid: oid.toHexString() }); + + const replacement = { + _id: { $oid: oid.toHexString() }, + title: "Replaced Title", + author: "Alice", + active: true, + }; + + const replaceRes = await mongoConnector.replaceDocument(config, { + collection, + filter: { _id: { $oid: oid.toHexString() } }, + replacement, + options: { returnDocument: "after" }, + }); + + expect(replaceRes.matchedCount).toBe(1); + expect(replaceRes.modifiedCount).toBe(1); + expect(replaceRes.updatedDocument?.title).toBe("Replaced Title"); + expect(replaceRes.updatedDocument?.author).toBe("Alice"); + expect(replaceRes.updatedDocument?.version).toBeUndefined(); + expect(replaceRes.updatedDocument?.tags).toBeUndefined(); + + const found = await mongoConnector.findDocuments(config, { + collection, + filter: { _id: { $oid: oid.toHexString() } }, + }); + expect(found.documents).toHaveLength(1); + expect(found.documents[0].title).toBe("Replaced Title"); + expect(found.documents[0].version).toBeUndefined(); + }); + + test("rejects dropping system collection", async () => { + if (!connected) return; + await expect( + mongoConnector.dropCollection(config, { collection: "system.views" }), + ).rejects.toMatchObject({ code: "MONGO_FORBIDDEN" }); + }); + + test("drops collection successfully", async () => { + if (!connected) return; + const dropRes = await mongoConnector.dropCollection(config, { collection }); + expect(dropRes.ok).toBe(true); + expect(dropRes.collection).toBe(collection); + + const list = await mongoConnector.listCollections(config); + expect(list.some((c) => c.name === collection)).toBe(false); + }); +}); diff --git a/bridge/__tests__/mongoBson.test.ts b/bridge/__tests__/mongoBson.test.ts new file mode 100644 index 00000000..4514aebe --- /dev/null +++ b/bridge/__tests__/mongoBson.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "@jest/globals"; +import { ObjectId, Decimal128, Long, Timestamp, Int32, Binary } from "mongodb"; +import { serializeMongoValue, serializeMongoDocument } from "../src/connectors/mongoBson"; + +describe("serializeMongoValue", () => { + test("passes primitives through unchanged", () => { + expect(serializeMongoValue(null)).toBeNull(); + expect(serializeMongoValue(true)).toBe(true); + expect(serializeMongoValue(42)).toBe(42); + expect(serializeMongoValue("hi")).toBe("hi"); + }); + + test("drops non-finite numbers to stay JSON-safe", () => { + expect(serializeMongoValue(Number.NaN)).toBeNull(); + expect(serializeMongoValue(Number.POSITIVE_INFINITY)).toBeNull(); + }); + + test("maps undefined to the wire convention", () => { + expect(serializeMongoValue(undefined)).toEqual({ $undefined: true }); + }); + + test("recurses into arrays", () => { + expect(serializeMongoValue([1, "two", null])).toEqual([1, "two", null]); + }); + + test("serializes ObjectId, Date, Decimal128, Long, Timestamp", () => { + const oid = new ObjectId("507f1f77bcf86cd799439011"); + expect(serializeMongoValue(oid)).toEqual({ $oid: "507f1f77bcf86cd799439011" }); + + const date = new Date("2020-01-02T03:04:05.000Z"); + expect(serializeMongoValue(date)).toEqual({ $date: "2020-01-02T03:04:05.000Z" }); + + expect(serializeMongoValue(new Decimal128("123.45"))).toEqual({ $numberDecimal: "123.45" }); + + expect(serializeMongoValue(Long.fromNumber(42))).toEqual({ $numberLong: "42" }); + + const ts = new Timestamp({ t: 1234567890, i: 1 }); + expect(serializeMongoValue(ts)).toEqual({ $timestamp: { t: 1234567890, i: 1 } }); + }); + + test("flattens Int32 to a plain number", () => { + expect(serializeMongoValue(new Int32(7))).toBe(7); + expect(serializeMongoValue(new Int32(7)).constructor).toBe(Number); + }); + + test("serializes Binary and raw buffers to $binary", () => { + const bin = new Binary(Buffer.from("hello")); + const out = serializeMongoValue(bin) as { $binary: { base64: string; subtype: string } }; + expect(out.$binary.base64).toBe(Buffer.from("hello").toString("base64")); + expect(out.$binary.subtype).toBe("00"); + + const bufOut = serializeMongoValue(Buffer.from("hi")) as { + $binary: { base64: string; subtype: string }; + }; + expect(bufOut.$binary.base64).toBe(Buffer.from("hi").toString("base64")); + + const u8Out = serializeMongoValue(new Uint8Array([1, 2, 3])) as { + $binary: { base64: string; subtype: string }; + }; + expect(u8Out.$binary.base64).toBe(Buffer.from([1, 2, 3]).toString("base64")); + }); + + test("recurses through nested documents and arrays", () => { + const value = serializeMongoValue({ + _id: new ObjectId("507f1f77bcf86cd799439011"), + nested: { when: new Date("2021-06-07T08:09:10.000Z"), tags: ["a", "b"] }, + list: [1, { deep: true }], + }); + expect(value).toEqual({ + _id: { $oid: "507f1f77bcf86cd799439011" }, + nested: { when: { $date: "2021-06-07T08:09:10.000Z" }, tags: ["a", "b"] }, + list: [1, { deep: true }], + }); + }); + + test("output is always JSON.stringify-safe", () => { + const value = serializeMongoValue({ + _id: new ObjectId(), + when: new Date(), + big: Long.fromString("9007199254740993"), + dec: new Decimal128("3.14"), + raw: Buffer.from("bytes"), + missing: undefined, + deep: { arr: [new ObjectId(), new Date()] }, + }); + expect(() => JSON.stringify(value)).not.toThrow(); + expect(JSON.parse(JSON.stringify(value))).toBeDefined(); + }); +}); + +describe("serializeMongoDocument", () => { + test("serializes a document and omits the internal _bsontype marker", () => { + const doc = { + _id: new ObjectId("507f1f77bcf86cd799439011"), + name: "test", + createdAt: new Date("2022-03-04T05:06:07.000Z"), + }; + const out = serializeMongoDocument(doc); + expect(out).toEqual({ + _id: { $oid: "507f1f77bcf86cd799439011" }, + name: "test", + createdAt: { $date: "2022-03-04T05:06:07.000Z" }, + }); + expect(Object.keys(out)).not.toContain("_bsontype"); + }); +}); diff --git a/bridge/__tests__/mongoBulkEngine.test.ts b/bridge/__tests__/mongoBulkEngine.test.ts new file mode 100644 index 00000000..5af5a6af --- /dev/null +++ b/bridge/__tests__/mongoBulkEngine.test.ts @@ -0,0 +1,1649 @@ +import { describe, expect, test, beforeEach, afterEach, jest } from "@jest/globals"; +import fs from "fs"; +import path from "path"; +import os from "os"; +import * as mongoConnector from "../src/connectors/mongo"; +import { + BulkOperationManager, + MongoBulkEngine, + parseImportPayload, + parseCsv, + isValidUuidV4, + MAX_IMPORT_CHUNK_DOCS, + MAX_IMPORT_CHUNK_SIZE_BYTES, + MAX_BULK_INSERT_DOCS, + EXPORT_DIR, +} from "../src/services/mongoBulkEngine"; +import { DBType } from "../src/types"; + +describe("MongoDB M7 Phase 3: Bulk Operations Bridge Engine", () => { + let opManager: BulkOperationManager; + let engine: MongoBulkEngine; + let mockDbService: any; + let mockCollection: any; + let mockNativeDb: any; + + beforeEach(() => { + opManager = new BulkOperationManager(60 * 60 * 1000); // 1-hour TTL + + mockCollection = { + insertMany: jest.fn().mockResolvedValue({ + insertedCount: 2, + insertedIds: { 0: "id1", 1: "id2" }, + }), + updateMany: jest.fn().mockResolvedValue({ + matchedCount: 5, + modifiedCount: 3, + upsertedId: null, + }), + deleteMany: jest.fn().mockResolvedValue({ + deletedCount: 15, + }), + find: jest.fn(), + }; + + mockNativeDb = { + collection: jest.fn().mockReturnValue(mockCollection), + }; + + mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + db: { id: "test-db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test" }, + dbType: DBType.MONGODB, + }), + acquireConnection: jest.fn().mockResolvedValue({ + db: { id: "test-db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test" }, + dbType: DBType.MONGODB, + release: jest.fn(), + }), + }; + + jest.spyOn(mongoConnector, "selectDb").mockReturnValue(mockNativeDb as any); + + engine = new MongoBulkEngine(mockDbService, opManager); + jest.spyOn(engine, "getDb").mockResolvedValue({ + db: { id: "test-db", readonly: false }, + conn: { uri: "mongodb://localhost:27017/test" }, + dbType: DBType.MONGODB, + nativeDb: mockNativeDb, + }); + }); + + afterEach(() => { + opManager.dispose(); + }); + + // ========================================================================= + // 1. BulkOperationManager Lifecycle & State Machine + // ========================================================================= + describe("BulkOperationManager", () => { + test("validates and generates valid UUIDv4 operationIds", () => { + const op = opManager.createOperation("import", "db-1", "users", "json"); + expect(isValidUuidV4(op.operation.operationId)).toBe(true); + expect(op.operation.status).toBe("pending"); + expect(op.operation.collection).toBe("users"); + }); + + test("enforces safe state machine transitions", () => { + const op = opManager.createOperation("import", "db-1", "users", "json"); + const id = op.operation.operationId; + + // pending -> in_progress + opManager.transition(id, "in_progress"); + expect(opManager.getOperation(id)?.operation.status).toBe("in_progress"); + + // in_progress -> completed + opManager.transition(id, "completed"); + expect(opManager.getOperation(id)?.operation.status).toBe("completed"); + + // terminal state cannot transition again + expect(() => opManager.transition(id, "in_progress")).toThrow( + /Cannot transition terminal operation/, + ); + }); + + test("handles cancellation and avoids concurrent cancellation races", () => { + const op = opManager.createOperation("export", "db-1", "orders", "json"); + const id = op.operation.operationId; + + const cancelRes = opManager.cancel(id); + expect(cancelRes.status).toBe("cancelled"); + expect(op.controller.signal.aborted).toBe(true); + + // Repeat cancel is idempotent and safe + const repeatCancel = opManager.cancel(id); + expect(repeatCancel.status).toBe("cancelled"); + expect(repeatCancel.message).toContain("already cancelled"); + }); + + test("rejects path traversal attempts in operationId", () => { + expect(() => opManager.getOperation("../etc/passwd")).toThrow(/Invalid operationId format/); + expect(() => opManager.cleanup("invalid-uuid\0")).toThrow(/Invalid operationId format/); + }); + + test("respects 1-hour TTL expiration", () => { + const shortTtlManager = new BulkOperationManager(10); // 10ms TTL + const op = shortTtlManager.createOperation("import", "db-1", "users", "json"); + const id = op.operation.operationId; + + return new Promise((resolve) => { + setTimeout(() => { + const fetched = shortTtlManager.getOperation(id); + expect(fetched).toBeNull(); + shortTtlManager.dispose(); + resolve(); + }, 30); + }); + }); + }); + + // ========================================================================= + // 2. Parsing, Limits & Prototype Pollution Defense + // ========================================================================= + describe("Parsing & Payload Guards", () => { + test("parses JSON array format correctly", () => { + const payload = JSON.stringify([ + { name: "Alice", age: 30 }, + { name: "Bob", age: 25 }, + ]); + const docs = parseImportPayload(payload, "json"); + expect(docs).toHaveLength(2); + expect(docs[0].name).toBe("Alice"); + }); + + test("rejects non-array JSON payloads", () => { + const payload = JSON.stringify({ name: "Alice" }); + expect(() => parseImportPayload(payload, "json")).toThrow(/must be an array of documents/); + }); + + test("parses NDJSON format correctly", () => { + const payload = '{"name":"Alice","score":100}\n{"name":"Bob","score":85}\n'; + const docs = parseImportPayload(payload, "ndjson"); + expect(docs).toHaveLength(2); + expect(docs[1].score).toBe(85); + }); + + test("rejects malformed NDJSON lines", () => { + const payload = '{"name":"Alice"}\nINVALID_JSON_LINE\n'; + expect(() => parseImportPayload(payload, "ndjson")).toThrow(/Invalid NDJSON at line 2/); + }); + + test("parses CSV with un-flattened dot notation, types, quotes, and newlines", () => { + const csv = + "user.name,user.age,active,notes\n" + + '"Smith, Alice",30,true,"Line 1\nLine 2"\n' + + "Bob,25,false,null\n"; + + const docs = parseCsv(csv); + expect(docs).toHaveLength(2); + expect(docs[0].user).toEqual({ name: "Smith, Alice", age: 30 }); + expect(docs[0].active).toBe(true); + expect(docs[0].notes).toBe("Line 1\nLine 2"); + expect(docs[1].user).toEqual({ name: "Bob", age: 25 }); + expect(docs[1].active).toBe(false); + expect(docs[1].notes).toBeNull(); + }); + + test("detects and rejects prototype pollution keys (__proto__, constructor)", () => { + const payloadWithProto = JSON.stringify([ + { title: "Normal" }, + JSON.parse('{"__proto__": {"admin": true}, "title": "Hacked"}'), + ]); + expect(() => parseImportPayload(payloadWithProto, "json")).toThrow(/Prototype pollution/); + + const payloadWithConstructor = JSON.stringify([ + { title: "Normal" }, + { constructor: { name: "Evil" } }, + ]); + expect(() => parseImportPayload(payloadWithConstructor, "json")).toThrow( + /Prototype pollution/, + ); + + const csvWithProto = "name,__proto__.isAdmin\nAlice,true\n"; + expect(() => parseCsv(csvWithProto)).toThrow(/Prototype pollution/); + }); + + test("enforces chunk document limit (max 1,000 docs)", () => { + const oversizedArray = new Array(MAX_IMPORT_CHUNK_DOCS + 1).fill({ item: "test" }); + expect(() => parseImportPayload(JSON.stringify(oversizedArray), "json")).toThrow( + /exceeds maximum limit of 1000 documents/, + ); + }); + + test("enforces chunk payload size limit (max 4MB)", () => { + const largeDoc = "x".repeat(MAX_IMPORT_CHUNK_SIZE_BYTES + 10); + expect(() => parseImportPayload(largeDoc, "json")).toThrow( + /exceeds maximum limit of 4194304 bytes/, + ); + }); + + test("blocks system collections (system.*)", async () => { + await expect( + engine.importStart({ dbId: "db-1", collection: "system.views", format: "json" }), + ).rejects.toThrow(/Operations on system collections are forbidden/); + }); + }); + + // ========================================================================= + // 3. Chunk Replay Protection & Write Batching + // ========================================================================= + describe("Import Engine Write & Replay Protections", () => { + test("idempotently handles chunk replay without double insert", async () => { + const startRes = await engine.importStart({ + dbId: "db-1", + collection: "items", + format: "json", + }); + const opId = startRes.operationId; + const payload = JSON.stringify([{ id: 1 }, { id: 2 }]); + + // First chunk invocation + const res1 = await engine.importChunk({ + operationId: opId, + chunkIndex: 0, + payload, + }); + expect(res1.insertedCount).toBe(2); + expect(mockCollection.insertMany).toHaveBeenCalledTimes(1); + + // Replaying the exact same chunkIndex: must return cached response without insert + const res2 = await engine.importChunk({ + operationId: opId, + chunkIndex: 0, + payload, + }); + expect(res2.insertedCount).toBe(2); + expect(mockCollection.insertMany).toHaveBeenCalledTimes(1); // Not called again! + }); + + test("handles out-of-order chunks cleanly", async () => { + const startRes = await engine.importStart({ + dbId: "db-1", + collection: "items", + format: "json", + }); + const opId = startRes.operationId; + + // Chunk 1 arrives before Chunk 0 + const chunk1 = await engine.importChunk({ + operationId: opId, + chunkIndex: 1, + isLastChunk: true, + payload: JSON.stringify([{ id: 3 }, { id: 4 }]), + }); + expect(chunk1.isComplete).toBe(true); + + // Chunk 0 arrives after + const chunk0 = await engine.importChunk({ + operationId: opId, + chunkIndex: 0, + payload: JSON.stringify([{ id: 1 }, { id: 2 }]), + }); + expect(chunk0.insertedCount).toBe(2); + expect(mockCollection.insertMany).toHaveBeenCalledTimes(2); + }); + + test("handles duplicate key partial failures with ordered: false", async () => { + mockCollection.insertMany.mockRejectedValueOnce({ + name: "MongoBulkWriteError", + insertedCount: 1, + writeErrors: [ + { + index: 1, + code: 11000, + errmsg: + "E11000 duplicate key error collection: test.items index: _id_ dup key: { _id: 2 }", + }, + ], + }); + + const startRes = await engine.importStart({ + dbId: "db-1", + collection: "items", + format: "json", + }); + + const chunkRes = await engine.importChunk({ + operationId: startRes.operationId, + chunkIndex: 0, + payload: JSON.stringify([{ _id: 1 }, { _id: 2 }]), + }); + + expect(chunkRes.insertedCount).toBe(1); + expect(chunkRes.failedCount).toBe(1); + expect(chunkRes.errors).toHaveLength(1); + expect(chunkRes.errors![0].code).toBe("MONGO_DUPLICATE_KEY"); + }); + + test("cancels active batch writes cleanly between batches", async () => { + const startRes = await engine.importStart({ + dbId: "db-1", + collection: "items", + format: "json", + }); + const opState = opManager.getOperation(startRes.operationId)!; + + // Abort signal right before batch execution + opState.controller.abort(); + + await expect( + engine.importChunk({ + operationId: startRes.operationId, + chunkIndex: 0, + payload: JSON.stringify([{ id: 1 }]), + }), + ).rejects.toThrow(/Operation was cancelled/); + }); + + test("enforces read-only database protection on import writes", async () => { + jest.spyOn(engine, "getDb").mockResolvedValueOnce({ + db: { id: "readonly-db", readonly: true }, + conn: { uri: "mongodb://localhost:27017/test", readonly: true }, + dbType: DBType.MONGODB, + nativeDb: mockNativeDb, + }); + + await expect( + engine.importStart({ dbId: "readonly-db", collection: "items", format: "json" }), + ).rejects.toThrow(/Database is in read-only mode/); + }); + }); + + // ========================================================================= + // 4. Cursor Streaming Bulk Export (Zero .toArray()) + // ========================================================================= + describe("Export Engine Cursor Streaming", () => { + test("streams JSON array directly to file and closes cursor in all paths", async () => { + const mockCursorDocs = [ + { _id: "1", name: "Alpha" }, + { _id: "2", name: "Beta" }, + ]; + let cursorClosed = false; + + const mockCursor = { + [Symbol.asyncIterator]: async function* () { + for (const doc of mockCursorDocs) { + yield doc; + } + }, + batchSize: jest.fn().mockReturnThis(), + sort: jest.fn().mockReturnThis(), + project: jest.fn().mockReturnThis(), + limit: jest.fn().mockReturnThis(), + close: jest.fn().mockImplementation(async () => { + cursorClosed = true; + }), + }; + mockCollection.find.mockReturnValue(mockCursor); + + const res = await engine.exportStart({ + dbId: "test-db", + collection: "items", + format: "json", + }); + + expect(res.operationId).toBeDefined(); + + // Wait for background stream pipeline to complete + for (let i = 0; i < 50; i++) { + const opStatus = await engine.getOperationStatus(res.operationId); + if (opStatus.status === "completed" || opStatus.status === "failed") break; + await new Promise((r) => setTimeout(r, 20)); + } + + const status = await engine.getOperationStatus(res.operationId); + expect(status.status).toBe("completed"); + expect(status.successfulDocuments).toBe(2); + expect(cursorClosed).toBe(true); + + // Verify temp file content + const filePath = path.join(EXPORT_DIR, `export-${res.operationId}.json`); + expect(fs.existsSync(filePath)).toBe(true); + const content = fs.readFileSync(filePath, "utf8"); + expect(content).toContain('"_id":"1"'); + expect(content).toContain('"_id":"2"'); + + // Cleanup + await engine.cleanupOperation(res.operationId); + expect(fs.existsSync(filePath)).toBe(false); + }); + + test("streams CSV with dot-notation flattening and escaping", async () => { + const mockCursorDocs = [ + { _id: "101", user: { name: "Alice", location: "NYC" } }, + { _id: "102", user: { name: "Bob", location: "SF, CA" } }, + ]; + + const mockCursor = { + [Symbol.asyncIterator]: async function* () { + for (const doc of mockCursorDocs) { + yield doc; + } + }, + batchSize: jest.fn().mockReturnThis(), + close: jest.fn().mockResolvedValue(undefined), + }; + mockCollection.find.mockReturnValue(mockCursor); + + const res = await engine.exportStart({ + dbId: "test-db", + collection: "users", + format: "csv", + }); + + // Wait for background stream pipeline to complete + for (let i = 0; i < 50; i++) { + const opStatus = await engine.getOperationStatus(res.operationId); + if (opStatus.status === "completed" || opStatus.status === "failed") break; + await new Promise((r) => setTimeout(r, 20)); + } + + const filePath = path.join(EXPORT_DIR, `export-${res.operationId}.csv`); + expect(fs.existsSync(filePath)).toBe(true); + const csv = fs.readFileSync(filePath, "utf8"); + + expect(csv).toContain("_id,user.location,user.name"); + expect(csv).toContain('"SF, CA"'); // Escaped comma + + await engine.cleanupOperation(res.operationId); + }); + }); + + // ========================================================================= + // 5. Supporting Bulk Mutations + // ========================================================================= + describe("Supporting Bulk Mutations", () => { + test("bulkInsert rejects > 5,000 documents and protects against prototype pollution", async () => { + const oversized = new Array(MAX_BULK_INSERT_DOCS + 1).fill({ a: 1 }); + await expect( + engine.bulkInsert({ dbId: "test-db", collection: "items", documents: oversized }), + ).rejects.toThrow(/exceeds maximum allowed of 5000/); + + const polluted = [{ normal: true }, JSON.parse('{"__proto__": {"admin": true}}')]; + await expect( + engine.bulkInsert({ dbId: "test-db", collection: "items", documents: polluted }), + ).rejects.toThrow(/Prototype pollution/); + }); + + test("bulkUpdate enforces operator document ($) and restricts empty filter", async () => { + // Must start with $ + await expect( + engine.bulkUpdate({ + dbId: "test-db", + collection: "items", + filter: { status: "pending" }, + update: { status: "processed" }, // Missing $set! + }), + ).rejects.toThrow(/top-level operators must begin with '\$'/); + + // Empty filter without allowUnrestricted is forbidden + await expect( + engine.bulkUpdate({ + dbId: "test-db", + collection: "items", + filter: {}, + update: { $set: { status: "processed" } }, + }), + ).rejects.toThrow(/allowUnrestricted: true/); + + // Empty filter with allowUnrestricted emits advisory warning + const res = await engine.bulkUpdate({ + dbId: "test-db", + collection: "items", + filter: {}, + update: { $set: { status: "processed" } }, + options: { allowUnrestricted: true }, + }); + expect(res.advisoryWarning).toContain("Unrestricted bulk update"); + }); + + test("bulkDelete restricts empty filter and emits advisory warning when deletedCount > 100", async () => { + // Empty filter without allowUnrestricted is forbidden + await expect( + engine.bulkDelete({ + dbId: "test-db", + collection: "items", + filter: {}, + }), + ).rejects.toThrow(/allowUnrestricted: true/); + + // Large delete count warning + mockCollection.deleteMany.mockResolvedValueOnce({ deletedCount: 150 }); + const res = await engine.bulkDelete({ + dbId: "test-db", + collection: "items", + filter: { obsolete: true }, + }); + expect(res.deletedCount).toBe(150); + expect(res.advisoryWarning).toContain( + "Broad delete operation affected more than 100 documents", + ); + }); + }); + + // ========================================================================= + // 5. M7 Phase 4: Stage 1 Export Save-To & Staging Cleanup Tests + // ========================================================================= + describe("M7 Phase 4: Stage 1 Export Save-To & Staging Cleanup", () => { + let testDestDir: string; + const createdExportFiles: string[] = []; + + beforeEach(() => { + testDestDir = fs.mkdtempSync(path.join(os.tmpdir(), "relwave-test-dest-")); + }); + + afterEach(() => { + jest.restoreAllMocks(); + try { + fs.rmSync(testDestDir, { recursive: true, force: true }); + } catch {} + for (const f of createdExportFiles) { + try { + if (fs.existsSync(f)) fs.unlinkSync(f); + } catch {} + } + createdExportFiles.length = 0; + opManager.clearPendingStagingCleanups(); + }); + + function createCompletedExport(format = "json", content = '{"test":123}') { + const state = opManager.createOperation("export", "test-db", "users", format as any); + const id = state.operation.operationId; + opManager.transition(id, "in_progress"); + + if (!fs.existsSync(EXPORT_DIR)) { + fs.mkdirSync(EXPORT_DIR, { recursive: true }); + } + const sourceFilePath = path.join(EXPORT_DIR, `${id}.${format}`); + fs.writeFileSync(sourceFilePath, content, "utf8"); + createdExportFiles.push(sourceFilePath); + + state.operation.tempFilePath = sourceFilePath; + state.operation.fileSizeBytes = Buffer.byteLength(content, "utf8"); + opManager.transition(id, "completed"); + + return { id, state, sourceFilePath }; + } + + // ----------------------------------------------------------------------- + // Validation & Safety Checks + // ----------------------------------------------------------------------- + describe("Validation & Safety Checks", () => { + test("rejects relative or whitespace destinationPath", async () => { + const { id } = createCompletedExport(); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: "relative/path.json" }), + ).rejects.toThrow(/destinationPath must be a non-empty absolute path/); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: " " }), + ).rejects.toThrow(/destinationPath must be a non-empty absolute path/); + }); + + test("rejects non-existent or inaccessible parent directory", async () => { + const { id } = createCompletedExport(); + const nonExistentDir = path.join(testDestDir, "does-not-exist", "export.json"); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: nonExistentDir }), + ).rejects.toThrow(/Parent directory does not exist or is not accessible/); + }); + + test("rejects non-export operations", async () => { + const importOp = opManager.createOperation("import", "test-db", "users", "json"); + opManager.transition(importOp.operation.operationId, "in_progress"); + opManager.transition(importOp.operation.operationId, "completed"); + + await expect( + engine.exportSaveTo({ + operationId: importOp.operation.operationId, + destinationPath: path.join(testDestDir, "out.json"), + }), + ).rejects.toThrow(/Cannot save: operation is not an export/); + }); + + test("rejects non-completed export operations", async () => { + const state = opManager.createOperation("export", "test-db", "users", "json"); + await expect( + engine.exportSaveTo({ + operationId: state.operation.operationId, + destinationPath: path.join(testDestDir, "out.json"), + }), + ).rejects.toThrow( + /Cannot save export file: operation status is 'pending', expected 'completed'/, + ); + }); + + test("rejects missing export source temporary file", async () => { + const { id, sourceFilePath } = createCompletedExport(); + fs.unlinkSync(sourceFilePath); + + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: path.join(testDestDir, "out.json"), + }), + ).rejects.toThrow(/Export source temporary file does not exist/); + }); + }); + + // ----------------------------------------------------------------------- + // Destination Inspection & Type Discrimination + // ----------------------------------------------------------------------- + describe("Destination Inspection & Type Discrimination", () => { + test("proceeds when destination does not exist (ENOENT)", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "new-file.json"); + + // Spy on link to simulate success + jest.spyOn(fs.promises, "link").mockImplementation(async (src, target) => { + fs.copyFileSync(src as string, target as string); + }); + + const res = await engine.exportSaveTo({ operationId: id, destinationPath: dest }); + expect(res.success).toBe(true); + expect(res.destinationPath).toBe(path.normalize(dest)); + expect(res.bytesWritten).toBeGreaterThan(0); + expect(res.warning).toBeUndefined(); + }); + + test("rejects when destination inspection throws EACCES or EIO", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "forbidden.json"); + + const lstatSpy = jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (p === path.normalize(dest)) { + const err: any = new Error("Permission denied"); + err.code = "EACCES"; + throw err; + } + return lstatSpy.getMockImplementation()!(p); + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Failed to inspect destination path \(EACCES\)/); + }); + + test("rejects when destination is an existing directory", async () => { + const { id } = createCompletedExport(); + const dirDest = path.join(testDestDir, "sub-dir"); + fs.mkdirSync(dirDest); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dirDest }), + ).rejects.toThrow( + /Destination is an existing directory; cannot overwrite a directory with a file/, + ); + }); + + test("rejects when destination is a special device", async () => { + const { id } = createCompletedExport(); + const specialDest = path.join(testDestDir, "special-dev"); + + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (p === path.normalize(specialDest)) { + return { + isFile: () => false, + isDirectory: () => false, + isFIFO: () => true, + isSocket: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isSymbolicLink: () => false, + } as any; + } + return fs.lstatSync(p); + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: specialDest }), + ).rejects.toThrow(/Destination is a special device or socket/); + }); + + test("rejects existing destination when overwrite !== true", async () => { + const { id } = createCompletedExport(); + const existingDest = path.join(testDestDir, "existing.json"); + fs.writeFileSync(existingDest, "old content", "utf8"); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: existingDest }), + ).rejects.toThrow(/Destination file already exists/); + }); + + test("existing destination symlink with overwrite false throws MONGO_FILE_EXISTS", async () => { + const { id } = createCompletedExport(); + const targetFile = path.join(testDestDir, "target-false.txt"); + fs.writeFileSync(targetFile, "target data", "utf8"); + const destSymlink = path.join(testDestDir, "link-to-target-false.json"); + + let realSymlinkCreated = false; + try { + fs.symlinkSync(targetFile, destSymlink, "file"); + realSymlinkCreated = true; + } catch {} + + if (!realSymlinkCreated) { + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (p === path.normalize(destSymlink)) { + return { + isFile: () => false, + isDirectory: () => false, + isSymbolicLink: () => true, + isFIFO: () => false, + isSocket: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + } as any; + } + return fs.lstatSync(p); + }); + } + + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: destSymlink, + options: { overwrite: false }, + }), + ).rejects.toThrow(/Destination file already exists/); + + expect(fs.readFileSync(targetFile, "utf8")).toBe("target data"); + }); + }); + + // ----------------------------------------------------------------------- + // Atomic Publication Commit & Decoupled Cleanup (overwrite: false) + // ----------------------------------------------------------------------- + describe("Atomic Publication Commit & Decoupled Cleanup (overwrite: false)", () => { + test("hard-link publication succeeds with clean staging unlink", async () => { + const { id } = createCompletedExport("json", '{"a":1}'); + const dest = path.join(testDestDir, "linked.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async (src, target) => { + fs.copyFileSync(src as string, target as string); + }); + + const res = await engine.exportSaveTo({ operationId: id, destinationPath: dest }); + expect(res.success).toBe(true); + expect(res.warning).toBeUndefined(); + expect(fs.existsSync(dest)).toBe(true); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("hard-link publication succeeds but post-publication staging unlink fails", async () => { + const { id, sourceFilePath } = createCompletedExport("json", '{"b":2}'); + const dest = path.join(testDestDir, "linked-warn.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async (src, target) => { + fs.copyFileSync(src as string, target as string); + }); + + // Spy on unlink to fail on staging file + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + const err: any = new Error("Resource busy"); + err.code = "EBUSY"; + throw err; + } + return origUnlink(p); + }); + + const res = await engine.exportSaveTo({ operationId: id, destinationPath: dest }); + expect(res.success).toBe(true); + expect(res.warning).toContain("queued for identity-verified retry cleanup"); + expect(fs.existsSync(dest)).toBe(true); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + expect(fs.existsSync(sourceFilePath)).toBe(true); // source retained + expect(opManager.getPendingStagingCleanupsCount()).toBe(1); + }); + + test("pre-publication link collision (EEXIST) throws MONGO_FILE_EXISTS and unlinks staging", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "collision.json"); + + let stagingPathUnlinked: string | null = null; + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + stagingPathUnlinked = p; + } + return origUnlink(p); + }); + + jest.spyOn(fs.promises, "link").mockImplementation(async () => { + const err: any = new Error("File exists"); + err.code = "EEXIST"; + throw err; + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Destination file already exists/); + + expect(stagingPathUnlinked).not.toBeNull(); + expect(fs.existsSync(sourceFilePath)).toBe(true); // source preserved + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("pre-publication link failure on confirmed unsupported filesystem (ENOSYS) throws MONGO_UNSUPPORTED_FILESYSTEM without copyFile fallback", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "unsupported.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async () => { + const err: any = new Error("Function not implemented"); + err.code = "ENOSYS"; + throw err; + }); + + const copyFileSpy = jest.spyOn(fs.promises, "copyFile"); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow( + /The destination filesystem does not support atomic hard-link publication/, + ); + + expect(copyFileSpy).not.toHaveBeenCalled(); + expect(fs.existsSync(dest)).toBe(false); + expect(fs.existsSync(sourceFilePath)).toBe(true); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("pre-publication link failure with EPERM or EACCES throws MONGO_SAVE_FAILED without claiming unsupported filesystem", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "perm-denied.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async () => { + const err: any = new Error("Operation not permitted"); + err.code = "EPERM"; + throw err; + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow( + /Permission denied or operation not permitted during hard-link publication \(EPERM\)/, + ); + }); + + test("pre-publication link failure with EXDEV throws MONGO_SAVE_FAILED without claiming unsupported filesystem", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "cross-dev.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async () => { + const err: any = new Error("Cross-device link"); + err.code = "EXDEV"; + throw err; + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Cross-device link error \(EXDEV\)/); + }); + + test("pre-publication link failure where staging cleanup fails preserves original link error and records identity", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "double-fail.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async () => { + const err: any = new Error("File exists"); + err.code = "EEXIST"; + throw err; + }); + + jest.spyOn(fs.promises, "unlink").mockImplementation(async () => { + const err: any = new Error("Access denied on unlink"); + err.code = "EACCES"; + throw err; + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Destination file already exists/); + + expect(opManager.getPendingStagingCleanupsCount()).toBe(1); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + }); + + // ----------------------------------------------------------------------- + // Atomic Replacement (overwrite: true) + // ----------------------------------------------------------------------- + describe("Atomic Replacement (overwrite: true)", () => { + test("atomic rename replaces destination directly and skips staging unlink", async () => { + const { id } = createCompletedExport("json", '{"version":2}'); + const dest = path.join(testDestDir, "overwrite-target.json"); + fs.writeFileSync(dest, '{"version":1}', "utf8"); + + const unlinkSpy = jest.spyOn(fs.promises, "unlink"); + + const res = await engine.exportSaveTo({ + operationId: id, + destinationPath: dest, + options: { overwrite: true }, + }); + + expect(res.success).toBe(true); + expect(res.warning).toBeUndefined(); + expect(fs.readFileSync(dest, "utf8")).toBe('{"version":2}'); + // unlink should NOT be called for staging file because rename already consumed it + for (const call of unlinkSpy.mock.calls) { + expect(String(call[0])).not.toContain(".relwave-tmp-"); + } + }); + + test("post-commit audit failure surfaces advisory warning without failing operation", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "post-audit-warn.json"); + + // Make rename succeed, but post-commit lstat on dest return non-file + const realLstat = fs.promises.lstat; + let renameCommitted = false; + jest.spyOn(fs.promises, "rename").mockImplementation(async (src, target) => { + fs.copyFileSync(src as string, target as string); + fs.unlinkSync(src as string); + renameCommitted = true; + }); + + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (renameCommitted && p === path.normalize(dest)) { + return { + isFile: () => false, + isDirectory: () => true, + isSymbolicLink: () => false, + } as any; + } + return realLstat(p); + }); + + const res = await engine.exportSaveTo({ + operationId: id, + destinationPath: dest, + options: { overwrite: true }, + }); + + expect(res.success).toBe(true); + expect(res.warning).toContain("post-commit audit detected unexpected file attributes"); + }); + + test("pre-publication rename failure throws MONGO_SAVE_FAILED and unlinks staging", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "rename-fail.json"); + + let stagingUnlinked = false; + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + stagingUnlinked = true; + } + return origUnlink(p); + }); + + jest.spyOn(fs.promises, "rename").mockImplementation(async () => { + const err: any = new Error("Device locked"); + err.code = "EBUSY"; + throw err; + }); + + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: dest, + options: { overwrite: true }, + }), + ).rejects.toThrow(/Failed to atomically replace destination file \(EBUSY\)/); + + expect(stagingUnlinked).toBe(true); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("existing destination symlink with overwrite true: rename replaces the symlink entry itself, while external target remains unchanged", async () => { + const { id } = createCompletedExport("json", '{"newData":true}'); + const targetFile = path.join(testDestDir, "target-true.txt"); + fs.writeFileSync(targetFile, "original target content", "utf8"); + const destSymlink = path.join(testDestDir, "dest-symlink-true.json"); + + let realSymlinkCreated = false; + try { + fs.symlinkSync(targetFile, destSymlink, "file"); + realSymlinkCreated = true; + } catch {} + + if (realSymlinkCreated) { + const res = await engine.exportSaveTo({ + operationId: id, + destinationPath: destSymlink, + options: { overwrite: true }, + }); + expect(res.success).toBe(true); + expect(fs.lstatSync(destSymlink).isSymbolicLink()).toBe(false); + expect(fs.readFileSync(destSymlink, "utf8")).toBe('{"newData":true}'); + expect(fs.readFileSync(targetFile, "utf8")).toBe("original target content"); + } else { + let renameTargetPassed: string | null = null; + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (p === path.normalize(destSymlink)) { + return { + isFile: () => false, + isDirectory: () => false, + isSymbolicLink: () => true, + isFIFO: () => false, + isSocket: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + } as any; + } + return fs.lstatSync(p); + }); + jest.spyOn(fs.promises, "rename").mockImplementation(async (src, target) => { + renameTargetPassed = target as string; + fs.copyFileSync(src as string, target as string); + fs.unlinkSync(src as string); + }); + + const res = await engine.exportSaveTo({ + operationId: id, + destinationPath: destSymlink, + options: { overwrite: true }, + }); + expect(res.success).toBe(true); + expect(renameTargetPassed).toBe(path.normalize(destSymlink)); + expect(fs.readFileSync(targetFile, "utf8")).toBe("original target content"); + } + }); + + test("directory appearing between destination inspection and rename throws MONGO_SAVE_FAILED", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "race-dir-dest.json"); + + jest.spyOn(fs.promises, "rename").mockImplementation(async () => { + const err: any = new Error("Is a directory"); + err.code = "EISDIR"; + throw err; + }); + + let stagingUnlinked = false; + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + stagingUnlinked = true; + } + return origUnlink(p); + }); + + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: dest, + options: { overwrite: true }, + }), + ).rejects.toThrow(/Failed to atomically replace destination file \(EISDIR\)/); + + expect(stagingUnlinked).toBe(true); + expect(fs.existsSync(sourceFilePath)).toBe(true); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("post-commit audit lstat throwing an error returns success with an advisory warning", async () => { + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "audit-throw.json"); + + let renameCommitted = false; + jest.spyOn(fs.promises, "rename").mockImplementation(async (src, target) => { + fs.copyFileSync(src as string, target as string); + fs.unlinkSync(src as string); + renameCommitted = true; + }); + + const realLstat = fs.promises.lstat; + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (renameCommitted && p === path.normalize(dest)) { + const err: any = new Error("Resource temporarily locked"); + err.code = "EBUSY"; + throw err; + } + return realLstat(p); + }); + + const res = await engine.exportSaveTo({ + operationId: id, + destinationPath: dest, + options: { overwrite: true }, + }); + + expect(res.success).toBe(true); + expect(res.warning).toContain("post-commit verification failed (EBUSY)"); + expect(fs.existsSync(dest)).toBe(true); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + }); + + // ----------------------------------------------------------------------- + // Streaming Lifecycle & Staging Failure Handling + // ----------------------------------------------------------------------- + describe("Streaming Lifecycle & Staging Failure Handling", () => { + test("stream failure followed by successful staging unlink", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "stream-fail-unlink-ok.json"); + + let stagingUnlinked = false; + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + stagingUnlinked = true; + } + return origUnlink(p); + }); + + jest.spyOn(fs, "createReadStream").mockImplementationOnce(() => { + throw new Error("Disk I/O error during read"); + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow( + /Failed to stream export data to staging file: Disk I\/O error during read/, + ); + + expect(stagingUnlinked).toBe(true); + expect(fs.existsSync(sourceFilePath)).toBe(true); // Source file preserved + expect(fs.existsSync(dest)).toBe(false); // Destination never published + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("stream failure followed by staging unlink failure and successful queue registration", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "stream-fail-unlink-fail.json"); + + jest.spyOn(fs, "createReadStream").mockImplementationOnce(() => { + throw new Error("Read stream broken pipe"); + }); + + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + const err: any = new Error("File locked by process"); + err.code = "EBUSY"; + throw err; + } + return origUnlink(p); + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Failed to stream export data to staging file: Read stream broken pipe/); + + expect(opManager.getPendingStagingCleanupsCount()).toBe(1); + expect(fs.existsSync(sourceFilePath)).toBe(true); + expect(fs.existsSync(dest)).toBe(false); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("stream failure where identity capture also fails", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "stream-id-capture-fail.json"); + + jest.spyOn(fs, "createReadStream").mockImplementationOnce(() => { + throw new Error("Stream abort error"); + }); + + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + const err: any = new Error("Unlink failed"); + err.code = "EACCES"; + throw err; + } + return origUnlink(p); + }); + + const origLstat = fs.promises.lstat; + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + const err: any = new Error("Cannot lstat staging file"); + err.code = "EPERM"; + throw err; + } + return origLstat(p); + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Failed to stream export data to staging file: Stream abort error/); + + // Identity capture failed -> registered with fail-closed identity (dev: 0, ino: 0) + expect(opManager.getPendingStagingCleanupsCount()).toBe(1); + const entries = Array.from((opManager as any).pendingStagingCleanups.values()) as any[]; + expect(entries[0].dev).toBe(0); + expect(entries[0].ino).toBe(0); + + expect(fs.existsSync(sourceFilePath)).toBe(true); + expect(fs.existsSync(dest)).toBe(false); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("stream failure where cleanup queue capacity is exhausted", async () => { + const { id, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "stream-queue-full.json"); + + // Fill queue to capacity (100) + for (let i = 0; i < 100; i++) { + opManager.recordPendingStagingCleanup(`/tmp/stream-full-${i}.tmp`); + } + expect(opManager.getPendingStagingCleanupsCount()).toBe(100); + + jest.spyOn(fs, "createReadStream").mockImplementationOnce(() => { + throw new Error("Premature stream EOF"); + }); + + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + const err: any = new Error("Unlink failed"); + err.code = "EBUSY"; + throw err; + } + return origUnlink(p); + }); + + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: dest }), + ).rejects.toThrow(/Failed to stream export data to staging file: Premature stream EOF/); + + // Queue refused 101st entry and retained capacity 100 + expect(opManager.getPendingStagingCleanupsCount()).toBe(100); + expect(fs.existsSync(sourceFilePath)).toBe(true); + expect(fs.existsSync(dest)).toBe(false); + expect(opManager.getOperation(id)?.isSaving).toBe(false); + }); + + test("every stream failure preserves the original error classification and releases the lock", async () => { + const { id, state, sourceFilePath } = createCompletedExport(); + const dest = path.join(testDestDir, "stream-preserve-err.json"); + + jest.spyOn(fs, "createReadStream").mockImplementationOnce(() => { + throw new Error("Specific custom streaming corruption error"); + }); + + let caughtErr: any; + try { + await engine.exportSaveTo({ operationId: id, destinationPath: dest }); + } catch (e) { + caughtErr = e; + } + + expect(caughtErr).toBeDefined(); + expect(caughtErr.code).toBe("MONGO_SAVE_FAILED"); + expect(caughtErr.message).toContain("Specific custom streaming corruption error"); + expect(state.isSaving).toBe(false); + expect(fs.existsSync(sourceFilePath)).toBe(true); + }); + }); + + // ----------------------------------------------------------------------- + // Universal Mutex & Lock Release Across All Paths + // ----------------------------------------------------------------------- + describe("Universal Mutex & Lock Release Across All Paths", () => { + test("rejects concurrent save while save is active", async () => { + const { id, state } = createCompletedExport(); + state.isSaving = true; + + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: path.join(testDestDir, "concurrent.json"), + }), + ).rejects.toThrow(/Another save operation is already in progress/); + + state.isSaving = false; + }); + + test("active save blocks cancelOperation and cleanupOperation", () => { + const { id, state } = createCompletedExport(); + state.isSaving = true; + + expect(() => opManager.cancel(id)).toThrow( + /Cannot cancel an export operation that is currently saving/, + ); + + expect(() => opManager.cleanup(id)).toThrow( + /Cannot clean up an export operation that is currently saving/, + ); + + state.isSaving = false; + }); + + test("lock is released in finally across multiple failure paths", async () => { + const { id, state } = createCompletedExport(); + + // 1. Validation failure + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: "relative.json" }), + ).rejects.toThrow(); + expect(state.isSaving ?? false).toBe(false); + + // 2. Inspection failure + const badDest = path.join(testDestDir, "bad-dir"); + fs.mkdirSync(badDest); + await expect( + engine.exportSaveTo({ operationId: id, destinationPath: badDest }), + ).rejects.toThrow(); + expect(state.isSaving).toBe(false); + + // 3. Link collision + jest.spyOn(fs.promises, "link").mockRejectedValueOnce({ code: "EEXIST" }); + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: path.join(testDestDir, "link-fail.json"), + }), + ).rejects.toThrow(); + expect(state.isSaving).toBe(false); + + // 4. Stream error + jest.spyOn(fs, "createReadStream").mockImplementationOnce(() => { + throw new Error("Stream read error"); + }); + await expect( + engine.exportSaveTo({ + operationId: id, + destinationPath: path.join(testDestDir, "stream-fail.json"), + }), + ).rejects.toThrow(); + expect(state.isSaving).toBe(false); + }); + }); + + // ----------------------------------------------------------------------- + // Bounded Staging Cleanup Queue & Capacity Overflow + // ----------------------------------------------------------------------- + describe("Bounded Staging Cleanup Queue & Capacity Overflow", () => { + test("records identity up to 100 entries and refuses beyond capacity", () => { + for (let i = 0; i < 100; i++) { + const tracked = opManager.recordPendingStagingCleanup(`/tmp/staging-${i}.tmp`); + expect(tracked).toBe(true); + } + expect(opManager.getPendingStagingCleanupsCount()).toBe(100); + + // 101st entry must be refused and return false + const overflowResult = opManager.recordPendingStagingCleanup("/tmp/staging-overflow.tmp"); + expect(overflowResult).toBe(false); + expect(opManager.getPendingStagingCleanupsCount()).toBe(100); + }); + + test("duplicate path registration is an idempotent no-op preserving authoritative identity", () => { + const initialIdentity = { + exactPath: "/tmp/staging-dup.tmp", + dev: 111, + ino: 222, + size: 500, + birthtimeMs: 12345, + registeredAt: 1000, + retryCount: 0, + }; + + const res1 = opManager.recordPendingStagingCleanup("/tmp/staging-dup.tmp", initialIdentity); + expect(res1).toBe(true); + expect(opManager.getPendingStagingCleanupsCount()).toBe(1); + + // Re-register with different metadata + const modifiedIdentity = { + exactPath: "/tmp/staging-dup.tmp", + dev: 999, + ino: 888, + size: 1000, + birthtimeMs: 54321, + registeredAt: 2000, + retryCount: 1, + }; + const res2 = opManager.recordPendingStagingCleanup( + "/tmp/staging-dup.tmp", + modifiedIdentity, + ); + expect(res2).toBe(true); + expect(opManager.getPendingStagingCleanupsCount()).toBe(1); + + // Original identity metadata MUST be preserved! + const recorded = opManager.getPendingStagingCleanup("/tmp/staging-dup.tmp"); + expect(recorded?.dev).toBe(111); + expect(recorded?.ino).toBe(222); + expect(recorded?.size).toBe(500); + }); + + test("post-publication save surfaces distinct overflow warning when queue is at capacity", async () => { + // Fill queue to 100 + for (let i = 0; i < 100; i++) { + opManager.recordPendingStagingCleanup(`/tmp/full-${i}.tmp`); + } + expect(opManager.getPendingStagingCleanupsCount()).toBe(100); + + const { id } = createCompletedExport(); + const dest = path.join(testDestDir, "overflow-save.json"); + + jest.spyOn(fs.promises, "link").mockImplementation(async (src, target) => { + fs.copyFileSync(src as string, target as string); + }); + + // Fail staging unlink + const origUnlink = fs.promises.unlink; + jest.spyOn(fs.promises, "unlink").mockImplementation(async (p) => { + if (typeof p === "string" && p.includes(".relwave-tmp-")) { + const err: any = new Error("Busy"); + err.code = "EBUSY"; + throw err; + } + return origUnlink(p); + }); + + const res = await engine.exportSaveTo({ operationId: id, destinationPath: dest }); + expect(res.success).toBe(true); // Publication committed! + expect(res.warning).toContain("cleanup queue is at capacity (100)"); + expect(res.warning).toContain("untracked orphan"); + }); + }); + + // ----------------------------------------------------------------------- + // Pre-Retry Identity Verification Loop (sweepExpired) + // ----------------------------------------------------------------------- + describe("Pre-Retry Identity Verification Loop (sweepExpired)", () => { + test("safely unlinks and evicts file when identity matches", async () => { + const stagingFile = path.join(testDestDir, "staging-match.tmp"); + fs.writeFileSync(stagingFile, "staging data", "utf8"); + const st = fs.lstatSync(stagingFile); + + opManager.recordPendingStagingCleanup(stagingFile, { + exactPath: stagingFile, + dev: st.dev, + ino: st.ino, + size: st.size, + registeredAt: Date.now(), + retryCount: 0, + }); + + await opManager.sweepExpired(); + expect(fs.existsSync(stagingFile)).toBe(false); + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); + }); + + test("safely evicts absent file (ENOENT)", async () => { + const nonExistent = path.join(testDestDir, "absent.tmp"); + opManager.recordPendingStagingCleanup(nonExistent, { + exactPath: nonExistent, + dev: 123, + ino: 456, + size: 100, + registeredAt: Date.now(), + retryCount: 0, + }); + + await opManager.sweepExpired(); + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); + }); + + test("aborts unlinking, preserves file on disk, and evicts from tracking on identity mismatch", async () => { + const stagingFile = path.join(testDestDir, "staging-mismatch.tmp"); + fs.writeFileSync(stagingFile, "real file", "utf8"); + const st = fs.lstatSync(stagingFile); + + opManager.recordPendingStagingCleanup(stagingFile, { + exactPath: stagingFile, + dev: st.dev, + ino: st.ino + 9999, // mismatched inode! + size: st.size, + registeredAt: Date.now(), + retryCount: 0, + }); + + await opManager.sweepExpired(); + // File on disk MUST be untouched! + expect(fs.existsSync(stagingFile)).toBe(true); + expect(fs.readFileSync(stagingFile, "utf8")).toBe("real file"); + // Evicted from automated tracking! + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); + }); + + test("aborts unlinking, preserves symlink on disk, and evicts from tracking on symlink detection", async () => { + const stagingFile = path.join(testDestDir, "staging-symlink.tmp"); + const realTarget = path.join(testDestDir, "target.txt"); + fs.writeFileSync(realTarget, "target content", "utf8"); + + // Mock lstat to report symlink + jest.spyOn(fs.promises, "lstat").mockImplementation(async (p) => { + if (p === stagingFile) { + return { + isFile: () => false, + isSymbolicLink: () => true, + isDirectory: () => false, + dev: 100, + ino: 200, + } as any; + } + return fs.lstatSync(p); + }); + + opManager.recordPendingStagingCleanup(stagingFile, { + exactPath: stagingFile, + dev: 100, + ino: 200, + size: 10, + registeredAt: Date.now(), + retryCount: 0, + }); + + await opManager.sweepExpired(); + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); + }); + + test("fails closed when identity is unpopulated (dev: 0, ino: 0)", async () => { + const stagingFile = path.join(testDestDir, "staging-zero-id.tmp"); + fs.writeFileSync(stagingFile, "content", "utf8"); + + opManager.recordPendingStagingCleanup(stagingFile, { + exactPath: stagingFile, + dev: 0, + ino: 0, + size: 7, + registeredAt: Date.now(), + retryCount: 0, + }); + + await opManager.sweepExpired(); + expect(fs.existsSync(stagingFile)).toBe(true); // Untouched on disk + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); // Evicted from queue + }); + + test("evicts after 5 failed retry attempts", async () => { + const stagingFile = path.join(testDestDir, "staging-locked.tmp"); + fs.writeFileSync(stagingFile, "locked", "utf8"); + const st = fs.lstatSync(stagingFile); + + opManager.recordPendingStagingCleanup(stagingFile, { + exactPath: stagingFile, + dev: st.dev, + ino: st.ino, + size: st.size, + registeredAt: Date.now(), + retryCount: 4, // 5th retry attempt + }); + + jest.spyOn(fs.promises, "unlink").mockRejectedValue(new Error("Still locked")); + + await opManager.sweepExpired(); + // After 5 retries, abandoned as untracked orphan + expect(opManager.getPendingStagingCleanupsCount()).toBe(0); + expect(fs.existsSync(stagingFile)).toBe(true); + }); + }); + + // ----------------------------------------------------------------------- + // Export Source Lifecycle & Sweep Coordination + // ----------------------------------------------------------------------- + describe("Export Source Lifecycle & Sweep Coordination", () => { + test("sweepExpired skips unlinking while isSaving === true", async () => { + const { id, state, sourceFilePath } = createCompletedExport(); + state.isSaving = true; + + // Force expiration + state.operation.expiresAt = new Date(Date.now() - 1000).toISOString(); + + await opManager.sweepExpired(); + expect(fs.existsSync(sourceFilePath)).toBe(true); + expect(opManager.getOperation(id)).not.toBeNull(); + + state.isSaving = false; + }); + + test("save completing after TTL expiry commits publication and triggers deferred source eviction", async () => { + const { id, state, sourceFilePath } = createCompletedExport("json", '{"expired":true}'); + const dest = path.join(testDestDir, "expired-save.json"); + + // Expire during save while isSaving === true + jest.spyOn(fs.promises, "rename").mockImplementation(async (src, target) => { + state.operation.expiresAt = new Date(Date.now() - 1000).toISOString(); + fs.copyFileSync(src as string, target as string); + fs.unlinkSync(src as string); + }); + + const res = await engine.exportSaveTo({ + operationId: id, + destinationPath: dest, + options: { overwrite: true }, + }); + + expect(res.success).toBe(true); + expect(fs.existsSync(dest)).toBe(true); + // Deferred cleanup unlinks source file + expect(fs.existsSync(sourceFilePath)).toBe(false); + // Operation evicted + expect(opManager.getOperation(id)).toBeNull(); + }); + + test("cleanupOperation called after TTL eviction returns alreadyCleaned: true idempotently", async () => { + const randomId = "11111111-2222-4333-8444-555555555555"; + const res = await engine.cleanupOperation(randomId); + expect(res.success).toBe(true); + expect(res.alreadyCleaned).toBe(true); + }); + + test("dispose() during an active save does not prematurely unlink the source", () => { + const { state, sourceFilePath } = createCompletedExport(); + state.isSaving = true; + + opManager.dispose(); + + // Source file MUST NOT be unlinked! + expect(fs.existsSync(sourceFilePath)).toBe(true); + // Active save protection is preserved, not forcibly reset + expect(state.isSaving).toBe(true); + + state.isSaving = false; + }); + }); + }); +}); diff --git a/bridge/__tests__/mongoIndex.test.ts b/bridge/__tests__/mongoIndex.test.ts new file mode 100644 index 00000000..d181c516 --- /dev/null +++ b/bridge/__tests__/mongoIndex.test.ts @@ -0,0 +1,1505 @@ +import { describe, expect, test, beforeEach, afterEach, jest } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { + evaluateIndexRedundancy, + mongoConnector, + MongoApiError, + mongodbCache, +} from "../src/connectors/mongo"; +import { QueryExecutor } from "../src/services/queryExecutor"; +import { IndexHandlers } from "../src/handlers/indexHandlers"; +import { DBType } from "../src/types"; +import type { MongoConfig, MongoIndexDetails } from "../src/types/mongo"; + +describe("MongoDB Index Management Unit Tests (M6 Spec §3, §9, §11)", () => { + const dummyConfig: MongoConfig = { + uri: "mongodb://localhost:27017/testdb", + database: "testdb", + }; + + // ========================================================================= + // 1. evaluateIndexRedundancy + // ========================================================================= + describe("evaluateIndexRedundancy", () => { + test("primary key index (_id_) is never marked redundant", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "_id_", + key: { _id: 1 }, + sizeBytes: 4096, + unique: true, + sparse: false, + isIdIndex: true, + }, + { + name: "compound_id", + key: { _id: 1, email: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated[0].isRedundant).toBe(false); + }); + + test("non-B-tree indexes (text, 2dsphere, hashed) are never covered by B-tree prefixes", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "text_idx", + key: { bio: "text" }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "geo_idx", + key: { location: "2dsphere" }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "hashed_idx", + key: { userId: "hashed" }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "compound_all", + key: { bio: 1, location: 1, userId: 1 }, + sizeBytes: 12000, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "text_idx")?.isRedundant).toBe(false); + expect(evaluated.find((i) => i.name === "geo_idx")?.isRedundant).toBe(false); + expect(evaluated.find((i) => i.name === "hashed_idx")?.isRedundant).toBe(false); + }); + + test("hidden compound index cannot make another index redundant", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_user", + key: { userId: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_user_status_hidden", + key: { userId: 1, status: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + hidden: true, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_user")?.isRedundant).toBe(false); + }); + + test("compound index covers single-field prefix and flags it as redundant", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_user", + key: { userId: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_user_status", + key: { userId: 1, status: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + const userIdx = evaluated.find((i) => i.name === "idx_user"); + expect(userIdx?.isRedundant).toBe(true); + expect(userIdx?.redundantReason).toContain('compound index "idx_user_status"'); + }); + + test("compound index covers shorter compound index with matching key order and directions", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_org_dept", + key: { orgId: 1, deptId: -1 }, + sizeBytes: 6000, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_org_dept_role", + key: { orgId: 1, deptId: -1, role: 1 }, + sizeBytes: 10000, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_org_dept")?.isRedundant).toBe(true); + }); + + test("compound index with different direction on second key does NOT make prefix redundant", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_org_dept", + key: { orgId: 1, deptId: 1 }, + sizeBytes: 6000, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_org_dept_role", + key: { orgId: 1, deptId: -1, role: 1 }, + sizeBytes: 10000, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_org_dept")?.isRedundant).toBe(false); + }); + + test("unique index is NEVER considered redundant with non-unique compound index", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_email_unique", + key: { email: 1 }, + sizeBytes: 4096, + unique: true, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_email_created", + key: { email: 1, createdAt: -1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_email_unique")?.isRedundant).toBe(false); + }); + + test("non-sparse index is NOT redundant with sparse compound index", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_phone", + key: { phone: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_phone_name_sparse", + key: { phone: 1, name: 1 }, + sizeBytes: 8192, + unique: false, + sparse: true, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_phone")?.isRedundant).toBe(false); + }); + + test("sparse index IS redundant with non-sparse compound index", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_phone_sparse", + key: { phone: 1 }, + sizeBytes: 4096, + unique: false, + sparse: true, + isIdIndex: false, + }, + { + name: "idx_phone_name", + key: { phone: 1, name: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_phone_sparse")?.isRedundant).toBe(true); + }); + + test("differing partialFilterExpression prevents redundancy", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_active_users", + key: { userId: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + partialFilterExpression: { active: true }, + }, + { + name: "idx_verified_users", + key: { userId: 1, verified: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + partialFilterExpression: { verified: true }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_active_users")?.isRedundant).toBe(false); + }); + + test("identical partialFilterExpression allows redundancy", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_active_user", + key: { userId: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + partialFilterExpression: { active: true }, + }, + { + name: "idx_active_user_date", + key: { userId: 1, date: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + partialFilterExpression: { active: true }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_active_user")?.isRedundant).toBe(true); + }); + + test("differing collation prevents redundancy", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_name_ci", + key: { name: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + collation: { locale: "en", strength: 2 }, + }, + { + name: "idx_name_age_cs", + key: { name: 1, age: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + collation: { locale: "en", strength: 3 }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_name_ci")?.isRedundant).toBe(false); + }); + + test("compound index with inverted directions (reverse traversal) covers compound prefix", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_rev_prefix", + key: { orgId: -1, deptId: 1 }, + sizeBytes: 6000, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_compound_full", + key: { orgId: 1, deptId: -1, role: 1 }, + sizeBytes: 12000, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_rev_prefix")?.isRedundant).toBe(true); + }); + + test("compound other index with special key (text, 2dsphere, hashed) cannot make B-tree prefix redundant", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_category", + key: { category: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_category_title_text", + key: { category: 1, title: "text" }, + sizeBytes: 10000, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_category_loc_geo", + key: { category: 1, loc: "2dsphere" }, + sizeBytes: 10000, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_category_id_hash", + key: { category: 1, id: "hashed" }, + sizeBytes: 10000, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_category")?.isRedundant).toBe(false); + }); + + test("default collation (undefined) matches explicit simple collation ({ locale: 'simple' })", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_user_default_coll", + key: { user: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_user_age_simple_coll", + key: { user: 1, age: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + collation: { locale: "simple" }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_user_default_coll")?.isRedundant).toBe(true); + }); + + test("default collation (undefined) does NOT match explicit non-simple collation ({ locale: 'en' })", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_user_default_coll", + key: { user: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_user_age_en_coll", + key: { user: 1, age: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + collation: { locale: "en", strength: 2 }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_user_default_coll")?.isRedundant).toBe(false); + }); + + test("partialFilterExpression with differing key order is recognized as equivalent", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_partial_a", + key: { status: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + partialFilterExpression: { a: 1, b: 2 }, + }, + { + name: "idx_partial_b", + key: { status: 1, date: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + partialFilterExpression: { b: 2, a: 1 }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_partial_a")?.isRedundant).toBe(true); + }); + + test("unique compound candidate does NOT cover shorter non-unique index (conservative constraint boundary)", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_email_non_unique", + key: { email: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_email_tenant_unique", + key: { email: 1, tenantId: 1 }, + sizeBytes: 8192, + unique: true, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes); + expect(evaluated.find((i) => i.name === "idx_email_non_unique")?.isRedundant).toBe(false); + }); + + test("effective collation: collection default non-simple collation propagates and prevents false redundancy with simple index", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_name_inherited", + key: { name: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + // collation omitted: inherits collection default + }, + { + name: "idx_name_age_simple", + key: { name: 1, age: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + collation: { locale: "simple" }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes, { + collectionDefaultCollation: { locale: "fr", strength: 2 }, + collectionCollationUnknown: false, + }); + expect(evaluated.find((i) => i.name === "idx_name_inherited")?.isRedundant).toBe(false); + }); + + test("effective collation: both indexes inherit same collection default and allow redundancy", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_name_inherited", + key: { name: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_name_age_inherited", + key: { name: 1, age: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes, { + collectionDefaultCollation: { locale: "fr", strength: 2 }, + collectionCollationUnknown: false, + }); + expect(evaluated.find((i) => i.name === "idx_name_inherited")?.isRedundant).toBe(true); + }); + + test("effective collation: suppresses redundancy when collection collation metadata is unknown", () => { + const indexes: MongoIndexDetails[] = [ + { + name: "idx_name_missing_coll", + key: { name: 1 }, + sizeBytes: 4096, + unique: false, + sparse: false, + isIdIndex: false, + }, + { + name: "idx_name_age_simple", + key: { name: 1, age: 1 }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + collation: { locale: "simple" }, + }, + ]; + const evaluated = evaluateIndexRedundancy(indexes, { + collectionDefaultCollation: null, + collectionCollationUnknown: true, // metadata unavailable + }); + expect(evaluated.find((i) => i.name === "idx_name_missing_coll")?.isRedundant).toBe(false); + }); + }); + + // ========================================================================= + // 2. MongoConnector Index Methods + // ========================================================================= + describe("MongoConnector Index Methods", () => { + let mockDb: any; + let mockCollection: any; + let mockRawIndexes: any[]; + let mockIndexSizes: Record; + let mockTotalIndexSize: number; + let mockStatsDocs: any[]; + let dbSpy: any; + + beforeEach(() => { + mockRawIndexes = [ + { v: 2, key: { _id: 1 }, name: "_id_" }, + { v: 2, key: { email: 1 }, name: "idx_email", unique: true }, + { v: 2, key: { email: 1, name: 1 }, name: "idx_email_name" }, + ]; + mockIndexSizes = { + _id_: 4096, + idx_email: 8192, + idx_email_name: 12288, + }; + mockTotalIndexSize = 24576; + mockStatsDocs = [ + { + name: "idx_email", + key: { email: 1 }, + host: "localhost:27017", + accesses: { ops: 50, since: new Date("2026-10-30T00:00:00.000Z") }, + }, + ]; + + mockCollection = { + listIndexes: jest.fn().mockReturnValue({ + toArray: jest.fn().mockImplementation(() => Promise.resolve(mockRawIndexes)), + }), + createIndex: jest + .fn() + .mockImplementation((_key: any, opts: any) => + Promise.resolve(opts?.name || "idx_created"), + ), + dropIndex: jest.fn().mockImplementation((_name: any) => Promise.resolve({ ok: 1 })), + aggregate: jest.fn().mockReturnValue({ + toArray: jest.fn().mockImplementation(() => Promise.resolve(mockStatsDocs)), + }), + }; + + mockDb = { + collection: jest.fn().mockReturnValue(mockCollection), + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ name: "users", options: {} }]), + }), + command: jest.fn().mockImplementation((cmd: any) => { + if (cmd.collStats) { + return Promise.resolve({ + indexSizes: mockIndexSizes, + totalIndexSize: mockTotalIndexSize, + }); + } + if (cmd.collMod) { + return Promise.resolve({ ok: 1 }); + } + return Promise.resolve({}); + }), + }; + + dbSpy = jest.spyOn(MongoClient.prototype, "db").mockReturnValue(mockDb as any); + }); + + afterEach(() => { + dbSpy.mockRestore(); + }); + + describe("listIndexes", () => { + test("rejects invalid collection name", async () => { + await expect(mongoConnector.listIndexes(dummyConfig, { collection: "" })).rejects.toThrow( + MongoApiError, + ); + await expect( + mongoConnector.listIndexes(dummyConfig, { collection: "system.views" }), + ).rejects.toThrow(MongoApiError); + }); + + test("returns mapped indexes with sizes and redundancy flags", async () => { + const result = await mongoConnector.listIndexes(dummyConfig, { collection: "users" }); + expect(result.collection).toBe("users"); + expect(result.indexes).toHaveLength(3); + expect(result.totalSizeBytes).toBe(24576); + + const idIdx = result.indexes.find((i) => i.name === "_id_"); + expect(idIdx).toBeDefined(); + expect(idIdx?.isIdIndex).toBe(true); + expect(idIdx?.sizeBytes).toBe(4096); + expect(idIdx?.isRedundant).toBe(false); + + const emailIdx = result.indexes.find((i) => i.name === "idx_email"); + expect(emailIdx?.unique).toBe(true); + expect(emailIdx?.sizeBytes).toBe(8192); + // Unique index is never redundant even though prefix matches + expect(emailIdx?.isRedundant).toBe(false); + }); + + test("handles NamespaceNotFound gracefully by returning empty indexes array", async () => { + mockCollection.listIndexes = jest.fn().mockReturnValue({ + toArray: jest.fn().mockRejectedValue({ + code: 26, + codeName: "NamespaceNotFound", + message: "ns not found", + }), + }); + + const result = await mongoConnector.listIndexes(dummyConfig, { + collection: "missing_coll", + }); + expect(result.indexes).toEqual([]); + expect(result.totalSizeBytes).toBe(0); + }); + + test("handles collStats failure gracefully by defaulting sizes to 0", async () => { + mockDb.command = jest + .fn() + .mockRejectedValue(new Error("collStats not allowed on view")); + + const result = await mongoConnector.listIndexes(dummyConfig, { collection: "users" }); + expect(result.indexes).toHaveLength(3); + expect(result.totalSizeBytes).toBe(0); + expect(result.indexes[0].sizeBytes).toBe(0); + }); + }); + + describe("createIndex", () => { + test("rejects when connection is read-only", async () => { + await expect( + mongoConnector.createIndex( + dummyConfig, + { collection: "users", key: { email: 1 } }, + { readOnly: true }, + ), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + }); + + test("rejects invalid collection name", async () => { + await expect( + mongoConnector.createIndex(dummyConfig, { collection: "$invalid", key: { email: 1 } }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects non-object or empty key", async () => { + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: null as unknown as any, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + await expect( + mongoConnector.createIndex(dummyConfig, { collection: "users", key: {} }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects invalid direction in key spec", async () => { + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 2 as any }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects invalid options (negative expireAfterSeconds, empty name)", async () => { + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { createdAt: 1 }, + options: { expireAfterSeconds: -10 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1 }, + options: { name: " " }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("creates single, compound, and special index types forwarding driver options", async () => { + const clearCacheSpy = jest.spyOn(mongodbCache, "clearCollectionCache"); + + const res = await mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1, age: -1 }, + options: { + name: "idx_custom_email_age", + unique: true, + sparse: true, + partialFilterExpression: { active: true }, + collation: { locale: "en", strength: 2 }, + hidden: false, + }, + }); + + expect(res.success).toBe(true); + expect(res.indexName).toBe("idx_custom_email_age"); + expect(res.executionTimeMs).toBeGreaterThanOrEqual(0); + expect(mockCollection.createIndex).toHaveBeenCalledWith( + { email: 1, age: -1 }, + expect.objectContaining({ + name: "idx_custom_email_age", + unique: true, + sparse: true, + partialFilterExpression: { active: true }, + collation: { locale: "en", strength: 2 }, + hidden: false, + }), + ); + expect(clearCacheSpy).toHaveBeenCalledWith(dummyConfig, "users"); + }); + + test("creates single-field TTL index", async () => { + const res = await mongoConnector.createIndex(dummyConfig, { + collection: "sessions", + key: { createdAt: 1 }, + options: { + name: "idx_ttl", + expireAfterSeconds: 3600, + }, + }); + expect(res.success).toBe(true); + expect(mockCollection.createIndex).toHaveBeenCalledWith( + { createdAt: 1 }, + expect.objectContaining({ expireAfterSeconds: 3600 }), + ); + }); + + test("rejects compound TTL index with MONGO_VALIDATION_ERROR", async () => { + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "sessions", + key: { createdAt: 1, userId: 1 }, + options: { + expireAfterSeconds: 3600, + }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("single field"), + }); + }); + + test("supports special index directions: text, 2dsphere, hashed", async () => { + const textRes = await mongoConnector.createIndex(dummyConfig, { + collection: "articles", + key: { title: "text", body: "text" }, + }); + expect(textRes.success).toBe(true); + + const geoRes = await mongoConnector.createIndex(dummyConfig, { + collection: "places", + key: { location: "2dsphere" }, + }); + expect(geoRes.success).toBe(true); + + const hashRes = await mongoConnector.createIndex(dummyConfig, { + collection: "accounts", + key: { accountId: "hashed" }, + }); + expect(hashRes.success).toBe(true); + }); + + test("rejects illegal special index combinations with MONGO_VALIDATION_ERROR", async () => { + // Text + 2dsphere + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { title: "text", loc: "2dsphere" }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("text index with 2dsphere"), + }); + + // Text + hashed + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { title: "text", id: "hashed" }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("text index with 2dsphere or hashed"), + }); + + // 2dsphere + hashed + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { loc: "2dsphere", id: "hashed" }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("2dsphere index with hashed"), + }); + + // Multiple hashed + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { id1: "hashed", id2: "hashed" }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("more than one hashed key"), + }); + + // Text with unique + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { title: "text" }, + options: { unique: true }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("Text indexes cannot enforce unique"), + }); + + // TTL on _id + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { _id: 1 }, + options: { expireAfterSeconds: 60 }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("immutable primary key '_id'"), + }); + + // TTL on special key + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "items", + key: { title: "text" }, + options: { expireAfterSeconds: 60 }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("special non-B-tree key types"), + }); + }); + + test("rejects malformed partial filter expressions and collations", async () => { + // Malformed partialFilter (primitive) + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1 }, + options: { partialFilterExpression: "not-an-object" as any }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("non-null object"), + }); + + // Malformed partialFilter (prohibited JS / $where) + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1 }, + options: { partialFilterExpression: { $where: "sleep(1000)" } }, + }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN_OPERATOR", + }); + + // Malformed collation (empty locale) + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1 }, + options: { collation: { locale: " " } }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("collation.locale"), + }); + + // Malformed collation (invalid strength) + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1 }, + options: { collation: { locale: "en", strength: 99 as any } }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("collation.strength"), + }); + }); + + test("maps driver InvalidOptions on hidden: true to MONGO_UNSUPPORTED_OPERATION (< MongoDB 4.4)", async () => { + mockCollection.createIndex = jest.fn().mockRejectedValue({ + code: 72, + codeName: "InvalidOptions", + message: "unrecognized option: hidden", + }); + + await expect( + mongoConnector.createIndex(dummyConfig, { + collection: "users", + key: { email: 1 }, + options: { hidden: true }, + }), + ).rejects.toMatchObject({ + code: "MONGO_UNSUPPORTED_OPERATION", + message: expect.stringContaining("MongoDB 4.4"), + }); + }); + + test("rejects when config.readonly is true even without options.readOnly", async () => { + const readOnlyConfig = { ...dummyConfig, readonly: true }; + await expect( + mongoConnector.createIndex(readOnlyConfig, { + collection: "users", + key: { email: 1 }, + }), + ).rejects.toMatchObject({ + code: "MONGO_READ_ONLY", + }); + }); + }); + + describe("dropIndex", () => { + test("rejects when connection is read-only", async () => { + await expect( + mongoConnector.dropIndex( + dummyConfig, + { collection: "users", indexName: "idx_email" }, + { readOnly: true }, + ), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + }); + + test("strictly prevents dropping the immutable default primary key index _id_", async () => { + await expect( + mongoConnector.dropIndex(dummyConfig, { collection: "users", indexName: "_id_" }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: expect.stringContaining("_id_"), + }); + expect(mockCollection.dropIndex).not.toHaveBeenCalled(); + }); + + test("strictly prevents dropping wildcard '*' index with MONGO_FORBIDDEN", async () => { + await expect( + mongoConnector.dropIndex(dummyConfig, { collection: "users", indexName: "*" }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: expect.stringContaining("Wildcard"), + }); + expect(mockCollection.dropIndex).not.toHaveBeenCalled(); + }); + + test("rejects empty or whitespace indexName", async () => { + await expect( + mongoConnector.dropIndex(dummyConfig, { collection: "users", indexName: "" }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + await expect( + mongoConnector.dropIndex(dummyConfig, { collection: "users", indexName: " " }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("drops secondary index and clears caches", async () => { + const clearCacheSpy = jest.spyOn(mongodbCache, "clearCollectionCache"); + + const res = await mongoConnector.dropIndex(dummyConfig, { + collection: "users", + indexName: "idx_email", + }); + + expect(res.success).toBe(true); + expect(res.droppedIndexName).toBe("idx_email"); + expect(res.executionTimeMs).toBeGreaterThanOrEqual(0); + expect(mockCollection.dropIndex).toHaveBeenCalledWith("idx_email", expect.any(Object)); + expect(clearCacheSpy).toHaveBeenCalledWith(dummyConfig, "users"); + }); + + test("maps driver IndexNotFound (code 27) to MONGO_NOT_FOUND", async () => { + mockCollection.dropIndex = jest.fn().mockRejectedValue({ + code: 27, + codeName: "IndexNotFound", + message: "index not found with name [idx_missing]", + }); + + await expect( + mongoConnector.dropIndex(dummyConfig, { + collection: "users", + indexName: "idx_missing", + }), + ).rejects.toMatchObject({ code: "MONGO_NOT_FOUND" }); + }); + }); + + describe("getIndexStats", () => { + test("rejects invalid collection name", async () => { + await expect(mongoConnector.getIndexStats(dummyConfig, { collection: "" })).rejects.toThrow( + MongoApiError, + ); + }); + + test("aggregates $indexStats and maps results into stats items", async () => { + const res = await mongoConnector.getIndexStats(dummyConfig, { collection: "users" }); + expect(res.collection).toBe("users"); + expect(res.stats.idx_email).toBeDefined(); + expect(res.stats.idx_email.ops).toBe(50); + expect(res.stats.idx_email.host).toBe("localhost:27017"); + expect(res.stats.idx_email.since).toBe("2026-10-30T00:00:00.000Z"); + expect(mockCollection.aggregate).toHaveBeenCalledWith( + [{ $indexStats: {} }], + expect.any(Object), + ); + }); + + test("handles NamespaceNotFound by returning empty stats", async () => { + mockCollection.aggregate = jest.fn().mockReturnValue({ + toArray: jest.fn().mockRejectedValue({ code: 26, codeName: "NamespaceNotFound" }), + }); + + const res = await mongoConnector.getIndexStats(dummyConfig, { collection: "missing" }); + expect(res.stats).toEqual({}); + }); + }); + + describe("hideIndex", () => { + test("rejects when connection is read-only", async () => { + await expect( + mongoConnector.hideIndex( + dummyConfig, + { collection: "users", indexName: "idx_email", hidden: true }, + { readOnly: true }, + ), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + }); + + test("rejects hiding the immutable default primary key index _id_", async () => { + await expect( + mongoConnector.hideIndex(dummyConfig, { + collection: "users", + indexName: "_id_", + hidden: true, + }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: expect.stringContaining("_id_"), + }); + expect(mockDb.command).not.toHaveBeenCalled(); + }); + + test("rejects non-boolean hidden parameter", async () => { + await expect( + mongoConnector.hideIndex(dummyConfig, { + collection: "users", + indexName: "idx_email", + hidden: "yes" as any, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("issues collMod command to toggle hidden status and clears caches", async () => { + const clearCacheSpy = jest.spyOn(mongodbCache, "clearCollectionCache"); + + const res = await mongoConnector.hideIndex(dummyConfig, { + collection: "users", + indexName: "idx_email", + hidden: true, + }); + + expect(res.success).toBe(true); + expect(res.indexName).toBe("idx_email"); + expect(res.hidden).toBe(true); + expect(mockDb.command).toHaveBeenCalledWith({ + collMod: "users", + index: { + name: "idx_email", + hidden: true, + }, + }); + expect(clearCacheSpy).toHaveBeenCalledWith(dummyConfig, "users"); + }); + + test("maps IndexNotFound error to MONGO_NOT_FOUND", async () => { + mockDb.command = jest.fn().mockRejectedValue({ + code: 27, + codeName: "IndexNotFound", + message: "cannot find index to hide", + }); + + await expect( + mongoConnector.hideIndex(dummyConfig, { + collection: "users", + indexName: "idx_missing", + hidden: false, + }), + ).rejects.toMatchObject({ code: "MONGO_NOT_FOUND" }); + }); + + test("maps unsupported collMod option (< MongoDB 4.4) to MONGO_UNSUPPORTED_OPERATION", async () => { + mockDb.command = jest.fn().mockRejectedValue({ + code: 72, + codeName: "InvalidOptions", + message: "no option found to collMod: index", + }); + + await expect( + mongoConnector.hideIndex(dummyConfig, { + collection: "users", + indexName: "idx_email", + hidden: true, + }), + ).rejects.toMatchObject({ + code: "MONGO_UNSUPPORTED_OPERATION", + message: expect.stringContaining("MongoDB 4.4"), + }); + }); + }); + }); + + // ========================================================================= + // 3. QueryExecutor Delegation & Type Check Tests + // ========================================================================= + describe("QueryExecutor Index Delegation", () => { + let executor: QueryExecutor; + + beforeEach(() => { + executor = new QueryExecutor(); + }); + + test("throws MONGO_UNSUPPORTED_OPERATION when dbType is not MONGODB", async () => { + const nonMongoTypes = [DBType.POSTGRES, DBType.MYSQL, DBType.SQLITE, DBType.DUCKDB]; + + for (const t of nonMongoTypes) { + await expect( + executor.listIndexes(dummyConfig, t, { collection: "users" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + + await expect( + executor.createIndex(dummyConfig, t, { collection: "users", key: { a: 1 } }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + + await expect( + executor.dropIndex(dummyConfig, t, { collection: "users", indexName: "idx_a" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + + await expect( + executor.getIndexStats(dummyConfig, t, { collection: "users" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + + await expect( + executor.hideIndex(dummyConfig, t, { + collection: "users", + indexName: "idx_a", + hidden: true, + }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + } + }); + + test("delegates to mongoConnector when dbType is MONGODB", async () => { + const listSpy = jest + .spyOn(mongoConnector, "listIndexes") + .mockResolvedValueOnce({ collection: "users", indexes: [], totalSizeBytes: 0 }); + const createSpy = jest + .spyOn(mongoConnector, "createIndex") + .mockResolvedValueOnce({ success: true, indexName: "idx_a", executionTimeMs: 5 }); + const dropSpy = jest + .spyOn(mongoConnector, "dropIndex") + .mockResolvedValueOnce({ success: true, droppedIndexName: "idx_a", executionTimeMs: 5 }); + const statsSpy = jest + .spyOn(mongoConnector, "getIndexStats") + .mockResolvedValueOnce({ collection: "users", stats: {} }); + const hideSpy = jest + .spyOn(mongoConnector, "hideIndex") + .mockResolvedValueOnce({ success: true, indexName: "idx_a", hidden: true }); + + await executor.listIndexes(dummyConfig, DBType.MONGODB, { collection: "users" }); + expect(listSpy).toHaveBeenCalledWith(dummyConfig, { collection: "users" }); + + await executor.createIndex( + dummyConfig, + DBType.MONGODB, + { collection: "users", key: { a: 1 } }, + { readOnly: false }, + ); + expect(createSpy).toHaveBeenCalledWith( + dummyConfig, + { collection: "users", key: { a: 1 } }, + { readOnly: false }, + ); + + await executor.dropIndex( + dummyConfig, + DBType.MONGODB, + { collection: "users", indexName: "idx_a" }, + { readOnly: false }, + ); + expect(dropSpy).toHaveBeenCalledWith( + dummyConfig, + { collection: "users", indexName: "idx_a" }, + { readOnly: false }, + ); + + await executor.getIndexStats(dummyConfig, DBType.MONGODB, { collection: "users" }); + expect(statsSpy).toHaveBeenCalledWith(dummyConfig, { collection: "users" }); + + await executor.hideIndex( + dummyConfig, + DBType.MONGODB, + { collection: "users", indexName: "idx_a", hidden: true }, + { readOnly: false }, + ); + expect(hideSpy).toHaveBeenCalledWith( + dummyConfig, + { collection: "users", indexName: "idx_a", hidden: true }, + { readOnly: false }, + ); + }); + }); + + // ========================================================================= + // 4. IndexHandlers RPC Dispatch Tests + // ========================================================================= + describe("IndexHandlers RPC Dispatch", () => { + let mockRpc: any; + let mockLogger: any; + let mockDbService: any; + let mockQueryExecutor: any; + let handlers: IndexHandlers; + + beforeEach(() => { + mockRpc = { + sendResponse: jest.fn(), + sendError: jest.fn(), + }; + mockLogger = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + }; + mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + db: { id: "mongo-1", name: "testdb", readonly: false }, + conn: dummyConfig, + dbType: DBType.MONGODB, + }), + withConnection: jest.fn().mockImplementation(async (dbId: string, fn: any) => { + const connInfo = await mockDbService.getDatabaseConnection(dbId); + return fn(connInfo.conn, connInfo.db, connInfo.dbType); + }), + }; + mockQueryExecutor = { + listIndexes: jest.fn().mockResolvedValue({ + collection: "users", + indexes: [], + totalSizeBytes: 0, + }), + createIndex: jest.fn().mockResolvedValue({ + success: true, + indexName: "idx_test", + executionTimeMs: 10, + }), + dropIndex: jest.fn().mockResolvedValue({ + success: true, + droppedIndexName: "idx_test", + executionTimeMs: 8, + }), + getIndexStats: jest.fn().mockResolvedValue({ + collection: "users", + stats: {}, + }), + hideIndex: jest.fn().mockResolvedValue({ + success: true, + indexName: "idx_test", + hidden: true, + }), + }; + + handlers = new IndexHandlers(mockRpc, mockLogger, mockDbService, mockQueryExecutor); + }); + + test("handleListIndexes dispatches successfully", async () => { + await handlers.handleListIndexes({ dbId: "mongo-1", collection: "users" }, 1); + expect(mockQueryExecutor.listIndexes).toHaveBeenCalledWith(dummyConfig, DBType.MONGODB, { + collection: "users", + }); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(1, { + ok: true, + data: expect.objectContaining({ collection: "users" }), + }); + }); + + test("handleListIndexes rejects missing dbId or collection", async () => { + await handlers.handleListIndexes({ collection: "users" }, 2); + expect(mockRpc.sendError).toHaveBeenCalledWith(2, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + + await handlers.handleListIndexes({ dbId: "mongo-1" }, 3); + expect(mockRpc.sendError).toHaveBeenCalledWith(3, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + }); + + test("handleCreateIndex extracts readOnly and dispatches successfully", async () => { + await handlers.handleCreateIndex( + { dbId: "mongo-1", collection: "users", key: { email: 1 } }, + 4, + ); + expect(mockQueryExecutor.createIndex).toHaveBeenCalledWith( + dummyConfig, + DBType.MONGODB, + { collection: "users", key: { email: 1 }, options: undefined }, + { readOnly: false }, + ); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(4, { + ok: true, + data: expect.objectContaining({ success: true }), + }); + }); + + test("handleCreateIndex extracts readOnly: true from db metadata", async () => { + mockDbService.getDatabaseConnection.mockResolvedValueOnce({ + db: { id: "mongo-ro", name: "testdb", readonly: true }, + conn: dummyConfig, + dbType: DBType.MONGODB, + }); + + await handlers.handleCreateIndex( + { dbId: "mongo-ro", collection: "users", key: { email: 1 } }, + 5, + ); + expect(mockQueryExecutor.createIndex).toHaveBeenCalledWith( + dummyConfig, + DBType.MONGODB, + { collection: "users", key: { email: 1 }, options: undefined }, + { readOnly: true }, + ); + }); + + test("handleCreateIndex rejects missing parameters", async () => { + await handlers.handleCreateIndex({ dbId: "mongo-1", collection: "users" }, 6); + expect(mockRpc.sendError).toHaveBeenCalledWith(6, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or key", + }); + }); + + test("handleDropIndex dispatches successfully", async () => { + await handlers.handleDropIndex( + { dbId: "mongo-1", collection: "users", indexName: "idx_email" }, + 7, + ); + expect(mockQueryExecutor.dropIndex).toHaveBeenCalledWith( + dummyConfig, + DBType.MONGODB, + { collection: "users", indexName: "idx_email" }, + { readOnly: false }, + ); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(7, { + ok: true, + data: expect.objectContaining({ success: true }), + }); + }); + + test("handleDropIndex rejects missing parameters", async () => { + await handlers.handleDropIndex({ dbId: "mongo-1", collection: "users" }, 8); + expect(mockRpc.sendError).toHaveBeenCalledWith(8, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or indexName", + }); + }); + + test("handleGetIndexStats dispatches successfully", async () => { + await handlers.handleGetIndexStats({ dbId: "mongo-1", collection: "users" }, 9); + expect(mockQueryExecutor.getIndexStats).toHaveBeenCalledWith(dummyConfig, DBType.MONGODB, { + collection: "users", + }); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(9, { + ok: true, + data: expect.objectContaining({ collection: "users" }), + }); + }); + + test("handleHideIndex validates hidden boolean and dispatches successfully", async () => { + await handlers.handleHideIndex( + { dbId: "mongo-1", collection: "users", indexName: "idx_email", hidden: true }, + 10, + ); + expect(mockQueryExecutor.hideIndex).toHaveBeenCalledWith( + dummyConfig, + DBType.MONGODB, + { collection: "users", indexName: "idx_email", hidden: true }, + { readOnly: false }, + ); + expect(mockRpc.sendResponse).toHaveBeenCalledWith(10, { + ok: true, + data: expect.objectContaining({ success: true, hidden: true }), + }); + }); + + test("handleHideIndex rejects non-boolean hidden parameter", async () => { + await handlers.handleHideIndex( + { dbId: "mongo-1", collection: "users", indexName: "idx_email", hidden: "invalid" }, + 11, + ); + expect(mockRpc.sendError).toHaveBeenCalledWith(11, { + code: "MONGO_VALIDATION_ERROR", + message: expect.stringContaining("hidden (must be a boolean)"), + }); + }); + + test("maps errors using sendMongoError", async () => { + mockQueryExecutor.listIndexes.mockRejectedValueOnce( + new MongoApiError("MONGO_NOT_FOUND", "Collection does not exist"), + ); + + await handlers.handleListIndexes({ dbId: "mongo-1", collection: "missing" }, 12); + expect(mockRpc.sendError).toHaveBeenCalledWith(12, { + code: "MONGO_NOT_FOUND", + message: "Collection does not exist", + }); + expect(mockLogger.error).toHaveBeenCalled(); + }); + }); +}); diff --git a/bridge/__tests__/mongoMigrationEngine.test.ts b/bridge/__tests__/mongoMigrationEngine.test.ts new file mode 100644 index 00000000..a3e73004 --- /dev/null +++ b/bridge/__tests__/mongoMigrationEngine.test.ts @@ -0,0 +1,1417 @@ +import { describe, expect, test, beforeEach, afterEach } from "@jest/globals"; +import fs from "fs"; +import path from "path"; +import os from "os"; +import { Db } from "mongodb"; +import { + validateManifest, + assertValidManifest, + computeManifestChecksum, + validateSafeFilePath, + listMigrations, + applyMigration, + rollbackMigration, + rebuildMigrationSummary, + getMigrationStatus, + compensateStep, + MAX_MANIFEST_SIZE_BYTES, + EVENTS_COLLECTION, + MIGRATIONS_COLLECTION, +} from "../src/services/mongoMigrationEngine"; +import { LOCK_COLLECTION, LOCK_DOC_ID } from "../src/services/mongoMigrationLock"; +import { MongoApiError } from "../src/connectors/mongo"; +import { + MongoMigrationManifest, + MongoMigrationSummary, + MongoMigrationEvent, + MongoMigrationLockDoc, +} from "../src/types/mongoMigration"; + +/** + * Creates an in-memory mock MongoDB database for migration engine testing. + */ +function createMockEngineDb() { + const locks = new Map(); + const summaries = new Map(); + const events: MongoMigrationEvent[] = []; + const collectionsData = new Map(); + const collectionsIndexes = new Map(); + const collectionsOptions = new Map(); + + const getOrCreateCollData = (name: string) => { + if (!collectionsData.has(name)) collectionsData.set(name, []); + return collectionsData.get(name)!; + }; + + const getOrCreateCollIndexes = (name: string) => { + if (!collectionsIndexes.has(name)) { + collectionsIndexes.set(name, [{ name: "_id_", key: { _id: 1 }, v: 2 }]); + } + return collectionsIndexes.get(name)!; + }; + + const db: any = { + async createCollection(name: string, options?: any) { + if (collectionsData.has(name)) { + const err: any = new Error(`Collection ${name} already exists`); + err.code = 48; + throw err; + } + collectionsData.set(name, []); + collectionsOptions.set(name, options || {}); + getOrCreateCollIndexes(name); + return { collectionName: name }; + }, + + listCollections(filter?: any) { + return { + async toArray() { + const list: any[] = []; + for (const [name, options] of collectionsOptions.entries()) { + if (filter?.name && filter.name !== name) continue; + list.push({ name, options }); + } + return list; + }, + }; + }, + + async command(cmd: any) { + if (cmd.collMod) { + const collName = cmd.collMod; + const currentOptions = collectionsOptions.get(collName) || {}; + if (cmd.validator) currentOptions.validator = cmd.validator; + if (cmd.validationLevel) currentOptions.validationLevel = cmd.validationLevel; + if (cmd.validationAction) currentOptions.validationAction = cmd.validationAction; + collectionsOptions.set(collName, currentOptions); + return { ok: 1 }; + } + return { ok: 1 }; + }, + + collection(name: string) { + if (name === LOCK_COLLECTION) { + return { + async findOneAndUpdate(filter: any, update: any, options?: any) { + const existing = locks.get(LOCK_DOC_ID); + const now = new Date(); + + if (!existing) { + const fencingToken = update.$inc?.fencingToken || 1; + const newDoc: MongoMigrationLockDoc = { + _id: LOCK_DOC_ID, + ownerId: update.$set?.ownerId || null, + migrationId: update.$set?.migrationId || null, + acquiredAt: update.$set?.acquiredAt || now, + heartbeatAt: update.$set?.heartbeatAt || now, + expiresAt: update.$set?.expiresAt || new Date(now.getTime() + 60000), + fencingToken, + }; + locks.set(LOCK_DOC_ID, newDoc); + return newDoc; + } + + const ownerMatch = filter.$or?.some((cond: any) => { + if ("ownerId" in cond && cond.ownerId === existing.ownerId) return true; + if ("ownerId" in cond && cond.ownerId === null && existing.ownerId === null) + return true; + if ( + cond.expiresAt?.$lt && + existing.expiresAt && + new Date(existing.expiresAt).getTime() < new Date(cond.expiresAt.$lt).getTime() + ) { + return true; + } + return false; + }); + + if (ownerMatch) { + const fencingToken = (existing.fencingToken || 0) + (update.$inc?.fencingToken || 1); + const updatedDoc: MongoMigrationLockDoc = { + ...existing, + ownerId: + update.$set?.ownerId !== undefined ? update.$set.ownerId : existing.ownerId, + migrationId: + update.$set?.migrationId !== undefined + ? update.$set.migrationId + : existing.migrationId, + acquiredAt: update.$set?.acquiredAt || now, + heartbeatAt: update.$set?.heartbeatAt || now, + expiresAt: update.$set?.expiresAt || existing.expiresAt, + fencingToken, + }; + locks.set(LOCK_DOC_ID, updatedDoc); + return updatedDoc; + } + + if (options?.upsert) { + const err: any = new Error("E11000 duplicate key error"); + err.code = 11000; + throw err; + } + return null; + }, + + async findOne(filter: any) { + if (filter._id === LOCK_DOC_ID) { + const doc = locks.get(LOCK_DOC_ID); + if (!doc) return null; + if (filter.ownerId && doc.ownerId !== filter.ownerId) return null; + if (filter.fencingToken !== undefined && doc.fencingToken !== filter.fencingToken) + return null; + if (filter.expiresAt?.$gt) { + if ( + !doc.expiresAt || + new Date(doc.expiresAt).getTime() <= new Date(filter.expiresAt.$gt).getTime() + ) { + return null; + } + } + return { ...doc }; + } + return null; + }, + + async updateOne(filter: any, update: any) { + const doc = locks.get(LOCK_DOC_ID); + if (!doc) return { matchedCount: 0, modifiedCount: 0 }; + if (filter.ownerId && doc.ownerId !== filter.ownerId) + return { matchedCount: 0, modifiedCount: 0 }; + if (filter.fencingToken !== undefined && doc.fencingToken !== filter.fencingToken) { + return { matchedCount: 0, modifiedCount: 0 }; + } + if (filter.expiresAt?.$gt) { + if ( + !doc.expiresAt || + new Date(doc.expiresAt).getTime() <= new Date(filter.expiresAt.$gt).getTime() + ) { + return { matchedCount: 0, modifiedCount: 0 }; + } + } + if (update.$set) Object.assign(doc, update.$set); + return { matchedCount: 1, modifiedCount: 1 }; + }, + + async deleteOne(filter: any) { + const doc = locks.get(LOCK_DOC_ID); + if (!doc) return { deletedCount: 0 }; + if (filter.ownerId && doc.ownerId !== filter.ownerId) return { deletedCount: 0 }; + if (filter.fencingToken !== undefined && doc.fencingToken !== filter.fencingToken) { + return { deletedCount: 0 }; + } + locks.delete(LOCK_DOC_ID); + return { deletedCount: 1 }; + }, + }; + } + + if (name === MIGRATIONS_COLLECTION) { + return { + async findOne(filter: any) { + if (filter._id) { + const doc = summaries.get(filter._id); + return doc ? { ...doc } : null; + } + return null; + }, + + find(filter: any = {}) { + let list = Array.from(summaries.values()); + if (filter.status?.$ne) { + list = list.filter((s) => s.status !== filter.status.$ne); + } else if (filter.status) { + list = list.filter((s) => s.status === filter.status); + } + return { + sort(sortObj: any) { + if (sortObj._id === 1) { + list.sort((a, b) => a._id.localeCompare(b._id)); + } else if (sortObj._id === -1) { + list.sort((a, b) => b._id.localeCompare(a._id)); + } + return { + limit(n: number) { + list = list.slice(0, n); + return { + async toArray() { + return list.map((x) => ({ ...x })); + }, + }; + }, + async toArray() { + return list.map((x) => ({ ...x })); + }, + }; + }, + async toArray() { + return list.map((x) => ({ ...x })); + }, + }; + }, + + async updateOne(filter: any, update: any, options?: any) { + const id = filter._id; + let doc = summaries.get(id); + if (!doc && options?.upsert) { + doc = { + _id: id, + name: update.$set?.name || "", + checksum: update.$set?.checksum || "", + status: update.$set?.status || "PENDING", + startedAt: update.$set?.startedAt || new Date().toISOString(), + executedSteps: update.$set?.executedSteps || 0, + totalSteps: update.$set?.totalSteps || 0, + fencingToken: update.$set?.fencingToken || 1, + }; + summaries.set(id, doc); + } + if (doc && update.$set) { + Object.assign(doc, update.$set); + } + return { matchedCount: doc ? 1 : 0, modifiedCount: 1 }; + }, + + async replaceOne(filter: any, doc: any, _options?: any) { + summaries.set(filter._id, { ...doc }); + return { matchedCount: 1, modifiedCount: 1 }; + }, + + async countDocuments(filter: any = {}) { + let list = Array.from(summaries.values()); + if (filter.status?.$ne) { + list = list.filter((s) => s.status !== filter.status.$ne); + } + return list.length; + }, + }; + } + + if (name === EVENTS_COLLECTION) { + return { + async insertOne(doc: any) { + events.push({ ...doc }); + return { insertedId: doc._id }; + }, + + find(filter: any = {}) { + let list = events.filter((e) => { + if (filter.migrationId && e.migrationId !== filter.migrationId) return false; + return true; + }); + return { + sort(sortObj: any) { + if (sortObj.attemptNumber !== undefined) { + list.sort((a, b) => { + if (a.attemptNumber !== b.attemptNumber) { + return sortObj.attemptNumber === 1 + ? a.attemptNumber - b.attemptNumber + : b.attemptNumber - a.attemptNumber; + } + if (sortObj.sequenceNumber !== undefined) { + return sortObj.sequenceNumber === 1 + ? (a.sequenceNumber || 0) - (b.sequenceNumber || 0) + : (b.sequenceNumber || 0) - (a.sequenceNumber || 0); + } + return 0; + }); + } else if (sortObj.timestamp) { + list.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + } + return { + limit(n: number) { + list = list.slice(0, n); + return { + async toArray() { + return list.map((x) => ({ ...x })); + }, + }; + }, + async toArray() { + return list.map((x) => ({ ...x })); + }, + }; + }, + async toArray() { + return list.map((x) => ({ ...x })); + }, + }; + }, + }; + } + + // User collection mock + return { + async countDocuments(_filter: any = {}) { + const data = collectionsData.get(name); + return data ? data.length : 0; + }, + + async drop() { + if (!collectionsData.has(name) && !collectionsOptions.has(name)) { + const err: any = new Error(`Collection ${name} not found`); + err.code = 26; + throw err; + } + collectionsData.delete(name); + collectionsOptions.delete(name); + collectionsIndexes.delete(name); + return true; + }, + + async createIndex(key: any, options?: any) { + const idxs = getOrCreateCollIndexes(name); + const indexName = + options?.name || + Object.entries(key) + .map(([k, v]) => `${k}_${v}`) + .join("_"); + idxs.push({ + name: indexName, + key, + ...options, + }); + return indexName; + }, + + async dropIndex(indexName: string) { + const idxs = getOrCreateCollIndexes(name); + const idx = idxs.findIndex((i) => i.name === indexName); + if (idx === -1) { + const err: any = new Error(`Index ${indexName} not found`); + err.code = 27; + throw err; + } + idxs.splice(idx, 1); + return { ok: 1 }; + }, + + async indexes() { + const idxs = getOrCreateCollIndexes(name); + return idxs.map((i) => ({ ...i })); + }, + + async updateMany(filter: any, update: any, _options?: any) { + const data = getOrCreateCollData(name); + let matched = 0; + for (const item of data) { + matched++; + if (update.$set) Object.assign(item, update.$set); + if (update.$unset) { + for (const k of Object.keys(update.$unset)) { + delete item[k]; + } + } + } + return { matchedCount: matched, modifiedCount: matched }; + }, + + async insertMany(docs: any[]) { + const data = getOrCreateCollData(name); + data.push(...docs); + return { insertedCount: docs.length }; + }, + }; + }, + }; + + return { + db: db as Db, + locks, + summaries, + events, + collectionsData, + collectionsIndexes, + collectionsOptions, + }; +} + +describe("mongoMigrationEngine Unit Tests (M6 Spec §4-§8)", () => { + let tempDir: string; + let mongoDir: string; + let mockEngine: ReturnType; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "relwave_engine_test_")); + mongoDir = path.join(tempDir, "migrations", "mongo"); + fs.mkdirSync(mongoDir, { recursive: true }); + mockEngine = createMockEngineDb(); + }); + + afterEach(() => { + try { + fs.rmSync(tempDir, { recursive: true, force: true }); + } catch { + // ignore + } + }); + + // ========================================================================= + // 1. Manifest Validation & Security Sandbox + // ========================================================================= + describe("Manifest Validation & AST Security", () => { + test("valid manifest with DDL and updateMany with down step passes validation", () => { + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "setup_users", + up: [ + { type: "createCollection", collection: "users" }, + { + type: "createIndex", + collection: "users", + key: { email: 1 }, + options: { unique: true }, + }, + { + type: "updateMany", + collection: "users", + filter: { active: { $exists: false } }, + update: { $set: { active: true } }, + }, + ], + down: [ + { + type: "updateMany", + collection: "users", + filter: { active: true }, + update: { $unset: { active: "" } }, + }, + ], + }; + + const result = validateManifest(manifest); + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + expect(result.stepsBreakdown.ddlCount).toBe(2); + expect(result.stepsBreakdown.dataTransformCount).toBe(1); + expect(result.stepsBreakdown.reversibleCount).toBe(3); + }); + + test("missing required manifest fields fails validation", () => { + const result = validateManifest({ id: "", up: [] }); + expect(result.valid).toBe(false); + expect(result.errors.length).toBeGreaterThan(0); + }); + + test("checksum calculation normalizes CRLF and LF", () => { + const contentCRLF = '{\r\n "version": "1.0",\r\n "name": "test"\r\n}'; + const contentLF = '{\n "version": "1.0",\n "name": "test"\n}'; + expect(computeManifestChecksum(contentCRLF)).toBe(computeManifestChecksum(contentLF)); + }); + + test("rejects deleteMany step permanently", () => { + const manifest: any = { + version: "1.0", + id: "20261031120000", + name: "delete_test", + up: [{ type: "deleteMany", collection: "users", filter: { old: true } }], + }; + const result = validateManifest(manifest); + expect(result.valid).toBe(false); + expect(result.errors[0]).toContain("deleteMany is permanently excluded"); + + expect(() => assertValidManifest(manifest)).toThrow(MongoApiError); + }); + + test("rejects forbidden expressions: $where, $function, $accumulator in updateMany", () => { + const whereManifest: any = { + version: "1.0", + id: "20261031120000", + name: "where_test", + up: [ + { + type: "updateMany", + collection: "users", + filter: { $where: "this.age > 21" }, + update: { $set: { adult: true } }, + }, + ], + down: [ + { + type: "updateMany", + collection: "users", + filter: { adult: true }, + update: { $unset: { adult: "" } }, + }, + ], + }; + const res = validateManifest(whereManifest); + expect(res.valid).toBe(false); + expect(res.errors.some((e) => e.includes("forbidden") && e.includes("$where"))).toBe(true); + expect(() => assertValidManifest(whereManifest)).toThrow(MongoApiError); + + const funcManifest: any = { + version: "1.0", + id: "20261031120000", + name: "func_test", + up: [ + { + type: "updateMany", + collection: "users", + filter: { age: { $gt: 20 } }, + update: { $set: { fn: { $function: { body: "function() { return 1; }" } } } }, + }, + ], + down: [ + { + type: "updateMany", + collection: "users", + filter: {}, + update: { $unset: { fn: "" } }, + }, + ], + }; + expect(validateManifest(funcManifest).valid).toBe(false); + expect(() => assertValidManifest(funcManifest)).toThrow(MongoApiError); + }); + + test("dropCollection requires allowIrreversible: true", () => { + const badDrop: any = { + version: "1.0", + id: "20261031120000", + name: "drop_test", + up: [{ type: "dropCollection", collection: "old_users" }], + }; + const resBad = validateManifest(badDrop); + expect(resBad.valid).toBe(false); + expect(resBad.errors[0]).toContain("allowIrreversible: true"); + + const goodDrop: any = { + ...badDrop, + allowIrreversible: true, + }; + expect(validateManifest(goodDrop).valid).toBe(true); + }); + + test("dropIndex rejects dropping immutable _id_ index", () => { + const dropIdManifest: any = { + version: "1.0", + id: "20261031120000", + name: "drop_id_test", + up: [{ type: "dropIndex", collection: "users", indexName: "_id_" }], + }; + const res = validateManifest(dropIdManifest); + expect(res.valid).toBe(false); + expect(res.errors[0]).toContain("cannot drop default _id_ index"); + }); + + test("updateMany requires an author-provided down step", () => { + const noDownManifest: any = { + version: "1.0", + id: "20261031120000", + name: "no_down_test", + up: [ + { + type: "updateMany", + collection: "users", + filter: { active: false }, + update: { $set: { active: true } }, + }, + ], + }; + const res = validateManifest(noDownManifest); + expect(res.valid).toBe(false); + expect(res.errors[0]).toContain("requires an author-provided down step"); + }); + }); + + // ========================================================================= + // 2. Filesystem Security: Path Traversal, Symlinks & Max Size + // ========================================================================= + describe("Filesystem Security & Discovery", () => { + test("rejects path traversal containing '..' or null bytes", () => { + expect(() => validateSafeFilePath(path.join(mongoDir, "../secret.json"), mongoDir)).toThrow( + MongoApiError, + ); + + expect(() => validateSafeFilePath(path.join(mongoDir, "manifest\0.json"), mongoDir)).toThrow( + MongoApiError, + ); + }); + + test("rejects files outside migration directory", () => { + const outsideFile = path.join(tempDir, "outside.mql.json"); + fs.writeFileSync(outsideFile, "{}"); + + expect(() => validateSafeFilePath(outsideFile, mongoDir)).toThrow(MongoApiError); + }); + + test("rejects oversized manifests (>1MB)", async () => { + const bigFile = path.join(mongoDir, "20261031120000_huge.mql.json"); + // Create a file > 1MB + const buffer = Buffer.alloc(MAX_MANIFEST_SIZE_BYTES + 10, "x"); + fs.writeFileSync(bigFile, buffer); + + await expect(applyMigration(mockEngine.db, tempDir, "20261031120000")).rejects.toThrow( + MongoApiError, + ); + + try { + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + } catch (err: any) { + expect(err.code).toBe("MANIFEST_SIZE_EXCEEDED"); + } + }); + + test("listMigrations discovers valid files and correctly reports pending", async () => { + const file1 = path.join(mongoDir, "20261031120000_init_users.mql.json"); + const manifest1: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "init_users", + up: [{ type: "createCollection", collection: "users" }], + }; + fs.writeFileSync(file1, JSON.stringify(manifest1, null, 2)); + + const listRes = await listMigrations(mockEngine.db, tempDir); + expect(listRes.pendingMigrations).toHaveLength(1); + expect(listRes.pendingMigrations[0].id).toBe("20261031120000"); + expect(listRes.pendingMigrations[0].name).toBe("init_users"); + expect(listRes.appliedMigrations).toHaveLength(0); + expect(listRes.isLocked).toBe(false); + }); + }); + + // ========================================================================= + // 3. Multi-Step Execution, Checkpoints & Journaling + // ========================================================================= + describe("Multi-Step Execution & Checkpoints", () => { + test("applies multi-step migration and records per-step events and summary", async () => { + const file = path.join(mongoDir, "20261031120000_setup.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "setup_app", + up: [ + { type: "createCollection", collection: "users" }, + { + type: "createIndex", + collection: "users", + key: { email: 1 }, + options: { unique: true }, + }, + { + type: "setValidator", + collection: "users", + validator: { $jsonSchema: { bsonType: "object", required: ["email"] } }, + }, + ], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + const res = await applyMigration(mockEngine.db, tempDir, "20261031120000"); + expect(res.success).toBe(true); + expect(res.appliedCount).toBe(1); + expect(res.results[0].status).toBe("APPLIED"); + expect(res.results[0].executedSteps).toBe(3); + + // Verify materialized summary in _relwave_migrations + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary).toBeDefined(); + expect(summary?.status).toBe("APPLIED"); + expect(summary?.executedSteps).toBe(3); + expect(summary?.fencingToken).toBe(1); + + // Verify events in _relwave_migration_events + const eventTypes = mockEngine.events.map((e) => e.eventType); + expect(eventTypes).toContain("VALIDATION_SUCCESS"); + expect(eventTypes).toContain("LOCK_ACQUIRED"); + expect(eventTypes).toContain("MIGRATION_START"); + expect(eventTypes).toContain("STEP_START"); + expect(eventTypes).toContain("STEP_COMPLETE"); + expect(eventTypes).toContain("MIGRATION_APPLIED"); + expect(eventTypes).toContain("LOCK_RELEASED"); + + // Verify lock was safely released + expect(mockEngine.locks.get(LOCK_DOC_ID)?.ownerId).toBeNull(); + }); + + test("cannot re-apply an already applied migration", async () => { + const file = path.join(mongoDir, "20261031120000_setup.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "setup", + up: [{ type: "createCollection", collection: "items" }], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + + await expect(applyMigration(mockEngine.db, tempDir, "20261031120000")).rejects.toThrow( + MongoApiError, + ); + + try { + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + } catch (err: any) { + expect(err.code).toBe("MIGRATION_ALREADY_APPLIED"); + } + }); + + test("rejects if manifest content was tampered with after applying (checksum mismatch)", async () => { + const file = path.join(mongoDir, "20261031120000_setup.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "setup", + up: [{ type: "createCollection", collection: "items" }], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + + // Tamper with file + manifest.name = "tampered_name"; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + await expect(applyMigration(mockEngine.db, tempDir, "20261031120000")).rejects.toThrow( + MongoApiError, + ); + + try { + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + } catch (err: any) { + expect(err.code).toBe("CHECKSUM_MISMATCH"); + } + }); + }); + + // ========================================================================= + // 4. Failure Unwind & Reverse Compensation State Machine + // ========================================================================= + describe("Failure Unwind & Reverse Compensation", () => { + test("intermediate step failure triggers reverse compensation -> ROLLED_BACK", async () => { + const file = path.join(mongoDir, "20261031120000_failing.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "partial_fail", + up: [ + { type: "createCollection", collection: "accounts" }, + { + type: "createIndex", + collection: "accounts", + key: { username: 1 }, + options: { name: "idx_user" }, + }, + // Step 3 will fail because collection "nonexistent" is missing for dropIndex + { type: "dropIndex", collection: "nonexistent_collection", indexName: "nonexistent_idx" }, + ], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + await expect(applyMigration(mockEngine.db, tempDir, "20261031120000")).rejects.toThrow(); + + // Check summary + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary?.status).toBe("ROLLED_BACK"); + + // Check events recorded reverse compensation + const eventTypes = mockEngine.events.map((e) => e.eventType); + expect(eventTypes).toContain("STEP_FAILED"); + expect(eventTypes).toContain("COMPENSATION_START"); + expect(eventTypes).toContain("COMPENSATION_COMPLETE"); + expect(eventTypes).toContain("MIGRATION_ROLLED_BACK"); + + // Verify that step 1 (createCollection) was compensated by dropping collection + expect(mockEngine.collectionsData.has("accounts")).toBe(false); + // Verify lock was released + expect(mockEngine.locks.get(LOCK_DOC_ID)?.ownerId).toBeNull(); + }); + + test("count-guarded createCollection compensation aborts when external data present -> FAILED_MANUAL_INTERVENTION_REQUIRED", async () => { + const file = path.join(mongoDir, "20261031120000_countguard.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "count_guard_test", + up: [ + { type: "createCollection", collection: "wallets" }, + // Step 2 fails deliberately + { type: "dropIndex", collection: "missing_coll", indexName: "missing_idx" }, + ], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + // Hook into collection creation to simulate external data insertion before compensation + const originalCreateCollection = mockEngine.db.createCollection; + mockEngine.db.createCollection = async (name: string, options?: any) => { + const res = await originalCreateCollection.call(mockEngine.db, name, options); + if (name === "wallets") { + // Simulate concurrent insertion of an external document + mockEngine.collectionsData.get("wallets")!.push({ _id: "doc1", balance: 100 }); + } + return res; + }; + + await expect(applyMigration(mockEngine.db, tempDir, "20261031120000")).rejects.toThrow(); + + // Summary must transition to FAILED_MANUAL_INTERVENTION_REQUIRED + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary?.status).toBe("FAILED_MANUAL_INTERVENTION_REQUIRED"); + + const eventTypes = mockEngine.events.map((e) => e.eventType); + expect(eventTypes).toContain("COMPENSATION_FAILED"); + expect(eventTypes).toContain("MANUAL_INTERVENTION_REQUIRED"); + + // Wallets collection was NOT dropped (data preserved!) + expect(mockEngine.collectionsData.has("wallets")).toBe(true); + expect(mockEngine.collectionsData.get("wallets")!).toHaveLength(1); + }); + + test("pre-step fencing loss marks STEP_UNCERTAIN and halts execution immediately", async () => { + const file = path.join(mongoDir, "20261031120000_fencing.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "fencing_loss_test", + up: [ + { type: "createCollection", collection: "products" }, + { type: "createIndex", collection: "products", key: { sku: 1 } }, + ], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + // Hook step 1 to simulate lock stolen by another client + const origCreateCollection = mockEngine.db.createCollection; + mockEngine.db.createCollection = async (name: string, options?: any) => { + const res = await origCreateCollection.call(mockEngine.db, name, options); + // Another client steals the lock and increments fencingToken + const lock = mockEngine.locks.get(LOCK_DOC_ID)!; + lock.ownerId = "client-B"; + lock.fencingToken = lock.fencingToken + 1; + return res; + }; + + let thrownError: any; + try { + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + } catch (err: any) { + thrownError = err; + } + expect(thrownError).toBeInstanceOf(MongoApiError); + expect(thrownError.code).toBe("LOCK_LOST_FENCING_VIOLATION"); + + const eventTypes = mockEngine.events.map((e) => e.eventType); + expect(eventTypes).toContain("STEP_UNCERTAIN"); + expect(eventTypes).toContain("MANUAL_INTERVENTION_REQUIRED"); + + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary?.status).toBe("CRASHED_UNCERTAIN"); + }); + }); + + // ========================================================================= + // 5. Rollback Method & Read-Only Checks + // ========================================================================= + describe("Rollback & Read-Only Enforcement", () => { + test("rollback cleans up applied steps in reverse order", async () => { + const file = path.join(mongoDir, "20261031120000_rollback.mql.json"); + const manifest: MongoMigrationManifest = { + version: "1.0", + id: "20261031120000", + name: "for_rollback", + up: [ + { type: "createCollection", collection: "books" }, + { + type: "createIndex", + collection: "books", + key: { isbn: 1 }, + options: { name: "idx_isbn" }, + }, + ], + }; + fs.writeFileSync(file, JSON.stringify(manifest, null, 2)); + + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + expect(mockEngine.collectionsData.has("books")).toBe(true); + + const rollRes = await rollbackMigration(mockEngine.db, "20261031120000", tempDir); + expect(rollRes.success).toBe(true); + expect(rollRes.status).toBe("ROLLED_BACK"); + expect(rollRes.stepsCompensated).toBe(2); + + expect(mockEngine.collectionsData.has("books")).toBe(false); + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary?.status).toBe("ROLLED_BACK"); + }); + + test("rejects apply and rollback on read-only database connections", async () => { + const file = path.join(mongoDir, "20261031120000_readonly.mql.json"); + fs.writeFileSync( + file, + JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "ro_test", + up: [{ type: "createCollection", collection: "ro_coll" }], + }), + ); + + await expect( + applyMigration(mockEngine.db, tempDir, "20261031120000", { readOnly: true }), + ).rejects.toThrow(MongoApiError); + + try { + await applyMigration(mockEngine.db, tempDir, "20261031120000", { readOnly: true }); + } catch (err: any) { + expect(err.code).toBe("MONGO_READ_ONLY"); + } + + await expect( + rollbackMigration(mockEngine.db, "20261031120000", tempDir, { readOnly: true }), + ).rejects.toThrow(MongoApiError); + + try { + await rollbackMigration(mockEngine.db, "20261031120000", tempDir, { readOnly: true }); + } catch (err: any) { + expect(err.code).toBe("MONGO_READ_ONLY"); + } + }); + }); + + // ========================================================================= + // 6. rebuildMigrationSummary Projection Rebuild + // ========================================================================= + describe("rebuildMigrationSummary", () => { + test("rebuilds summary from event stream correctly after simulated crash", async () => { + const migrationId = "20261031120000"; + // Simulate events recorded in _relwave_migration_events + mockEngine.events.push( + { + _id: `${migrationId}_meta_MIGRATION_START_1`, + eventId: "e1", + migrationId, + attemptId: "att1", + attemptNumber: 1, + sequenceNumber: 1, + eventType: "MIGRATION_START", + timestamp: "2026-10-31T12:00:00.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + { + _id: `${migrationId}_0_STEP_START_1`, + eventId: "e2", + migrationId, + attemptId: "att1", + attemptNumber: 1, + sequenceNumber: 2, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_START", + timestamp: "2026-10-31T12:00:01.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + { + _id: `${migrationId}_0_STEP_COMPLETE_1`, + eventId: "e3", + migrationId, + attemptId: "att1", + attemptNumber: 1, + sequenceNumber: 3, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_COMPLETE", + timestamp: "2026-10-31T12:00:02.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + metadata: { durationMs: 25 }, + }, + { + _id: `${migrationId}_meta_MIGRATION_APPLIED_1`, + eventId: "e4", + migrationId, + attemptId: "att1", + attemptNumber: 1, + sequenceNumber: 4, + eventType: "MIGRATION_APPLIED", + timestamp: "2026-10-31T12:00:03.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + ); + + // Rebuild summary + const rebuilt = await rebuildMigrationSummary(mockEngine.db, migrationId); + expect(rebuilt._id).toBe(migrationId); + expect(rebuilt.status).toBe("APPLIED"); + expect(rebuilt.executedSteps).toBe(1); + expect(rebuilt.durationMs).toBe(25); + expect(rebuilt.fencingToken).toBe(1); + expect(rebuilt.appliedBy).toBe("worker-1"); + + const saved = mockEngine.summaries.get(migrationId); + expect(saved?.status).toBe("APPLIED"); + }); + + test("detects dangling STEP_START as CRASHED_UNCERTAIN", async () => { + const migrationId = "20261031999999"; + mockEngine.events.push( + { + _id: `${migrationId}_meta_MIGRATION_START_1`, + eventId: "e1", + migrationId, + attemptId: "att1", + attemptNumber: 1, + sequenceNumber: 1, + eventType: "MIGRATION_START", + timestamp: "2026-10-31T12:00:00.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + { + _id: `${migrationId}_0_STEP_START_1`, + eventId: "e2", + migrationId, + attemptId: "att1", + attemptNumber: 1, + sequenceNumber: 2, + stepIndex: 0, + stepType: "createIndex", + eventType: "STEP_START", + timestamp: "2026-10-31T12:00:01.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + ); + + const rebuilt = await rebuildMigrationSummary(mockEngine.db, migrationId); + expect(rebuilt.status).toBe("CRASHED_UNCERTAIN"); + }); + + test("handles multiple attempts: failed attempt followed by successful attempt rebuilds to APPLIED", async () => { + const migrationId = "20261031888888"; + // Attempt 1: Started and rolled back + mockEngine.events.push( + { + _id: `${migrationId}_a1_s1`, + eventId: "att1-e1", + migrationId, + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 1, + eventType: "MIGRATION_START", + timestamp: "2026-10-31T12:00:00.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + { + _id: `${migrationId}_a1_s2`, + eventId: "att1-e2", + migrationId, + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 2, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_START", + timestamp: "2026-10-31T12:00:01.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + { + _id: `${migrationId}_a1_s3`, + eventId: "att1-e3", + migrationId, + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 3, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_FAILED", + timestamp: "2026-10-31T12:00:02.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + metadata: { error: { message: "Simulated failure in attempt 1" } }, + }, + { + _id: `${migrationId}_a1_s4`, + eventId: "att1-e4", + migrationId, + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 4, + eventType: "MIGRATION_ROLLED_BACK", + timestamp: "2026-10-31T12:00:03.000Z", + lockOwnerId: "worker-1", + fencingToken: 1, + }, + ); + + // Attempt 2: Succeeds + mockEngine.events.push( + { + _id: `${migrationId}_a2_s1`, + eventId: "att2-e1", + migrationId, + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 1, + eventType: "MIGRATION_START", + timestamp: "2026-10-31T12:01:00.000Z", + lockOwnerId: "worker-2", + fencingToken: 2, + }, + { + _id: `${migrationId}_a2_s2`, + eventId: "att2-e2", + migrationId, + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 2, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_START", + timestamp: "2026-10-31T12:01:01.000Z", + lockOwnerId: "worker-2", + fencingToken: 2, + }, + { + _id: `${migrationId}_a2_s3`, + eventId: "att2-e3", + migrationId, + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 3, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_COMPLETE", + timestamp: "2026-10-31T12:01:02.000Z", + lockOwnerId: "worker-2", + fencingToken: 2, + metadata: { durationMs: 15 }, + }, + { + _id: `${migrationId}_a2_s4`, + eventId: "att2-e4", + migrationId, + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 4, + eventType: "MIGRATION_APPLIED", + timestamp: "2026-10-31T12:01:03.000Z", + lockOwnerId: "worker-2", + fencingToken: 2, + }, + ); + + const rebuilt = await rebuildMigrationSummary(mockEngine.db, migrationId, { + version: "1.0", + id: migrationId, + name: "multi_attempt_test", + up: [{ type: "createCollection", collection: "multi_coll" }], + }); + + expect(rebuilt.status).toBe("APPLIED"); + expect(rebuilt.executedSteps).toBe(1); + expect(rebuilt.attemptNumber).toBe(2); + expect(rebuilt.attemptId).toBe("att-2"); + expect(rebuilt.appliedBy).toBe("worker-2"); + }); + }); + + // ========================================================================= + // 7. getMigrationStatus + // ========================================================================= + describe("getMigrationStatus", () => { + test("returns summary, events, and lock status", async () => { + const file = path.join(mongoDir, "20261031120000_status.mql.json"); + fs.writeFileSync( + file, + JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "status_test", + up: [{ type: "createCollection", collection: "status_coll" }], + }), + ); + + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + + const statusRes = await getMigrationStatus(mockEngine.db, "20261031120000"); + expect(statusRes.migration?.status).toBe("APPLIED"); + expect(statusRes.events?.length).toBeGreaterThan(0); + expect(statusRes.lockStatus.isLocked).toBe(false); + expect(statusRes.lastAppliedVersion).toBe("20261031120000"); + }); + }); + + // ========================================================================= + // 8. Safety Audit Hardening & Compensation (Phase 3) + // ========================================================================= + describe("Safety Audit Hardening & Compensation (Phase 3)", () => { + test("event journal assigns unique deterministic _id and monotonic sequenceNumber", async () => { + const file = path.join(mongoDir, "20261031120000_sequential_events.mql.json"); + fs.writeFileSync( + file, + JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "sequential_events", + up: [ + { type: "createCollection", collection: "seq_coll1" }, + { type: "createCollection", collection: "seq_coll2" }, + ], + }), + ); + + const res = await applyMigration(mockEngine.db, tempDir, "20261031120000"); + expect(res.success).toBe(true); + + const events = mockEngine.events.filter((e) => e.migrationId === "20261031120000"); + expect(events.length).toBeGreaterThanOrEqual(4); + + const seenIds = new Set(); + let expectedSeq = 1; + for (const ev of events) { + expect(ev.sequenceNumber).toBe(expectedSeq); + expect(ev._id).toBe(`20261031120000_a1_s${expectedSeq}`); + expect(seenIds.has(ev._id)).toBe(false); + seenIds.add(ev._id); + expectedSeq++; + } + }); + + test("dry-run does not acquire lock or write events", async () => { + const file = path.join(mongoDir, "20261031120000_dry_run.mql.json"); + fs.writeFileSync( + file, + JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "dry_run_test", + up: [{ type: "createCollection", collection: "dry_coll" }], + }), + ); + + const result = await applyMigration(mockEngine.db, tempDir, "20261031120000", { + dryRun: true, + }); + expect(result.success).toBe(true); + expect(result.results[0].status).toBe("PENDING"); + + // Lock was not acquired + expect(mockEngine.locks.size).toBe(0); + + // No events written + expect(mockEngine.events.length).toBe(0); + + // No collection created + expect(mockEngine.collectionsData.has("dry_coll")).toBe(false); + }); + + test("compensation of createCollection halts when collection is non-empty", async () => { + const file = path.join(mongoDir, "20261031120000_guard_coll.mql.json"); + fs.writeFileSync( + file, + JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "guard_coll_test", + up: [ + { type: "createCollection", collection: "guard_coll" }, + { type: "createCollection", collection: "failing_coll" }, + ], + }), + ); + + const origCreateCollection = mockEngine.db.createCollection; + mockEngine.db.createCollection = async function (name: string, options?: any) { + if (name === "failing_coll") { + // Simulate user documents written to guard_coll before second step fails + mockEngine.collectionsData.get("guard_coll")!.push({ _id: 1, name: "external doc" }); + throw new Error("Simulated failure on second step"); + } + return origCreateCollection.call(this, name, options); + }; + + await expect(applyMigration(mockEngine.db, tempDir, "20261031120000")).rejects.toThrow( + "Simulated failure on second step", + ); + + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary?.status).toBe("FAILED_MANUAL_INTERVENTION_REQUIRED"); + expect((summary?.error as any)?.message).toContain( + "contains 1 documents added after creation", + ); + + // guard_coll must NOT be dropped + expect(mockEngine.collectionsData.has("guard_coll")).toBe(true); + expect(mockEngine.collectionsData.get("guard_coll")?.length).toBe(1); + }); + + test("compensation of dropIndex throws when pre-state is missing and down block is absent", async () => { + await expect( + compensateStep( + mockEngine.db, + { type: "dropIndex", collection: "orders", indexName: "idx_custom" }, + null, // missing preState + undefined, // no down block + ), + ).rejects.toThrow(MongoApiError); + + try { + await compensateStep( + mockEngine.db, + { type: "dropIndex", collection: "orders", indexName: "idx_custom" }, + null, + undefined, + ); + } catch (err: any) { + expect(err.code).toBe("COMPENSATION_FAILED"); + expect(err.message).toContain( + "pre-state missing and no matching createIndex found in down steps", + ); + } + }); + + test("in-flight heartbeat loss causes pre-step fencing failure", async () => { + const file = path.join(mongoDir, "20261031120000_fencing_loss.mql.json"); + fs.writeFileSync( + file, + JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "fencing_loss_test", + up: [ + { type: "createCollection", collection: "fence_coll1" }, + { type: "createCollection", collection: "fence_coll2" }, + ], + }), + ); + + let step = 0; + const origCreateCollection = mockEngine.db.createCollection; + mockEngine.db.createCollection = async function (name: string, options?: any) { + step++; + const res = await origCreateCollection.call(this, name, options); + if (step === 1) { + // Lease stolen by another worker + const lockDoc = mockEngine.locks.get(LOCK_DOC_ID); + if (lockDoc) { + lockDoc.ownerId = "stealing-worker"; + lockDoc.fencingToken = 99; + } + } + return res; + }; + + try { + await applyMigration(mockEngine.db, tempDir, "20261031120000"); + expect(true).toBe(false); // should not reach + } catch (err: any) { + expect(err).toBeInstanceOf(MongoApiError); + expect(err.code).toBe("LOCK_LOST_FENCING_VIOLATION"); + } + + const summary = mockEngine.summaries.get("20261031120000"); + expect(summary?.status).toBe("CRASHED_UNCERTAIN"); + + const events = mockEngine.events.filter((e) => e.migrationId === "20261031120000"); + expect(events.some((e) => e.eventType === "STEP_UNCERTAIN")).toBe(true); + expect(events.some((e) => e.eventType === "MANUAL_INTERVENTION_REQUIRED")).toBe(true); + }); + }); +}); diff --git a/bridge/__tests__/mongoMigrationLock.test.ts b/bridge/__tests__/mongoMigrationLock.test.ts new file mode 100644 index 00000000..d963db8d --- /dev/null +++ b/bridge/__tests__/mongoMigrationLock.test.ts @@ -0,0 +1,395 @@ +import { describe, expect, test, beforeEach } from "@jest/globals"; +import { Db } from "mongodb"; +import { + acquireLock, + renewLockHeartbeat, + releaseLock, + checkLockStatus, + verifyLock, + LOCK_COLLECTION, + LOCK_DOC_ID, +} from "../src/services/mongoMigrationLock"; +import { MongoApiError } from "../src/connectors/mongo"; +import { MongoMigrationLockDoc } from "../src/types/mongoMigration"; + +/** + * Creates an in-memory mock MongoDB Db for lock testing. + */ +function createMockLockDb(): { db: Db; store: Map } { + const store = new Map(); + + const collectionMock = { + async findOneAndUpdate(filter: any, update: any, options?: any) { + const existing = store.get(LOCK_DOC_ID); + const now = new Date(); + + if (!existing) { + // Document does not exist: upsert + const fencingToken = update.$inc?.fencingToken || 1; + const newDoc: MongoMigrationLockDoc = { + _id: LOCK_DOC_ID, + ownerId: update.$set?.ownerId || null, + migrationId: update.$set?.migrationId || null, + acquiredAt: update.$set?.acquiredAt || now, + heartbeatAt: update.$set?.heartbeatAt || now, + expiresAt: update.$set?.expiresAt || new Date(now.getTime() + 60000), + fencingToken, + }; + store.set(LOCK_DOC_ID, newDoc); + return newDoc; + } + + // Check filter condition + const ownerMatch = filter.$or?.some((cond: any) => { + if ("ownerId" in cond && cond.ownerId === existing.ownerId) return true; + if ("ownerId" in cond && cond.ownerId === null && existing.ownerId === null) return true; + if ( + cond.expiresAt?.$lt && + existing.expiresAt && + new Date(existing.expiresAt).getTime() < new Date(cond.expiresAt.$lt).getTime() + ) { + return true; + } + return false; + }); + + if (ownerMatch) { + const fencingToken = (existing.fencingToken || 0) + (update.$inc?.fencingToken || 1); + const updatedDoc: MongoMigrationLockDoc = { + ...existing, + ownerId: update.$set?.ownerId !== undefined ? update.$set.ownerId : existing.ownerId, + migrationId: + update.$set?.migrationId !== undefined ? update.$set.migrationId : existing.migrationId, + acquiredAt: update.$set?.acquiredAt || now, + heartbeatAt: update.$set?.heartbeatAt || now, + expiresAt: update.$set?.expiresAt || existing.expiresAt, + fencingToken, + }; + store.set(LOCK_DOC_ID, updatedDoc); + return updatedDoc; + } + + // If document exists with _id: "migration_lock", but filter doesn't match and upsert: true, + // MongoDB throws E11000 duplicate key error. + if (options?.upsert) { + const err: any = new Error("E11000 duplicate key error collection"); + err.code = 11000; + throw err; + } + + return null; + }, + + async findOne(filter: any) { + if (filter._id === LOCK_DOC_ID) { + const doc = store.get(LOCK_DOC_ID); + if (!doc) return null; + + if (filter.ownerId && doc.ownerId !== filter.ownerId) return null; + if (filter.fencingToken !== undefined && doc.fencingToken !== filter.fencingToken) + return null; + if (filter.expiresAt?.$gt) { + if ( + !doc.expiresAt || + new Date(doc.expiresAt).getTime() <= new Date(filter.expiresAt.$gt).getTime() + ) { + return null; + } + } + return { ...doc }; + } + return null; + }, + + async updateOne(filter: any, update: any) { + const doc = store.get(LOCK_DOC_ID); + if (!doc) return { matchedCount: 0, modifiedCount: 0 }; + + if (filter.ownerId && doc.ownerId !== filter.ownerId) + return { matchedCount: 0, modifiedCount: 0 }; + if (filter.fencingToken !== undefined && doc.fencingToken !== filter.fencingToken) { + return { matchedCount: 0, modifiedCount: 0 }; + } + if (filter.expiresAt?.$gt) { + if ( + !doc.expiresAt || + new Date(doc.expiresAt).getTime() <= new Date(filter.expiresAt.$gt).getTime() + ) { + return { matchedCount: 0, modifiedCount: 0 }; + } + } + + if (update.$set) { + Object.assign(doc, update.$set); + } + return { matchedCount: 1, modifiedCount: 1 }; + }, + + async deleteOne(filter: any) { + const doc = store.get(LOCK_DOC_ID); + if (!doc) return { deletedCount: 0 }; + + if (filter.ownerId && doc.ownerId !== filter.ownerId) return { deletedCount: 0 }; + if (filter.fencingToken !== undefined && doc.fencingToken !== filter.fencingToken) { + return { deletedCount: 0 }; + } + + store.delete(LOCK_DOC_ID); + return { deletedCount: 1 }; + }, + }; + + const dbMock: any = { + collection(name: string) { + if (name === LOCK_COLLECTION) { + return collectionMock; + } + throw new Error(`Collection '${name}' not implemented in mock`); + }, + }; + + return { db: dbMock, store }; +} + +describe("mongoMigrationLock Unit Tests (M6 Spec §6)", () => { + let mockDb: Db; + let store: Map; + + beforeEach(() => { + const mock = createMockLockDb(); + mockDb = mock.db; + store = mock.store; + }); + + test("acquires lock successfully when unheld", async () => { + const res = await acquireLock(mockDb, "client-A", "20261031120000", 60000); + expect(res.success).toBe(true); + expect(res.fencingToken).toBe(1); + expect(res.expiresAt.getTime()).toBeGreaterThan(Date.now()); + + const doc = store.get(LOCK_DOC_ID); + expect(doc?.ownerId).toBe("client-A"); + expect(doc?.migrationId).toBe("20261031120000"); + expect(doc?.fencingToken).toBe(1); + }); + + test("second client is blocked while lock is owned by an active lease", async () => { + await acquireLock(mockDb, "client-A", "20261031120000", 60000); + + await expect(acquireLock(mockDb, "client-B", "20261031120000", 60000, 1)).rejects.toThrow( + MongoApiError, + ); + + try { + await acquireLock(mockDb, "client-B", "20261031120000", 60000, 1); + } catch (err: any) { + expect(err).toBeInstanceOf(MongoApiError); + expect(err.code).toBe("LOCK_ALREADY_HELD"); + expect(err.message).toContain("Migration lock held by client-A"); + } + }); + + test("retries on E11000 duplicate key error and acquires lock when released", async () => { + let callCount = 0; + const originalFindOneAndUpdate = (mockDb.collection(LOCK_COLLECTION) as any).findOneAndUpdate; + + (mockDb.collection(LOCK_COLLECTION) as any).findOneAndUpdate = async function (...args: any[]) { + callCount++; + if (callCount === 1) { + const err: any = new Error("E11000 duplicate key error on concurrent upsert"); + err.code = 11000; + throw err; + } + return originalFindOneAndUpdate.apply(this, args); + }; + + const res = await acquireLock(mockDb, "client-retry", "20261031120000", 60000, 3); + expect(res.success).toBe(true); + expect(res.fencingToken).toBe(1); + expect(callCount).toBe(2); + }); + + test("acquires lock after lease expires and increments fencing token", async () => { + // Client A acquires with expired lease in the past + store.set(LOCK_DOC_ID, { + _id: LOCK_DOC_ID, + ownerId: "stale-client", + fencingToken: 5, + acquiredAt: new Date(Date.now() - 100000), + heartbeatAt: new Date(Date.now() - 90000), + expiresAt: new Date(Date.now() - 30000), // Expired 30 seconds ago + migrationId: "20261031000000", + }); + + const res = await acquireLock(mockDb, "new-client", "20261031120000", 60000); + expect(res.success).toBe(true); + expect(res.fencingToken).toBe(6); // Incremented from 5 to 6 + + const doc = store.get(LOCK_DOC_ID); + expect(doc?.ownerId).toBe("new-client"); + expect(doc?.fencingToken).toBe(6); + }); + + test("renews lock heartbeat for valid owner and fencing token", async () => { + const lock = await acquireLock(mockDb, "client-A", "20261031120000", 60000); + const initialExpires = lock.expiresAt; + + // Small delay to ensure timestamp progression + await new Promise((r) => setTimeout(r, 10)); + + const renewed = await renewLockHeartbeat(mockDb, "client-A", lock.fencingToken, 120000); + expect(renewed).toBe(true); + + const doc = store.get(LOCK_DOC_ID); + expect(new Date(doc!.expiresAt!).getTime()).toBeGreaterThan(initialExpires.getTime()); + + // Wrong owner cannot renew + const wrongOwnerRenewed = await renewLockHeartbeat( + mockDb, + "impostor", + lock.fencingToken, + 60000, + ); + expect(wrongOwnerRenewed).toBe(false); + + // Wrong fencing token cannot renew + const wrongTokenRenewed = await renewLockHeartbeat( + mockDb, + "client-A", + lock.fencingToken + 99, + 60000, + ); + expect(wrongTokenRenewed).toBe(false); + }); + + test("releases lock conditionally matching owner and fencing token", async () => { + const lock = await acquireLock(mockDb, "client-A", "20261031120000", 60000); + + // Impostor cannot release + const impostorRelease = await releaseLock(mockDb, "impostor", lock.fencingToken); + expect(impostorRelease).toBe(false); + expect(store.has(LOCK_DOC_ID)).toBe(true); + + // Wrong fencing token cannot release + const wrongTokenRelease = await releaseLock(mockDb, "client-A", 999); + expect(wrongTokenRelease).toBe(false); + expect(store.has(LOCK_DOC_ID)).toBe(true); + + // True owner releases cleanly (leaves doc, sets ownerId: null, preserves fencingToken) + const released = await releaseLock(mockDb, "client-A", lock.fencingToken); + expect(released).toBe(true); + expect(store.has(LOCK_DOC_ID)).toBe(true); + expect(store.get(LOCK_DOC_ID)?.ownerId).toBeNull(); + expect(store.get(LOCK_DOC_ID)?.fencingToken).toBe(1); + expect(store.get(LOCK_DOC_ID)?.releasedAt).toBeDefined(); + }); + + test("renewLockHeartbeat returns false and does not revive lease when expiresAt <= now", async () => { + const lock = await acquireLock(mockDb, "client-A", "20261031120000", 60000); + + // Expire lease + store.get(LOCK_DOC_ID)!.expiresAt = new Date(Date.now() - 5000); + + const renewed = await renewLockHeartbeat(mockDb, "client-A", lock.fencingToken, 60000); + expect(renewed).toBe(false); + + // Ensure lease was not revived + const doc = store.get(LOCK_DOC_ID); + expect(new Date(doc!.expiresAt!).getTime()).toBeLessThan(Date.now()); + }); + + test("releaseLock leaves document and preserves fencingToken monotonically across multiple sequential acquires", async () => { + // Token 1 + const lock1 = await acquireLock(mockDb, "client-1", "m1", 60000); + expect(lock1.fencingToken).toBe(1); + + const rel1 = await releaseLock(mockDb, "client-1", lock1.fencingToken); + expect(rel1).toBe(true); + expect(store.has(LOCK_DOC_ID)).toBe(true); + expect(store.get(LOCK_DOC_ID)?.ownerId).toBeNull(); + expect(store.get(LOCK_DOC_ID)?.fencingToken).toBe(1); + + // Token 2 + const lock2 = await acquireLock(mockDb, "client-2", "m2", 60000); + expect(lock2.fencingToken).toBe(2); + + const rel2 = await releaseLock(mockDb, "client-2", lock2.fencingToken); + expect(rel2).toBe(true); + expect(store.get(LOCK_DOC_ID)?.ownerId).toBeNull(); + expect(store.get(LOCK_DOC_ID)?.fencingToken).toBe(2); + + // Token 3 + const lock3 = await acquireLock(mockDb, "client-3", "m3", 60000); + expect(lock3.fencingToken).toBe(3); + + const rel3 = await releaseLock(mockDb, "client-3", lock3.fencingToken); + expect(rel3).toBe(true); + expect(store.get(LOCK_DOC_ID)?.ownerId).toBeNull(); + expect(store.get(LOCK_DOC_ID)?.fencingToken).toBe(3); + }); + + test("previous owner cannot release newer owner's lock", async () => { + const lock1 = await acquireLock(mockDb, "client-1", "m1", 100); + + // Expire lease and client-2 acquires lock2 with token 2 + store.get(LOCK_DOC_ID)!.expiresAt = new Date(Date.now() - 1000); + const lock2 = await acquireLock(mockDb, "client-2", "m1", 60000); + expect(lock2.fencingToken).toBe(2); + + // Client 1 cannot release with its old token + const client1Release = await releaseLock(mockDb, "client-1", lock1.fencingToken); + expect(client1Release).toBe(false); + + // Client 1 cannot release even if it guesses lock2's token + const client1ReleaseToken2 = await releaseLock(mockDb, "client-1", lock2.fencingToken); + expect(client1ReleaseToken2).toBe(false); + + // Lock remains held by client-2 + const doc = store.get(LOCK_DOC_ID); + expect(doc?.ownerId).toBe("client-2"); + expect(doc?.fencingToken).toBe(2); + }); + + test("verifies lock status and detects lock loss / stale token", async () => { + const lock = await acquireLock(mockDb, "client-A", "20261031120000", 60000); + + // Active lease returns true + const valid = await verifyLock(mockDb, "client-A", lock.fencingToken); + expect(valid).toBe(true); + + // Wrong owner returns false + const wrongOwner = await verifyLock(mockDb, "other", lock.fencingToken); + expect(wrongOwner).toBe(false); + + // Stale fencing token returns false + const staleToken = await verifyLock(mockDb, "client-A", lock.fencingToken - 1); + expect(staleToken).toBe(false); + + // Expired lease returns false + store.get(LOCK_DOC_ID)!.expiresAt = new Date(Date.now() - 1000); + const expired = await verifyLock(mockDb, "client-A", lock.fencingToken); + expect(expired).toBe(false); + }); + + test("checkLockStatus inspects without mutating document", async () => { + // When no lock document exists + const emptyStatus = await checkLockStatus(mockDb); + expect(emptyStatus.isLocked).toBe(false); + expect(emptyStatus.isExpired).toBe(false); + + // When held by active client + await acquireLock(mockDb, "client-A", "20261031120000", 60000); + const heldStatus = await checkLockStatus(mockDb); + expect(heldStatus.isLocked).toBe(true); + expect(heldStatus.isExpired).toBe(false); + expect(heldStatus.ownerId).toBe("client-A"); + expect(heldStatus.fencingToken).toBe(1); + expect(heldStatus.migrationId).toBe("20261031120000"); + + // When lease is expired + store.get(LOCK_DOC_ID)!.expiresAt = new Date(Date.now() - 5000); + const expiredStatus = await checkLockStatus(mockDb); + expect(expiredStatus.isLocked).toBe(false); + expect(expiredStatus.isExpired).toBe(true); + }); +}); diff --git a/bridge/__tests__/mongoRelationshipEngine.test.ts b/bridge/__tests__/mongoRelationshipEngine.test.ts new file mode 100644 index 00000000..2b94dad4 --- /dev/null +++ b/bridge/__tests__/mongoRelationshipEngine.test.ts @@ -0,0 +1,631 @@ +import { describe, expect, test, beforeEach, jest } from "@jest/globals"; +import { ObjectId } from "mongodb"; +import { MongoRelationshipEngine } from "../src/services/mongoRelationshipEngine"; +import { MongoRelationshipsFile } from "../src/types/mongo"; + +describe("MongoRelationshipEngine Unit & Heuristic Tests (M7 Phase 1 Remediation)", () => { + let engine: MongoRelationshipEngine; + let mockDbService: any; + let mockProjectStore: any; + let storedRelationships: MongoRelationshipsFile; + + beforeEach(() => { + storedRelationships = { + version: 1, + databaseId: "test-db", + inferred: [], + confirmed: [], + rejectedIds: [], + manual: [], + insufficientEvidence: [], + updatedAt: new Date().toISOString(), + }; + + mockProjectStore = { + getMongoRelationships: jest.fn().mockImplementation(async () => storedRelationships), + saveMongoRelationships: jest.fn().mockImplementation(async (file: any) => { + storedRelationships = file; + }), + }; + + mockDbService = { + getDatabaseConnection: jest.fn(), + }; + + engine = new MongoRelationshipEngine(mockDbService, mockProjectStore); + }); + + describe("Candidate Name Derivation & Normalization", () => { + test("derives candidate collection names and normalizes ObjectIds", () => { + expect(engine.deriveCandidateCollectionNames("userId")).toEqual( + expect.arrayContaining(["user", "users"]), + ); + expect(engine.deriveCandidateCollectionNames("items.productId")).toEqual( + expect.arrayContaining(["product", "products"]), + ); + const oid = new ObjectId("507f1f77bcf86cd799439011"); + expect(engine.normalizeIdValue(oid)).toBe("507f1f77bcf86cd799439011"); + expect(engine.normalizeIdValue({ $oid: "507f1f77bcf86cd799439011" })).toBe( + "507f1f77bcf86cd799439011", + ); + expect(engine.normalizeIdValue("507f1f77bcf86cd799439011")).toBe("507f1f77bcf86cd799439011"); + }); + }); + + describe("Embedded Subdocument Arrays & Dotted Paths (LOW-02)", () => { + test("extracts embedded subdocument array fields and filters out embedded _ids", () => { + const p1 = new ObjectId(); + const docs = [ + { + _id: new ObjectId(), + items: [ + { + _id: new ObjectId(), // Embedded _id MUST be ignored + productId: p1, + title: "Widget", + }, + ], + }, + ]; + + const candidates = engine.extractCandidateFields(docs); + const paths = candidates.map((c) => c.sourcePath); + + expect(paths).toContain("items.productId"); + expect(paths).not.toContain("items._id"); + expect(paths).not.toContain("_id"); + + // Verify extractFieldValue extracts from array of objects + const extracted = engine.extractFieldValue(docs[0], "items.productId"); + expect(extracted).toEqual([p1]); + }); + }); + + describe("Unique Index Cardinality Rigor (MED-01)", () => { + test("infers one-to-one only for ordinary single-field unique index", async () => { + const targetOid = new ObjectId(); + + const mockProfilesCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: targetOid }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockUsersCol = { + find: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ _id: new ObjectId(), profile_id: targetOid }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([ + { key: { profile_id: 1 }, unique: true }, // valid single-field unique + ]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ name: "users" }, { name: "profiles" }]), + }), + collection: jest.fn((colName: string) => { + if (colName === "users") return mockUsersCol; + if (colName === "profiles") return mockProfilesCol; + throw new Error(`Unknown col ${colName}`); + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const res = await engine.inferRelationships("test-db"); + const rel = res.relationships.find((r) => r.id === "users.profile_id->profiles._id")!; + expect(rel).toBeDefined(); + expect(rel.cardinality).toBe("one-to-one"); + expect(rel.evidence.hasUniqueIndexOnSource).toBe(true); + }); + + test("excludes sparse, partial, and compound unique indexes from one-to-one", async () => { + const targetOid = new ObjectId(); + + const mockTargetCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: targetOid }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockSourceCol = { + find: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ _id: new ObjectId(), target_id: targetOid }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([ + { key: { target_id: 1 }, unique: true, sparse: true }, // sparse -> excluded! + { key: { target_id: 1 }, unique: true, partialFilterExpression: { active: true } }, // partial -> excluded! + { key: { target_id: 1, tenant_id: 1 }, unique: true }, // compound -> excluded! + ]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ name: "sources" }, { name: "targets" }]), + }), + collection: jest.fn((colName: string) => { + if (colName === "sources") return mockSourceCol; + if (colName === "targets") return mockTargetCol; + throw new Error(`Unknown col ${colName}`); + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const res = await engine.inferRelationships("test-db"); + const rel = res.relationships.find((r) => r.id === "sources.target_id->targets._id")!; + expect(rel).toBeDefined(); + expect(rel.cardinality).not.toBe("one-to-one"); // Must NOT be one-to-one + expect(rel.evidence.hasUniqueIndexOnSource).toBe(false); + }); + }); + + describe("Insufficient Evidence Semantics (MED-02)", () => { + test("separates insufficient evidence (< 0.65) into diagnostic list", async () => { + const mockCodesCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: "non_hex_custom_1" }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + // Source field has matching name to "codes" (nameScore = 1.0) and generic string type (typeScore = 0.3) + // HitRatio = 0.50 (1/2 matched) + // Confidence = 0.55 * 0.5 + 0.25 * 1.0 + 0.20 * 0.3 = 0.275 + 0.25 + 0.06 = 0.585 (< 0.65) + const mockSourcesCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([ + { _id: new ObjectId(), code_id: "non_hex_custom_1" }, + { _id: new ObjectId(), code_id: "non_matching_value_123" }, + ]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ name: "sources" }, { name: "codes" }]), + }), + collection: jest.fn((colName: string) => { + if (colName === "sources") return mockSourcesCol; + if (colName === "codes") return mockCodesCol; + throw new Error(`Unknown col ${colName}`); + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const res = await engine.inferRelationships("test-db"); + // Must not be in actionable relationships + expect(res.relationships.some((r) => r.sourceField === "code_id")).toBe(false); + // Must be in insufficientEvidence diagnostic list + const diag = res.insufficientEvidence.find((r) => r.sourceField === "code_id"); + expect(diag).toBeDefined(); + expect(diag?.cardinality).toBe("insufficient-evidence"); + expect(diag?.confidence).toBeLessThan(0.65); + }); + }); + + describe("Bounded Target Discovery & Collection Scoping (HIGH-01 / LOW-01)", () => { + test("does not query all unrelated collections when candidate name matches", async () => { + const u1 = new ObjectId(); + const mockUsersCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: u1 }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + const mockOrdersCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: new ObjectId(), userId: u1 }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + const mockUnrelatedCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([ + { name: "users" }, + { name: "orders" }, + { name: "logs" }, + { name: "analytics" }, + { name: "audit" }, + ]), + }), + collection: jest.fn((colName: string) => { + if (colName === "users") return mockUsersCol; + if (colName === "orders") return mockOrdersCol; + return mockUnrelatedCol; + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + await engine.inferRelationships("test-db"); + + // logs, analytics, audit should NOT have target find queries for userId! + expect(mockUsersCol.find).toHaveBeenCalled(); + }); + + test("scopes inference to requested collections when provided", async () => { + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ name: "users" }, { name: "orders" }, { name: "products" }]), + }), + collection: jest.fn().mockReturnValue({ + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const res = await engine.inferRelationships("test-db", { + collections: ["users", "orders"], + }); + + expect(res.collectionsScanned).toBe(2); + expect(mockDb.collection).not.toHaveBeenCalledWith("products"); + }); + + test("does not query arbitrary collections when candidate has no matching target (LOW-REM-02)", async () => { + // Source collection has an externalId field (ends in Id), but no collection named externals/external exists + const mockSourcesCol = { + find: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ _id: new ObjectId(), externalId: new ObjectId() }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + const mockProductsCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + const mockOrdersCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ name: "sources" }, { name: "products" }, { name: "orders" }]), + }), + collection: jest.fn((colName: string) => { + if (colName === "sources") return mockSourcesCol; + if (colName === "products") return mockProductsCol; + if (colName === "orders") return mockOrdersCol; + throw new Error(`Unknown col ${colName}`); + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const res = await engine.inferRelationships("test-db"); + + // Target collections should NOT be probed for externalId because neither products nor orders match "external" + expect(mockProductsCol.find).not.toHaveBeenCalledWith( + expect.objectContaining({ _id: expect.anything() }), + expect.anything(), + ); + expect(mockOrdersCol.find).not.toHaveBeenCalledWith( + expect.objectContaining({ _id: expect.anything() }), + expect.anything(), + ); + expect(res.relationships.length).toBe(0); + }); + + test("stops inference and reports timedOut when overall deadline is exceeded without persisting partial results (MED-REM-01)", async () => { + // Seed pre-existing stored relationships + const preExistingInferred = [ + { + id: "orders.customer_id->customers._id", + sourceCollection: "orders", + sourceField: "customer_id", + targetCollection: "customers", + targetField: "_id", + cardinality: "many-to-one" as const, + confidence: 0.9, + evidence: {} as any, + status: "inferred" as const, + inferredAt: "2026-01-01T00:00:00.000Z", + }, + ]; + storedRelationships.inferred = [...preExistingInferred]; + storedRelationships.insufficientEvidence = [ + { + id: "orders.code->codes._id", + sourceCollection: "orders", + sourceField: "code", + targetCollection: "codes", + targetField: "_id", + cardinality: "insufficient-evidence" as const, + confidence: 0.4, + evidence: {} as any, + status: "inferred" as const, + inferredAt: "2026-01-01T00:00:00.000Z", + }, + ]; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ name: "col_1" }, { name: "col_2" }, { name: "col_3" }]), + }), + collection: jest.fn().mockReturnValue({ + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + // Force Date.now to simulate deadline expiry after first collection + let calls = 0; + const originalNow = Date.now; + jest.spyOn(Date, "now").mockImplementation(() => { + calls++; + return calls > 3 ? originalNow() + 100000 : originalNow(); + }); + + const res = await engine.inferRelationships("test-db", { timeoutMs: 1000 }); + expect(res.timedOut).toBe(true); + + // Verify saveMongoRelationships was NOT called on timeout (MED-REM-01) + expect(mockProjectStore.saveMongoRelationships).not.toHaveBeenCalled(); + + // Verify stored relationships in memory remain untouched + expect(storedRelationships.inferred).toEqual(preExistingInferred); + expect(storedRelationships.insufficientEvidence?.length).toBe(1); + + jest.spyOn(Date, "now").mockRestore(); + }); + + test("passes exact sub-100ms remaining deadline to maxTimeMS (LOW-REM-01)", async () => { + let capturedMaxTimeMS: number | undefined; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ name: "single_col" }]), + }), + collection: jest.fn().mockReturnValue({ + find: jest.fn().mockImplementation((_filter: any, options: any) => { + capturedMaxTimeMS = options?.maxTimeMS; + return { + toArray: jest.fn().mockResolvedValue([]), + limit: jest.fn().mockReturnThis(), + }; + }), + indexes: jest.fn().mockResolvedValue([]), + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const baseNow = 1000000; + let callCount = 0; + jest.spyOn(Date, "now").mockImplementation(() => { + callCount++; + // On collection loop (callCount >= 2), remaining time is 45ms (less than 100ms) + if (callCount >= 2) return baseNow + 1000 - 45; + return baseNow; + }); + + await engine.inferRelationships("test-db", { timeoutMs: 1000 }); + + // Must NOT be floored to 100; must be exact remaining 45ms + expect(capturedMaxTimeMS).toBe(45); + + jest.spyOn(Date, "now").mockRestore(); + }); + }); + + describe("String Hex vs BSON ObjectId Normalization", () => { + test("matches target collection where _id is stored as 24-char hex string", async () => { + const hexId = "507f1f77bcf86cd799439011"; + + const mockCustomersCol = { + find: jest.fn().mockImplementation((filter: any) => { + // Verify that query filter contains both ObjectId and raw string + const values = filter?._id?.$in || []; + const hasStringMatch = values.includes(hexId); + return { + toArray: jest.fn().mockResolvedValue(hasStringMatch ? [{ _id: hexId }] : []), + limit: jest.fn().mockReturnThis(), + }; + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockInvoicesCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: new ObjectId(), customer_id: hexId }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ name: "customers" }, { name: "invoices" }]), + }), + collection: jest.fn((colName: string) => { + if (colName === "customers") return mockCustomersCol; + if (colName === "invoices") return mockInvoicesCol; + throw new Error(`Unknown col ${colName}`); + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + const res = await engine.inferRelationships("test-db"); + const rel = res.relationships.find((r) => r.id === "invoices.customer_id->customers._id"); + expect(rel).toBeDefined(); + expect(rel?.evidence.hitRatio).toBe(1.0); + }); + }); + + describe("Ambiguous Target Penalty", () => { + test("applies 0.30 ambiguity penalty when multiple targets match", async () => { + const sharedId = new ObjectId(); + + const mockColA = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: sharedId }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + const mockColB = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: sharedId }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + const mockSourceCol = { + find: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([{ _id: new ObjectId(), target_id: sharedId }]), + limit: jest.fn().mockReturnThis(), + }), + indexes: jest.fn().mockResolvedValue([]), + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest + .fn() + .mockResolvedValue([{ name: "source" }, { name: "target" }, { name: "targets" }]), + }), + collection: jest.fn((colName: string) => { + if (colName === "source") return mockSourceCol; + if (colName === "target") return mockColA; + if (colName === "targets") return mockColB; + throw new Error(`Unknown col ${colName}`); + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + // HitRatio = 1.0, TypeScore = 1.0, NameScore = 1.0 (both match "target" / "targets") + // Without penalty: 0.55 * 1 + 0.25 * 1 + 0.20 * 1 = 1.00 + // With ambiguity penalty: 1.00 - 0.30 = 0.70 + const res = await engine.inferRelationships("test-db"); + expect(res.relationships.length).toBe(2); + expect(res.relationships[0].confidence).toBe(0.7); + expect(res.relationships[1].confidence).toBe(0.7); + }); + }); + + describe("State Retention Across Refresh & CRUD", () => { + test("preserves confirmed, rejected, manual, and insufficient definitions", async () => { + storedRelationships = { + version: 1, + databaseId: "test-db", + inferred: [], + confirmed: [ + { + id: "orders.customer_id->customers._id", + sourceCollection: "orders", + sourceField: "customer_id", + targetCollection: "customers", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + rejectedIds: ["orders.bad_id->products._id"], + manual: [], + insufficientEvidence: [], + updatedAt: "2026-01-01T00:00:00.000Z", + }; + + const mockDb: any = { + listCollections: jest.fn().mockReturnValue({ + toArray: jest.fn().mockResolvedValue([]), + }), + }; + jest.spyOn(engine, "getDb").mockResolvedValue(mockDb); + + await engine.inferRelationships("test-db"); + + expect(mockProjectStore.saveMongoRelationships).toHaveBeenCalledWith( + expect.objectContaining({ + confirmed: expect.arrayContaining([ + expect.objectContaining({ id: "orders.customer_id->customers._id" }), + ]), + rejectedIds: ["orders.bad_id->products._id"], + }), + undefined, + "test-db", + ); + }); + + test("sets status and manages manual CRUD", async () => { + storedRelationships.inferred = [ + { + id: "orders.user_id->users._id", + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + confidence: 0.95, + evidence: {} as any, + status: "inferred", + inferredAt: new Date().toISOString(), + }, + ]; + + const confRes = await engine.setRelationshipStatus( + "test-db", + "orders.user_id->users._id", + "confirmed", + ); + expect(confRes.status).toBe("confirmed"); + expect(storedRelationships.confirmed.length).toBe(1); + + const rejRes = await engine.setRelationshipStatus( + "test-db", + "orders.user_id->users._id", + "rejected", + ); + expect(rejRes.status).toBe("rejected"); + expect(storedRelationships.rejectedIds).toContain("orders.user_id->users._id"); + expect(storedRelationships.confirmed.length).toBe(0); + }); + }); +}); diff --git a/bridge/__tests__/mongoUri.test.ts b/bridge/__tests__/mongoUri.test.ts new file mode 100644 index 00000000..03a8c9c0 --- /dev/null +++ b/bridge/__tests__/mongoUri.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, test } from "@jest/globals"; +import { + isMongoUri, + extractMongoCredentials, + sanitizeMongoUri, + injectMongoPassword, + normalizeMongoUriMeta, + splitMongoUriCredentials, + buildMongoConnectionUri, +} from "../src/utils/mongoUri"; + +describe("mongoUri", () => { + test("isMongoUri recognizes mongodb and mongodb+srv schemes", () => { + expect(isMongoUri("mongodb://localhost:27017")).toBe(true); + expect(isMongoUri("mongodb+srv://cluster/db")).toBe(true); + expect(isMongoUri("postgres://localhost")).toBe(false); + expect(isMongoUri(undefined)).toBe(false); + expect(isMongoUri("")).toBe(false); + }); + + test("extractMongoCredentials decodes percent-encoded credentials", () => { + const creds = extractMongoCredentials("mongodb://user:p%40ss%23word@localhost:27017/testdb"); + expect(creds.user).toBe("user"); + expect(creds.password).toBe("p@ss#word"); + }); + + test("sanitizeMongoUri removes the password but keeps user and options", () => { + const sanitized = sanitizeMongoUri( + "mongodb://user:secret@localhost:27017/testdb?authSource=admin&replicaSet=rs0", + ); + expect(sanitized).toBe("mongodb://user@localhost:27017/testdb?authSource=admin&replicaSet=rs0"); + expect(sanitized).not.toContain("secret"); + }); + + test("sanitizeMongoUri is idempotent and safe for SRV URIs", () => { + const srv = "mongodb+srv://alice:p%40ss%23word@cluster0.example.net/prod"; + const s1 = sanitizeMongoUri(srv); + expect(s1).toBe("mongodb+srv://alice@cluster0.example.net/prod"); + expect(sanitizeMongoUri(s1)).toBe(s1); + }); + + test("injectMongoPassword re-injects and round-trips to the original", () => { + const original = "mongodb://user:p%40ss@localhost:27017/testdb"; + const sanitized = sanitizeMongoUri(original); + const reconstructed = injectMongoPassword(sanitized, "p@ss"); + // decoded password is re-encoded identically + expect(decodeURIComponent(reconstructed)).toBe(decodeURIComponent(original)); + expect(reconstructed).toContain("p%40ss"); + }); + + test("injectMongoPassword without a username returns the URI unchanged", () => { + const uri = "mongodb://localhost:27017/testdb"; + expect(injectMongoPassword(uri, "secret")).toBe(uri); + }); + + test("normalizeMongoUriMeta derives relational-meta fields", () => { + const meta = normalizeMongoUriMeta( + "mongodb://user:p%40ss%23word@example.net:27018/proddb?authSource=admin&replicaSet=rs0", + ); + expect(meta.host).toBe("example.net"); + expect(meta.port).toBe(27018); + expect(meta.user).toBe("user"); + expect(meta.database).toBe("proddb"); + expect(meta.authSource).toBe("admin"); + expect(meta.replicaSet).toBe("rs0"); + }); + + test("normalizeMongoUriMeta defaults port to 27017", () => { + const meta = normalizeMongoUriMeta("mongodb://localhost/testdb"); + expect(meta.port).toBe(27017); + expect(meta.database).toBe("testdb"); + }); + + test("splitMongoUriCredentials separates sanitized URI from the password", () => { + const parts = splitMongoUriCredentials("mongodb://u:p%40ss%23word@h:27017/db"); + expect(parts.isMongoUri).toBe(true); + expect(parts.hasCredentials).toBe(true); + expect(parts.sanitizedUri).toBe("mongodb://u@h:27017/db"); + expect(parts.password).toBe("p@ss#word"); + }); + + test("splitMongoUriCredentials passes non-URI values through", () => { + const parts = splitMongoUriCredentials("not-a-uri"); + expect(parts.isMongoUri).toBe(false); + expect(parts.hasCredentials).toBe(false); + expect(parts.sanitizedUri).toBe("not-a-uri"); + }); + + test("buildMongoConnectionUri re-injects runtime password into a sanitized URI", () => { + const sanitized = "mongodb://testuser@127.0.0.1:27017/testdb"; + const uri = buildMongoConnectionUri({ url: sanitized }, "runtime-pw"); + expect(uri).toContain("testuser"); + expect(decodeURIComponent(uri)).toContain("runtime-pw"); + }); + + test("buildMongoConnectionUri leaves a passwordless sanitized URI untouched", () => { + const sanitized = "mongodb://127.0.0.1:27017/testdb"; + expect(buildMongoConnectionUri({ url: sanitized }, null)).toBe(sanitized); + }); + + test("buildMongoConnectionUri composes from host/port/user/password", () => { + const uri = buildMongoConnectionUri( + { host: "localhost", port: 27017, user: "u", database: "db" }, + "p@ss", + ); + expect(uri).toBe("mongodb://u:p%40ss@localhost:27017/db"); + }); + + test("buildMongoConnectionUri throws when neither uri nor host is present", () => { + expect(() => buildMongoConnectionUri({}, "pw")).toThrow(/requires a url/i); + }); + + test("parses multi-host replica set URI without throwing ERR_INVALID_URL", () => { + const uri = + "mongodb://user:secret@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin"; + expect(() => extractMongoCredentials(uri)).not.toThrow(); + const meta = normalizeMongoUriMeta(uri); + expect(meta.host).toBe("host1"); + expect(meta.port).toBe(27017); + expect(meta.user).toBe("user"); + expect(meta.database).toBe("testdb"); + expect(meta.replicaSet).toBe("rs0"); + expect(meta.authSource).toBe("admin"); + }); + + test("multi-host credentials extraction and sanitization", () => { + const uri = + "mongodb://user:secret@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin"; + const creds = extractMongoCredentials(uri); + expect(creds.user).toBe("user"); + expect(creds.password).toBe("secret"); + + const sanitized = sanitizeMongoUri(uri); + expect(sanitized).toBe( + "mongodb://user@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin", + ); + expect(sanitized).not.toContain("secret"); + expect(sanitized).toContain("host1:27017,host2:27017,host3:27017"); + expect(sanitized).toContain("replicaSet=rs0"); + expect(sanitized).toContain("authSource=admin"); + }); + + test("re-injects password into sanitized multi-host URI with buildMongoConnectionUri", () => { + const sanitized = + "mongodb://user@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin"; + const reconstructed = buildMongoConnectionUri({ url: sanitized }, "secretPass"); + expect(reconstructed).toBe( + "mongodb://user:secretPass@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin", + ); + }); +}); diff --git a/bridge/__tests__/services/mongoBulkEngineCleanup.test.ts b/bridge/__tests__/services/mongoBulkEngineCleanup.test.ts new file mode 100644 index 00000000..e03d0dca --- /dev/null +++ b/bridge/__tests__/services/mongoBulkEngineCleanup.test.ts @@ -0,0 +1,219 @@ +import { describe, expect, test, beforeEach, afterEach } from "@jest/globals"; +import path from "path"; +import fs from "fs"; +import os from "os"; +import { + ExportCoordinationManager, + MongoBulkEngine, + EXPORT_ORPHAN_TTL_MS, +} from "../../src/services/mongoBulkEngine"; + +describe("MongoBulkEngine Export Orphan Cleanup & Coordination Suite (AUD-09)", () => { + let testExportDir: string; + + beforeEach(async () => { + testExportDir = path.join( + os.tmpdir(), + `relwave-test-exports-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + ); + await fs.promises.mkdir(testExportDir, { recursive: true }); + ExportCoordinationManager.resetForTesting(); + }); + + afterEach(async () => { + ExportCoordinationManager.resetForTesting(); + try { + await fs.promises.rm(testExportDir, { recursive: true, force: true }); + } catch { + // safe cleanup + } + }); + + test("1. ExportCoordinationManager process-wide singleton identity", () => { + const inst1 = ExportCoordinationManager.getInstance(); + const inst2 = ExportCoordinationManager.getInstance(); + expect(inst1).toBe(inst2); + + const _engineA = new MongoBulkEngine(); + const _engineB = new MongoBulkEngine(); + // Both engines interact with the same underlying singleton + expect(ExportCoordinationManager.getInstance()).toBe(inst1); + }); + + test("2. Cross-engine job protection: Active job registered in Engine A is protected from purge by Engine B even when > 2 hours old", async () => { + const manager = ExportCoordinationManager.getInstance(); + const _engineA = new MongoBulkEngine(); + const engineB = new MongoBulkEngine(); + + const jobId = "job-active-cross-engine-12345"; + const filePath = path.join(testExportDir, `export-${jobId}.json`); + + // Register job under Engine A / singleton + await manager.registerJobBeforeFileCreation(jobId, filePath); + await fs.promises.writeFile(filePath, '{"test": 1}'); + + // Set file mtime to 3 hours ago (> 2h threshold) + const threeHoursAgo = Date.now() - (EXPORT_ORPHAN_TTL_MS + 3600000); + await fs.promises.utimes(filePath, new Date(threeHoursAgo), new Date(threeHoursAgo)); + + // Engine B executes purge + const result = await engineB.purgeOrphanedExports(testExportDir); + + expect(result.purged).toBe(0); + expect(result.skipped).toBe(1); + expect(fs.existsSync(filePath)).toBe(true); + + // Unregister and purge again + await manager.unregisterJob(jobId); + const resultAfterUnregister = await engineB.purgeOrphanedExports(testExportDir); + expect(resultAfterUnregister.purged).toBe(1); + expect(fs.existsSync(filePath)).toBe(false); + }); + + test("3. Pre-creation registration under critical section prevents purge race", async () => { + const manager = ExportCoordinationManager.getInstance(); + const jobId = "pre-reg-job-999"; + const filePath = path.join(testExportDir, `export-${jobId}.tmp`); + + // Register job BEFORE file creation + await manager.registerJobBeforeFileCreation(jobId, filePath); + expect(manager.isJobActive(jobId)).toBe(true); + + // File is created + await fs.promises.writeFile(filePath, "pre-created data"); + + // Even if timestamp is artificially aged, active registry shield holds + const fourHoursAgo = Date.now() - EXPORT_ORPHAN_TTL_MS * 2; + await fs.promises.utimes(filePath, new Date(fourHoursAgo), new Date(fourHoursAgo)); + + const purgeResult = await manager.purgeOrphanedExportsAsync(testExportDir); + expect(purgeResult.purged).toBe(0); + expect(purgeResult.skipped).toBe(1); + expect(fs.existsSync(filePath)).toBe(true); + + await manager.unregisterJob(jobId); + }); + + test("4. Two-hour TTL age boundary (Option A: 2 * 60 * 60 * 1000 ms)", async () => { + const manager = ExportCoordinationManager.getInstance(); + + const freshJobId = "fresh-job-uuid-111"; + const staleJobId = "stale-job-uuid-222"; + + const freshFile = path.join(testExportDir, `export-${freshJobId}.json`); + const staleFile = path.join(testExportDir, `export-${staleJobId}.json`); + + await fs.promises.writeFile(freshFile, "fresh export data"); + await fs.promises.writeFile(staleFile, "stale export data"); + + const now = Date.now(); + // Fresh: 1 hour ago (< 2h) + const oneHourAgo = now - 3600000; + await fs.promises.utimes(freshFile, new Date(oneHourAgo), new Date(oneHourAgo)); + + // Stale: 2 hours and 5 minutes ago (> 2h) + const twoHoursFiveMinAgo = now - (EXPORT_ORPHAN_TTL_MS + 300000); + await fs.promises.utimes(staleFile, new Date(twoHoursFiveMinAgo), new Date(twoHoursFiveMinAgo)); + + const result = await manager.purgeOrphanedExportsAsync(testExportDir); + + expect(result.purged).toBe(1); // Only stale file purged + expect(result.skipped).toBe(1); // Fresh file skipped + + expect(fs.existsSync(freshFile)).toBe(true); + expect(fs.existsSync(staleFile)).toBe(false); + }); + + test("5. Symlink rejection via lstat() protects target files", async () => { + const manager = ExportCoordinationManager.getInstance(); + + const targetFile = path.join(testExportDir, "important-target-data.json"); + await fs.promises.writeFile(targetFile, "CRITICAL DATA NEVER TOUCH"); + + const symlinkPath = path.join(testExportDir, "export-symlink-target.json"); + + try { + await fs.promises.symlink(targetFile, symlinkPath); + } catch (err: any) { + // In some Windows environments without dev privileges, symlinks might require elevation + if (err.code === "EPERM") { + return; // skip if OS forbids symlink creation + } + throw err; + } + + // Age the symlink target or symlink + const threeHoursAgo = Date.now() - (EXPORT_ORPHAN_TTL_MS + 3600000); + try { + await fs.promises.utimes(symlinkPath, new Date(threeHoursAgo), new Date(threeHoursAgo)); + } catch {} + + await manager.purgeOrphanedExportsAsync(testExportDir); + + // Symlinks are skipped + expect(fs.existsSync(targetFile)).toBe(true); + expect(fs.existsSync(symlinkPath)).toBe(true); + }); + + test("6. Keyed per-directory mutex serializes concurrent purge sweeps", async () => { + const manager = ExportCoordinationManager.getInstance(); + const executionOrder: string[] = []; + + const p1 = manager.withDirLock(testExportDir, async () => { + executionOrder.push("p1_start"); + await new Promise((resolve) => setTimeout(resolve, 50)); + executionOrder.push("p1_end"); + }); + + const p2 = manager.withDirLock(testExportDir, async () => { + executionOrder.push("p2_start"); + executionOrder.push("p2_end"); + }); + + await Promise.all([p1, p2]); + + expect(executionOrder).toEqual(["p1_start", "p1_end", "p2_start", "p2_end"]); + }); + + test("7. Strict export filename regex matching ignores unrelated files", async () => { + const manager = ExportCoordinationManager.getInstance(); + + const validExport1 = path.join(testExportDir, "export-abc-123.csv"); + const validExport2 = path.join(testExportDir, "export-xyz_999.ndjson"); + const validExport3 = path.join(testExportDir, "export-temp-session.tmp"); + const invalidFile1 = path.join(testExportDir, "my-other-document.json"); + const invalidFile2 = path.join(testExportDir, "export.json"); + const invalidFile3 = path.join(testExportDir, "export-.csv"); + + await fs.promises.writeFile(validExport1, "1"); + await fs.promises.writeFile(validExport2, "2"); + await fs.promises.writeFile(validExport3, "3"); + await fs.promises.writeFile(invalidFile1, "4"); + await fs.promises.writeFile(invalidFile2, "5"); + await fs.promises.writeFile(invalidFile3, "6"); + + const ancientTime = Date.now() - (EXPORT_ORPHAN_TTL_MS + 10000000); + for (const f of [ + validExport1, + validExport2, + validExport3, + invalidFile1, + invalidFile2, + invalidFile3, + ]) { + await fs.promises.utimes(f, new Date(ancientTime), new Date(ancientTime)); + } + + const result = await manager.purgeOrphanedExportsAsync(testExportDir); + + expect(result.purged).toBe(3); // only validExport 1, 2, 3 + expect(result.skipped).toBe(3); // invalidFile 1, 2, 3 skipped + + expect(fs.existsSync(validExport1)).toBe(false); + expect(fs.existsSync(validExport2)).toBe(false); + expect(fs.existsSync(validExport3)).toBe(false); + expect(fs.existsSync(invalidFile1)).toBe(true); + expect(fs.existsSync(invalidFile2)).toBe(true); + expect(fs.existsSync(invalidFile3)).toBe(true); + }); +}); diff --git a/bridge/__tests__/services/mongoConnectionLifecycle.test.ts b/bridge/__tests__/services/mongoConnectionLifecycle.test.ts new file mode 100644 index 00000000..49c8bf91 --- /dev/null +++ b/bridge/__tests__/services/mongoConnectionLifecycle.test.ts @@ -0,0 +1,970 @@ +import { describe, expect, test, jest, beforeEach, afterEach } from "@jest/globals"; +import path from "path"; +import fs from "fs"; +import { MongoClient } from "mongodb"; +import { + getClientKey, + clientRegistry, + closingClients, + closeMongoClientByKey, + closeAllMongoClients, + forceDestroyMongoSockets, + getMongoClient, + testConnection, + setMongoShuttingDown, + mongoConnector, +} from "../../src/connectors/mongo"; +import { ConnectionPool, PoolEntry } from "../../src/services/connectionPool"; +import { DatabaseService } from "../../src/services/databaseService"; +import { ConnectionBuilder } from "../../src/services/connectionBuilder"; +import { dbStoreInstance } from "../../src/services/dbStore"; +import { keyringServiceInstance } from "../../src/services/keyringService"; +import { MongoBulkEngine } from "../../src/services/mongoBulkEngine"; +import { MigrationHandlers } from "../../src/handlers/migrationHandlers"; +import { DBType } from "../../src/types"; +import { MongoConfig } from "../../src/types/mongo"; + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: any) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe("MongoDB Connection Lifecycle & Concurrency Deterministic Test Suite", () => { + let dbService: DatabaseService; + let pool: ConnectionPool; + + beforeEach(() => { + clientRegistry.clear(); + closingClients.clear(); + setMongoShuttingDown(false); + + pool = new ConnectionPool(); + dbService = new DatabaseService(); + (dbService as any).connectionPool = pool; + jest.restoreAllMocks(); + }); + + afterEach(async () => { + try { + pool.destroy(); + } catch {} + clientRegistry.clear(); + closingClients.clear(); + setMongoShuttingDown(false); + jest.useRealTimers(); + }); + + // --------------------------------------------------------------------------- + // 1. test_concurrent_callers_receive_distinct_leases_and_correct_active_count + // --------------------------------------------------------------------------- + test("1. test_concurrent_callers_receive_distinct_leases_and_correct_active_count", async () => { + const dbId = "test_db_1"; + const barriers = Array.from({ length: 5 }, () => createDeferred()); + let buildCount = 0; + + const mockDb = { + id: dbId, + name: "TestDB", + type: "mongodb", + url: "mongodb://localhost:27017/testdb", + database: "testdb", + }; + + jest.spyOn(dbStoreInstance, "getDB").mockResolvedValue(mockDb as any); + jest.spyOn(dbStoreInstance, "getPasswordFor").mockResolvedValue(null as any); + jest.spyOn(ConnectionBuilder, "buildConnection").mockImplementation(async () => { + buildCount++; + return { + config: { uri: "mongodb://localhost:27017/testdb", database: "testdb" }, + tunnel: undefined, + }; + }); + + const callers = barriers.map((barrier, index) => + dbService.withConnection(dbId, async (_conn, _db, _type) => { + await barrier.promise; + return index; + }), + ); + + // Yield to allow all 5 callers to acquire leases and pause at their barrier + await new Promise((r) => setImmediate(r)); + + expect(buildCount).toBe(1); + const poolEntries = (pool as any).pool as Map; + const entry = poolEntries.get(dbId); + expect(entry).toBeDefined(); + expect(entry!.activeOperations).toBe(5); + + // Release caller 1 + barriers[0].resolve(); + const result0 = await callers[0]; + expect(result0).toBe(0); + expect(entry!.activeOperations).toBe(4); + + // Invalidate profile while callers 2-5 are still holding their leases + await pool.invalidate(dbId); + expect(entry!.disposed).toBe(true); + const pendingDisposal = (pool as any).pendingDisposal as Set; + expect(pendingDisposal.has(entry!)).toBe(true); + + const safeCloseSpy = jest.spyOn(pool, "safeCloseMongoClient"); + + // Release callers 2 to 5 + barriers[1].resolve(); + barriers[2].resolve(); + barriers[3].resolve(); + barriers[4].resolve(); + + await Promise.all(callers.slice(1)); + expect(entry!.activeOperations).toBe(0); + expect(pendingDisposal.has(entry!)).toBe(false); + + // Give microtask tick for deferred cleanup + await new Promise((r) => setImmediate(r)); + expect(safeCloseSpy).toHaveBeenCalled(); + }); + + // --------------------------------------------------------------------------- + // 2. test_identity_safe_inflight_promise_removal + // --------------------------------------------------------------------------- + test("2. test_identity_safe_inflight_promise_removal", async () => { + const dbId = "test_db_2"; + const barrierA = createDeferred(); + const barrierB = createDeferred(); + + const mockDb = { + id: dbId, + name: "TestDB", + type: "mongodb", + url: "mongodb://localhost:27017/testdb", + database: "testdb", + }; + + jest.spyOn(dbStoreInstance, "getDB").mockResolvedValue(mockDb as any); + jest.spyOn(dbStoreInstance, "getPasswordFor").mockResolvedValue(null as any); + + let invocation = 0; + jest.spyOn(ConnectionBuilder, "buildConnection").mockImplementation(async () => { + invocation++; + if (invocation === 1) { + await barrierA.promise; + return { config: { uri: "mongodb://localhost:27017/testdb_a", database: "testdb" } }; + } else { + await barrierB.promise; + return { config: { uri: "mongodb://localhost:27017/testdb_b", database: "testdb" } }; + } + }); + + const inFlightMap = (dbService as any).inFlightConnections as Map>; + + // Start Builder A + const builderAPromise = (dbService as any).ensureConnectionBuilt(dbId); + expect(inFlightMap.has(dbId)).toBe(true); + const promiseA = inFlightMap.get(dbId); + + // Allow Builder A to progress past getDB and pause inside buildConnection + for (let i = 0; i < 5; i++) await new Promise((r) => setImmediate(r)); + expect(invocation).toBe(1); + + // Invalidate connection (bumps generation and clears inFlightConnections) + await dbService.invalidateConnection(dbId); + expect(inFlightMap.has(dbId)).toBe(false); + + // Start Builder B + const builderBPromise = (dbService as any).ensureConnectionBuilt(dbId); + expect(inFlightMap.has(dbId)).toBe(true); + const promiseB = inFlightMap.get(dbId); + expect(promiseB).not.toBe(promiseA); + + // Allow Builder B to progress past getDB and pause inside buildConnection + for (let i = 0; i < 5; i++) await new Promise((r) => setImmediate(r)); + expect(invocation).toBe(2); + + // Resolve Builder A + barrierA.resolve(); + await expect(builderAPromise).rejects.toThrow(/Connection build aborted/); + + // Identity safety check: Builder A settling must NOT have deleted Builder B's promise + expect(inFlightMap.get(dbId)).toBe(promiseB); + + // Caller C joins Builder B without starting Builder C + const builderCPromise = (dbService as any).ensureConnectionBuilt(dbId); + expect(inFlightMap.get(dbId)).toBe(promiseB); + expect(invocation).toBe(2); + + // Settle Builder B + barrierB.resolve(); + await Promise.all([builderBPromise, builderCPromise]); + + // Map is cleanly emptied after settle + expect(inFlightMap.has(dbId)).toBe(false); + }); + + // --------------------------------------------------------------------------- + // 3. test_new_profile_acquire_during_final_reference_cleanup + // --------------------------------------------------------------------------- + test("3. test_new_profile_acquire_during_final_reference_cleanup", async () => { + const clientKey = getClientKey({ + uri: "mongodb://localhost:27017/shared_db", + database: "shared_db", + }); + const closeBarrier = createDeferred(); + + const mockOldClient: any = { + close: jest.fn().mockImplementation(async () => { + await closeBarrier.promise; + }), + topology: { s: { servers: new Map() } }, + }; + + clientRegistry.set(clientKey, { client: mockOldClient, state: "closing" }); + const closePromise = closeMongoClientByKey(clientKey, { force: true }); + closingClients.set(clientKey, closePromise); + + let acquiredNewClient: MongoClient | undefined; + const acquirePromise = (async () => { + acquiredNewClient = await getMongoClient({ + uri: "mongodb://localhost:27017/shared_db", + database: "shared_db", + }); + })(); + + await new Promise((r) => setImmediate(r)); + // Profile B is awaiting in-flight closure + expect(acquiredNewClient).toBeUndefined(); + + // Settle close barrier + closeBarrier.resolve(); + await closePromise; + await acquirePromise; + + expect(acquiredNewClient).toBeDefined(); + expect(acquiredNewClient).not.toBe(mockOldClient); + expect(clientRegistry.get(clientKey)?.state).toBe("active"); + }); + + // --------------------------------------------------------------------------- + // 4. test_single_flight_force_close_recovery_and_single_replacement_client + // --------------------------------------------------------------------------- + test("4. test_single_flight_force_close_recovery_and_single_replacement_client", async () => { + const cfg: MongoConfig = { + uri: "mongodb://localhost:27017/recovery_db", + database: "recovery_db", + }; + const key = getClientKey(cfg); + const closeBarrier = createDeferred(); + + const mockFailedClient: any = { + close: jest.fn().mockImplementation(async () => { + await closeBarrier.promise; + }), + topology: { s: { servers: new Map() } }, + }; + + clientRegistry.set(key, { + client: mockFailedClient, + state: "close_failed", + closeError: new Error("Previous hang"), + }); + + // Launch two concurrent callers + const caller1Promise = getMongoClient(cfg); + const caller2Promise = getMongoClient(cfg); + + await new Promise((r) => setImmediate(r)); + + // Exactly one close call was dispatched + expect(mockFailedClient.close).toHaveBeenCalledTimes(1); + + // Resolve close barrier + closeBarrier.resolve(); + + const [client1, client2] = await Promise.all([caller1Promise, caller2Promise]); + + expect(client1).toBeDefined(); + expect(client1).toBe(client2); + expect(clientRegistry.size).toBe(1); + expect(clientRegistry.get(key)?.state).toBe("active"); + + // Failure case: when force-close rejects, verify close_failed is preserved and zero replacement created + const unconfirmedClient: any = { + close: jest.fn().mockRejectedValue(new Error("Socket severance failed")), + topology: { s: { servers: "not-a-map" } }, + }; + clientRegistry.set(key, { client: unconfirmedClient, state: "close_failed" }); + + await expect(getMongoClient(cfg)).rejects.toThrow( + /prior client for key .* failed to force-close/, + ); + expect(clientRegistry.get(key)?.state).toBe("close_failed"); + }); + + // --------------------------------------------------------------------------- + // 5. test_dynamically_allocated_ssh_ports_and_lifecycle_interleaving + // --------------------------------------------------------------------------- + test("5. test_dynamically_allocated_ssh_ports_and_lifecycle_interleaving", async () => { + const dbA = { + id: "ssh_db_a", + name: "SSH_A", + type: "mongodb", + url: "mongodb://remote-server:27017/db_a", + database: "db_a", + ssh: { host: "bastion", port: 22, user: "tunnel", authMethod: "password" }, + }; + const dbB = { + id: "ssh_db_b", + name: "SSH_B", + type: "mongodb", + url: "mongodb://remote-server:27017/db_b", + database: "db_b", + ssh: { host: "bastion", port: 22, user: "tunnel", authMethod: "password" }, + }; + + jest + .spyOn(dbStoreInstance, "getDB") + .mockImplementation(async (id) => (id === "ssh_db_a" ? dbA : dbB) as any); + jest.spyOn(dbStoreInstance, "getPasswordFor").mockResolvedValue(null as any); + jest.spyOn(keyringServiceInstance, "getCredential").mockResolvedValue("ssh_secret"); + + const mockTunnelA = { localPort: 54321, close: jest.fn() }; + const mockTunnelB = { localPort: 54322, close: jest.fn() }; + + jest.spyOn(ConnectionBuilder, "buildConnection").mockImplementation(async (db: any) => { + const tunnel = db.id === "ssh_db_a" ? mockTunnelA : mockTunnelB; + return { + config: { + uri: `mongodb://127.0.0.1:${tunnel.localPort}/${db.database}`, + database: db.database, + }, + tunnel, + }; + }); + + const leaseA = await dbService.acquireConnection("ssh_db_a"); + const leaseB = await dbService.acquireConnection("ssh_db_b"); + + const keyA = getClientKey(leaseA.conn); + const keyB = getClientKey(leaseB.conn); + + expect(keyA).toContain("54321"); + expect(keyB).toContain("54322"); + expect(keyA).not.toBe(keyB); + + leaseA.release(); + leaseB.release(); + + // Interleaved cleanup: invalidating Profile A does NOT affect Profile B + await pool.invalidate("ssh_db_a"); + expect(mockTunnelA.close).toHaveBeenCalled(); + expect(mockTunnelB.close).not.toHaveBeenCalled(); + expect(pool.get("ssh_db_b")).toBeDefined(); + }); + + // --------------------------------------------------------------------------- + // 6. test_stale_builder_separate_tunnel_and_client_adoption_checks + // --------------------------------------------------------------------------- + describe("6. test_stale_builder_separate_tunnel_and_client_adoption_checks", () => { + test("Sub-scenario A: Same tunnel object adopted, unshared client", async () => { + const builtTunnel = { localPort: 50001, close: jest.fn() }; + const builtConfig = { uri: "mongodb://127.0.0.1:50001/unshared_db", database: "unshared_db" }; + + // Adopted in pool under distinct profile + pool.set( + "other_profile", + { uri: "mongodb://127.0.0.1:50001/other_db", database: "other_db" }, + DBType.MONGODB, + builtTunnel as any, + ); + + const safeCloseSpy = jest.spyOn(pool, "safeCloseMongoClient"); + + // Builder runs check 3 logic + const isTunnelAdopted = Array.from(pool.getAllEntries()).some( + (e) => e.tunnel === builtTunnel, + ); + expect(isTunnelAdopted).toBe(true); + if (!isTunnelAdopted) builtTunnel.close(); + + await pool.safeCloseMongoClient(builtConfig); + + expect(builtTunnel.close).not.toHaveBeenCalled(); + expect(safeCloseSpy).toHaveBeenCalledWith(builtConfig); + }); + + test("Sub-scenario B: Similar endpoint, different tunnel object", async () => { + const builtTunnel = { localPort: 50002, close: jest.fn() }; + const otherTunnel = { localPort: 50002, close: jest.fn() }; + + pool.set( + "other_profile", + { uri: "mongodb://127.0.0.1:50002/db", database: "db" }, + DBType.MONGODB, + otherTunnel as any, + ); + + // Identity check: must be strict object equality + const isTunnelAdopted = Array.from(pool.getAllEntries()).some( + (e) => e.tunnel === builtTunnel, + ); + expect(isTunnelAdopted).toBe(false); + + if (!isTunnelAdopted) builtTunnel.close(); + + expect(builtTunnel.close).toHaveBeenCalled(); + expect(otherTunnel.close).not.toHaveBeenCalled(); + }); + + test("Sub-scenario C: Shared MongoClient, unshared tunnel", async () => { + const builtTunnel = { localPort: 50003, close: jest.fn() }; + const sharedConfig = { uri: "mongodb://127.0.0.1:50003/shared_c", database: "shared_c" }; + + // Other profile shares clientKey but has different or no tunnel + pool.set("adopting_profile", sharedConfig, DBType.MONGODB, undefined); + + const isTunnelAdopted = Array.from(pool.getAllEntries()).some( + (e) => e.tunnel === builtTunnel, + ); + if (!isTunnelAdopted) builtTunnel.close(); + + const closeKeySpy = jest.spyOn(pool, "safeCloseMongoClient"); + await pool.safeCloseMongoClient(sharedConfig); + + expect(builtTunnel.close).toHaveBeenCalled(); + expect(closeKeySpy).toHaveBeenCalled(); + }); + + test("Sub-scenario D: Shared tunnel, unshared MongoClient", async () => { + const builtTunnel = { localPort: 50004, close: jest.fn() }; + const unsharedConfig = { + uri: "mongodb://127.0.0.1:50004/distinct_d", + database: "distinct_d", + }; + + pool.set( + "tunnel_holder", + { uri: "mongodb://127.0.0.1:50004/holder_db", database: "holder_db" }, + DBType.MONGODB, + builtTunnel as any, + ); + + const isTunnelAdopted = Array.from(pool.getAllEntries()).some( + (e) => e.tunnel === builtTunnel, + ); + expect(isTunnelAdopted).toBe(true); + if (!isTunnelAdopted) builtTunnel.close(); + + await pool.safeCloseMongoClient(unsharedConfig); + + expect(builtTunnel.close).not.toHaveBeenCalled(); + }); + }); + + // --------------------------------------------------------------------------- + // 7. test_fatal_error_invalidation_awaited_and_original_error_preserved + // --------------------------------------------------------------------------- + test("7. test_fatal_error_invalidation_awaited_and_original_error_preserved", async () => { + const dbId = "fatal_db"; + const mockDb = { + id: dbId, + name: "FatalDB", + type: "mongodb", + url: "mongodb://localhost:27017/fatal", + database: "fatal", + }; + + jest.spyOn(dbStoreInstance, "getDB").mockResolvedValue(mockDb as any); + jest.spyOn(dbStoreInstance, "getPasswordFor").mockResolvedValue(null as any); + jest.spyOn(ConnectionBuilder, "buildConnection").mockResolvedValue({ + config: { uri: "mongodb://localhost:27017/fatal", database: "fatal" }, + }); + + const fatalError = new Error("connection reset by peer"); + (fatalError as any).name = "MongoNetworkError"; + + const invalidateSpy = jest.spyOn(dbService, "invalidateConnection"); + + await expect( + dbService.withConnection(dbId, async () => { + throw fatalError; + }), + ).rejects.toThrow("connection reset by peer"); + + expect(invalidateSpy).toHaveBeenCalledWith(dbId); + + // Verify when invalidation rejects, original error is still preserved + invalidateSpy.mockRejectedValueOnce(new Error("invalidation failure")); + + await expect( + dbService.withConnection(dbId, async () => { + throw fatalError; + }), + ).rejects.toThrow("connection reset by peer"); + }); + + // --------------------------------------------------------------------------- + // 8. test_non_settling_client_close_forced_socket_destruction_and_bounded_shutdown + // --------------------------------------------------------------------------- + test("8. test_non_settling_client_close_forced_socket_destruction_and_bounded_shutdown", async () => { + const key = "non_settling_key"; + const mockSocket = { + destroyed: false, + destroy: jest.fn(() => { + mockSocket.destroyed = true; + }), + }; + + const mockClient: any = { + close: jest.fn(() => new Promise(() => {})), // Never settles + topology: { + s: { + servers: new Map([ + ["server1", { pool: { connections: [{ socket: mockSocket }], checkedOut: new Set() } }], + ]), + }, + }, + }; + + clientRegistry.set(key, { client: mockClient, state: "active" }); + + const phase4Deadline = Date.now() + 50; + await closeAllMongoClients({ force: true, globalDeadline: phase4Deadline }); + + expect(mockSocket.destroy).toHaveBeenCalled(); + expect(clientRegistry.has(key)).toBe(false); // destroy_confirmed deletes from registry + }); + + // --------------------------------------------------------------------------- + // 9. test_shutdown_escalates_inflight_normal_close_without_competing_operation + // --------------------------------------------------------------------------- + test("9. test_shutdown_escalates_inflight_normal_close_without_competing_operation", async () => { + const key = "escalation_key"; + const mockSocket = { + destroyed: false, + destroy: jest.fn(() => { + mockSocket.destroyed = true; + }), + }; + + const ordinaryCloseBarrier = createDeferred(); + const mockClient: any = { + close: jest.fn(async () => { + await ordinaryCloseBarrier.promise; + }), + topology: { + s: { + servers: new Map([ + ["server1", { pool: { connections: [{ socket: mockSocket }], checkedOut: new Set() } }], + ]), + }, + }, + }; + + clientRegistry.set(key, { client: mockClient, state: "active" }); + + // Initiate ordinary close + const normalClosePromise = closeMongoClientByKey(key); + + // Shutdown arrives while close is in-flight + const phase4Deadline = Date.now() + 40; + await closeAllMongoClients({ force: true, globalDeadline: phase4Deadline }); + + // Client.close was called exactly ONCE (no competing second call) + expect(mockClient.close).toHaveBeenCalledTimes(1); + expect(mockSocket.destroy).toHaveBeenCalled(); + expect(clientRegistry.has(key)).toBe(false); + + ordinaryCloseBarrier.resolve(); + await normalClosePromise; + }); + + // --------------------------------------------------------------------------- + // 10. test_multiple_non_settling_clients_global_bounded_teardown + // --------------------------------------------------------------------------- + test("10. test_multiple_non_settling_clients_global_bounded_teardown", async () => { + const mockSockets: any[] = []; + for (let i = 0; i < 5; i++) { + const sock = { + destroyed: false, + destroy: jest.fn(function (this: any) { + this.destroyed = true; + }), + }; + mockSockets.push(sock); + + const client: any = { + close: jest.fn(() => new Promise(() => {})), + topology: { + s: { + servers: new Map([ + [`server_${i}`, { pool: { connections: [{ socket: sock }], checkedOut: new Set() } }], + ]), + }, + }, + }; + + clientRegistry.set(`client_${i}`, { client, state: "active" }); + } + + const start = Date.now(); + const phase4Deadline = start + 50; + + await closeAllMongoClients({ force: true, globalDeadline: phase4Deadline }); + const elapsed = Date.now() - start; + + expect(elapsed).toBeLessThan(300); // Must be bounded globally, not 5 * 50ms serial + for (const sock of mockSockets) { + expect(sock.destroy).toHaveBeenCalled(); + } + }); + + // --------------------------------------------------------------------------- + // 11. test_unconfirmed_socket_destruction_retains_registry_entry + // --------------------------------------------------------------------------- + test("11. test_unconfirmed_socket_destruction_retains_registry_entry", async () => { + const key = getClientKey({ uri: "mongodb://stubborn/db", database: "db" }); + + // Client whose socket destruction fails to observe .destroyed === true + const stubbornSocket = { destroyed: false, destroy: jest.fn() }; + const mockClient: any = { + close: jest.fn().mockRejectedValue(new Error("Socket severance failed")), + topology: { + s: { + servers: new Map([ + [ + "server1", + { pool: { connections: [{ socket: stubbornSocket }], checkedOut: new Set() } }, + ], + ]), + }, + }, + }; + + clientRegistry.set(key, { client: mockClient, state: "active" }); + + await expect( + closeMongoClientByKey(key, { force: true, globalDeadline: Date.now() + 100 }), + ).rejects.toThrow(); + + const entry = clientRegistry.get(key); + expect(entry).toBeDefined(); + expect(entry?.state).toBe("close_failed"); + expect(stubbornSocket.destroy).toHaveBeenCalled(); + + // Subsequent acquire rejects + await expect( + getMongoClient({ uri: "mongodb://stubborn/db", database: "db" } as any), + ).rejects.toThrow(/prior client for key .* failed to force-close/); + }); + + // --------------------------------------------------------------------------- + // 12. test_phase_4_cleanup_sync_bounded_to_200ms_before_forced_teardown + // --------------------------------------------------------------------------- + test("12. test_phase_4_cleanup_sync_bounded_to_200ms_before_forced_teardown", async () => { + jest.useFakeTimers(); + + const hangingCleanup = new Promise(() => {}); + pool.getInProgressCleanups().add(hangingCleanup); + + const shutdownPromise = (dbService as any).executePhase4Teardown(); + + // Advance 200ms + jest.advanceTimersByTime(200); + await Promise.resolve(); + + // Advance past phase 4 client close + jest.advanceTimersByTime(100); + await shutdownPromise; + + expect(pool.isForcedTeardownInitiated).toBe(true); + jest.useRealTimers(); + }); + + // --------------------------------------------------------------------------- + // 13. test_canonical_client_key_identity_across_sanitization_and_casing + // --------------------------------------------------------------------------- + test("13. test_canonical_client_key_identity_across_sanitization_and_casing", async () => { + const key1 = getClientKey({ + uri: "mongodb://user:secret@localhost:27017/ProductionDB", + database: "ProductionDB", + }); + const key2 = getClientKey({ + uri: "mongodb://user@localhost:27017/ProductionDB", + database: " productiondb ", + }); + const key3 = getClientKey({ + uri: "mongodb://user:secret@localhost:27017/productiondb", + database: "productiondb", + }); + + expect(key1).toBe(key2); + expect(key2).toBe(key3); + + const pingSpy = jest.spyOn(MongoClient.prototype, "db").mockReturnValue({ + command: jest.fn().mockResolvedValue({ ok: 1 }), + } as any); + + // Verify testConnection is completely transient and does not touch clientRegistry + const initialSize = clientRegistry.size; + await testConnection({ uri: "mongodb://localhost:27017/test", database: "test" }); + expect(clientRegistry.size).toBe(initialSize); + pingSpy.mockRestore(); + }); + + // --------------------------------------------------------------------------- + // 14. test_shutdown_blocks_resource_creation_after_credential_lookup + // --------------------------------------------------------------------------- + test("14. test_shutdown_blocks_resource_creation_after_credential_lookup", async () => { + const dbId = "shutdown_db_check2"; + const keyringBarrier = createDeferred(); + + jest.spyOn(dbStoreInstance, "getDB").mockResolvedValue({ + id: dbId, + name: "Check2DB", + type: "mongodb", + database: "check2", + ssh: { host: "bastion", authMethod: "password" }, + } as any); + jest.spyOn(dbStoreInstance, "getPasswordFor").mockResolvedValue(null as any); + jest.spyOn(keyringServiceInstance, "getCredential").mockImplementation(async () => { + return keyringBarrier.promise; + }); + + const buildSpy = jest.spyOn(ConnectionBuilder, "buildConnection"); + + const buildPromise = (dbService as any).ensureConnectionBuilt(dbId); + + // Trigger shutdown while waiting for keyring + (dbService as any).isShuttingDown = true; + keyringBarrier.resolve("pass"); + + await expect(buildPromise).rejects.toThrow(/Bridge is shutting down|Connection build aborted/); + expect(buildSpy).not.toHaveBeenCalled(); + }); + + // --------------------------------------------------------------------------- + // 15. test_shutdown_1500ms_deadline_enforces_forced_teardown + // --------------------------------------------------------------------------- + test("15. test_shutdown_1500ms_deadline_enforces_forced_teardown", async () => { + jest.useFakeTimers(); + + const hangingBuilder = new Promise(() => {}); + (dbService as any).activeBuilders.add(hangingBuilder); + + const shutdownPromise = dbService.shutdown(); + + // Advance 1500ms for phase 3 drain + jest.advanceTimersByTime(1500); + await Promise.resolve(); + + // Advance 200ms for phase 4 cleanup sync + jest.advanceTimersByTime(200); + await Promise.resolve(); + + // Advance 100ms for phase 4 forced teardown + jest.advanceTimersByTime(100); + await shutdownPromise; + + expect((dbService as any).isShuttingDown).toBe(true); + expect(pool.isForcedTeardownInitiated).toBe(true); + jest.useRealTimers(); + }); + + // --------------------------------------------------------------------------- + // 16. test_all_invalidation_entry_points_advance_generation + // --------------------------------------------------------------------------- + test("16. test_all_invalidation_entry_points_advance_generation", async () => { + const dbId = "gen_db"; + expect(dbService.getGeneration(dbId)).toBe(0); + + // Invalidation 1: updateDatabase + jest.spyOn(dbStoreInstance, "updateDB").mockResolvedValue({} as any); + await dbService.updateDatabase(dbId, { name: "NewName" }); + expect(dbService.getGeneration(dbId)).toBe(1); + + // Invalidation 2: fatal error in withConnection + jest.spyOn(dbStoreInstance, "getDB").mockResolvedValue({ id: dbId, database: "test" } as any); + jest + .spyOn(ConnectionBuilder, "buildConnection") + .mockResolvedValue({ config: { database: "test" } }); + + const fatal = new Error("ECONNRESET"); + (fatal as any).name = "MongoNetworkError"; + + await expect( + dbService.withConnection(dbId, async () => { + throw fatal; + }), + ).rejects.toThrow("ECONNRESET"); + expect(dbService.getGeneration(dbId)).toBe(2); + + // Invalidation 3: deleteDatabase + jest.spyOn(dbStoreInstance, "deleteDB").mockResolvedValue({ ok: true } as any); + jest.spyOn(keyringServiceInstance, "deleteCredential").mockResolvedValue(); + await dbService.deleteDatabase(dbId); + expect(dbService.getGeneration(dbId)).toBe(3); + }); + + // --------------------------------------------------------------------------- + // 17. test_read_only_rejection_across_all_15_mutating_operations + // --------------------------------------------------------------------------- + test("17. test_read_only_rejection_across_all_15_mutating_operations", async () => { + const readOnlyConn: MongoConfig = { + uri: "mongodb://localhost:27017/ro_db", + database: "ro_db", + readonly: true, + }; + + // 1-9: Connector Guards in mongo.ts + await expect( + mongoConnector.insertDocument(readOnlyConn, { collection: "c", document: { a: 1 } }), + ).rejects.toThrow("Database is configured as read-only. 'insertDocument' is prohibited."); + + await expect( + mongoConnector.updateDocument(readOnlyConn, { + collection: "c", + filter: { a: 1 }, + update: { $set: { b: 2 } }, + }), + ).rejects.toThrow("Database is configured as read-only. 'updateDocument' is prohibited."); + + await expect( + mongoConnector.replaceDocument(readOnlyConn, { + collection: "c", + filter: { a: 1 }, + replacement: { b: 2 }, + }), + ).rejects.toThrow("Database is configured as read-only. 'replaceDocument' is prohibited."); + + await expect( + mongoConnector.deleteDocument(readOnlyConn, { collection: "c", filter: { a: 1 } }), + ).rejects.toThrow("Database is configured as read-only. 'deleteDocument' is prohibited."); + + await expect( + mongoConnector.createCollection(readOnlyConn, { collection: "c_new" }), + ).rejects.toThrow("Database is configured as read-only. 'createCollection' is prohibited."); + + await expect( + mongoConnector.dropCollection(readOnlyConn, { collection: "c_old" }), + ).rejects.toThrow("Database is configured as read-only. 'dropCollection' is prohibited."); + + await expect( + mongoConnector.createIndexes(readOnlyConn, { collection: "c", key: { a: 1 } }), + ).rejects.toThrow("Database is configured as read-only. 'createIndexes' is prohibited."); + + await expect( + mongoConnector.dropIndex(readOnlyConn, { collection: "c", indexName: "idx_1" }), + ).rejects.toThrow("Database is configured as read-only. 'dropIndex' is prohibited."); + + await expect( + mongoConnector.hideIndex(readOnlyConn, { collection: "c", indexName: "idx_1", hidden: true }), + ).rejects.toThrow("Database is configured as read-only. 'hideIndex' is prohibited."); + + // 10-13: Bulk Engine Guards in mongoBulkEngine.ts + const mockDbService = { + getDatabaseConnection: jest.fn().mockResolvedValue({ + dbType: DBType.MONGODB, + db: { id: "ro_bulk_db", readonly: true }, + conn: readOnlyConn, + }), + } as any; + const bulkEngine = new MongoBulkEngine(mockDbService); + + await expect( + bulkEngine.bulkInsert({ dbId: "ro_bulk_db", collection: "c", documents: [{ a: 1 }] }), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + + await expect( + bulkEngine.bulkUpdate({ + dbId: "ro_bulk_db", + collection: "c", + filter: { a: 1 }, + update: { $set: { b: 2 } }, + }), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + + await expect( + bulkEngine.bulkDelete({ dbId: "ro_bulk_db", collection: "c", filter: { a: 1 } }), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + + const opState = bulkEngine.opManager.createOperation("import", "ro_bulk_db", "c", "json"); + await expect( + bulkEngine.importChunk({ + operationId: opState.operation.operationId, + chunkIndex: 0, + isLastChunk: true, + payload: "[]", + }), + ).rejects.toMatchObject({ code: "MONGO_READ_ONLY" }); + + // 14-15: Migration Handler Guards in migrationHandlers.ts + const mockRpc = { sendResponse: jest.fn(), sendError: jest.fn() }; + const mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() } as any; + const migHandlers = new MigrationHandlers(mockRpc as any, mockLogger, mockDbService, {} as any); + + await migHandlers.handleApplyMigration({ dbId: "ro_bulk_db", version: 1 }, 101); + expect(mockRpc.sendError).toHaveBeenCalledWith( + 101, + expect.objectContaining({ code: "MONGO_READ_ONLY" }), + ); + + await migHandlers.handleRollbackMigration({ dbId: "ro_bulk_db", version: 1 }, 102); + expect(mockRpc.sendError).toHaveBeenCalledWith( + 102, + expect.objectContaining({ code: "MONGO_READ_ONLY" }), + ); + }); + + // --------------------------------------------------------------------------- + // 18. test_implementation_time_driver_version_verification + // --------------------------------------------------------------------------- + test("18. test_implementation_time_driver_version_verification", async () => { + // 1. Check resolved mongodb version + + const driverPackage = require("mongodb/package.json"); + expect(driverPackage.version).toBe("6.21.0"); + + // 2. Check bridge/package.json specification + const pkgJsonPath = path.resolve(__dirname, "../../package.json"); + const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8")); + expect(pkgJson.dependencies.mongodb).toMatch(/6\.21\.0/); + + // 3. Runtime topology and connection pool structure assertion + const mockSocket = { destroyed: false, destroy: jest.fn() }; + const serverMap = new Map(); + const mockPool = { + connections: [{ socket: mockSocket }], + checkedOut: new Set(), + destroy: jest.fn(), + }; + serverMap.set("node1", { pool: mockPool }); + + const client: any = new MongoClient("mongodb://localhost:27017/test"); + // Attach simulated 6.21.0 topology structure + client.topology = { + s: { + servers: serverMap, + }, + }; + + expect(client.topology.s.servers instanceof Map).toBe(true); + expect(mockPool.connections[0].socket).toBeDefined(); + + // Verify forceDestroyMongoSockets correctly processes standard 6.21.0 layout + const result = forceDestroyMongoSockets(client); + expect(result.status).toBe("destroy_requested"); + expect(result.totalDiscoveredSockets).toBe(1); + expect(mockSocket.destroy).toHaveBeenCalled(); + + // Verify structural deviation throws explicit diagnostic error / unconfirmed status + const corruptClient: any = { topology: { s: { servers: "invalid_structure" } } }; + const corruptResult = forceDestroyMongoSockets(corruptClient); + expect(corruptResult.status).toBe("unconfirmed"); + expect(corruptResult.error?.message).toContain("Unexpected MongoDB driver topology structure"); + }); +}); diff --git a/bridge/__tests__/services/mongoMigrationDrift.test.ts b/bridge/__tests__/services/mongoMigrationDrift.test.ts new file mode 100644 index 00000000..fa6e0c80 --- /dev/null +++ b/bridge/__tests__/services/mongoMigrationDrift.test.ts @@ -0,0 +1,416 @@ +import { describe, expect, test, beforeEach, afterEach } from "@jest/globals"; +import fs from "fs"; +import path from "path"; +import os from "os"; +import { Db } from "mongodb"; +import { + computeManifestChecksum, + listMigrations, + MAX_MANIFEST_SIZE_BYTES, + MIGRATIONS_COLLECTION, +} from "../../src/services/mongoMigrationEngine"; +import { LOCK_COLLECTION } from "../../src/services/mongoMigrationLock"; +import { MongoMigrationSummary } from "../../src/types/mongoMigration"; + +function createDriftTestDb(summaries: MongoMigrationSummary[]) { + const summaryMap = new Map(); + for (const s of summaries) { + summaryMap.set(s._id, { ...s }); + } + + const writeCalls: string[] = []; + + const db: any = { + collection(name: string) { + if (name === LOCK_COLLECTION) { + return { + async findOne() { + return null; // No active lock + }, + }; + } + + if (name === MIGRATIONS_COLLECTION) { + return { + find() { + return { + sort() { + return { + async toArray() { + return Array.from(summaryMap.values()); + }, + }; + }, + }; + }, + async updateOne() { + writeCalls.push("updateOne"); + return { matchedCount: 1, modifiedCount: 1 }; + }, + async findOneAndUpdate() { + writeCalls.push("findOneAndUpdate"); + return null; + }, + async insertOne() { + writeCalls.push("insertOne"); + return { insertedId: "test" }; + }, + async deleteOne() { + writeCalls.push("deleteOne"); + return { deletedCount: 1 }; + }, + }; + } + + return { + find() { + return { + toArray: async () => [], + }; + }, + }; + }, + getWriteCalls: () => writeCalls, + }; + + return db; +} + +describe("mongoMigrationDrift", () => { + let tempProjectRoot: string; + let migrationsDir: string; + + beforeEach(() => { + tempProjectRoot = fs.mkdtempSync(path.join(os.tmpdir(), "relwave-drift-")); + migrationsDir = path.join(tempProjectRoot, "migrations", "mongo"); + fs.mkdirSync(migrationsDir, { recursive: true }); + }); + + afterEach(() => { + try { + fs.rmSync(tempProjectRoot, { recursive: true, force: true }); + } catch { + // Cleanup best effort + } + }); + + describe("Three-State Precedence & Drift Classification", () => { + test("Precedence 1: flags UNVERIFIABLE with MANIFEST_MISSING when applied migration has no on-disk file", async () => { + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Add Users Collection", + checksum: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + expect(res.appliedMigrations.length).toBe(1); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("UNVERIFIABLE"); + expect(applied.drifted).toBe(false); + expect(applied.driftDetails?.reason).toBe("MANIFEST_MISSING"); + expect(applied.driftDetails?.message).toContain("not found"); + }); + + test("Precedence 2: flags UNVERIFIABLE with MANIFEST_PARSE_ERROR when multiple on-disk manifest files produce duplicate migration IDs", async () => { + const manifestContent = JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "Duplicate One", + up: [{ type: "createCollection", collection: "test" }], + }); + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_a.mql.json"), + manifestContent, + "utf8", + ); + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_b.mql.json"), + manifestContent, + "utf8", + ); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Duplicate Test", + checksum: computeManifestChecksum(manifestContent), + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("UNVERIFIABLE"); + expect(applied.drifted).toBe(false); + expect(applied.driftDetails?.reason).toBe("MANIFEST_PARSE_ERROR"); + expect(applied.driftDetails?.message).toContain("Multiple manifest files"); + expect(applied.driftDetails?.message).toContain("20261031120000_a.mql.json"); + expect(applied.driftDetails?.message).toContain("20261031120000_b.mql.json"); + expect(res.pendingMigrations.length).toBe(0); + }); + + test("Precedence 2: flags UNVERIFIABLE with MANIFEST_PARSE_ERROR when manifest exceeds 1MB without reading file content", async () => { + const filename = "20261031120000_oversized.mql.json"; + const filePath = path.join(migrationsDir, filename); + + const header = Buffer.from('{"version":"1.0","id":"20261031120000","name":"large","pad":"'); + const footer = Buffer.from('"}'); + const padSize = MAX_MANIFEST_SIZE_BYTES + 100 - header.length - footer.length; + const pad = Buffer.alloc(padSize, "x"); + const fullContent = Buffer.concat([header, pad, footer]); + fs.writeFileSync(filePath, fullContent); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Oversized Test", + checksum: "somehash", + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("UNVERIFIABLE"); + expect(applied.drifted).toBe(false); + expect(applied.driftDetails?.reason).toBe("MANIFEST_PARSE_ERROR"); + expect(applied.driftDetails?.message).toContain("exceeds 1MB limit"); + }); + + test("Precedence 2: flags UNVERIFIABLE with MANIFEST_PARSE_ERROR when on-disk manifest contains invalid JSON", async () => { + const filename = "20261031120000_corrupt.mql.json"; + fs.writeFileSync(path.join(migrationsDir, filename), "{ corrupt json: not closed", "utf8"); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Corrupt Test", + checksum: "somehash", + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("UNVERIFIABLE"); + expect(applied.drifted).toBe(false); + expect(applied.driftDetails?.reason).toBe("MANIFEST_PARSE_ERROR"); + expect(applied.driftDetails?.message).toContain("Failed to parse manifest JSON"); + }); + + test("Precedence 2: associates corrupt JSON manifest with applied migration via filename-derived 14-digit timestamp ID", async () => { + const filename = "20261031154500_corrupt_manifest.mql.json"; + fs.writeFileSync(path.join(migrationsDir, filename), "broken json syntax", "utf8"); + + const db = createDriftTestDb([ + { + _id: "20261031154500", + name: "Corrupt Test Timestamp", + checksum: "hash123", + status: "APPLIED", + startedAt: "2026-10-31T15:45:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied._id).toBe("20261031154500"); + expect(applied.driftStatus).toBe("UNVERIFIABLE"); + expect(applied.driftDetails?.reason).toBe("MANIFEST_PARSE_ERROR"); + }); + + test("Precedence 3: flags UNVERIFIABLE with RECORDED_CHECKSUM_MISSING when database record has empty checksum", async () => { + const manifestContent = JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "Legacy Migration", + up: [{ type: "createCollection", collection: "orders" }], + }); + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_orders.mql.json"), + manifestContent, + "utf8", + ); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Legacy Migration", + checksum: "", // Empty baseline checksum + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("UNVERIFIABLE"); + expect(applied.drifted).toBe(false); + expect(applied.driftDetails?.reason).toBe("RECORDED_CHECKSUM_MISSING"); + expect(applied.driftDetails?.actualChecksum).toBe(computeManifestChecksum(manifestContent)); + expect(applied.driftDetails?.message).toContain("no baseline checksum"); + }); + + test("Precedence 4: flags VERIFIED with CHECKSUM_MATCH when on-disk manifest SHA-256 matches recorded checksum", async () => { + const manifestContent = JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "Verified Migration", + up: [{ type: "createCollection", collection: "users" }], + }); + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_users.mql.json"), + manifestContent, + "utf8", + ); + const computedHash = computeManifestChecksum(manifestContent); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Verified Migration", + checksum: computedHash, + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("VERIFIED"); + expect(applied.drifted).toBe(false); + expect(applied.driftDetails?.reason).toBe("CHECKSUM_MATCH"); + expect(applied.driftDetails?.expectedChecksum).toBe(computedHash); + expect(applied.driftDetails?.actualChecksum).toBe(computedHash); + }); + + test("Precedence 5: flags DRIFTED with CHECKSUM_MISMATCH when on-disk manifest content has been altered", async () => { + const manifestContent = JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "Drifted Migration", + up: [{ type: "createCollection", collection: "altered" }], + }); + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_drifted.mql.json"), + manifestContent, + "utf8", + ); + const onDiskHash = computeManifestChecksum(manifestContent); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Drifted Migration", + checksum: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + const applied = res.appliedMigrations[0]; + expect(applied.driftStatus).toBe("DRIFTED"); + expect(applied.drifted).toBe(true); + expect(applied.driftDetails?.reason).toBe("CHECKSUM_MISMATCH"); + expect(applied.driftDetails?.expectedChecksum).toBe( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ); + expect(applied.driftDetails?.actualChecksum).toBe(onDiskHash); + }); + }); + + describe("Execution Semantics Immutability", () => { + test("does not modify recorded checksums or write to _relwave_migrations during listMigrations", async () => { + const manifestContent = JSON.stringify({ + version: "1.0", + id: "20261031120000", + name: "Immutability Test", + up: [], + }); + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_immut.mql.json"), + manifestContent, + "utf8", + ); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Immutability Test", + checksum: "hash_differs_from_disk", + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 0, + totalSteps: 0, + fencingToken: 1, + }, + ]); + + await listMigrations(db as Db, tempProjectRoot); + expect(db.getWriteCalls().length).toBe(0); + }); + + test("does not change migration execution status from APPLIED", async () => { + const manifestContent = "altered content"; + fs.writeFileSync( + path.join(migrationsDir, "20261031120000_status.mql.json"), + manifestContent, + "utf8", + ); + + const db = createDriftTestDb([ + { + _id: "20261031120000", + name: "Status Test", + checksum: "oldhash", + status: "APPLIED", + startedAt: "2026-10-31T12:00:00Z", + executedSteps: 1, + totalSteps: 1, + fencingToken: 1, + }, + ]); + + const res = await listMigrations(db as Db, tempProjectRoot); + expect(res.appliedMigrations[0].status).toBe("APPLIED"); + }); + + test("reuses identical computeManifestChecksum function and CRLF-normalization as apply-time path", () => { + const lfContent = '{\n "version": "1.0"\n}\n'; + const crlfContent = '{\r\n "version": "1.0"\r\n}\r\n'; + expect(computeManifestChecksum(lfContent)).toBe(computeManifestChecksum(crlfContent)); + }); + }); +}); diff --git a/bridge/__tests__/services/mongoQueryCancellation.test.ts b/bridge/__tests__/services/mongoQueryCancellation.test.ts new file mode 100644 index 00000000..ec909713 --- /dev/null +++ b/bridge/__tests__/services/mongoQueryCancellation.test.ts @@ -0,0 +1,391 @@ +import { describe, expect, test, jest, beforeEach, afterEach } from "@jest/globals"; +import { + createDriverSettlementSignal, + ZombieOperationRegistry, + UnrecoverableZombieRegistry, + executeIndependentTeardown, + OperationLifecycleSupervisor, +} from "../../src/handlers/queryHandlers"; +import { SessionManager } from "../../src/sessionManager"; +import { OperationLifecycleHandle, TeardownOutcome } from "../../src/types/mongo"; + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: any) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe("MongoDB Stage 8 Phase 3 Query Cancellation & Lifecycle Supervision Suite", () => { + beforeEach(() => { + ZombieOperationRegistry.resetForTesting(); + UnrecoverableZombieRegistry.resetForTesting(); + jest.restoreAllMocks(); + }); + + afterEach(() => { + ZombieOperationRegistry.resetForTesting(); + UnrecoverableZombieRegistry.resetForTesting(); + }); + + // ------------------------------------------------------------------------- + // 1. Authoritative Lease-Release Invariant Tests + // ------------------------------------------------------------------------- + + test("1. Condition A: Natural driver settlement permits release, exactly once", () => { + let releaseCount = 0; + const { signal, markSettled } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-1", + dbId: "db-1", + state: "running", + settlementSignal: signal, + }; + + const supervisor = new OperationLifecycleSupervisor( + handle, + () => { + releaseCount++; + }, + {} as any, + { uri: "mongodb://localhost:27017", database: "testdb" }, + ); + + // Prior to settlement, safeReleaseLease does nothing + supervisor.safeReleaseLease(); + expect(releaseCount).toBe(0); + expect(handle.hasReleasedLease).toBeFalsy(); + + // Natural settlement occurs + markSettled("fulfilled"); + expect(signal.isSettled).toBe(true); + + supervisor.safeReleaseLease(); + expect(releaseCount).toBe(1); + expect(handle.hasReleasedLease).toBe(true); + expect(handle.state).toBe("completed"); + + // Repeated calls are idempotent (exactly once) + supervisor.safeReleaseLease(); + expect(releaseCount).toBe(1); + }); + + test("2. Condition B: Confirmed socket destruction must await actual driver settlement before releasing lease", async () => { + let releaseCount = 0; + const { signal, markSettled } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-2", + dbId: "db-1", + state: "running", + settlementSignal: signal, + }; + + const mockDbService = { + connectionPool: { getActiveOperationsCount: () => 1 }, + invalidateConnection: jest.fn().mockImplementation(() => Promise.resolve()), + }; + + const supervisor = new OperationLifecycleSupervisor( + handle, + () => { + releaseCount++; + }, + mockDbService as any, + { uri: "mongodb://localhost:27017", database: "testdb" }, + ); + + // Mock executeDebouncedTeardown to return destroy_confirmed + const registry = ZombieOperationRegistry.getInstance(); + jest.spyOn(registry, "executeDebouncedTeardown").mockResolvedValue({ + status: "destroy_confirmed", + invalidationSucceeded: true, + socketsDestroyed: 2, + durationMs: 15, + }); + + const teardownPromise = supervisor.runTeardownLoop("client-key-test"); + + // Sockets are confirmed destroyed, but driver promise has not settled yet! + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(releaseCount).toBe(0); + expect(signal.isSettled).toBe(false); + + // Driver promise rejects due to MongoNetworkError from severed socket + markSettled("rejected", new Error("MongoNetworkError: connection closed")); + + await teardownPromise; + expect(releaseCount).toBe(1); + expect(handle.hasReleasedLease).toBe(true); + }); + + test("3. Strict Prohibitions: Failed teardown and shutdown handoff NEVER release lease", async () => { + let releaseCount = 0; + const { signal } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-3", + dbId: "db-1", + state: "running", + settlementSignal: signal, + }; + + const supervisor = new OperationLifecycleSupervisor( + handle, + () => { + releaseCount++; + }, + {} as any, + { uri: "mongodb://localhost:27017", database: "testdb" }, + ); + + const registry = ZombieOperationRegistry.getInstance(); + + // 3a: Shutdown handoff + jest.spyOn(registry, "executeDebouncedTeardown").mockResolvedValueOnce({ + status: "shutdown_handoff", + reason: "shutdown_adopted", + durationMs: 0, + }); + + await supervisor.runTeardownLoop("client-key-test"); + expect(handle.state).toBe("shutdown_adopted"); + expect(releaseCount).toBe(0); + expect(handle.hasReleasedLease).toBeFalsy(); + + // 3b: Unrecoverable failure + jest.spyOn(registry, "executeDebouncedTeardown").mockResolvedValue({ + status: "unrecoverable_failure", + invalidationSucceeded: false, + totalAttempts: 3, + durationMs: 50, + }); + + await supervisor.runTeardownLoop("client-key-test"); + expect(handle.state).toBe("unrecoverable_pinned"); + expect(releaseCount).toBe(0); + expect(UnrecoverableZombieRegistry.getInstance().has("op-3")).toBe(true); + }); + + // ------------------------------------------------------------------------- + // 2. Pre-Driver Cancellation Gates & Signal Racing Tests + // ------------------------------------------------------------------------- + + test("4. Pre-driver cancellation gate: Cancel before driver promise unwinds and releases lease", async () => { + let releaseCount = 0; + const { signal } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-gate", + dbId: "db-gate", + state: "running", + settlementSignal: signal, + }; + + const supervisor = new OperationLifecycleSupervisor( + handle, + () => { + releaseCount++; + }, + {} as any, + { uri: "mongodb://localhost:27017", database: "testdb" }, + ); + + // Rejected outer promise prior to driver invocation + const outerRejection = Promise.reject(new Error("Pre-driver aborted")); + + await supervisor.superviseTimeoutOrCancel(undefined, outerRejection); + + expect(signal.isSettled).toBe(true); + expect(releaseCount).toBe(1); + expect(handle.hasReleasedLease).toBe(true); + }); + + test("5. Fast Pre-Check: Already settled driver returns status: driver_settled immediately", async () => { + const { signal, markSettled } = createDriverSettlementSignal(); + markSettled("fulfilled"); + + const handle: OperationLifecycleHandle = { + id: "op-fast", + dbId: "db-fast", + state: "running", + settlementSignal: signal, + }; + + const outcome = await executeIndependentTeardown({} as any, "db-fast", {}, handle, 1, 3); + + expect(outcome.status).toBe("driver_settled"); + }); + + test("6. Invalidation racing: Driver settling during pool invalidation aborts teardown", async () => { + const { signal, markSettled } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-inval-race", + dbId: "db-inval", + state: "running", + settlementSignal: signal, + }; + + const invalDeferred = createDeferred(); + const mockDbService = { + invalidateConnection: jest.fn().mockReturnValue(invalDeferred.promise), + }; + + const teardownPromise = executeIndependentTeardown( + mockDbService as any, + "db-inval", + {}, + handle, + 1, + 3, + ); + + // Driver settles while invalidation is in-flight + markSettled("fulfilled"); + invalDeferred.resolve(); + + const outcome = await teardownPromise; + expect(outcome.status).toBe("driver_settled"); + }); + + // ------------------------------------------------------------------------- + // 3. Concurrency, Zombie Registry & Escalation + // ------------------------------------------------------------------------- + + test("7. Deadlock-free sole-op recovery: poolActiveOps <= 1 enters teardown immediately", async () => { + const { signal } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-sole", + dbId: "db-sole", + state: "running", + settlementSignal: signal, + }; + + const mockDbService = { + connectionPool: { getActiveOperationsCount: () => 1 }, + }; + + const supervisor = new OperationLifecycleSupervisor(handle, () => {}, mockDbService as any, { + uri: "mongodb://localhost:27017", + database: "testdb", + }); + + const teardownSpy = jest.spyOn(supervisor, "runTeardownLoop").mockResolvedValue(); + + const outerPromise = Promise.reject(new Error("Timeout")).catch(() => {}); + const driverDeferred = createDeferred(); + + await supervisor.superviseTimeoutOrCancel(driverDeferred.promise, outerPromise as any, 10); + + // Teardown should have been invoked immediately without registering a 60s zombie + expect(teardownSpy).toHaveBeenCalledTimes(1); + expect(ZombieOperationRegistry.getInstance().getZombiesForDb("db-sole")).toHaveLength(0); + }); + + test("8. Sibling concurrency: poolActiveOps > 1 pins zombie with active 60s escalation timer", async () => { + const { signal } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-sibling", + dbId: "db-sibling", + state: "running", + settlementSignal: signal, + }; + + let escalated = false; + const registry = ZombieOperationRegistry.getInstance(); + registry.registerZombie(handle, "client-key-sibling", () => { + escalated = true; + }); + + expect(handle.state).toBe("zombie_pinned"); + expect(registry.getZombiesForDb("db-sibling")).toHaveLength(1); + expect(escalated).toBe(false); + + // Unregister cancels the timer cleanly + registry.unregisterZombie(handle.id); + expect(registry.getZombiesForDb("db-sibling")).toHaveLength(0); + }); + + test("9. Shutdown handoff: snapshotAndDisarm disarms timers and snapshots in-flight teardowns", async () => { + const registry = ZombieOperationRegistry.getInstance(); + + const { signal } = createDriverSettlementSignal(); + const handle: OperationLifecycleHandle = { + id: "op-shutdown", + dbId: "db-shutdown", + state: "running", + settlementSignal: signal, + }; + + registry.registerZombie(handle, "key-shut", () => {}); + expect(registry.getZombiesForDb("db-shutdown")).toHaveLength(1); + + // Mock an in-flight teardown + const inFlightOutcome: TeardownOutcome = { + status: "destroy_confirmed", + invalidationSucceeded: true, + socketsDestroyed: 1, + durationMs: 20, + }; + registry.executeDebouncedTeardown("db-shutdown", "key-shut", async () => inFlightOutcome); + + const adopted = registry.snapshotAndDisarm(); + expect(adopted.length).toBeGreaterThanOrEqual(1); + expect(registry.getZombiesForDb("db-shutdown")).toHaveLength(0); + + // Adoption with Promise.allSettled is rejection-safe + const results = await Promise.allSettled(adopted); + expect(results[0].status).toBe("fulfilled"); + }); + + // ------------------------------------------------------------------------- + // 4. SessionManager Synchronous Non-Blocking Cancel + // ------------------------------------------------------------------------- + + test("10. SessionManager synchronous non-blocking cancel and at-most-once callback", () => { + const sm = new SessionManager(); + const sid = "sess-123"; + sm.create(sid); + + let callbackCount = 0; + sm.registerCancel(sid, () => { + callbackCount++; + }); + + // Synchronous cancel returns boolean + const res1 = sm.cancel(sid); + expect(res1).toBe(true); + expect(callbackCount).toBe(1); + + // Repeated cancel is idempotent and does not dispatch duplicate callback + const res2 = sm.cancel(sid); + expect(res2).toBe(true); + expect(callbackCount).toBe(1); + + // Unknown session returns false + const resUnknown = sm.cancel("unknown-session-id"); + expect(resUnknown).toBe(false); + + sm.destroy(); + }); + + test("11. SessionManager post-registration cancel dispatch", () => { + const sm = new SessionManager(); + const sid = "sess-early-cancel"; + sm.create(sid); + + // Cancel arrives before cancel callback is registered + const cancelled = sm.cancel(sid); + expect(cancelled).toBe(true); + + let callbackFired = false; + sm.registerCancel(sid, () => { + callbackFired = true; + }); + + // Callback is dispatched immediately upon registration + expect(callbackFired).toBe(true); + + sm.destroy(); + }); +}); diff --git a/bridge/docker-compose.test.yml b/bridge/docker-compose.test.yml index 190aab41..0b63e4c6 100644 --- a/bridge/docker-compose.test.yml +++ b/bridge/docker-compose.test.yml @@ -67,6 +67,19 @@ services: timeout: 5s retries: 5 + mongodb: + image: mongo:7 + environment: + # Runs without auth so the credential-free integration test can connect. + MONGO_INITDB_DATABASE: testdb + ports: + - "27017:27017" + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping')"] + interval: 10s + timeout: 5s + retries: 5 + ssh-server: image: linuxserver/openssh-server environment: diff --git a/bridge/package.json b/bridge/package.json index c2b36d48..3e615dfa 100644 --- a/bridge/package.json +++ b/bridge/package.json @@ -1,6 +1,6 @@ { "name": "relwave-bridge", - "version": "1.2.0", + "version": "1.3.0", "type": "commonjs", "main": "dist/index.cjs", "scripts": { @@ -30,6 +30,7 @@ "better-sqlite3": "^11.9.0", "dotenv": "^17.2.3", "groq-sdk": "^1.2.1", + "mongodb": "^6.21.0", "mysql2": "^3.15.3", "ollama": "^0.6.3", "openai": "^6.41.0", diff --git a/bridge/pnpm-lock.yaml b/bridge/pnpm-lock.yaml index 47fc76ed..aec0cbca 100644 --- a/bridge/pnpm-lock.yaml +++ b/bridge/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: groq-sdk: specifier: ^1.2.1 version: 1.2.1 + mongodb: + specifier: ^6.21.0 + version: 6.21.0 mysql2: specifier: ^3.15.3 version: 3.15.3 @@ -629,6 +632,9 @@ packages: '@mistralai/mistralai@2.2.5': resolution: {integrity: sha512-ATbWzKkNzNAZ+gtw9MI/c/ULTMG80tKUiRNIbQFfg4OP0uEZZpTfXZeBCNfs5Dq0uqMQ/tQWc4o6RRJQtMrpDA==} + '@mongodb-js/saslprep@1.6.1': + resolution: {integrity: sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg==} + '@napi-rs/keyring-darwin-arm64@1.2.0': resolution: {integrity: sha512-CA83rDeyONDADO25JLZsh3eHY8yTEtm/RS6ecPsY+1v+dSawzT9GywBMu2r6uOp1IEhQs/xAfxgybGAFr17lSA==} engines: {node: '>= 10'} @@ -799,6 +805,12 @@ packages: '@types/strip-json-comments@0.0.30': resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==} + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@11.0.5': + resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} + '@types/yargs-parser@21.0.3': resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -1056,6 +1068,10 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + bson@6.10.4: + resolution: {integrity: sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==} + engines: {node: '>=16.20.1'} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -1665,6 +1681,9 @@ packages: makeerror@1.0.12: resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==} + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -1702,6 +1721,36 @@ packages: engines: {node: '>=10'} hasBin: true + mongodb-connection-string-url@3.0.2: + resolution: {integrity: sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==} + + mongodb@6.21.0: + resolution: {integrity: sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 || ^2.0.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.3.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1944,6 +1993,10 @@ packages: pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + pure-rand@7.0.1: resolution: {integrity: sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==} @@ -2057,6 +2110,9 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -2172,6 +2228,10 @@ packages: tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -2301,9 +2361,17 @@ packages: webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + whatwg-fetch@3.6.20: resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -2959,6 +3027,10 @@ snapshots: - bufferutil - utf-8-validate + '@mongodb-js/saslprep@1.6.1': + dependencies: + sparse-bitfield: 3.0.3 + '@napi-rs/keyring-darwin-arm64@1.2.0': optional: true @@ -3113,6 +3185,12 @@ snapshots: '@types/strip-json-comments@0.0.30': {} + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@11.0.5': + dependencies: + '@types/webidl-conversions': 7.0.3 + '@types/yargs-parser@21.0.3': {} '@types/yargs@17.0.35': @@ -3369,6 +3447,8 @@ snapshots: dependencies: node-int64: 0.4.0 + bson@6.10.4: {} + buffer-from@1.1.2: {} buffer@5.7.1: @@ -4136,6 +4216,8 @@ snapshots: dependencies: tmpl: 1.0.5 + memory-pager@1.5.0: {} + merge-stream@2.0.0: {} micromatch@4.0.8: @@ -4163,6 +4245,17 @@ snapshots: mkdirp@1.0.4: {} + mongodb-connection-string-url@3.0.2: + dependencies: + '@types/whatwg-url': 11.0.5 + whatwg-url: 14.2.0 + + mongodb@6.21.0: + dependencies: + '@mongodb-js/saslprep': 1.6.1 + bson: 6.10.4 + mongodb-connection-string-url: 3.0.2 + ms@2.1.3: {} multistream@4.1.0: @@ -4392,6 +4485,8 @@ snapshots: end-of-stream: 1.4.5 once: 1.4.0 + punycode@2.3.1: {} + pure-rand@7.0.1: {} quick-format-unescaped@4.0.4: {} @@ -4496,6 +4591,10 @@ snapshots: source-map@0.6.1: {} + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + split2@4.2.0: {} sprintf-js@1.0.3: {} @@ -4618,6 +4717,10 @@ snapshots: tr46@0.0.3: {} + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} ts-algebra@2.0.0: {} @@ -4758,8 +4861,15 @@ snapshots: webidl-conversions@3.0.1: {} + webidl-conversions@7.0.0: {} + whatwg-fetch@3.6.20: {} + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + whatwg-url@5.0.0: dependencies: tr46: 0.0.3 diff --git a/bridge/scripts/seed-mongodb.cjs b/bridge/scripts/seed-mongodb.cjs new file mode 100644 index 00000000..3435a891 --- /dev/null +++ b/bridge/scripts/seed-mongodb.cjs @@ -0,0 +1,150 @@ +/* + * Standalone MongoDB seed script (manual tool). + * + * Seeds the test database with a couple of collections plus a view so the + * MongoDB connector's listing/stats can be exercised: + * - users (flat documents) + * - orders (nested documents with an ObjectId reference + array + Date) + * - active_users_view (a view over users, to exercise VIEW detection) + * + * Run with: node scripts/seed-mongodb.cjs + * Reads REAL_MONGODB_* from .env (loaded via dotenv). + */ + +const fs = require("dotenv"); +fs.config({ path: require("path").join(__dirname, "..", ".env") }); + +const { MongoClient, ObjectId } = require("mongodb"); + +function wait(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function seed() { + const host = process.env.REAL_MONGODB_HOST || "127.0.0.1"; + const port = process.env.REAL_MONGODB_PORT || "27017"; + const user = process.env.REAL_MONGODB_USER; + const password = process.env.REAL_MONGODB_PASSWORD; + const database = process.env.REAL_MONGODB_DATABASE || "testdb"; + + // Build candidate URIs: prefer explicit REAL_MONGODB_URI, then credentialed if provided, with unauthenticated fallback + const uris = []; + if (process.env.REAL_MONGODB_URI) { + uris.push(process.env.REAL_MONGODB_URI); + } else { + if (user && password) { + uris.push( + `mongodb://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:${port}/${database}?authSource=admin`, + ); + } + uris.push(`mongodb://${host}:${port}/${database}`); + } + + let client = null; + let connected = false; + + for (let attempt = 1; attempt <= 15; attempt++) { + for (const testUri of uris) { + try { + const candidateClient = new MongoClient(testUri, { + serverSelectionTimeoutMS: 2000, + connectTimeoutMS: 2000, + }); + await candidateClient.connect(); + await candidateClient.db(database).command({ ping: 1 }); + client = candidateClient; + connected = true; + break; + } catch { + // try next URI candidate or retry + } + } + if (connected) break; + await wait(1000); + } + + if (!connected || !client) { + throw new Error(`MongoDB at ${host}:${port} was not reachable after 15 attempts.`); + } + + const db = client.db(database); + + // Idempotent: drop existing fixtures first. + for (const name of ["users", "orders", "active_users_view"]) { + try { + await db.dropCollection(name); + } catch { + // may not exist + } + } + + const userIds = []; + const users = [ + { + _id: new ObjectId(), + name: "John Doe", + email: "john@example.com", + age: 30, + active: true, + createdAt: new Date(), + }, + { + _id: new ObjectId(), + name: "Jane Smith", + email: "jane@example.com", + age: 27, + active: true, + createdAt: new Date(), + }, + { + _id: new ObjectId(), + name: "Bob Lee", + email: "bob@example.com", + age: 45, + active: false, + createdAt: new Date(), + }, + ]; + const userInsert = await db.collection("users").insertMany(users); + for (const id of Object.values(userInsert.insertedIds)) { + userIds.push(id); + } + + const orders = [ + { + _id: new ObjectId(), + userId: userIds[0], + items: [ + { sku: "ABC-1", qty: 2, price: 9.99 }, + { sku: "XYZ-9", qty: 1, price: 19.5 }, + ], + total: 39.48, + status: "placed", + orderedAt: new Date(), + }, + { + _id: new ObjectId(), + userId: userIds[1], + items: [{ sku: "ABC-1", qty: 5, price: 9.99 }], + total: 49.95, + status: "shipped", + orderedAt: new Date(), + }, + ]; + await db.collection("orders").insertMany(orders); + + // A view to exercise VIEW detection in listTables/listCollections. + await db.command({ + create: "active_users_view", + viewOn: "users", + pipeline: [{ $match: { active: true } }], + }); + + await client.close(); + console.log("MongoDB test database seeded successfully."); +} + +seed().catch((err) => { + console.error("Failed to seed MongoDB:", err); + process.exit(1); +}); diff --git a/bridge/src/connectors/mongo.ts b/bridge/src/connectors/mongo.ts new file mode 100644 index 00000000..a137c55c --- /dev/null +++ b/bridge/src/connectors/mongo.ts @@ -0,0 +1,3269 @@ +/** + * MongoDB connector (Milestone 1 foundation). + * + * Implements the connection / listing / statistics subset required to wire + * MongoDB into the existing structural connector registry: + * - client pool (per uri+database) + * - testConnection + * - listCollections (native, document-oriented) + * - listTables (relational compatibility adapter → TableInfo[]) + * - getDBStats (→ DBStats) + * - listSchemaNames (single pseudo-schema) + * - connectToDatabase (minimal; no baseline, no schema_migrations) + * - mongodbCache (satisfies the `${dbType}Cache` naming convention) + * + * Document-native operations (findDocuments / insertDocument / updateDocument / + * deleteDocument / runAggregation / inferCollectionSchema) and the row/table + * compatibility adapters land in a later milestone. They are intentionally NOT + * stubbed here. + * + * Credentials never appear in logs or error messages: connection errors are + * sanitized and the raw config is never logged. + */ + +import { + MongoClient, + Collection, + FindOptions, + AggregateOptions, + Document, + CreateIndexesOptions, + Db, + MongoClientOptions, +} from "mongodb"; +import { sanitizeMongoUri } from "../utils/mongoUri"; +import { createHash } from "crypto"; +import { + CursorCloseOutcome, + MongoExecutionContext, + MongoConfig, + MongoCollectionInfo, + MongoCollectionStats, + MongoBsonType, + MongoInferSchemaRequest, + MongoCollectionSchema, + MongoInferredField, + EJSONDocument, + EJsonValue, + MongoFindRequest, + MongoFindResult, + MongoInsertRequest, + MongoInsertResult, + MongoUpdateRequest, + MongoUpdateResult, + MongoDeleteRequest, + MongoDeleteResult, + MongoReplaceRequest, + MongoReplaceResult, + MongoCreateCollectionRequest, + MongoCreateCollectionResult, + MongoDropCollectionRequest, + MongoDropCollectionResult, + MongoAggregationRequest, + MongoAggregationResult, + MongoSortSpec, + MongoProjection, + MongoPageCacheKey, + MongoFindOpPayload, + MongoAggregateOpPayload, + MongoCountOpPayload, + MongoDistinctOpPayload, + MongoMqlExecuteRequest, + MongoMqlExecuteResult, + MongoExplainRequest, + MongoExplainResult, + MongoIndexCollation, + MongoIndexDetails, + MongoIndexListRequest, + MongoIndexListResult, + MongoIndexCreateRequest, + MongoIndexCreateResult, + MongoIndexDropRequest, + MongoIndexDropResult, + MongoIndexStatsItem, + MongoIndexStatsRequest, + MongoIndexStatsResult, + MongoIndexHideRequest, + MongoIndexHideResult, + MAX_PAGE_SIZE, + MAX_PAGE, + DEFAULT_PAGE_SIZE, + MAX_FILTER_SIZE_BYTES, + MAX_SORT_FIELDS, + MAX_PROJECTION_FIELDS, + MAX_AGG_STAGES, + MAX_AGG_STAGE_SIZE_BYTES, + MAX_AGG_RESULTS, + MAX_DISTINCT_VALUES, + MAX_EXPRESSION_DEPTH, + MAX_DOCUMENT_SIZE_BYTES, + DEFAULT_OP_TIMEOUT_MS, + MAX_OP_TIMEOUT_MS, + MAX_SAMPLE_SIZE, + MAX_INFERENCE_DEPTH, + MAX_INFERRED_FIELDS, + DEFAULT_SAMPLE_SIZE, + DEFAULT_MAX_DEPTH, +} from "../types/mongo"; +import { TableInfo, DBStats, SchemaInfo } from "../types/common"; +import { CacheEntry } from "../types/cache"; +import logger from "../services/logger"; +import { + serializeMongoDocument, + serializeMongoValue, + serializeMongoEJSONDocument, + serializeMongoFilter, + serializeMongoUpdateOperand, + serializeMongoPipelineStage, + mongoBSONtoEJSON, + bsonTypeOf, + bsonNumericKind, +} from "./mongoBson"; + +// --------------------------------------------------------------------------- +// Error sanitization — decision #8: never expose credentials in errors/logs. +// --------------------------------------------------------------------------- + +/** Redact any embedded credentials if a message ever echoes the connection URI. */ +export function safeMongoError(err: unknown): string { + const base = err instanceof Error ? err.message : String(err); + return base.replace(/(mongodb\+?srv?:\/\/)[^@\s]+@/gi, "$1@"); +} + +// --------------------------------------------------------------------------- +// CACHING +// --------------------------------------------------------------------------- + +const CACHE_TTL = 30_000; +const STATS_CACHE_TTL = 10_000; +const DOC_PAGE_TTL = 5_000; // M2 opt-in document-page cache (spec §14) +const SCHEMA_CACHE_TTL = 300_000; // M3 inferred schema cache (5 min, spec §4.4, §9) + +class MongoCacheManager { + private tableListCache = new Map>(); + private dbStatsCache = new Map>(); + // M2 collection-scoped caches (spec §14) + private collectionStatsCache = new Map>(); + private docPageCache = new Map>(); + // M3 inferred schema cache + private inferredSchemaCache = new Map< + string, + CacheEntry & { connHash: string; collection: string } + >(); + + private getConfigKey(cfg: MongoConfig): string { + return `${cfg.uri || ""}:${cfg.database || ""}`; + } + + /** Credential-free connection identifier for M2 query-result cache keys (spec §15). */ + private getConnHash(cfg: MongoConfig): string { + return createHash("sha256") + .update(cfg.uri || "") + .digest("hex"); + } + + private getSchemaCacheKey( + cfg: MongoConfig, + collection: string, + sampleSize: number, + maxDepth: number, + ): string { + return createHash("sha256") + .update( + `${this.getConnHash(cfg)}:${cfg.database || ""}:${collection}:${sampleSize}:${maxDepth}`, + ) + .digest("hex"); + } + + private isValid(entry: CacheEntry | undefined): boolean { + return !!entry && Date.now() - entry.timestamp < entry.ttl; + } + + getTableList(cfg: MongoConfig): TableInfo[] | null { + const entry = this.tableListCache.get(this.getConfigKey(cfg)); + return this.isValid(entry) ? entry!.data : null; + } + setTableList(cfg: MongoConfig, data: TableInfo[]): void { + this.tableListCache.set(this.getConfigKey(cfg), { + data, + timestamp: Date.now(), + ttl: CACHE_TTL, + }); + } + + getDBStats(cfg: MongoConfig): DBStats | null { + const entry = this.dbStatsCache.get(this.getConfigKey(cfg)); + return this.isValid(entry) ? entry!.data : null; + } + setDBStats(cfg: MongoConfig, data: DBStats): void { + this.dbStatsCache.set(this.getConfigKey(cfg), { + data, + timestamp: Date.now(), + ttl: STATS_CACHE_TTL, + }); + } + + // ---- M2: collection-scoped caches (spec §14 / D-M2-5) ---- + + getCollectionStats(cfg: MongoConfig, collection: string): MongoCollectionStats | null { + const entry = this.collectionStatsCache.get(`${this.getConfigKey(cfg)}:${collection}`); + return this.isValid(entry) ? entry!.data : null; + } + setCollectionStats(cfg: MongoConfig, collection: string, data: MongoCollectionStats): void { + this.collectionStatsCache.set(`${this.getConfigKey(cfg)}:${collection}`, { + data, + timestamp: Date.now(), + ttl: STATS_CACHE_TTL, + }); + } + + getDocumentPage(cfg: MongoConfig, key: MongoPageCacheKey): EJSONDocument[] | null { + const entry = this.docPageCache.get(`${this.getConnHash(cfg)}:${key}`); + return this.isValid(entry) ? entry!.data : null; + } + setDocumentPage(cfg: MongoConfig, key: MongoPageCacheKey, docs: EJSONDocument[]): void { + this.docPageCache.set(`${this.getConnHash(cfg)}:${key}`, { + data: docs, + timestamp: Date.now(), + ttl: DOC_PAGE_TTL, + }); + } + + getInferredSchema( + cfg: MongoConfig, + collection: string, + sampleSize: number, + maxDepth: number, + ): MongoCollectionSchema | null { + const key = this.getSchemaCacheKey(cfg, collection, sampleSize, maxDepth); + const entry = this.inferredSchemaCache.get(key); + return this.isValid(entry) ? entry!.data : null; + } + + setInferredSchema( + cfg: MongoConfig, + collection: string, + sampleSize: number, + maxDepth: number, + data: MongoCollectionSchema, + ): void { + const key = this.getSchemaCacheKey(cfg, collection, sampleSize, maxDepth); + this.inferredSchemaCache.set(key, { + data, + timestamp: Date.now(), + ttl: SCHEMA_CACHE_TTL, + connHash: this.getConnHash(cfg), + collection, + }); + } + + /** Evict a collection's stats + all its cached document pages (called on mutation). */ + clearCollectionCache(cfg: MongoConfig, collection: string): void { + this.collectionStatsCache.delete(`${this.getConfigKey(cfg)}:${collection}`); + const connHash = this.getConnHash(cfg); + for (const k of Array.from(this.docPageCache.keys())) { + if (k.startsWith(`${connHash}:`)) this.docPageCache.delete(k); + } + for (const [k, v] of Array.from(this.inferredSchemaCache.entries())) { + if (v.connHash === connHash && v.collection === collection) { + this.inferredSchemaCache.delete(k); + } + } + } + + clearForConnection(cfg: MongoConfig): void { + const key = this.getConfigKey(cfg); + for (const [k] of this.tableListCache) { + if (k.startsWith(key)) this.tableListCache.delete(k); + } + this.dbStatsCache.delete(key); + for (const k of Array.from(this.collectionStatsCache.keys())) { + if (k.startsWith(key)) this.collectionStatsCache.delete(k); + } + const connHash = this.getConnHash(cfg); + for (const k of Array.from(this.docPageCache.keys())) { + if (k.startsWith(`${connHash}:`)) this.docPageCache.delete(k); + } + for (const [k, v] of Array.from(this.inferredSchemaCache.entries())) { + if (v.connHash === connHash) { + this.inferredSchemaCache.delete(k); + } + } + } + + clearAll(): void { + this.tableListCache.clear(); + this.dbStatsCache.clear(); + this.collectionStatsCache.clear(); + this.docPageCache.clear(); + this.inferredSchemaCache.clear(); + } + + flushConnection(cfg: MongoConfig): void { + this.clearForConnection(cfg); + } +} + +export const mongodbCache = new MongoCacheManager(); + +// --------------------------------------------------------------------------- +// CLIENT CONNECTION POOL +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// CANONICAL CLIENT KEY FUNCTION (Section 4.3) +// --------------------------------------------------------------------------- + +export function getClientKey(cfg: { uri?: string; database?: string }): string { + const normalizedDb = (cfg.database || "").trim().toLowerCase(); + if (!cfg.uri) { + return `::${normalizedDb}`; + } + + try { + const raw = cfg.uri.trim(); + const isSrv = raw.toLowerCase().startsWith("mongodb+srv://"); + const scheme = isSrv ? "mongodb+srv" : "mongodb"; + const prefix = isSrv ? "mongodb+srv://" : "mongodb://"; + const rest = raw.slice(prefix.length); + + const qIdx = rest.indexOf("?"); + const hIdx = rest.indexOf("#"); + let endIdx = rest.length; + if (qIdx !== -1) endIdx = qIdx; + if (hIdx !== -1 && (qIdx === -1 || hIdx < qIdx)) endIdx = hIdx; + const authAndPath = rest.slice(0, endIdx); + + const slashIdx = authAndPath.indexOf("/"); + const authority = slashIdx !== -1 ? authAndPath.slice(0, slashIdx) : authAndPath; + + let userPart = ""; + let hostsPart = authority; + const atIdx = authority.lastIndexOf("@"); + if (atIdx !== -1) { + const creds = authority.slice(0, atIdx); + hostsPart = authority.slice(atIdx + 1); + const colonIdx = creds.indexOf(":"); + const rawUser = colonIdx !== -1 ? creds.slice(0, colonIdx) : creds; + if (rawUser) { + userPart = `${encodeURIComponent(decodeURIComponent(rawUser))}@`; + } + } + + const normalizedHosts = hostsPart + .split(",") + .map((hp) => { + const trimmed = hp.trim().toLowerCase(); + if (!trimmed) return ""; + if (isSrv) return trimmed; + if (trimmed.startsWith("[")) { + const closeBracket = trimmed.indexOf("]"); + if (closeBracket !== -1) { + const h = trimmed.slice(0, closeBracket + 1); + const after = trimmed.slice(closeBracket + 1); + if (after.startsWith(":")) { + const p = parseInt(after.slice(1), 10); + return `${h}:${isNaN(p) ? 27017 : p}`; + } + return `${h}:27017`; + } + } + const lastColon = trimmed.lastIndexOf(":"); + if (lastColon !== -1) { + const h = trimmed.slice(0, lastColon); + const p = trimmed.slice(lastColon + 1); + const portNum = parseInt(p, 10); + return `${h}:${isNaN(portNum) ? 27017 : portNum}`; + } + return `${trimmed}:27017`; + }) + .filter(Boolean) + .join(","); + + const normalizedUri = `${scheme}://${userPart}${normalizedHosts}`; + return `${normalizedUri}::${normalizedDb}`; + } catch { + const sanitizedUri = sanitizeMongoUri(cfg.uri); + return `${sanitizedUri}::${normalizedDb}`; + } +} + +// --------------------------------------------------------------------------- +// ATOMIC SHARED MONGO CLIENT REGISTRY & LIFECYCLE (Section 5) +// --------------------------------------------------------------------------- + +export type SocketTeardownStatus = "destroy_confirmed" | "destroy_requested" | "unconfirmed"; + +export interface SocketDestructionResult { + status: SocketTeardownStatus; + destroyedSockets: number; + totalDiscoveredSockets: number; + error?: Error; +} + +export interface ManagedMongoClient { + client: MongoClient; + state: "active" | "closing" | "close_failed" | "closed"; + closeError?: Error; + closePromise?: Promise; +} + +export const clientRegistry = new Map(); +export const closingClients = new Map>(); + +let isMongoShuttingDown = false; +export function setMongoShuttingDown(val: boolean): void { + isMongoShuttingDown = val; +} + +export const DEFAULT_MONGO_CLIENT_OPTIONS: MongoClientOptions = { + connectTimeoutMS: 5000, + socketTimeoutMS: 30000, + serverSelectionTimeoutMS: 5000, + maxIdleTimeMS: 60000, + maxPoolSize: 20, +}; + +export function getDriverOptions(_cfg?: MongoConfig): MongoClientOptions { + return { + ...DEFAULT_MONGO_CLIENT_OPTIONS, + }; +} + +export function forceDestroyMongoSockets(client: MongoClient): SocketDestructionResult { + let destroyedSockets = 0; + try { + const topology = (client as any).topology; + if (!topology) { + return { status: "destroy_confirmed", destroyedSockets: 0, totalDiscoveredSockets: 0 }; + } + + const servers = topology.s?.servers; + if (!(servers instanceof Map)) { + return { + status: "unconfirmed", + destroyedSockets: 0, + totalDiscoveredSockets: 0, + error: new Error("Unexpected MongoDB driver topology structure: servers Map not found"), + }; + } + + if (servers.size === 0) { + return { status: "destroy_confirmed", destroyedSockets: 0, totalDiscoveredSockets: 0 }; + } + + const uniqueConns = new Set(); + const uniqueSockets = new Set(); + + for (const server of servers.values()) { + const pool = server.pool || server.s?.pool; + if (pool) { + if (pool.connections && typeof pool.connections[Symbol.iterator] === "function") { + for (const conn of pool.connections) { + if (conn) uniqueConns.add(conn); + } + } + if (pool.checkedOut instanceof Set) { + for (const conn of pool.checkedOut) { + if (conn) uniqueConns.add(conn); + } + } + if (typeof pool.destroy === "function") { + try { + pool.destroy(); + } catch {} + } + } + } + + for (const conn of uniqueConns) { + if (conn?.socket && typeof conn.socket.destroy === "function") { + uniqueSockets.add(conn.socket); + } + } + + if (uniqueSockets.size === 0) { + return { status: "destroy_confirmed", destroyedSockets: 0, totalDiscoveredSockets: 0 }; + } + + const totalDiscoveredSockets = uniqueSockets.size; + let allDestroyedObserved = true; + + for (const socket of uniqueSockets) { + try { + if (!socket.destroyed) { + socket.destroy(); + } + destroyedSockets++; + if (!socket.destroyed) { + allDestroyedObserved = false; + } + } catch (sockErr) { + allDestroyedObserved = false; + logger.warn({ sockErr }, "Failed invoking destroy on individual MongoDB socket"); + } + } + + if (allDestroyedObserved && destroyedSockets === totalDiscoveredSockets) { + return { status: "destroy_confirmed", destroyedSockets, totalDiscoveredSockets }; + } + + return { status: "destroy_requested", destroyedSockets, totalDiscoveredSockets }; + } catch (err: any) { + logger.warn({ err: safeMongoError(err) }, "Error during forceDestroyMongoSockets traversal"); + return { status: "unconfirmed", destroyedSockets, totalDiscoveredSockets: 0, error: err }; + } +} + +export function closeMongoClientByKey( + clientKey: string, + options?: { force?: boolean; timeoutMs?: number; globalDeadline?: number }, +): Promise { + const existingClose = closingClients.get(clientKey); + const entry = clientRegistry.get(clientKey); + + if (!entry || entry.state === "closed") { + return Promise.resolve(); + } + + const remainingMs = + options?.globalDeadline !== undefined + ? Math.max(0, options.globalDeadline - Date.now()) + : options?.timeoutMs !== undefined + ? options.timeoutMs + : options?.force + ? 100 + : undefined; + + if (existingClose) { + if (remainingMs === undefined) { + return existingClose; + } + + return (async () => { + let timer: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("Close timeout")), remainingMs); + }); + + try { + await Promise.race([existingClose, timeoutPromise]); + } catch (err: any) { + logger.warn( + { clientKey, err: safeMongoError(err) }, + "In-flight MongoClient close exceeded deadline; escalating to forced socket destruction", + ); + const result = forceDestroyMongoSockets(entry.client); + if (result.status === "destroy_confirmed") { + entry.state = "closed"; + entry.closeError = undefined; + clientRegistry.delete(clientKey); + } else { + entry.state = "close_failed"; + entry.closeError = + result.error || new Error(`Forced socket destruction state: ${result.status}`); + logger.error( + { clientKey, status: result.status, err: safeMongoError(entry.closeError) }, + "Forced socket destruction not confirmed; retaining MongoClient in close_failed state", + ); + } + } finally { + if (timer) clearTimeout(timer); + } + })(); + } + + entry.state = "closing"; + + const closePromise = (async () => { + let timer: NodeJS.Timeout | undefined; + try { + const driverClosePromise = entry.client.close(options?.force); + if (remainingMs !== undefined) { + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("Close timeout")), remainingMs); + }); + await Promise.race([driverClosePromise, timeoutPromise]); + } else { + await driverClosePromise; + } + + entry.state = "closed"; + entry.closeError = undefined; + clientRegistry.delete(clientKey); + } catch (err: any) { + logger.warn( + { clientKey, err: safeMongoError(err) }, + "MongoClient.close() rejected or timed out; escalating to forced socket destruction", + ); + const result = forceDestroyMongoSockets(entry.client); + if (result.status === "destroy_confirmed") { + entry.state = "closed"; + entry.closeError = undefined; + clientRegistry.delete(clientKey); + } else { + entry.state = "close_failed"; + entry.closeError = result.error || err; + logger.error( + { err: safeMongoError(entry.closeError), status: result.status, clientKey }, + "Failed to confirm MongoClient socket destruction; retained in close_failed state for recovery/watchdog", + ); + throw err; + } + } finally { + if (timer) clearTimeout(timer); + closingClients.delete(clientKey); + entry.closePromise = undefined; + } + })(); + + entry.closePromise = closePromise; + closingClients.set(clientKey, closePromise); + return closePromise; +} + +export function closeMongoClient(cfg: MongoConfig): Promise { + return closeMongoClientByKey(getClientKey(cfg)); +} + +export async function closeAllMongoClients(options?: { + force?: boolean; + timeoutMs?: number; + globalDeadline?: number; +}): Promise { + const globalDeadline = options?.globalDeadline ?? Date.now() + (options?.timeoutMs ?? 100); + const remainingBudget = Math.max(0, globalDeadline - Date.now()); + + let globalTimer: NodeJS.Timeout | undefined; + const globalTimeoutPromise = new Promise((resolve) => { + globalTimer = setTimeout(resolve, remainingBudget); + }); + + const keys = Array.from(clientRegistry.keys()); + const batchClosePromise = Promise.allSettled( + keys.map((key) => + closeMongoClientByKey(key, { + force: options?.force, + globalDeadline, + }), + ), + ); + + await Promise.race([batchClosePromise, globalTimeoutPromise]); + if (globalTimer) clearTimeout(globalTimer); + + for (const [key, entry] of clientRegistry.entries()) { + if (entry.state !== "closed") { + const result = forceDestroyMongoSockets(entry.client); + if (result.status === "destroy_confirmed") { + entry.state = "closed"; + entry.closeError = undefined; + clientRegistry.delete(key); + } else { + entry.state = "close_failed"; + if (!entry.closeError) { + entry.closeError = + result.error || new Error(`Forced socket destruction state: ${result.status}`); + } + } + } + } +} + +export function getMongoClientSync(cfg: MongoConfig): MongoClient { + if (isMongoShuttingDown) { + throw new Error("Bridge is shutting down: MongoClient creation rejected"); + } + + const key = getClientKey(cfg); + let entry = clientRegistry.get(key); + + if (entry && entry.state === "close_failed") { + throw new Error( + `Cannot create MongoClient: prior client for key '${key}' failed to force-close (${entry.closeError?.message || entry.closeError}). Resource ownership retained in registry; manual recovery or bridge restart required.`, + ); + } + + if (entry && entry.state === "closing") { + throw new Error(`MongoClient for key '${key}' is currently closing.`); + } + + if (entry && entry.state === "active") { + return entry.client; + } + + if (!entry || entry.state === "closed") { + if (!cfg.uri) throw new Error("MongoDB connection requires a connection URI"); + const client = new MongoClient(cfg.uri, getDriverOptions(cfg)); + entry = { client, state: "active" }; + clientRegistry.set(key, entry); + } + + return entry.client; +} + +export async function getMongoClient(cfg: MongoConfig): Promise { + if (isMongoShuttingDown) { + throw new Error("Bridge is shutting down: MongoClient creation rejected"); + } + + const key = getClientKey(cfg); + + const pendingClose = closingClients.get(key); + if (pendingClose) { + try { + await pendingClose; + } catch {} + } + + let entry = clientRegistry.get(key); + if (entry && entry.state === "close_failed") { + try { + await closeMongoClientByKey(key, { force: true, timeoutMs: 100 }); + } catch (err: any) { + throw new Error( + `Cannot create MongoClient: prior client for key '${key}' failed to force-close (${err?.message || err}). Resource ownership retained in registry; manual recovery or bridge restart required.`, + ); + } + entry = clientRegistry.get(key); + if (entry && entry.state === "close_failed") { + throw new Error( + `Cannot create MongoClient: prior client for key '${key}' failed to force-close. Resource ownership retained in registry; manual recovery or bridge restart required.`, + ); + } + } + + if (entry && entry.state === "active") { + return entry.client; + } + + if (!entry || entry.state === "closed") { + if (!cfg.uri) throw new Error("MongoDB connection requires a connection URI"); + const client = new MongoClient(cfg.uri, getDriverOptions(cfg)); + entry = { client, state: "active" }; + clientRegistry.set(key, entry); + } + + return entry.client; +} + +/** Select the target database, defaulting to the configured one. */ +export function selectDb(cfg: MongoConfig): Db { + const client = getMongoClientSync(cfg); + return (cfg.database ? client.db(cfg.database) : client.db()) as Db; +} + +// --------------------------------------------------------------------------- +// READ-ONLY & SAFE EXPRESSION GUARDS (Section 7 & 8) +// --------------------------------------------------------------------------- + +export function assertMongoNotReadOnly(conn: MongoConfig, operation: string): void { + if (conn.readonly || (conn as any).readOnly) { + throw new MongoApiError( + "MONGO_READ_ONLY", + `Database is configured as read-only. '${operation}' is prohibited.`, + ); + } +} + +// --------------------------------------------------------------------------- +// CORE CONNECTOR OPERATIONS (M1 subset) +// --------------------------------------------------------------------------- + +export async function testConnection( + cfg: MongoConfig, +): Promise<{ ok: boolean; message?: string; status: "connected" | "disconnected" }> { + if (!cfg.uri) { + return { ok: false, status: "disconnected", message: "Missing connection URI" }; + } + + const tempClient = new MongoClient(cfg.uri, getDriverOptions(cfg)); + try { + const targetDb = cfg.database ? tempClient.db(cfg.database) : tempClient.db(); + await targetDb.command({ ping: 1 }); + return { ok: true, status: "connected", message: "Connection successful" }; + } catch (err) { + return { ok: false, status: "disconnected", message: safeMongoError(err) }; + } finally { + try { + await tempClient.close(); + } catch { + // ignore + } + } +} + +/** Native: list collections (and views) in the configured database. */ +export async function listCollections(cfg: MongoConfig): Promise { + const cached = mongodbCache.getTableList(cfg); + if (cached) { + // Derive collection info from the cached relational adapter view. + return cached.map((t) => ({ name: t.name, type: t.type === "VIEW" ? "view" : "collection" })); + } + + const db = selectDb(cfg); + const infos = await db.listCollections({}, { nameOnly: true }).toArray(); + + const result: MongoCollectionInfo[] = infos.map((c) => { + const isView = c.type === "view"; + return { name: c.name, type: isView ? "view" : "collection" }; + }); + + // Prime the table-list cache from the native result. + mongodbCache.setTableList( + cfg, + result.map((c) => ({ + schema: cfg.database || "default", + name: c.name, + type: c.type === "view" ? "VIEW" : "BASE TABLE", + })), + ); + return result; +} + +/** + * Relational compatibility adapter: map collections → `TableInfo[]` so the + * existing `db.listTables` RPC and schema tree keep working. The relational + * `schema` filter is ignored (Mongo is not schema-namespaced). + */ +export async function listTables(cfg: MongoConfig, _schema?: string): Promise { + const cached = mongodbCache.getTableList(cfg); + if (cached) return cached; + + const collections = await listCollections(cfg); + const tables: TableInfo[] = collections.map((c) => ({ + schema: cfg.database || "default", + name: c.name, + type: c.type === "view" ? "VIEW" : "BASE TABLE", + })); + mongodbCache.setTableList(cfg, tables); + return tables; +} + +/** Basic database statistics mapped onto the shared `DBStats` shape. */ +export async function getDBStats(cfg: MongoConfig): Promise { + const cached = mongodbCache.getDBStats(cfg); + if (cached) return cached; + + const db = selectDb(cfg); + const stats = await db.stats(); + + let total_tables = 0; + try { + const collInfos = await db.listCollections({}, { nameOnly: true }).toArray(); + total_tables = collInfos.length; + } catch (err) { + logger.warn({ err: safeMongoError(err) }, "MongoDB countCollections failed"); + } + + const total_rows = typeof stats.objects === "number" ? stats.objects : 0; + const total_db_size_mb = + typeof stats.size === "number" ? Math.round((stats.size / (1024 * 1024)) * 100) / 100 : 0; + + const result: DBStats = { total_tables, total_rows, total_db_size_mb }; + mongodbCache.setDBStats(cfg, result); + return result; +} + +/** Single pseudo-schema (the database name) for the existing `db.listSchemas`. */ +export async function listSchemaNames(cfg: MongoConfig): Promise { + return [cfg.database || "default"]; +} + +/** + * Minimal connect bootstrap. Returns collections as pseudo-schemas and empty + * migration lists. No baseline and no `schema_migrations` — MongoDB migrations + * are out of scope for this milestone. + */ +export async function connectToDatabase( + cfg: MongoConfig, + _connectionId: string, + _options?: { readOnly?: boolean }, +): Promise<{ + baselined: boolean; + schema: SchemaInfo[]; + migrations: { local: unknown[]; applied: unknown[] }; +}> { + const collections = await listCollections(cfg); + const schema: SchemaInfo[] = collections.map((c) => ({ name: c.name })); + return { + baselined: false, + schema, + migrations: { local: [], applied: [] }, + }; +} + +// =========================================================================== +// M2 — MongoDB Document Engine (findDocuments / insertDocument / updateDocument +// / deleteDocument / runAggregation). See docs/MONGODB_M2_SPEC.md. +// +// Document-native operations, distinct from the M1 row/table compatibility +// adapters above. They run through a small MongoConnector class so the +// executor/handler layer has one typed surface to call (spec §13). Errors map to +// the stable MongoApiError codes (spec §11) and never carry credentials (§15). +// =========================================================================== + +/** Stable JSON encoding with sorted keys, for size measurement + cache keys. */ +function canonicalJSON(value: unknown): string { + const sortKeys = (v: unknown): unknown => { + if (v && typeof v === "object" && !Array.isArray(v)) { + const obj = v as Record; + const sorted: Record = {}; + for (const k of Object.keys(obj).sort()) sorted[k] = sortKeys(obj[k]); + return sorted; + } + if (Array.isArray(v)) return v.map(sortKeys); + return v; + }; + return JSON.stringify(sortKeys(value)); +} + +function jsonByteSize(value: unknown): number { + return Buffer.byteLength(canonicalJSON(value), "utf8"); +} + +/** Stable, credential-free error carrying a M2 error code (spec §11). */ +export class MongoApiError extends Error { + constructor( + public readonly code: string, + message: string, + ) { + super(message); + this.name = "MongoApiError"; + } +} + +export function assertCollectionName(collection: unknown): asserts collection is string { + if (typeof collection !== "string" || collection.length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection must be a non-empty string"); + } + if (Buffer.byteLength(collection, "utf8") > 120) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection name must not exceed 120 bytes"); + } + if (collection.includes("$") || collection.includes("\0")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not contain '$' or null bytes", + ); + } + if (collection.startsWith("system.")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not start with 'system.'", + ); + } +} + +export function assertNewCollectionName(collection: unknown): asserts collection is string { + if (typeof collection !== "string" || collection.length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection must be a non-empty string"); + } + if (collection.length > 120) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not exceed 120 characters", + ); + } + if (collection.includes("$") || collection.includes("\0")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not contain '$' or null bytes", + ); + } + if (collection.startsWith("system.")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not start with 'system.'", + ); + } + const VALID_NAME_REGEX = /^[a-zA-Z_][a-zA-Z0-9_-]*$/; + if (!VALID_NAME_REGEX.test(collection)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `collection name '${collection}' is invalid; must start with a letter or underscore and contain only alphanumeric, underscores, or hyphens`, + ); + } +} + +export function assertNoOperatorKeys(doc: Record): void { + for (const key of Object.keys(doc)) { + if (key.startsWith("$")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "replacement document must not contain top-level operator keys", + ); + } + } +} + +function assertSize(value: unknown, max: number, what: string): void { + if (jsonByteSize(value) > max) { + throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `${what} exceeds the ${max}-byte limit`); + } +} + +function validateSort(sort: MongoSortSpec | undefined): Record | undefined { + if (sort === undefined || sort === null) return undefined; + if (typeof sort !== "object" || Array.isArray(sort)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "sort must be an object"); + } + const entries = Object.entries(sort); + if (entries.length > MAX_SORT_FIELDS) { + throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `sort exceeds ${MAX_SORT_FIELDS} fields`); + } + const out: Record = {}; + for (const [field, dir] of entries) { + if (field.includes("$") || field.startsWith(".")) { + throw new MongoApiError("MONGO_INVALID_FILTER", `invalid sort field '${field}'`); + } + if (dir === 1 || dir === "ASC") out[field] = 1; + else if (dir === -1 || dir === "DESC") out[field] = -1; + else throw new MongoApiError("MONGO_INVALID_FILTER", `invalid sort direction for '${field}'`); + } + return out; +} + +function validateProjection( + projection: MongoProjection | undefined, +): Record | undefined { + if (projection === undefined || projection === null) return undefined; + if (typeof projection !== "object" || Array.isArray(projection)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "projection must be an object"); + } + const entries = Object.entries(projection); + if (entries.length > MAX_PROJECTION_FIELDS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `projection exceeds ${MAX_PROJECTION_FIELDS} fields`, + ); + } + const includes: string[] = []; + const excludes: string[] = []; + const out: Record = {}; + for (const [field, mode] of entries) { + if (mode === 1 || mode === true) { + out[field] = 1; + includes.push(field); + } else if (mode === 0 || mode === false) { + out[field] = 0; + excludes.push(field); + } else { + throw new MongoApiError("MONGO_INVALID_FILTER", `invalid projection mode for '${field}'`); + } + } + if (includes.length > 0 && excludes.filter((f) => f !== "_id").length > 0) { + throw new MongoApiError( + "MONGO_INVALID_FILTER", + "cannot mix inclusion and exclusion (except _id)", + ); + } + return out; +} + +function assertNonEmptyFilter(filter: EJSONDocument): void { + if (typeof filter !== "object" || filter === null || Array.isArray(filter)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + if (Object.keys(filter).length === 0) { + throw new MongoApiError( + "MONGO_BULK_OP_FORBIDDEN", + "update/delete requires a non-empty filter to prevent accidental whole-collection operations", + ); + } +} + +function assertOperatorDocument(update: EJSONDocument): void { + if (typeof update !== "object" || update === null || Array.isArray(update)) { + throw new MongoApiError("MONGO_INVALID_UPDATE_OPERAND", "update must be an operator document"); + } + const keys = Object.keys(update); + if (keys.length === 0) { + throw new MongoApiError("MONGO_INVALID_UPDATE_OPERAND", "No update operators provided"); + } + for (const key of keys) { + if (!key.startsWith("$")) { + throw new MongoApiError( + "MONGO_INVALID_UPDATE_OPERAND", + `'${key}' is not an update operator (full-document replacement is not supported)`, + ); + } + } +} + +export const FORBIDDEN_PIPELINE_STAGES = new Set(["$out", "$merge"]); +export const FORBIDDEN_EXPRESSION_OPS = new Set(["$function", "$accumulator", "$where"]); + +/** + * Recursively scans an object or array for forbidden expression operators ($function, $accumulator, $where). + */ +export function assertSafeExpressions(value: unknown, depth = 0): void { + if (depth > MAX_EXPRESSION_DEPTH) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `Expression nesting exceeds maximum depth of ${MAX_EXPRESSION_DEPTH}`, + ); + } + if (!value || typeof value !== "object") return; + + if (Array.isArray(value)) { + for (const item of value) { + assertSafeExpressions(item, depth + 1); + } + return; + } + + for (const [key, val] of Object.entries(value)) { + if (FORBIDDEN_EXPRESSION_OPS.has(key)) { + throw new MongoApiError( + "MONGO_FORBIDDEN_OPERATOR", + `Operator '${key}' is forbidden (server-side JavaScript execution is not permitted)`, + ); + } + assertSafeExpressions(val, depth + 1); + } +} + +/** + * Recursively validates aggregation pipeline stages and nested sub-pipelines. + * Permits empty pipelines aggregate([]) as valid read operations. + */ +export function assertReadOnlyPipeline(pipeline: unknown[], depth = 0): void { + if (depth > MAX_EXPRESSION_DEPTH) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `Pipeline nesting exceeds maximum depth of ${MAX_EXPRESSION_DEPTH}`, + ); + } + if (!Array.isArray(pipeline)) { + throw new MongoApiError("MONGO_INVALID_PIPELINE", "Pipeline must be an array"); + } + // Empty pipeline is valid in MongoDB; returns all documents + if (pipeline.length === 0) { + return; + } + if (pipeline.length > MAX_AGG_STAGES) { + throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `Pipeline exceeds ${MAX_AGG_STAGES} stages`); + } + + for (const stage of pipeline) { + if (typeof stage !== "object" || stage === null || Array.isArray(stage)) { + throw new MongoApiError( + "MONGO_INVALID_PIPELINE", + "Each pipeline stage must be a non-null object", + ); + } + const keys = Object.keys(stage); + if (keys.length !== 1) { + throw new MongoApiError( + "MONGO_INVALID_PIPELINE", + "Each pipeline stage must have exactly one operator key", + ); + } + const stageName = keys[0]; + if (!stageName.startsWith("$")) { + throw new MongoApiError( + "MONGO_INVALID_PIPELINE", + `Invalid stage name '${stageName}'; must begin with '$'`, + ); + } + if (FORBIDDEN_PIPELINE_STAGES.has(stageName)) { + throw new MongoApiError( + "MONGO_WRITE_STAGE_FORBIDDEN", + `Stage '${stageName}' is forbidden in read-only aggregation queries`, + ); + } + + assertSize(stage, MAX_AGG_STAGE_SIZE_BYTES, `Pipeline stage ${stageName}`); + + // Scan all expressions inside this stage for forbidden operators + const stageBody = (stage as Record)[stageName]; + assertSafeExpressions(stageBody, depth + 1); + + // Recurse into known nested sub-pipeline containers + if ( + stageName === "$lookup" && + stageBody && + typeof stageBody === "object" && + !Array.isArray(stageBody) + ) { + if (stageBody.pipeline !== undefined) { + if (!Array.isArray(stageBody.pipeline)) { + throw new MongoApiError("MONGO_INVALID_PIPELINE", "$lookup.pipeline must be an array"); + } + assertReadOnlyPipeline(stageBody.pipeline, depth + 1); + } + } + if ( + stageName === "$unionWith" && + stageBody && + typeof stageBody === "object" && + !Array.isArray(stageBody) + ) { + if (stageBody.pipeline !== undefined) { + if (!Array.isArray(stageBody.pipeline)) { + throw new MongoApiError("MONGO_INVALID_PIPELINE", "$unionWith.pipeline must be an array"); + } + assertReadOnlyPipeline(stageBody.pipeline, depth + 1); + } + } + if (stageName === "$facet") { + if (typeof stageBody !== "object" || stageBody === null || Array.isArray(stageBody)) { + throw new MongoApiError( + "MONGO_INVALID_PIPELINE", + "$facet stage specification must be an object", + ); + } + for (const [facetName, facetPipeline] of Object.entries(stageBody)) { + if (!Array.isArray(facetPipeline)) { + throw new MongoApiError( + "MONGO_INVALID_PIPELINE", + `$facet pipeline for '${facetName}' must be an array`, + ); + } + assertReadOnlyPipeline(facetPipeline as EJSONDocument[], depth + 1); + } + } + } +} + +export function validatePipeline(pipeline: unknown[]): unknown[] { + assertReadOnlyPipeline(pipeline); + return pipeline; +} + +export function clampMqlTimeout(ms: number | undefined, defaultMs: number, maxMs = 60000): number { + if (ms === undefined || ms === null) { + return defaultMs; + } + if (typeof ms !== "number" || !Number.isFinite(ms) || !Number.isInteger(ms) || ms <= 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "maxTimeMS must be a positive integer"); + } + return Math.min(Math.max(1, ms), maxMs); +} + +export const WHITELIST_READ_OPS = new Set([ + "find", + "aggregate", + "countDocuments", + "estimatedDocumentCount", + "distinct", +]); + +function clampPageSize(pageSize: number | undefined): number { + const n = + typeof pageSize === "number" && Number.isFinite(pageSize) + ? Math.floor(pageSize) + : DEFAULT_PAGE_SIZE; + if (n < 1) return 1; + if (n > MAX_PAGE_SIZE) return MAX_PAGE_SIZE; + return n; +} + +function clampPage(page: number | undefined): number { + const n = typeof page === "number" && Number.isFinite(page) ? Math.floor(page) : 1; + return n < 1 ? 1 : n; +} + +function clampTimeout(ms?: number): number { + const base = typeof ms === "number" && ms > 0 ? ms : DEFAULT_OP_TIMEOUT_MS; + return Math.min(base, MAX_OP_TIMEOUT_MS); +} + +// ---- M3 Clamping Helpers (docs/MONGODB_M3_SPEC.md §4.4, §4.6) ---- + +export function clampInferSampleSize(size?: number): number { + if (typeof size !== "number" || isNaN(size)) return DEFAULT_SAMPLE_SIZE; + return Math.min(Math.max(1, Math.floor(size)), MAX_SAMPLE_SIZE); +} + +export function clampInferMaxDepth(depth?: number): number { + if (typeof depth !== "number" || isNaN(depth)) return DEFAULT_MAX_DEPTH; + return Math.min(Math.max(1, Math.floor(depth)), MAX_INFERENCE_DEPTH); +} + +function isPlainRecord(v: unknown): v is Record { + return ( + typeof v === "object" && + v !== null && + !Array.isArray(v) && + !(v instanceof Date) && + !Buffer.isBuffer(v) && + !(v instanceof Uint8Array) && + !(v as any)._bsontype + ); +} + +interface FieldAccumulatorStats { + path: string; + depth: number; + parentPath?: string; + isArray: boolean; + isNested: boolean; + count: number; + nullCount: number; + typeCounts: Map; + numericKinds: Set<"int" | "double">; + arrayElementTypes: Set; + sampleValue?: EJSONDocument | any; + maxChildDepth?: number; + truncated?: boolean; +} + +/** + * Pure statistical accumulator over driver-native documents for schema inference. + * Never makes network or driver calls (docs/MONGODB_M3_SPEC.md §4.4). + */ +export class FieldAccumulator { + private fieldMap = new Map(); + + constructor( + public readonly sampleSize: number, + public readonly maxDepth: number, + ) {} + + ingest(doc: Document | Record, parentPath = "", depth = 0): void { + if (!doc || typeof doc !== "object" || Array.isArray(doc)) return; + + for (const [key, value] of Object.entries(doc)) { + if (key === "_bsontype") continue; + + const path = parentPath ? `${parentPath}.${key}` : key; + + if (!this.fieldMap.has(path)) { + if (this.fieldMap.size >= MAX_INFERRED_FIELDS) { + if (parentPath && this.fieldMap.has(parentPath)) { + this.fieldMap.get(parentPath)!.truncated = true; + } + continue; + } + this.fieldMap.set(path, { + path, + depth, + parentPath: parentPath || undefined, + isArray: false, + isNested: false, + count: 0, + nullCount: 0, + typeCounts: new Map(), + numericKinds: new Set<"int" | "double">(), + arrayElementTypes: new Set(), + }); + } + + const stats = this.fieldMap.get(path)!; + + if (parentPath) { + let p: string | undefined = parentPath; + while (p) { + const ancestor = this.fieldMap.get(p); + if (ancestor) { + ancestor.maxChildDepth = Math.max(ancestor.maxChildDepth ?? 0, depth); + p = ancestor.parentPath; + } else { + break; + } + } + } + + const bType = bsonTypeOf(value); + stats.typeCounts.set(bType, (stats.typeCounts.get(bType) || 0) + 1); + + if (value !== undefined) { + stats.count++; + } + if (value === null) { + stats.nullCount++; + } + + if (stats.sampleValue === undefined && value !== undefined && value !== null) { + stats.sampleValue = serializeMongoValue(value); + } + + if (bType === "number") { + const numKind = bsonNumericKind(value); + if (numKind) { + stats.numericKinds.add(numKind); + } + } + + if (Array.isArray(value)) { + stats.isArray = true; + for (const item of value) { + stats.arrayElementTypes.add(bsonTypeOf(item)); + } + } else if (isPlainRecord(value)) { + stats.isNested = true; + if (depth < this.maxDepth) { + this.ingest(value, path, depth + 1); + } else { + stats.truncated = true; + } + } + } + } + + toFields(): MongoInferredField[] { + const fields: MongoInferredField[] = []; + + for (const stats of this.fieldMap.values()) { + const sortedTypes = Array.from(stats.typeCounts.entries()).sort((a, b) => { + if (b[1] !== a[1]) return b[1] - a[1]; + if (a[0] === "null" || a[0] === "undefined") return 1; + if (b[0] === "null" || b[0] === "undefined") return -1; + return a[0].localeCompare(b[0]); + }); + + const dominantType = sortedTypes.length > 0 ? sortedTypes[0][0] : "null"; + const types = sortedTypes.map(([t]) => t); + + const typeConflicts = + types.length > 1 ? sortedTypes.map(([bsonType, count]) => ({ bsonType, count })) : []; + + const presencePct = + this.sampleSize > 0 ? Math.round((stats.count / this.sampleSize) * 1000) / 10 : 0; + + const field: MongoInferredField = { + path: stats.path, + bsonType: dominantType, + types, + numericKinds: + stats.numericKinds.size > 0 + ? (Array.from(stats.numericKinds).sort() as Array<"int" | "double">) + : undefined, + depth: stats.depth, + parentPath: stats.parentPath, + isArray: stats.isArray, + isNested: stats.isNested, + count: stats.count, + nullCount: stats.nullCount, + sampleSize: this.sampleSize, + presencePct, + nullable: stats.nullCount > 0, + optional: stats.count < this.sampleSize, + typeConflicts, + arrayElementTypes: stats.isArray ? Array.from(stats.arrayElementTypes).sort() : undefined, + sampleValue: stats.sampleValue, + maxChildDepth: stats.maxChildDepth, + truncated: stats.truncated ? true : undefined, + }; + + fields.push(field); + } + + fields.sort((a, b) => { + if (a.depth !== b.depth) return a.depth - b.depth; + return a.path.localeCompare(b.path); + }); + + return fields; + } +} + +function isEmptyObject(obj: unknown): boolean { + return ( + typeof obj === "object" && obj !== null && !Array.isArray(obj) && Object.keys(obj).length === 0 + ); +} + +/** + * Run a driver operation under a client-side timeout. On expiry the in-flight + * driver promise is aborted and a MONGO_OPERATION_TIMEOUT error is thrown + * (spec §12). Mirrors the AbortController pattern of the SQL streamQueryCancelable. + */ +export function withMongoTimeout( + ms: number, + fn: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(); + reject( + new MongoApiError("MONGO_OPERATION_TIMEOUT", `MongoDB operation timed out after ${ms}ms`), + ); + }, ms); + }); + const run = (async () => { + try { + return await fn(controller.signal); + } finally { + if (timer) clearTimeout(timer); + } + })(); + run.catch(() => {}); // avoid an unhandled rejection if the timeout wins the race + return Promise.race([run, timeout]); +} + +/** Defensive: strip any credential fragments from an error message (spec §11/§15). */ +function stripCredentialTokens(message: string): string { + return message + .replace(/password[=:]\S+/gi, "password=") + .replace(/\bpasswd\b[=:]\S+/gi, "passwd="); +} + +/** Map a driver error to a stable, credential-free M2 error (spec §11). */ +export function mapMongoError(err: unknown): MongoApiError { + if (err instanceof MongoApiError) return err; + const anyErr = err as { codeName?: string; code?: number; name?: string; message?: string }; + const codeName = anyErr?.codeName; + const code = anyErr?.code; + let mapped: { code: string; message: string }; + if (anyErr?.name === "MongoServerError") { + if (codeName === "NamespaceNotFound" || code === 26) { + mapped = { code: "MONGO_NS_NOT_FOUND", message: "Namespace not found" }; + } else if (code === 11000 || code === 11001) { + mapped = { code: "MONGO_DUPLICATE_KEY", message: "Duplicate key" }; + } else if (codeName === "AuthenticationFailed" || code === 18) { + mapped = { code: "MONGO_AUTH_ERROR", message: "Authentication failed" }; + } else if (codeName === "MaxTimeMSExpired" || code === 50) { + mapped = { code: "MONGO_OPERATION_TIMEOUT", message: "Operation timed out" }; + } else { + mapped = { code: "MONGO_UNKNOWN_ERROR", message: safeMongoError(anyErr?.message) }; + } + } else { + mapped = { code: "MONGO_CONNECTION_ERROR", message: safeMongoError(err) }; + } + return new MongoApiError(mapped.code, stripCredentialTokens(mapped.message)); +} + +function selectCollection(config: MongoConfig, collection: string): Collection { + return selectDb(config).collection(collection); +} + +/** Whether a collection (or view) exists in the configured database. */ +export async function collectionExists(config: MongoConfig, collection: string): Promise { + const db = selectDb(config); + const infos = await db.listCollections({ name: collection }, { nameOnly: true }).toArray(); + return infos.length > 0; +} + +/** Credential-free cache key for a document page (spec §15). */ +function pageCacheKey( + collection: string, + filter: EJSONDocument, + sort: Record | undefined, + projection: Record | undefined, + page: number, + pageSize: number, +): string { + const canonical = JSON.stringify([ + collection, + filter, + sort ?? null, + projection ?? null, + page, + pageSize, + ]); + return createHash("sha256").update(canonical).digest("hex"); +} + +function buildFindResult( + collection: string, + documents: EJSONDocument[], + total: number, + page: number, + pageSize: number, +): MongoFindResult { + const totalPages = total === 0 ? 0 : Math.ceil(total / pageSize); + return { + collection, + documents, + total, + page, + pageSize, + totalPages, + hasMore: page * pageSize < total, + }; +} + +/** + * BSON deserialization options that keep numeric/binary values in their typed + * form (int32 -> Int32, int64 -> Long, double -> Double, binary -> Binary) + * instead of promoting them to plain JS numbers. Without this the driver + * collapses every numeric BSON value to a JS number, so the EJSON converter + * could not emit $numberInt / $numberLong / $numberDouble and the documented + * round-trip (§7.1) would silently lose type information. + */ +const PRESERVE_BSON_TYPES = { + promoteValues: false, + promoteLongs: false, + promoteBuffers: false, +} as const; + +export class MongoConnector { + async findDocuments(config: MongoConfig, req: MongoFindRequest): Promise { + assertCollectionName(req.collection); + const pageSize = clampPageSize(req.pageSize); + const page = clampPage(req.page); + if (page > MAX_PAGE) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `page ${page} exceeds the maximum of ${MAX_PAGE} (use sort by _id + range for deeper pagination)`, + ); + } + if ( + req.filter !== undefined && + (typeof req.filter !== "object" || req.filter === null || Array.isArray(req.filter)) + ) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + const filter: EJSONDocument = req.filter ?? {}; + assertSize(filter, MAX_FILTER_SIZE_BYTES, "filter"); + if (req.filter) assertSafeExpressions(req.filter); + const sort = validateSort(req.sort); + const projection = validateProjection(req.projection); + const bsonFilter = serializeMongoFilter(filter); + const skip = Math.max(0, (page - 1) * pageSize); + + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const nativeDocs = await withMongoTimeout(timeout, async (signal) => { + const options: FindOptions & { signal?: AbortSignal } = { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }; + if (sort) options.sort = sort; + if (projection) options.projection = projection; + const cursor = coll.find(bsonFilter, options).skip(skip).limit(pageSize); + return (await cursor.toArray()) as unknown[]; + }); + + const total = isEmptyObject(filter) + ? await coll.estimatedDocumentCount({ maxTimeMS: timeout }) + : await coll.countDocuments(bsonFilter, { maxTimeMS: timeout }); + + if (config.cacheDocuments) { + const key = pageCacheKey(req.collection, filter, sort, projection, page, pageSize); + const cached = mongodbCache.getDocumentPage(config, key); + if (cached) return buildFindResult(req.collection, cached, total, page, pageSize); + } + + const documents = nativeDocs.map((d) => mongoBSONtoEJSON(d) as EJSONDocument); + + if (config.cacheDocuments) { + const key = pageCacheKey(req.collection, filter, sort, projection, page, pageSize); + mongodbCache.setDocumentPage(config, key, documents); + } + + return buildFindResult(req.collection, documents, total, page, pageSize); + } + + async insertDocument(config: MongoConfig, req: MongoInsertRequest): Promise { + assertMongoNotReadOnly(config, "insertDocument"); + assertCollectionName(req.collection); + if (typeof req.document !== "object" || req.document === null || Array.isArray(req.document)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "document must be an object"); + } + assertSize(req.document, MAX_DOCUMENT_SIZE_BYTES, "document"); + assertSafeExpressions(req.document); + const doc = serializeMongoEJSONDocument(req.document); + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const res = await withMongoTimeout(timeout, () => coll.insertOne(doc, { maxTimeMS: timeout })); + + const insertedId = mongoBSONtoEJSON(res.insertedId); + const echoed = mongoBSONtoEJSON({ ...doc, _id: res.insertedId }) as EJSONDocument; + mongodbCache.clearCollectionCache(config, req.collection); + return { collection: req.collection, insertedId, document: echoed }; + } + + async updateDocument(config: MongoConfig, req: MongoUpdateRequest): Promise { + assertMongoNotReadOnly(config, "updateDocument"); + assertCollectionName(req.collection); + assertNonEmptyFilter(req.filter); + assertOperatorDocument(req.update); + assertSafeExpressions(req.filter); + assertSafeExpressions(req.update); + const bsonFilter = serializeMongoFilter(req.filter); + const bsonUpdate = serializeMongoUpdateOperand(req.update); + const multi = req.options?.multi ?? false; + const returnDocument = req.options?.returnDocument ?? "after"; + const upsert = req.options?.upsert ?? false; + const arrayFilters = req.options?.arrayFilters + ? req.options.arrayFilters.map((f) => serializeMongoFilter(f)) + : undefined; + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const result = await withMongoTimeout(timeout, async (signal) => { + let updatedDocument: EJSONDocument | undefined; + if (returnDocument === "before") { + const before = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = before ? (mongoBSONtoEJSON(before) as EJSONDocument) : undefined; + } + const updateOptions = { + maxTimeMS: timeout, + upsert, + arrayFilters, + }; + const r = multi + ? await coll.updateMany(bsonFilter, bsonUpdate, updateOptions) + : await coll.updateOne(bsonFilter, bsonUpdate, updateOptions); + if (returnDocument !== "before") { + // Best-effort per spec §7.2: if the follow-up read fails, omit + // updatedDocument but still return the driver counts. + try { + const after = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = after ? (mongoBSONtoEJSON(after) as EJSONDocument) : undefined; + } catch (readErr) { + logger.warn( + { collection: req.collection, err: readErr }, + "mongo updateDocument follow-up read failed; returning counts only", + ); + } + } + return { r, updatedDocument }; + }); + + const upsertedRaw = result.r.upsertedId; + const upsertedId = + upsertedRaw && Object.keys(upsertedRaw).length > 0 + ? (mongoBSONtoEJSON(Object.values(upsertedRaw)[0]) as EJsonValue) + : undefined; + + mongodbCache.clearCollectionCache(config, req.collection); + return { + collection: req.collection, + matchedCount: result.r.matchedCount, + modifiedCount: result.r.modifiedCount, + upsertedId, + updatedDocument: result.updatedDocument, + }; + } + + async replaceDocument( + config: MongoConfig, + req: MongoReplaceRequest, + ): Promise { + assertMongoNotReadOnly(config, "replaceDocument"); + assertCollectionName(req.collection); + assertNonEmptyFilter(req.filter); + if ( + typeof req.replacement !== "object" || + req.replacement === null || + Array.isArray(req.replacement) + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "replacement document must be an object"); + } + assertNoOperatorKeys(req.replacement); + assertSize(req.replacement, MAX_DOCUMENT_SIZE_BYTES, "replacement"); + assertSafeExpressions(req.filter); + assertSafeExpressions(req.replacement); + + if (req.replacement._id !== undefined && req.filter._id !== undefined) { + if (canonicalJSON(req.replacement._id) !== canonicalJSON(req.filter._id)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "Cannot change _id of a document"); + } + } + + const bsonFilter = serializeMongoFilter(req.filter); + const doc = serializeMongoEJSONDocument(req.replacement); + const upsert = req.options?.upsert ?? false; + const returnDocument = req.options?.returnDocument ?? "after"; + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const result = await withMongoTimeout(timeout, async (signal) => { + let updatedDocument: EJSONDocument | undefined; + if (returnDocument === "before") { + const before = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = before ? (mongoBSONtoEJSON(before) as EJSONDocument) : undefined; + } + const replaceOptions = { + maxTimeMS: timeout, + upsert, + }; + const r = await coll.replaceOne(bsonFilter, doc, replaceOptions); + if (returnDocument !== "before") { + try { + const after = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = after ? (mongoBSONtoEJSON(after) as EJSONDocument) : undefined; + } catch (readErr) { + logger.warn( + { collection: req.collection, err: readErr }, + "mongo replaceDocument follow-up read failed; returning counts only", + ); + } + } + return { r, updatedDocument }; + }); + + const upsertedRaw = result.r.upsertedId; + const upsertedId = + upsertedRaw && Object.keys(upsertedRaw).length > 0 + ? (mongoBSONtoEJSON(Object.values(upsertedRaw)[0]) as EJsonValue) + : upsertedRaw + ? (mongoBSONtoEJSON(upsertedRaw) as EJsonValue) + : undefined; + + mongodbCache.clearCollectionCache(config, req.collection); + return { + collection: req.collection, + matchedCount: result.r.matchedCount, + modifiedCount: result.r.modifiedCount, + upsertedId, + updatedDocument: result.updatedDocument, + }; + } + + async deleteDocument(config: MongoConfig, req: MongoDeleteRequest): Promise { + assertMongoNotReadOnly(config, "deleteDocument"); + assertCollectionName(req.collection); + assertNonEmptyFilter(req.filter); + assertSafeExpressions(req.filter); + const bsonFilter = serializeMongoFilter(req.filter); + const multi = req.options?.multi ?? false; + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const res = await withMongoTimeout(timeout, () => + multi + ? coll.deleteMany(bsonFilter, { maxTimeMS: timeout }) + : coll.deleteOne(bsonFilter, { maxTimeMS: timeout }), + ); + + logger.debug( + { collection: req.collection, deletedCount: res.deletedCount }, + "mongo deleteDocument", + ); + mongodbCache.clearCollectionCache(config, req.collection); + return { collection: req.collection, deletedCount: res.deletedCount }; + } + + async runAggregation( + config: MongoConfig, + req: MongoAggregationRequest, + ): Promise { + assertCollectionName(req.collection); + assertReadOnlyPipeline(req.pipeline); + if (req.pipeline) { + for (const stage of req.pipeline) { + assertSafeExpressions(stage); + } + } + const stages = (req.pipeline || []).map((stage) => serializeMongoPipelineStage(stage)); + const timeout = clampMqlTimeout(req.options?.maxTimeMS, 10000, 60000); + const allowDiskUse = + typeof req.options?.allowDiskUse === "boolean" ? req.options.allowDiskUse : false; + const coll = selectCollection(config, req.collection); + + const started = Date.now(); + const nativeDocs: Document[] = []; + await withMongoTimeout(timeout, async (signal) => { + const options: AggregateOptions & { signal?: AbortSignal } = { + signal, + maxTimeMS: timeout, + allowDiskUse, + ...PRESERVE_BSON_TYPES, + }; + if ( + typeof req.options?.batchSize === "number" && + Number.isInteger(req.options.batchSize) && + req.options.batchSize > 0 + ) { + options.batchSize = req.options.batchSize; + } + const cursor = coll.aggregate(stages, options); + try { + for await (const doc of cursor) { + nativeDocs.push(doc); + if (nativeDocs.length > MAX_AGG_RESULTS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `aggregation returned more than ${MAX_AGG_RESULTS} documents (max ${MAX_AGG_RESULTS}); add a $limit stage`, + ); + } + } + } finally { + await cursor.close().catch(() => {}); + } + }); + + const documents = nativeDocs.map((d) => mongoBSONtoEJSON(d) as EJSONDocument); + return { + collection: req.collection, + documents, + stageCount: stages.length, + executionTimeMs: Date.now() - started, + }; + } + + async executeMql( + config: MongoConfig, + req: MongoMqlExecuteRequest, + context?: MongoExecutionContext, + ): Promise { + return executeMql(config, req, context); + } + + async explainMql( + config: MongoConfig, + req: MongoExplainRequest, + context?: MongoExecutionContext, + ): Promise { + return explainMql(config, req, context); + } + + async listCollections(config: MongoConfig): Promise { + return listCollections(config); + } + + async inferCollectionSchema( + config: MongoConfig, + req: MongoInferSchemaRequest, + ): Promise { + assertCollectionName(req.collection); + const sampleSize = clampInferSampleSize(req.sampleSize); + const maxDepth = clampInferMaxDepth(req.maxDepth); + + const cached = mongodbCache.getInferredSchema(config, req.collection, sampleSize, maxDepth); + if (cached) { + return cached; + } + + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const nativeDocs = (await withMongoTimeout(timeout, (signal) => + coll + .find({}, { signal, maxTimeMS: timeout, limit: sampleSize, ...PRESERVE_BSON_TYPES }) + .toArray(), + )) as Array>; + + const actualSampleSize = nativeDocs.length; + const acc = new FieldAccumulator(actualSampleSize, maxDepth); + for (const doc of nativeDocs) { + acc.ingest(doc, "", 0); + } + const fields = acc.toFields(); + + let totalDocs = 0; + try { + totalDocs = await coll.estimatedDocumentCount({ maxTimeMS: timeout }); + } catch { + try { + totalDocs = await coll.countDocuments({}, { maxTimeMS: timeout }); + } catch { + totalDocs = actualSampleSize; + } + } + + const sampleDocuments = nativeDocs + .slice(0, 5) + .map((d) => serializeMongoDocument(d as Document)); + + const schema: MongoCollectionSchema = { + collection: req.collection, + sampleSize: actualSampleSize, + totalDocs, + inferredAt: new Date().toISOString(), + fieldCount: fields.length, + nestedFieldCount: fields.filter((f) => f.depth > 0).length, + arrayFieldCount: fields.filter((f) => f.isArray).length, + fields, + sampleDocuments, + }; + + if (config.cacheDocuments !== false) { + mongodbCache.setInferredSchema(config, req.collection, sampleSize, maxDepth, schema); + } + + return schema; + } + + async createCollection( + config: MongoConfig, + req: MongoCreateCollectionRequest, + ): Promise { + assertMongoNotReadOnly(config, "createCollection"); + assertNewCollectionName(req.collection); + if (req.options?.capped) { + const { size, max } = req.options; + if (typeof size !== "number" || !Number.isInteger(size) || size < 4096) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Capped collection size must be an integer >= 4096 bytes", + ); + } + if (max !== undefined && (typeof max !== "number" || !Number.isInteger(max) || max <= 0)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Capped collection max must be a positive integer", + ); + } + } + const db = selectDb(config); + const timeout = clampTimeout(); + await withMongoTimeout(timeout, () => db.createCollection(req.collection, req.options as any)); + mongodbCache.flushConnection(config); + return { ok: true, collection: req.collection }; + } + + async dropCollection( + config: MongoConfig, + req: MongoDropCollectionRequest, + ): Promise { + assertMongoNotReadOnly(config, "dropCollection"); + if (typeof req.collection === "string" && req.collection.startsWith("system.")) { + throw new MongoApiError("MONGO_FORBIDDEN", "Cannot drop system collection"); + } + assertCollectionName(req.collection); + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + let dropped: boolean = true; + try { + dropped = await withMongoTimeout(timeout, () => coll.drop()); + } catch (err: any) { + if (err?.codeName === "NamespaceNotFound" || err?.code === 26) { + throw new MongoApiError("MONGO_NOT_FOUND", `Collection '${req.collection}' does not exist`); + } + throw err; + } + if (dropped === false) { + throw new MongoApiError("MONGO_NOT_FOUND", `Collection '${req.collection}' does not exist`); + } + mongodbCache.clearCollectionCache(config, req.collection); + mongodbCache.flushConnection(config); + return { ok: true, collection: req.collection }; + } + + // =========================================================================== + // M6 — Secondary Index Management (docs/MONGODB_M6_SPEC.md §3, §9) + // =========================================================================== + + async listIndexes( + config: MongoConfig, + req: MongoIndexListRequest, + ): Promise { + assertCollectionName(req.collection); + const coll = selectCollection(config, req.collection); + const db = selectDb(config); + const timeout = clampTimeout(); + + let rawIndexes: any[] = []; + try { + rawIndexes = await withMongoTimeout(timeout, () => + coll.listIndexes({ maxTimeMS: timeout }).toArray(), + ); + } catch (err: any) { + if (err?.codeName === "NamespaceNotFound" || err?.code === 26) { + return { + collection: req.collection, + indexes: [], + totalSizeBytes: 0, + }; + } + throw err; + } + + let indexSizes: Record = {}; + let totalSizeBytes = 0; + try { + const stats = await withMongoTimeout(timeout, () => + db.command({ collStats: req.collection }), + ); + if (stats?.indexSizes && typeof stats.indexSizes === "object") { + indexSizes = stats.indexSizes; + } + if (typeof stats?.totalIndexSize === "number") { + totalSizeBytes = stats.totalIndexSize; + } + } catch { + // collStats is best-effort (e.g. Atlas / views / permissions) + } + + if (totalSizeBytes === 0 && Object.keys(indexSizes).length > 0) { + totalSizeBytes = Object.values(indexSizes).reduce( + (sum, sz) => sum + (typeof sz === "number" ? sz : 0), + 0, + ); + } + + let defaultCollation: MongoIndexCollation | null = null; + let collectionCollationEstablished = false; + try { + const collInfos = await withMongoTimeout(timeout, () => + db.listCollections({ name: req.collection }, { maxTimeMS: timeout }).toArray(), + ); + if (collInfos.length > 0) { + collectionCollationEstablished = true; + defaultCollation = (collInfos[0] as any)?.options?.collation || null; + } + } catch { + // Best-effort: if listCollections fails, collectionCollationEstablished remains false + } + + const indexDetailsList: MongoIndexDetails[] = rawIndexes.map((raw) => { + const isIdIndex = raw.name === "_id_"; + const sizeBytes = typeof indexSizes[raw.name] === "number" ? indexSizes[raw.name] : 0; + return { + name: raw.name, + key: raw.key, + sizeBytes, + unique: raw.unique ?? (isIdIndex ? true : false), + sparse: raw.sparse ?? false, + isIdIndex, + expireAfterSeconds: raw.expireAfterSeconds, + partialFilterExpression: raw.partialFilterExpression + ? (mongoBSONtoEJSON(raw.partialFilterExpression) as EJSONDocument) + : undefined, + collation: raw.collation, + hidden: raw.hidden, + v: raw.v, + textIndexVersion: raw.textIndexVersion, + weights: raw.weights, + default_language: raw.default_language, + language_override: raw.language_override, + }; + }); + + const evaluatedIndexes = evaluateIndexRedundancy(indexDetailsList, { + collectionDefaultCollation: defaultCollation, + collectionCollationUnknown: !collectionCollationEstablished, + }); + + return { + collection: req.collection, + indexes: evaluatedIndexes, + totalSizeBytes, + }; + } + + async createIndex( + config: MongoConfig, + req: MongoIndexCreateRequest, + options?: { readOnly?: boolean }, + ): Promise { + assertMongoNotReadOnly(config, "createIndexes"); + if (options?.readOnly) { + throw new MongoApiError( + "MONGO_READ_ONLY", + "Database is configured as read-only. 'createIndexes' is prohibited.", + ); + } + assertCollectionName(req.collection); + if (typeof req.key !== "object" || req.key === null || Array.isArray(req.key)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "key must be an object"); + } + const keyEntries = Object.entries(req.key); + if (keyEntries.length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "key must contain at least one field"); + } + const VALID_DIRECTIONS = new Set([1, -1, "text", "2dsphere", "hashed"]); + for (const [field, dir] of keyEntries) { + if (typeof field !== "string" || !field.trim()) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "key field name must be a non-empty string", + ); + } + if (!VALID_DIRECTIONS.has(dir as any)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `invalid direction for field '${field}': ${dir}`, + ); + } + } + + // Special index combination validation + const textFields = keyEntries.filter(([_, dir]) => dir === "text"); + const geoFields = keyEntries.filter(([_, dir]) => dir === "2dsphere"); + const hashedFields = keyEntries.filter(([_, dir]) => dir === "hashed"); + + if (textFields.length > 0 && (geoFields.length > 0 || hashedFields.length > 0)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Cannot combine text index with 2dsphere or hashed keys", + ); + } + if (geoFields.length > 0 && hashedFields.length > 0) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Cannot combine 2dsphere index with hashed keys", + ); + } + if (hashedFields.length > 1) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "A compound index cannot contain more than one hashed key", + ); + } + if (textFields.length > 0 && req.options?.unique) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Text indexes cannot enforce unique constraints", + ); + } + + if (req.options?.expireAfterSeconds !== undefined) { + if ( + typeof req.options.expireAfterSeconds !== "number" || + req.options.expireAfterSeconds < 0 || + !Number.isFinite(req.options.expireAfterSeconds) + ) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "expireAfterSeconds must be a non-negative number", + ); + } + if (keyEntries.length > 1) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "TTL index (expireAfterSeconds) can only be created on a single field", + ); + } + if (keyEntries[0][0] === "_id") { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Cannot create TTL index on immutable primary key '_id'", + ); + } + if (textFields.length > 0 || geoFields.length > 0 || hashedFields.length > 0) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "TTL index cannot be created on special non-B-tree key types", + ); + } + } + + if (req.options?.name !== undefined) { + if (typeof req.options.name !== "string" || !req.options.name.trim()) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "index name must be a non-empty string"); + } + } + + if (req.options?.partialFilterExpression !== undefined) { + if ( + typeof req.options.partialFilterExpression !== "object" || + req.options.partialFilterExpression === null || + Array.isArray(req.options.partialFilterExpression) + ) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "partialFilterExpression must be a non-null object", + ); + } + assertSafeExpressions(req.options.partialFilterExpression); + assertSize( + req.options.partialFilterExpression, + MAX_FILTER_SIZE_BYTES, + "partialFilterExpression", + ); + } + + if (req.options?.collation !== undefined) { + if ( + typeof req.options.collation !== "object" || + req.options.collation === null || + Array.isArray(req.options.collation) + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collation must be a non-null object"); + } + if ( + typeof req.options.collation.locale !== "string" || + !req.options.collation.locale.trim() + ) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collation.locale must be a non-empty string", + ); + } + if (req.options.collation.strength !== undefined) { + const s = req.options.collation.strength; + if (![1, 2, 3, 4, 5].includes(s)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collation.strength must be an integer between 1 and 5", + ); + } + } + if (req.options.collation.caseFirst !== undefined) { + if (!["upper", "lower", "off"].includes(req.options.collation.caseFirst)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collation.caseFirst must be 'upper', 'lower', or 'off'", + ); + } + } + if (req.options.collation.alternate !== undefined) { + if (!["non-ignorable", "shifted"].includes(req.options.collation.alternate)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collation.alternate must be 'non-ignorable' or 'shifted'", + ); + } + } + if (req.options.collation.maxVariable !== undefined) { + if (!["punct", "space"].includes(req.options.collation.maxVariable)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collation.maxVariable must be 'punct' or 'space'", + ); + } + } + } + + const driverOptions: CreateIndexesOptions = {}; + if (req.options?.name) driverOptions.name = req.options.name; + if (typeof req.options?.unique === "boolean") driverOptions.unique = req.options.unique; + if (typeof req.options?.sparse === "boolean") driverOptions.sparse = req.options.sparse; + if (typeof req.options?.expireAfterSeconds === "number") { + driverOptions.expireAfterSeconds = req.options.expireAfterSeconds; + } + if (req.options?.partialFilterExpression) { + driverOptions.partialFilterExpression = serializeMongoFilter( + req.options.partialFilterExpression, + ); + } + if (req.options?.collation) driverOptions.collation = req.options.collation; + if (typeof req.options?.hidden === "boolean") driverOptions.hidden = req.options.hidden; + if (typeof req.options?.background === "boolean") { + (driverOptions as any).background = req.options.background; + } + + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + const started = Date.now(); + + let indexName: string; + try { + indexName = await withMongoTimeout(timeout, () => + coll.createIndex(req.key as any, driverOptions), + ); + } catch (err: any) { + if ( + req.options?.hidden && + (err?.codeName === "InvalidOptions" || + err?.code === 72 || + (typeof err?.message === "string" && + (err.message.includes("hidden") || + err.message.includes("unrecognized option") || + err.message.includes("unknown option")))) + ) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Hidden indexes require MongoDB 4.4 or higher", + ); + } + throw err; + } + + mongodbCache.clearCollectionCache(config, req.collection); + mongodbCache.flushConnection(config); + + return { + success: true, + indexName, + executionTimeMs: Date.now() - started, + }; + } + + async createIndexes( + config: MongoConfig, + req: MongoIndexCreateRequest, + options?: { readOnly?: boolean }, + ): Promise { + return this.createIndex(config, req, options); + } + + async dropIndex( + config: MongoConfig, + req: MongoIndexDropRequest, + options?: { readOnly?: boolean }, + ): Promise { + assertMongoNotReadOnly(config, "dropIndex"); + if (options?.readOnly) { + throw new MongoApiError( + "MONGO_READ_ONLY", + "Database is configured as read-only. 'dropIndex' is prohibited.", + ); + } + assertCollectionName(req.collection); + if (typeof req.indexName !== "string" || !req.indexName.trim()) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "indexName must be a non-empty string"); + } + if (req.indexName === "_id_" || req.indexName === "*") { + throw new MongoApiError( + "MONGO_FORBIDDEN", + req.indexName === "_id_" + ? "Cannot drop immutable default primary key index '_id_'" + : "Wildcard index drop '*' is forbidden", + ); + } + + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + const started = Date.now(); + + try { + await withMongoTimeout(timeout, () => coll.dropIndex(req.indexName, { maxTimeMS: timeout })); + } catch (err: any) { + if (err?.codeName === "IndexNotFound" || err?.code === 27) { + throw new MongoApiError("MONGO_NOT_FOUND", `Index '${req.indexName}' not found`); + } + throw err; + } + + mongodbCache.clearCollectionCache(config, req.collection); + mongodbCache.flushConnection(config); + + return { + success: true, + droppedIndexName: req.indexName, + executionTimeMs: Date.now() - started, + }; + } + + async getIndexStats( + config: MongoConfig, + req: MongoIndexStatsRequest, + ): Promise { + assertCollectionName(req.collection); + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + let nativeDocs: any[] = []; + try { + nativeDocs = await withMongoTimeout(timeout, () => + coll.aggregate([{ $indexStats: {} }], { maxTimeMS: timeout }).toArray(), + ); + } catch (err: any) { + if (err?.codeName === "NamespaceNotFound" || err?.code === 26) { + return { + collection: req.collection, + stats: {}, + }; + } + throw err; + } + + const stats: Record = {}; + for (const doc of nativeDocs) { + const name = doc.name; + if (!name) continue; + const ops = + typeof doc.accesses?.ops === "number" ? doc.accesses.ops : Number(doc.accesses?.ops || 0); + const since = + doc.accesses?.since instanceof Date + ? doc.accesses.since.toISOString() + : String(doc.accesses?.since || new Date().toISOString()); + + stats[name] = { + name, + key: doc.key, + ops, + since, + host: doc.host, + }; + } + + return { + collection: req.collection, + stats, + }; + } + + async hideIndex( + config: MongoConfig, + req: MongoIndexHideRequest, + options?: { readOnly?: boolean }, + ): Promise { + assertMongoNotReadOnly(config, "hideIndex"); + if (options?.readOnly) { + throw new MongoApiError( + "MONGO_READ_ONLY", + "Database is configured as read-only. 'hideIndex' is prohibited.", + ); + } + assertCollectionName(req.collection); + if (typeof req.indexName !== "string" || !req.indexName.trim()) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "indexName must be a non-empty string"); + } + if (req.indexName === "_id_") { + throw new MongoApiError("MONGO_FORBIDDEN", "Cannot hide default primary key index '_id_'"); + } + if (typeof req.hidden !== "boolean") { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "hidden must be a boolean"); + } + + const db = selectDb(config); + const timeout = clampTimeout(); + + try { + await withMongoTimeout(timeout, () => + db.command({ + collMod: req.collection, + index: { + name: req.indexName, + hidden: req.hidden, + }, + }), + ); + } catch (err: any) { + if (err?.codeName === "IndexNotFound" || err?.code === 27) { + throw new MongoApiError("MONGO_NOT_FOUND", `Index '${req.indexName}' not found`); + } + if ( + err?.codeName === "InvalidOptions" || + err?.code === 72 || + (typeof err?.message === "string" && + (err.message.includes("no option found to collMod") || + err.message.includes("unrecognized option") || + err.message.includes("unknown option") || + err.message.includes("index option not supported"))) + ) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Hiding indexes requires MongoDB 4.4 or higher", + ); + } + throw err; + } + + mongodbCache.clearCollectionCache(config, req.collection); + mongodbCache.flushConnection(config); + + return { + success: true, + indexName: req.indexName, + hidden: req.hidden, + }; + } +} + +/** + * Canonical comparison of collation specs. + * Missing collation, empty collation, or { locale: "simple" } represents default binary collation. + */ +function areCollationsEquivalent( + a?: MongoIndexCollation | null, + b?: MongoIndexCollation | null, +): boolean { + const isSimpleA = !a || !a.locale || a.locale === "simple"; + const isSimpleB = !b || !b.locale || b.locale === "simple"; + if (isSimpleA && isSimpleB) return true; + if (isSimpleA !== isSimpleB) return false; + + const nonNullA = a!; + const nonNullB = b!; + return ( + nonNullA.locale === nonNullB.locale && + (nonNullA.strength ?? 3) === (nonNullB.strength ?? 3) && + Boolean(nonNullA.caseLevel) === Boolean(nonNullB.caseLevel) && + (nonNullA.caseFirst ?? "off") === (nonNullB.caseFirst ?? "off") && + Boolean(nonNullA.numericOrdering) === Boolean(nonNullB.numericOrdering) && + (nonNullA.alternate ?? "non-ignorable") === (nonNullB.alternate ?? "non-ignorable") && + (nonNullA.maxVariable ?? "punct") === (nonNullB.maxVariable ?? "punct") && + Boolean(nonNullA.backwards) === Boolean(nonNullB.backwards) + ); +} + +/** + * Resolves effective collation for an index, accounting for collection-level defaults. + * Returns "UNKNOWN" if collection metadata is missing/unavailable and the index lacks explicit collation. + */ +function getEffectiveCollation( + idx: MongoIndexDetails, + collectionDefault?: MongoIndexCollation | null, + collectionCollationUnknown?: boolean, +): MongoIndexCollation | null | "UNKNOWN" { + if (idx.collation) { + return idx.collation; + } + if (collectionCollationUnknown) { + return "UNKNOWN"; + } + if (collectionDefault) { + return collectionDefault; + } + return { locale: "simple" }; +} + +/** + * Canonical comparison of partial filter expressions. + * Recursively sorts object keys for deterministic structural equality. + */ +function areFiltersEquivalent(a?: Record, b?: Record): boolean { + if (!a && !b) return true; + if (!a || !b) return false; + + const normalize = (obj: any): any => { + if (obj === null || typeof obj !== "object") return obj; + if (Array.isArray(obj)) return obj.map(normalize); + const sortedKeys = Object.keys(obj).sort(); + const result: Record = {}; + for (const key of sortedKeys) { + result[key] = normalize(obj[key]); + } + return result; + }; + + return JSON.stringify(normalize(a)) === JSON.stringify(normalize(b)); +} + +export interface EvaluateRedundancyOptions { + collectionDefaultCollation?: MongoIndexCollation | null; + collectionCollationUnknown?: boolean; +} + +/** + * Multi-dimensional redundant index detection (docs/MONGODB_M6_SPEC.md §9). + * Evaluates prefix matching, directional alignment (forward & reverse traversal), + * unique/sparse/partialFilter/collation/hidden dimensions. + */ +export function evaluateIndexRedundancy( + indexes: MongoIndexDetails[], + options?: EvaluateRedundancyOptions, +): MongoIndexDetails[] { + return indexes.map((target) => { + // 1. Primary key index is never redundant + if (target.isIdIndex || target.name === "_id_") { + return { ...target, isRedundant: false }; + } + + // 2. Special non-B-tree index types (text, 2dsphere, hashed) on target are never covered by B-tree prefixes + const targetValues = Object.values(target.key); + if ( + targetValues.some((v) => typeof v === "string" && ["text", "2dsphere", "hashed"].includes(v)) + ) { + return { ...target, isRedundant: false }; + } + + const targetKeyEntries = Object.entries(target.key); + + for (const other of indexes) { + if (other.name === target.name) continue; + + // 3. Hidden indexes cannot be used by query planner; cannot make anything redundant + if (other.hidden) continue; + + // 4. If other contains ANY special non-B-tree key (text, 2dsphere, hashed), + // it cannot safely cover a standard B-tree index (different execution plans & query restrictions) + const otherValues = Object.values(other.key); + if ( + otherValues.some((v) => typeof v === "string" && ["text", "2dsphere", "hashed"].includes(v)) + ) { + continue; + } + + const otherKeyEntries = Object.entries(other.key); + + // 5. Must be a strict prefix (fewer keys) + if (targetKeyEntries.length >= otherKeyEntries.length) continue; + + // 6. Prefix match & direction consistency (supporting both forward and reverse traversal): + // Single-field prefix: B-tree can be traversed forward or backward for 1 or -1 + // Multi-field prefix: all keys must match in sequence, and directions must be + // either ALL identical (forward) OR ALL inverted (reverse traversal). + let isPrefix = false; + if (targetKeyEntries.length === 1) { + isPrefix = + targetKeyEntries[0][0] === otherKeyEntries[0][0] && + typeof otherKeyEntries[0][1] === "number"; + } else { + const keysMatch = targetKeyEntries.every( + ([key], idx) => otherKeyEntries[idx] && otherKeyEntries[idx][0] === key, + ); + if (keysMatch) { + const allIdentical = targetKeyEntries.every( + ([_, dir], idx) => otherKeyEntries[idx][1] === dir, + ); + const allInverted = targetKeyEntries.every( + ([_, dir], idx) => + typeof dir === "number" && + typeof otherKeyEntries[idx][1] === "number" && + (otherKeyEntries[idx][1] as number) === -1 * (dir as number), + ); + isPrefix = allIdentical || allInverted; + } + } + if (!isPrefix) continue; + + // 7. Unique Constraint Dimension: + // A unique target index is NEVER redundant with a compound index (must keep unique constraint). + // Furthermore, a unique compound index does NOT provide equivalent semantics to a non-unique + // shorter index (different constraint semantics, array/multikey restrictions, and distinct lifecycle). + if (target.unique || other.unique) continue; + + // 8. Sparsity Dimension: + // A non-sparse target index cannot be covered by a sparse index (sparse omits docs). + if (!target.sparse && other.sparse) continue; + + // 9. Partial Filter Expression Dimension: + // Both must have identical partial filter expressions (or neither). + if (!areFiltersEquivalent(target.partialFilterExpression, other.partialFilterExpression)) { + continue; + } + + // 10. Collation Dimension: + // Establish effective collation taking collection defaults into account. + // If effective collation cannot be established, prefer suppressing redundancy over an unsafe recommendation. + const targetEffective = getEffectiveCollation( + target, + options?.collectionDefaultCollation, + options?.collectionCollationUnknown, + ); + const otherEffective = getEffectiveCollation( + other, + options?.collectionDefaultCollation, + options?.collectionCollationUnknown, + ); + + if (targetEffective === "UNKNOWN" || otherEffective === "UNKNOWN") { + continue; + } + + if (!areCollationsEquivalent(targetEffective, otherEffective)) { + continue; + } + + // If all checks pass, target is genuinely redundant + return { + ...target, + isRedundant: true, + redundantReason: `Prefix is completely covered by compound index "${other.name}". Drops write overhead without sacrificing query plan optimality.`, + }; + } + + return { ...target, isRedundant: false }; + }); +} + +export const mongoConnector = new MongoConnector(); + +// =========================================================================== +// Stage 8 Phase 3 — Cursor Close Helper with 2000ms Bound +// =========================================================================== + +export async function initiateCursorClose(cursor: any): Promise { + const startedAt = Date.now(); + if (!cursor || typeof cursor.close !== "function") { + return { status: "closed_confirmed", durationMs: 0 }; + } + let timer: NodeJS.Timeout | undefined; + try { + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("Cursor close timeout")), 2000); + }); + await Promise.race([cursor.close(), timeoutPromise]); + return { status: "closed_confirmed", durationMs: Date.now() - startedAt }; + } catch (err: any) { + if (err?.message === "Cursor close timeout") { + return { status: "closed_timed_out", durationMs: Date.now() - startedAt }; + } + return { status: "closed_failed", error: err, durationMs: Date.now() - startedAt }; + } finally { + if (timer) clearTimeout(timer); + } +} + +// =========================================================================== +// M5 — MQL Query Workspace & Read-Only Engine (docs/MONGODB_M5_SPEC.md §3, §5) +// =========================================================================== + +export async function executeMql( + config: MongoConfig, + req: MongoMqlExecuteRequest, + context?: MongoExecutionContext, +): Promise { + // Gate 0: Entry check + context?.signal?.throwIfAborted(); + + const op = req?.op; + const opName = (op as any)?.operation; + + if (!opName || !WHITELIST_READ_OPS.has(opName)) { + throw new MongoApiError( + "MONGO_FORBIDDEN_OPERATION", + `Operation '${opName || "unknown"}' is forbidden in read-only query workspace. Use RelWave document mutation actions.`, + ); + } + + assertCollectionName(req.collection); + // Gate 1: Pre-collection selection check + context?.signal?.throwIfAborted(); + const coll = selectCollection(config, req.collection); + const started = Date.now(); + + switch (op.operation) { + case "find": { + const payload: MongoFindOpPayload = op.payload ?? { filter: {} }; + if (typeof payload !== "object" || payload === null || Array.isArray(payload)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "find payload must be an object"); + } + if ( + payload.filter !== undefined && + (typeof payload.filter !== "object" || + payload.filter === null || + Array.isArray(payload.filter)) + ) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + const filter: EJSONDocument = payload.filter ?? {}; + assertSafeExpressions(filter); + assertSize(filter, MAX_FILTER_SIZE_BYTES, "filter"); + + const sort = validateSort(payload.sort); + const projection = validateProjection(payload.projection); + + if (payload.skip !== undefined) { + if ( + typeof payload.skip !== "number" || + !Number.isFinite(payload.skip) || + !Number.isInteger(payload.skip) || + payload.skip < 0 + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "skip must be a non-negative integer"); + } + } + + let limit = DEFAULT_PAGE_SIZE; + if (typeof payload.limit === "number" && Number.isFinite(payload.limit)) { + const floored = Math.floor(payload.limit); + if (floored >= 1) { + limit = Math.min(floored, MAX_PAGE_SIZE); + } + } + + const timeout = clampMqlTimeout(req.options?.maxTimeMS, 5000, 60000); + + // Gate 2: Pre-serialization check + context?.signal?.throwIfAborted(); + const bsonFilter = serializeMongoFilter(filter); + + // Gate 3: Pre-driver invocation check + context?.signal?.throwIfAborted(); + + const nativeDocs = await withMongoTimeout(timeout, (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + + const options: FindOptions & { signal?: AbortSignal } = { + signal, + maxTimeMS: timeout, + limit, + ...PRESERVE_BSON_TYPES, + }; + if (sort) options.sort = sort; + if (projection) options.projection = projection; + if (payload.skip !== undefined) options.skip = payload.skip; + + const cursor = coll.find(bsonFilter, options); + + // Gate 4: Cursor creation check + if (signal.aborted || context?.signal?.aborted) { + initiateCursorClose(cursor); + if (signal.aborted) signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + } + + context?.onCursorCreated?.(cursor); + + const driverPromise = cursor.toArray(); + context?.onDriverPromiseCreated?.(driverPromise); + + return driverPromise; + }); + + const documents = nativeDocs.map((d) => mongoBSONtoEJSON(d) as EJSONDocument); + return { + operation: "find", + collection: req.collection, + documents, + totalReturned: documents.length, + executionTimeMs: Date.now() - started, + }; + } + + case "aggregate": { + const payload: MongoAggregateOpPayload = op.payload; + if (typeof payload !== "object" || payload === null || Array.isArray(payload)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "aggregate payload must be an object"); + } + assertReadOnlyPipeline(payload.pipeline); + const stages = (payload.pipeline || []).map((stage) => serializeMongoPipelineStage(stage)); + const timeout = clampMqlTimeout(req.options?.maxTimeMS, 10000, 60000); + const allowDiskUse = typeof payload.allowDiskUse === "boolean" ? payload.allowDiskUse : false; + + // Gate 3: Pre-driver invocation check + context?.signal?.throwIfAborted(); + + const nativeDocs: Document[] = []; + await withMongoTimeout(timeout, async (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + + const options: AggregateOptions & { signal?: AbortSignal } = { + signal, + maxTimeMS: timeout, + allowDiskUse, + ...PRESERVE_BSON_TYPES, + }; + if ( + typeof payload.batchSize === "number" && + Number.isInteger(payload.batchSize) && + payload.batchSize > 0 + ) { + options.batchSize = payload.batchSize; + } + const cursor = coll.aggregate(stages, options); + + if (signal.aborted || context?.signal?.aborted) { + initiateCursorClose(cursor); + if (signal.aborted) signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + } + + context?.onCursorCreated?.(cursor); + + try { + const streamPromise = (async () => { + for await (const doc of cursor) { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + nativeDocs.push(doc); + if (nativeDocs.length > MAX_AGG_RESULTS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `aggregation returned more than ${MAX_AGG_RESULTS} documents (max ${MAX_AGG_RESULTS}); add a $limit stage`, + ); + } + } + })(); + context?.onDriverPromiseCreated?.(streamPromise); + await streamPromise; + } finally { + await initiateCursorClose(cursor); + } + }); + + const documents = nativeDocs.map((d) => mongoBSONtoEJSON(d) as EJSONDocument); + return { + operation: "aggregate", + collection: req.collection, + documents, + stageCount: stages.length, + executionTimeMs: Date.now() - started, + }; + } + + case "countDocuments": { + const payload: MongoCountOpPayload = op.payload ?? {}; + if (typeof payload !== "object" || payload === null || Array.isArray(payload)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "countDocuments payload must be an object", + ); + } + if ( + payload.filter !== undefined && + (typeof payload.filter !== "object" || + payload.filter === null || + Array.isArray(payload.filter)) + ) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + const filter: EJSONDocument = payload.filter ?? {}; + assertSafeExpressions(filter); + assertSize(filter, MAX_FILTER_SIZE_BYTES, "filter"); + + if (payload.skip !== undefined) { + if ( + typeof payload.skip !== "number" || + !Number.isFinite(payload.skip) || + !Number.isInteger(payload.skip) || + payload.skip < 0 + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "skip must be a non-negative integer"); + } + } + + if (payload.limit !== undefined) { + if ( + typeof payload.limit !== "number" || + !Number.isFinite(payload.limit) || + !Number.isInteger(payload.limit) || + payload.limit < 0 + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "limit must be a non-negative integer"); + } + } + + const timeout = clampMqlTimeout(req.options?.maxTimeMS, 5000, 60000); + + // Gate 2: Pre-serialization check + context?.signal?.throwIfAborted(); + const bsonFilter = serializeMongoFilter(filter); + + // Gate 3: Pre-driver invocation check + context?.signal?.throwIfAborted(); + + const count = await withMongoTimeout(timeout, (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + + const options: { maxTimeMS?: number; signal?: AbortSignal; skip?: number; limit?: number } = + { + maxTimeMS: timeout, + signal, + }; + if (payload.skip !== undefined) options.skip = payload.skip; + if (payload.limit !== undefined) options.limit = payload.limit; + + const p = coll.countDocuments(bsonFilter, options); + context?.onDriverPromiseCreated?.(p); + return p; + }); + + return { + operation: "countDocuments", + collection: req.collection, + count, + executionTimeMs: Date.now() - started, + }; + } + + case "estimatedDocumentCount": { + if ( + op.payload !== undefined && + (typeof op.payload !== "object" || + op.payload === null || + Array.isArray(op.payload) || + Object.keys(op.payload).length > 0) + ) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "estimatedDocumentCount does not accept a filter; payload must be empty or undefined. Use countDocuments for filtered counts.", + ); + } + const timeout = clampMqlTimeout(req.options?.maxTimeMS, 5000, 60000); + + // Gate 3: Pre-driver invocation check + context?.signal?.throwIfAborted(); + + const count = await withMongoTimeout(timeout, (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + + const p = coll.estimatedDocumentCount({ maxTimeMS: timeout, signal } as any); + context?.onDriverPromiseCreated?.(p); + return p; + }); + + return { + operation: "estimatedDocumentCount", + collection: req.collection, + count, + executionTimeMs: Date.now() - started, + }; + } + + case "distinct": { + const payload: MongoDistinctOpPayload = op.payload; + if (typeof payload !== "object" || payload === null || Array.isArray(payload)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "distinct payload must be an object"); + } + if (typeof payload.field !== "string" || payload.field.trim().length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "field must be a non-empty string"); + } + if ( + payload.filter !== undefined && + (typeof payload.filter !== "object" || + payload.filter === null || + Array.isArray(payload.filter)) + ) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + const filter: EJSONDocument = payload.filter ?? {}; + assertSafeExpressions(filter); + assertSize(filter, MAX_FILTER_SIZE_BYTES, "filter"); + + const timeout = clampMqlTimeout(req.options?.maxTimeMS, 5000, 60000); + + // Gate 2: Pre-serialization check + context?.signal?.throwIfAborted(); + const bsonFilter = serializeMongoFilter(filter); + + // Gate 3: Pre-driver invocation check + context?.signal?.throwIfAborted(); + + let rawValues = await withMongoTimeout(timeout, (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + + const p = coll.distinct(payload.field, bsonFilter, { maxTimeMS: timeout, signal } as any); + context?.onDriverPromiseCreated?.(p); + return p; + }); + + if (rawValues.length > MAX_DISTINCT_VALUES) { + rawValues = rawValues.slice(0, MAX_DISTINCT_VALUES); + } + + const values = rawValues.map((v) => mongoBSONtoEJSON(v) as EJsonValue); + return { + operation: "distinct", + collection: req.collection, + field: payload.field, + values, + count: values.length, + executionTimeMs: Date.now() - started, + }; + } + } +} + +export async function explainMql( + config: MongoConfig, + req: MongoExplainRequest, + context?: MongoExecutionContext, +): Promise { + context?.signal?.throwIfAborted(); + const target = req?.target; + const op = (target as any)?.operation; + + if (op !== "find" && op !== "aggregate") { + throw new MongoApiError( + "MONGO_FORBIDDEN_OPERATION", + "Explain is only supported for 'find' and 'aggregate' operations", + ); + } + + assertCollectionName(req.collection); + context?.signal?.throwIfAborted(); + const coll = selectCollection(config, req.collection); + + const verbosity = req.verbosity || "queryPlanner"; + if (!["queryPlanner", "executionStats", "allPlansExecution"].includes(verbosity)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", `Invalid explain verbosity '${verbosity}'`); + } + + const defaultTimeout = + verbosity === "queryPlanner" ? 5000 : verbosity === "executionStats" ? 10000 : 15000; + const timeout = clampMqlTimeout(req.options?.maxTimeMS, defaultTimeout, 60000); + const started = Date.now(); + + let raw: any; + if (target.operation === "find") { + if ( + target.filter !== undefined && + (typeof target.filter !== "object" || target.filter === null || Array.isArray(target.filter)) + ) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + const filter: EJSONDocument = target.filter ?? {}; + assertSafeExpressions(filter); + assertSize(filter, MAX_FILTER_SIZE_BYTES, "filter"); + + const sort = validateSort(target.sort); + const projection = validateProjection(target.projection); + + if (target.skip !== undefined) { + if ( + typeof target.skip !== "number" || + !Number.isFinite(target.skip) || + !Number.isInteger(target.skip) || + target.skip < 0 + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "skip must be a non-negative integer"); + } + } + + context?.signal?.throwIfAborted(); + const bsonFilter = serializeMongoFilter(filter); + const cursor = coll.find(bsonFilter, { maxTimeMS: timeout, ...PRESERVE_BSON_TYPES }); + if (sort) cursor.sort(sort); + if (projection) cursor.project(projection); + if (target.skip !== undefined) cursor.skip(target.skip); + if (typeof target.limit === "number" && Number.isFinite(target.limit)) { + const floored = Math.floor(target.limit); + if (floored >= 1) cursor.limit(Math.min(floored, MAX_PAGE_SIZE)); + } + + context?.onCursorCreated?.(cursor); + raw = await withMongoTimeout(timeout, (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + const p = cursor.explain(verbosity); + context?.onDriverPromiseCreated?.(p); + return p; + }); + } else { + // aggregate + assertReadOnlyPipeline(target.pipeline); + const stages = (target.pipeline || []).map((stage) => serializeMongoPipelineStage(stage)); + context?.signal?.throwIfAborted(); + const cursor = coll.aggregate(stages, { + maxTimeMS: timeout, + allowDiskUse: target.allowDiskUse ?? false, + ...PRESERVE_BSON_TYPES, + }); + + context?.onCursorCreated?.(cursor); + raw = await withMongoTimeout(timeout, (signal) => { + signal.throwIfAborted(); + context?.signal?.throwIfAborted(); + const p = cursor.explain(verbosity); + context?.onDriverPromiseCreated?.(p); + return p; + }); + } + + const rawObj = (raw || {}) as Record; + const queryPlanner = rawObj.queryPlanner || { + plannerVersion: 1, + namespace: `${config.database || ""}.${req.collection}`, + indexFilterSet: false, + winningPlan: rawObj.stages ? { stages: rawObj.stages } : rawObj.winningPlan || {}, + }; + + const executionStats = rawObj.executionStats + ? { + executionSuccess: rawObj.executionStats.executionSuccess ?? true, + nReturned: rawObj.executionStats.nReturned ?? 0, + executionTimeMillis: rawObj.executionStats.executionTimeMillis ?? 0, + totalKeysExamined: rawObj.executionStats.totalKeysExamined ?? 0, + totalDocsExamined: rawObj.executionStats.totalDocsExamined ?? 0, + executionStages: rawObj.executionStats.executionStages || {}, + allPlansExecution: rawObj.executionStats.allPlansExecution, + } + : undefined; + + const serverInfo = rawObj.serverInfo + ? { + host: rawObj.serverInfo.host, + port: rawObj.serverInfo.port, + version: rawObj.serverInfo.version, + } + : undefined; + + return { + collection: req.collection, + operation: target.operation, + queryPlanner, + executionStats, + serverInfo, + rawExplain: (mongoBSONtoEJSON(raw) as Record) || {}, + executionTimeMs: Date.now() - started, + }; +} diff --git a/bridge/src/connectors/mongoBson.ts b/bridge/src/connectors/mongoBson.ts new file mode 100644 index 00000000..f1b251a4 --- /dev/null +++ b/bridge/src/connectors/mongoBson.ts @@ -0,0 +1,593 @@ +/** + * BSON-to-wire serializer for MongoDB documents. + * + * Converts a driver document into a lossless, JSON-safe `SerializedMongoDocument` + * that can travel over the newline-delimited JSON-RPC stdio channel. ObjectId, + * Decimal128, Long, Timestamp, Binary, and `undefined` are not JSON-representable + * and are mapped to MongoDB Extended JSON forms; plain primitives pass through. + * + * Type detection is by BSON `_bsontype` (stable across driver/bson versions) plus + * `instanceof Date` / `Buffer.isBuffer`, so this module is pure and unit-testable + * without a live driver. No value it returns fails `JSON.stringify`. + */ + +import { ObjectId, Int32, Long, Decimal128, Binary, Timestamp } from "mongodb"; +import type { Document } from "mongodb"; +import type { + SerializedMongoDocument, + MongoWireValue, + MongoWireObject, + MongoWireBinary, + EJsonValue, + EJSONDocument, + MongoBsonType, +} from "../types/mongo"; + +type WithBsontype = { _bsontype?: string }; + +/** Minimal structural views so we never touch `any`. */ +interface ObjectIdLike { + _bsontype: "ObjectId"; + toHexString(): string; +} +interface Decimal128Like { + _bsontype: "Decimal128"; + toString(): string; +} +interface LongLike { + _bsontype: "Long"; + toString(): string; +} +interface TimestampLike { + _bsontype: "Timestamp"; + t: number; + i: number; +} +interface IntLike { + _bsontype: "Int32" | "Double"; + valueOf(): number; +} +interface BinaryLike { + _bsontype: "Binary"; + buffer: Uint8Array; + sub_type?: number; +} + +function toFiniteNumber(n: number): number | null { + return Number.isFinite(n) ? n : null; +} + +function serializeBinary(value: BinaryLike): MongoWireBinary { + const buf = Buffer.isBuffer(value.buffer) ? value.buffer : Buffer.from(value.buffer); + const subtype = typeof value.sub_type === "number" ? value.sub_type : 0; + return { + $binary: { + base64: buf.toString("base64"), + subtype: subtype.toString(16).padStart(2, "0"), + }, + }; +} + +/** + * Serialize a single BSON value into a JSON-safe wire value. Arrays and nested + * documents recurse. + */ +export function serializeMongoValue(value: unknown): MongoWireValue { + if (value === null) return null; + + const kind = typeof value; + if (kind === "string" || kind === "boolean") { + return value as MongoWireValue; + } + if (kind === "number") { + return toFiniteNumber(value as number) as MongoWireValue; + } + if (kind === "undefined") { + return { $undefined: true }; + } + if (Array.isArray(value)) { + return value.map((item) => serializeMongoValue(item)); + } + + const bsontype = (value as WithBsontype)._bsontype; + if (bsontype) { + switch (bsontype) { + case "ObjectId": + return { $oid: (value as ObjectIdLike).toHexString() }; + case "Decimal128": + return { $numberDecimal: (value as Decimal128Like).toString() }; + case "Long": + return { $numberLong: (value as LongLike).toString() }; + case "Timestamp": { + const ts = value as TimestampLike; + return { $timestamp: { t: ts.t | 0, i: ts.i | 0 } }; + } + case "Int32": + case "Double": + return toFiniteNumber((value as IntLike).valueOf()) as MongoWireValue; + case "Binary": + return serializeBinary(value as BinaryLike); + default: + // Unknown BSON subtype (e.g. DBRef) falls through to object serialization. + break; + } + } + + if (value instanceof Date) { + return { $date: value.toISOString() }; + } + if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + return serializeBinary({ _bsontype: "Binary", buffer: value, sub_type: 0 }); + } + + if (kind === "object") { + const out: MongoWireObject = {}; + for (const [key, item] of Object.entries(value as Record)) { + if (key === "_bsontype") continue; + out[key] = serializeMongoValue(item); + } + return out; + } + + // functions / symbols / bigint are not representable on the wire. + return null; +} + +/** + * Serialize a whole document. Own enumerable keys are serialized; the internal + * `_bsontype` marker is dropped. + */ +export function serializeMongoDocument(doc: Document): SerializedMongoDocument { + const out: SerializedMongoDocument = {}; + for (const [key, item] of Object.entries(doc as Record)) { + if (key === "_bsontype") continue; + out[key] = serializeMongoValue(item); + } + return out; +} + +// =========================================================================== +// M3 — BSON Type Inference helpers (pure, docs/MONGODB_M3_SPEC.md §4.2) +// =========================================================================== + +/** + * Determine the canonical BSON type of a value without requiring a live driver. + * Never throws; unknown BSON types fall back to "object". + */ +export function bsonTypeOf(value: unknown): MongoBsonType { + if (value === null) return "null"; + if (value === undefined) return "undefined"; + + const kind = typeof value; + if (kind === "string") return "string"; + if (kind === "boolean") return "bool"; + if (kind === "number") return "number"; + + if (Array.isArray(value)) return "array"; + + const bsontype = (value as WithBsontype)?._bsontype; + if (bsontype) { + switch (bsontype) { + case "ObjectId": + return "objectId"; + case "Int32": + case "Double": + return "number"; + case "Long": + return "long"; + case "Decimal128": + return "decimal"; + case "Timestamp": + return "timestamp"; + case "Binary": + return "binary"; + default: + break; + } + } + + if (value instanceof Date) return "date"; + if (Buffer.isBuffer(value) || value instanceof Uint8Array) return "binary"; + + if (kind === "object") return "object"; + + return "object"; +} + +/** + * Determine the numeric kind ("int" | "double") for numeric values. + */ +export function bsonNumericKind(value: unknown): "int" | "double" | undefined { + const bsontype = (value as WithBsontype)?._bsontype; + if (bsontype === "Int32") return "int"; + if (bsontype === "Double") return "double"; + if (typeof value === "number") { + return Number.isInteger(value) ? "int" : "double"; + } + return undefined; +} + +// =========================================================================== +// M2 — Extended JSON (EJSON) <-> driver-native conversion. +// +// serializeMongoValue/serializeMongoDocument above go driver-native -> wire/EJSON +// using the driver's wire shapes for the JSON-RPC response channel. The M2 +// document engine needs the canonical EJSON v2 round-trip in BOTH directions +// (docs/MONGODB_M2_SPEC.md §4.3): mongoEJSONtoBSON parses request inputs; +// mongoBSONtoEJSON renders results so that mongoBSONtoEJSON(mongoEJSONtoBSON(x)) +// is structurally equal to x for the supported type set. Pure and unit-testable. +// =========================================================================== + +const EJSON_WRAPPER_KEYS = new Set([ + "$oid", + "$date", + "$numberInt", + "$numberLong", + "$numberDouble", + "$numberDecimal", + "$binary", + "$regex", + "$timestamp", + "$undefined", +]); + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` is an EJSON object with exactly one known wrapper key. */ +function isEJSONWrapper(value: unknown): value is Record { + if (!isPlainRecord(value)) return false; + const keys = Object.keys(value); + return keys.length === 1 && EJSON_WRAPPER_KEYS.has(keys[0]); +} + +/** + * Validates and normalizes BSON binary subtype. + * Subtype must be an integer between 0 and 255 (numeric or 1-2 character hex string). + */ +export function validateAndParseSubtype(rawSubtype: unknown): number { + if (typeof rawSubtype === "number") { + if (!Number.isSafeInteger(rawSubtype) || rawSubtype < 0 || rawSubtype > 255) { + throw new TypeError( + `Invalid EJSON $binary: subtype must be a safe integer between 0 and 255, received ${rawSubtype}`, + ); + } + return rawSubtype; + } + + if (typeof rawSubtype === "string") { + if (!/^(?:0x)?[0-9a-fA-F]{1,2}$/i.test(rawSubtype)) { + throw new TypeError( + `Invalid EJSON $binary: subtype must be a valid 1-2 digit hexadecimal string (0x00-0xFF), received '${rawSubtype}'`, + ); + } + const cleanHex = + rawSubtype.startsWith("0x") || rawSubtype.startsWith("0X") ? rawSubtype.slice(2) : rawSubtype; + const val = parseInt(cleanHex, 16); + if (!Number.isSafeInteger(val) || val < 0 || val > 255) { + throw new TypeError( + `Invalid EJSON $binary: subtype must be between 0 and 255, received '${rawSubtype}'`, + ); + } + return val; + } + + throw new TypeError( + `Invalid EJSON $binary: subtype must be a number or hexadecimal string, received ${typeof rawSubtype}`, + ); +} + +/** + * Validates structural RFC 4648 Base64 syntax before decoding, and canonical + * post-decoding re-encoding (ensuring unused pad bits in the final quantum are zero). + */ +export function validateAndDecodeBase64(rawBase64: unknown): Buffer { + if (typeof rawBase64 !== "string") { + throw new TypeError( + `Invalid EJSON $binary: base64 payload must be a string, received ${typeof rawBase64}`, + ); + } + + if (rawBase64 === "") { + return Buffer.alloc(0); + } + + // Reject truncated base64 (modulo 4 cannot be 1 in RFC 4648) + if (rawBase64.length % 4 === 1) { + throw new TypeError( + `Invalid EJSON $binary: invalid base64 length (${rawBase64.length}), modulo 4 cannot be 1`, + ); + } + + let paddedStr: string; + + if (rawBase64.includes("=")) { + // Padded base64 must have total length divisible by 4 + if (rawBase64.length % 4 !== 0) { + throw new TypeError( + `Invalid EJSON $binary: padded base64 length (${rawBase64.length}) must be a multiple of 4`, + ); + } + // Must match valid padded base64 structure: blocks of 4 chars ending with == (preceded by 2 chars) or = (preceded by 3 chars) + const PADDED_BASE64_REGEX = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)$/; + if (!PADDED_BASE64_REGEX.test(rawBase64)) { + throw new TypeError( + `Invalid EJSON $binary: invalid base64 padding or characters in '${rawBase64}'`, + ); + } + paddedStr = rawBase64; + } else { + // Unpadded base64: characters must be strictly RFC 4648 alphabet without padding + if (!/^[A-Za-z0-9+/]+$/.test(rawBase64)) { + throw new TypeError( + `Invalid EJSON $binary: invalid characters in unpadded base64 '${rawBase64}'`, + ); + } + const remainder = rawBase64.length % 4; + if (remainder === 2) { + paddedStr = rawBase64 + "=="; + } else if (remainder === 3) { + paddedStr = rawBase64 + "="; + } else { + paddedStr = rawBase64; + } + } + + const buf = Buffer.from(paddedStr, "base64"); + + // Canonical re-encoding validation post-decoding (unused pad bits must be zero) + if (buf.toString("base64") !== paddedStr) { + throw new TypeError( + "Invalid EJSON $binary: non-canonical base64 encoding (unused pad bits must be zero)", + ); + } + + return buf; +} + +/** + * Recursively convert an EJSON value to driver-native BSON types. Resolves the + * EJSON v2 wrappers; plain JSON passes through; a bare 24-hex string stays a + * string (only $oid yields an ObjectId — spec §4.2 / D-M2-6); unknown $-wrappers + * are returned unchanged for the driver to handle. + */ +export function mongoEJSONtoBSON(doc: EJSONDocument): Record; +export function mongoEJSONtoBSON(value: EJsonValue): unknown; +export function mongoEJSONtoBSON(value: EJsonValue): unknown { + if (value === null || value === undefined) return value; + + const kind = typeof value; + if (kind === "string" || kind === "boolean") { + return value; + } + if (kind === "number") { + return value; + } + if (Array.isArray(value)) { + return value.map((item) => mongoEJSONtoBSON(item)); + } + if (kind === "bigint") { + return (value as unknown as bigint).toString(); + } + + if (isEJSONWrapper(value)) { + const key = Object.keys(value)[0]; + const v = value as Record; + switch (key) { + case "$oid": + return new ObjectId(v["$oid"] as string); + case "$date": { + const d = v["$date"]; + if (isPlainRecord(d) && typeof d["$numberLong"] === "string") { + return new Date(Long.fromString(d["$numberLong"]).toNumber()); + } + return new Date(d as string); + } + case "$numberInt": + return new Int32(Number(v["$numberInt"])); + case "$numberLong": + return Long.fromString(v["$numberLong"] as string); + case "$numberDouble": + return Number(v["$numberDouble"] as string); + case "$numberDecimal": + return new Decimal128(v["$numberDecimal"] as string); + case "$binary": { + const b = v["$binary"]; + if (typeof b === "string") { + const buf = validateAndDecodeBase64(b); + return new Binary(buf, 0); + } + + if (isPlainRecord(b)) { + const base64Candidates: { key: string; val: unknown }[] = []; + if ("base64" in b && b.base64 !== undefined) { + base64Candidates.push({ key: "base64", val: b.base64 }); + } + if ("$base64" in b && b.$base64 !== undefined) { + base64Candidates.push({ key: "$base64", val: b.$base64 }); + } + + if (base64Candidates.length === 0) { + throw new TypeError("Invalid EJSON $binary: missing base64 data"); + } + + // Validate EVERY defined candidate independently (a valid alias cannot conceal an invalid alias) + const decodedBuffers: Buffer[] = []; + for (const cand of base64Candidates) { + decodedBuffers.push(validateAndDecodeBase64(cand.val)); + } + + if ( + base64Candidates.length === 2 && + base64Candidates[0].val !== base64Candidates[1].val + ) { + throw new TypeError("Invalid EJSON $binary: conflicting base64 payload properties"); + } + + const selectedBuffer = decodedBuffers[0]; + + // Subtype candidates: subtype, subType, $type, sub_type + const subtypeCandidates: { key: string; val: unknown }[] = []; + for (const key of ["subtype", "subType", "$type", "sub_type"] as const) { + if (key in b && b[key] !== undefined) { + subtypeCandidates.push({ key, val: b[key] }); + } + } + + let selectedSubtype = 0; + if (subtypeCandidates.length > 0) { + const parsedSubtypes: number[] = []; + for (const cand of subtypeCandidates) { + parsedSubtypes.push(validateAndParseSubtype(cand.val)); + } + + const firstSubtype = parsedSubtypes[0]; + for (let i = 1; i < parsedSubtypes.length; i++) { + if (parsedSubtypes[i] !== firstSubtype) { + throw new TypeError( + "Invalid EJSON $binary: conflicting subtype payload properties", + ); + } + } + selectedSubtype = firstSubtype; + } + + return new Binary(selectedBuffer, selectedSubtype); + } + + throw new TypeError("Invalid EJSON $binary: missing base64 data"); + } + case "$regex": + return new RegExp(v["$regex"] as string, (v["$options"] as string) || ""); + case "$timestamp": { + const t = v["$timestamp"] as { $t?: string; $i?: string }; + return new Timestamp({ t: Number(t["$t"] || 0), i: Number(t["$i"] || 0) }); + } + case "$undefined": + return undefined; + default: + return value; // unreachable: isEJSONWrapper matched a known key + } + } + + if (kind === "object") { + const out: Record = {}; + for (const [k, item] of Object.entries(value as unknown as Record)) { + out[k] = mongoEJSONtoBSON(item as EJsonValue); + } + return out; + } + return value; +} + +/** + * Convert a driver-native value to canonical EJSON v2 for the JSON-RPC response. + * Int32 -> $numberInt and Long -> $numberLong are preserved (so int types + * round-trip); Double -> plain JSON number (the accepted double normalization). + */ +export function mongoBSONtoEJSON(value: unknown): EJsonValue { + if (value === null || value === undefined) return null; + + const kind = typeof value; + if (kind === "string" || kind === "boolean") { + return value as EJsonValue; + } + if (kind === "number") { + return toFiniteNumber(value as number) as EJsonValue; + } + if (kind === "bigint") { + return (value as bigint).toString() as EJsonValue; + } + if (Array.isArray(value)) { + return value.map((item) => mongoBSONtoEJSON(item)); + } + + if (value instanceof Date) { + return { $date: value.toISOString() }; + } + if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + const buf = Buffer.isBuffer(value) ? value : Buffer.from(value); + return { $binary: { $type: "00", $base64: buf.toString("base64") } }; + } + + const bsontype = (value as WithBsontype)._bsontype; + if (bsontype) { + switch (bsontype) { + case "ObjectId": + return { $oid: (value as ObjectIdLike).toHexString() }; + case "Decimal128": + return { $numberDecimal: (value as Decimal128Like).toString() }; + case "Long": + return { $numberLong: (value as LongLike).toString() }; + case "Timestamp": { + const ts = value as TimestampLike; + return { $timestamp: { $t: String(ts.t), $i: String(ts.i) } }; + } + case "Int32": + return { $numberInt: String((value as IntLike).valueOf()) }; + case "Double": + return toFiniteNumber((value as IntLike).valueOf()) as EJsonValue; + case "Binary": { + const b = value as BinaryLike; + const buf = Buffer.isBuffer(b.buffer) ? b.buffer : Buffer.from(b.buffer); + const subtype = typeof b.sub_type === "number" ? b.sub_type : 0; + return { + $binary: { + $type: subtype.toString(16).padStart(2, "0"), + $base64: buf.toString("base64"), + }, + }; + } + default: + break; + } + } + + if (kind === "object") { + const out: EJSONDocument = {}; + for (const [key, item] of Object.entries(value as Record)) { + if (key === "_bsontype") continue; + out[key] = mongoBSONtoEJSON(item); + } + return out; + } + return null; +} + +/** + * Convert an insert document (EJSON) to driver-native form. Named distinctly + * from serializeMongoDocument (which goes the opposite direction) to avoid the + * name collision flagged during design. + */ +export function serializeMongoEJSONDocument(doc: EJSONDocument): Record { + return mongoEJSONtoBSON(doc); +} + +/** Convert a query filter (EJSON) to driver-native form. */ +export function serializeMongoFilter(filter: EJSONDocument): Record { + return mongoEJSONtoBSON(filter); +} + +/** + * Convert an update operator document (e.g. { $set: {...}, $inc: {...} }) to + * driver-native form. Operator KEYS are preserved verbatim; their leaf VALUES + * are resolved via mongoEJSONtoBSON. Never promotes a plain-field object to an + * operator document (the connector rejects those). + */ +export function serializeMongoUpdateOperand(update: EJSONDocument): Record { + const out: Record = {}; + for (const [op, body] of Object.entries(update)) { + out[op] = mongoEJSONtoBSON(body as EJsonValue); + } + return out; +} + +/** + * Convert one aggregation pipeline stage to driver-native form. Resolves EJSON + * wrappers but does NOT auto-$literal-wrap plain-object values — the caller is + * responsible for MQL expression structure (spec §5.6 / D-M2-4). + */ +export function serializeMongoPipelineStage(stage: EJSONDocument): Record { + return mongoEJSONtoBSON(stage); +} diff --git a/bridge/src/handlers/aiHandlers.ts b/bridge/src/handlers/aiHandlers.ts index 65acd12b..219b1deb 100644 --- a/bridge/src/handlers/aiHandlers.ts +++ b/bridge/src/handlers/aiHandlers.ts @@ -20,7 +20,6 @@ import { buildSchemaAnalysisPrompt } from "../ai/prompts/schema-analysis"; import { buildQueryExplanationPrompt } from "../ai/prompts/query-explanation"; import { buildChartRecommendationPrompt } from "../ai/prompts/chart-recommendation"; import { parseChartRecommendation } from "../ai/prompts/chart-recommendation"; -import fs from "fs/promises"; import fsSync from "fs"; import { AI_SETTINGS_FILE, CONFIG_FOLDER, ensureDir } from "../utils/config"; diff --git a/bridge/src/handlers/databaseHandlers.ts b/bridge/src/handlers/databaseHandlers.ts index 0fb13c02..c5676bdb 100644 --- a/bridge/src/handlers/databaseHandlers.ts +++ b/bridge/src/handlers/databaseHandlers.ts @@ -85,8 +85,9 @@ export class DatabaseHandlers { }); } - const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); - const tables = await this.queryExecutor.listTables(conn, dbType, params.schema); + const tables = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.listTables(conn, dbType, params.schema), + ); this.rpc.sendResponse(id, { ok: true, data: tables }); } catch (e: any) { this.logger?.error({ e }, "db.listTables failed"); @@ -103,8 +104,9 @@ export class DatabaseHandlers { message: "Missing id", }); } - const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); - const schemas = await this.queryExecutor.listSchemaNames(conn, dbType); + const schemas = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.listSchemaNames(conn, dbType), + ); this.rpc.sendResponse(id, { ok: true, data: schemas }); } catch (e: any) { this.logger?.error({ e }, "db.listSchemas failed"); @@ -121,8 +123,9 @@ export class DatabaseHandlers { message: "Missing id", }); } - const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); - const dbSchema = await this.queryExecutor.listSchemas(conn, dbType); + const dbSchema = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.listSchemas(conn, dbType), + ); this.rpc.sendResponse(id, { ok: true, data: dbSchema }); } catch (e: any) { this.logger?.error({ e }, "db.getSchema failed"); @@ -171,13 +174,14 @@ export class DatabaseHandlers { }); } - let conn, dbType; + let result; if (dbId) { - const result = await this.dbService.getDatabaseConnection(dbId); - conn = result.conn; - dbType = result.dbType; + result = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.testConnection(conn, dbType), + ); } else { + let dbType; // Detect DB type from the raw connection object if (connection.type?.toLowerCase().includes("mariadb")) { dbType = "mariadb"; @@ -198,10 +202,10 @@ export class DatabaseHandlers { connection.password ?? null, dbType, ); - conn = buildResult.config; + const conn = buildResult.config; tunnelToClose = buildResult.tunnel; + result = await this.queryExecutor.testConnection(conn, dbType); } - const result = await this.queryExecutor.testConnection(conn, dbType); this.rpc.sendResponse(id, { ok: true, data: result }); } catch (err: any) { this.logger.error({ err }, "[Handler] testConnection error"); diff --git a/bridge/src/handlers/indexHandlers.ts b/bridge/src/handlers/indexHandlers.ts new file mode 100644 index 00000000..2a996c0c --- /dev/null +++ b/bridge/src/handlers/indexHandlers.ts @@ -0,0 +1,140 @@ +import { Logger } from "pino"; +import { Rpc } from "../types"; +import { DatabaseService } from "../services/databaseService"; +import { QueryExecutor } from "../services/queryExecutor"; +import { MongoApiError, mapMongoError } from "../connectors/mongo"; + +export class IndexHandlers { + constructor( + private rpc: Rpc, + private logger: Logger, + private dbService: DatabaseService, + private queryExecutor: QueryExecutor, + ) {} + + /** Map a MongoApiError (or driver error) to the stable error envelope. */ + private sendMongoError(id: number | string, e: unknown): void { + const mapped = + e instanceof MongoApiError ? { code: e.code, message: e.message } : mapMongoError(e); + this.logger.error({ err: e, code: mapped.code }, "MongoDB index operation failed"); + this.rpc.sendError(id, { code: mapped.code, message: mapped.message }); + } + + async handleListIndexes(params: any, id: number | string): Promise { + try { + const { dbId, collection } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.listIndexes(conn, dbType, { collection }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleCreateIndex(params: any, id: number | string): Promise { + try { + const { dbId, collection, key, options } = params || {}; + if (!dbId || !collection || !key) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or key", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, db, dbType) => { + const readOnly = Boolean( + (db as any)?.readonly || + (db as any)?.readOnly || + (conn as any)?.readonly || + (conn as any)?.readOnly, + ); + return this.queryExecutor.createIndex( + conn, + dbType, + { collection, key, options }, + { readOnly }, + ); + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleDropIndex(params: any, id: number | string): Promise { + try { + const { dbId, collection, indexName } = params || {}; + if (!dbId || !collection || !indexName) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or indexName", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, db, dbType) => { + const readOnly = Boolean( + (db as any)?.readonly || + (db as any)?.readOnly || + (conn as any)?.readonly || + (conn as any)?.readOnly, + ); + return this.queryExecutor.dropIndex(conn, dbType, { collection, indexName }, { readOnly }); + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleGetIndexStats(params: any, id: number | string): Promise { + try { + const { dbId, collection } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.getIndexStats(conn, dbType, { collection }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleHideIndex(params: any, id: number | string): Promise { + try { + const { dbId, collection, indexName, hidden } = params || {}; + if (!dbId || !collection || !indexName || typeof hidden !== "boolean") { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, indexName, or hidden (must be a boolean)", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, db, dbType) => { + const readOnly = Boolean( + (db as any)?.readonly || + (db as any)?.readOnly || + (conn as any)?.readonly || + (conn as any)?.readOnly, + ); + return this.queryExecutor.hideIndex( + conn, + dbType, + { collection, indexName, hidden }, + { readOnly }, + ); + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } +} diff --git a/bridge/src/handlers/migrationHandlers.ts b/bridge/src/handlers/migrationHandlers.ts index e893ee89..ba04cc0a 100644 --- a/bridge/src/handlers/migrationHandlers.ts +++ b/bridge/src/handlers/migrationHandlers.ts @@ -6,6 +6,8 @@ import { Logger } from "pino"; import path from "path"; import fs from "fs"; import { projectStoreInstance } from "../services/projectStore"; +import { mongoMigrationEngine } from "../services/mongoMigrationEngine"; +import { selectDb, MongoApiError } from "../connectors/mongo"; export class MigrationHandlers { constructor( @@ -15,6 +17,115 @@ export class MigrationHandlers { private queryExecutor: QueryExecutor, ) {} + private sendMigrationError(id: number | string, e: any, contextMsg: string): void { + this.logger?.error({ e }, contextMsg); + if (e instanceof MongoApiError) { + this.rpc.sendError(id, { code: e.code, message: e.message }); + } else { + this.rpc.sendError(id, { + code: e?.code || "IO_ERROR", + message: String(e?.message || e), + }); + } + } + + private async resolveProjectRoot(dbId: string, customRoot?: string): Promise { + if (customRoot) return customRoot; + const project = await projectStoreInstance.getProjectByDatabaseId(dbId); + if (project) { + return projectStoreInstance.resolveProjectDir(project.id); + } + const { CONFIG_FOLDER } = await import("../utils/config"); + return path.join(CONFIG_FOLDER, "projects", dbId); + } + + async handleListMigrations(params: any, id: number | string) { + try { + const { dbId, projectRoot: customRoot } = params || {}; + if (!dbId) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing dbId", + }); + } + + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + const db = selectDb(conn); + const projectRoot = await this.resolveProjectRoot(dbId, customRoot); + const data = await mongoMigrationEngine.listMigrations(db, projectRoot); + return this.rpc.sendResponse(id, { ok: true, data }); + } + + // SQL fallback + const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); + const { loadLocalMigrations } = await import("../utils/baselineMigration"); + const localMigrations = await loadLocalMigrations(migrationsDir); + const connector = getConnector(dbType) as any; + const appliedMigrations = await connector.listAppliedMigrations(conn); + this.rpc.sendResponse(id, { + ok: true, + data: { + appliedMigrations, + localMigrations, + }, + }); + } catch (e: any) { + this.sendMigrationError(id, e, "migration.list failed"); + } + } + + async handleValidateMigration(params: any, id: number | string) { + try { + const { dbId, manifest } = params || {}; + if (!manifest) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing manifest", + }); + } + + if (dbId) { + const { dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + const data = mongoMigrationEngine.validateManifest(manifest); + return this.rpc.sendResponse(id, { ok: true, data }); + } + } + + const data = mongoMigrationEngine.validateManifest(manifest); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e: any) { + this.sendMigrationError(id, e, "migration.validate failed"); + } + } + + async handleGetMigrationStatus(params: any, id: number | string) { + try { + const { dbId, migrationId } = params || {}; + if (!dbId) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing dbId", + }); + } + + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + const db = selectDb(conn); + const data = await mongoMigrationEngine.getMigrationStatus(db, migrationId); + return this.rpc.sendResponse(id, { ok: true, data }); + } + + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "migration.getStatus is currently supported only for MongoDB", + }); + } catch (e: any) { + this.sendMigrationError(id, e, "migration.getStatus failed"); + } + } + async handleGenerateCreateMigration(params: any, id: number | string) { try { const { dbId, schemaName, tableName, columns, foreignKeys = [] } = params || {}; @@ -27,6 +138,12 @@ export class MigrationHandlers { } const { dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Migrations are not supported for MongoDB databases", + }); + } const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); // Generate migration file @@ -68,6 +185,12 @@ export class MigrationHandlers { } const { dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Migrations are not supported for MongoDB databases", + }); + } const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); // Generate migration file @@ -115,6 +238,12 @@ export class MigrationHandlers { } const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Migrations are not supported for MongoDB databases", + }); + } const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); let columns = passedColumns; @@ -206,16 +335,48 @@ export class MigrationHandlers { async handleApplyMigration(params: any, id: number | string) { try { - const { dbId, version } = params || {}; + const { dbId, version, dryRun, projectRoot: customRoot, ownerId } = params || {}; - if (!dbId || !version) { + if (!dbId) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing dbId", + }); + } + + const dbConn = await this.dbService.getDatabaseConnection(dbId); + if (dbConn.dbType === "mongodb") { + const readOnly = Boolean( + (dbConn.db as any)?.readonly || + (dbConn.db as any)?.readOnly || + (dbConn.conn as any)?.readonly || + (dbConn.conn as any)?.readOnly, + ); + if (readOnly) { + return this.rpc.sendError(id, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + } + + const db = selectDb(dbConn.conn); + const projectRoot = await this.resolveProjectRoot(dbId, customRoot); + const data = await mongoMigrationEngine.applyMigration(db, projectRoot, version, { + dryRun, + readOnly, + ownerId, + }); + return this.rpc.sendResponse(id, { ok: true, data }); + } + + if (!version) { return this.rpc.sendError(id, { code: "BAD_REQUEST", message: "Missing dbId or version", }); } - const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const { conn, dbType } = dbConn; const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); // Find migration file @@ -296,8 +457,7 @@ export class MigrationHandlers { this.rpc.sendResponse(id, { ok: true }); } catch (e: any) { - this.logger?.error({ e }, "migration.apply failed"); - this.rpc.sendError(id, { code: "IO_ERROR", message: String(e) }); + this.sendMigrationError(id, e, "migration.apply failed"); } } @@ -475,7 +635,7 @@ export class MigrationHandlers { async handleRollbackMigration(params: any, id: number | string) { try { - const { dbId, version } = params || {}; + const { dbId, version, force, projectRoot: customRoot, ownerId } = params || {}; if (!dbId || !version) { return this.rpc.sendError(id, { @@ -484,7 +644,32 @@ export class MigrationHandlers { }); } - const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const dbConn = await this.dbService.getDatabaseConnection(dbId); + if (dbConn.dbType === "mongodb") { + const readOnly = Boolean( + (dbConn.db as any)?.readonly || + (dbConn.db as any)?.readOnly || + (dbConn.conn as any)?.readonly || + (dbConn.conn as any)?.readOnly, + ); + if (readOnly) { + return this.rpc.sendError(id, { + code: "MONGO_READ_ONLY", + message: "Database is in read-only mode", + }); + } + + const db = selectDb(dbConn.conn); + const projectRoot = await this.resolveProjectRoot(dbId, customRoot); + const data = await mongoMigrationEngine.rollbackMigration(db, version, projectRoot, { + readOnly, + force, + ownerId, + }); + return this.rpc.sendResponse(id, { ok: true, data }); + } + + const { conn, dbType } = dbConn; const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); // Find migration file @@ -564,8 +749,7 @@ export class MigrationHandlers { this.rpc.sendResponse(id, { ok: true }); } catch (e: any) { - this.logger?.error({ e }, "migration.rollback failed"); - this.rpc.sendError(id, { code: "IO_ERROR", message: String(e) }); + this.sendMigrationError(id, e, "migration.rollback failed"); } } diff --git a/bridge/src/handlers/mongoBulkHandlers.ts b/bridge/src/handlers/mongoBulkHandlers.ts new file mode 100644 index 00000000..5a58fbf2 --- /dev/null +++ b/bridge/src/handlers/mongoBulkHandlers.ts @@ -0,0 +1,339 @@ +/** + * MongoDB Milestone 7 Phase 3 — Bulk Operations JSON-RPC Handlers. + * + * Exposes methods under the `mongo.bulk.*` namespace: + * - mongo.bulk.importStart + * - mongo.bulk.importChunk + * - mongo.bulk.exportStart + * - mongo.bulk.getOperationStatus + * - mongo.bulk.cancelOperation + * - mongo.bulk.cleanupOperation + * - mongo.bulk.insert + * - mongo.bulk.update + * - mongo.bulk.delete + */ + +import { Logger } from "pino"; +import { Rpc } from "../types"; +import { MongoBulkEngine, isValidUuidV4 } from "../services/mongoBulkEngine"; +import { MongoBulkFormat } from "../types/mongoBulk"; +import { MongoApiError, safeMongoError } from "../connectors/mongo"; + +function isNonEmptyString(val: any): val is string { + return typeof val === "string" && val.trim().length > 0; +} + +export class MongoBulkHandlers { + constructor( + private rpc: Rpc, + private logger: Logger, + private engine: MongoBulkEngine, + ) {} + + private sendError(id: number | string, err: any, contextMsg: string): void { + this.logger?.error({ err }, contextMsg); + if (err instanceof MongoApiError) { + this.rpc.sendError(id, { code: err.code, message: err.message }); + } else { + this.rpc.sendError(id, { + code: err?.code || "INTERNAL_ERROR", + message: err?.message ? safeMongoError(err.message) : String(err), + }); + } + } + + async handleImportStart(params: any, id: number | string): Promise { + try { + const { dbId, collection, format, options } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(collection)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: collection", + }); + } + if (!isNonEmptyString(format) || !["json", "ndjson", "csv"].includes(format)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: + "Missing or invalid required parameter: format must be 'json', 'ndjson', or 'csv'", + }); + } + + const data = await this.engine.importStart({ + dbId, + collection, + format: format as MongoBulkFormat, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.importStart failed"); + } + } + + async handleImportChunk(params: any, id: number | string): Promise { + try { + const { operationId, chunkIndex, isLastChunk, payload } = params || {}; + if (!isNonEmptyString(operationId) || !isValidUuidV4(operationId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: operationId must be a valid UUIDv4", + }); + } + if (typeof chunkIndex !== "number" || !Number.isInteger(chunkIndex) || chunkIndex < 0) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: + "Missing or invalid required parameter: chunkIndex must be a non-negative integer", + }); + } + if (payload === undefined || payload === null) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing required parameter: payload", + }); + } + + const data = await this.engine.importChunk({ + operationId, + chunkIndex, + isLastChunk, + payload, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.importChunk failed"); + } + } + + async handleExportStart(params: any, id: number | string): Promise { + try { + const { dbId, collection, format, filter, projection, sort, limit } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(collection)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: collection", + }); + } + if (!isNonEmptyString(format) || !["json", "ndjson", "csv"].includes(format)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: + "Missing or invalid required parameter: format must be 'json', 'ndjson', or 'csv'", + }); + } + + const data = await this.engine.exportStart({ + dbId, + collection, + format: format as MongoBulkFormat, + filter, + projection, + sort, + limit, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.exportStart failed"); + } + } + + async handleGetOperationStatus(params: any, id: number | string): Promise { + try { + const { operationId } = params || {}; + if (!isNonEmptyString(operationId) || !isValidUuidV4(operationId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: operationId must be a valid UUIDv4", + }); + } + + const data = await this.engine.getOperationStatus(operationId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.getOperationStatus failed"); + } + } + + async handleCancelOperation(params: any, id: number | string): Promise { + try { + const { operationId } = params || {}; + if (!isNonEmptyString(operationId) || !isValidUuidV4(operationId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: operationId must be a valid UUIDv4", + }); + } + + const data = await this.engine.cancelOperation(operationId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.cancelOperation failed"); + } + } + + async handleCleanupOperation(params: any, id: number | string): Promise { + try { + const { operationId } = params || {}; + if (!isNonEmptyString(operationId) || !isValidUuidV4(operationId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: operationId must be a valid UUIDv4", + }); + } + + const data = await this.engine.cleanupOperation(operationId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.cleanupOperation failed"); + } + } + + async handleExportSaveTo(params: any, id: number | string): Promise { + try { + const { operationId, destinationPath, options } = params || {}; + if (!isNonEmptyString(operationId) || !isValidUuidV4(operationId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: operationId must be a valid UUIDv4", + }); + } + if (!isNonEmptyString(destinationPath)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: + "Missing or invalid required parameter: destinationPath must be a non-empty string", + }); + } + if ( + options !== undefined && + (typeof options !== "object" || options === null || Array.isArray(options)) + ) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid optional parameter: options must be an object", + }); + } + if (options?.overwrite !== undefined && typeof options.overwrite !== "boolean") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid optional parameter: options.overwrite must be a boolean", + }); + } + + const data = await this.engine.exportSaveTo({ + operationId, + destinationPath, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.exportSaveTo failed"); + } + } + + async handleBulkInsert(params: any, id: number | string): Promise { + try { + const { dbId, collection, documents, options } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(collection)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: collection", + }); + } + if (!Array.isArray(documents) || documents.length === 0) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: documents must be a non-empty array", + }); + } + + const data = await this.engine.bulkInsert({ dbId, collection, documents, options }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.insert failed"); + } + } + + async handleBulkUpdate(params: any, id: number | string): Promise { + try { + const { dbId, collection, filter, update, options } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(collection)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: collection", + }); + } + if (typeof filter !== "object" || filter === null || Array.isArray(filter)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: filter must be an object", + }); + } + if (typeof update !== "object" || update === null || Array.isArray(update)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: update must be an object", + }); + } + + const data = await this.engine.bulkUpdate({ dbId, collection, filter, update, options }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.update failed"); + } + } + + async handleBulkDelete(params: any, id: number | string): Promise { + try { + const { dbId, collection, filter, options } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(collection)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: collection", + }); + } + if (typeof filter !== "object" || filter === null || Array.isArray(filter)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: filter must be an object", + }); + } + + const data = await this.engine.bulkDelete({ dbId, collection, filter, options }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.sendError(id, err, "mongo.bulk.delete failed"); + } + } +} diff --git a/bridge/src/handlers/mongoRelationshipHandlers.ts b/bridge/src/handlers/mongoRelationshipHandlers.ts new file mode 100644 index 00000000..1f1cac90 --- /dev/null +++ b/bridge/src/handlers/mongoRelationshipHandlers.ts @@ -0,0 +1,246 @@ +import { Logger } from "pino"; +import { Rpc } from "../types"; +import { MongoRelationshipEngine } from "../services/mongoRelationshipEngine"; + +function isNonEmptyString(val: any): val is string { + return typeof val === "string" && val.trim().length > 0; +} + +export class MongoRelationshipHandlers { + constructor( + private rpc: Rpc, + private logger: Logger, + private engine: MongoRelationshipEngine, + ) {} + + async handleInfer(params: any, id: number | string): Promise { + try { + const { dbId, projectId, collections, sampleSize, timeoutMs, maxCollections } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (projectId !== undefined && !isNonEmptyString(projectId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: projectId must be a non-empty string", + }); + } + if ( + collections !== undefined && + (!Array.isArray(collections) || !collections.every((c) => isNonEmptyString(c))) + ) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: collections must be an array of non-empty strings", + }); + } + if ( + sampleSize !== undefined && + (typeof sampleSize !== "number" || isNaN(sampleSize) || sampleSize <= 0) + ) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: sampleSize must be a positive number", + }); + } + if ( + timeoutMs !== undefined && + (typeof timeoutMs !== "number" || isNaN(timeoutMs) || timeoutMs <= 0) + ) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: timeoutMs must be a positive number", + }); + } + + const data = await this.engine.inferRelationships(dbId, { + projectId, + collections, + sampleSize, + timeoutMs, + maxCollections: typeof maxCollections === "number" ? maxCollections : undefined, + }); + + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.logger.error({ err, params }, "mongo.relationship.infer failed"); + this.rpc.sendError(id, { + code: err?.code || "INTERNAL_ERROR", + message: err?.message || String(err), + }); + } + } + + async handleList(params: any, id: number | string): Promise { + try { + const { dbId, projectId } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (projectId !== undefined && !isNonEmptyString(projectId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: projectId must be a non-empty string", + }); + } + + const data = await this.engine.listRelationships(dbId, projectId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.logger.error({ err, params }, "mongo.relationship.list failed"); + this.rpc.sendError(id, { + code: err?.code || "INTERNAL_ERROR", + message: err?.message || String(err), + }); + } + } + + async handleSetStatus(params: any, id: number | string): Promise { + try { + const { dbId, projectId, relationshipId, status } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(relationshipId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: relationshipId", + }); + } + if (status !== "confirmed" && status !== "rejected") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: 'Parameter status must be either "confirmed" or "rejected"', + }); + } + if (projectId !== undefined && !isNonEmptyString(projectId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: projectId must be a non-empty string", + }); + } + + const data = await this.engine.setRelationshipStatus(dbId, relationshipId, status, projectId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.logger.error({ err, params }, "mongo.relationship.setStatus failed"); + this.rpc.sendError(id, { + code: err?.code || "INTERNAL_ERROR", + message: err?.message || String(err), + }); + } + } + + async handleSave(params: any, id: number | string): Promise { + try { + const { dbId, projectId, relationship } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (projectId !== undefined && !isNonEmptyString(projectId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: projectId must be a non-empty string", + }); + } + if (!relationship || typeof relationship !== "object" || Array.isArray(relationship)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: relationship", + }); + } + + const { + id: relId, + sourceCollection, + sourceField, + targetCollection, + targetField, + cardinality, + } = relationship; + if ( + !isNonEmptyString(relId) || + !isNonEmptyString(sourceCollection) || + !isNonEmptyString(sourceField) || + !isNonEmptyString(targetCollection) || + !isNonEmptyString(targetField) + ) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: + "Relationship object missing required non-empty string fields (id, sourceCollection, sourceField, targetCollection, targetField)", + }); + } + + const validCardinalities = [ + "one-to-one", + "one-to-many", + "many-to-one", + "many-to-many", + "unknown", + "insufficient-evidence", + ]; + if (!validCardinalities.includes(cardinality)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: + 'Invalid cardinality: must be one of "one-to-one", "one-to-many", "many-to-one", "many-to-many", "unknown", "insufficient-evidence"', + }); + } + + const data = await this.engine.saveRelationship(dbId, relationship, projectId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.logger.error({ err, params }, "mongo.relationship.save failed"); + this.rpc.sendError(id, { + code: err?.code || "INTERNAL_ERROR", + message: err?.message || String(err), + }); + } + } + + async handleDelete(params: any, id: number | string): Promise { + try { + const { dbId, projectId, relationshipId } = params || {}; + if (!isNonEmptyString(dbId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: dbId", + }); + } + if (!isNonEmptyString(relationshipId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Missing or invalid required parameter: relationshipId", + }); + } + if (projectId !== undefined && !isNonEmptyString(projectId)) { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Invalid parameter: projectId must be a non-empty string", + }); + } + + const data = await this.engine.deleteRelationship(dbId, relationshipId, projectId); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (err: any) { + this.logger.error({ err, params }, "mongo.relationship.delete failed"); + this.rpc.sendError(id, { + code: err?.code || "INTERNAL_ERROR", + message: err?.message || String(err), + }); + } + } +} diff --git a/bridge/src/handlers/queryHandlers.ts b/bridge/src/handlers/queryHandlers.ts index 9252370a..931849b1 100644 --- a/bridge/src/handlers/queryHandlers.ts +++ b/bridge/src/handlers/queryHandlers.ts @@ -3,7 +3,532 @@ import { DatabaseService } from "../services/databaseService"; import { QueryExecutor } from "../services/queryExecutor"; import { SessionManager } from "../sessionManager"; import { getConnector } from "../services/connectorRegistry"; +import { + MongoApiError, + mapMongoError, + getClientKey, + closeMongoClientByKey, + initiateCursorClose, +} from "../connectors/mongo"; import { Logger } from "pino"; +import logger from "../services/logger"; +import { v4 as uuidv4 } from "uuid"; +import { + DriverSettlementSignal, + MongoExecutionContext, + OperationLifecycleHandle, + TeardownOutcome, +} from "../types/mongo"; + +// --------------------------------------------------------------------------- +// Driver Settlement Signal Constructor +// --------------------------------------------------------------------------- + +export function createDriverSettlementSignal(driverPromise?: Promise): { + signal: DriverSettlementSignal; + markSettled: (outcome?: "fulfilled" | "rejected", error?: unknown) => void; + attachPromise: (promise: Promise) => void; +} { + let isSettled = false; + let settledAt: number | undefined; + let settledOutcome: "fulfilled" | "rejected" | undefined; + let settledError: unknown; + let resolveSettlement!: (val: { outcome: "fulfilled" | "rejected"; error?: unknown }) => void; + + const settlementPromise = new Promise<{ outcome: "fulfilled" | "rejected"; error?: unknown }>( + (resolve) => { + resolveSettlement = resolve; + }, + ); + + const markSettled = (outcome: "fulfilled" | "rejected" = "fulfilled", error?: unknown) => { + if (isSettled) return; + isSettled = true; + settledAt = Date.now(); + settledOutcome = outcome; + settledError = error; + resolveSettlement({ outcome, error }); + }; + + const attachPromise = (promise: Promise) => { + promise.then(() => markSettled("fulfilled")).catch((err) => markSettled("rejected", err)); + }; + + if (driverPromise) { + attachPromise(driverPromise); + } + + const signal: DriverSettlementSignal = { + get isSettled() { + return isSettled; + }, + get settledAt() { + return settledAt; + }, + get outcome() { + return settledOutcome; + }, + get error() { + return settledError; + }, + promise: settlementPromise, + markSettled, + }; + + return { signal, markSettled, attachPromise }; +} + +// --------------------------------------------------------------------------- +// Zombie Operation Registry & Active 60s Escalation +// --------------------------------------------------------------------------- + +export interface ZombieOperation { + dbId: string; + clientKey: string; + handle: OperationLifecycleHandle; + timer: NodeJS.Timeout; + registeredAt: number; +} + +export class ZombieOperationRegistry { + private static instance: ZombieOperationRegistry; + private zombies = new Map(); + private teardownInFlight = new Map>(); + public clientTeardownInFlight = new Map>(); + private isShuttingDown = false; + + public static readonly MAX_ZOMBIES_PER_DB = 5; + public static readonly ZOMBIE_MAX_TTL_MS = 60000; + + public static getInstance(): ZombieOperationRegistry { + if (!ZombieOperationRegistry.instance) { + ZombieOperationRegistry.instance = new ZombieOperationRegistry(); + } + return ZombieOperationRegistry.instance; + } + + public static resetForTesting(): void { + if (ZombieOperationRegistry.instance) { + ZombieOperationRegistry.instance.snapshotAndDisarm(); + ZombieOperationRegistry.instance = new ZombieOperationRegistry(); + } + } + + registerZombie( + handle: OperationLifecycleHandle, + clientKey: string, + onEscalate: () => void, + ): boolean { + if (this.isShuttingDown) return false; + + // Disarm previous timer if any + const existing = this.zombies.get(handle.id); + if (existing) { + clearTimeout(existing.timer); + } + + const timer = setTimeout(() => { + this.zombies.delete(handle.id); + onEscalate(); + }, ZombieOperationRegistry.ZOMBIE_MAX_TTL_MS); + + if (timer.unref) timer.unref(); + + this.zombies.set(handle.id, { + dbId: handle.dbId, + clientKey, + handle, + timer, + registeredAt: Date.now(), + }); + + handle.state = "zombie_pinned"; + return true; + } + + unregisterZombie(handleId: string): void { + const zombie = this.zombies.get(handleId); + if (zombie) { + clearTimeout(zombie.timer); + this.zombies.delete(handleId); + } + } + + getZombiesForDb(dbId: string): ZombieOperation[] { + return Array.from(this.zombies.values()).filter((z) => z.dbId === dbId); + } + + snapshotAndDisarm(): Promise[] { + this.isShuttingDown = true; + + // Disarm all active 60s timers and retry timers + for (const zombie of this.zombies.values()) { + clearTimeout(zombie.timer); + if (zombie.handle.retryTimer) { + clearTimeout(zombie.handle.retryTimer); + } + } + this.zombies.clear(); + + // Snapshot in-flight teardown promises (adopted by shutdown drain via Promise.allSettled) + return Array.from(this.teardownInFlight.values()); + } + + async executeDebouncedTeardown( + dbId: string, + clientKey: string, + action: () => Promise, + ): Promise { + if (this.isShuttingDown) { + return { status: "shutdown_handoff", reason: "shutdown_adopted", durationMs: 0 }; + } + + let pending = this.teardownInFlight.get(dbId); + if (!pending) { + pending = (async () => { + try { + return await action(); + } finally { + this.teardownInFlight.delete(dbId); + this.clientTeardownInFlight.delete(clientKey); + } + })(); + this.teardownInFlight.set(dbId, pending); + this.clientTeardownInFlight.set(clientKey, pending); + } + return pending; + } +} + +// --------------------------------------------------------------------------- +// Unrecoverable Zombie Registry +// --------------------------------------------------------------------------- + +export class UnrecoverableZombieRegistry { + private static instance: UnrecoverableZombieRegistry; + private unrecoverable = new Map(); + + public static getInstance(): UnrecoverableZombieRegistry { + if (!UnrecoverableZombieRegistry.instance) { + UnrecoverableZombieRegistry.instance = new UnrecoverableZombieRegistry(); + } + return UnrecoverableZombieRegistry.instance; + } + + public static resetForTesting(): void { + UnrecoverableZombieRegistry.instance = new UnrecoverableZombieRegistry(); + } + + register(handle: OperationLifecycleHandle): void { + handle.state = "unrecoverable_pinned"; + this.unrecoverable.set(handle.id, handle); + } + + unregister(handleId: string): void { + // Strictly does NOT release lease! + this.unrecoverable.delete(handleId); + } + + get(handleId: string): OperationLifecycleHandle | undefined { + return this.unrecoverable.get(handleId); + } + + has(handleId: string): boolean { + return this.unrecoverable.has(handleId); + } + + size(): number { + return this.unrecoverable.size; + } +} + +// --------------------------------------------------------------------------- +// Resilient Independent Teardown Engine with Signal Racing +// --------------------------------------------------------------------------- + +export async function executeIndependentTeardown( + dbService: DatabaseService, + dbId: string, + conn: any, + handle: OperationLifecycleHandle, + attempt: number, + maxAttempts: number, +): Promise { + const startedAt = Date.now(); + + // 1. Fast Pre-Check: Was the driver promise already settled? + if (handle.settlementSignal?.isSettled) { + return { status: "driver_settled", durationMs: Date.now() - startedAt }; + } + + let invalidationSucceeded = false; + let invalidationError: unknown; + + // 2. Race Invalidation against Natural Settlement + try { + const invalPromise = dbService.invalidateConnection(dbId); + const raceResult = await Promise.race([ + invalPromise.then(() => ({ type: "inval_done" as const })), + handle.settlementSignal.promise.then(() => ({ type: "driver_settled" as const })), + ]); + + if (raceResult.type === "driver_settled" || handle.settlementSignal.isSettled) { + return { status: "driver_settled", durationMs: Date.now() - startedAt }; + } + invalidationSucceeded = true; + } catch (err) { + invalidationError = err; + logger.warn( + { dbId, err }, + "Pool invalidation threw; proceeding to independent socket teardown", + ); + } + + // 3. Post-Invalidation Settlement Check + if (handle.settlementSignal.isSettled) { + return { status: "driver_settled", durationMs: Date.now() - startedAt }; + } + + // 4. Race Forced Socket Destruction against Natural Settlement + let socketResult: { status: string; destroyedCount?: number; error?: unknown }; + try { + const clientKey = getClientKey(conn); + const socketDestructionPromise = closeMongoClientByKey(clientKey, { + force: true, + timeoutMs: 100, + }); + + const socketRace = await Promise.race([ + socketDestructionPromise.then((res: any) => ({ type: "destroyed" as const, res })), + handle.settlementSignal.promise.then(() => ({ type: "driver_settled" as const })), + ]); + + if (socketRace.type === "driver_settled" || handle.settlementSignal.isSettled) { + return { status: "driver_settled", durationMs: Date.now() - startedAt }; + } + + socketResult = (socketRace as any).res || { status: "destroy_confirmed" }; + } catch (err) { + socketResult = { status: "destroy_failed", error: err }; + } + + const durationMs = Date.now() - startedAt; + + // 5. Outcome Evaluation + if (socketResult.status === "destroy_confirmed") { + return { + status: "destroy_confirmed", + invalidationSucceeded, + socketsDestroyed: socketResult.destroyedCount ?? 1, + durationMs, + }; + } + + if (attempt < maxAttempts) { + return { + status: "retryable_failure", + invalidationSucceeded, + socketError: socketResult.error, + invalidationError, + attempt, + durationMs, + }; + } + + return { + status: "unrecoverable_failure", + invalidationSucceeded, + socketError: socketResult.error, + invalidationError, + totalAttempts: attempt, + durationMs, + }; +} + +// --------------------------------------------------------------------------- +// Authoritative Operation Lifecycle Supervisor +// --------------------------------------------------------------------------- + +export class OperationLifecycleSupervisor { + private hasReleasedLease = false; + private readonly maxTeardownAttempts = 3; + + constructor( + private handle: OperationLifecycleHandle, + private leaseReleaseFn: () => void, + private dbService: DatabaseService, + private conn: any, + private zombieRegistry: ZombieOperationRegistry = ZombieOperationRegistry.getInstance(), + private unrecoverableRegistry: UnrecoverableZombieRegistry = UnrecoverableZombieRegistry.getInstance(), + ) {} + + /** + * The Authoritative Lease-Release Invariant: + * Exclusive Ownership: OperationLifecycleSupervisor alone possesses and calls safeReleaseLease(). + * Condition A: Driver promise settled naturally (handle.settlementSignal.isSettled === true). + * Condition B: Sockets confirmed destroyed (status === "destroy_confirmed"), AND supervisor has subsequently awaited actual driver settlement (await handle.settlementSignal.promise). + */ + public safeReleaseLease(): void { + if (this.hasReleasedLease) return; + if (!this.handle.settlementSignal.isSettled) { + // Lease release is forbidden while driver promise remains unsettled! + return; + } + this.hasReleasedLease = true; + this.handle.hasReleasedLease = true; + this.handle.state = "completed"; + try { + this.leaseReleaseFn(); + } catch (err) { + logger.error({ err, handleId: this.handle.id }, "Error releasing connection lease"); + } + } + + async superviseTimeoutOrCancel( + driverPromiseRef: Promise | undefined, + outerPromise: Promise, + cleanRecoveryTimeoutMs: number = 5000, + ): Promise { + this.handle.state = "rpc_rejected_driver_pending"; + + // Pre-driver cancellation gate: If cancelled before driver promise creation + if (!driverPromiseRef) { + await outerPromise.catch(() => {}); + this.handle.settlementSignal.markSettled(); + this.safeReleaseLease(); + return; + } + + // Launch cursor close with 2000ms deadline if cursor exists + if (this.handle.cursor) { + initiateCursorClose(this.handle.cursor) + .then((outcome) => { + this.handle.cursorCloseOutcome = outcome; + }) + .catch(() => {}); + } + + // Fast check: did driver promise already settle? + if (this.handle.settlementSignal.isSettled) { + this.safeReleaseLease(); + return; + } + + // clean recovery deadline racing + let deadlineTimer: NodeJS.Timeout | undefined; + const cleanDeadlinePromise = new Promise<{ type: "deadline" }>((resolve) => { + deadlineTimer = setTimeout(() => resolve({ type: "deadline" }), cleanRecoveryTimeoutMs); + }); + + const race5s = await Promise.race([ + this.handle.settlementSignal.promise.then(() => ({ type: "driver_settled" as const })), + cleanDeadlinePromise, + ]); + + if (deadlineTimer) clearTimeout(deadlineTimer); + + if (race5s.type === "driver_settled" || this.handle.settlementSignal.isSettled) { + this.safeReleaseLease(); + return; + } + + // Operation is hung past deadline. Check sibling concurrency. + const clientKey = getClientKey(this.conn); + const poolActiveOps = (this.dbService as any).connectionPool?.getActiveOperationsCount() ?? 1; + + if (poolActiveOps <= 1) { + // Deadlock-free sole op recovery: initiate teardown immediately + await this.runTeardownLoop(clientKey); + } else { + // Sibling concurrency: pin as zombie with active 60s escalation timer + this.zombieRegistry.registerZombie(this.handle, clientKey, async () => { + await this.runTeardownLoop(clientKey); + }); + } + } + + public async runTeardownLoop(clientKey: string): Promise { + this.handle.state = "teardown_initiated"; + + const delays = [0, 1000, 3000]; + + for (let attempt = 1; attempt <= this.maxTeardownAttempts; attempt++) { + // Check settlement before retry delay + if (this.handle.settlementSignal.isSettled) { + this.safeReleaseLease(); + return; + } + + const delay = delays[attempt - 1]; + if (delay > 0) { + let retryTimer: NodeJS.Timeout | undefined; + const delayPromise = new Promise<{ type: "delay" }>((resolve) => { + retryTimer = setTimeout(() => resolve({ type: "delay" }), delay); + }); + this.handle.retryTimer = retryTimer; + + const delayRace = await Promise.race([ + this.handle.settlementSignal.promise.then(() => ({ type: "driver_settled" as const })), + delayPromise, + ]); + + if (retryTimer) clearTimeout(retryTimer); + this.handle.retryTimer = undefined; + + if (delayRace.type === "driver_settled" || this.handle.settlementSignal.isSettled) { + this.safeReleaseLease(); + return; + } + } + + const outcome = await this.zombieRegistry.executeDebouncedTeardown( + this.handle.dbId, + clientKey, + () => + executeIndependentTeardown( + this.dbService, + this.handle.dbId, + this.conn, + this.handle, + attempt, + this.maxTeardownAttempts, + ), + ); + + if (outcome.status === "driver_settled" || this.handle.settlementSignal.isSettled) { + this.safeReleaseLease(); + return; + } + + if (outcome.status === "destroy_confirmed") { + // Condition B: Sockets confirmed destroyed. + // Await actual driver settlement (caused by MongoNetworkError from destroyed socket) + await this.handle.settlementSignal.promise; + this.safeReleaseLease(); + return; + } + + if (outcome.status === "shutdown_handoff") { + this.handle.state = "shutdown_adopted"; + // Lease remains held until Phase 4 socket destruction unblocks driver + return; + } + + if (outcome.status === "retryable_failure") { + this.handle.state = "teardown_failed_pinned"; + // Continue to next retry attempt + continue; + } + + if (outcome.status === "unrecoverable_failure") { + this.handle.state = "unrecoverable_pinned"; + this.unrecoverableRegistry.register(this.handle); + // Lease remains held until driver settlement or process termination + return; + } + } + } +} export class QueryHandlers { constructor( @@ -336,4 +861,475 @@ export class QueryHandlers { this.rpc.sendError(id, { code: "IO_ERROR", message: String(e) }); } } + + // ---- M2: MongoDB document engine handlers (docs/MONGODB_M2_SPEC.md §13) ---- + + /** Map a MongoApiError (or driver error) to the stable M2 error envelope. */ + private sendMongoError(id: number | string, e: unknown): void { + const mapped = + e instanceof MongoApiError ? { code: e.code, message: e.message } : mapMongoError(e); + this.logger.error({ err: e, code: mapped.code }, "MongoDB document operation failed"); + this.rpc.sendError(id, { code: mapped.code, message: mapped.message }); + } + + async handleFetchDocuments(params: any, id: number | string) { + try { + const { dbId, collection, filter, sort, projection, page, pageSize } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.findDocuments(conn, dbType, { + collection, + filter, + sort, + projection, + page, + pageSize, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleInsertDocument(params: any, id: number | string) { + try { + const { dbId, collection, document } = params || {}; + if (!dbId || !collection || !document) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or document", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.insertDocument(conn, dbType, { collection, document }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleUpdateDocument(params: any, id: number | string) { + try { + const { dbId, collection, filter, update, options } = params || {}; + if (!dbId || !collection || !filter || !update) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, filter, or update", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.updateDocument(conn, dbType, { + collection, + filter, + update, + options, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleDeleteDocument(params: any, id: number | string) { + try { + const { dbId, collection, filter, options } = params || {}; + if (!dbId || !collection || !filter) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or filter", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.deleteDocument(conn, dbType, { + collection, + filter, + options, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleRunAggregation(params: any, id: number | string) { + try { + const { dbId, collection, pipeline, options } = params || {}; + if (!dbId || !collection || !pipeline) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or pipeline", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.runAggregation(conn, dbType, { + collection, + pipeline, + options, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleListCollections(params: any, id: number | string) { + try { + const { dbId } = params || {}; + if (!dbId) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.listCollections(conn, dbType, params), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleGetInferredSchema(params: any, id: number | string) { + try { + const { dbId, collection, sampleSize, maxDepth } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.inferCollectionSchema(conn, dbType, { + collection, + sampleSize, + maxDepth, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleReplaceDocument(params: any, id: number | string) { + try { + const { dbId, collection, filter, replacement, options } = params || {}; + if (!dbId || !collection || !filter || !replacement) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, filter, or replacement", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.replaceDocument(conn, dbType, { + collection, + filter, + replacement, + options, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleCreateCollection(params: any, id: number | string) { + try { + const { dbId, collection, options } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.createCollection(conn, dbType, { + collection, + options, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleDropCollection(params: any, id: number | string) { + try { + const { dbId, collection } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const data = await this.dbService.withConnection(dbId, (conn, _db, dbType) => + this.queryExecutor.dropCollection(conn, dbType, { + collection, + }), + ); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleExecuteMql(params: any, id: number | string) { + const { dbId, collection, op, options, sessionId } = params || {}; + if (!dbId || !collection || !op) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or op", + }); + } + + const sid = sessionId || uuidv4(); + let lease: { conn: any; db: any; dbType: any; release: () => void } | null = null; + let driverPromiseRef: Promise | undefined; + + try { + if (typeof (this.dbService as any).acquireConnection === "function") { + lease = await this.dbService.acquireConnection(dbId); + } else { + const connInfo = await this.dbService.getDatabaseConnection(dbId); + lease = { + conn: connInfo.conn, + dbType: connInfo.dbType, + db: undefined, + release: () => {}, + }; + } + } catch (err: any) { + return this.sendMongoError(id, err); + } + + const abortController = new AbortController(); + const { + signal: settlementSignal, + markSettled: _markSettled, + attachPromise, + } = createDriverSettlementSignal(); + + const handle: OperationLifecycleHandle = { + id: sid, + dbId, + state: "running", + settlementSignal, + }; + + const supervisor = new OperationLifecycleSupervisor( + handle, + () => { + if (lease) { + lease.release(); + lease = null; + } + }, + this.dbService, + lease.conn, + ); + + let isRpcCompleted = false; + + // Register cancel in SessionManager + this.sessions.registerCancel(sid, () => { + abortController.abort(); + }); + + const context: MongoExecutionContext = { + signal: abortController.signal, + onCursorCreated: (cursor) => { + handle.cursor = cursor; + }, + onDriverPromiseCreated: (p) => { + driverPromiseRef = p; + attachPromise(p); + }, + }; + + const outerPromise = this.queryExecutor.executeMql( + lease.conn, + lease.dbType, + { + collection, + op, + options, + }, + context, + ); + + try { + const data = await outerPromise; + if (!isRpcCompleted) { + isRpcCompleted = true; + this.rpc.sendResponse(id, { ok: true, data }); + } + supervisor.safeReleaseLease(); + this.sessions?.remove?.(sid); + } catch (err: any) { + if (!isRpcCompleted) { + isRpcCompleted = true; + if (abortController.signal.aborted) { + this.rpc.sendError(id, { + code: "MONGO_OPERATION_CANCELLED", + message: "Operation was cancelled by user", + }); + } else if ( + err?.code === "MONGO_OPERATION_TIMEOUT" || + String(err?.message).includes("timed out") + ) { + this.rpc.sendError(id, { + code: "MONGO_OPERATION_TIMEOUT", + message: "Operation timed out", + }); + } else { + this.sendMongoError(id, err); + } + } + + if ( + abortController.signal.aborted || + err?.code === "MONGO_OPERATION_TIMEOUT" || + String(err?.message).includes("timed out") + ) { + await supervisor.superviseTimeoutOrCancel(driverPromiseRef, outerPromise); + } else { + supervisor.safeReleaseLease(); + } + this.sessions?.remove?.(sid); + } + } + + async handleExplainMql(params: any, id: number | string) { + const { dbId, collection, target, verbosity, options, sessionId } = params || {}; + if (!dbId || !collection || !target) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or target", + }); + } + + const sid = sessionId || uuidv4(); + let lease: { conn: any; db: any; dbType: any; release: () => void } | null = null; + let driverPromiseRef: Promise | undefined; + + try { + if (typeof (this.dbService as any).acquireConnection === "function") { + lease = await this.dbService.acquireConnection(dbId); + } else { + const connInfo = await this.dbService.getDatabaseConnection(dbId); + lease = { + conn: connInfo.conn, + dbType: connInfo.dbType, + db: undefined, + release: () => {}, + }; + } + } catch (err: any) { + return this.sendMongoError(id, err); + } + + const abortController = new AbortController(); + const { + signal: settlementSignal, + markSettled: _markSettled, + attachPromise, + } = createDriverSettlementSignal(); + + const handle: OperationLifecycleHandle = { + id: sid, + dbId, + state: "running", + settlementSignal, + }; + + const supervisor = new OperationLifecycleSupervisor( + handle, + () => { + if (lease) { + lease.release(); + lease = null; + } + }, + this.dbService, + lease.conn, + ); + + let isRpcCompleted = false; + + this.sessions.registerCancel(sid, () => { + abortController.abort(); + }); + + const context: MongoExecutionContext = { + signal: abortController.signal, + onCursorCreated: (cursor) => { + handle.cursor = cursor; + }, + onDriverPromiseCreated: (p) => { + driverPromiseRef = p; + attachPromise(p); + }, + }; + + const outerPromise = this.queryExecutor.explainMql( + lease.conn, + lease.dbType, + { collection, target, verbosity, options }, + context, + ); + + try { + const data = await outerPromise; + if (!isRpcCompleted) { + isRpcCompleted = true; + this.rpc.sendResponse(id, { ok: true, data }); + } + supervisor.safeReleaseLease(); + this.sessions?.remove?.(sid); + } catch (err: any) { + if (!isRpcCompleted) { + isRpcCompleted = true; + if (abortController.signal.aborted) { + this.rpc.sendError(id, { + code: "MONGO_OPERATION_CANCELLED", + message: "Operation was cancelled by user", + }); + } else if ( + err?.code === "MONGO_OPERATION_TIMEOUT" || + String(err?.message).includes("timed out") + ) { + this.rpc.sendError(id, { + code: "MONGO_OPERATION_TIMEOUT", + message: "Operation timed out", + }); + } else { + this.sendMongoError(id, err); + } + } + + if ( + abortController.signal.aborted || + err?.code === "MONGO_OPERATION_TIMEOUT" || + String(err?.message).includes("timed out") + ) { + await supervisor.superviseTimeoutOrCancel(driverPromiseRef, outerPromise); + } else { + supervisor.safeReleaseLease(); + } + this.sessions?.remove?.(sid); + } + } } diff --git a/bridge/src/index.ts b/bridge/src/index.ts index 8a26ce14..7e1f1912 100644 --- a/bridge/src/index.ts +++ b/bridge/src/index.ts @@ -4,7 +4,7 @@ import { JsonStdio } from "./jsonRpc"; import logger from "./services/logger"; import { registerDbHandlers } from "./jsonRpcHandler"; import { SessionManager } from "./sessionManager"; -import { connectionPool } from "./services/connectionPool"; +import { databaseServiceInstance } from "./services/databaseService"; const rpc = new JsonStdio(); const sessions = new SessionManager(); @@ -73,36 +73,55 @@ rpc.on("request", async (req: any) => { }); // Graceful shutdown +const SHUTDOWN_WATCHDOG_MS = 2000; let isShuttingDown = false; -function shutdown(signal: string) { + +async function shutdown(signal: string): Promise { if (isShuttingDown) return; isShuttingDown = true; - logger.info(`Bridge received ${signal} — shutting down`); + logger.info(`Bridge received ${signal} — initiating graceful shutdown`); + + // Unconditional watchdog: forces exit if teardown hangs + const watchdog = setTimeout(() => { + logger.error("Graceful shutdown exceeded 2000ms deadline — forcing exit"); + process.exit(0); + }, SHUTDOWN_WATCHDOG_MS); + if (watchdog.unref) watchdog.unref(); + + // 1. Session cleanup (isolated try/catch) try { - sessions.destroy(); + await Promise.resolve(sessions.destroy()); } catch (e) { logger.error({ e }, "Error destroying sessions"); } + + // 2. Database service & connection pool drain/teardown (authoritatively awaited) try { - connectionPool.destroy(); + await databaseServiceInstance.shutdown(); } catch (e) { - logger.error({ e }, "Error destroying connection pool"); + logger.error({ e }, "Error during database service shutdown"); } - // Hard exit fallback in case anything hangs in event loop - setTimeout(() => process.exit(0), 100).unref(); + + // 3. Normal clean exit + clearTimeout(watchdog); process.exit(0); } -process.on("SIGINT", () => shutdown("SIGINT")); -process.on("SIGTERM", () => shutdown("SIGTERM")); + +process.on("SIGINT", () => { + void shutdown("SIGINT"); +}); +process.on("SIGTERM", () => { + void shutdown("SIGTERM"); +}); rpc.on("end", () => { logger.info("JSON-RPC stream ended — shutting down"); - shutdown("RPC_END"); + void shutdown("RPC_END"); }); rpc.on("error", (err: any) => { logger.error({ err }, "JSON-RPC stream error — shutting down"); - shutdown("RPC_ERROR"); + void shutdown("RPC_ERROR"); }); // Parent process watchdog: if parent process (RelWave/Tauri) dies, bridge exits immediately @@ -117,7 +136,7 @@ if (parentPid && parentPid > 0 && parentPid !== process.pid) { process.kill(parentPid, 0); } catch { logger.info({ parentPid }, "Parent process has exited — shutting down bridge"); - shutdown("PARENT_EXITED"); + void shutdown("PARENT_EXITED"); } }, 1000); watchdog.unref(); diff --git a/bridge/src/jsonRpcHandler.ts b/bridge/src/jsonRpcHandler.ts index dfcdb8fc..468de2dc 100644 --- a/bridge/src/jsonRpcHandler.ts +++ b/bridge/src/jsonRpcHandler.ts @@ -1,9 +1,14 @@ import { Rpc } from "./types"; import { SessionManager } from "./sessionManager"; -import { DatabaseService } from "./services/databaseService"; +import { databaseServiceInstance } from "./services/databaseService"; import { QueryExecutor } from "./services/queryExecutor"; import { QueryHandlers } from "./handlers/queryHandlers"; import { DatabaseHandlers } from "./handlers/databaseHandlers"; +import { IndexHandlers } from "./handlers/indexHandlers"; +import { MongoRelationshipEngine } from "./services/mongoRelationshipEngine"; +import { MongoRelationshipHandlers } from "./handlers/mongoRelationshipHandlers"; +import { MongoBulkEngine } from "./services/mongoBulkEngine"; +import { MongoBulkHandlers } from "./handlers/mongoBulkHandlers"; import { SessionHandlers } from "./handlers/sessionHandlers"; import { StatsHandlers } from "./handlers/statsHandlers"; import { MigrationHandlers } from "./handlers/migrationHandlers"; @@ -24,7 +29,7 @@ import { Logger } from "pino"; */ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionManager) { // Initialize services - const dbService = new DatabaseService(); + const dbService = databaseServiceInstance; const queryExecutor = new QueryExecutor(); const monitoringService = new MonitoringService(); const monitoringWebSocketServer = new MonitoringWebSocketServer( @@ -45,6 +50,15 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa const gitAdvancedHandlers = new GitAdvancedHandlers(rpc, logger); const monitoringHandlers = new MonitoringHandlers(rpc, logger, dbService, monitoringService); const aiHandlers = new (require("./handlers/aiHandlers").AIHandlers)(rpc, logger, dbService); + const indexHandlers = new IndexHandlers(rpc, logger, dbService, queryExecutor); + const mongoRelationshipEngine = new MongoRelationshipEngine(dbService); + const mongoRelationshipHandlers = new MongoRelationshipHandlers( + rpc, + logger, + mongoRelationshipEngine, + ); + const mongoBulkEngine = new MongoBulkEngine(dbService); + const mongoBulkHandlers = new MongoBulkHandlers(rpc, logger, mongoBulkEngine); // ========================================== // SESSION MANAGEMENT HANDLERS @@ -70,6 +84,17 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "query.updateRow", (p, id) => queryHandlers.handleUpdateRow(p, id)); rpcRegister(rpc, "query.deleteRow", (p, id) => queryHandlers.handleDeleteRow(p, id)); rpcRegister(rpc, "query.searchTable", (p, id) => queryHandlers.handleSearchTable(p, id)); + // M2: MongoDB document engine (docs/MONGODB_M2_SPEC.md §13) + rpcRegister(rpc, "query.fetchDocuments", (p, id) => queryHandlers.handleFetchDocuments(p, id)); + rpcRegister(rpc, "query.findDocuments", (p, id) => queryHandlers.handleFetchDocuments(p, id)); + rpcRegister(rpc, "query.insertDocument", (p, id) => queryHandlers.handleInsertDocument(p, id)); + rpcRegister(rpc, "query.updateDocument", (p, id) => queryHandlers.handleUpdateDocument(p, id)); + rpcRegister(rpc, "query.replaceDocument", (p, id) => queryHandlers.handleReplaceDocument(p, id)); + rpcRegister(rpc, "query.deleteDocument", (p, id) => queryHandlers.handleDeleteDocument(p, id)); + rpcRegister(rpc, "query.runAggregation", (p, id) => queryHandlers.handleRunAggregation(p, id)); + // M5: MongoDB MQL Workspace & Read-Only Engine (docs/MONGODB_M5_SPEC.md §5) + rpcRegister(rpc, "query.executeMql", (p, id) => queryHandlers.handleExecuteMql(p, id)); + rpcRegister(rpc, "query.explainMql", (p, id) => queryHandlers.handleExplainMql(p, id)); // ========================================== // DATABASE CRUD HANDLERS @@ -87,6 +112,60 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "db.listTables", (p, id) => databaseHandlers.handleListTables(p, id)); rpcRegister(rpc, "db.getSchema", (p, id) => databaseHandlers.handleGetSchema(p, id)); rpcRegister(rpc, "db.listSchemas", (p, id) => databaseHandlers.handleListSchemas(p, id)); + rpcRegister(rpc, "db.listCollections", (p, id) => queryHandlers.handleListCollections(p, id)); + rpcRegister(rpc, "db.getInferredSchema", (p, id) => queryHandlers.handleGetInferredSchema(p, id)); + rpcRegister(rpc, "db.createCollection", (p, id) => queryHandlers.handleCreateCollection(p, id)); + rpcRegister(rpc, "db.dropCollection", (p, id) => queryHandlers.handleDropCollection(p, id)); + + // ========================================== + // INDEX MANAGEMENT HANDLERS (docs/MONGODB_M6_SPEC.md §11) + // ========================================== + rpcRegister(rpc, "index.list", (p, id) => indexHandlers.handleListIndexes(p, id)); + rpcRegister(rpc, "index.create", (p, id) => indexHandlers.handleCreateIndex(p, id)); + rpcRegister(rpc, "index.drop", (p, id) => indexHandlers.handleDropIndex(p, id)); + rpcRegister(rpc, "index.getStats", (p, id) => indexHandlers.handleGetIndexStats(p, id)); + rpcRegister(rpc, "index.hide", (p, id) => indexHandlers.handleHideIndex(p, id)); + + // ========================================== + // MONGO RELATIONSHIP HANDLERS (M7 Phase 1) + // ========================================== + rpcRegister(rpc, "mongo.relationship.infer", (p, id) => + mongoRelationshipHandlers.handleInfer(p, id), + ); + rpcRegister(rpc, "mongo.relationship.list", (p, id) => + mongoRelationshipHandlers.handleList(p, id), + ); + rpcRegister(rpc, "mongo.relationship.setStatus", (p, id) => + mongoRelationshipHandlers.handleSetStatus(p, id), + ); + rpcRegister(rpc, "mongo.relationship.save", (p, id) => + mongoRelationshipHandlers.handleSave(p, id), + ); + rpcRegister(rpc, "mongo.relationship.delete", (p, id) => + mongoRelationshipHandlers.handleDelete(p, id), + ); + + // ========================================== + // MONGO BULK OPERATIONS HANDLERS (M7 Phase 3) + // ========================================== + rpcRegister(rpc, "mongo.bulk.importStart", (p, id) => mongoBulkHandlers.handleImportStart(p, id)); + rpcRegister(rpc, "mongo.bulk.importChunk", (p, id) => mongoBulkHandlers.handleImportChunk(p, id)); + rpcRegister(rpc, "mongo.bulk.exportStart", (p, id) => mongoBulkHandlers.handleExportStart(p, id)); + rpcRegister(rpc, "mongo.bulk.getOperationStatus", (p, id) => + mongoBulkHandlers.handleGetOperationStatus(p, id), + ); + rpcRegister(rpc, "mongo.bulk.cancelOperation", (p, id) => + mongoBulkHandlers.handleCancelOperation(p, id), + ); + rpcRegister(rpc, "mongo.bulk.cleanupOperation", (p, id) => + mongoBulkHandlers.handleCleanupOperation(p, id), + ); + rpcRegister(rpc, "mongo.bulk.exportSaveTo", (p, id) => + mongoBulkHandlers.handleExportSaveTo(p, id), + ); + rpcRegister(rpc, "mongo.bulk.insert", (p, id) => mongoBulkHandlers.handleBulkInsert(p, id)); + rpcRegister(rpc, "mongo.bulk.update", (p, id) => mongoBulkHandlers.handleBulkUpdate(p, id)); + rpcRegister(rpc, "mongo.bulk.delete", (p, id) => mongoBulkHandlers.handleBulkDelete(p, id)); // ========================================== // MIGRATION HANDLERS @@ -100,6 +179,13 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "migration.generateDrop", (p, id) => migrationHandlers.handleGenerateDropMigration(p, id), ); + rpcRegister(rpc, "migration.list", (p, id) => migrationHandlers.handleListMigrations(p, id)); + rpcRegister(rpc, "migration.validate", (p, id) => + migrationHandlers.handleValidateMigration(p, id), + ); + rpcRegister(rpc, "migration.getStatus", (p, id) => + migrationHandlers.handleGetMigrationStatus(p, id), + ); rpcRegister(rpc, "migration.apply", (p, id) => migrationHandlers.handleApplyMigration(p, id)); rpcRegister(rpc, "migration.applyMigrations", (p, id) => migrationHandlers.handleApplyMigrations(p, id), diff --git a/bridge/src/services/connectionBuilder.ts b/bridge/src/services/connectionBuilder.ts index 3e91e98d..d53b3919 100644 --- a/bridge/src/services/connectionBuilder.ts +++ b/bridge/src/services/connectionBuilder.ts @@ -1,14 +1,41 @@ import { DBType, DatabaseConfig } from "../types"; import { SQLiteConfig } from "../types/sqlite"; +import { MongoConfig } from "../types/mongo"; import { isWindowsDriveRootPath, normalizeSQLitePath } from "../utils/sqlitePath"; import { parseClickHouseUrl } from "../utils/clickhouseUrl"; +import { buildMongoConnectionUri } from "../utils/mongoUri"; import { sshTunnelServiceInstance, TunnelInfo } from "./sshTunnelService"; +import { ConnectionPool } from "./connectionPool"; export interface BuildResult { - config: DatabaseConfig | SQLiteConfig; + config: DatabaseConfig | SQLiteConfig | MongoConfig; tunnel?: TunnelInfo; } +export async function cleanupStaleBuilderResources( + builtTunnel: any, + builtConfig: any, + builtDbType: DBType | undefined, + connectionPool: ConnectionPool, +): Promise { + // 1. Separate SSH Tunnel Adoption Check (Strict Object Identity Equality): + if (builtTunnel) { + const isTunnelAdopted = Array.from(connectionPool.getAllEntries()).some( + (entry) => entry.tunnel === builtTunnel, + ); + if (!isTunnelAdopted) { + try { + builtTunnel.close(); + } catch {} + } + } + + // 2. Separate MongoClient Adoption Check (Canonical Client-Key Based): + if (builtConfig && builtDbType === DBType.MONGODB) { + await connectionPool.safeCloseMongoClient(builtConfig); + } +} + export class ConnectionBuilder { static async buildConnection(db: any, pwd: string | null, dbType: DBType): Promise { if (dbType === DBType.SQLITE || dbType === DBType.DUCKDB) { @@ -52,6 +79,44 @@ export class ConnectionBuilder { }; } + if (dbType === DBType.MONGODB) { + if (db.ssh) { + const remoteHost = db.host || "127.0.0.1"; + const remotePort = db.port || 27017; + const tunnel = await sshTunnelServiceInstance.createSSHTunnel( + db.ssh, + remoteHost, + remotePort, + ); + const tunnelDb = { ...db, host: "127.0.0.1", port: tunnel.localPort }; + const uri = buildMongoConnectionUri(tunnelDb, pwd); + const config: MongoConfig = { + uri, + database: db.database, + authSource: db.authSource, + replicaSet: db.replicaSet, + tls: db.tls, + directConnection: db.directConnection, + readonly: Boolean(db.readonly || db.readOnly), + }; + return { config, tunnel }; + } + + // The persisted URI is credential-free; the runtime password is re-injected + // here (in memory only) via the sanitized URI + decrypted pwd. + const uri = buildMongoConnectionUri(db, pwd); + const config: MongoConfig = { + uri, + database: db.database, + authSource: db.authSource, + replicaSet: db.replicaSet, + tls: db.tls, + directConnection: db.directConnection, + readonly: Boolean(db.readonly || db.readOnly), + }; + return { config }; + } + const defaultPort = dbType === DBType.MYSQL || dbType === DBType.MARIADB ? 3306 : 5432; const base: DatabaseConfig = { diff --git a/bridge/src/services/connectionPool.ts b/bridge/src/services/connectionPool.ts index 49896b55..4c13d52d 100644 --- a/bridge/src/services/connectionPool.ts +++ b/bridge/src/services/connectionPool.ts @@ -2,108 +2,242 @@ // services/connectionPool.ts // ---------------------------- // -// Lightweight connection pool keyed by dbId. -// Reuses the connector config object so each database is connected once -// per bridge lifetime and is reconnected on-demand after idle eviction. +// Lightweight connection pool keyed by dbId with independent per-caller lease tracking. import { DBType } from "../types"; import { TunnelInfo } from "./sshTunnelService"; +import logger from "./logger"; +import { getClientKey, closeMongoClientByKey, safeMongoError } from "../connectors/mongo"; -interface PoolEntry { - conn: unknown; +export interface PoolEntry { + conn: any; dbType: DBType; lastUsed: number; tunnel?: TunnelInfo; + activeOperations: number; + disposed: boolean; } -/** Idle timeout before a connection is evicted (10 minutes) */ -const IDLE_TIMEOUT_MS = 10 * 60 * 1000; +export interface ConnectionLease { + connection: PoolEntry; + release: () => void; +} -/** How frequently the sweeper checks for idle connections (5 minutes) */ +const IDLE_TIMEOUT_MS = 10 * 60 * 1000; const SWEEP_INTERVAL_MS = 5 * 60 * 1000; export class ConnectionPool { private pool = new Map(); + private pendingDisposal = new Set(); + private buildingClientKeys = new Map(); + private inProgressCleanups = new Set>(); + private isShuttingDown = false; + private _isForcedTeardownInitiated = false; private sweepTimer: ReturnType | null = null; constructor() { this.sweepTimer = setInterval(() => this.sweep(), SWEEP_INTERVAL_MS); - // Allow the process to exit even if this timer is still running if (this.sweepTimer?.unref) this.sweepTimer.unref(); } - /** - * Get a cached connection for the given dbId, or null if not cached. - * Updates lastUsed on hit. - */ - get(dbId: string): { conn: unknown; dbType: DBType } | null { + get isForcedTeardownInitiated(): boolean { + return this._isForcedTeardownInitiated; + } + + setForcedTeardownInitiated(val: boolean): void { + this._isForcedTeardownInitiated = val; + } + + setShuttingDown(val: boolean): void { + this.isShuttingDown = val; + } + + registerBuildingClient(clientKey: string): void { + const current = this.buildingClientKeys.get(clientKey) || 0; + this.buildingClientKeys.set(clientKey, current + 1); + } + + unregisterBuildingClient(clientKey: string): void { + const current = (this.buildingClientKeys.get(clientKey) || 0) - 1; + if (current <= 0) { + this.buildingClientKeys.delete(clientKey); + } else { + this.buildingClientKeys.set(clientKey, current); + } + } + + getBuildingClientCount(clientKey: string): number { + return this.buildingClientKeys.get(clientKey) || 0; + } + + getAllEntries(): PoolEntry[] { + const set = new Set(); + for (const e of this.pool.values()) set.add(e); + for (const e of this.pendingDisposal) set.add(e); + return Array.from(set); + } + + getInProgressCleanups(): Set> { + return this.inProgressCleanups; + } + + getActiveOperationsCount(): number { + let count = 0; + for (const entry of this.pool.values()) count += entry.activeOperations; + for (const entry of this.pendingDisposal) count += entry.activeOperations; + return count; + } + + acquire(dbId: string): ConnectionLease | null { + if (this.isShuttingDown) return null; const entry = this.pool.get(dbId); - if (!entry) return null; + if (!entry || entry.disposed) return null; + + entry.activeOperations++; entry.lastUsed = Date.now(); - return { conn: entry.conn, dbType: entry.dbType }; + + let released = false; + const release = () => { + if (released) return; + released = true; + this.release(dbId, entry); + }; + + return { connection: entry, release }; + } + + release(dbId: string, entry: PoolEntry): void { + entry.activeOperations = Math.max(0, entry.activeOperations - 1); + if ((entry.disposed || this.pendingDisposal.has(entry)) && entry.activeOperations === 0) { + this.pendingDisposal.delete(entry); + this.triggerDeferredCleanup(entry); + } + } + + private triggerDeferredCleanup(entry: PoolEntry): void { + if (this._isForcedTeardownInitiated) return; + + const cleanupPromise = (async () => { + try { + if (this._isForcedTeardownInitiated) return; + if (entry.tunnel) { + try { + entry.tunnel.close(); + } catch {} + } + if (entry.dbType === DBType.MONGODB && entry.conn) { + await this.safeCloseMongoClient(entry.conn); + } + } catch (err: any) { + logger.warn({ err: safeMongoError(err) }, "Deferred cleanup error in ConnectionPool"); + } + })(); + + this.inProgressCleanups.add(cleanupPromise); + cleanupPromise.finally(() => { + this.inProgressCleanups.delete(cleanupPromise); + }); + } + + async safeCloseMongoClient(cfg: any): Promise { + if (this._isForcedTeardownInitiated) return; + const clientKey = getClientKey(cfg); + + const inPool = Array.from(this.pool.values()).some( + (e) => + !e.disposed && e.dbType === DBType.MONGODB && e.conn && getClientKey(e.conn) === clientKey, + ); + const inPending = Array.from(this.pendingDisposal).some( + (e) => + e.activeOperations > 0 && + e.dbType === DBType.MONGODB && + e.conn && + getClientKey(e.conn) === clientKey, + ); + const buildingCount = this.buildingClientKeys.get(clientKey) || 0; + + if (inPool || inPending || buildingCount > 0) { + return; + } + + await closeMongoClientByKey(clientKey, { force: true }); } - /** - * Store a connection for the given dbId. - */ set(dbId: string, conn: unknown, dbType: DBType, tunnel?: TunnelInfo): void { - // If there's an existing entry with a tunnel, close it first + if (this.isShuttingDown) { + throw new Error("Bridge is shutting down: connection publication rejected"); + } const existing = this.pool.get(dbId); - if (existing?.tunnel) { - existing.tunnel.close(); + if (existing) { + existing.disposed = true; + if (existing.activeOperations > 0) { + this.pendingDisposal.add(existing); + } else { + this.triggerDeferredCleanup(existing); + } } - this.pool.set(dbId, { conn, dbType, lastUsed: Date.now(), tunnel }); + this.pool.set(dbId, { + conn, + dbType, + lastUsed: Date.now(), + tunnel, + activeOperations: 0, + disposed: false, + }); } - /** - * Invalidate (evict) the connection for a specific dbId. - * Call this after db.delete, db.update (when credentials change), etc. - */ - invalidate(dbId: string): void { + get(dbId: string): { conn: unknown; dbType: DBType } | null { const entry = this.pool.get(dbId); - if (entry?.tunnel) { - entry.tunnel.close(); - } - this.pool.delete(dbId); + if (!entry || entry.disposed) return null; + entry.lastUsed = Date.now(); + return { conn: entry.conn, dbType: entry.dbType }; } - /** - * Invalidate all cached connections. - * Call this on bridge shutdown. - */ - invalidateAll(): void { - for (const entry of this.pool.values()) { - if (entry.tunnel) { - entry.tunnel.close(); - } + async invalidate(dbId: string): Promise { + const entry = this.pool.get(dbId); + if (!entry) return; + + this.pool.delete(dbId); + entry.disposed = true; + + if (entry.activeOperations > 0) { + this.pendingDisposal.add(entry); + } else { + this.triggerDeferredCleanup(entry); } - this.pool.clear(); } - /** - * Remove connections that have been idle longer than IDLE_TIMEOUT_MS. - */ private sweep(): void { const now = Date.now(); - for (const [id, entry] of this.pool) { - if (now - entry.lastUsed > IDLE_TIMEOUT_MS) { - if (entry.tunnel) { - entry.tunnel.close(); - } + for (const [id, entry] of Array.from(this.pool.entries())) { + if (entry.activeOperations === 0 && now - entry.lastUsed > IDLE_TIMEOUT_MS) { this.pool.delete(id); + entry.disposed = true; + this.triggerDeferredCleanup(entry); } } } - /** Stop the background sweep timer (call on shutdown). */ + clear(): void { + this.pool.clear(); + this.pendingDisposal.clear(); + this.buildingClientKeys.clear(); + } + destroy(): void { if (this.sweepTimer) { clearInterval(this.sweepTimer); this.sweepTimer = null; } - this.invalidateAll(); + for (const entry of this.getAllEntries()) { + if (entry.tunnel) { + try { + entry.tunnel.close(); + } catch {} + } + } + this.clear(); } } -/** Singleton pool instance shared across the bridge process */ export const connectionPool = new ConnectionPool(); diff --git a/bridge/src/services/connectorKind.ts b/bridge/src/services/connectorKind.ts new file mode 100644 index 00000000..37db5830 --- /dev/null +++ b/bridge/src/services/connectorKind.ts @@ -0,0 +1,26 @@ +/** + * Database kind discriminator. + * + * A single switch key for "relational vs document" behavior, replacing scattered + * per-engine checks. M1 only defines it; later milestones consume it to gate + * SQL-only features (migrations, relational schema, SQL editor/builder, ER + * auto-edges, monitoring whitelists, NL-to-SQL). + */ + +import { DBType } from "../types"; + +export type DatabaseKind = "sql" | "nosql"; + +const KIND: Record = { + [DBType.POSTGRES]: "sql", + [DBType.MYSQL]: "sql", + [DBType.MARIADB]: "sql", + [DBType.SQLITE]: "sql", + [DBType.DUCKDB]: "sql", + [DBType.CLICKHOUSE]: "sql", + [DBType.MONGODB]: "nosql", +}; + +export function connectorKind(dbType: DBType): DatabaseKind { + return KIND[dbType]; +} diff --git a/bridge/src/services/connectorRegistry.ts b/bridge/src/services/connectorRegistry.ts index fe84ca45..7f60b577 100644 --- a/bridge/src/services/connectorRegistry.ts +++ b/bridge/src/services/connectorRegistry.ts @@ -13,6 +13,7 @@ import * as mariadbConnector from "../connectors/mariadb"; import * as sqliteConnector from "../connectors/sqlite"; import * as duckdbConnector from "../connectors/duckdb"; import * as clickhouseConnector from "../connectors/clickhouse"; +import * as mongoConnector from "../connectors/mongo"; export type Connector = | typeof postgresConnector @@ -20,7 +21,8 @@ export type Connector = | typeof mariadbConnector | typeof sqliteConnector | typeof duckdbConnector - | typeof clickhouseConnector; + | typeof clickhouseConnector + | typeof mongoConnector; const registry: Record = { [DBType.POSTGRES]: postgresConnector, @@ -29,6 +31,7 @@ const registry: Record = { [DBType.SQLITE]: sqliteConnector, [DBType.DUCKDB]: duckdbConnector, [DBType.CLICKHOUSE]: clickhouseConnector, + [DBType.MONGODB]: mongoConnector, }; /** diff --git a/bridge/src/services/databaseService.ts b/bridge/src/services/databaseService.ts index e23bf6ba..39768d6f 100644 --- a/bridge/src/services/databaseService.ts +++ b/bridge/src/services/databaseService.ts @@ -1,56 +1,243 @@ import { ConnectionBuilder } from "./connectionBuilder"; import { DBTypeDetector } from "../utils/dbTypeDetector"; import { DBType } from "../types"; -import { dbStoreInstance } from "./dbStore"; // always use the singleton -import { connectionPool } from "./connectionPool"; +import { + isMongoUri, + extractMongoCredentials, + sanitizeMongoUri, + normalizeMongoUriMeta, + buildMongoConnectionUri, +} from "../utils/mongoUri"; +import { dbStoreInstance } from "./dbStore"; +import { connectionPool, ConnectionPool } from "./connectionPool"; import { keyringServiceInstance } from "./keyringService"; +import { + getClientKey, + closeAllMongoClients, + setMongoShuttingDown, + safeMongoError, +} from "../connectors/mongo"; +import logger from "./logger"; export class DatabaseService { - /** - * Get a single database's metadata (no password). - */ - async getDatabase(dbId: string) { - return dbStoreInstance.getDB(dbId); + private dbGenerations = new Map(); + private inFlightConnections = new Map>(); + private activeBuilders = new Set>(); + private isShuttingDown = false; + private connectionPool: ConnectionPool = connectionPool; + + getGeneration(dbId: string): number { + return this.dbGenerations.get(dbId) || 0; + } + + async invalidateConnection(dbId: string): Promise { + const nextGen = (this.dbGenerations.get(dbId) || 0) + 1; + this.dbGenerations.set(dbId, nextGen); + this.inFlightConnections.delete(dbId); + await this.connectionPool.invalidate(dbId); + } + + private async ensureConnectionBuilt(dbId: string): Promise { + if (this.isShuttingDown) { + throw new Error("Bridge is shutting down: connection acquisition rejected"); + } + + let pending = this.inFlightConnections.get(dbId); + if (!pending) { + const buildGeneration = this.getGeneration(dbId); + + pending = (async () => { + let builtTunnel: any; + let builtConfig: any; + let builtDbType: DBType | undefined; + let registeredClientKey: string | undefined; + + try { + if (this.isShuttingDown) { + throw new Error("Bridge is shutting down: connection build aborted"); + } + + const db = await dbStoreInstance.getDB(dbId); + if (!db) throw new Error("Database not found"); + const pwd = await dbStoreInstance.getPasswordFor(db); + const dbType = DBTypeDetector.detect(db); + builtDbType = dbType; + + if (db.ssh) { + if (db.ssh.authMethod === "password") { + db.ssh.password = + (await keyringServiceInstance.getCredential(`${dbId}_ssh_pwd`)) || undefined; + } else if (db.ssh.authMethod === "privateKey") { + db.ssh.passphrase = + (await keyringServiceInstance.getCredential(`${dbId}_ssh_pass`)) || undefined; + } + } + + if (this.isShuttingDown || this.getGeneration(dbId) !== buildGeneration) { + throw new Error( + "Connection build aborted: profile was invalidated or bridge is shutting down", + ); + } + + if (dbType === DBType.MONGODB && !db.ssh) { + registeredClientKey = getClientKey({ + uri: buildMongoConnectionUri(db, null), + database: db.database, + }); + this.connectionPool.registerBuildingClient(registeredClientKey); + } + + const res = await ConnectionBuilder.buildConnection(db, pwd, dbType); + builtConfig = res.config; + builtTunnel = res.tunnel; + + if (dbType === DBType.MONGODB && db.ssh && builtConfig) { + registeredClientKey = getClientKey(builtConfig); + this.connectionPool.registerBuildingClient(registeredClientKey); + } + + if (this.isShuttingDown || this.getGeneration(dbId) !== buildGeneration) { + if (builtTunnel) { + const isTunnelAdopted = Array.from(this.connectionPool.getAllEntries()).some( + (entry) => entry.tunnel === builtTunnel, + ); + if (!isTunnelAdopted) { + try { + builtTunnel.close(); + } catch {} + } + } + + if (registeredClientKey) { + this.connectionPool.unregisterBuildingClient(registeredClientKey); + registeredClientKey = undefined; + } + + if (builtConfig && builtDbType === DBType.MONGODB) { + await this.connectionPool.safeCloseMongoClient(builtConfig); + } + + throw new Error( + "Connection build aborted: profile was invalidated or bridge is shutting down", + ); + } + + this.connectionPool.set(dbId, builtConfig, builtDbType, builtTunnel); + } finally { + if (registeredClientKey) { + this.connectionPool.unregisterBuildingClient(registeredClientKey); + } + if (this.inFlightConnections.get(dbId) === pending) { + this.inFlightConnections.delete(dbId); + } + this.activeBuilders.delete(pending); + } + })(); + + this.inFlightConnections.set(dbId, pending); + this.activeBuilders.add(pending); + } + + await pending; + } + + isFatalConnectionError(err: any): boolean { + if (!err) return false; + const name = err.name || ""; + const msg = err.message || ""; + return ( + name === "MongoServerSelectionError" || + name === "MongoNetworkError" || + name === "MongoNetworkTimeoutError" || + name === "MongoTopologyClosedError" || + msg.includes("connection reset by peer") || + msg.includes("ECONNRESET") || + msg.includes("ECONNREFUSED") || + msg.includes("ETIMEDOUT") || + msg.includes("Topology is closed") + ); + } + + async withConnection( + dbId: string, + operation: (conn: any, db: any, dbType: DBType) => Promise, + ): Promise { + if (this.isShuttingDown) { + throw new Error("Bridge is shutting down: connection acquisition rejected"); + } + + let lease = this.connectionPool.acquire(dbId); + + if (!lease) { + await this.ensureConnectionBuilt(dbId); + lease = this.connectionPool.acquire(dbId); + if (!lease) { + throw new Error(`Failed to acquire connection lease for '${dbId}' after build`); + } + } + + const db = await dbStoreInstance.getDB(dbId); + + try { + return await operation(lease.connection.conn, db, lease.connection.dbType); + } catch (err: any) { + if (this.isFatalConnectionError(err)) { + try { + await this.invalidateConnection(dbId); + } catch (invalErr: any) { + logger.error( + { dbId, err: safeMongoError(invalErr), originalErr: safeMongoError(err) }, + "Failed to invalidate connection following fatal database operation error", + ); + } + } + throw err; + } finally { + lease.release(); + } + } + + async acquireConnection( + dbId: string, + ): Promise<{ conn: any; db: any; dbType: DBType; release: () => void }> { + if (this.isShuttingDown) { + throw new Error("Bridge is shutting down: connection acquisition rejected"); + } + + let lease = this.connectionPool.acquire(dbId); + if (!lease) { + await this.ensureConnectionBuilt(dbId); + lease = this.connectionPool.acquire(dbId); + if (!lease) { + throw new Error(`Failed to acquire connection lease for '${dbId}' after build`); + } + } + + const db = await dbStoreInstance.getDB(dbId); + return { + conn: lease.connection.conn, + db, + dbType: lease.connection.dbType, + release: lease.release, + }; } - /** - * Return a connection config for the given dbId. - * Hits the in-memory pool first; builds a fresh config on miss. - * - * NOTE: the connectors create their own client on every call using the - * config object, so the "connection" here is a config descriptor, not an - * open socket. The pool avoids repeated dbStore + password lookups. - */ async getDatabaseConnection(dbId: string): Promise<{ db: unknown; conn: any; dbType: DBType }> { - // Pool hit — skip disk IO entirely - const cached = connectionPool.get(dbId); + const cached = this.connectionPool.get(dbId); if (cached) { const db = await dbStoreInstance.getDB(dbId); return { db, conn: cached.conn, dbType: cached.dbType }; } - // Pool miss — build config and cache it + await this.ensureConnectionBuilt(dbId); + const published = this.connectionPool.get(dbId); + if (!published) throw new Error("Failed to build connection"); const db = await dbStoreInstance.getDB(dbId); - if (!db) throw new Error("Database not found"); - - const pwd = await dbStoreInstance.getPasswordFor(db); - const dbType = DBTypeDetector.detect(db); - - // If SSH is present, retrieve credentials from keyring - if (db.ssh) { - if (db.ssh.authMethod === "password") { - db.ssh.password = - (await keyringServiceInstance.getCredential(`${dbId}_ssh_pwd`)) || undefined; - } else if (db.ssh.authMethod === "privateKey") { - db.ssh.passphrase = - (await keyringServiceInstance.getCredential(`${dbId}_ssh_pass`)) || undefined; - } - } - - const { config, tunnel } = await ConnectionBuilder.buildConnection(db, pwd, dbType); + return { db, conn: published.conn, dbType: published.dbType }; + } - connectionPool.set(dbId, config, dbType, tunnel); - return { db, conn: config, dbType }; + async getDatabase(dbId: string) { + return dbStoreInstance.getDB(dbId); } async listDatabases() { @@ -63,16 +250,16 @@ export class DatabaseService { (payload.type as string | undefined)?.toLowerCase() || "", ); const isClickHouse = (payload.type as string | undefined)?.toLowerCase() === "clickhouse"; + const isMongo = (payload.type as string | undefined)?.toLowerCase() === "mongodb"; const required = isFileBasedDB ? ["name", "database", "type"] - : isClickHouse + : isClickHouse || isMongo ? ["name", "type"] : ["name", "host", "port", "user", "database", "type"]; for (const field of required) { if (!payload[field]) throw new Error(`Missing required field: ${field}`); } - // Extract SSH credentials before saving to dbStore const sshPwd = payload.ssh?.password; const sshPass = payload.ssh?.passphrase; if (payload.ssh) { @@ -81,7 +268,6 @@ export class DatabaseService { } if (isFileBasedDB) { - // NOTE: ConnectionBuilder.buildSQLiteConnection works for duckdb paths too const { config } = await ConnectionBuilder.buildSQLiteConnection(payload); payload = { ...payload, @@ -102,11 +288,33 @@ export class DatabaseService { database: payload.database || parsed.database || "default", ssl: payload.ssl !== undefined ? payload.ssl : parsed.ssl, }; + } else if (isMongo) { + if (!payload.url && !payload.host) { + throw new Error("Missing required field: url"); + } + if (payload.url && isMongoUri(payload.url)) { + const creds = extractMongoCredentials(payload.url); + const meta = normalizeMongoUriMeta(payload.url); + payload = { + ...payload, + url: sanitizeMongoUri(payload.url), + host: payload.host || meta.host, + port: payload.port || meta.port, + user: payload.user || meta.user, + database: payload.database || meta.database, + authSource: payload.authSource ?? meta.authSource, + replicaSet: payload.replicaSet ?? meta.replicaSet, + }; + if (creds.password) { + payload = { ...payload, password: creds.password }; + } + } else { + payload = { ...payload, port: payload.port || 27017, database: payload.database || "" }; + } } const res = await dbStoreInstance.addDB(payload as Parameters[0]); - // Store SSH credentials in keyring if (sshPwd) { await keyringServiceInstance.storeCredential(`${res.id}_ssh_pwd`, sshPwd); } @@ -123,8 +331,8 @@ export class DatabaseService { (payload.type as string | undefined)?.toLowerCase() || "", ); const isClickHouse = (payload.type as string | undefined)?.toLowerCase() === "clickhouse"; + const isMongo = (payload.type as string | undefined)?.toLowerCase() === "mongodb"; - // Extract SSH credentials const sshPwd = payload.ssh?.password; const sshPass = payload.ssh?.passphrase; if (payload.ssh) { @@ -144,6 +352,13 @@ export class DatabaseService { ssl: payload.ssl !== undefined ? payload.ssl : parsed.ssl, }; } + if (isMongo && payload.url && isMongoUri(payload.url)) { + const creds = extractMongoCredentials(payload.url); + payload = { ...payload, url: sanitizeMongoUri(payload.url) }; + if (creds.password && payload.password === undefined) { + payload = { ...payload, password: creds.password }; + } + } if (payload.ssh) { delete payload.ssh.password; delete payload.ssh.passphrase; @@ -166,13 +381,12 @@ export class DatabaseService { } } - connectionPool.invalidate(id); // evict stale cached config + await this.invalidateConnection(id); const res = await dbStoreInstance.updateDB( id, payload as Parameters[1], ); - // Update SSH credentials in keyring if (sshPwd !== undefined) { if (sshPwd) { await keyringServiceInstance.storeCredential(`${id}_ssh_pwd`, sshPwd); @@ -193,10 +407,9 @@ export class DatabaseService { async deleteDatabase(id: string) { if (!id) throw new Error("Missing id"); - connectionPool.invalidate(id); // evict before removing from store + await this.invalidateConnection(id); const res = await dbStoreInstance.deleteDB(id); - // Delete SSH credentials from keyring await keyringServiceInstance.deleteCredential(`${id}_ssh_pwd`); await keyringServiceInstance.deleteCredential(`${id}_ssh_pass`); @@ -207,4 +420,66 @@ export class DatabaseService { if (!id) throw new Error("Missing id"); return dbStoreInstance.touchDB(id); } + + private async drainActiveLeases(): Promise { + while (this.connectionPool.getActiveOperationsCount() > 0) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + } + + private async executePhase4Teardown(): Promise { + const inProgressList = Array.from(this.connectionPool.getInProgressCleanups()); + if (inProgressList.length > 0) { + let syncTimer: NodeJS.Timeout | undefined; + const syncTimeout = new Promise((resolve) => { + syncTimer = setTimeout(resolve, 200); + }); + await Promise.race([Promise.allSettled(inProgressList), syncTimeout]); + if (syncTimer) clearTimeout(syncTimer); + } + + this.connectionPool.setForcedTeardownInitiated(true); + + const uniqueTunnels = new Set(); + for (const entry of this.connectionPool.getAllEntries()) { + if (entry.tunnel) uniqueTunnels.add(entry.tunnel); + } + for (const tunnel of uniqueTunnels) { + try { + tunnel.close(); + } catch {} + } + + const phase4Deadline = Date.now() + 100; + await closeAllMongoClients({ force: true, globalDeadline: phase4Deadline }); + + this.connectionPool.clear(); + this.activeBuilders.clear(); + } + + async shutdown(): Promise { + this.isShuttingDown = true; + this.connectionPool.setShuttingDown(true); + setMongoShuttingDown(true); + + for (const dbId of Array.from(this.dbGenerations.keys())) { + this.dbGenerations.set(dbId, (this.dbGenerations.get(dbId) || 0) + 1); + } + this.inFlightConnections.clear(); + + const leaseDrainPromise = this.drainActiveLeases(); + const builderDrainPromise = Promise.allSettled(Array.from(this.activeBuilders)); + + let drainTimer: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((resolve) => { + drainTimer = setTimeout(resolve, 1500); + }); + + await Promise.race([Promise.all([leaseDrainPromise, builderDrainPromise]), timeoutPromise]); + if (drainTimer) clearTimeout(drainTimer); + + await this.executePhase4Teardown(); + } } + +export const databaseServiceInstance = new DatabaseService(); diff --git a/bridge/src/services/mongoBulkEngine.ts b/bridge/src/services/mongoBulkEngine.ts new file mode 100644 index 00000000..3f29f87d --- /dev/null +++ b/bridge/src/services/mongoBulkEngine.ts @@ -0,0 +1,2019 @@ +/** + * MongoDB Milestone 7 Phase 3 — Bulk Operations Bridge Engine. + * + * Implements: + * - BulkOperationManager: UUIDv4 validation, 1-hour TTL, safe state machine, + * idempotent chunk replay protection, AbortController lifecycle. + * - Bulk Import Engine: JSON, NDJSON, CSV parsing, prototype pollution defense, + * system collection blocking, bounded batch writes (<= 500 docs), duplicate key handling. + * - Bulk Export Engine: Cursor streaming (batchSize 1000) directly to temp file, + * dot-notation flattening, sanitization of filesystem paths, cursor/file closure. + * - Supporting Bulk Mutations: bulk insert, bulk update, bulk delete with broad-filter guards. + * - Read-only database enforcement. + */ + +import fs from "fs"; +import path from "path"; +import os from "os"; +import { v4 as uuidv4 } from "uuid"; +import { Collection, Db, Document } from "mongodb"; +import { DatabaseService } from "./databaseService"; +import logger from "./logger"; +import { selectDb, MongoApiError, safeMongoError, mongodbCache } from "../connectors/mongo"; +import { + serializeMongoEJSONDocument, + mongoBSONtoEJSON, + serializeMongoFilter, + serializeMongoUpdateOperand, +} from "../connectors/mongoBson"; +import { DBType } from "../types"; +import { + MongoBulkFormat, + MongoBulkOperationType, + MongoBulkOperationStatus, + MongoBulkOperation, + MongoBulkImportStartRequest, + MongoBulkImportStartResult, + MongoBulkImportChunkRequest, + MongoBulkImportChunkResult, + MongoBulkExportStartRequest, + MongoBulkExportStartResult, + StagingFileIdentity, + MongoBulkExportSaveToRequest, + MongoBulkExportSaveToResult, + MongoBulkOperationStatusResult, + MongoBulkCancelResult, + MongoBulkCleanupResult, + MongoBulkInsertRequest, + MongoBulkInsertResult, + MongoBulkUpdateRequest, + MongoBulkUpdateResult, + MongoBulkDeleteRequest, + MongoBulkDeleteResult, +} from "../types/mongoBulk"; + +export const MAX_IMPORT_CHUNK_DOCS = 1_000; +export const MAX_IMPORT_CHUNK_SIZE_BYTES = 4 * 1024 * 1024; // 4 MB +export const MAX_IMPORT_TOTAL_SIZE_BYTES = 16 * 1024 * 1024; // 16 MB absolute cap +export const MAX_BULK_INSERT_DOCS = 5_000; +export const MAX_BULK_INSERT_SIZE_BYTES = 16 * 1024 * 1024; // 16 MB +export const BATCH_WRITE_SIZE = 500; +export const OPERATION_TTL_MS = 60 * 60 * 1000; // 1 hour +export const EXPORT_CURSOR_BATCH_SIZE = 1_000; + +export const EXPORT_DIR = path.join(os.tmpdir(), "relwave-exports"); + +export const EXPORT_ORPHAN_TTL_MS = 2 * 60 * 60 * 1000; // 2 hours +export const EXPORT_FILENAME_REGEX = /^export-([a-zA-Z0-9_-]+)\.(json|ndjson|csv|tmp)$/; + +export interface ActiveExportJob { + jobId: string; + startedAt: number; + lastHeartbeat: number; + filePath: string; +} + +export class ExportCoordinationManager { + private static instance: ExportCoordinationManager; + private activeJobs = new Map(); + private dirLocks = new Map>(); + + private constructor() {} + + /** Process-wide Singleton Accessor */ + public static getInstance(): ExportCoordinationManager { + if (!ExportCoordinationManager.instance) { + ExportCoordinationManager.instance = new ExportCoordinationManager(); + } + return ExportCoordinationManager.instance; + } + + public static resetForTesting(): void { + ExportCoordinationManager.instance = new ExportCoordinationManager(); + } + + /** Keyed Per-Directory Mutex */ + public async withDirLock(dirPath: string, fn: () => Promise): Promise { + const normalizedDir = path.resolve(dirPath); + const prevLock = this.dirLocks.get(normalizedDir) || Promise.resolve(); + let releaseLock!: () => void; + const newLock = new Promise((resolve) => { + releaseLock = resolve; + }); + this.dirLocks.set(normalizedDir, newLock); + + await prevLock; + try { + return await fn(); + } finally { + releaseLock(); + if (this.dirLocks.get(normalizedDir) === newLock) { + this.dirLocks.delete(normalizedDir); + } + } + } + + /** Pre-Creation Registration: Registered under critical section BEFORE file creation */ + async registerJobBeforeFileCreation(jobId: string, filePath: string): Promise { + const dir = path.dirname(filePath); + await this.withDirLock(dir, async () => { + this.activeJobs.set(jobId, { + jobId, + startedAt: Date.now(), + lastHeartbeat: Date.now(), + filePath, + }); + }); + } + + updateJobHeartbeat(jobId: string): void { + const job = this.activeJobs.get(jobId); + if (job) job.lastHeartbeat = Date.now(); + } + + async unregisterJob(jobId: string): Promise { + const job = this.activeJobs.get(jobId); + const dir = job ? path.dirname(job.filePath) : process.cwd(); + await this.withDirLock(dir, async () => { + this.activeJobs.delete(jobId); + }); + } + + isJobActive(jobId: string): boolean { + return this.activeJobs.has(jobId); + } + + /** Race-Safe Purge: Synchronized with registration under dirLock */ + async purgeOrphanedExportsAsync(exportDir: string): Promise<{ purged: number; skipped: number }> { + return this.withDirLock(exportDir, async () => { + let files: string[]; + try { + files = await fs.promises.readdir(exportDir); + } catch (err: any) { + if (err.code === "ENOENT") return { purged: 0, skipped: 0 }; + throw err; + } + + const now = Date.now(); + let purged = 0; + let skipped = 0; + + for (const file of files) { + const match = file.match(EXPORT_FILENAME_REGEX); + if (!match) { + skipped++; + continue; + } + + const fullPath = path.join(exportDir, file); + let stat: fs.Stats; + try { + stat = await fs.promises.lstat(fullPath); + } catch (err: any) { + if (err.code === "ENOENT") continue; + skipped++; + continue; + } + + // 1. Symlink rejection + if (stat.isSymbolicLink()) { + skipped++; + continue; + } + + const jobId = match[1]; + + // 2. In-memory Active Job Protection (Protected even if run > 2 hours!) + if (this.activeJobs.has(jobId)) { + skipped++; + continue; + } + + // 3. Two-Hour TTL Age Gate + if (now - stat.mtimeMs < EXPORT_ORPHAN_TTL_MS) { + skipped++; + continue; + } + + // 4. Atomic Unlink with ENOENT tolerance + try { + await fs.promises.unlink(fullPath); + purged++; + } catch (err: any) { + if (err.code === "ENOENT") { + // Deleted concurrently or already rotated + continue; + } + skipped++; + logger.warn({ fullPath, err }, "Failed to unlink orphaned export file"); + } + } + + return { purged, skipped }; + }); + } +} + +const UUID_V4_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export function isValidUuidV4(id: unknown): id is string { + return typeof id === "string" && UUID_V4_REGEX.test(id); +} + +export function assertValidOperationId(id: unknown): asserts id is string { + if (typeof id !== "string" || !UUID_V4_REGEX.test(id)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Invalid operationId format: must be a valid UUIDv4 string", + ); + } +} + +export function assertValidCollectionName(collection: unknown): asserts collection is string { + if (typeof collection !== "string" || collection.trim().length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection must be a non-empty string"); + } + if (Buffer.byteLength(collection, "utf8") > 120) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection name must not exceed 120 bytes"); + } + if (collection.includes("$") || collection.includes("\0")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not contain '$' or null bytes", + ); + } + if (collection.startsWith("system.")) { + throw new MongoApiError("MONGO_FORBIDDEN", "Operations on system collections are forbidden"); + } +} + +export function assertNoPrototypePollution(value: unknown): void { + if (!value || typeof value !== "object") return; + if (Array.isArray(value)) { + for (const item of value) { + assertNoPrototypePollution(item); + } + return; + } + for (const key of Object.getOwnPropertyNames(value)) { + if (key === "__proto__" || key === "constructor" || key === "prototype") { + throw new MongoApiError( + "SECURITY_VIOLATION", + `Prototype pollution attempt detected with property '${key}'`, + ); + } + assertNoPrototypePollution((value as Record)[key]); + } +} + +export function assertNotReadOnly(db: any, conn: any): void { + const isReadOnly = Boolean(db?.readonly || db?.readOnly || conn?.readonly || conn?.readOnly); + if (isReadOnly) { + throw new MongoApiError("MONGO_READ_ONLY", "Database is in read-only mode"); + } +} + +// =========================================================================== +// Parsing Utilities (JSON array, NDJSON, CSV) +// =========================================================================== + +export function parseJsonArray(textOrObj: unknown): Record[] { + let parsed: any = textOrObj; + if (typeof textOrObj === "string") { + try { + parsed = JSON.parse(textOrObj); + } catch (err: any) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", `Invalid JSON payload: ${err.message}`); + } + } + if (!Array.isArray(parsed)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "JSON bulk import payload must be an array of documents", + ); + } + for (let i = 0; i < parsed.length; i++) { + const doc = parsed[i]; + if (typeof doc !== "object" || doc === null || Array.isArray(doc)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `JSON array element at index ${i} must be an object`, + ); + } + assertNoPrototypePollution(doc); + } + return parsed; +} + +export function parseNdjson(text: string): Record[] { + const lines = text.split(/\r?\n/).filter((l) => l.trim().length > 0); + const docs: Record[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + let doc: any; + try { + doc = JSON.parse(line); + } catch (err: any) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `Invalid NDJSON at line ${i + 1}: ${err.message}`, + ); + } + if (typeof doc !== "object" || doc === null || Array.isArray(doc)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `NDJSON line ${i + 1} must be a JSON object`, + ); + } + assertNoPrototypePollution(doc); + docs.push(doc); + } + return docs; +} + +function setDottedPath(target: Record, pathStr: string, value: any): void { + const parts = pathStr.split("."); + let cur = target; + for (let i = 0; i < parts.length - 1; i++) { + const part = parts[i]; + if (part === "__proto__" || part === "constructor" || part === "prototype") { + throw new MongoApiError( + "SECURITY_VIOLATION", + `Prototype pollution attempt with key '${part}'`, + ); + } + if (cur[part] === undefined || cur[part] === null || typeof cur[part] !== "object") { + cur[part] = {}; + } + cur = cur[part]; + } + const last = parts[parts.length - 1]; + if (last === "__proto__" || last === "constructor" || last === "prototype") { + throw new MongoApiError("SECURITY_VIOLATION", `Prototype pollution attempt with key '${last}'`); + } + cur[last] = value; +} + +function coerceCsvValue(val: string): any { + const trimmed = val.trim(); + if (trimmed === "" || trimmed.toLowerCase() === "null") { + return null; + } + if (trimmed.toLowerCase() === "true") return true; + if (trimmed.toLowerCase() === "false") return false; + if (/^-?\d+(\.\d+)?$/.test(trimmed)) { + const num = Number(trimmed); + if (Number.isFinite(num)) return num; + } + if ( + (trimmed.startsWith("{") && trimmed.endsWith("}")) || + (trimmed.startsWith("[") && trimmed.endsWith("]")) + ) { + try { + const parsed = JSON.parse(trimmed); + assertNoPrototypePollution(parsed); + return parsed; + } catch { + // Retain as string + } + } + return val; +} + +export function parseCsv(csvText: string): Record[] { + const rows: string[][] = []; + let currentRow: string[] = []; + let currentField = ""; + let insideQuotes = false; + let i = 0; + const len = csvText.length; + + while (i < len) { + const char = csvText[i]; + const nextChar = i + 1 < len ? csvText[i + 1] : ""; + + if (insideQuotes) { + if (char === '"') { + if (nextChar === '"') { + currentField += '"'; + i += 2; + continue; + } else { + insideQuotes = false; + i++; + continue; + } + } else { + currentField += char; + i++; + continue; + } + } else { + if (char === '"') { + insideQuotes = true; + i++; + continue; + } else if (char === ",") { + currentRow.push(currentField); + currentField = ""; + i++; + continue; + } else if (char === "\r") { + if (nextChar === "\n") i++; + currentRow.push(currentField); + currentField = ""; + rows.push(currentRow); + currentRow = []; + i++; + continue; + } else if (char === "\n") { + currentRow.push(currentField); + currentField = ""; + rows.push(currentRow); + currentRow = []; + i++; + continue; + } else { + currentField += char; + i++; + continue; + } + } + } + + if (currentField.length > 0 || currentRow.length > 0) { + currentRow.push(currentField); + rows.push(currentRow); + } + + const cleanRows = rows.filter((r) => r.length > 1 || (r.length === 1 && r[0].trim().length > 0)); + if (cleanRows.length === 0) { + return []; + } + + const rawHeaders = cleanRows[0].map((h) => h.trim()); + for (const h of rawHeaders) { + if (!h) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "CSV contains empty column header"); + } + if (h.includes("__proto__") || h.includes("constructor") || h.includes("prototype")) { + throw new MongoApiError( + "SECURITY_VIOLATION", + `Prototype pollution attempt detected in CSV header: ${h}`, + ); + } + } + + const documents: Record[] = []; + for (let rIdx = 1; rIdx < cleanRows.length; rIdx++) { + const row = cleanRows[rIdx]; + const doc: Record = {}; + + for (let cIdx = 0; cIdx < rawHeaders.length; cIdx++) { + const header = rawHeaders[cIdx]; + const rawVal = cIdx < row.length ? row[cIdx] : ""; + const val = coerceCsvValue(rawVal); + setDottedPath(doc, header, val); + } + documents.push(doc); + } + + return documents; +} + +export function parseImportPayload( + payload: unknown, + format: MongoBulkFormat, +): Record[] { + if (payload === undefined || payload === null) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "Payload cannot be null or undefined"); + } + + let byteSize = 0; + if (typeof payload === "string") { + byteSize = Buffer.byteLength(payload, "utf8"); + } else if (Buffer.isBuffer(payload)) { + byteSize = payload.length; + } else { + byteSize = Buffer.byteLength(JSON.stringify(payload), "utf8"); + } + + if (byteSize > MAX_IMPORT_CHUNK_SIZE_BYTES) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `Chunk payload size ${byteSize} bytes exceeds maximum limit of ${MAX_IMPORT_CHUNK_SIZE_BYTES} bytes (4MB)`, + ); + } + + const text = + typeof payload === "string" + ? payload + : Buffer.isBuffer(payload) + ? payload.toString("utf8") + : ""; + + let docs: Record[]; + if (format === "json") { + docs = parseJsonArray(text || payload); + } else if (format === "ndjson") { + if (typeof payload !== "string" && !Buffer.isBuffer(payload)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "NDJSON payload must be a string or buffer", + ); + } + docs = parseNdjson(text); + } else if (format === "csv") { + if (typeof payload !== "string" && !Buffer.isBuffer(payload)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "CSV payload must be a string or buffer"); + } + docs = parseCsv(text); + } else { + throw new MongoApiError("MONGO_VALIDATION_ERROR", `Unsupported bulk import format: ${format}`); + } + + if (docs.length > MAX_IMPORT_CHUNK_DOCS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `Chunk document count ${docs.length} exceeds maximum limit of ${MAX_IMPORT_CHUNK_DOCS} documents`, + ); + } + + return docs; +} + +export function flattenObject(obj: Record, prefix = ""): Record { + const result: Record = {}; + for (const [key, val] of Object.entries(obj)) { + if (key === "__proto__" || key === "constructor" || key === "prototype") continue; + const newKey = prefix ? `${prefix}.${key}` : key; + if ( + val !== null && + typeof val === "object" && + !Array.isArray(val) && + !(val instanceof Date) && + !(val as any)._bsontype && + !(val as any).$oid && + !(val as any).$date + ) { + const nested = flattenObject(val, newKey); + Object.assign(result, nested); + } else { + result[newKey] = val; + } + } + return result; +} + +export function escapeCsvValue(val: unknown): string { + if (val === null || val === undefined) return ""; + let str: string; + if (val instanceof Date) { + str = val.toISOString(); + } else if (typeof val === "object") { + if ((val as any)._bsontype === "ObjectId" && typeof (val as any).toHexString === "function") { + str = (val as any).toHexString(); + } else if (typeof (val as any).$oid === "string") { + str = (val as any).$oid; + } else if (typeof (val as any).$date === "string") { + str = (val as any).$date; + } else { + str = JSON.stringify(val); + } + } else { + str = String(val); + } + if (str.includes(",") || str.includes('"') || str.includes("\n") || str.includes("\r")) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; +} + +async function writeStreamDrain(stream: fs.WriteStream, chunk: string): Promise { + if (!stream.write(chunk)) { + await new Promise((resolve, reject) => { + const onDrain = () => { + stream.removeListener("error", onError); + resolve(); + }; + const onError = (err: any) => { + stream.removeListener("drain", onDrain); + reject(err); + }; + stream.once("drain", onDrain); + stream.once("error", onError); + }); + } +} + +// =========================================================================== +// Operation Lifecycle Manager +// =========================================================================== + +export interface InternalOperationState { + operation: MongoBulkOperation; + controller: AbortController; + processedChunkIndices: Set; + cachedChunkResponses: Map; + totalBytesProcessed: number; + isSaving?: boolean; +} + +export class BulkOperationManager { + private operations = new Map(); + private pendingStagingCleanups = new Map(); + private static readonly MAX_PENDING_STAGING_CLEANUPS = 100; + private sweepTimer: NodeJS.Timeout | null = null; + + constructor(private ttlMs: number = OPERATION_TTL_MS) { + this.sweepTimer = setInterval( + () => { + this.sweepExpired().catch(() => {}); + }, + 5 * 60 * 1000, + ); + if (this.sweepTimer.unref) { + this.sweepTimer.unref(); + } + } + + recordPendingStagingCleanup(stagingPath: string, identity?: StagingFileIdentity): boolean { + // Duplicate path registration: idempotent no-op, preserves original authoritative identity and does not consume extra capacity + if (this.pendingStagingCleanups.has(stagingPath)) { + return true; + } + + // Below capacity: track for automated retry + if (this.pendingStagingCleanups.size < BulkOperationManager.MAX_PENDING_STAGING_CLEANUPS) { + this.pendingStagingCleanups.set( + stagingPath, + identity ?? { + exactPath: stagingPath, + dev: 0, + ino: 0, + size: 0, + registeredAt: Date.now(), + retryCount: 0, + }, + ); + logger.info( + { stagingPath, currentCount: this.pendingStagingCleanups.size }, + "Recorded staging file identity for bounded retry cleanup", + ); + return true; + } + + // Capacity overflow (queue full): honest refusal and observable orphan logging + logger.warn( + { + stagingPath, + capacity: BulkOperationManager.MAX_PENDING_STAGING_CLEANUPS, + dropped: true, + }, + "Pending staging cleanups queue capacity reached (100); temporary staging file abandoned as untracked orphan requiring manual administrative removal", + ); + return false; + } + + getPendingStagingCleanupsCount(): number { + return this.pendingStagingCleanups.size; + } + + getPendingStagingCleanup(stagingPath: string): StagingFileIdentity | undefined { + return this.pendingStagingCleanups.get(stagingPath); + } + + clearPendingStagingCleanups(): void { + this.pendingStagingCleanups.clear(); + } + + createOperation( + type: MongoBulkOperationType, + dbId: string, + collection: string, + format: MongoBulkFormat, + ): InternalOperationState { + assertValidCollectionName(collection); + const operationId = uuidv4(); + const now = new Date(); + const expiresAt = new Date(now.getTime() + this.ttlMs).toISOString(); + + const operation: MongoBulkOperation = { + operationId, + type, + status: "pending", + dbId, + collection, + format, + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + expiresAt, + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + }; + + const state: InternalOperationState = { + operation, + controller: new AbortController(), + processedChunkIndices: new Set(), + cachedChunkResponses: new Map(), + totalBytesProcessed: 0, + }; + + this.operations.set(operationId, state); + return state; + } + + getOperation(operationId: string): InternalOperationState | null { + assertValidOperationId(operationId); + const state = this.operations.get(operationId); + if (!state) return null; + + if (Date.now() > new Date(state.operation.expiresAt).getTime()) { + if (state.isSaving === true) { + return state; + } + this.cleanup(operationId); + return null; + } + + return state; + } + + transition(operationId: string, newStatus: MongoBulkOperationStatus, error?: string): void { + const state = this.getOperation(operationId); + if (!state) { + throw new MongoApiError("MONGO_NOT_FOUND", `Operation '${operationId}' not found or expired`); + } + + const currentStatus = state.operation.status; + if ( + currentStatus === "completed" || + currentStatus === "failed" || + currentStatus === "cancelled" + ) { + throw new MongoApiError( + "MONGO_INVALID_STATE", + `Cannot transition terminal operation '${operationId}' from '${currentStatus}' to '${newStatus}'`, + ); + } + + if ( + currentStatus === "pending" && + !["in_progress", "cancelled", "failed"].includes(newStatus) + ) { + throw new MongoApiError( + "MONGO_INVALID_STATE", + `Invalid transition from '${currentStatus}' to '${newStatus}'`, + ); + } + + if ( + currentStatus === "in_progress" && + !["completed", "cancelled", "failed"].includes(newStatus) + ) { + throw new MongoApiError( + "MONGO_INVALID_STATE", + `Invalid transition from '${currentStatus}' to '${newStatus}'`, + ); + } + + state.operation.status = newStatus; + state.operation.updatedAt = new Date().toISOString(); + if (error) { + state.operation.error = safeMongoError(error); + } + } + + cancel(operationId: string): MongoBulkCancelResult { + const state = this.getOperation(operationId); + if (!state) { + throw new MongoApiError("MONGO_NOT_FOUND", `Operation '${operationId}' not found or expired`); + } + + if (state.isSaving === true) { + throw new MongoApiError( + "MONGO_CONCURRENT_SAVE", + "Cannot cancel an export operation that is currently saving", + ); + } + + if (state.operation.status === "cancelled") { + return { + operationId, + status: "cancelled", + message: "Operation was already cancelled", + }; + } + + if (state.operation.status === "completed" || state.operation.status === "failed") { + throw new MongoApiError( + "MONGO_INVALID_STATE", + `Cannot cancel an operation that has already terminated with status '${state.operation.status}'`, + ); + } + + state.operation.status = "cancelled"; + state.operation.updatedAt = new Date().toISOString(); + state.controller.abort(); + + if (state.operation.tempFilePath) { + try { + if (fs.existsSync(state.operation.tempFilePath)) { + fs.unlinkSync(state.operation.tempFilePath); + } + } catch { + // safe ignore + } + } + + return { + operationId, + status: "cancelled", + message: "Operation cancelled successfully", + }; + } + + cleanup(operationId: string): MongoBulkCleanupResult { + assertValidOperationId(operationId); + const state = this.operations.get(operationId); + if (state) { + if (state.isSaving === true) { + throw new MongoApiError( + "MONGO_CONCURRENT_SAVE", + "Cannot clean up an export operation that is currently saving", + ); + } + state.controller.abort(); + if (state.operation.tempFilePath) { + try { + if (fs.existsSync(state.operation.tempFilePath)) { + fs.unlinkSync(state.operation.tempFilePath); + } + } catch { + // safe ignore + } + } + this.operations.delete(operationId); + return { operationId, success: true }; + } + return { operationId, success: true, alreadyCleaned: true }; + } + + async sweepExpired(): Promise { + const now = Date.now(); + for (const [id, state] of Array.from(this.operations.entries())) { + if (state.isSaving === true) { + continue; + } + if (now > new Date(state.operation.expiresAt).getTime()) { + try { + this.cleanup(id); + } catch { + // safe ignore + } + } + } + + for (const [stagingPath, expected] of Array.from(this.pendingStagingCleanups.entries())) { + try { + // Step A: Inspect path via lstat (NEVER follow symlinks) + let currentStat: fs.Stats; + try { + currentStat = await fs.promises.lstat(stagingPath); + } catch (statErr: any) { + if (statErr?.code === "ENOENT") { + // File already removed externally; safely evict stale entry and free capacity + this.pendingStagingCleanups.delete(stagingPath); + logger.info( + { stagingPath }, + "Staging file already absent (ENOENT); evicted from cleanup queue", + ); + continue; + } + logger.warn( + { err: statErr, stagingPath }, + "Failed to lstat staging path during retry verification; retaining", + ); + continue; + } + + // Step B: Must be a regular file; reject symlinks, directories, FIFOs, and devices + // Filesystem entry is left untouched on disk (never moved or isolated) and evicted from cleanup tracking + if (!currentStat.isFile() || currentStat.isSymbolicLink()) { + logger.error( + { + stagingPath, + isSymbolicLink: currentStat.isSymbolicLink(), + isDirectory: currentStat.isDirectory(), + }, + "SECURITY ALERT: Staging path is no longer a regular file (symlink or directory substituted); unlinking aborted, filesystem entry left untouched on disk, and entry evicted from automated cleanup tracking", + ); + this.pendingStagingCleanups.delete(stagingPath); + continue; + } + + // Step C: Verify filesystem identity (dev & ino) + const hasValidExpected = expected.dev !== 0 && expected.ino !== 0; + const hasValidCurrent = currentStat.dev !== 0 && currentStat.ino !== 0; + + if (hasValidExpected && hasValidCurrent) { + if (currentStat.dev !== expected.dev || currentStat.ino !== expected.ino) { + logger.error( + { + stagingPath, + expectedDev: expected.dev, + actualDev: currentStat.dev, + expectedIno: expected.ino, + actualIno: currentStat.ino, + }, + "SECURITY ALERT: Staging file identity mismatch (dev/ino substitution detected); unlinking aborted, filesystem entry left untouched on disk, and entry evicted from automated cleanup tracking", + ); + this.pendingStagingCleanups.delete(stagingPath); + continue; + } + } else { + // Fail-Closed Fallback for unsupported or unpopulated filesystem identity + logger.warn( + { stagingPath, dev: currentStat.dev, ino: currentStat.ino }, + "Filesystem identity (dev/ino) unpopulated or unsupported on host volume; failing closed, refusing automated retry unlink, and evicting entry from cleanup tracking", + ); + this.pendingStagingCleanups.delete(stagingPath); + continue; + } + + // Step D: Authoritative unlink of verified staging file + await fs.promises.unlink(stagingPath); + this.pendingStagingCleanups.delete(stagingPath); + logger.info( + { stagingPath }, + "Successfully verified identity and unlinked leftover staging file on retry", + ); + } catch (retryErr: any) { + if (retryErr?.code === "ENOENT") { + this.pendingStagingCleanups.delete(stagingPath); + } else { + expected.retryCount = (expected.retryCount ?? 0) + 1; + if (expected.retryCount >= 5) { + logger.warn( + { stagingPath, retries: expected.retryCount }, + "Staging file retry limit reached (5); abandoning as untracked orphan", + ); + this.pendingStagingCleanups.delete(stagingPath); + } else { + logger.warn( + { err: retryErr, stagingPath }, + "Staging cleanup retry failed; retaining in retry queue", + ); + } + } + } + } + } + + dispose(): void { + if (this.sweepTimer) { + clearInterval(this.sweepTimer); + this.sweepTimer = null; + } + for (const id of Array.from(this.operations.keys())) { + try { + const state = this.operations.get(id); + if (state?.isSaving === true) { + logger.warn( + { operationId: id }, + "Operation has an active save in progress during manager disposal; active save protection preserved and source file retained", + ); + continue; + } + this.cleanup(id); + } catch { + // safe ignore during dispose + } + } + this.pendingStagingCleanups.clear(); + } +} + +// =========================================================================== +// MongoBulkEngine Main Service +// =========================================================================== + +export class MongoBulkEngine { + constructor( + private dbService: DatabaseService = new DatabaseService(), + public readonly opManager: BulkOperationManager = new BulkOperationManager(), + ) { + if (!fs.existsSync(EXPORT_DIR)) { + try { + fs.mkdirSync(EXPORT_DIR, { recursive: true }); + } catch { + // handled in runtime + } + } + } + + async getDb(dbId: string): Promise<{ db: any; conn: any; dbType: DBType; nativeDb: Db }> { + const { db, conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_INVALID_DATABASE", + `Database ${dbId} is not a MongoDB database (type: ${dbType})`, + ); + } + const nativeDb = selectDb(conn); + return { db, conn, dbType, nativeDb }; + } + + // ========================================================================= + // 1. Bulk Import Engine + // ========================================================================= + + async importStart(req: MongoBulkImportStartRequest): Promise { + const { dbId, collection, format } = req; + assertValidCollectionName(collection); + + if (!["json", "ndjson", "csv"].includes(format)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", `Invalid bulk format '${format}'`); + } + + const { db, conn } = await this.getDb(dbId); + assertNotReadOnly(db, conn); + + const state = this.opManager.createOperation("import", dbId, collection, format); + + return { + operationId: state.operation.operationId, + status: state.operation.status, + format: state.operation.format, + collection: state.operation.collection, + }; + } + + async importChunk(req: MongoBulkImportChunkRequest): Promise { + const { operationId, chunkIndex, isLastChunk, payload } = req; + assertValidOperationId(operationId); + + if (typeof chunkIndex !== "number" || !Number.isInteger(chunkIndex) || chunkIndex < 0) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "chunkIndex must be a non-negative integer", + ); + } + + const state = this.opManager.getOperation(operationId); + if (!state) { + throw new MongoApiError("MONGO_NOT_FOUND", `Operation '${operationId}' not found or expired`); + } + + if (state.operation.type !== "import") { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `Operation '${operationId}' is not an import operation`, + ); + } + + if (state.operation.status === "cancelled") { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Operation has been cancelled"); + } + + if (state.operation.status === "failed") { + throw new MongoApiError( + "MONGO_OPERATION_FAILED", + `Operation has failed: ${state.operation.error}`, + ); + } + + // Replay protection: repeated chunkIndex returns cached result without double write + if (state.processedChunkIndices.has(chunkIndex)) { + const cached = state.cachedChunkResponses.get(chunkIndex); + if (cached) { + return cached; + } + } + + const { db, conn, nativeDb } = await this.getDb(state.operation.dbId); + assertNotReadOnly(db, conn); + + if (state.operation.status === "pending") { + this.opManager.transition(operationId, "in_progress"); + } + + let byteSize = 0; + if (typeof payload === "string") { + byteSize = Buffer.byteLength(payload, "utf8"); + } else if (Buffer.isBuffer(payload)) { + byteSize = payload.length; + } else { + byteSize = Buffer.byteLength(JSON.stringify(payload), "utf8"); + } + + state.totalBytesProcessed += byteSize; + if (state.totalBytesProcessed > MAX_IMPORT_TOTAL_SIZE_BYTES) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `Total cumulative import payload exceeds ${MAX_IMPORT_TOTAL_SIZE_BYTES} bytes (16MB)`, + ); + } + + const docs = parseImportPayload(payload, state.operation.format); + + const coll = nativeDb.collection(state.operation.collection); + const writeResult = await this.writeBatches(coll, docs, state.controller); + + mongodbCache.clearCollectionCache(conn, state.operation.collection); + + state.operation.processedDocuments += docs.length; + state.operation.successfulDocuments += writeResult.insertedCount; + state.operation.failedDocuments += writeResult.failedCount; + state.operation.updatedAt = new Date().toISOString(); + + const isComplete = Boolean(isLastChunk); + if (isComplete) { + this.opManager.transition(operationId, "completed"); + } + + const chunkResult: MongoBulkImportChunkResult = { + operationId, + chunkIndex, + status: state.operation.status, + insertedCount: writeResult.insertedCount, + failedCount: writeResult.failedCount, + errors: writeResult.errors.length > 0 ? writeResult.errors : undefined, + isComplete, + totalInserted: state.operation.successfulDocuments, + totalFailed: state.operation.failedDocuments, + }; + + state.processedChunkIndices.add(chunkIndex); + state.cachedChunkResponses.set(chunkIndex, chunkResult); + + return chunkResult; + } + + private async writeBatches( + coll: Collection, + docs: Record[], + controller: AbortController, + ): Promise<{ + insertedCount: number; + failedCount: number; + errors: Array<{ index: number; code: string; message: string }>; + }> { + let insertedCount = 0; + let failedCount = 0; + const errors: Array<{ index: number; code: string; message: string }> = []; + + for (let i = 0; i < docs.length; i += BATCH_WRITE_SIZE) { + if (controller.signal.aborted) { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Operation was cancelled"); + } + + const batch = docs.slice(i, i + BATCH_WRITE_SIZE); + const bsonDocs = batch.map((d) => serializeMongoEJSONDocument(d)); + + try { + const res = await coll.insertMany(bsonDocs, { ordered: false }); + insertedCount += res.insertedCount ?? Object.keys(res.insertedIds || {}).length; + } catch (err: any) { + if ( + err?.name === "MongoBulkWriteError" || + err?.name === "BulkWriteError" || + err?.writeErrors + ) { + const nInserted = err.result?.nInserted ?? err.insertedCount ?? 0; + insertedCount += nInserted; + const writeErrors = err.writeErrors || []; + for (const we of writeErrors) { + const docIdx = i + (we.index ?? 0); + const isDup = we.code === 11000 || we.code === 11001; + errors.push({ + index: docIdx, + code: isDup ? "MONGO_DUPLICATE_KEY" : "MONGO_WRITE_ERROR", + message: safeMongoError(we.errmsg || we.message || "Write error"), + }); + } + failedCount += batch.length - nInserted; + } else { + throw err; + } + } + } + + return { insertedCount, failedCount, errors }; + } + + // ========================================================================= + // 2. Bulk Export Engine (Streaming directly to temporary file) + // ========================================================================= + + async purgeOrphanedExports( + exportDir: string = EXPORT_DIR, + ): Promise<{ purged: number; skipped: number }> { + return ExportCoordinationManager.getInstance().purgeOrphanedExportsAsync(exportDir); + } + + async exportStart(req: MongoBulkExportStartRequest): Promise { + const { dbId, collection, format, filter = {}, projection, sort, limit } = req; + assertValidCollectionName(collection); + + if (!["json", "ndjson", "csv"].includes(format)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", `Invalid bulk export format '${format}'`); + } + + const lease = await this.dbService.acquireConnection(dbId); + const nativeDb = selectDb(lease.conn); + + const state = this.opManager.createOperation("export", dbId, collection, format); + const operationId = state.operation.operationId; + + if (!fs.existsSync(EXPORT_DIR)) { + fs.mkdirSync(EXPORT_DIR, { recursive: true }); + } + const exportCoord = ExportCoordinationManager.getInstance(); + const tempFilePath = path.join(EXPORT_DIR, `export-${operationId}.${format}`); + state.operation.tempFilePath = tempFilePath; + + // Register job BEFORE temporary file creation under critical section + await exportCoord.registerJobBeforeFileCreation(operationId, tempFilePath); + + const bsonFilter = serializeMongoFilter(filter); + const coll = nativeDb.collection(collection); + + // Run export stream asynchronously in background + (async () => { + let writeStream: fs.WriteStream | null = null; + let cursor: any = null; + try { + this.opManager.transition(operationId, "in_progress"); + + writeStream = fs.createWriteStream(tempFilePath, { encoding: "utf8" }); + cursor = coll.find(bsonFilter, { + batchSize: EXPORT_CURSOR_BATCH_SIZE, + promoteValues: false, + promoteLongs: false, + promoteBuffers: false, + }); + + if (sort) cursor.sort(sort); + if (projection) cursor.project(projection); + if (limit && limit > 0) cursor.limit(limit); + + if (format === "json") { + await this.streamExportJson(cursor, writeStream, state); + } else if (format === "ndjson") { + await this.streamExportNdjson(cursor, writeStream, state); + } else if (format === "csv") { + await this.streamExportCsv(cursor, writeStream, state); + } + + await new Promise((resolve, reject) => { + writeStream!.end(() => resolve()); + writeStream!.on("error", reject); + }); + + const stat = await fs.promises.stat(tempFilePath).catch(() => null); + state.operation.fileSizeBytes = stat?.size ?? 0; + state.operation.updatedAt = new Date().toISOString(); + + const currentOp = this.opManager.getOperation(operationId); + if (currentOp && currentOp.operation.status === "in_progress") { + this.opManager.transition(operationId, "completed"); + } + } catch (err: any) { + const currentOp = this.opManager.getOperation(operationId); + if ( + currentOp && + currentOp.operation.status !== "completed" && + currentOp.operation.status !== "failed" && + currentOp.operation.status !== "cancelled" + ) { + if (state.controller.signal.aborted || err?.code === "MONGO_OPERATION_CANCELLED") { + this.opManager.transition(operationId, "cancelled"); + } else { + this.opManager.transition(operationId, "failed", err?.message || String(err)); + } + } + await fs.promises.unlink(tempFilePath).catch(() => {}); + } finally { + if (cursor) { + await cursor.close().catch(() => {}); + } + if (writeStream && !writeStream.destroyed) { + writeStream.destroy(); + } + await exportCoord.unregisterJob(operationId); + lease.release(); + } + })(); + + return { + operationId, + status: state.operation.status, + collection, + format, + }; + } + + private async streamExportJson( + cursor: any, + stream: fs.WriteStream, + state: InternalOperationState, + ): Promise { + const exportCoord = ExportCoordinationManager.getInstance(); + await writeStreamDrain(stream, "[\n"); + let isFirst = true; + + for await (const doc of cursor) { + if (state.controller.signal.aborted) { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Export operation was cancelled"); + } + exportCoord.updateJobHeartbeat(state.operation.operationId); + const serialized = JSON.stringify(mongoBSONtoEJSON(doc)); + const chunk = isFirst ? ` ${serialized}` : `,\n ${serialized}`; + isFirst = false; + await writeStreamDrain(stream, chunk); + state.operation.processedDocuments++; + state.operation.successfulDocuments++; + } + + await writeStreamDrain(stream, "\n]\n"); + } + + private async streamExportNdjson( + cursor: any, + stream: fs.WriteStream, + state: InternalOperationState, + ): Promise { + const exportCoord = ExportCoordinationManager.getInstance(); + for await (const doc of cursor) { + if (state.controller.signal.aborted) { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Export operation was cancelled"); + } + exportCoord.updateJobHeartbeat(state.operation.operationId); + const line = JSON.stringify(mongoBSONtoEJSON(doc)) + "\n"; + await writeStreamDrain(stream, line); + state.operation.processedDocuments++; + state.operation.successfulDocuments++; + } + } + + private async streamExportCsv( + cursor: any, + stream: fs.WriteStream, + state: InternalOperationState, + ): Promise { + const exportCoord = ExportCoordinationManager.getInstance(); + const bufferedDocs: Document[] = []; + const headerSet = new Set(); + + for await (const doc of cursor) { + if (state.controller.signal.aborted) { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Export operation was cancelled"); + } + exportCoord.updateJobHeartbeat(state.operation.operationId); + bufferedDocs.push(doc); + const flattened = flattenObject(doc); + for (const k of Object.keys(flattened)) { + headerSet.add(k); + } + if (bufferedDocs.length >= 100) { + break; + } + } + + const headers = Array.from(headerSet).sort((a, b) => { + if (a === "_id") return -1; + if (b === "_id") return 1; + return a.localeCompare(b); + }); + + if (headers.length > 0) { + const headerLine = headers.map(escapeCsvValue).join(",") + "\n"; + await writeStreamDrain(stream, headerLine); + } + + for (const doc of bufferedDocs) { + if (state.controller.signal.aborted) { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Export operation was cancelled"); + } + exportCoord.updateJobHeartbeat(state.operation.operationId); + const flattened = flattenObject(doc); + const row = headers.map((h) => escapeCsvValue(flattened[h])).join(",") + "\n"; + await writeStreamDrain(stream, row); + state.operation.processedDocuments++; + state.operation.successfulDocuments++; + } + + for await (const doc of cursor) { + if (state.controller.signal.aborted) { + throw new MongoApiError("MONGO_OPERATION_CANCELLED", "Export operation was cancelled"); + } + exportCoord.updateJobHeartbeat(state.operation.operationId); + const flattened = flattenObject(doc); + const row = headers.map((h) => escapeCsvValue(flattened[h])).join(",") + "\n"; + await writeStreamDrain(stream, row); + state.operation.processedDocuments++; + state.operation.successfulDocuments++; + } + } + + // ========================================================================= + // 3. Operation Lifecycle Management + // ========================================================================= + + async getOperationStatus(operationId: string): Promise { + assertValidOperationId(operationId); + const state = this.opManager.getOperation(operationId); + if (!state) { + throw new MongoApiError("MONGO_NOT_FOUND", `Operation '${operationId}' not found or expired`); + } + + const op = state.operation; + return { + operationId: op.operationId, + type: op.type, + status: op.status, + collection: op.collection, + format: op.format, + processedDocuments: op.processedDocuments, + successfulDocuments: op.successfulDocuments, + failedDocuments: op.failedDocuments, + fileSizeBytes: op.fileSizeBytes, + error: op.error, + createdAt: op.createdAt, + updatedAt: op.updatedAt, + expiresAt: op.expiresAt, + }; + } + + async cancelOperation(operationId: string): Promise { + assertValidOperationId(operationId); + return this.opManager.cancel(operationId); + } + + async cleanupOperation(operationId: string): Promise { + assertValidOperationId(operationId); + return this.opManager.cleanup(operationId); + } + + async exportSaveTo(params: MongoBulkExportSaveToRequest): Promise { + const { operationId, destinationPath, options } = params || {}; + assertValidOperationId(operationId); + + if ( + typeof destinationPath !== "string" || + destinationPath.trim().length === 0 || + !path.isAbsolute(destinationPath) + ) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "destinationPath must be a non-empty absolute path", + ); + } + + const state = this.opManager.getOperation(operationId); + if (!state) { + throw new MongoApiError("MONGO_NOT_FOUND", `Operation '${operationId}' not found or expired`); + } + + if (state.operation.type !== "export") { + throw new MongoApiError("MONGO_INVALID_STATE", "Cannot save: operation is not an export"); + } + + if (state.operation.status !== "completed") { + throw new MongoApiError( + "MONGO_INVALID_STATE", + `Cannot save export file: operation status is '${state.operation.status}', expected 'completed'`, + ); + } + + const sourceFilePath = state.operation.tempFilePath; + if (!sourceFilePath || !fs.existsSync(sourceFilePath)) { + throw new MongoApiError("MONGO_NOT_FOUND", "Export source temporary file does not exist"); + } + + if (state.isSaving === true) { + throw new MongoApiError( + "MONGO_CONCURRENT_SAVE", + "Another save operation is already in progress for this export", + ); + } + + // Synchronously acquire lock to eliminate race window + state.isSaving = true; + + try { + const normalizedDestPath = path.normalize(destinationPath); + const parentDir = path.dirname(normalizedDestPath); + + let resolvedParent: string; + try { + resolvedParent = await fs.promises.realpath(parentDir); + } catch (err: any) { + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Parent directory does not exist or is not accessible: ${err?.message}`, + ); + } + + let parentStat: fs.Stats; + try { + parentStat = await fs.promises.stat(resolvedParent); + } catch (err: any) { + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Failed to inspect parent directory: ${err?.message}`, + ); + } + if (!parentStat.isDirectory()) { + throw new MongoApiError("MONGO_SAVE_FAILED", "Parent path is not a directory"); + } + + // Destination Type Inspection (prior to staging) with strict error discrimination + let destStat: fs.Stats | null = null; + try { + destStat = await fs.promises.lstat(normalizedDestPath); + } catch (err: any) { + if (err?.code === "ENOENT") { + destStat = null; + } else { + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Failed to inspect destination path (${err.code ?? "UNKNOWN"}): ${err.message}`, + ); + } + } + + if (destStat) { + if (destStat.isDirectory()) { + throw new MongoApiError( + "MONGO_FILE_EXISTS", + "Destination is an existing directory; cannot overwrite a directory with a file", + ); + } + if ( + destStat.isFIFO() || + destStat.isSocket() || + destStat.isBlockDevice() || + destStat.isCharacterDevice() + ) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Destination is a special device or socket; overwriting unsupported file types is forbidden", + ); + } + if (options?.overwrite !== true) { + throw new MongoApiError("MONGO_FILE_EXISTS", "Destination file already exists"); + } + } + + // Generate staging filename in same parent dir + const stagingFileName = `${path.basename(normalizedDestPath)}.relwave-tmp-${uuidv4()}`; + const tempDestPath = path.join(resolvedParent, stagingFileName); + + let stagingWriteStream: fs.WriteStream | null = null; + let sourceReadStream: fs.ReadStream | null = null; + + try { + stagingWriteStream = fs.createWriteStream(tempDestPath, { + flags: "wx", + mode: 0o600, + }); + stagingWriteStream.on("error", () => {}); + + sourceReadStream = fs.createReadStream(sourceFilePath); + sourceReadStream.on("error", () => {}); + + await new Promise((resolve, reject) => { + stagingWriteStream!.once("error", reject); + sourceReadStream!.once("error", reject); + stagingWriteStream!.once("finish", resolve); + sourceReadStream!.pipe(stagingWriteStream!); + }); + } catch (streamErr: any) { + if (stagingWriteStream) { + if (!stagingWriteStream.destroyed) { + stagingWriteStream.destroy(); + } + if (!stagingWriteStream.closed) { + await new Promise((res) => { + stagingWriteStream!.once("close", () => res()); + setTimeout(res, 50); + }); + } + } + if (sourceReadStream && !sourceReadStream.destroyed) { + sourceReadStream.destroy(); + } + + // Attempt filesystem cleanup of temporary staging file + try { + await fs.promises.unlink(tempDestPath); + } catch (cleanupErr: any) { + if (cleanupErr?.code !== "ENOENT") { + let streamFailureIdentity: StagingFileIdentity | undefined; + try { + const st = await fs.promises.lstat(tempDestPath); + streamFailureIdentity = { + exactPath: tempDestPath, + dev: st.dev, + ino: st.ino, + size: st.size, + birthtimeMs: st.birthtimeMs, + registeredAt: Date.now(), + retryCount: 0, + }; + } catch { + // Identity capture failed: streamFailureIdentity remains undefined. + // Existing fail-closed behavior will apply during retry verification (dev: 0, ino: 0). + } + + const isTracked = this.opManager.recordPendingStagingCleanup( + tempDestPath, + streamFailureIdentity, + ); + if (!isTracked) { + logger.warn( + { tempDestPath, err: cleanupErr }, + "Failed to clean up staging file on streaming failure and retry queue at capacity; file abandoned as untracked orphan", + ); + } + } + } + + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Failed to stream export data to staging file: ${streamErr?.message}`, + ); + } finally { + if (stagingWriteStream && !stagingWriteStream.destroyed) { + stagingWriteStream.destroy(); + } + if (sourceReadStream && !sourceReadStream.destroyed) { + sourceReadStream.destroy(); + } + } + + // Capture staging file identity for cleanup tracking + let stagingIdentity: StagingFileIdentity | undefined; + let stagedStat: fs.Stats | undefined; + try { + stagedStat = await fs.promises.lstat(tempDestPath); + stagingIdentity = { + exactPath: tempDestPath, + dev: stagedStat.dev, + ino: stagedStat.ino, + size: stagedStat.size, + birthtimeMs: stagedStat.birthtimeMs, + registeredAt: Date.now(), + retryCount: 0, + }; + } catch { + // Identity remains undefined if lstat fails; fail-closed rules will apply if cleanup retried + } + const bytesWritten = stagedStat?.size ?? 0; + + // STEP 1: Authoritative publication operation + if (options?.overwrite === true) { + // Atomic replacement via rename + try { + await fs.promises.rename(tempDestPath, normalizedDestPath); + } catch (renameErr: any) { + // Pre-publication failure: attempt staging cleanup + try { + await fs.promises.unlink(tempDestPath); + } catch (cleanupErr: any) { + if (cleanupErr?.code !== "ENOENT") { + const isTracked = this.opManager.recordPendingStagingCleanup( + tempDestPath, + stagingIdentity, + ); + if (!isTracked) { + logger.warn( + { tempDestPath, err: cleanupErr }, + "Failed to clean up staging file on pre-publication failure and retry queue at capacity; file abandoned as untracked orphan", + ); + } + } + } + + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Failed to atomically replace destination file (${renameErr?.code ?? "UNKNOWN"}): ${renameErr?.message}`, + ); + } + } else { + // Atomic hard-link publication for overwrite: false + try { + await fs.promises.link(tempDestPath, normalizedDestPath); + } catch (linkErr: any) { + // Pre-publication failure: attempt staging cleanup + try { + await fs.promises.unlink(tempDestPath); + } catch (cleanupErr: any) { + if (cleanupErr?.code !== "ENOENT") { + const isTracked = this.opManager.recordPendingStagingCleanup( + tempDestPath, + stagingIdentity, + ); + if (!isTracked) { + logger.warn( + { tempDestPath, err: cleanupErr }, + "Failed to clean up staging file on pre-publication failure and retry queue at capacity; file abandoned as untracked orphan", + ); + } + } + } + + if (linkErr?.code === "EEXIST") { + throw new MongoApiError("MONGO_FILE_EXISTS", "Destination file already exists"); + } + + if ( + linkErr?.code === "ENOSYS" || + linkErr?.code === "EOPNOTSUPP" || + linkErr?.code === "ENOTSUP" + ) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_FILESYSTEM", + "The destination filesystem does not support atomic hard-link publication (e.g. FAT32, exFAT, or unsupported network share). " + + "Please choose a destination on an NTFS or POSIX filesystem, or export with explicit overwrite enabled.", + ); + } + + if (linkErr?.code === "EACCES" || linkErr?.code === "EPERM") { + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Permission denied or operation not permitted during hard-link publication (${linkErr.code}): ${linkErr.message}`, + ); + } + + if (linkErr?.code === "EXDEV") { + throw new MongoApiError( + "MONGO_SAVE_FAILED", + "Cross-device link error (EXDEV): destination spans a volume, junction, or mount boundary. " + + "Please choose a destination on a single local filesystem volume, or export with explicit overwrite enabled.", + ); + } + + throw new MongoApiError( + "MONGO_SAVE_FAILED", + `Failed to publish export file (${linkErr?.code ?? "UNKNOWN"}): ${linkErr?.message}`, + ); + } + } + + // STEP 2: Post-publication staging cleanup & audit (Destination is COMMITTED) + let warning: string | undefined; + + // Staging unlink is only required for hard-link publication (!options?.overwrite). + // Atomic rename already moved/consumed tempDestPath; skipping unlink avoids an artificial ENOENT error. + if (!options?.overwrite) { + try { + await fs.promises.unlink(tempDestPath); + } catch (cleanupErr: any) { + if (cleanupErr?.code !== "ENOENT") { + const isTracked = this.opManager.recordPendingStagingCleanup( + tempDestPath, + stagingIdentity, + ); + if (isTracked) { + warning = `Destination file published successfully, but temporary staging file could not be removed (${cleanupErr.code ?? "UNKNOWN"}); queued for identity-verified retry cleanup.`; + } else { + warning = `Destination file published successfully, but temporary staging file could not be removed (${cleanupErr.code ?? "UNKNOWN"}) and cleanup queue is at capacity (100); file remains as an untracked orphan requiring manual removal: ${tempDestPath}`; + } + } + } + } + + try { + const postStat = await fs.promises.lstat(normalizedDestPath); + if (!postStat.isFile()) { + const auditMsg = + "Destination published, but post-commit audit detected unexpected file attributes"; + logger.warn({ destinationPath: normalizedDestPath }, auditMsg); + warning = warning ? `${warning}; ${auditMsg}` : auditMsg; + } + } catch (auditErr: any) { + const auditMsg = `Destination published, but post-commit verification failed (${auditErr.code ?? "UNKNOWN"}): ${auditErr.message}`; + logger.warn({ err: auditErr, destinationPath: normalizedDestPath }, auditMsg); + warning = warning ? `${warning}; ${auditMsg}` : auditMsg; + } + + return { + operationId, + destinationPath: normalizedDestPath, + bytesWritten, + success: true, + warning, + }; + } finally { + state.isSaving = false; + if (Date.now() > new Date(state.operation.expiresAt).getTime()) { + try { + this.opManager.cleanup(operationId); + } catch { + // safe ignore + } + } + } + } + + // ========================================================================= + // 4. Supporting Bulk Mutations + // ========================================================================= + + async bulkInsert(req: MongoBulkInsertRequest): Promise { + const { dbId, collection, documents, options } = req; + assertValidCollectionName(collection); + + if (!Array.isArray(documents) || documents.length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "documents must be a non-empty array"); + } + + if (documents.length > MAX_BULK_INSERT_DOCS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `documents count ${documents.length} exceeds maximum allowed of ${MAX_BULK_INSERT_DOCS}`, + ); + } + + const byteSize = Buffer.byteLength(JSON.stringify(documents), "utf8"); + if (byteSize > MAX_BULK_INSERT_SIZE_BYTES) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `documents payload size ${byteSize} exceeds 16MB limit`, + ); + } + + for (const doc of documents) { + if (typeof doc !== "object" || doc === null || Array.isArray(doc)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Each item in documents must be an object", + ); + } + assertNoPrototypePollution(doc); + } + + const { db, conn, nativeDb } = await this.getDb(dbId); + assertNotReadOnly(db, conn); + + const coll = nativeDb.collection(collection); + const ordered = options?.ordered ?? false; + let insertedCount = 0; + let failedCount = 0; + const errors: Array<{ index: number; code: string; message: string }> = []; + + for (let i = 0; i < documents.length; i += BATCH_WRITE_SIZE) { + const batch = documents.slice(i, i + BATCH_WRITE_SIZE); + const bsonDocs = batch.map((d) => serializeMongoEJSONDocument(d)); + + try { + const res = await coll.insertMany(bsonDocs, { ordered }); + insertedCount += res.insertedCount ?? Object.keys(res.insertedIds || {}).length; + } catch (err: any) { + if ( + err?.name === "MongoBulkWriteError" || + err?.name === "BulkWriteError" || + err?.writeErrors + ) { + const nInserted = err.result?.nInserted ?? err.insertedCount ?? 0; + insertedCount += nInserted; + const writeErrors = err.writeErrors || []; + for (const we of writeErrors) { + const isDup = we.code === 11000 || we.code === 11001; + errors.push({ + index: i + (we.index ?? 0), + code: isDup ? "MONGO_DUPLICATE_KEY" : "MONGO_WRITE_ERROR", + message: safeMongoError(we.errmsg || we.message || "Write error"), + }); + } + failedCount += batch.length - nInserted; + if (ordered) { + break; + } + } else { + throw err; + } + } + } + + mongodbCache.clearCollectionCache(conn, collection); + + return { + collection, + insertedCount, + failedCount, + errors: errors.length > 0 ? errors : undefined, + }; + } + + async bulkUpdate(req: MongoBulkUpdateRequest): Promise { + const { dbId, collection, filter, update, options } = req; + assertValidCollectionName(collection); + + if (typeof filter !== "object" || filter === null || Array.isArray(filter)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + + if (typeof update !== "object" || update === null || Array.isArray(update)) { + throw new MongoApiError( + "MONGO_INVALID_UPDATE_OPERAND", + "update must be an operator document", + ); + } + + const updateKeys = Object.keys(update); + if (updateKeys.length === 0) { + throw new MongoApiError("MONGO_INVALID_UPDATE_OPERAND", "No update operators provided"); + } + for (const k of updateKeys) { + if (!k.startsWith("$")) { + throw new MongoApiError( + "MONGO_INVALID_UPDATE_OPERAND", + `'${k}' is not an update operator; top-level operators must begin with '$'`, + ); + } + } + + const isFilterEmpty = Object.keys(filter).length === 0; + if (isFilterEmpty && options?.allowUnrestricted !== true) { + throw new MongoApiError( + "MONGO_BULK_OP_FORBIDDEN", + "update requires a non-empty filter or allowUnrestricted: true to prevent accidental whole-collection operations", + ); + } + + assertNoPrototypePollution(filter); + assertNoPrototypePollution(update); + + const { db, conn, nativeDb } = await this.getDb(dbId); + assertNotReadOnly(db, conn); + + const bsonFilter = serializeMongoFilter(filter); + const bsonUpdate = serializeMongoUpdateOperand(update); + const coll = nativeDb.collection(collection); + + const res = await coll.updateMany(bsonFilter, bsonUpdate, { + upsert: options?.upsert ?? false, + }); + + mongodbCache.clearCollectionCache(conn, collection); + + let advisoryWarning: string | undefined; + if (isFilterEmpty && options?.allowUnrestricted === true) { + advisoryWarning = "Unrestricted bulk update performed across entire collection"; + } + + return { + collection, + matchedCount: res.matchedCount, + modifiedCount: res.modifiedCount, + upsertedId: res.upsertedId ? (mongoBSONtoEJSON(res.upsertedId) as any) : undefined, + advisoryWarning, + }; + } + + async bulkDelete(req: MongoBulkDeleteRequest): Promise { + const { dbId, collection, filter, options } = req; + assertValidCollectionName(collection); + + if (typeof filter !== "object" || filter === null || Array.isArray(filter)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + + const isFilterEmpty = Object.keys(filter).length === 0; + if (isFilterEmpty && options?.allowUnrestricted !== true) { + throw new MongoApiError( + "MONGO_BULK_OP_FORBIDDEN", + "delete requires a non-empty filter or allowUnrestricted: true to prevent accidental whole-collection operations", + ); + } + + assertNoPrototypePollution(filter); + + const { db, conn, nativeDb } = await this.getDb(dbId); + assertNotReadOnly(db, conn); + + const bsonFilter = serializeMongoFilter(filter); + const coll = nativeDb.collection(collection); + + const res = await coll.deleteMany(bsonFilter); + + mongodbCache.clearCollectionCache(conn, collection); + + let advisoryWarning: string | undefined; + if (res.deletedCount > 100) { + advisoryWarning = `Broad delete operation affected more than 100 documents (${res.deletedCount} deleted)`; + } + + return { + collection, + deletedCount: res.deletedCount, + advisoryWarning, + }; + } +} diff --git a/bridge/src/services/mongoMigrationEngine.ts b/bridge/src/services/mongoMigrationEngine.ts new file mode 100644 index 00000000..91edd33f --- /dev/null +++ b/bridge/src/services/mongoMigrationEngine.ts @@ -0,0 +1,1690 @@ +/** + * MongoDB Milestone 6 — Declarative Migration Engine. + * + * Implements: + * - Filesystem discovery and security sandbox + * - Static manifest preflight validation + * - Distributed locking integration with fencing token verification + * - Step runner with immutable event journaling (_relwave_migration_events) + * - Materialized summary projection (_relwave_migrations) and rebuild algorithm + * - Reverse compensation state machine with count-guarded collection rollback + * - Rollback execution and crash reconciliation + */ + +import fs from "fs"; +import path from "path"; +import crypto from "crypto"; +import { v4 as uuidv4 } from "uuid"; +import { Db } from "mongodb"; +import { MongoApiError, assertSafeExpressions } from "../connectors/mongo"; +import { + acquireLock, + renewLockHeartbeat, + releaseLock, + checkLockStatus, + verifyLock, + DEFAULT_HEARTBEAT_MS, +} from "./mongoMigrationLock"; +import { + MongoMigrationStep, + MongoMigrationManifest, + MongoMigrationEvent, + MongoMigrationEventType, + MongoMigrationSummary, + MongoMigrationStatus, + MongoMigrationListResult, + MongoMigrationValidateResult, + MongoMigrationApplyResult, + MongoMigrationRollbackResult, + MongoMigrationGetStatusResult, +} from "../types/mongoMigration"; + +export const MAX_MANIFEST_SIZE_BYTES = 1_048_576; // 1 MB +export const MANIFEST_FILENAME_REGEX = /^\d{14}_[a-zA-Z0-9_-]+\.mql\.json$/; +export const EVENTS_COLLECTION = "_relwave_migration_events"; +export const MIGRATIONS_COLLECTION = "_relwave_migrations"; + +/** + * Normalizes CRLF to LF and computes SHA-256 checksum. + */ +export function computeManifestChecksum(content: string | Buffer): string { + const text = typeof content === "string" ? content : content.toString("utf8"); + const normalized = text.replace(/\r\n/g, "\n"); + return crypto.createHash("sha256").update(normalized, "utf8").digest("hex"); +} + +/** + * Validates canonical file path to prevent directory traversal or external symlink escape. + */ +export function validateSafeFilePath(filePath: string, allowedDir: string): void { + if (filePath.includes("\0") || filePath.includes("..")) { + throw new MongoApiError("SECURITY_VIOLATION", "Path traversal or null byte detected"); + } + + const canonicalDir = fs.existsSync(allowedDir) + ? fs.realpathSync(allowedDir) + : path.resolve(allowedDir); + const resolvedPath = path.resolve(filePath); + + if (!fs.existsSync(resolvedPath)) { + const dirOfFile = path.dirname(resolvedPath); + const canonicalDirOfFile = fs.existsSync(dirOfFile) + ? fs.realpathSync(dirOfFile) + : path.resolve(dirOfFile); + + if ( + !canonicalDirOfFile.startsWith(canonicalDir + path.sep) && + canonicalDirOfFile !== canonicalDir + ) { + throw new MongoApiError( + "SECURITY_VIOLATION", + "File path escapes allowed migration directory", + ); + } + return; + } + + const canonicalFilePath = fs.realpathSync(resolvedPath); + if ( + !canonicalFilePath.startsWith(canonicalDir + path.sep) && + canonicalFilePath !== canonicalDir + ) { + throw new MongoApiError( + "SECURITY_VIOLATION", + "Symlink or path traversal outside allowed migration directory", + ); + } +} + +/** + * Resolves the mongo migration directory for a given project root. + */ +export function resolveMongoMigrationsDir(projectRoot: string): string { + return path.resolve(projectRoot, "migrations", "mongo"); +} + +/** + * Generates an index name from a key specification if name is not explicitly provided. + */ +export function getIndexNameFromKey(key: Record): string { + return Object.entries(key) + .map(([k, v]) => `${k}_${v}`) + .join("_"); +} + +/** + * Validates a migration manifest against schema and security policies. + */ +export function validateManifest(manifest: any): MongoMigrationValidateResult { + const errors: string[] = []; + const warnings: string[] = []; + let ddlCount = 0; + let dataTransformCount = 0; + let reversibleCount = 0; + let irreversibleCount = 0; + + if (!manifest || typeof manifest !== "object") { + return { + valid: false, + errors: ["Manifest must be a non-null object"], + warnings, + stepsBreakdown: { ddlCount, dataTransformCount, reversibleCount, irreversibleCount }, + }; + } + + if (!manifest.id || typeof manifest.id !== "string") { + errors.push("Manifest requires a non-empty string 'id'"); + } + if (!manifest.name || typeof manifest.name !== "string") { + errors.push("Manifest requires a non-empty string 'name'"); + } + if (!manifest.version || typeof manifest.version !== "string") { + errors.push("Manifest requires a non-empty string 'version'"); + } + if (!Array.isArray(manifest.up)) { + errors.push("Manifest requires an array of 'up' steps"); + } else if (manifest.up.length === 0) { + errors.push("Manifest 'up' array must not be empty"); + } + + const upSteps: any[] = Array.isArray(manifest.up) ? manifest.up : []; + const downSteps: any[] = Array.isArray(manifest.down) ? manifest.down : []; + + for (let idx = 0; idx < upSteps.length; idx++) { + const step = upSteps[idx]; + if (!step || typeof step !== "object") { + errors.push(`Step at index ${idx} must be a non-null object`); + continue; + } + + if (step.type === "deleteMany") { + errors.push( + `Step at index ${idx}: deleteMany is permanently excluded from migration manifests to prevent unrecoverable data loss`, + ); + continue; + } + + const ALLOWED_TYPES = [ + "createCollection", + "dropCollection", + "createIndex", + "dropIndex", + "setValidator", + "updateMany", + ]; + if (!ALLOWED_TYPES.includes(step.type)) { + errors.push(`Step at index ${idx}: unsupported step type '${step.type}'`); + continue; + } + + switch (step.type) { + case "createCollection": { + ddlCount++; + reversibleCount++; + if (!step.collection || typeof step.collection !== "string") { + errors.push( + `Step at index ${idx} (createCollection): requires non-empty collection name`, + ); + } else if ( + step.collection.includes("$") || + step.collection.includes("\0") || + step.collection.startsWith("system.") + ) { + errors.push( + `Step at index ${idx} (createCollection): invalid collection name '${step.collection}'`, + ); + } + if (step.options?.capped) { + if (typeof step.options.size !== "number" || step.options.size < 4096) { + errors.push( + `Step at index ${idx} (createCollection): capped collection requires size >= 4096`, + ); + } + } + break; + } + case "dropCollection": { + ddlCount++; + irreversibleCount++; + if (!step.collection || typeof step.collection !== "string") { + errors.push(`Step at index ${idx} (dropCollection): requires non-empty collection name`); + } else if (step.collection.startsWith("system.")) { + errors.push(`Step at index ${idx} (dropCollection): cannot drop system collection`); + } + if (!manifest.allowIrreversible) { + errors.push( + `Step at index ${idx} (dropCollection): requires allowIrreversible: true in manifest`, + ); + } + break; + } + case "createIndex": { + ddlCount++; + reversibleCount++; + if (!step.collection || typeof step.collection !== "string") { + errors.push(`Step at index ${idx} (createIndex): requires non-empty collection name`); + } + if (!step.key || typeof step.key !== "object" || Object.keys(step.key).length === 0) { + errors.push(`Step at index ${idx} (createIndex): requires non-empty key specification`); + } else { + for (const [k, dir] of Object.entries(step.key)) { + if (![1, -1, "text", "2dsphere", "hashed"].includes(dir as any)) { + errors.push( + `Step at index ${idx} (createIndex): invalid direction '${dir}' for key '${k}'`, + ); + } + } + } + if (step.options?.partialFilterExpression) { + try { + assertSafeExpressions(step.options.partialFilterExpression); + } catch (err: any) { + errors.push( + `Step at index ${idx} (createIndex): forbidden expression in partialFilterExpression: ${err.message}`, + ); + } + } + break; + } + case "dropIndex": { + ddlCount++; + reversibleCount++; + if (!step.collection || typeof step.collection !== "string") { + errors.push(`Step at index ${idx} (dropIndex): requires non-empty collection name`); + } + if (!step.indexName || typeof step.indexName !== "string") { + errors.push(`Step at index ${idx} (dropIndex): requires non-empty indexName`); + } else if (step.indexName === "_id_") { + errors.push(`Step at index ${idx} (dropIndex): cannot drop default _id_ index`); + } + break; + } + case "setValidator": { + ddlCount++; + reversibleCount++; + if (!step.collection || typeof step.collection !== "string") { + errors.push(`Step at index ${idx} (setValidator): requires non-empty collection name`); + } + if (!step.validator || typeof step.validator !== "object") { + errors.push(`Step at index ${idx} (setValidator): requires validator object`); + } else { + try { + assertSafeExpressions(step.validator); + } catch (err: any) { + errors.push( + `Step at index ${idx} (setValidator): forbidden expression in validator: ${err.message}`, + ); + } + } + break; + } + case "updateMany": { + dataTransformCount++; + if (!step.collection || typeof step.collection !== "string") { + errors.push(`Step at index ${idx} (updateMany): requires non-empty collection name`); + } + if (!step.filter || typeof step.filter !== "object") { + errors.push(`Step at index ${idx} (updateMany): requires filter object`); + } + if (!step.update || typeof step.update !== "object") { + errors.push(`Step at index ${idx} (updateMany): requires update operator document`); + } + + // Validate operator keys and AST safety + if (step.filter) { + try { + assertSafeExpressions(step.filter); + } catch (err: any) { + errors.push( + `Step at index ${idx} (updateMany): forbidden expression in filter: ${err.message}`, + ); + } + } + if (step.update) { + try { + assertSafeExpressions(step.update); + } catch (err: any) { + errors.push( + `Step at index ${idx} (updateMany): forbidden expression in update: ${err.message}`, + ); + } + const hasOpKeys = Object.keys(step.update).some((k) => k.startsWith("$")); + if (!hasOpKeys) { + errors.push( + `Step at index ${idx} (updateMany): update operand must contain atomic operators ($set, $unset, etc.)`, + ); + } + } + + // Enforce author-provided down step + const hasDown = downSteps.some( + (d) => d && d.type === "updateMany" && d.collection === step.collection, + ); + if (!hasDown) { + errors.push( + `Step at index ${idx} (updateMany): updateMany requires an author-provided down step for collection '${step.collection}'`, + ); + irreversibleCount++; + } else { + reversibleCount++; + } + break; + } + } + } + + return { + valid: errors.length === 0, + errors, + warnings, + stepsBreakdown: { + ddlCount, + dataTransformCount, + reversibleCount, + irreversibleCount, + }, + }; +} + +/** + * Asserts that a manifest is valid, throwing an appropriate MongoApiError if not. + */ +export function assertValidManifest(manifest: any): void { + const result = validateManifest(manifest); + if (!result.valid) { + const forbiddenErr = result.errors.find( + (e) => + e.includes("forbidden") || + e.includes("$where") || + e.includes("$function") || + e.includes("$accumulator"), + ); + if (forbiddenErr) { + throw new MongoApiError("MONGO_FORBIDDEN_OPERATOR", forbiddenErr); + } + if (result.errors.some((e) => e.includes("deleteMany"))) { + throw new MongoApiError( + "FORBIDDEN_STEP_TYPE", + "deleteMany is permanently excluded from migration manifests", + ); + } + if (result.errors.some((e) => e.includes("allowIrreversible"))) { + throw new MongoApiError( + "IRREVERSIBLE_MIGRATION", + "dropCollection requires allowIrreversible: true in manifest", + ); + } + throw new MongoApiError("INVALID_MANIFEST", result.errors.join("; ")); + } +} + +/** + * Compensates a single step during rollback or forward-failure unwind. + */ +export async function compensateStep( + db: Db, + step: MongoMigrationStep, + preState: any, + manifestDown?: MongoMigrationStep[], +): Promise { + switch (step.type) { + case "createIndex": { + const indexName = step.options?.name || getIndexNameFromKey(step.key); + try { + await db.collection(step.collection).dropIndex(indexName); + } catch (err: any) { + if (err.code !== 27 && err.codeName !== "IndexNotFound") { + throw err; + } + } + break; + } + case "dropIndex": { + if (preState && preState.key) { + const { key, name: _name, v: _v, ns: _ns, ...restOptions } = preState; + await db + .collection(step.collection) + .createIndex(key, { ...restOptions, name: step.indexName }); + } else { + const downStep = manifestDown?.find( + (d) => + d.type === "createIndex" && + d.collection === step.collection && + (d.options?.name === step.indexName || + (!d.options?.name && getIndexNameFromKey(d.key) === step.indexName)), + ); + if (downStep && downStep.type === "createIndex") { + await db + .collection(downStep.collection) + .createIndex(downStep.key as any, downStep.options as any); + } else { + throw new MongoApiError( + "COMPENSATION_FAILED", + `Cannot compensate dropIndex for '${step.indexName}' on '${step.collection}': pre-state missing and no matching createIndex found in down steps`, + ); + } + } + break; + } + case "createCollection": { + // COUNT-GUARD: check existence and verify collection has not received external data + const collections = await db.listCollections({ name: step.collection }).toArray(); + if (collections.length > 0) { + const count = await db.collection(step.collection).countDocuments({}); + if (count > 0) { + throw new MongoApiError( + "COMPENSATION_FAILED", + `Compensation aborted: Collection '${step.collection}' contains ${count} documents added after creation. Manual intervention required to prevent unintended data loss.`, + ); + } + await db.collection(step.collection).drop(); + } + break; + } + case "dropCollection": { + throw new MongoApiError( + "IRREVERSIBLE_STEP", + `Cannot compensate dropCollection for '${step.collection}'. Data was destroyed.`, + ); + } + case "setValidator": { + const previousValidator = preState?.validator || {}; + const command: Record = { + collMod: step.collection, + validator: previousValidator, + }; + if (preState?.validationLevel !== undefined) { + command.validationLevel = preState.validationLevel; + } + if (preState?.validationAction !== undefined) { + command.validationAction = preState.validationAction; + } + await db.command(command); + break; + } + case "updateMany": { + const downStep = manifestDown?.find( + (d) => d.type === "updateMany" && d.collection === step.collection, + ); + if (!downStep || downStep.type !== "updateMany") { + throw new MongoApiError( + "COMPENSATION_FAILED", + `Missing author-provided down step for updateMany on collection '${step.collection}'`, + ); + } + await db + .collection(downStep.collection) + .updateMany(downStep.filter, downStep.update, downStep.options as any); + break; + } + } +} + +/** + * Executes a single migration manifest through the checkpointed state machine. + */ +async function applySingleManifest( + db: Db, + manifest: MongoMigrationManifest, + checksum: string, + options?: { + dryRun?: boolean; + ownerId?: string; + }, +): Promise<{ + id: string; + name: string; + executionTimeMs: number; + status: MongoMigrationStatus; + executedSteps: number; + error?: string; +}> { + assertValidManifest(manifest); + + const existing = await db + .collection(MIGRATIONS_COLLECTION) + .findOne({ _id: manifest.id }); + + if (existing && existing.status === "APPLIED") { + if (existing.checksum && existing.checksum !== checksum) { + throw new MongoApiError( + "CHECKSUM_MISMATCH", + `Migration '${manifest.id}' was previously applied with checksum '${existing.checksum}', but manifest has checksum '${checksum}'`, + ); + } + throw new MongoApiError( + "MIGRATION_ALREADY_APPLIED", + `Migration '${manifest.id}' has already been applied`, + ); + } + + if (options?.dryRun) { + return { + id: manifest.id, + name: manifest.name, + executionTimeMs: 0, + status: "PENDING", + executedSteps: 0, + }; + } + + const ownerId = options?.ownerId || uuidv4(); + const { fencingToken } = await acquireLock(db, ownerId, manifest.id); + + let heartbeatInFlight = false; + let lockLost = false; + const heartbeatTimer: NodeJS.Timeout | null = setInterval(async () => { + if (heartbeatInFlight || lockLost) return; + heartbeatInFlight = true; + try { + const renewed = await renewLockHeartbeat(db, ownerId, fencingToken); + if (!renewed) { + lockLost = true; + if (heartbeatTimer) clearInterval(heartbeatTimer); + } + } catch { + // transient network error; verifyLock guards pre-step + } finally { + heartbeatInFlight = false; + } + }, DEFAULT_HEARTBEAT_MS); + if (heartbeatTimer.unref) heartbeatTimer.unref(); + + const attemptId = uuidv4(); + const prevEvents = await db + .collection(EVENTS_COLLECTION) + .find({ migrationId: manifest.id }) + .sort({ attemptNumber: -1 }) + .limit(1) + .toArray(); + const attemptNumber = (prevEvents[0]?.attemptNumber || 0) + 1; + let sequenceNumber = 0; + + const recordEvent = async ( + eventType: MongoMigrationEventType, + stepIndex?: number, + stepType?: string, + metadata?: Record, + ) => { + sequenceNumber++; + const eventDoc: MongoMigrationEvent = { + _id: `${manifest.id}_a${attemptNumber}_s${sequenceNumber}`, + eventId: uuidv4(), + migrationId: manifest.id, + attemptId, + attemptNumber, + sequenceNumber, + stepIndex, + stepType, + eventType, + timestamp: new Date().toISOString(), + lockOwnerId: ownerId, + fencingToken, + metadata, + }; + await db.collection(EVENTS_COLLECTION).insertOne(eventDoc as any); + return eventDoc; + }; + + await recordEvent("VALIDATION_SUCCESS"); + await recordEvent("LOCK_ACQUIRED"); + await recordEvent("MIGRATION_START"); + + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + name: manifest.name, + checksum, + status: "APPLYING", + startedAt: new Date().toISOString(), + executedSteps: 0, + totalSteps: manifest.up.length, + fencingToken, + attemptNumber, + attemptId, + appliedBy: ownerId, + isReversible: manifest.up.every((s) => s.type !== "dropCollection"), + }, + }, + { upsert: true }, + ); + + const executedPreStates: Array<{ step: MongoMigrationStep; preState: any }> = []; + const startTime = Date.now(); + let executedSteps = 0; + + try { + for (let i = 0; i < manifest.up.length; i++) { + const step = manifest.up[i]; + + // Pre-step fencing check! + if (lockLost || !(await verifyLock(db, ownerId, fencingToken))) { + await recordEvent("STEP_UNCERTAIN", i, step.type, { + reason: "Pre-step fencing check failed; lock lease expired or stolen", + }); + await recordEvent("MANUAL_INTERVENTION_REQUIRED", i, step.type, { + reason: "Fencing token violation mid-migration", + }); + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + status: "CRASHED_UNCERTAIN", + lastExecutedStepIndex: i, + error: "LOCK_LOST_FENCING_VIOLATION: Lock lease expired or stolen by another client", + }, + }, + ); + throw new MongoApiError( + "LOCK_LOST_FENCING_VIOLATION", + `Migration lock lease lost before executing step ${i}`, + ); + } + + // Pre-state capture before mutation + let preState: any = null; + if (step.type === "dropIndex") { + try { + const coll = db.collection(step.collection); + const indexes = await coll.indexes(); + const targetIdx = indexes.find((idx) => idx.name === step.indexName); + if (targetIdx) { + const { v: _v, ns: _ns, ...rest } = targetIdx; + preState = rest; + } + } catch { + // ignore if collection doesn't exist + } + } else if (step.type === "setValidator") { + try { + const collInfos = await db.listCollections({ name: step.collection }).toArray(); + const firstColl = collInfos[0] as any; + if (firstColl && firstColl.options) { + preState = { + validator: firstColl.options.validator || {}, + validationLevel: firstColl.options.validationLevel, + validationAction: firstColl.options.validationAction, + }; + } else { + preState = { + validator: {}, + validationLevel: undefined, + validationAction: undefined, + }; + } + } catch { + preState = { + validator: {}, + validationLevel: undefined, + validationAction: undefined, + }; + } + } + + await recordEvent("STEP_START", i, step.type, { preState }); + const stepStart = Date.now(); + + try { + switch (step.type) { + case "createCollection": { + await db.createCollection(step.collection, step.options as any); + break; + } + case "dropCollection": { + await db.collection(step.collection).drop(); + break; + } + case "createIndex": { + await db.collection(step.collection).createIndex(step.key as any, step.options as any); + break; + } + case "dropIndex": { + await db.collection(step.collection).dropIndex(step.indexName); + break; + } + case "setValidator": { + const command: Record = { + collMod: step.collection, + validator: step.validator, + }; + if (step.validationLevel) command.validationLevel = step.validationLevel; + if (step.validationAction) command.validationAction = step.validationAction; + await db.command(command); + break; + } + case "updateMany": { + await db + .collection(step.collection) + .updateMany(step.filter, step.update, step.options as any); + break; + } + } + + const stepDuration = Date.now() - stepStart; + await recordEvent("STEP_COMPLETE", i, step.type, { durationMs: stepDuration }); + executedPreStates.push({ step, preState }); + executedSteps++; + + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + executedSteps, + lastExecutedStepIndex: i, + }, + }, + ); + } catch (stepErr: any) { + await recordEvent("STEP_FAILED", i, step.type, { + error: { message: stepErr.message, stack: stepErr.stack }, + }); + + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + status: "FAILED_PARTIAL", + lastExecutedStepIndex: i, + error: { + stepIndex: i, + stepType: step.type, + message: stepErr.message, + }, + }, + }, + ); + + // Reverse compensation + await recordEvent("COMPENSATION_START", i, step.type); + await db + .collection(MIGRATIONS_COLLECTION) + .updateOne({ _id: manifest.id }, { $set: { status: "ROLLING_BACK" } }); + + let compensationFailed = false; + let compError: any = null; + + for (let j = executedPreStates.length - 1; j >= 0; j--) { + const { step: completedStep, preState: savedPreState } = executedPreStates[j]; + if (lockLost || !(await verifyLock(db, ownerId, fencingToken))) { + compensationFailed = true; + compError = new MongoApiError( + "LOCK_LOST_FENCING_VIOLATION", + "Lock lease lost during reverse compensation", + ); + break; + } + try { + await compensateStep(db, completedStep, savedPreState, manifest.down); + } catch (cErr: any) { + compensationFailed = true; + compError = cErr; + break; + } + } + + if (compensationFailed) { + await recordEvent("COMPENSATION_FAILED", i, step.type, { + error: { message: compError?.message }, + }); + await recordEvent("MANUAL_INTERVENTION_REQUIRED", i, step.type, { + reason: compError?.message || "Reverse compensation failed", + }); + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + status: "FAILED_MANUAL_INTERVENTION_REQUIRED", + error: { + stepIndex: i, + stepType: step.type, + message: `Step failed: ${stepErr.message}. Compensation failed: ${compError?.message}`, + }, + }, + }, + ); + } else { + await recordEvent("COMPENSATION_COMPLETE", i, step.type); + await recordEvent("MIGRATION_ROLLED_BACK", i, step.type); + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + status: "ROLLED_BACK", + }, + }, + ); + } + + throw stepErr; + } + } + + const totalDuration = Date.now() - startTime; + await recordEvent("MIGRATION_APPLIED", undefined, undefined, { + durationMs: totalDuration, + }); + + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: manifest.id }, + { + $set: { + status: "APPLIED", + completedAt: new Date().toISOString(), + durationMs: totalDuration, + totalDurationMs: totalDuration, + executedSteps: manifest.up.length, + lastExecutedStepIndex: manifest.up.length - 1, + attemptNumber, + attemptId, + }, + }, + ); + + return { + id: manifest.id, + name: manifest.name, + executionTimeMs: totalDuration, + status: "APPLIED", + executedSteps: manifest.up.length, + }; + } finally { + if (heartbeatTimer) clearInterval(heartbeatTimer); + await releaseLock(db, ownerId, fencingToken); + await recordEvent("LOCK_RELEASED"); + } +} + +/** + * List applied and pending migrations for a MongoDB database. + * Computes deterministic drift classification for applied migrations + * strictly adhering to 5-precedence order without mutating database state. + */ +export async function listMigrations( + db: Db, + projectRoot: string, +): Promise { + const lockStatus = await checkLockStatus(db); + const migrationsDir = resolveMongoMigrationsDir(projectRoot); + + const appliedSummaries = await db + .collection(MIGRATIONS_COLLECTION) + .find({}) + .sort({ _id: 1 }) + .toArray(); + + const appliedMap = new Map(); + for (const s of appliedSummaries) { + appliedMap.set(s._id, s); + } + + const pendingMigrations: MongoMigrationListResult["pendingMigrations"] = []; + + // If migrationsDir does not exist, all applied migrations become UNVERIFIABLE (MANIFEST_MISSING) + if (!fs.existsSync(migrationsDir)) { + for (const applied of appliedSummaries) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_MISSING", + expectedChecksum: applied.checksum || undefined, + message: "Manifest file not found in migrations directory", + }; + } + + return { + appliedMigrations: appliedSummaries, + pendingMigrations, + isLocked: lockStatus.isLocked, + lockDetails: lockStatus.isLocked + ? { + ownerId: lockStatus.ownerId, + fencingToken: lockStatus.fencingToken, + expiresAt: lockStatus.expiresAt, + migrationId: lockStatus.migrationId, + } + : undefined, + }; + } + + const rawFiles = fs.readdirSync(migrationsDir); + const candidateFiles = rawFiles.filter((f) => MANIFEST_FILENAME_REGEX.test(f)); + // Deterministically pre-sort filenames alphabetically + candidateFiles.sort((a, b) => a.localeCompare(b)); + + // Group candidate files by filename-derived ID (split on first '_') + const idToFilesMap = new Map(); + for (const file of candidateFiles) { + const filenameId = file.split("_")[0]; + const list = idToFilesMap.get(filenameId) || []; + list.push(file); + idToFilesMap.set(filenameId, list); + } + + const duplicateIds = new Set(); + for (const [id, fileList] of idToFilesMap.entries()) { + if (fileList.length > 1) { + duplicateIds.add(id); + const applied = appliedMap.get(id); + if (applied) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + expectedChecksum: applied.checksum || undefined, + message: `Multiple manifest files on disk resolve to migration ID '${id}': [${fileList.join(", ")}]`, + }; + } + } + } + + // Map to store valid parsed manifests on disk for applied drift verification + const validDiskFiles = new Map< + string, + { checksum: string; manifest: MongoMigrationManifest; file: string } + >(); + + for (const file of candidateFiles) { + const filenameId = file.split("_")[0]; + if (duplicateIds.has(filenameId)) { + // Duplicate collisions are flagged on applied migration (if any) + // and skipped from valid pending migrations. + continue; + } + + const fullPath = path.join(migrationsDir, file); + validateSafeFilePath(fullPath, migrationsDir); + + let fd: number; + try { + fd = fs.openSync(fullPath, "r"); + } catch (err: any) { + const applied = appliedMap.get(filenameId); + if (applied && !applied.driftStatus) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + expectedChecksum: applied.checksum || undefined, + message: `Failed to open manifest file '${file}': ${err.message}`, + }; + } + continue; + } + + let stat: fs.Stats; + try { + stat = fs.fstatSync(fd); + } catch (err: any) { + fs.closeSync(fd); + const applied = appliedMap.get(filenameId); + if (applied && !applied.driftStatus) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + expectedChecksum: applied.checksum || undefined, + message: `Failed to stat manifest file '${file}': ${err.message}`, + }; + } + continue; + } + + // Size Safety & TOCTOU Prevention: + // If stat.size > MAX_MANIFEST_SIZE_BYTES (1MB), close fd immediately and bypass buffer read. + if (stat.size > MAX_MANIFEST_SIZE_BYTES) { + fs.closeSync(fd); + const applied = appliedMap.get(filenameId); + if (applied && !applied.driftStatus) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + expectedChecksum: applied.checksum || undefined, + message: `Manifest file '${file}' exceeds 1MB limit (${stat.size} bytes)`, + }; + } + continue; + } + + const buf = Buffer.alloc(stat.size); + try { + fs.readSync(fd, buf, 0, stat.size, 0); + } finally { + fs.closeSync(fd); + } + + const rawContent = buf.toString("utf8"); + const checksum = computeManifestChecksum(rawContent); + + let manifest: MongoMigrationManifest; + try { + manifest = JSON.parse(rawContent); + } catch (err: any) { + const applied = appliedMap.get(filenameId); + if (applied && !applied.driftStatus) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + expectedChecksum: applied.checksum || undefined, + message: `Failed to parse manifest JSON in '${file}': ${err.message}`, + }; + } + continue; + } + + const id = manifest.id || filenameId; + validDiskFiles.set(id, { checksum, manifest, file }); + if (id !== filenameId) { + validDiskFiles.set(filenameId, { checksum, manifest, file }); + } + + const applied = appliedMap.get(id) || appliedMap.get(filenameId); + if (!applied) { + pendingMigrations.push({ + id, + name: manifest.name || file, + filename: file, + checksum, + stepCount: manifest.up?.length || 0, + isReversible: manifest.up?.every((s) => s.type !== "dropCollection") ?? false, + }); + } + } + + // Evaluate applied migrations according to strict deterministic precedence: + for (const applied of appliedSummaries) { + // If already marked UNVERIFIABLE via Precedence 2 (duplicate collision, oversized, or parse error), preserve it. + if (applied.driftStatus) { + continue; + } + + const filesForId = idToFilesMap.get(applied._id); + + // Precedence 1: MANIFEST_MISSING + if (!filesForId || filesForId.length === 0) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_MISSING", + expectedChecksum: applied.checksum || undefined, + message: "Manifest file not found in migrations directory", + }; + continue; + } + + const diskInfo = validDiskFiles.get(applied._id); + if (!diskInfo) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + expectedChecksum: applied.checksum || undefined, + message: `Failed to validate manifest file for migration ID '${applied._id}'`, + }; + continue; + } + + // Precedence 3: RECORDED_CHECKSUM_MISSING + if (!applied.checksum) { + applied.driftStatus = "UNVERIFIABLE"; + applied.drifted = false; + applied.driftDetails = { + status: "UNVERIFIABLE", + reason: "RECORDED_CHECKSUM_MISSING", + actualChecksum: diskInfo.checksum, + message: "Recorded migration summary contains no baseline checksum", + }; + } else if (applied.checksum === diskInfo.checksum) { + // Precedence 4: CHECKSUM_MATCH + applied.driftStatus = "VERIFIED"; + applied.drifted = false; + applied.driftDetails = { + status: "VERIFIED", + reason: "CHECKSUM_MATCH", + expectedChecksum: applied.checksum, + actualChecksum: diskInfo.checksum, + }; + } else { + // Precedence 5: CHECKSUM_MISMATCH + applied.driftStatus = "DRIFTED"; + applied.drifted = true; + applied.driftDetails = { + status: "DRIFTED", + reason: "CHECKSUM_MISMATCH", + expectedChecksum: applied.checksum, + actualChecksum: diskInfo.checksum, + message: "On-disk manifest content has been altered since application", + }; + } + } + + pendingMigrations.sort((a, b) => a.id.localeCompare(b.id)); + + return { + appliedMigrations: appliedSummaries, + pendingMigrations, + isLocked: lockStatus.isLocked, + lockDetails: lockStatus.isLocked + ? { + ownerId: lockStatus.ownerId, + fencingToken: lockStatus.fencingToken, + expiresAt: lockStatus.expiresAt, + migrationId: lockStatus.migrationId, + } + : undefined, + }; +} + +/** + * Applies a specific migration or all pending migrations in order. + */ +export async function applyMigration( + db: Db, + projectRoot: string, + version?: string, + options?: { + dryRun?: boolean; + readOnly?: boolean; + ownerId?: string; + }, +): Promise { + if (options?.readOnly) { + throw new MongoApiError("MONGO_READ_ONLY", "Database is in read-only mode"); + } + + const migrationsDir = resolveMongoMigrationsDir(projectRoot); + const results: MongoMigrationApplyResult["results"] = []; + + if (version) { + if (!fs.existsSync(migrationsDir)) { + throw new MongoApiError( + "MIGRATION_NOT_FOUND", + `Migration directory not found: ${migrationsDir}`, + ); + } + + const files = fs.readdirSync(migrationsDir); + const targetFile = files.find( + (f) => + (f.startsWith(version + "_") || f.startsWith(version + ".")) && + MANIFEST_FILENAME_REGEX.test(f), + ); + + if (!targetFile) { + throw new MongoApiError( + "MIGRATION_NOT_FOUND", + `Migration file not found for version '${version}'`, + ); + } + + const fullPath = path.join(migrationsDir, targetFile); + validateSafeFilePath(fullPath, migrationsDir); + + const stats = fs.statSync(fullPath); + if (stats.size > MAX_MANIFEST_SIZE_BYTES) { + throw new MongoApiError( + "MANIFEST_SIZE_EXCEEDED", + `Manifest exceeds 1MB limit (${stats.size} bytes)`, + ); + } + + const rawContent = fs.readFileSync(fullPath, "utf8"); + const checksum = computeManifestChecksum(rawContent); + + let manifest: MongoMigrationManifest; + try { + manifest = JSON.parse(rawContent); + } catch (err: any) { + throw new MongoApiError( + "INVALID_MANIFEST", + `Invalid JSON in manifest '${targetFile}': ${err.message}`, + ); + } + + const singleResult = await applySingleManifest(db, manifest, checksum, options); + results.push(singleResult); + } else { + // Apply all pending migrations in chronological order + const list = await listMigrations(db, projectRoot); + for (const pending of list.pendingMigrations) { + const fullPath = path.join(migrationsDir, pending.filename); + validateSafeFilePath(fullPath, migrationsDir); + + const rawContent = fs.readFileSync(fullPath, "utf8"); + const checksum = computeManifestChecksum(rawContent); + const manifest: MongoMigrationManifest = JSON.parse(rawContent); + + const singleResult = await applySingleManifest(db, manifest, checksum, options); + results.push(singleResult); + } + } + + return { + success: results.every((r) => r.status === "APPLIED" || r.status === "PENDING"), + appliedCount: results.filter((r) => r.status === "APPLIED").length, + results, + }; +} + +/** + * Rolls back an applied or partially failed migration. + */ +export async function rollbackMigration( + db: Db, + migrationId: string, + projectRoot?: string, + options?: { + readOnly?: boolean; + force?: boolean; + ownerId?: string; + }, +): Promise { + if (options?.readOnly) { + throw new MongoApiError("MONGO_READ_ONLY", "Database is in read-only mode"); + } + + const summary = await db + .collection(MIGRATIONS_COLLECTION) + .findOne({ _id: migrationId }); + + if (!summary) { + throw new MongoApiError( + "MIGRATION_NOT_FOUND", + `Migration '${migrationId}' not found in database records`, + ); + } + if (summary.status === "ROLLED_BACK") { + throw new MongoApiError( + "ALREADY_ROLLED_BACK", + `Migration '${migrationId}' has already been rolled back`, + ); + } + + // Load manifest if projectRoot provided + let manifest: MongoMigrationManifest | undefined; + if (projectRoot) { + const migrationsDir = resolveMongoMigrationsDir(projectRoot); + if (fs.existsSync(migrationsDir)) { + const files = fs.readdirSync(migrationsDir); + const file = files.find( + (f) => + (f.startsWith(migrationId + "_") || f.startsWith(migrationId + ".")) && + MANIFEST_FILENAME_REGEX.test(f), + ); + if (file) { + const fullPath = path.join(migrationsDir, file); + validateSafeFilePath(fullPath, migrationsDir); + manifest = JSON.parse(fs.readFileSync(fullPath, "utf8")); + } + } + } + + if (summary.isReversible === false && !options?.force) { + throw new MongoApiError( + "IRREVERSIBLE_MIGRATION", + `Migration '${migrationId}' contains irreversible steps and cannot be rolled back without force flag`, + ); + } + + const ownerId = options?.ownerId || uuidv4(); + const { fencingToken } = await acquireLock(db, ownerId, migrationId); + + let heartbeatInFlight = false; + let lockLost = false; + const heartbeatTimer: NodeJS.Timeout | null = setInterval(async () => { + if (heartbeatInFlight || lockLost) return; + heartbeatInFlight = true; + try { + const renewed = await renewLockHeartbeat(db, ownerId, fencingToken); + if (!renewed) { + lockLost = true; + if (heartbeatTimer) clearInterval(heartbeatTimer); + } + } catch { + // transient network error; verifyLock guards pre-step + } finally { + heartbeatInFlight = false; + } + }, DEFAULT_HEARTBEAT_MS); + if (heartbeatTimer.unref) heartbeatTimer.unref(); + + const attemptId = uuidv4(); + const prevEvents = await db + .collection(EVENTS_COLLECTION) + .find({ migrationId }) + .sort({ attemptNumber: -1 }) + .limit(1) + .toArray(); + const attemptNumber = (prevEvents[0]?.attemptNumber || 0) + 1; + let sequenceNumber = 0; + + const recordEvent = async ( + eventType: MongoMigrationEventType, + stepIndex?: number, + stepType?: string, + metadata?: Record, + ) => { + sequenceNumber++; + const eventDoc: MongoMigrationEvent = { + _id: `${migrationId}_a${attemptNumber}_s${sequenceNumber}`, + eventId: uuidv4(), + migrationId, + attemptId, + attemptNumber, + sequenceNumber, + stepIndex, + stepType, + eventType, + timestamp: new Date().toISOString(), + lockOwnerId: ownerId, + fencingToken, + metadata, + }; + await db.collection(EVENTS_COLLECTION).insertOne(eventDoc as any); + return eventDoc; + }; + + const startTime = Date.now(); + let stepsCompensated = 0; + + try { + await recordEvent("COMPENSATION_START"); + await db + .collection(MIGRATIONS_COLLECTION) + .updateOne({ _id: migrationId }, { $set: { status: "ROLLING_BACK" } }); + + // Retrieve events to find completed steps and pre-states + const events = await db + .collection(EVENTS_COLLECTION) + .find({ migrationId }) + .sort({ attemptNumber: 1, sequenceNumber: 1 } as any) + .toArray(); + + const completedSteps: Array<{ + index: number; + type: string; + preState: any; + step?: MongoMigrationStep; + }> = []; + const completedStepIndices = new Set(); + + for (const ev of events) { + if (ev.eventType === "STEP_COMPLETE" && ev.stepIndex !== undefined) { + completedStepIndices.add(ev.stepIndex); + } + } + + for (const ev of events) { + if ( + ev.eventType === "STEP_START" && + ev.stepIndex !== undefined && + completedStepIndices.has(ev.stepIndex) + ) { + const stepDef = manifest?.up[ev.stepIndex]; + completedSteps.push({ + index: ev.stepIndex, + type: ev.stepType || stepDef?.type || "unknown", + preState: ev.metadata?.preState, + step: stepDef, + }); + } + } + + // Rollback completed steps in reverse order + for (let i = completedSteps.length - 1; i >= 0; i--) { + const stepInfo = completedSteps[i]; + + // Pre-step fencing check + if (lockLost || !(await verifyLock(db, ownerId, fencingToken))) { + await recordEvent("STEP_UNCERTAIN", stepInfo.index, stepInfo.type, { + reason: "Pre-step fencing check failed during rollback; lock lease expired or stolen", + }); + await recordEvent("MANUAL_INTERVENTION_REQUIRED", stepInfo.index, stepInfo.type, { + reason: "Lock lost during rollback", + }); + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: migrationId }, + { + $set: { + status: "CRASHED_UNCERTAIN", + lastExecutedStepIndex: stepInfo.index, + error: `LOCK_LOST_FENCING_VIOLATION: Lock lease lost during rollback of step ${stepInfo.index}`, + }, + }, + ); + throw new MongoApiError( + "LOCK_LOST_FENCING_VIOLATION", + `Migration lock lease lost during rollback of step ${stepInfo.index}`, + ); + } + + if (stepInfo.step) { + await compensateStep(db, stepInfo.step, stepInfo.preState, manifest?.down); + } else { + // Fallback compensation using captured preState + if (stepInfo.type === "createIndex" && stepInfo.preState?.name) { + await db.collection(stepInfo.preState.collection).dropIndex(stepInfo.preState.name); + } else if (stepInfo.type === "setValidator" && stepInfo.preState) { + await db.command({ + collMod: stepInfo.preState.collection, + validator: stepInfo.preState.validator || {}, + }); + } + } + stepsCompensated++; + } + + await recordEvent("COMPENSATION_COMPLETE"); + await recordEvent("MIGRATION_ROLLED_BACK"); + + const duration = Date.now() - startTime; + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: migrationId }, + { + $set: { + status: "ROLLED_BACK", + completedAt: new Date().toISOString(), + durationMs: duration, + }, + }, + ); + + return { + success: true, + version: migrationId, + status: "ROLLED_BACK", + executionTimeMs: duration, + stepsCompensated, + }; + } catch (err: any) { + await recordEvent("COMPENSATION_FAILED", undefined, undefined, { error: err.message }); + await recordEvent("MANUAL_INTERVENTION_REQUIRED", undefined, undefined, { + reason: err.message, + }); + + await db.collection(MIGRATIONS_COLLECTION).updateOne( + { _id: migrationId }, + { + $set: { + status: "FAILED_MANUAL_INTERVENTION_REQUIRED", + error: { + stepIndex: stepsCompensated, + stepType: "compensation", + message: err.message, + }, + }, + }, + ); + + throw err; + } finally { + if (heartbeatTimer) clearInterval(heartbeatTimer); + await releaseLock(db, ownerId, fencingToken); + await recordEvent("LOCK_RELEASED"); + } +} + +/** + * Deterministically rebuilds the summary projection in `_relwave_migrations` + * by replaying the chronological event stream from `_relwave_migration_events`. + */ +export async function rebuildMigrationSummary( + db: Db, + migrationId: string, + manifest?: MongoMigrationManifest, +): Promise { + const events = await db + .collection(EVENTS_COLLECTION) + .find({ migrationId }) + .sort({ attemptNumber: 1, sequenceNumber: 1 } as any) + .toArray(); + + if (events.length === 0) { + throw new MongoApiError("EVENT_STREAM_EMPTY", `No events found for migration '${migrationId}'`); + } + + let status: MongoMigrationStatus = "PENDING"; + let lastExecutedStepIndex = -1; + let executedSteps = 0; + let startedAt = events[0].timestamp; + let completedAt: string | undefined; + let durationMs = 0; + let error: string | undefined; + let fencingToken = events[0].fencingToken || 0; + let currentAttemptNumber = events[0].attemptNumber ?? 1; + let currentAttemptId = events[0].attemptId; + let inFlightStep = false; + + for (const event of events) { + fencingToken = Math.max(fencingToken, event.fencingToken || 0); + + if (event.attemptNumber !== undefined && event.attemptNumber > currentAttemptNumber) { + currentAttemptNumber = event.attemptNumber; + currentAttemptId = event.attemptId; + executedSteps = 0; + lastExecutedStepIndex = -1; + startedAt = event.timestamp; + completedAt = undefined; + durationMs = 0; + error = undefined; + status = "PENDING"; + inFlightStep = false; + } + + switch (event.eventType) { + case "MIGRATION_START": + status = "APPLYING"; + inFlightStep = false; + break; + case "STEP_START": + lastExecutedStepIndex = event.stepIndex ?? lastExecutedStepIndex; + inFlightStep = true; + break; + case "STEP_COMPLETE": + executedSteps++; + inFlightStep = false; + if (event.metadata?.durationMs) { + durationMs += event.metadata.durationMs; + } + break; + case "STEP_FAILED": + status = "FAILED_PARTIAL"; + inFlightStep = false; + error = event.metadata?.error?.message || "Step execution failed"; + break; + case "STEP_UNCERTAIN": + status = "CRASHED_UNCERTAIN"; + inFlightStep = false; + error = event.metadata?.reason || "Step outcome uncertain"; + break; + case "COMPENSATION_START": + status = "ROLLING_BACK"; + break; + case "COMPENSATION_COMPLETE": + status = "ROLLED_BACK"; + break; + case "COMPENSATION_FAILED": + status = "FAILED_MANUAL_INTERVENTION_REQUIRED"; + error = event.metadata?.error?.message || "Compensation step failed"; + break; + case "MIGRATION_APPLIED": + status = "APPLIED"; + completedAt = event.timestamp; + inFlightStep = false; + break; + case "MIGRATION_ROLLED_BACK": + status = "ROLLED_BACK"; + completedAt = event.timestamp; + inFlightStep = false; + break; + case "MANUAL_INTERVENTION_REQUIRED": + status = "FAILED_MANUAL_INTERVENTION_REQUIRED"; + error = event.metadata?.reason || "Manual intervention required"; + break; + } + } + + // Detect dangling STEP_START within the latest attempt + if (inFlightStep) { + status = "CRASHED_UNCERTAIN"; + } + + const lastEvent = events[events.length - 1]; + const summary: MongoMigrationSummary = { + _id: migrationId, + name: manifest?.name || (events[0].metadata?.name as string) || "rebuilt_migration", + checksum: manifest ? computeManifestChecksum(JSON.stringify(manifest)) : "reconstructed", + status, + startedAt, + completedAt, + durationMs, + totalDurationMs: durationMs, + executedSteps, + totalSteps: manifest?.up.length || Math.max(executedSteps, 1), + lastExecutedStepIndex, + fencingToken, + attemptNumber: currentAttemptNumber, + attemptId: currentAttemptId, + error, + isReversible: manifest?.up.every((s) => s.type !== "dropCollection") ?? false, + appliedBy: lastEvent.lockOwnerId || events[0].lockOwnerId, + }; + + await db + .collection(MIGRATIONS_COLLECTION) + .replaceOne({ _id: migrationId }, summary, { upsert: true }); + + return summary; +} + +/** + * Retrieves the migration status, recent events, and distributed lock status. + */ +export async function getMigrationStatus( + db: Db, + migrationId?: string, +): Promise { + const lockStatus = await checkLockStatus(db); + + let migration: MongoMigrationSummary | undefined; + let events: MongoMigrationEvent[] | undefined; + + if (migrationId) { + migration = + (await db + .collection(MIGRATIONS_COLLECTION) + .findOne({ _id: migrationId })) || undefined; + + events = await db + .collection(EVENTS_COLLECTION) + .find({ migrationId }) + .sort({ attemptNumber: 1, sequenceNumber: 1 } as any) + .toArray(); + } + + const lastAppliedDocs = await db + .collection(MIGRATIONS_COLLECTION) + .find({ status: "APPLIED" }) + .sort({ _id: -1 }) + .limit(1) + .toArray(); + + const lastApplied = lastAppliedDocs[0]; + + const pendingDocs = await db + .collection(MIGRATIONS_COLLECTION) + .countDocuments({ status: { $ne: "APPLIED" } }); + + return { + migration, + events, + lockStatus, + lastAppliedVersion: lastApplied?._id, + lastAppliedAt: lastApplied?.completedAt || lastApplied?.startedAt, + pendingCount: pendingDocs, + }; +} + +export const mongoMigrationEngine = { + validateManifest, + assertValidManifest, + computeManifestChecksum, + validateSafeFilePath, + resolveMongoMigrationsDir, + listMigrations, + applyMigration, + rollbackMigration, + rebuildMigrationSummary, + getMigrationStatus, + compensateStep, +}; diff --git a/bridge/src/services/mongoMigrationLock.ts b/bridge/src/services/mongoMigrationLock.ts new file mode 100644 index 00000000..ce54402f --- /dev/null +++ b/bridge/src/services/mongoMigrationLock.ts @@ -0,0 +1,217 @@ +/** + * MongoDB Milestone 6 — Distributed Migration Lock Manager. + * + * Implements atomic distributed locking on `_relwave_migrations_lock` + * with fencing tokens, bounded E11000 retry backoff, active heartbeats, + * and owner-conditional releases. + */ + +import { Db } from "mongodb"; +import { MongoApiError } from "../connectors/mongo"; +import { MongoMigrationLockDoc, MongoMigrationLockStatus } from "../types/mongoMigration"; + +export const LOCK_COLLECTION = "_relwave_migrations_lock"; +export const LOCK_DOC_ID = "migration_lock"; +export const DEFAULT_LEASE_MS = 60_000; // 60s +export const DEFAULT_HEARTBEAT_MS = 15_000; // 15s +export const DEFAULT_MAX_RETRIES = 3; + +/** + * Acquire the distributed migration lock atomically using findOneAndUpdate. + * + * Handles concurrent first-time upsert races (MongoDB error 11000) with + * bounded exponential backoff and jitter. If an active owner holds the lease, + * throws `MongoApiError("LOCK_ALREADY_HELD", ...)`. + */ +export async function acquireLock( + db: Db, + ownerId: string, + migrationId?: string, + leaseMs: number = DEFAULT_LEASE_MS, + maxRetries: number = DEFAULT_MAX_RETRIES, +): Promise<{ success: boolean; fencingToken: number; expiresAt: Date }> { + const collection = db.collection(LOCK_COLLECTION); + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + const now = new Date(); + const expiresAt = new Date(now.getTime() + leaseMs); + + try { + const result = await collection.findOneAndUpdate( + { + _id: LOCK_DOC_ID, + $or: [{ ownerId }, { ownerId: null }, { expiresAt: { $lt: now } }], + }, + { + $set: { + ownerId, + migrationId: migrationId || null, + acquiredAt: now, + heartbeatAt: now, + expiresAt, + }, + $inc: { fencingToken: 1 }, + }, + { upsert: true, returnDocument: "after" }, + ); + + const doc = result && "value" in (result as any) ? (result as any).value : result; + if (doc) { + return { + success: true, + fencingToken: doc.fencingToken, + expiresAt: new Date(doc.expiresAt), + }; + } + + // If findOneAndUpdate returned null, lock is held by another active owner + const activeLock = await collection.findOne({ _id: LOCK_DOC_ID }); + if ( + activeLock && + activeLock.expiresAt && + new Date(activeLock.expiresAt).getTime() >= now.getTime() + ) { + if (activeLock.ownerId !== ownerId) { + if (attempt < maxRetries) { + const jitter = Math.floor(Math.random() * 150) + 50 * attempt; + await new Promise((resolve) => setTimeout(resolve, jitter)); + continue; + } + throw new MongoApiError( + "LOCK_ALREADY_HELD", + `Migration lock held by ${activeLock.ownerId} (expires at ${new Date(activeLock.expiresAt).toISOString()})`, + ); + } + } + } catch (err: any) { + if (err instanceof MongoApiError) { + throw err; + } + + // MongoDB duplicate key error (code 11000) on concurrent first-time upserts + if (err.code === 11000 || err.codeName === "DuplicateKey") { + if (attempt < maxRetries) { + const jitter = Math.floor(Math.random() * 150) + 50 * attempt; + await new Promise((resolve) => setTimeout(resolve, jitter)); + continue; + } + + const activeLock = await collection.findOne({ _id: LOCK_DOC_ID }); + if ( + activeLock && + activeLock.expiresAt && + new Date(activeLock.expiresAt).getTime() >= Date.now() + ) { + throw new MongoApiError( + "LOCK_ALREADY_HELD", + `Migration lock held by ${activeLock.ownerId} (expires at ${new Date(activeLock.expiresAt).toISOString()})`, + ); + } + } + + throw err; + } + } + + // If retries exhausted and not acquired + const activeLock = await collection.findOne({ _id: LOCK_DOC_ID }); + if ( + activeLock && + activeLock.expiresAt && + new Date(activeLock.expiresAt).getTime() >= Date.now() + ) { + throw new MongoApiError( + "LOCK_ALREADY_HELD", + `Migration lock held by ${activeLock.ownerId} (expires at ${new Date(activeLock.expiresAt).toISOString()})`, + ); + } + + throw new MongoApiError("LOCK_ALREADY_HELD", "Failed to acquire migration lock after retries"); +} + +/** + * Renew lock heartbeat atomically if ownerId and fencingToken match and lease has not expired. + */ +export async function renewLockHeartbeat( + db: Db, + ownerId: string, + fencingToken: number, + leaseMs: number = DEFAULT_LEASE_MS, +): Promise { + const collection = db.collection(LOCK_COLLECTION); + const now = new Date(); + const expiresAt = new Date(now.getTime() + leaseMs); + + const result = await collection.updateOne( + { _id: LOCK_DOC_ID, ownerId, fencingToken, expiresAt: { $gt: now } }, + { $set: { heartbeatAt: now, expiresAt } }, + ); + + return result.matchedCount > 0; +} + +/** + * Release lock atomically if ownerId and fencingToken match. + * Preserves fencingToken across releases. + */ +export async function releaseLock(db: Db, ownerId: string, fencingToken: number): Promise { + const collection = db.collection(LOCK_COLLECTION); + const result = await collection.updateOne( + { _id: LOCK_DOC_ID, ownerId, fencingToken }, + { + $set: { + ownerId: null, + migrationId: null, + expiresAt: null, + releasedAt: new Date(), + }, + }, + ); + return result.matchedCount > 0; +} + +/** + * Non-mutating inspection of the distributed lock status. + */ +export async function checkLockStatus(db: Db): Promise { + const collection = db.collection(LOCK_COLLECTION); + const doc = await collection.findOne({ _id: LOCK_DOC_ID }); + + if (!doc || !doc.ownerId) { + return { + isLocked: false, + isExpired: false, + }; + } + + const now = new Date(); + const expiresAtDate = doc.expiresAt ? new Date(doc.expiresAt) : new Date(0); + const isExpired = expiresAtDate.getTime() < now.getTime(); + + return { + isLocked: !isExpired, + ownerId: doc.ownerId, + fencingToken: doc.fencingToken, + expiresAt: doc.expiresAt ? new Date(doc.expiresAt).toISOString() : undefined, + isExpired, + migrationId: doc.migrationId || undefined, + }; +} + +/** + * Pre-step fencing guard: verifies the current client still holds the lease. + * Returns true if the doc exists matching ownerId and fencingToken and expiresAt > now. + */ +export async function verifyLock(db: Db, ownerId: string, fencingToken: number): Promise { + const collection = db.collection(LOCK_COLLECTION); + const now = new Date(); + + const doc = await collection.findOne({ + _id: LOCK_DOC_ID, + ownerId, + fencingToken, + expiresAt: { $gt: now }, + }); + + return doc !== null; +} diff --git a/bridge/src/services/mongoRelationshipEngine.ts b/bridge/src/services/mongoRelationshipEngine.ts new file mode 100644 index 00000000..71e305fd --- /dev/null +++ b/bridge/src/services/mongoRelationshipEngine.ts @@ -0,0 +1,852 @@ +import { Db, ObjectId, Document } from "mongodb"; +import { DatabaseService } from "./databaseService"; +import { projectStoreInstance, ProjectStore } from "./projectStore"; +import { selectDb } from "../connectors/mongo"; +import { DBType } from "../types"; +import { + MongoCardinality, + MongoRelationshipEvidence, + MongoInferredRelationship, + MongoRelationship, + MongoRelationshipsFile, + MongoInferRelationshipsRequest, + MongoInferRelationshipsResult, + MongoListRelationshipsResult, + MongoSetRelationshipStatusResult, + MongoSaveRelationshipResult, + MongoDeleteRelationshipResult, +} from "../types/mongo"; + +interface CandidateField { + sourcePath: string; + isArray: boolean; + sampleValues: unknown[]; + targetNameCandidates: string[]; +} + +export class MongoRelationshipEngine { + constructor( + private dbService: DatabaseService = new DatabaseService(), + private projectStore: ProjectStore = projectStoreInstance, + ) {} + + /** + * Acquire native MongoDB Db object via connection config + */ + async getDb(dbId: string): Promise { + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType !== DBType.MONGODB) { + throw new Error(`Database ${dbId} is not a MongoDB database (type: ${dbType})`); + } + return selectDb(conn); + } + + /** + * Derive candidate collection names from a field name (e.g. userId -> [users, user]) + */ + deriveCandidateCollectionNames(fieldPath: string): string[] { + const leaf = fieldPath.split(".").pop() || fieldPath; + const stripped = leaf.replace(/(_ids|_id|Ids|Id)$/i, ""); + if (!stripped) return []; + + const lower = stripped.toLowerCase(); + const candidates = new Set(); + candidates.add(lower); + + // Common pluralization / singularization rules + if (lower.endsWith("y")) { + candidates.add(`${lower.slice(0, -1)}ies`); + } else if ( + lower.endsWith("s") || + lower.endsWith("sh") || + lower.endsWith("ch") || + lower.endsWith("x") || + lower.endsWith("z") + ) { + candidates.add(`${lower}es`); + if (lower.endsWith("s") && lower.length > 2) { + candidates.add(lower.slice(0, -1)); + } + } else { + candidates.add(`${lower}s`); + } + + return Array.from(candidates); + } + + /** + * Test if a value looks like an ObjectId or 24-char hex string + */ + isObjectIdOrHex(val: unknown): boolean { + if (!val) return false; + if ( + val instanceof ObjectId || + (typeof val === "object" && (val as any)._bsontype === "ObjectId") + ) { + return true; + } + if (typeof val === "object" && typeof (val as any).$oid === "string") { + return true; + } + if (typeof val === "string" && /^[0-9a-fA-F]{24}$/.test(val)) { + return true; + } + return false; + } + + /** + * Normalize an ObjectId or hex string to a string ID + */ + normalizeIdValue(val: unknown): string | null { + if (val === null || val === undefined) return null; + if (val instanceof ObjectId) return val.toHexString(); + if (typeof val === "object" && (val as any)._bsontype === "ObjectId") { + return (val as any).toHexString?.() || String(val); + } + if (typeof val === "object" && typeof (val as any).$oid === "string") { + return (val as any).$oid; + } + if (typeof val === "string") return val; + return String(val); + } + + /** + * Convert normalized string ID to BSON query value (ObjectId if hex) + */ + toQueryId(val: string): unknown { + if (/^[0-9a-fA-F]{24}$/.test(val)) { + try { + return new ObjectId(val); + } catch { + return val; + } + } + return val; + } + + /** + * Extract candidate reference fields from sampled documents (LOW-02: Embedded Subdocument Arrays) + */ + extractCandidateFields(docs: Document[]): CandidateField[] { + const fieldMap = new Map(); + + const traverse = (obj: any, prefix = "", depth = 1, inArray = false) => { + if (!obj || typeof obj !== "object" || depth > 3) return; + + for (const [key, value] of Object.entries(obj)) { + if (key === "_bsontype") continue; + const currentPath = prefix ? `${prefix}.${key}` : key; + + // Skip root document's own _id and embedded subdocument _id + if (key === "_id" || currentPath === "_id" || currentPath.endsWith("._id")) { + continue; + } + + const isArray = Array.isArray(value); + const hasFkNamePattern = /(_ids|_id|Ids|Id)$/i.test(key) || key.toLowerCase() === "id"; + + if (isArray) { + const arr = value as unknown[]; + const sampleSlice = arr.slice(0, 5); + + // Check if elements are subdocuments + const isArrayOfObjects = sampleSlice.some( + (el) => + el && + typeof el === "object" && + !(el instanceof ObjectId) && + !(el as any)._bsontype && + !(el as any).$oid && + !(el instanceof Date), + ); + + if (isArrayOfObjects) { + for (const item of sampleSlice) { + if ( + item && + typeof item === "object" && + !(item instanceof ObjectId) && + !(item as any)._bsontype && + !(item as any).$oid && + !(item instanceof Date) + ) { + traverse(item, currentPath, depth + 1, true); + } + } + } else { + const hasRefElements = sampleSlice.some( + (el) => this.isObjectIdOrHex(el) || (hasFkNamePattern && typeof el === "string"), + ); + if (hasRefElements || hasFkNamePattern) { + if (!fieldMap.has(currentPath)) { + fieldMap.set(currentPath, { isArray: true, values: [] }); + } + const entry = fieldMap.get(currentPath)!; + entry.isArray = true; + for (const item of arr) { + if (item !== null && item !== undefined) entry.values.push(item); + } + } + } + } else if ( + value && + typeof value === "object" && + !(value instanceof ObjectId) && + !(value as any)._bsontype && + !(value as any).$oid && + !(value instanceof Date) + ) { + traverse(value, currentPath, depth + 1, inArray); + } else { + const isRefVal = this.isObjectIdOrHex(value); + if (isRefVal || (hasFkNamePattern && typeof value === "string")) { + if (!fieldMap.has(currentPath)) { + fieldMap.set(currentPath, { isArray: inArray, values: [] }); + } + const entry = fieldMap.get(currentPath)!; + if (inArray) entry.isArray = true; + if (value !== null && value !== undefined) entry.values.push(value); + } + } + } + }; + + for (const doc of docs) { + traverse(doc, "", 1, false); + } + + const candidates: CandidateField[] = []; + for (const [path, info] of fieldMap.entries()) { + candidates.push({ + sourcePath: path, + isArray: info.isArray, + sampleValues: info.values, + targetNameCandidates: this.deriveCandidateCollectionNames(path), + }); + } + + return candidates; + } + + /** + * Helper to safely extract nested dot-path value from an object, + * supporting traversal through arrays of objects (LOW-02). + */ + extractFieldValue(obj: any, path: string): unknown { + if (!obj || typeof obj !== "object") return undefined; + const parts = path.split("."); + + const resolve = (current: any, index: number): unknown => { + if (current === null || current === undefined) return undefined; + if (index >= parts.length) return current; + + const part = parts[index]; + if (Array.isArray(current)) { + const remainingPath = parts.slice(index).join("."); + const mapped = current + .map((item) => this.extractFieldValue(item, remainingPath)) + .filter((v) => v !== undefined); + return mapped.flat(); + } + + const nextVal = current[part]; + if (Array.isArray(nextVal) && index + 1 < parts.length) { + const remainingPath = parts.slice(index + 1).join("."); + const mapped = nextVal + .map((item) => this.extractFieldValue(item, remainingPath)) + .filter((v) => v !== undefined); + return mapped.flat(); + } + + return resolve(nextVal, index + 1); + }; + + return resolve(obj, 0); + } + + /** + * Infer relationships across collections in a MongoDB database + */ + async inferRelationships( + dbId: string, + options?: Omit, + ): Promise { + const db = await this.getDb(dbId); + const sampleSize = Math.max(10, Math.min(options?.sampleSize ?? 100, 500)); + const maxCollections = Math.max(1, Math.min(options?.maxCollections ?? 50, 200)); + + // Overall deadline (HIGH-01 / LOW-01) + const timeoutMs = Math.max(1000, Math.min(options?.timeoutMs ?? 30000, 120000)); + const deadline = Date.now() + timeoutMs; + let timedOut = false; + + // 1. Retrieve and filter collection names + const allCollections = await db.listCollections({}, { nameOnly: true }).toArray(); + const allCollectionNames = allCollections + .map((c) => c.name) + .filter((name) => !name.startsWith("system.")); + + let collectionNames: string[]; + if (options?.collections) { + collectionNames = options.collections.filter((c) => allCollectionNames.includes(c)); + if (collectionNames.length === 0) { + return { + relationships: [], + insufficientEvidence: [], + inferredAt: new Date().toISOString(), + collectionsScanned: 0, + }; + } + } else { + collectionNames = allCollectionNames.slice(0, maxCollections); + } + + // 2. Sample each collection and collect unique indexes + const collectionDocs = new Map(); + const collectionUniqueIndexes = new Map>(); + + for (const colName of collectionNames) { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + timedOut = true; + break; + } + try { + const queryTimeMs = Math.max(1, Math.min(5000, remainingMs)); + const docs = await db + .collection(colName) + .find({}, { limit: sampleSize, maxTimeMS: queryTimeMs }) + .toArray(); + collectionDocs.set(colName, docs); + + // Scan indexes for single-field unconditional uniqueness (MED-01) + const indexes = await db + .collection(colName) + .indexes() + .catch(() => []); + const uniqueFields = new Set(); + for (const idx of indexes) { + if (idx.unique && !idx.sparse && !idx.partialFilterExpression && idx.key) { + const keys = Object.keys(idx.key); + if (keys.length === 1) { + uniqueFields.add(keys[0]); + } + } + } + collectionUniqueIndexes.set(colName, uniqueFields); + } catch { + collectionDocs.set(colName, []); + collectionUniqueIndexes.set(colName, new Set()); + } + } + + // 3. Extract candidate fields per collection + const candidateMap = new Map(); + for (const colName of collectionNames) { + const docs = collectionDocs.get(colName) || []; + candidateMap.set(colName, this.extractCandidateFields(docs)); + } + + interface EvaluatedHit { + sourceCol: string; + candidate: CandidateField; + targetCol: string; + queryValues: unknown[]; + hitCount: number; + hitRatio: number; + matchedIds: Set; + sampleMatches: Array<{ sourceValue: string; targetId: string }>; + } + + const rawHits: EvaluatedHit[] = []; + + // 4. Cross-collection reference verification + for (const sourceCol of collectionNames) { + if (Date.now() >= deadline) { + timedOut = true; + break; + } + const candidates = candidateMap.get(sourceCol) || []; + const sourceDocs = collectionDocs.get(sourceCol) || []; + + for (const cand of candidates) { + if (Date.now() >= deadline) { + timedOut = true; + break; + } + + // Collect distinct source values (up to 50) + const distinctNormalized = Array.from( + new Set( + cand.sampleValues + .map((v) => this.normalizeIdValue(v)) + .filter((v): v is string => v !== null && v !== ""), + ), + ).slice(0, 50); + + if (distinctNormalized.length === 0) continue; + + // Bounded target discovery (HIGH-01 / LOW-REM-02) + const MAX_TARGETS_PER_CANDIDATE = 3; + const matchingTargets: string[] = []; + for (const targetCand of cand.targetNameCandidates) { + const matched = collectionNames.find((c) => c.toLowerCase() === targetCand.toLowerCase()); + if (matched && !matchingTargets.includes(matched)) { + matchingTargets.push(matched); + } + } + + // Test only target collections matching derived candidate names (up to cap) + // If no candidate name matches, do not probe arbitrary collections (LOW-REM-02) + const targetCollectionsToTest = matchingTargets.slice(0, MAX_TARGETS_PER_CANDIDATE); + + // Test each candidate target collection + for (const targetCol of targetCollectionsToTest) { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + timedOut = true; + break; + } + try { + // Correctness: 24-Hex String & BSON ObjectId Normalization + const queryValues: unknown[] = []; + const seenOids = new Set(); + const seenRaw = new Set(); + + for (const id of distinctNormalized) { + if (/^[0-9a-fA-F]{24}$/.test(id)) { + try { + if (!seenOids.has(id)) { + queryValues.push(new ObjectId(id)); + seenOids.add(id); + } + } catch { + // ignore + } + if (!seenRaw.has(id)) { + queryValues.push(id); + seenRaw.add(id); + } + } else { + if (!seenRaw.has(id)) { + queryValues.push(id); + seenRaw.add(id); + } + } + } + + const queryFilter: any = { + _id: { $in: queryValues }, + }; + + const queryTimeMs = Math.max(1, Math.min(5000, remainingMs)); + const hits = await db + .collection(targetCol) + .find(queryFilter, { projection: { _id: 1 }, maxTimeMS: queryTimeMs }) + .limit(50) + .toArray(); + + const matchedIds = new Set( + hits + .map((h) => this.normalizeIdValue(h._id)) + .filter((id): id is string => id !== null), + ); + const hitCount = matchedIds.size; + const hitRatio = hitCount / distinctNormalized.length; + + // Reject if hitRatio < 0.50 + if (hitRatio < 0.5) continue; + + // Construct sample match pairs + const sampleMatches: Array<{ sourceValue: string; targetId: string }> = []; + for (const doc of sourceDocs) { + const val = this.extractFieldValue(doc, cand.sourcePath); + if (Array.isArray(val)) { + for (const item of val) { + const norm = this.normalizeIdValue(item); + if (norm && matchedIds.has(norm) && sampleMatches.length < 5) { + sampleMatches.push({ sourceValue: norm, targetId: norm }); + } + } + } else { + const norm = this.normalizeIdValue(val); + if (norm && matchedIds.has(norm) && sampleMatches.length < 5) { + sampleMatches.push({ sourceValue: norm, targetId: norm }); + } + } + if (sampleMatches.length >= 5) break; + } + + rawHits.push({ + sourceCol, + candidate: cand, + targetCol, + queryValues, + hitCount, + hitRatio, + matchedIds, + sampleMatches, + }); + } catch { + // verification query error -> skip candidate + } + } + } + } + + // 5. Compute Confidence and Cardinality with Ambiguity Penalty + const inferredCandidates: MongoInferredRelationship[] = []; + const insufficientCandidates: MongoInferredRelationship[] = []; + + for (const hit of rawHits) { + const { + sourceCol, + candidate: cand, + targetCol, + hitRatio, + hitCount, + sampleMatches, + matchedIds, + } = hit; + const sourceDocs = collectionDocs.get(sourceCol) || []; + + // Check if source candidate values also matched in other target collections + const matchingTargetCount = rawHits.filter( + (h) => h.sourceCol === sourceCol && h.candidate.sourcePath === cand.sourcePath, + ).length; + const ambiguityPenalty = matchingTargetCount > 1 ? 0.3 : 0.0; + + // NameScore calculation + let nameScore = 0.0; + const targetLower = targetCol.toLowerCase(); + const hasDirectNameMatch = cand.targetNameCandidates.some( + (c) => c.toLowerCase() === targetLower, + ); + if (hasDirectNameMatch) { + nameScore = 1.0; + } else if ( + /(_ids|_id|Ids|Id)$/i.test(cand.sourcePath) || + cand.sourcePath.toLowerCase() === "id" + ) { + nameScore = 0.5; + } + + // TypeScore calculation + let typeScore = 0.3; // generic string default + const sampleVal = cand.sampleValues.find((v) => v !== null && v !== undefined); + if ( + sampleVal instanceof ObjectId || + (sampleVal && (sampleVal as any)._bsontype === "ObjectId") + ) { + typeScore = 1.0; + } else if (typeof sampleVal === "string" && /^[0-9a-fA-F]{24}$/.test(sampleVal)) { + typeScore = 0.7; + } + + // Final confidence formula: + // C = 0.55 * HitRatio + 0.25 * NameScore + 0.20 * TypeScore - AmbiguityPenalty + let confidence = 0.55 * hitRatio + 0.25 * nameScore + 0.2 * typeScore - ambiguityPenalty; + confidence = Math.max(0.0, Math.min(1.0, Math.round(confidence * 100) / 100)); + + // 6. Rigorous Cardinality Inference + let cardinality: MongoCardinality = "unknown"; + + // Check single-field unconditional unique index on source field (MED-01) + const uniqueIndexes = collectionUniqueIndexes.get(sourceCol) || new Set(); + const hasUniqueIndexOnSource = uniqueIndexes.has(cand.sourcePath); + + // Check if multiple distinct source documents reference the SAME target ID + const targetRefDocCounts = new Map(); + for (const doc of sourceDocs) { + const val = this.extractFieldValue(doc, cand.sourcePath); + const referencedInDoc = new Set(); + if (Array.isArray(val)) { + for (const item of val) { + const norm = this.normalizeIdValue(item); + if (norm && matchedIds.has(norm)) referencedInDoc.add(norm); + } + } else { + const norm = this.normalizeIdValue(val); + if (norm && matchedIds.has(norm)) referencedInDoc.add(norm); + } + for (const tId of referencedInDoc) { + targetRefDocCounts.set(tId, (targetRefDocCounts.get(tId) || 0) + 1); + } + } + + const multipleSourcesPerTarget = Array.from(targetRefDocCounts.values()).some( + (count) => count > 1, + ); + + if (cand.isArray) { + // DO NOT infer many-to-many from arrays alone! + if (multipleSourcesPerTarget) { + cardinality = "many-to-many"; + } else if (hitCount > 0) { + cardinality = "one-to-many"; + } else { + cardinality = "insufficient-evidence"; + } + } else { + // Scalar field + if (hasUniqueIndexOnSource) { + cardinality = "one-to-one"; + } else if (multipleSourcesPerTarget) { + cardinality = "many-to-one"; + } else if (hitCount > 0) { + cardinality = nameScore > 0 ? "many-to-one" : "unknown"; + } else { + cardinality = "insufficient-evidence"; + } + } + + const relId = `${sourceCol}.${cand.sourcePath}->${targetCol}._id`; + const evidence: MongoRelationshipEvidence = { + sourceCollection: sourceCol, + sourceField: cand.sourcePath, + targetCollection: targetCol, + targetField: "_id", + sampleCount: cand.sampleValues.length, + hitCount, + hitRatio: Math.round(hitRatio * 100) / 100, + hasNamePatternMatch: nameScore > 0, + isBsonTypeMatch: typeScore >= 0.7, + multipleSourcesPerTarget, + isArrayReference: cand.isArray, + hasUniqueIndexOnSource, + sampleMatches, + }; + + // Insufficient evidence cutoff (< 0.65) (MED-02) + if (confidence < 0.65) { + cardinality = "insufficient-evidence"; + insufficientCandidates.push({ + id: relId, + sourceCollection: sourceCol, + sourceField: cand.sourcePath, + targetCollection: targetCol, + targetField: "_id", + cardinality, + confidence, + evidence, + status: "inferred", + inferredAt: new Date().toISOString(), + }); + } else { + inferredCandidates.push({ + id: relId, + sourceCollection: sourceCol, + sourceField: cand.sourcePath, + targetCollection: targetCol, + targetField: "_id", + cardinality, + confidence, + evidence, + status: "inferred", + inferredAt: new Date().toISOString(), + }); + } + } + + // 7. State Retention Across Refresh + const existingFile = await this.projectStore.getMongoRelationships(options?.projectId, dbId); + const rejectedSet = new Set(existingFile.rejectedIds || []); + const confirmedMap = new Map((existingFile.confirmed || []).map((r) => [r.id, r])); + + // Filter out rejected relationships and already confirmed relationships + const prunedInferred = inferredCandidates.filter((cand) => { + if (rejectedSet.has(cand.id)) return false; + if (confirmedMap.has(cand.id)) return false; + return true; + }); + + const prunedInsufficient = insufficientCandidates.filter((cand) => { + if (rejectedSet.has(cand.id)) return false; + if (confirmedMap.has(cand.id)) return false; + return true; + }); + + const now = new Date().toISOString(); + const updatedFile: MongoRelationshipsFile = { + version: 1, + projectId: options?.projectId || existingFile.projectId, + databaseId: dbId, + inferred: prunedInferred, + confirmed: existingFile.confirmed || [], + rejectedIds: existingFile.rejectedIds || [], + manual: existingFile.manual || [], + insufficientEvidence: prunedInsufficient, + updatedAt: now, + }; + + // Fix 1 (MED-REM-01): Never persist partial inference on timeout + if (!timedOut) { + await this.projectStore.saveMongoRelationships(updatedFile, options?.projectId, dbId); + } + + return { + relationships: prunedInferred, + insufficientEvidence: prunedInsufficient, + inferredAt: now, + collectionsScanned: collectionNames.length, + ...(timedOut ? { timedOut: true } : {}), + }; + } + + /** + * List all stored relationships + */ + async listRelationships(dbId: string, projectId?: string): Promise { + const file = await this.projectStore.getMongoRelationships(projectId, dbId); + return { + inferred: file.inferred || [], + confirmed: file.confirmed || [], + rejectedIds: file.rejectedIds || [], + manual: file.manual || [], + insufficientEvidence: file.insufficientEvidence || [], + }; + } + + /** + * Update status of an inferred relationship (confirmed | rejected) + */ + async setRelationshipStatus( + dbId: string, + relationshipId: string, + status: "confirmed" | "rejected", + projectId?: string, + ): Promise { + const file = await this.projectStore.getMongoRelationships(projectId, dbId); + const inferred = file.inferred || []; + const confirmed = file.confirmed || []; + const rejectedIds = new Set(file.rejectedIds || []); + const insufficient = file.insufficientEvidence || []; + const now = new Date().toISOString(); + + const idx = inferred.findIndex((r) => r.id === relationshipId); + const insIdx = insufficient.findIndex((r) => r.id === relationshipId); + const candidate = idx >= 0 ? inferred[idx] : insIdx >= 0 ? insufficient[insIdx] : null; + + if (status === "confirmed") { + rejectedIds.delete(relationshipId); + if (candidate) { + if (idx >= 0) inferred.splice(idx, 1); + if (insIdx >= 0) insufficient.splice(insIdx, 1); + const existingConfIdx = confirmed.findIndex((r) => r.id === relationshipId); + if (existingConfIdx >= 0) { + confirmed[existingConfIdx].updatedAt = now; + } else { + confirmed.push({ + id: candidate.id, + sourceCollection: candidate.sourceCollection, + sourceField: candidate.sourceField, + targetCollection: candidate.targetCollection, + targetField: candidate.targetField, + cardinality: candidate.cardinality, + isManual: false, + createdAt: now, + updatedAt: now, + }); + } + } + } else if (status === "rejected") { + rejectedIds.add(relationshipId); + if (candidate) { + if (idx >= 0) inferred.splice(idx, 1); + if (insIdx >= 0) insufficient.splice(insIdx, 1); + } + const confIdx = confirmed.findIndex((r) => r.id === relationshipId); + if (confIdx >= 0) { + confirmed.splice(confIdx, 1); + } + } + + const updatedFile: MongoRelationshipsFile = { + ...file, + inferred, + confirmed, + rejectedIds: Array.from(rejectedIds), + insufficientEvidence: insufficient, + updatedAt: now, + }; + + await this.projectStore.saveMongoRelationships(updatedFile, projectId, dbId); + + return { + relationshipId, + status, + }; + } + + /** + * Save a user-defined manual relationship + */ + async saveRelationship( + dbId: string, + relationship: MongoRelationship, + projectId?: string, + ): Promise { + const file = await this.projectStore.getMongoRelationships(projectId, dbId); + const manual = file.manual || []; + const now = new Date().toISOString(); + + const normalized: MongoRelationship = { + ...relationship, + isManual: true, + createdAt: relationship.createdAt || now, + updatedAt: now, + }; + + const existingIdx = manual.findIndex((r) => r.id === relationship.id); + if (existingIdx >= 0) { + manual[existingIdx] = normalized; + } else { + manual.push(normalized); + } + + const updatedFile: MongoRelationshipsFile = { + ...file, + manual, + updatedAt: now, + }; + + await this.projectStore.saveMongoRelationships(updatedFile, projectId, dbId); + + return { + relationship: normalized, + }; + } + + /** + * Delete a relationship by ID + */ + async deleteRelationship( + dbId: string, + relationshipId: string, + projectId?: string, + ): Promise { + const file = await this.projectStore.getMongoRelationships(projectId, dbId); + const manual = (file.manual || []).filter((r) => r.id !== relationshipId); + const confirmed = (file.confirmed || []).filter((r) => r.id !== relationshipId); + const inferred = (file.inferred || []).filter((r) => r.id !== relationshipId); + const insufficient = (file.insufficientEvidence || []).filter((r) => r.id !== relationshipId); + const rejectedIds = (file.rejectedIds || []).filter((id) => id !== relationshipId); + + const updatedFile: MongoRelationshipsFile = { + ...file, + manual, + confirmed, + inferred, + insufficientEvidence: insufficient, + rejectedIds, + updatedAt: new Date().toISOString(), + }; + + await this.projectStore.saveMongoRelationships(updatedFile, projectId, dbId); + + return { + success: true, + relationshipId, + }; + } +} diff --git a/bridge/src/services/projectStore.ts b/bridge/src/services/projectStore.ts index 6efb71db..9391f96a 100644 --- a/bridge/src/services/projectStore.ts +++ b/bridge/src/services/projectStore.ts @@ -6,8 +6,15 @@ import path from "path"; import fs from "fs/promises"; import fsSync from "fs"; import { v4 as uuidv4 } from "uuid"; -import { PROJECTS_FOLDER, PROJECTS_INDEX_FILE, getProjectDir, ensureDir } from "../utils/config"; +import { + PROJECTS_FOLDER, + PROJECTS_INDEX_FILE, + getProjectDir, + ensureDir, + CONFIG_FOLDER, +} from "../utils/config"; import { dbStoreInstance, DBMeta } from "./dbStore"; +import type { MongoRelationshipsFile } from "../types/mongo"; // ========================================== // Types @@ -201,6 +208,7 @@ const PROJECT_FILES = { localConfig: "relwave.local.json", readme: "README.md", schema: path.join("schema", "schema.json"), + relationships: path.join("schema", "relationships.json"), erDiagram: path.join("diagrams", "er.json"), annotations: path.join("diagrams", "annotations.json"), queries: path.join("queries", "queries.json"), @@ -917,6 +925,105 @@ export class ProjectStore { return file; } + // ========================================== + // MongoDB Relationships (M7 Phase 1) + // ========================================== + + async getMongoRelationships(projectId?: string, dbId?: string): Promise { + let resolvedProjectId = projectId; + if (!resolvedProjectId && dbId) { + const project = await this.getProjectByDatabaseId(dbId); + if (project?.id) { + resolvedProjectId = project.id; + } + } + + let filePath: string; + if (resolvedProjectId) { + filePath = this.projectFile(resolvedProjectId, PROJECT_FILES.relationships); + } else if (dbId) { + filePath = path.join(CONFIG_FOLDER, "relationships", `${dbId}.json`); + } else { + return { + version: 1, + projectId: undefined, + databaseId: "", + inferred: [], + confirmed: [], + rejectedIds: [], + manual: [], + insufficientEvidence: [], + updatedAt: new Date().toISOString(), + }; + } + + const data = await this.readJSON(filePath); + if (!data) { + return { + version: 1, + projectId: resolvedProjectId || undefined, + databaseId: dbId || "", + inferred: [], + confirmed: [], + rejectedIds: [], + manual: [], + insufficientEvidence: [], + updatedAt: new Date().toISOString(), + }; + } + + return { + version: data.version ?? 1, + projectId: data.projectId ?? (resolvedProjectId || undefined), + databaseId: data.databaseId ?? (dbId || ""), + inferred: Array.isArray(data.inferred) ? data.inferred : [], + confirmed: Array.isArray(data.confirmed) ? data.confirmed : [], + rejectedIds: Array.isArray(data.rejectedIds) ? data.rejectedIds : [], + manual: Array.isArray(data.manual) ? data.manual : [], + insufficientEvidence: Array.isArray(data.insufficientEvidence) + ? data.insufficientEvidence + : [], + updatedAt: data.updatedAt ?? new Date().toISOString(), + }; + } + + async saveMongoRelationships( + file: MongoRelationshipsFile, + projectId?: string, + dbId?: string, + ): Promise { + let resolvedProjectId = projectId ?? file.projectId; + const resolvedDbId = dbId ?? file.databaseId; + + if (!resolvedProjectId && resolvedDbId) { + const project = await this.getProjectByDatabaseId(resolvedDbId); + if (project?.id) { + resolvedProjectId = project.id; + } + } + + let filePath: string; + if (resolvedProjectId) { + filePath = this.projectFile(resolvedProjectId, PROJECT_FILES.relationships); + } else if (resolvedDbId) { + filePath = path.join(CONFIG_FOLDER, "relationships", `${resolvedDbId}.json`); + } else { + throw new Error("Cannot save relationships: missing both projectId and dbId"); + } + + const fileToSave: MongoRelationshipsFile = { + ...file, + projectId: resolvedProjectId || undefined, + databaseId: resolvedDbId, + insufficientEvidence: Array.isArray(file.insufficientEvidence) + ? file.insufficientEvidence + : [], + updatedAt: new Date().toISOString(), + }; + + await this.writeJSON(filePath, fileToSave); + } + async getAnnotations(projectId: string): Promise { return this.readJSON(this.projectFile(projectId, PROJECT_FILES.annotations)); } diff --git a/bridge/src/services/queryExecutor.ts b/bridge/src/services/queryExecutor.ts index 5a625ef7..22d9e307 100644 --- a/bridge/src/services/queryExecutor.ts +++ b/bridge/src/services/queryExecutor.ts @@ -1,6 +1,44 @@ import { DBType, Rpc, QueryParams } from "../types"; import { getConnector } from "./connectorRegistry"; import logger from "./logger"; +import { mongoConnector, MongoApiError } from "../connectors/mongo"; +import { + MongoConfig, + MongoExecutionContext, + MongoCollectionInfo, + MongoInferSchemaRequest, + MongoCollectionSchema, + MongoFindRequest, + MongoFindResult, + MongoInsertRequest, + MongoInsertResult, + MongoUpdateRequest, + MongoUpdateResult, + MongoDeleteRequest, + MongoDeleteResult, + MongoReplaceRequest, + MongoReplaceResult, + MongoCreateCollectionRequest, + MongoCreateCollectionResult, + MongoDropCollectionRequest, + MongoDropCollectionResult, + MongoAggregationRequest, + MongoAggregationResult, + MongoMqlExecuteRequest, + MongoMqlExecuteResult, + MongoExplainRequest, + MongoExplainResult, + MongoIndexListRequest, + MongoIndexListResult, + MongoIndexCreateRequest, + MongoIndexCreateResult, + MongoIndexDropRequest, + MongoIndexDropResult, + MongoIndexStatsRequest, + MongoIndexStatsResult, + MongoIndexHideRequest, + MongoIndexHideResult, +} from "../types/mongo"; // Concurrency limit for parallel processing const PARALLEL_LIMIT = 5; @@ -169,4 +207,254 @@ export class QueryExecutor { return { name: dbName, schemas: finalSchemas.filter(Boolean) }; } + + // ---- M2: MongoDB document engine (docs/MONGODB_M2_SPEC.md §13) ---- + + async findDocuments( + conn: MongoConfig, + dbType: DBType, + req: MongoFindRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.findDocuments(conn, req); + } + + async insertDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoInsertRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.insertDocument(conn, req); + } + + async updateDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoUpdateRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.updateDocument(conn, req); + } + + async replaceDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoReplaceRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.replaceDocument(conn, req); + } + + async createCollection( + conn: MongoConfig, + dbType: DBType, + req: MongoCreateCollectionRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Collection operations require a MongoDB connection", + ); + } + return mongoConnector.createCollection(conn, req); + } + + async dropCollection( + conn: MongoConfig, + dbType: DBType, + req: MongoDropCollectionRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Collection operations require a MongoDB connection", + ); + } + return mongoConnector.dropCollection(conn, req); + } + + async deleteDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoDeleteRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.deleteDocument(conn, req); + } + + async runAggregation( + conn: MongoConfig, + dbType: DBType, + req: MongoAggregationRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "runAggregation requires a MongoDB connection", + ); + } + return mongoConnector.runAggregation(conn, req); + } + + async listCollections( + conn: MongoConfig, + dbType: DBType, + _req?: { collectionFilter?: string }, + ): Promise<{ collections: MongoCollectionInfo[] }> { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "listCollections requires a MongoDB connection", + ); + } + const collections = await mongoConnector.listCollections(conn); + return { collections }; + } + + async inferCollectionSchema( + conn: MongoConfig, + dbType: DBType, + req: MongoInferSchemaRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "inferCollectionSchema requires a MongoDB connection", + ); + } + return mongoConnector.inferCollectionSchema(conn, req); + } + + async executeMql( + conn: MongoConfig, + dbType: DBType, + req: MongoMqlExecuteRequest, + context?: MongoExecutionContext, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "executeMql requires a MongoDB connection", + ); + } + return mongoConnector.executeMql(conn, req, context); + } + + async explainMql( + conn: MongoConfig, + dbType: DBType, + req: MongoExplainRequest, + context?: MongoExecutionContext, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "explainMql requires a MongoDB connection", + ); + } + return mongoConnector.explainMql(conn, req, context); + } + + // =========================================================================== + // M6 — Secondary Index Management (docs/MONGODB_M6_SPEC.md §3) + // =========================================================================== + + async listIndexes( + conn: MongoConfig, + dbType: DBType, + req: MongoIndexListRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "listIndexes requires a MongoDB connection", + ); + } + return mongoConnector.listIndexes(conn, req); + } + + async createIndex( + conn: MongoConfig, + dbType: DBType, + req: MongoIndexCreateRequest, + options?: { readOnly?: boolean }, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "createIndex requires a MongoDB connection", + ); + } + return mongoConnector.createIndex(conn, req, options); + } + + async dropIndex( + conn: MongoConfig, + dbType: DBType, + req: MongoIndexDropRequest, + options?: { readOnly?: boolean }, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "dropIndex requires a MongoDB connection", + ); + } + return mongoConnector.dropIndex(conn, req, options); + } + + async getIndexStats( + conn: MongoConfig, + dbType: DBType, + req: MongoIndexStatsRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "getIndexStats requires a MongoDB connection", + ); + } + return mongoConnector.getIndexStats(conn, req); + } + + async hideIndex( + conn: MongoConfig, + dbType: DBType, + req: MongoIndexHideRequest, + options?: { readOnly?: boolean }, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "hideIndex requires a MongoDB connection", + ); + } + return mongoConnector.hideIndex(conn, req, options); + } } diff --git a/bridge/src/sessionManager.ts b/bridge/src/sessionManager.ts index e38f587e..93619cbf 100644 --- a/bridge/src/sessionManager.ts +++ b/bridge/src/sessionManager.ts @@ -1,9 +1,13 @@ -type Session = { +import logger from "./services/logger"; + +export interface Session { id: string; connectionId?: string; cancel?: () => Promise | void; + isCancelled?: boolean; + callbackDispatched?: boolean; createdAt: number; -}; +} /** Sessions idle longer than this are treated as stale and swept away */ const DEFAULT_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes @@ -32,26 +36,59 @@ export class SessionManager { get(id: string) { return this.sessions.get(id); } + remove(id: string) { return this.sessions.delete(id); } + list() { return Array.from(this.sessions.values()); } registerCancel(id: string, fn: () => Promise | void) { const s = this.sessions.get(id); - if (s) s.cancel = fn; + if (!s) return; + s.cancel = fn; + + // If cancel was called before callback registration, dispatch exactly once + if (s.isCancelled && !s.callbackDispatched) { + s.callbackDispatched = true; + try { + const res = fn(); + if (res && typeof (res as Promise).catch === "function") { + (res as Promise).catch(() => {}); + } + } catch (err) { + logger.warn({ id, err }, "Error in post-registered cancel callback"); + } + } } - async cancel(id: string) { + /** + * Synchronously triggers cancellation. + * Dispatches callback at-most-once. + * Returns false if session is unknown. + * Returns true if session exists. + * Repeated calls return true idempotently without duplicate callback dispatch. + */ + cancel(id: string): boolean { const s = this.sessions.get(id); if (!s) return false; - if (s.cancel) { - await s.cancel(); - return true; + if (s.isCancelled) return true; // Idempotent repeated cancel + + s.isCancelled = true; + if (s.cancel && !s.callbackDispatched) { + s.callbackDispatched = true; + try { + const res = s.cancel(); + if (res && typeof (res as Promise).catch === "function") { + (res as Promise).catch(() => {}); + } + } catch (err) { + logger.warn({ id, err }, "Error in synchronous cancel callback"); + } } - return false; + return true; } /** diff --git a/bridge/src/types/index.ts b/bridge/src/types/index.ts index b05ea952..cc95126d 100644 --- a/bridge/src/types/index.ts +++ b/bridge/src/types/index.ts @@ -16,6 +16,9 @@ export * from "./postgres"; export * from "./sqlite"; export * from "./duckdb"; export * from "./clickhouse"; +export * from "./mongo"; +export * from "./mongoMigration"; +export * from "./mongoBulk"; export * from "./ai"; export { SSHConfig } from "./common"; @@ -26,6 +29,7 @@ export enum DBType { SQLITE = "sqlite", DUCKDB = "duckdb", CLICKHOUSE = "clickhouse", + MONGODB = "mongodb", } export type Rpc = { diff --git a/bridge/src/types/mongo.ts b/bridge/src/types/mongo.ts new file mode 100644 index 00000000..4e4b385e --- /dev/null +++ b/bridge/src/types/mongo.ts @@ -0,0 +1,829 @@ +/** + * MongoDB-specific Types + * + * Single source of Mongo types for the bridge. Kept deliberately separate from + * `common.ts` (which models the relational catalog) so that "row / table / + * primary key" is never the conceptual MongoDB model. The connector reuses the + * shared `TableInfo` / `DBStats` shapes only as compatibility adapters for the + * existing relational RPC surface. + */ + +/** + * Connection configuration handed to the connector by ConnectionBuilder. + * `uri` is the authoritative field: it is the (runtime-reconstructed) + * authenticated `mongodb://` / `mongodb+srv://` connection string. The other + * fields are carried for completeness / composed-form connections. + */ +export type MongoConfig = { + uri?: string; // full authenticated connection string (preferred) + host?: string; // composed-form fallback + port?: number; + user?: string; + password?: string; // present only transiently when composing a URI; never persisted + database?: string; // default/auth database + authSource?: string; + replicaSet?: string; + tls?: boolean; + directConnection?: boolean; + readonly?: boolean; + /** + * M2 (opt-in): when true, findDocuments caches paginated pages briefly. + * Off by default to avoid stale reads (spec §14 / D-M2-5). + */ + cacheDocuments?: boolean; +}; + +// --------------------------------------------------------------------------- +// Typed BSON-to-wire representation (see connectors/mongoBson.ts). +// Every value here is safe to pass through JSON.stringify on the stdio channel. +// --------------------------------------------------------------------------- + +export type MongoWireObject = { [key: string]: MongoWireValue }; +export type MongoWireArray = MongoWireValue[]; + +export type MongoWireObjectId = { $oid: string }; +export type MongoWireDate = { $date: string }; // ISO-8601 UTC +export type MongoWireBinary = { $binary: { base64: string; subtype: string } }; +export type MongoWireDecimal = { $numberDecimal: string }; +export type MongoWireLong = { $numberLong: string }; +export type MongoWireTimestamp = { $timestamp: { t: number; i: number } }; +export type MongoWireUndefined = { $undefined: true }; + +export type MongoWireValue = + | null + | boolean + | number + | string + | MongoWireArray + | MongoWireObject + | MongoWireObjectId + | MongoWireDate + | MongoWireBinary + | MongoWireDecimal + | MongoWireLong + | MongoWireTimestamp + | MongoWireUndefined; + +export type SerializedMongoDocument = { [key: string]: MongoWireValue }; + +// --------------------------------------------------------------------------- +// Native document-oriented result shapes. +// `MongoCollectionInfo` is used in M1 (collection listing). The inferred-schema +// shapes are declared now so the connector contract is complete and typed; they +// are consumed starting in a later milestone. +// --------------------------------------------------------------------------- + +export type MongoCollectionInfo = { name: string; type: "collection" | "view" }; + +export type MongoBsonType = + | "objectId" + | "string" + | "number" + | "long" + | "decimal" + | "timestamp" + | "binary" + | "date" + | "bool" + | "null" + | "array" + | "object" + | "undefined"; + +export interface MongoInferSchemaRequest { + collection: string; + sampleSize?: number; // default 100, clamped [1, 1000] + maxDepth?: number; // default 8, clamped [1, 16] +} + +export interface MongoInferredField { + path: string; + bsonType: MongoBsonType; + types: MongoBsonType[]; + numericKinds?: Array<"int" | "double">; + depth: number; + parentPath?: string; + isArray: boolean; + isNested: boolean; + count: number; + nullCount: number; + sampleSize: number; + presencePct: number; + nullable: boolean; + optional: boolean; + typeConflicts: Array<{ bsonType: MongoBsonType; count: number }>; + arrayElementTypes?: MongoBsonType[]; + sampleValue?: EJSONDocument | MongoWireValue; + maxChildDepth?: number; + truncated?: boolean; +} + +export interface MongoCollectionSchema { + collection: string; + sampleSize: number; + totalDocs: number; + inferredAt: string; + fieldCount: number; + nestedFieldCount: number; + arrayFieldCount: number; + fields: MongoInferredField[]; + sampleDocuments: SerializedMongoDocument[]; +} +export type MongoInferredSchema = MongoCollectionSchema; +export type MongoCollectionListItem = MongoCollectionInfo; + +/** + * Result of parsing a MongoDB URI that may embed credentials. + * `sanitizedUri` never contains a password and is what gets persisted at rest. + */ +export type MongoUriParts = { + sanitizedUri: string; + username?: string; + password?: string; + hasCredentials: boolean; +}; + +// =========================================================================== +// M2 — MongoDB Document Engine (see docs/MONGODB_M2_SPEC.md) +// Extended-JSON document model, operation request/response types, and the +// safety/limit constants enforced by the connector. Everything below this line +// is new in M2; nothing above it changes. +// =========================================================================== + +/** + * MongoDB Extended JSON v2 type wrappers. A value crossing the JSON-RPC boundary + * uses these to represent BSON types plain JSON cannot carry. A bare 24-char hex + * string is a STRING, not an ObjectId — ObjectId must use $oid (spec §4.2 / D-M2-6). + */ +export type EJSONTypeWrapper = + | { $oid: string } + | { $date: string } + | { $date: { $numberLong: string } } + | { $numberInt: string } + | { $numberLong: string } + | { $numberDouble: string } + | { $numberDecimal: string } + | { $binary: { $type: string; $base64: string } } + | { $regex: string; $options?: string } + | { $timestamp: { $t: string; $i: string } } + | { $undefined: Record }; + +/** A JSON-RPC-safe value: plain JSON plus EJSON type wrappers. */ +export type EJsonValue = + string | number | boolean | null | EJsonValue[] | EJSONDocument | EJSONTypeWrapper; + +/** A JSON-RPC-safe document (string keys → EJSON values). */ +export type EJSONDocument = { [key: string]: EJsonValue }; + +/** Stable, credential-free key identifying a cached document page (spec §14). */ +export type MongoPageCacheKey = string; + +/** Per-collection statistics shape used by the M2 cache (spec §14). */ +export type MongoCollectionStats = { + collection: string; + count: number; + sizeBytes: number; + avgObjSize: number; +}; + +// --- Shared value objects (spec §5.1) --- + +export type MongoSortDirection = 1 | -1 | "ASC" | "DESC"; +export type MongoSortSpec = { [field: string]: MongoSortDirection }; +export type MongoProjection = { [field: string]: 0 | 1 | false | true }; + +// --- findDocuments (spec §5.2) --- + +export interface MongoFindRequest { + collection: string; + filter?: EJSONDocument; + sort?: MongoSortSpec; + projection?: MongoProjection; + page?: number; + pageSize?: number; +} + +export interface MongoFindResult { + collection: string; + documents: EJSONDocument[]; + total: number; + page: number; + pageSize: number; + totalPages: number; + hasMore: boolean; +} + +// --- insertDocument (spec §5.3) --- + +export interface MongoInsertRequest { + collection: string; + document: EJSONDocument; +} + +export interface MongoInsertResult { + collection: string; + insertedId: EJsonValue; + document: EJSONDocument; +} + +// --- updateDocument (spec §5.4) --- + +export interface MongoUpdateOptions { + upsert?: boolean; + multi?: boolean; + arrayFilters?: EJSONDocument[]; + returnDocument?: "before" | "after"; +} + +export interface MongoUpdateRequest { + collection: string; + filter: EJSONDocument; + update: EJSONDocument; + options?: MongoUpdateOptions; +} + +export interface MongoUpdateResult { + collection: string; + matchedCount: number; + modifiedCount: number; + upsertedId?: EJsonValue; + updatedDocument?: EJSONDocument; +} + +// --- deleteDocument (spec §5.5) --- + +export interface MongoDeleteOptions { + multi?: boolean; +} + +export interface MongoDeleteRequest { + collection: string; + filter: EJSONDocument; + options?: MongoDeleteOptions; +} + +export interface MongoDeleteResult { + collection: string; + deletedCount: number; +} + +// --- replaceDocument (spec §5.2.1) --- + +export interface MongoReplaceOptions { + upsert?: boolean; + returnDocument?: "before" | "after"; +} + +export interface MongoReplaceRequest { + collection: string; + filter: EJSONDocument; + replacement: EJSONDocument; + options?: MongoReplaceOptions; +} + +export interface MongoReplaceResult { + collection: string; + matchedCount: number; + modifiedCount: number; + upsertedId?: EJsonValue; + updatedDocument?: EJSONDocument; +} + +// --- collection lifecycle (spec §5.2.2 / §5.2.3) --- + +export interface MongoCreateCollectionOptions { + capped?: boolean; + size?: number; // bytes (min 4096) + max?: number; // max documents (positive integer) +} + +export interface MongoCreateCollectionRequest { + collection: string; + options?: MongoCreateCollectionOptions; +} + +export interface MongoCreateCollectionResult { + ok: boolean; + collection: string; +} + +export interface MongoDropCollectionRequest { + collection: string; +} + +export interface MongoDropCollectionResult { + ok: boolean; + collection: string; +} + +// --- runAggregation (spec §5.6) --- + +export interface MongoAggregationOptions { + allowDiskUse?: boolean; + maxTimeMS?: number; + batchSize?: number; +} + +export interface MongoAggregationRequest { + collection: string; + pipeline: EJSONDocument[]; + options?: MongoAggregationOptions; +} + +export interface MongoAggregationResult { + collection: string; + documents: EJSONDocument[]; + stageCount: number; + executionTimeMs?: number; +} + +// --- Safety / limit constants (spec §10) --- + +// --- Canonical Page Constants (M5 Spec §5.1) --- +export const DEFAULT_PAGE_SIZE = 50; +export const MAX_PAGE_SIZE = 100; +export const MAX_PAGE = 10; +export const MAX_SKIP = MAX_PAGE_SIZE * MAX_PAGE; // 1000 +export const MAX_FILTER_SIZE_BYTES = 8 * 1024; +export const MAX_SORT_FIELDS = 32; +export const MAX_PROJECTION_FIELDS = 256; +export const MAX_AGG_STAGES = 64; +export const MAX_AGG_STAGE_SIZE_BYTES = 16 * 1024; +export const MAX_AGG_RESULTS = 1000; +export const MAX_DISTINCT_VALUES = 10000; +export const MAX_EXPRESSION_DEPTH = 15; +export const MAX_DOCUMENT_SIZE_BYTES = 16 * 1024 * 1024; +export const DEFAULT_OP_TIMEOUT_MS = 30000; +export const MAX_OP_TIMEOUT_MS = 120000; +export const MAX_INSERT_DOCUMENTS = 1; + +// M3 schema inference limits (docs/MONGODB_M3_SPEC.md §4.6) +export const MAX_SAMPLE_SIZE = 1000; +export const MAX_INFERENCE_DEPTH = 16; +export const MAX_INFERRED_FIELDS = 5000; +export const DEFAULT_SAMPLE_SIZE = 100; +export const DEFAULT_MAX_DEPTH = 8; + +// --- M5 MQL Workspace & Builder Types (docs/MONGODB_M5_SPEC.md §5) --- + +export interface MongoFindOpPayload { + filter: EJSONDocument; + projection?: Record; + sort?: Record; + skip?: number; + limit?: number; // Clamped between 1 and MAX_PAGE_SIZE (100); defaults to DEFAULT_PAGE_SIZE (50) +} + +export interface MongoAggregateOpPayload { + pipeline: EJSONDocument[]; + allowDiskUse?: boolean; + batchSize?: number; +} + +export interface MongoCountOpPayload { + filter?: EJSONDocument; + skip?: number; + limit?: number; +} + +export type MongoEstimatedCountOpPayload = Record | undefined; + +export interface MongoDistinctOpPayload { + field: string; + filter?: EJSONDocument; +} + +// --- Tagged Union Request --- + +export type MongoMqlOperation = + | { operation: "find"; payload: MongoFindOpPayload } + | { operation: "aggregate"; payload: MongoAggregateOpPayload } + | { operation: "countDocuments"; payload: MongoCountOpPayload } + | { operation: "estimatedDocumentCount"; payload?: MongoEstimatedCountOpPayload } + | { operation: "distinct"; payload: MongoDistinctOpPayload }; + +export interface MongoMqlExecuteRequest { + dbId?: string; + collection: string; + op: MongoMqlOperation; + options?: { + maxTimeMS?: number; + }; +} + +// --- Typed Results --- + +export interface MongoMqlBaseResult { + collection: string; + executionTimeMs: number; +} + +export interface MongoFindMqlResult extends MongoMqlBaseResult { + operation: "find"; + documents: EJSONDocument[]; + totalReturned: number; +} + +export interface MongoAggregateMqlResult extends MongoMqlBaseResult { + operation: "aggregate"; + documents: EJSONDocument[]; + stageCount: number; +} + +export interface MongoCountMqlResult extends MongoMqlBaseResult { + operation: "countDocuments" | "estimatedDocumentCount"; + count: number; +} + +export interface MongoDistinctMqlResult extends MongoMqlBaseResult { + operation: "distinct"; + field: string; + values: EJsonValue[]; + count: number; +} + +export type MongoMqlExecuteResult = + MongoFindMqlResult | MongoAggregateMqlResult | MongoCountMqlResult | MongoDistinctMqlResult; + +export type MongoExplainTarget = + | { + operation: "find"; + filter: EJSONDocument; + projection?: Record; + sort?: Record; + skip?: number; + limit?: number; + } + | { + operation: "aggregate"; + pipeline: EJSONDocument[]; + allowDiskUse?: boolean; + }; + +export interface MongoExplainRequest { + dbId?: string; + collection: string; + target: MongoExplainTarget; + verbosity?: "queryPlanner" | "executionStats" | "allPlansExecution"; + options?: { + maxTimeMS?: number; + }; +} + +export interface MongoExplainResult { + collection: string; + operation: "find" | "aggregate"; + queryPlanner: { + plannerVersion: number; + namespace: string; + indexFilterSet: boolean; + winningPlan: Record; + rejectedPlans?: Record[]; + }; + executionStats?: { + executionSuccess: boolean; + nReturned: number; + executionTimeMillis: number; + totalKeysExamined: number; + totalDocsExamined: number; + executionStages: Record; + allPlansExecution?: Record[]; + }; + serverInfo?: { + host: string; + port: number; + version: string; + }; + rawExplain: Record; + executionTimeMs: number; +} + +// =========================================================================== +// M6 — MongoDB Index Management & Migrations (docs/MONGODB_M6_SPEC.md §3) +// =========================================================================== + +export type MongoIndexDirection = 1 | -1 | "text" | "2dsphere" | "hashed"; + +export interface MongoIndexCollation { + locale: string; + caseLevel?: boolean; + caseFirst?: "upper" | "lower" | "off"; + strength?: 1 | 2 | 3 | 4 | 5; + numericOrdering?: boolean; + alternate?: "non-ignorable" | "shifted"; + maxVariable?: "punct" | "space"; + backwards?: boolean; +} + +export interface MongoIndexOptions { + name?: string; + unique?: boolean; + sparse?: boolean; + expireAfterSeconds?: number; + partialFilterExpression?: EJSONDocument; + collation?: MongoIndexCollation; + hidden?: boolean; + background?: boolean; +} + +export interface MongoIndexDetails { + name: string; + key: Record; + sizeBytes: number; + unique: boolean; + sparse: boolean; + isIdIndex: boolean; + expireAfterSeconds?: number; + partialFilterExpression?: EJSONDocument; + collation?: MongoIndexCollation; + hidden?: boolean; + v?: number; + textIndexVersion?: number; + weights?: Record; + default_language?: string; + language_override?: string; + isRedundant?: boolean; + redundantReason?: string; +} + +export interface MongoIndexListRequest { + collection: string; +} + +export interface MongoIndexListResult { + collection: string; + indexes: MongoIndexDetails[]; + totalSizeBytes: number; +} + +export interface MongoIndexCreateRequest { + collection: string; + key: Record; + options?: MongoIndexOptions; +} + +export interface MongoIndexCreateResult { + success: boolean; + indexName: string; + executionTimeMs: number; +} + +export interface MongoIndexDropRequest { + collection: string; + indexName: string; +} + +export interface MongoIndexDropResult { + success: boolean; + droppedIndexName: string; + executionTimeMs: number; +} + +export interface MongoIndexStatsItem { + name: string; + key: Record; + ops: number; + since: string; + host?: string; +} + +export interface MongoIndexStatsRequest { + collection: string; +} + +export interface MongoIndexStatsResult { + collection: string; + stats: Record; +} + +export interface MongoIndexHideRequest { + collection: string; + indexName: string; + hidden: boolean; +} + +export interface MongoIndexHideResult { + success: boolean; + indexName: string; + hidden: boolean; +} + +// =========================================================================== +// M7 Phase 1 — MongoDB Relationship Inference & Management Types +// =========================================================================== + +export type MongoCardinality = + | "one-to-one" + | "one-to-many" + | "many-to-one" + | "many-to-many" + | "unknown" + | "insufficient-evidence"; + +export interface MongoRelationshipEvidence { + sourceCollection: string; + sourceField: string; + targetCollection: string; + targetField: string; + sampleCount: number; + hitCount: number; + hitRatio: number; // 0.0 - 1.0 (must be >= 0.50 to consider) + hasNamePatternMatch: boolean; + isBsonTypeMatch: boolean; + multipleSourcesPerTarget: boolean; // required for many-to-one or many-to-many + isArrayReference: boolean; + hasUniqueIndexOnSource: boolean; + sampleMatches: Array<{ sourceValue: string; targetId: string }>; +} + +export interface MongoInferredRelationship { + id: string; // e.g. "orders.user_id->users._id" + sourceCollection: string; + sourceField: string; + targetCollection: string; + targetField: string; // usually "_id" + cardinality: MongoCardinality; + confidence: number; // 0.0 - 1.0 (>= 0.65 to display) + evidence: MongoRelationshipEvidence; + status: "inferred" | "confirmed" | "rejected"; + inferredAt: string; +} + +export interface MongoRelationship { + id: string; + sourceCollection: string; + sourceField: string; + targetCollection: string; + targetField: string; + cardinality: MongoCardinality; + isManual?: boolean; + description?: string; + createdAt: string; + updatedAt: string; +} + +export interface MongoRelationshipsFile { + version: number; + projectId?: string; + databaseId: string; + inferred: MongoInferredRelationship[]; + confirmed: MongoRelationship[]; + rejectedIds: string[]; + manual: MongoRelationship[]; + insufficientEvidence?: MongoInferredRelationship[]; + updatedAt: string; +} + +// --- RPC Request / Response Interfaces --- + +export interface MongoInferRelationshipsRequest { + dbId: string; + projectId?: string; + collections?: string[]; + sampleSize?: number; + maxCollections?: number; + timeoutMs?: number; +} + +export interface MongoInferRelationshipsResult { + relationships: MongoInferredRelationship[]; // actionable (confidence >= 0.65) + insufficientEvidence: MongoInferredRelationship[]; // diagnostic (confidence < 0.65) + inferredAt: string; + collectionsScanned: number; + timedOut?: boolean; +} + +export interface MongoListRelationshipsRequest { + dbId: string; + projectId?: string; +} + +export interface MongoListRelationshipsResult { + inferred: MongoInferredRelationship[]; + confirmed: MongoRelationship[]; + rejectedIds: string[]; + manual: MongoRelationship[]; + insufficientEvidence?: MongoInferredRelationship[]; +} + +export interface MongoSetRelationshipStatusRequest { + dbId: string; + projectId?: string; + relationshipId: string; + status: "confirmed" | "rejected"; +} + +export interface MongoSetRelationshipStatusResult { + relationshipId: string; + status: "confirmed" | "rejected"; +} + +export interface MongoSaveRelationshipRequest { + dbId: string; + projectId?: string; + relationship: MongoRelationship; +} + +export interface MongoSaveRelationshipResult { + relationship: MongoRelationship; +} + +export interface MongoDeleteRelationshipRequest { + dbId: string; + projectId?: string; + relationshipId: string; +} + +export interface MongoDeleteRelationshipResult { + success: boolean; + relationshipId: string; +} + +// =========================================================================== +// Stage 8 Phase 3 — Cancellation, Lifecycle Supervision & Teardown Types +// =========================================================================== + +export type CursorCloseState = + "none" | "open" | "closing" | "closed_confirmed" | "closed_failed" | "closed_timed_out"; + +export type CursorCloseOutcome = + | { status: "closed_confirmed"; durationMs: number } + | { status: "closed_failed"; error: unknown; durationMs: number } + | { status: "closed_timed_out"; durationMs: number }; + +export interface MongoExecutionContext { + signal?: AbortSignal; + onCursorCreated?: (cursor: any) => void; + onDriverPromiseCreated?: (promise: Promise) => void; +} + +export interface DriverSettlementSignal { + readonly isSettled: boolean; + readonly settledAt?: number; + readonly outcome?: "fulfilled" | "rejected"; + readonly error?: unknown; + readonly promise: Promise<{ outcome: "fulfilled" | "rejected"; error?: unknown }>; + markSettled: (outcome?: "fulfilled" | "rejected", error?: unknown) => void; +} + +export type TeardownOutcomeStatus = + | "destroy_confirmed" + | "driver_settled" + | "shutdown_handoff" + | "retryable_failure" + | "unrecoverable_failure"; + +export type TeardownOutcome = + | { + status: "destroy_confirmed"; + invalidationSucceeded: boolean; + socketsDestroyed: number; + durationMs: number; + } + | { + status: "driver_settled"; + durationMs: number; + } + | { + status: "shutdown_handoff"; + reason: "shutdown_adopted"; + durationMs: number; + } + | { + status: "retryable_failure"; + invalidationSucceeded: boolean; + socketError?: unknown; + invalidationError?: unknown; + attempt: number; + durationMs: number; + } + | { + status: "unrecoverable_failure"; + invalidationSucceeded: boolean; + socketError?: unknown; + invalidationError?: unknown; + totalAttempts: number; + durationMs: number; + }; + +export type OperationLifecycleState = + | "running" + | "rpc_rejected_driver_pending" + | "zombie_pinned" + | "teardown_initiated" + | "teardown_failed_pinned" + | "unrecoverable_pinned" + | "shutdown_adopted" + | "completed" + | "closed"; + +export interface OperationLifecycleHandle { + id: string; + dbId: string; + state: OperationLifecycleState; + settlementSignal: DriverSettlementSignal; + retryTimer?: NodeJS.Timeout; + cursor?: any; + cursorCloseOutcome?: CursorCloseOutcome; + hasReleasedLease?: boolean; +} diff --git a/bridge/src/types/mongoBulk.ts b/bridge/src/types/mongoBulk.ts new file mode 100644 index 00000000..3cad117f --- /dev/null +++ b/bridge/src/types/mongoBulk.ts @@ -0,0 +1,221 @@ +/** + * MongoDB Milestone 7 Phase 3 — Bulk Operations Types + * + * Defines the request/response payloads, operation lifecycle states, + * and chunked import/export contracts for MongoDB bulk operations. + */ + +export type MongoBulkFormat = "json" | "ndjson" | "csv"; + +export type MongoBulkOperationType = "import" | "export"; + +export type MongoBulkOperationStatus = + "pending" | "in_progress" | "completed" | "failed" | "cancelled"; + +export interface MongoBulkOperation { + operationId: string; + type: MongoBulkOperationType; + status: MongoBulkOperationStatus; + dbId: string; + collection: string; + format: MongoBulkFormat; + createdAt: string; + updatedAt: string; + expiresAt: string; + totalDocuments?: number; + processedDocuments: number; + successfulDocuments: number; + failedDocuments: number; + fileSizeBytes?: number; + error?: string; + tempFilePath?: string; +} + +// =========================================================================== +// Import RPC Interfaces +// =========================================================================== + +export interface MongoBulkImportStartRequest { + dbId: string; + collection: string; + format: MongoBulkFormat; + options?: { + totalChunksExpected?: number; + }; +} + +export interface MongoBulkImportStartResult { + operationId: string; + status: MongoBulkOperationStatus; + format: MongoBulkFormat; + collection: string; +} + +export interface MongoBulkImportChunkRequest { + operationId: string; + chunkIndex: number; + isLastChunk?: boolean; + payload: string | Record[] | any; +} + +export interface MongoBulkImportChunkResult { + operationId: string; + chunkIndex: number; + status: MongoBulkOperationStatus; + insertedCount: number; + failedCount: number; + errors?: Array<{ index: number; code: string; message: string }>; + isComplete: boolean; + totalInserted: number; + totalFailed: number; +} + +// =========================================================================== +// Export RPC Interfaces +// =========================================================================== + +export interface MongoBulkExportStartRequest { + dbId: string; + collection: string; + format: MongoBulkFormat; + filter?: Record; + projection?: Record; + sort?: Record; + limit?: number; +} + +export interface MongoBulkExportStartResult { + operationId: string; + status: MongoBulkOperationStatus; + collection: string; + format: MongoBulkFormat; +} + +// =========================================================================== +// Export Save-To RPC Interfaces (Phase 4 Extension) +// =========================================================================== + +export interface StagingFileIdentity { + exactPath: string; + dev: number; + ino: number; + size: number; + birthtimeMs?: number; + registeredAt: number; + retryCount: number; +} + +export interface MongoBulkExportSaveToRequest { + operationId: string; + destinationPath: string; + options?: { + overwrite?: boolean; + }; +} + +export interface MongoBulkExportSaveToResult { + operationId: string; + destinationPath: string; + bytesWritten: number; + success: boolean; + warning?: string; +} + +// =========================================================================== +// Lifecycle & Management RPC Interfaces +// =========================================================================== + +export interface MongoBulkGetOperationStatusRequest { + operationId: string; +} + +export interface MongoBulkOperationStatusResult { + operationId: string; + type: MongoBulkOperationType; + status: MongoBulkOperationStatus; + collection: string; + format: MongoBulkFormat; + processedDocuments: number; + successfulDocuments: number; + failedDocuments: number; + fileSizeBytes?: number; + error?: string; + createdAt: string; + updatedAt: string; + expiresAt: string; +} + +export interface MongoBulkCancelRequest { + operationId: string; +} + +export interface MongoBulkCancelResult { + operationId: string; + status: "cancelled"; + message: string; +} + +export interface MongoBulkCleanupRequest { + operationId: string; +} + +export interface MongoBulkCleanupResult { + operationId: string; + success: boolean; + alreadyCleaned?: boolean; +} + +// =========================================================================== +// Supporting Bulk Mutation RPC Interfaces +// =========================================================================== + +export interface MongoBulkInsertRequest { + dbId: string; + collection: string; + documents: Record[]; + options?: { + ordered?: boolean; + }; +} + +export interface MongoBulkInsertResult { + collection: string; + insertedCount: number; + failedCount: number; + errors?: Array<{ index: number; code: string; message: string }>; +} + +export interface MongoBulkUpdateRequest { + dbId: string; + collection: string; + filter: Record; + update: Record; + options?: { + multi?: boolean; + upsert?: boolean; + allowUnrestricted?: boolean; + }; +} + +export interface MongoBulkUpdateResult { + collection: string; + matchedCount: number; + modifiedCount: number; + upsertedId?: any; + advisoryWarning?: string; +} + +export interface MongoBulkDeleteRequest { + dbId: string; + collection: string; + filter: Record; + options?: { + allowUnrestricted?: boolean; + }; +} + +export interface MongoBulkDeleteResult { + collection: string; + deletedCount: number; + advisoryWarning?: string; +} diff --git a/bridge/src/types/mongoMigration.ts b/bridge/src/types/mongoMigration.ts new file mode 100644 index 00000000..3e85eccf --- /dev/null +++ b/bridge/src/types/mongoMigration.ts @@ -0,0 +1,303 @@ +/** + * MongoDB Milestone 6 — Declarative Migration Engine Types. + * + * Defines the AST, schema, lock models, event log documents, + * and RPC interfaces for MongoDB migrations. + */ + +export type MongoMigrationStepType = + | "createCollection" + | "dropCollection" + | "createIndex" + | "dropIndex" + | "setValidator" + | "updateMany"; + +export interface MongoCreateCollectionStep { + type: "createCollection"; + collection: string; + options?: { + capped?: boolean; + size?: number; + max?: number; + validator?: Record; + validationLevel?: "off" | "strict" | "moderate"; + validationAction?: "error" | "warn"; + collation?: Record; + }; +} + +export interface MongoDropCollectionStep { + type: "dropCollection"; + collection: string; +} + +export interface MongoCreateIndexStep { + type: "createIndex"; + collection: string; + key: Record; + options?: { + name?: string; + unique?: boolean; + sparse?: boolean; + expireAfterSeconds?: number; + partialFilterExpression?: Record; + collation?: Record; + hidden?: boolean; + background?: boolean; + }; +} + +export interface MongoDropIndexStep { + type: "dropIndex"; + collection: string; + indexName: string; +} + +export interface MongoSetValidatorStep { + type: "setValidator"; + collection: string; + validator: Record; + validationLevel?: "off" | "strict" | "moderate"; + validationAction?: "error" | "warn"; +} + +export interface MongoUpdateManyStep { + type: "updateMany"; + collection: string; + filter: Record; + update: Record; + options?: { + upsert?: boolean; + arrayFilters?: Record[]; + collation?: Record; + }; +} + +/** + * Discriminated union of all supported migration steps. + * Note: `deleteMany` is permanently excluded. + */ +export type MongoMigrationStep = + | MongoCreateCollectionStep + | MongoDropCollectionStep + | MongoCreateIndexStep + | MongoDropIndexStep + | MongoSetValidatorStep + | MongoUpdateManyStep; + +export interface MongoMigrationManifest { + version: string; + id: string; + name: string; + description?: string; + author?: string; + allowIrreversible?: boolean; + up: MongoMigrationStep[]; + down?: MongoMigrationStep[]; +} + +export type MongoMigrationEventType = + | "VALIDATION_SUCCESS" + | "LOCK_ACQUIRED" + | "MIGRATION_START" + | "STEP_START" + | "STEP_COMPLETE" + | "STEP_FAILED" + | "STEP_UNCERTAIN" + | "COMPENSATION_START" + | "COMPENSATION_COMPLETE" + | "COMPENSATION_FAILED" + | "MIGRATION_APPLIED" + | "MIGRATION_ROLLED_BACK" + | "LOCK_RELEASED" + | "MANUAL_INTERVENTION_REQUIRED"; + +export interface MongoMigrationEvent { + _id: string; + eventId: string; + migrationId: string; + attemptId: string; + attemptNumber: number; + sequenceNumber: number; + stepIndex?: number; + stepType?: string; + eventType: MongoMigrationEventType; + timestamp: string; + lockOwnerId: string; + fencingToken: number; + metadata?: Record; +} + +export type MongoMigrationStatus = + | "PENDING" + | "ACQUIRING_LOCK" + | "APPLYING" + | "FAILED_PARTIAL" + | "CRASHED_UNCERTAIN" + | "ROLLING_BACK" + | "ROLLED_BACK" + | "APPLIED" + | "FAILED_MANUAL_INTERVENTION_REQUIRED"; + +export type MongoMigrationStepStatus = + "STEP_PENDING" | "STEP_RUNNING" | "STEP_COMPLETED" | "STEP_FAILED" | "STEP_UNCERTAIN"; + +export type MongoMigrationDriftStatus = "VERIFIED" | "DRIFTED" | "UNVERIFIABLE"; + +export type MongoMigrationDriftReason = + | "CHECKSUM_MATCH" + | "CHECKSUM_MISMATCH" + | "MANIFEST_MISSING" + | "RECORDED_CHECKSUM_MISSING" + | "MANIFEST_PARSE_ERROR"; + +export interface MongoMigrationDriftDetails { + status: MongoMigrationDriftStatus; + reason: MongoMigrationDriftReason; + expectedChecksum?: string; + actualChecksum?: string; + message?: string; +} + +export interface MongoMigrationSummary { + _id: string; // Matches manifest ID + name: string; + checksum: string; + status: MongoMigrationStatus; + startedAt: string; + completedAt?: string; + durationMs?: number; + totalDurationMs?: number; + executedSteps: number; + totalSteps: number; + lastExecutedStepIndex?: number; + fencingToken: number; + attemptNumber?: number; + attemptId?: string; + error?: string | { stepIndex?: number; stepType?: string; message: string }; + isReversible?: boolean; + appliedBy?: string; + // AUD-05 Additions: + driftStatus?: MongoMigrationDriftStatus; + drifted?: boolean; + driftDetails?: MongoMigrationDriftDetails; +} + +export interface MongoMigrationLockDoc { + _id: "migration_lock"; + ownerId: string | null; + fencingToken: number; + acquiredAt?: Date | string | null; + heartbeatAt?: Date | string | null; + expiresAt?: Date | string | null; + releasedAt?: Date | string | null; + migrationId?: string | null; +} + +export interface MongoMigrationLockStatus { + isLocked: boolean; + ownerId?: string; + fencingToken?: number; + expiresAt?: string; + isExpired: boolean; + migrationId?: string; +} + +// ========================================== +// RPC Request & Result Types +// ========================================== + +export interface MongoMigrationListRequest { + dbId: string; + projectRoot?: string; +} + +export interface MongoMigrationListResult { + appliedMigrations: MongoMigrationSummary[]; + pendingMigrations: { + id: string; + name: string; + filename: string; + checksum: string; + stepCount: number; + isReversible: boolean; + }[]; + isLocked: boolean; + lockDetails?: { + ownerId?: string; + fencingToken?: number; + acquiredAt?: string; + expiresAt?: string; + migrationId?: string; + }; +} + +export interface MongoMigrationValidateRequest { + dbId?: string; + manifest: MongoMigrationManifest; +} + +export interface MongoMigrationValidateResult { + valid: boolean; + errors: string[]; + warnings: string[]; + stepsBreakdown: { + ddlCount: number; + dataTransformCount: number; + reversibleCount: number; + irreversibleCount: number; + }; +} + +export interface MongoMigrationApplyRequest { + dbId: string; + version?: string; + dryRun?: boolean; + projectRoot?: string; + ownerId?: string; +} + +export interface MongoMigrationApplyResult { + success: boolean; + appliedCount: number; + results: { + id: string; + name: string; + executionTimeMs: number; + status: MongoMigrationStatus; + executedSteps: number; + error?: string; + }[]; +} + +export interface MongoMigrationRollbackRequest { + dbId: string; + version: string; + force?: boolean; + projectRoot?: string; + ownerId?: string; +} + +export interface MongoMigrationRollbackResult { + success: boolean; + version: string; + status: MongoMigrationStatus; + executionTimeMs: number; + stepsCompensated: number; + error?: string; +} + +export interface MongoMigrationGetStatusRequest { + dbId: string; + migrationId?: string; +} + +export interface MongoMigrationGetStatusResult { + migration?: MongoMigrationSummary; + events?: MongoMigrationEvent[]; + lockStatus: MongoMigrationLockStatus; + lastAppliedVersion?: string; + lastAppliedAt?: string; + pendingCount?: number; +} diff --git a/bridge/src/utils/dbTypeDetector.ts b/bridge/src/utils/dbTypeDetector.ts index 9df8046c..fd97c444 100644 --- a/bridge/src/utils/dbTypeDetector.ts +++ b/bridge/src/utils/dbTypeDetector.ts @@ -12,6 +12,7 @@ export class DBTypeDetector { if (normalized.includes("sqlite")) return DBType.SQLITE; if (normalized.includes("duckdb")) return DBType.DUCKDB; if (normalized.includes("clickhouse")) return DBType.CLICKHOUSE; + if (normalized.includes("mongo")) return DBType.MONGODB; } return DBType.POSTGRES; // default } diff --git a/bridge/src/utils/mongoUri.ts b/bridge/src/utils/mongoUri.ts new file mode 100644 index 00000000..224f2acd --- /dev/null +++ b/bridge/src/utils/mongoUri.ts @@ -0,0 +1,307 @@ +/** + * MongoDB URI utilities. + * + * Implements the decision-#11 credential handling: a MongoDB connection URI that + * embeds credentials is parsed, the password is extracted (to be stored through + * the existing dbStore AES path), and a *sanitized* URI (credentials removed) is + * what gets persisted in databases.json. The authenticated URI is reconstructed + * only at runtime from the sanitized URI + the decrypted password, and is never + * written back. + * + * Uses a zero-dependency parser that safely handles single-host, mongodb+srv://, + * and multi-host comma-separated seedlists (which WHATWG URL rejects with ERR_INVALID_URL). + * All returned strings are JSON-safe and free of plaintext passwords + * unless explicitly re-injecting at runtime. + */ + +import type { MongoUriParts } from "../types/mongo"; + +const MONGO_URI_RE = /^mongodb(\+srv)?:\/\//i; + +/** True when the string is a MongoDB connection URI. */ +export function isMongoUri(raw?: string): boolean { + return typeof raw === "string" && MONGO_URI_RE.test(raw.trim()); +} + +type MongoUriComponents = { + scheme: "mongodb" | "mongodb+srv"; + hosts: string; + host: string; + port?: number; + user?: string; + password?: string; // decoded + database?: string; + options: string; // raw query string, without leading '?' +}; + +/** Parse a MongoDB URI into its components (password decoded). */ +function parseMongoUriComponents(raw: string): MongoUriComponents { + const trimmed = raw.trim(); + const isSrv = trimmed.toLowerCase().startsWith("mongodb+srv://"); + const scheme: "mongodb" | "mongodb+srv" = isSrv ? "mongodb+srv" : "mongodb"; + const prefix = isSrv ? "mongodb+srv://" : "mongodb://"; + const rest = trimmed.toLowerCase().startsWith(prefix) + ? trimmed.slice(prefix.length) + : trimmed.replace(/^mongodb(\+srv)?:\/\//i, ""); + + const queryIndex = rest.indexOf("?"); + const hashIndex = rest.indexOf("#"); + let endOfPathIndex = rest.length; + if (queryIndex !== -1) endOfPathIndex = queryIndex; + if (hashIndex !== -1 && (queryIndex === -1 || hashIndex < queryIndex)) { + endOfPathIndex = hashIndex; + } + + const authorityAndPath = rest.slice(0, endOfPathIndex); + let options = ""; + if (queryIndex !== -1) { + if (hashIndex !== -1 && hashIndex > queryIndex) { + options = rest.slice(queryIndex + 1, hashIndex); + } else { + options = rest.slice(queryIndex + 1); + } + } + + const slashIndex = authorityAndPath.indexOf("/"); + const authority = slashIndex !== -1 ? authorityAndPath.slice(0, slashIndex) : authorityAndPath; + const rawDb = slashIndex !== -1 ? authorityAndPath.slice(slashIndex + 1) : ""; + let database: string | undefined = undefined; + if (rawDb) { + try { + database = decodeURIComponent(rawDb); + } catch { + database = rawDb; + } + } + + let user: string | undefined; + let password: string | undefined; + let hosts = authority; + + const atIndex = authority.lastIndexOf("@"); + if (atIndex !== -1) { + const creds = authority.slice(0, atIndex); + hosts = authority.slice(atIndex + 1); + const colonIndex = creds.indexOf(":"); + if (colonIndex !== -1) { + const rawUser = creds.slice(0, colonIndex); + const rawPass = creds.slice(colonIndex + 1); + try { + user = decodeURIComponent(rawUser); + } catch { + user = rawUser; + } + try { + password = decodeURIComponent(rawPass); + } catch { + password = rawPass; + } + } else { + try { + user = decodeURIComponent(creds); + } catch { + user = creds; + } + } + if (user === "") user = undefined; + if (password === "") password = undefined; + } + + let host = ""; + let port: number | undefined; + + const firstHost = hosts.split(",")[0].trim(); + if (firstHost.startsWith("[")) { + const closeBracket = firstHost.indexOf("]"); + if (closeBracket !== -1) { + host = firstHost.slice(1, closeBracket); + const after = firstHost.slice(closeBracket + 1); + if (after.startsWith(":")) { + const p = Number(after.slice(1)); + if (!isNaN(p)) port = p; + } + } else { + host = firstHost; + } + } else { + const colonIndex = firstHost.lastIndexOf(":"); + if (colonIndex !== -1) { + host = firstHost.slice(0, colonIndex); + const p = Number(firstHost.slice(colonIndex + 1)); + if (!isNaN(p)) port = p; + } else { + host = firstHost; + } + } + + return { scheme, hosts, host, port, user, password, database, options }; +} + +/** Rebuild a MongoDB URI string from components. */ +function buildMongoUriString(c: MongoUriComponents, includePassword: boolean): string { + let auth = ""; + if (c.user) { + auth += encodeURIComponent(c.user); + if (includePassword && c.password) { + auth += ":" + encodeURIComponent(c.password); + } + auth += "@"; + } + const hosts = c.hosts || (c.host ? (c.port ? `${c.host}:${c.port}` : c.host) : ""); + const path = c.database ? `/${encodeURIComponent(c.database)}` : ""; + const qs = c.options ? `?${c.options}` : ""; + return `${c.scheme}://${auth}${hosts}${path}${qs}`; +} + +type MongoUriOptions = { authSource?: string; replicaSet?: string }; + +function parseMongoOptions(rawOptions: string): MongoUriOptions { + const out: MongoUriOptions = {}; + if (!rawOptions) return out; + for (const pair of rawOptions.split("&")) { + const idx = pair.indexOf("="); + if (idx === -1) continue; + const key = pair.slice(0, idx); + const value = decodeURIComponent(pair.slice(idx + 1)); + if (key === "authSource") out.authSource = value; + else if (key === "replicaSet") out.replicaSet = value; + } + return out; +} + +/** + * Extract credentials from a MongoDB URI. The password returned here is meant to + * be stored through the dbStore AES credential path — never persisted in the URI. + */ +export function extractMongoCredentials(uri: string): { user?: string; password?: string } { + const c = parseMongoUriComponents(uri); + return { user: c.user, password: c.password }; +} + +/** + * Return a copy of the URI with any embedded password removed. The username and + * all query options are preserved (only the secret is stripped). + */ +export function sanitizeMongoUri(uri: string): string { + return buildMongoUriString(parseMongoUriComponents(uri), false); +} + +/** + * Reconstruct the authenticated URI at runtime by re-injecting the password into + * a sanitized URI. If the URI has no username there is nowhere to attach the + * password, so the URI is returned unchanged. + */ +export function injectMongoPassword(sanitizedUri: string, password: string): string { + const c = parseMongoUriComponents(sanitizedUri); + if (!c.user) return sanitizedUri; + return buildMongoUriString({ ...c, password }, true); +} + +/** + * Split a URI (if any) into a persisted-safe shape: the sanitized URI plus the + * extracted password (to be AES-stored). When the input is not a MongoDB URI the + * original value is passed through untouched with no credentials. + */ +export function splitMongoUriCredentials(uri: string): MongoUriParts & { isMongoUri: boolean } { + if (!isMongoUri(uri)) { + return { sanitizedUri: uri, hasCredentials: false, isMongoUri: false }; + } + const c = parseMongoUriComponents(uri); + const sanitizedUri = buildMongoUriString(c, false); + const hasCredentials = Boolean(c.password); + return { + sanitizedUri, + username: c.user, + password: c.password, + hasCredentials, + isMongoUri: true, + }; +} + +/** + * Derive the relational-meta fields (host/port/user/database/options) from a + * MongoDB URI so dbStore's required fields are populated. Options such as + * authSource / replicaSet remain in the (sanitized) URI query and are therefore + * preserved on the wire; they are also surfaced here for convenience. + */ +export function normalizeMongoUriMeta(uri: string): { + host: string; + port: number; + user: string; + database: string; + authSource?: string; + replicaSet?: string; +} { + const c = parseMongoUriComponents(uri); + const opts = parseMongoOptions(c.options); + return { + host: c.host, + port: c.port || 27017, + user: c.user || "", + database: c.database || "", + authSource: opts.authSource, + replicaSet: opts.replicaSet, + }; +} + +/** + * Build the authenticated connection URI handed to the driver. + * - If a MongoDB URI is present it is treated as the persisted (sanitized) form + * and the runtime password is re-injected when both a username and a password + * exist. + * - Otherwise a URI is composed from host/port/user/password. + */ +export function buildMongoConnectionUri( + db: { + url?: string; + uri?: string; + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + authSource?: string; + replicaSet?: string; + tls?: boolean; + directConnection?: boolean; + }, + pwd: string | null, +): string { + const rawUri = db.url || db.uri; + + if (rawUri && isMongoUri(rawUri)) { + const c = parseMongoUriComponents(rawUri); + const pass = pwd ?? undefined; + // Only attach the runtime password when there is a username to attach it to. + const finalPassword = c.user && pass ? pass : c.password; + return buildMongoUriString({ ...c, password: finalPassword }, true); + } + + const host = db.host; + if (!host) { + throw new Error("MongoDB connection requires a url (mongodb://...) or host"); + } + const port = db.port || 27017; + const user = db.user; + const pass = pwd ?? db.password; + + let auth = ""; + if (user) { + auth += encodeURIComponent(user); + if (pass) auth += ":" + encodeURIComponent(pass); + auth += "@"; + } + + let uri = `mongodb://${auth}${host}:${port}`; + if (db.database) uri += `/${encodeURIComponent(db.database)}`; + + const params = new URLSearchParams(); + if (db.authSource) params.set("authSource", db.authSource); + if (db.replicaSet) params.set("replicaSet", db.replicaSet); + if (db.tls) params.set("tls", "true"); + if (db.directConnection) params.set("directConnection", "true"); + const qs = params.toString(); + if (qs) uri += `?${qs}`; + + return uri; +} diff --git a/package.json b/package.json index 3f5e35df..14c87e5c 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,17 @@ { "name": "relwave", "private": true, - "version": "1.2.0", + "version": "1.3.0", "type": "module", "scripts": { "dev": "vite", "build": "tsc && vite build", "build:tauri-assets": "npm run bridge:package && npm run build", "preview": "vite preview", + "test": "npx tsx --test src/__tests__/*.test.ts src/features/mongodb/__tests__/*.test.ts", + "test:frontend": "npx tsx --test src/__tests__/*.test.ts src/features/mongodb/__tests__/*.test.ts", + "test:bridge": "npm --prefix bridge test", + "test:all": "npm run test && npm run test:bridge", "tauri": "tauri", "lint": "eslint .", "lint:fix": "eslint . --fix", @@ -20,6 +24,8 @@ }, "dependencies": { "@aptabase/tauri": "github:aptabase/tauri-plugin-aptabase", + "@codemirror/autocomplete": "^6.20.4", + "@codemirror/lang-json": "^6.0.2", "@codemirror/lang-sql": "^6.10.0", "@codemirror/theme-one-dark": "^6.1.3", "@codemirror/view": "^6.39.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e8562dfb..86436e2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,12 @@ importers: '@aptabase/tauri': specifier: github:aptabase/tauri-plugin-aptabase version: https://codeload.github.com/aptabase/tauri-plugin-aptabase/tar.gz/de6df6b8671568aad53eabaf4b49207a3d6c5622 + '@codemirror/autocomplete': + specifier: ^6.20.4 + version: 6.20.4 + '@codemirror/lang-json': + specifier: ^6.0.2 + version: 6.0.2 '@codemirror/lang-sql': specifier: ^6.10.0 version: 6.10.0 @@ -97,7 +103,7 @@ importers: version: 0.7.53 '@uiw/react-codemirror': specifier: ^4.25.4 - version: 4.25.4(@babel/runtime@7.28.4)(@codemirror/autocomplete@6.20.1)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/theme-one-dark@6.1.3)(@codemirror/view@6.39.16)(codemirror@6.0.2)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + version: 4.25.4(@babel/runtime@7.28.4)(@codemirror/autocomplete@6.20.4)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/theme-one-dark@6.1.3)(@codemirror/view@6.39.16)(codemirror@6.0.2)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) class-variance-authority: specifier: ^0.7.1 version: 0.7.1 @@ -307,12 +313,15 @@ packages: resolution: {integrity: sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==} engines: {node: '>=6.9.0'} - '@codemirror/autocomplete@6.20.1': - resolution: {integrity: sha512-1cvg3Vz1dSSToCNlJfRA2WSI4ht3K+WplO0UMOgmUYPivCyy2oueZY6Lx7M9wThm7SDUBViRmuT+OG/i8+ON9A==} + '@codemirror/autocomplete@6.20.4': + resolution: {integrity: sha512-yF8RhOf2SXlg00+qtupzLhrglcas61boqaONiNcbPIebHCpJocpqTBA6dyrtUrQO47hqyTdeob/3XnYilT7TaQ==} '@codemirror/commands@6.10.2': resolution: {integrity: sha512-vvX1fsih9HledO1c9zdotZYUZnE4xV0m6i3m25s5DIfXofuprk6cRcLUZvSk3CASUbwjQX21tOGbkY2BH8TpnQ==} + '@codemirror/lang-json@6.0.2': + resolution: {integrity: sha512-x2OtO+AvwEHrEwR0FyyPtfDUiloG3rnVTSZV1W8UteaLL8/MajQd8DpvUb2YVzC+/T18aSDv0H9mu+xw0EStoQ==} + '@codemirror/lang-sql@6.10.0': resolution: {integrity: sha512-6ayPkEd/yRw0XKBx5uAiToSgGECo/GY2NoJIHXIIQh1EVwLuKoU8BP/qK0qH5NLXAbtJRLuT73hx7P9X34iO4w==} @@ -616,6 +625,9 @@ packages: '@lezer/highlight@1.2.3': resolution: {integrity: sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==} + '@lezer/json@1.0.3': + resolution: {integrity: sha512-BP9KzdF9Y35PDpv04r0VeSTKDeox5vVr3efE7eBbx3r4s3oNLfunchejZhjArmeieBH+nVOpgIiBJpEAv8ilqQ==} + '@lezer/lr@1.4.5': resolution: {integrity: sha512-/YTRKP5yPPSo1xImYQk7AZZMAgap0kegzqCSYHjAL9x1AZ0ZQW+IpcEzMKagCsbTsLnVeWkxYrCNeXG8xEPrjg==} @@ -3699,7 +3711,7 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 - '@codemirror/autocomplete@6.20.1': + '@codemirror/autocomplete@6.20.4': dependencies: '@codemirror/language': 6.12.2 '@codemirror/state': 6.5.4 @@ -3713,9 +3725,14 @@ snapshots: '@codemirror/view': 6.39.16 '@lezer/common': 1.5.0 + '@codemirror/lang-json@6.0.2': + dependencies: + '@codemirror/language': 6.12.2 + '@lezer/json': 1.0.3 + '@codemirror/lang-sql@6.10.0': dependencies: - '@codemirror/autocomplete': 6.20.1 + '@codemirror/autocomplete': 6.20.4 '@codemirror/language': 6.12.2 '@codemirror/state': 6.5.4 '@lezer/common': 1.5.0 @@ -3976,6 +3993,12 @@ snapshots: dependencies: '@lezer/common': 1.5.0 + '@lezer/json@1.0.3': + dependencies: + '@lezer/common': 1.5.0 + '@lezer/highlight': 1.2.3 + '@lezer/lr': 1.4.5 + '@lezer/lr@1.4.5': dependencies: '@lezer/common': 1.5.0 @@ -5598,9 +5621,9 @@ snapshots: '@typescript-eslint/types': 8.70.0 eslint-visitor-keys: 5.0.1 - '@uiw/codemirror-extensions-basic-setup@4.25.4(@codemirror/autocomplete@6.20.1)(@codemirror/commands@6.10.2)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/view@6.39.16)': + '@uiw/codemirror-extensions-basic-setup@4.25.4(@codemirror/autocomplete@6.20.4)(@codemirror/commands@6.10.2)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/view@6.39.16)': dependencies: - '@codemirror/autocomplete': 6.20.1 + '@codemirror/autocomplete': 6.20.4 '@codemirror/commands': 6.10.2 '@codemirror/language': 6.12.2 '@codemirror/lint': 6.9.5 @@ -5608,14 +5631,14 @@ snapshots: '@codemirror/state': 6.5.4 '@codemirror/view': 6.39.16 - '@uiw/react-codemirror@4.25.4(@babel/runtime@7.28.4)(@codemirror/autocomplete@6.20.1)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/theme-one-dark@6.1.3)(@codemirror/view@6.39.16)(codemirror@6.0.2)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + '@uiw/react-codemirror@4.25.4(@babel/runtime@7.28.4)(@codemirror/autocomplete@6.20.4)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/theme-one-dark@6.1.3)(@codemirror/view@6.39.16)(codemirror@6.0.2)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@babel/runtime': 7.28.4 '@codemirror/commands': 6.10.2 '@codemirror/state': 6.5.4 '@codemirror/theme-one-dark': 6.1.3 '@codemirror/view': 6.39.16 - '@uiw/codemirror-extensions-basic-setup': 4.25.4(@codemirror/autocomplete@6.20.1)(@codemirror/commands@6.10.2)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/view@6.39.16) + '@uiw/codemirror-extensions-basic-setup': 4.25.4(@codemirror/autocomplete@6.20.4)(@codemirror/commands@6.10.2)(@codemirror/language@6.12.2)(@codemirror/lint@6.9.5)(@codemirror/search@6.6.0)(@codemirror/state@6.5.4)(@codemirror/view@6.39.16) codemirror: 6.0.2 react: 19.2.3 react-dom: 19.2.3(react@19.2.3) @@ -5747,7 +5770,7 @@ snapshots: codemirror@6.0.2: dependencies: - '@codemirror/autocomplete': 6.20.1 + '@codemirror/autocomplete': 6.20.4 '@codemirror/commands': 6.10.2 '@codemirror/language': 6.12.2 '@codemirror/lint': 6.9.5 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5ed0b5af..16625f0f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,3 @@ allowBuilds: + "@parcel/watcher": false esbuild: true diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 22d53560..a33ef62e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "relwave" -version = "1.0.0" +version = "1.3.0" description = "A powerful, cross-platform desktop application for database management with native Git version control." authors = ["yashh56"] edition = "2021" diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 1f8ea157..9a21698e 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "RelWave", - "version": "1.2.0", + "version": "1.3.0", "identifier": "tech.relwave.app", "build": { "beforeDevCommand": "vite", diff --git a/src/__tests__/parseConnectionUrl.mongo.test.ts b/src/__tests__/parseConnectionUrl.mongo.test.ts new file mode 100644 index 00000000..5f539ea5 --- /dev/null +++ b/src/__tests__/parseConnectionUrl.mongo.test.ts @@ -0,0 +1,130 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { parseConnectionUrl, buildConnectionUrl, parseMongoUrl } from "@/lib/parseConnectionUrl"; +import { MONGODB_REQUIRED_FIELDS } from "@/features/home/types"; + +function expect(actual: any) { + return { + toBe(expected: any) { + assert.strictEqual(actual, expected); + }, + toEqual(expected: any) { + assert.deepStrictEqual(actual, expected); + }, + toBeNull() { + assert.strictEqual(actual, null); + }, + toBeUndefined() { + assert.strictEqual(actual, undefined); + }, + toBeDefined() { + assert.notStrictEqual(actual, undefined); + }, + toBeTruthy() { + assert.ok(actual); + }, + toBeFalsy() { + assert.ok(!actual); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && actual.includes(substr), + `Expected "${actual}" to contain "${substr}"`, + ); + }, + not: { + toBeNull() { + assert.notStrictEqual(actual, null); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && !actual.includes(substr), + `Expected "${actual}" not to contain "${substr}"`, + ); + }, + }, + }; +} + +describe("MongoDB Connection URL Parsing & Round-Tripping", () => { + it("parses plain mongodb:// single-host URI and retains originalUri", () => { + const uri = "mongodb://localhost:27017/analytics"; + const parsed = parseConnectionUrl(uri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.host).toBe("localhost"); + expect(parsed!.port).toBe("27017"); + expect(parsed!.database).toBe("analytics"); + expect(parsed!.ssl).toBe(false); + expect(parsed!.originalUri).toBe(uri); + }); + + it("parses mongodb:// with authentication and query params", () => { + const uri = "mongodb://admin:secretPass@127.0.0.1:27017/authDb?authSource=admin&tls=true"; + const parsed = parseConnectionUrl(uri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.user).toBe("admin"); + expect(parsed!.password).toBe("secretPass"); + expect(parsed!.host).toBe("127.0.0.1"); + expect(parsed!.port).toBe("27017"); + expect(parsed!.database).toBe("authDb"); + expect(parsed!.ssl).toBe(true); + expect(parsed!.originalUri).toBe(uri); + }); + + it("CRITICAL SRV INVARIANT: parses mongodb+srv:// and preserves originalUri verbatim", () => { + const srvUri = + "mongodb+srv://user1:pwd%40123@cluster0.abcde.mongodb.net/production?retryWrites=true&w=majority"; + const parsed = parseConnectionUrl(srvUri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.ssl).toBe(true); + expect(parsed!.originalUri).toBe(srvUri); + expect(parsed!.originalUri).toBe(srvUri.trim()); + + // Proves buildConnectionUrl preserves originalUri verbatim + const rebuilt = buildConnectionUrl(parsed!); + expect(rebuilt).toBe(srvUri); + }); + + it("preserves replica-set seed list verbatim via originalUri", () => { + const rsUri = + "mongodb://node1:27017,node2:27017,node3:27017/replicaDb?replicaSet=myRs&readPreference=secondaryPreferred"; + const parsed = parseConnectionUrl(rsUri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.originalUri).toBe(rsUri); + + const rebuilt = buildConnectionUrl(parsed!); + expect(rebuilt).toBe(rsUri); + }); + + it("MONGODB_REQUIRED_FIELDS matches ['name', 'type', 'host', 'port']", () => { + expect(MONGODB_REQUIRED_FIELDS).toEqual(["name", "type", "host", "port"]); + }); + + it("parseMongoUrl retains originalUri verbatim for single-host, multi-host, and mongodb+srv://", () => { + const singleHost = "mongodb://localhost:27017/mydb"; + const multiHost = + "mongodb://user:pass@host1:27017,host2:27017,host3:27017/mydb?replicaSet=rs0&authSource=admin"; + const srv = + "mongodb+srv://user:pass@cluster0.abcde.mongodb.net/production?retryWrites=true&w=majority"; + + const parsedSingle = parseMongoUrl(singleHost); + expect(parsedSingle).not.toBeNull(); + expect(parsedSingle!.originalUri).toBe(singleHost); + + const parsedMulti = parseMongoUrl(multiHost); + expect(parsedMulti).not.toBeNull(); + expect(parsedMulti!.originalUri).toBe(multiHost); + + const parsedSrv = parseMongoUrl(srv); + expect(parsedSrv).not.toBeNull(); + expect(parsedSrv!.originalUri).toBe(srv); + }); +}); diff --git a/src/components/layout/VerticalIconBar.tsx b/src/components/layout/VerticalIconBar.tsx index 84ac2aed..d33e525a 100644 --- a/src/components/layout/VerticalIconBar.tsx +++ b/src/components/layout/VerticalIconBar.tsx @@ -1,32 +1,12 @@ import { useState } from "react"; -import { - Activity, - Home, - Database, - Search, - GitBranch, - GitCommitHorizontal, - Layers, - Terminal, - History, - Sparkles, - Settings, -} from "lucide-react"; +import { Home, Settings } from "lucide-react"; import { Link, useLocation } from "react-router-dom"; import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { SettingsDialog } from "@/features/settings/components"; +import { PanelType, getDatabasePanelItems } from "./panelItems"; -export type PanelType = - | "data" - | "sql-workspace" - | "query-builder" - | "schema-explorer" - | "er-diagram" - | "monitoring" - | "git-status" - | "migrations" - | "ai-workspace"; +export { type PanelType, getDatabasePanelItems }; interface VerticalIconBarProps { dbId?: string; @@ -37,37 +17,6 @@ interface VerticalIconBarProps { const globalNavigationItems = [{ icon: Home, label: "Dashboard", path: "/" }]; -function supportsMonitoring(databaseType?: string) { - const type = databaseType?.toLowerCase(); - return ( - type === "postgres" || - type === "postgresql" || - type === "mysql" || - type === "mariadb" || - type === "clickhouse" - ); -} - -export function getDatabasePanelItems(databaseType?: string): Array<{ - icon: any; - label: string; - panel: PanelType; -}> { - return [ - { icon: Layers, label: "Data View", panel: "data" }, - { icon: Terminal, label: "SQL Workspace", panel: "sql-workspace" }, - { icon: Sparkles, label: "AI Workspace", panel: "ai-workspace" }, - { icon: Search, label: "Query Builder", panel: "query-builder" }, - { icon: GitBranch, label: "Schema Explorer", panel: "schema-explorer" }, - { icon: Database, label: "ER Diagram", panel: "er-diagram" }, - ...(supportsMonitoring(databaseType) - ? [{ icon: Activity, label: "Monitoring", panel: "monitoring" as PanelType }] - : []), - { icon: History, label: "Migrations", panel: "migrations" }, - { icon: GitCommitHorizontal, label: "Git Status", panel: "git-status" }, - ]; -} - export default function VerticalIconBar({ dbId, databaseType, diff --git a/src/components/layout/panelItems.ts b/src/components/layout/panelItems.ts new file mode 100644 index 00000000..84b2dc9e --- /dev/null +++ b/src/components/layout/panelItems.ts @@ -0,0 +1,74 @@ +import { + Activity, + Database, + Search, + GitBranch, + GitCommitHorizontal, + Layers, + Terminal, + History, + Sparkles, + Network, + ArrowLeftRight, + LucideIcon, +} from "lucide-react"; +import { getDatabaseKind } from "@/lib/databases"; + +export type PanelType = + | "data" + | "sql-workspace" + | "query-builder" + | "schema-explorer" + | "er-diagram" + | "monitoring" + | "git-status" + | "migrations" + | "ai-workspace" + | "mongo-workspace" + | "mongo-aggregation" + | "mongo-migrations" + | "mongo-relationships" + | "mongo-bulk"; + +function supportsMonitoring(databaseType?: string) { + const type = databaseType?.toLowerCase(); + return ( + type === "postgres" || + type === "postgresql" || + type === "mysql" || + type === "mariadb" || + type === "clickhouse" + ); +} + +export function getDatabasePanelItems(databaseType?: string): Array<{ + icon: LucideIcon; + label: string; + panel: PanelType; +}> { + if (getDatabaseKind(databaseType) === "nosql") { + return [ + { icon: Layers, label: "Data View", panel: "data" }, + { icon: Terminal, label: "Query Workspace", panel: "mongo-workspace" }, + { icon: GitBranch, label: "Aggregation Builder", panel: "mongo-aggregation" }, + { icon: History, label: "Migrations", panel: "mongo-migrations" }, + { icon: Network, label: "Relationships", panel: "mongo-relationships" }, + { icon: ArrowLeftRight, label: "Bulk Operations", panel: "mongo-bulk" }, + { icon: GitCommitHorizontal, label: "Git Status", panel: "git-status" }, + ]; + } + + return [ + { icon: Layers, label: "Data View", panel: "data" }, + { icon: Terminal, label: "SQL Workspace", panel: "sql-workspace" }, + { icon: Sparkles, label: "AI Workspace", panel: "ai-workspace" }, + { icon: Search, label: "Query Builder", panel: "query-builder" }, + { icon: GitBranch, label: "Schema Explorer", panel: "schema-explorer" }, + { icon: Database, label: "ER Diagram", panel: "er-diagram" }, + ...(supportsMonitoring(databaseType) + ? [{ icon: Activity, label: "Monitoring", panel: "monitoring" as PanelType }] + : []), + { icon: History, label: "Migrations", panel: "migrations" }, + { icon: GitCommitHorizontal, label: "Git Status", panel: "git-status" }, + ]; +} diff --git a/src/components/shared/DataTable.tsx b/src/components/shared/DataTable.tsx index 9be2e3e6..9c7abbf2 100644 --- a/src/components/shared/DataTable.tsx +++ b/src/components/shared/DataTable.tsx @@ -38,6 +38,7 @@ interface DataTableProps { sortColumn?: string; sortDirection?: "asc" | "desc"; onSort?: (column: string) => void; + renderCell?: (value: any, column: string, row: Record) => React.ReactNode; } export const DataTable = ({ @@ -51,6 +52,7 @@ export const DataTable = ({ sortColumn, sortDirection, onSort, + renderCell, }: DataTableProps) => { const [copiedCellKey, setCopiedCellKey] = useState(null); @@ -199,7 +201,9 @@ export const DataTable = ({ } >
-
{formatCellValue(val, column)}
+
+ {renderCell ? renderCell(val, column, row) : formatCellValue(val, column)} +
{hasValue && (
')); + // Completed = 1 + assert.ok(html.includes('data-testid="metric-completed">1
')); + // Partial = 1 + assert.ok(html.includes('data-testid="metric-partial">1')); + // Failed = 1 + assert.ok(html.includes('data-testid="metric-failed">1')); + // Subtitle displays total + assert.ok(html.includes("Showing 4 bulk operations for database ApproachADatabase.")); + }); + + // 10. Partial Success UI Presentation + it("10. verifies MongoBulkOperationsHistory renders amber outline badge and inserted/failed counts for partial success", () => { + const now = new Date().toISOString(); + const partialMutation: TrackedSyncBulkMutation = { + kind: "sync", + id: "partial-123", + dbId: "db-hist", + collection: "customers", + type: "insert", + status: "partial_success", + timestamp: now, + insertedCount: 85, + failedCount: 15, + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoBulkOperationsHistory, { + operations: [partialMutation], + }), + ); + + assert.ok(html.includes("Partial Success")); + assert.ok(html.includes("85 inserted, 15 failed")); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoBulkNavigationIntegration.test.ts b/src/features/mongodb/__tests__/MongoBulkNavigationIntegration.test.ts new file mode 100644 index 00000000..fb9e197f --- /dev/null +++ b/src/features/mongodb/__tests__/MongoBulkNavigationIntegration.test.ts @@ -0,0 +1,1040 @@ +/** + * MongoDB Milestone 7 Phase 4 Stage 7 — Bulk Navigation & Contextual Integration Tests + * + * Exercises the 12 scenarios defined in docs/MONGODB_M7_SPEC.md §12: + * 1. Global Navigation Item Registration & Relational Isolation + * 2. Router Dispatch to MongoBulkOperationsPanel + * 3. Alt+6 Keyboard Shortcut Navigation + * 4. Keyboard Shortcut Input Field Suppression + * 5. Keyboard Shortcut Modal Dialog Suppression + * 6. Contextual Collection Toolbar Buttons Rendering + * 7. Contextual Toolbar Callback Invocations + * 8. System Collection Restrictions (Write Disabled, Read/Export Enabled) + * 9. Read-Only Mode Lock + * 10. Collection Prefill in Data View Dialogs + * 11. Background Execution & Unmount Survival with Single Poller Authority + * 12. State-Changing & Partial-Success View Refresh + */ + +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { ArrowLeftRight } from "lucide-react"; +import { getDatabasePanelItems, type PanelType } from "../../../components/layout/panelItems"; +import { MongoDocumentViewer } from "../components/MongoDocumentViewer"; +import { MongoBulkOperationsPanel } from "../components/bulk/MongoBulkOperationsPanel"; +import { + defaultMongoBulkOperationsStore, + createMongoBulkOperationsStore, +} from "../hooks/useMongoBulkOperationsStore"; +import type { + TrackedAsyncBulkOperation, + MongoBulkOperationStatus, + TrackedSyncBulkMutation, +} from "../types/bulk"; + +function findVdomNodes(node: any, predicate: (node: any) => boolean): any[] { + const matches: any[] = []; + if (!node || typeof node !== "object") return matches; + if (predicate(node)) { + matches.push(node); + } + if (Array.isArray(node.props?.children)) { + for (const child of node.props.children) { + matches.push(...findVdomNodes(child, predicate)); + } + } else if (node.props?.children) { + matches.push(...findVdomNodes(node.props.children, predicate)); + } + return matches; +} + +function renderViewerVdom(props: any): any { + let capturedVdom: any; + function Wrapper() { + capturedVdom = MongoDocumentViewer(props); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(Wrapper)); + return capturedVdom; +} + +describe("MongoDB M7 Stage 7: Bulk Navigation & Contextual Integration Tests", () => { + beforeEach(() => { + defaultMongoBulkOperationsStore.getSnapshot().resetStore(); + }); + + // Scenario 1: Navigation Item Registration & Relational Isolation + it("1. getDatabasePanelItems('mongodb') returns 7 items with 'mongo-bulk' at index 5 and 'git-status' at index 6; all 6 relational engines strictly exclude mongo-bulk", () => { + const mongoItems = getDatabasePanelItems("mongodb"); + assert.strictEqual(mongoItems.length, 7); + assert.strictEqual(mongoItems[5].panel, "mongo-bulk"); + assert.strictEqual(mongoItems[5].label, "Bulk Operations"); + assert.strictEqual(mongoItems[5].icon, ArrowLeftRight); + assert.strictEqual(mongoItems[6].panel, "git-status"); + + // Case-insensitivity + const mongoUpperItems = getDatabasePanelItems("MongoDB"); + assert.strictEqual(mongoUpperItems.length, 7); + assert.strictEqual(mongoUpperItems[5].panel, "mongo-bulk"); + assert.strictEqual(mongoUpperItems[6].panel, "git-status"); + + // All 6 relational engines (7 string identifiers) strictly exclude mongo-bulk + const relationalIdentifiers = [ + "postgres", + "postgresql", + "mysql", + "mariadb", + "sqlite", + "duckdb", + "clickhouse", + ]; + + for (const engine of relationalIdentifiers) { + const items = getDatabasePanelItems(engine); + const hasMongoBulk = items.some((i) => i.panel === "mongo-bulk"); + assert.strictEqual( + hasMongoBulk, + false, + `Relational engine ${engine} must strictly exclude mongo-bulk panel item`, + ); + } + }); + + // Scenario 2: Router Dispatch to MongoBulkOperationsPanel + it("2. Router dispatch in DatabaseDetails.tsx maps 'mongo-bulk' to MongoBulkOperationsPanel", () => { + const databaseDetailsPath = path.resolve(process.cwd(), "src/pages/DatabaseDetails.tsx"); + const content = fs.readFileSync(databaseDetailsPath, "utf-8"); + + // Assert file imports MongoBulkOperationsPanel + assert.ok( + content.includes("MongoBulkOperationsPanel"), + "DatabaseDetails.tsx must import MongoBulkOperationsPanel", + ); + + // Assert case 'mongo-bulk': is present in renderPanel router + assert.ok( + content.includes('case "mongo-bulk":'), + 'DatabaseDetails.tsx renderPanel must contain case "mongo-bulk":', + ); + + // Verify component renders without errors when given props + const vdom = React.createElement(MongoBulkOperationsPanel, { + key: "mbulk-test-1", + dbId: "test-db-1", + databaseName: "TestDB", + }); + const html = ReactDOMServer.renderToStaticMarkup(vdom); + assert.ok(html.includes("Bulk Operations")); + assert.ok(html.includes("TestDB")); + }); + + // Scenario 3: Alt+6 Keyboard Shortcut Navigation + it("3. Alt+6 keyboard shortcut selects 'mongo-bulk' when database is MongoDB", () => { + let activePanel: PanelType = "data"; + const setActivePanel = (panel: PanelType) => { + activePanel = panel; + }; + + // Simulate keydown handler from DatabaseDetails.tsx + const simulateKeyDown = (event: { + key: string; + altKey: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + shiftKey?: boolean; + targetIsInput?: boolean; + isDialogOpen?: boolean; + }) => { + if (event.targetIsInput) return; + if (event.isDialogOpen) return; + + if (event.altKey && !event.ctrlKey && !event.metaKey && !event.shiftKey) { + const num = parseInt(event.key, 10); + if (!isNaN(num) && num >= 1 && num <= 9) { + const items = getDatabasePanelItems("mongodb"); + const targetItem = items[num - 1]; + if (targetItem) { + setActivePanel(targetItem.panel); + } + } + } + }; + + simulateKeyDown({ key: "6", altKey: true }); + assert.strictEqual(activePanel, "mongo-bulk"); + }); + + // Scenario 4: Keyboard Shortcut Input Field Suppression + it("4. Alt+6 shortcut is suppressed when user focus is inside an input or textarea element", () => { + let activePanel: PanelType = "data"; + const setActivePanel = (panel: PanelType) => { + activePanel = panel; + }; + + const simulateKeyDown = (event: { + key: string; + altKey: boolean; + targetIsInput?: boolean; + isDialogOpen?: boolean; + }) => { + if (event.targetIsInput) return; + if (event.isDialogOpen) return; + + if (event.altKey) { + const num = parseInt(event.key, 10); + if (!isNaN(num) && num >= 1 && num <= 9) { + const items = getDatabasePanelItems("mongodb"); + const targetItem = items[num - 1]; + if (targetItem) { + setActivePanel(targetItem.panel); + } + } + } + }; + + // User is typing in input + simulateKeyDown({ key: "6", altKey: true, targetIsInput: true }); + assert.strictEqual( + activePanel, + "data", + "Active panel must remain unchanged when input is focused", + ); + }); + + // Scenario 5: Keyboard Shortcut Modal Dialog Suppression + it("5. Alt+6 shortcut is suppressed when a modal dialog is currently open", () => { + let activePanel: PanelType = "data"; + const setActivePanel = (panel: PanelType) => { + activePanel = panel; + }; + + const simulateKeyDown = (event: { + key: string; + altKey: boolean; + targetIsInput?: boolean; + isDialogOpen?: boolean; + }) => { + if (event.targetIsInput) return; + if (event.isDialogOpen) return; + + if (event.altKey) { + const num = parseInt(event.key, 10); + if (!isNaN(num) && num >= 1 && num <= 9) { + const items = getDatabasePanelItems("mongodb"); + const targetItem = items[num - 1]; + if (targetItem) { + setActivePanel(targetItem.panel); + } + } + } + }; + + // Modal dialog is open in DOM + simulateKeyDown({ key: "6", altKey: true, isDialogOpen: true }); + assert.strictEqual( + activePanel, + "data", + "Active panel must remain unchanged when dialog is open", + ); + }); + + // Scenario 6: Contextual Collection Toolbar Buttons Rendering + it("6. MongoDocumentViewer renders Import, Export, and Bulk Mutations toolbar buttons with data-testids", () => { + const vdom = renderViewerVdom({ + collection: "users", + documents: [{ _id: "doc-1", name: "Alice" }], + totalDocuments: 1, + page: 1, + pageSize: 50, + totalPages: 1, + isLoading: false, + sortField: null, + sortDirection: null, + onSort: () => {}, + onPageChange: () => {}, + onPageSizeChange: () => {}, + filterInput: "", + onFilterInputChange: () => {}, + onApplyFilter: () => {}, + onResetFilter: () => {}, + isFilterApplied: false, + onRefresh: () => {}, + onImportClick: () => {}, + onExportClick: () => {}, + onBulkMutationClick: () => {}, + }); + + const importBtn = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-import-button", + ); + const exportBtn = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-export-button", + ); + const mutationBtn = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-mutation-button", + ); + + assert.strictEqual(importBtn.length, 1, "Must render toolbar-bulk-import-button"); + assert.strictEqual(exportBtn.length, 1, "Must render toolbar-bulk-export-button"); + assert.strictEqual(mutationBtn.length, 1, "Must render toolbar-bulk-mutation-button"); + }); + + // Scenario 7: Contextual Toolbar Callback Invocations + it("7. Clicking toolbar buttons fires onImportClick, onExportClick, and onBulkMutationClick callbacks", () => { + let importClicked = false; + let exportClicked = false; + let mutationClicked = false; + + const vdom = renderViewerVdom({ + collection: "users", + documents: [{ _id: "doc-1", name: "Alice" }], + totalDocuments: 1, + page: 1, + pageSize: 50, + totalPages: 1, + isLoading: false, + sortField: null, + sortDirection: null, + onSort: () => {}, + onPageChange: () => {}, + onPageSizeChange: () => {}, + filterInput: "", + onFilterInputChange: () => {}, + onApplyFilter: () => {}, + onResetFilter: () => {}, + isFilterApplied: false, + onRefresh: () => {}, + onImportClick: () => { + importClicked = true; + }, + onExportClick: () => { + exportClicked = true; + }, + onBulkMutationClick: () => { + mutationClicked = true; + }, + }); + + const [importBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-import-button", + ); + const [exportBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-export-button", + ); + const [mutationBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-mutation-button", + ); + + importBtn.props.onClick(); + exportBtn.props.onClick(); + mutationBtn.props.onClick(); + + assert.strictEqual(importClicked, true, "onImportClick must be invoked"); + assert.strictEqual(exportClicked, true, "onExportClick must be invoked"); + assert.strictEqual(mutationClicked, true, "onBulkMutationClick must be invoked"); + }); + + // Scenario 8: System Collection Restrictions + it("8. When collection starts with 'system.', Import and Bulk Mutations are disabled with tooltip; Export remains enabled", () => { + const vdom = renderViewerVdom({ + collection: "system.views", + documents: [{ _id: "view-1" }], + totalDocuments: 1, + page: 1, + pageSize: 50, + totalPages: 1, + isLoading: false, + sortField: null, + sortDirection: null, + onSort: () => {}, + onPageChange: () => {}, + onPageSizeChange: () => {}, + filterInput: "", + onFilterInputChange: () => {}, + onApplyFilter: () => {}, + onResetFilter: () => {}, + isFilterApplied: false, + onRefresh: () => {}, + onImportClick: () => {}, + onExportClick: () => {}, + onBulkMutationClick: () => {}, + }); + + const [importBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-import-button", + ); + const [exportBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-export-button", + ); + const [mutationBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-mutation-button", + ); + + assert.strictEqual( + importBtn.props.disabled, + true, + "Import must be disabled on system collection", + ); + assert.strictEqual( + importBtn.props.title, + "System collections are protected against imports", + "Import must carry explanation title", + ); + + assert.strictEqual( + mutationBtn.props.disabled, + true, + "Bulk Mutations must be disabled on system collection", + ); + assert.strictEqual( + mutationBtn.props.title, + "System collections cannot be modified", + "Bulk Mutations must carry explanation title", + ); + + assert.strictEqual( + exportBtn.props.disabled, + false, + "Export must remain enabled for system collection inspection", + ); + }); + + // Scenario 9: Read-Only Mode Lock & Connection Propagation + it("9a. Derivation of effective read-only state from connection when no prop is supplied", () => { + const computeEffectiveReadOnly = (isReadOnlyProp: boolean | undefined, dbConn: any) => + Boolean(isReadOnlyProp) || Boolean(dbConn?.readOnly || dbConn?.readonly); + + // No prop supplied (undefined): checks readOnly and readonly property variations + assert.strictEqual(computeEffectiveReadOnly(undefined, { readOnly: true }), true); + assert.strictEqual(computeEffectiveReadOnly(undefined, { readonly: true }), true); + assert.strictEqual(computeEffectiveReadOnly(undefined, { readOnly: false }), false); + assert.strictEqual(computeEffectiveReadOnly(undefined, null), false); + assert.strictEqual(computeEffectiveReadOnly(undefined, undefined), false); + }); + + it("9b. Read-only connection cannot be overridden into writable mode by an explicit prop", () => { + const computeEffectiveReadOnly = (isReadOnlyProp: boolean | undefined, dbConn: any) => + Boolean(isReadOnlyProp) || Boolean(dbConn?.readOnly || dbConn?.readonly); + + // Explicit false prop cannot downgrade a read-only connection + assert.strictEqual(computeEffectiveReadOnly(false, { readOnly: true }), true); + assert.strictEqual(computeEffectiveReadOnly(false, { readonly: true }), true); + + // Explicit true prop locks down even if connection claims writable + assert.strictEqual(computeEffectiveReadOnly(true, { readOnly: false }), true); + assert.strictEqual(computeEffectiveReadOnly(true, null), true); + + // Only writable when both prop and connection are non-readonly + assert.strictEqual(computeEffectiveReadOnly(false, { readOnly: false }), false); + }); + + it("9c. When isReadOnly = true, Import and Bulk Mutations are disabled; Export remains enabled", () => { + const vdom = renderViewerVdom({ + collection: "orders", + documents: [{ _id: "order-1", total: 100 }], + totalDocuments: 1, + page: 1, + pageSize: 50, + totalPages: 1, + isLoading: false, + sortField: null, + sortDirection: null, + onSort: () => {}, + onPageChange: () => {}, + onPageSizeChange: () => {}, + filterInput: "", + onFilterInputChange: () => {}, + onApplyFilter: () => {}, + onResetFilter: () => {}, + isFilterApplied: false, + onRefresh: () => {}, + onImportClick: () => {}, + onExportClick: () => {}, + onBulkMutationClick: () => {}, + isReadOnly: true, + }); + + const [importBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-import-button", + ); + const [exportBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-export-button", + ); + const [mutationBtn] = findVdomNodes( + vdom, + (n) => n.props?.["data-testid"] === "toolbar-bulk-mutation-button", + ); + + assert.strictEqual(importBtn.props.disabled, true, "Import must be disabled in read-only mode"); + assert.strictEqual( + mutationBtn.props.disabled, + true, + "Bulk Mutations must be disabled in read-only mode", + ); + assert.strictEqual( + exportBtn.props.disabled, + false, + "Export must remain enabled in read-only mode", + ); + }); + + it("9d. MongoDataViewPanel propagates effective read-only to MongoDocumentViewer and MongoBulkMutationDialog", () => { + const dataViewPanelPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/MongoDataViewPanel.tsx", + ); + const content = fs.readFileSync(dataViewPanelPath, "utf-8"); + + assert.ok( + content.includes("isEffectiveReadOnly"), + "MongoDataViewPanel must compute isEffectiveReadOnly", + ); + assert.ok( + content.includes("isReadOnly={isEffectiveReadOnly}"), + "MongoDataViewPanel must pass isEffectiveReadOnly to delegated components", + ); + }); + + // Scenario 10: Collection Prefill in Data View Dialogs + it("10. MongoDataViewPanel pre-fills defaultCollection with the active selected collection", () => { + const dataViewPanelPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/MongoDataViewPanel.tsx", + ); + const content = fs.readFileSync(dataViewPanelPath, "utf-8"); + + // Verify all 3 bulk dialogs are imported and receive defaultCollection={selectedCollection} + assert.ok(content.includes("MongoBulkImportDialog")); + assert.ok(content.includes("MongoBulkExportDialog")); + assert.ok(content.includes("MongoBulkMutationDialog")); + assert.ok(content.includes("defaultCollection={selectedCollection}")); + }); + + // Scenario 11: Background Execution & Unmount Survival with Single Poller Authority + it("11. Dialogs forward cancelOnUnmount={false} and useMongoBulkOperationsStore enforces single poller authority", async () => { + const dataViewPanelPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/MongoDataViewPanel.tsx", + ); + const content = fs.readFileSync(dataViewPanelPath, "utf-8"); + + // Verify cancelOnUnmount={false} is wired to delegated dialogs + assert.ok( + content.includes("cancelOnUnmount={false}"), + "MongoDataViewPanel must pass cancelOnUnmount={false} to delegated bulk dialogs", + ); + + // Verify store deduplicates pollers and maintains single poller authority + const store = createMongoBulkOperationsStore(); + const now = new Date().toISOString(); + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-unmount-survival-1", + dbId: "db-unmount", + collection: "users", + type: "export", + status: "in_progress", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-unmount", op); + const registered = store.getSnapshot().operationsByDb[ + "db-unmount" + ]?.[0] as TrackedAsyncBulkOperation; + assert.strictEqual(registered?.status, "in_progress"); + + // Starting poller multiple times does not create duplicates + store._startPoller("op-unmount-survival-1"); + store._startPoller("op-unmount-survival-1"); + + // Store tracks single authoritative entry + assert.strictEqual(store._activePollers.has("op-unmount-survival-1"), true); + assert.strictEqual(store._activePollers.size, 1); + assert.strictEqual(store.getSnapshot().operationsByDb["db-unmount"].length, 1); + store.getSnapshot().resetStore(); + }); + + // Scenario 12: Closing the mutation dialog while submitting does not trigger early refetch + it("12. Closing the mutation dialog while submitting does not trigger early refetch", () => { + let documentRefreshed = 0; + let collectionsRefreshed = 0; + + const _refetchDocuments = () => { + documentRefreshed++; + }; + const _refetchCollections = () => { + collectionsRefreshed++; + }; + + // In MongoDataViewPanel, onOpenChange is wired directly to state setter: onOpenChange={setIsBulkMutationOpen} + // Closing the dialog (open = false) while in flight does not trigger any refetch + let isBulkMutationOpen = true; + const onOpenChange = (open: boolean) => { + isBulkMutationOpen = open; + }; + + onOpenChange(false); + assert.strictEqual(isBulkMutationOpen, false); + assert.strictEqual( + documentRefreshed, + 0, + "Closing mutation dialog must not trigger document refetch", + ); + assert.strictEqual( + collectionsRefreshed, + 0, + "Closing mutation dialog must not trigger collection refetch", + ); + + // Source audit confirms no refetch inside onOpenChange of MongoBulkMutationDialog + const dataViewPanelPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/MongoDataViewPanel.tsx", + ); + const content = fs.readFileSync(dataViewPanelPath, "utf-8"); + assert.ok( + content.includes("onOpenChange={setIsBulkMutationOpen}"), + "MongoBulkMutationDialog onOpenChange must only toggle modal open state without triggering refetch", + ); + }); + + // Scenario 13: Terminal mutation outcome (completed and partial_success) triggers refetch + it("13. Terminal mutation outcome (completed and partial_success) triggers query cache invalidation; failed does not", () => { + const invalidatedKeys: string[][] = []; + const mockQueryClient = { + invalidateQueries: ({ queryKey }: { queryKey: any[] }) => { + invalidatedKeys.push(queryKey); + }, + }; + + const store = createMongoBulkOperationsStore(); + const targetDbId = "db-mutations-test"; + + // Set up the router listener pattern from DatabaseDetails.tsx + let prevOperationsByDb = store.getSnapshot().operationsByDb; + const unsubscribe = store.subscribe(() => { + const currentOperationsByDb = store.getSnapshot().operationsByDb; + for (const [dbIdKey, ops] of Object.entries(currentOperationsByDb)) { + const prevOps = prevOperationsByDb[dbIdKey] || []; + for (const op of ops) { + if (op.kind === "sync") { + const prev = prevOps.find((p) => p.kind === "sync" && p.id === op.id); + if ( + prev && + prev.status === "running" && + (op.status === "completed" || op.status === "partial_success") + ) { + mockQueryClient.invalidateQueries({ queryKey: ["mongoDocuments", op.dbId] }); + mockQueryClient.invalidateQueries({ queryKey: ["mongoCollections", op.dbId] }); + mockQueryClient.invalidateQueries({ queryKey: ["mongoSchema", op.dbId] }); + } + } + } + } + prevOperationsByDb = currentOperationsByDb; + }); + + // 1. Register running mutation + const mutation1: TrackedSyncBulkMutation = { + kind: "sync", + id: "mut-1", + dbId: targetDbId, + collection: "users", + type: "update", + status: "running", + timestamp: new Date().toISOString(), + }; + store.getSnapshot().registerSyncMutation(targetDbId, mutation1); + assert.strictEqual(invalidatedKeys.length, 0, "Running mutation must not trigger invalidation"); + + // 2. Transition to completed -> invalidates documents, collections, and schema + store.getSnapshot().updateSyncMutation("mut-1", { + status: "completed", + matchedCount: 5, + modifiedCount: 5, + }); + assert.strictEqual(invalidatedKeys.length, 3); + assert.deepStrictEqual(invalidatedKeys[0], ["mongoDocuments", targetDbId]); + assert.deepStrictEqual(invalidatedKeys[1], ["mongoCollections", targetDbId]); + assert.deepStrictEqual(invalidatedKeys[2], ["mongoSchema", targetDbId]); + + // 3. Register running mutation that finishes with partial_success + invalidatedKeys.length = 0; + const mutation2: TrackedSyncBulkMutation = { + kind: "sync", + id: "mut-2", + dbId: targetDbId, + collection: "users", + type: "insert", + status: "running", + timestamp: new Date().toISOString(), + }; + store.getSnapshot().registerSyncMutation(targetDbId, mutation2); + store.getSnapshot().updateSyncMutation("mut-2", { + status: "partial_success", + insertedCount: 3, + errors: [{ index: 4, code: "11000", message: "duplicate key" }], + }); + assert.strictEqual(invalidatedKeys.length, 3, "partial_success must trigger invalidation"); + + // 4. Register running mutation that fails -> does NOT trigger invalidation + invalidatedKeys.length = 0; + const mutation3: TrackedSyncBulkMutation = { + kind: "sync", + id: "mut-3", + dbId: targetDbId, + collection: "users", + type: "delete", + status: "running", + timestamp: new Date().toISOString(), + }; + store.getSnapshot().registerSyncMutation(targetDbId, mutation3); + store.getSnapshot().updateSyncMutation("mut-3", { + status: "failed", + error: "Network disconnect", + }); + assert.strictEqual(invalidatedKeys.length, 0, "failed mutation must NOT trigger invalidation"); + + unsubscribe(); + }); + + // Scenario 14: Invalidation via store transition triggers cache invalidation for target op.dbId even after unmount + it("14. Invalidation via store transition preserves captured op.dbId even after dialog/panel unmount", () => { + const targetDbId = "db-background-mutation"; + const invalidatedKeys: string[][] = []; + const mockQueryClient = { + invalidateQueries: ({ queryKey }: { queryKey: any[] }) => { + invalidatedKeys.push(queryKey); + }, + }; + + const store = createMongoBulkOperationsStore(); + + // Persistent router subscriber + let prevOperationsByDb = store.getSnapshot().operationsByDb; + const unsubscribe = store.subscribe(() => { + const currentOperationsByDb = store.getSnapshot().operationsByDb; + for (const [dbIdKey, ops] of Object.entries(currentOperationsByDb)) { + const prevOps = prevOperationsByDb[dbIdKey] || []; + for (const op of ops) { + if (op.kind === "sync") { + const prev = prevOps.find((p) => p.kind === "sync" && p.id === op.id); + if ( + prev && + prev.status === "running" && + (op.status === "completed" || op.status === "partial_success") + ) { + mockQueryClient.invalidateQueries({ queryKey: ["mongoDocuments", op.dbId] }); + } + } + } + } + prevOperationsByDb = currentOperationsByDb; + }); + + // Mutation starts on targetDbId + store.getSnapshot().registerSyncMutation(targetDbId, { + kind: "sync", + id: "mut-unmounted", + dbId: targetDbId, + collection: "audit_logs", + type: "delete", + status: "running", + timestamp: new Date().toISOString(), + }); + + // Dialog unmounts, panel unmounts, user navigates to another database "db-other" + const activeDatabaseInUI = "db-other"; + + // Mutation settles in background + store.getSnapshot().updateSyncMutation("mut-unmounted", { + status: "completed", + deletedCount: 42, + }); + + // Verify invalidation targeted op.dbId, NOT activeDatabaseInUI + assert.strictEqual(invalidatedKeys.length, 1); + assert.deepStrictEqual(invalidatedKeys[0], ["mongoDocuments", targetDbId]); + assert.notDeepStrictEqual(invalidatedKeys[0], ["mongoDocuments", activeDatabaseInUI]); + + unsubscribe(); + }); + + // Scenario 15: Multiple chunk acknowledgements do not trigger refetch + it("15. Multiple chunk acknowledgements do not trigger refetch; only track cumulative count", () => { + let documentRefreshed = 0; + const _refetchDocuments = () => { + documentRefreshed++; + }; + + // Implementation matching MongoDataViewPanel + const importCommittedCountRef = { current: 0 }; + const onChunkAcknowledged = (progress: { totalInserted: number }) => { + importCommittedCountRef.current = progress.totalInserted; + }; + + // Simulate 50 incoming chunks during large bulk import stream + for (let i = 1; i <= 50; i++) { + onChunkAcknowledged({ totalInserted: i * 100 }); + } + + assert.strictEqual( + importCommittedCountRef.current, + 5000, + "Cumulative inserted count must be tracked", + ); + assert.strictEqual( + documentRefreshed, + 0, + "Streaming chunk acknowledgements must never invoke refetchDocuments", + ); + }); + + // Scenario 16: Complete successful import callback sequence eliminates duplicate refresh; single refresh on completed, cancelled with docs, and failed with docs + it("16. Complete successful import callback sequence eliminates duplicate refresh; single refresh on completed, cancelled with docs, and failed with docs", () => { + let documentRefreshed = 0; + let collectionsRefreshed = 0; + const refetchDocuments = () => { + documentRefreshed++; + }; + const refetchCollections = () => { + collectionsRefreshed++; + }; + + // Actual wiring matching MongoDataViewPanel.tsx + const importCommittedCountRef = { current: 0 }; + let isBulkImportOpen = true; + const onOpenChange = (open: boolean) => { + isBulkImportOpen = open; + }; + + const onChunkAcknowledged = (progress: { totalInserted: number }) => { + importCommittedCountRef.current = progress.totalInserted; + }; + + const onTerminalState = (_opId: string, status: MongoBulkOperationStatus) => { + if (status === "completed" || importCommittedCountRef.current > 0) { + refetchDocuments(); + refetchCollections(); + importCommittedCountRef.current = 0; + } + }; + + // Dialog component wiring (MongoBulkImportDialog.tsx): + // In MongoBulkImportDialog, handleFinish executes: onImportComplete?.(); onOpenChange(false); + // Since MongoDataViewPanel does not pass onImportComplete, onImportComplete is undefined. + const dialogProps = { + open: isBulkImportOpen, + onOpenChange, + onChunkAcknowledged, + onTerminalState, + onImportComplete: undefined as (() => void) | undefined, + }; + + const simulateDialogFinish = () => { + dialogProps.onImportComplete?.(); + dialogProps.onOpenChange(false); + }; + + // 1. Simulate acknowledged import chunks + dialogProps.onChunkAcknowledged({ totalInserted: 100 }); + assert.strictEqual(documentRefreshed, 0, "Chunk 1 must not trigger refresh"); + assert.strictEqual(collectionsRefreshed, 0, "Chunk 1 must not trigger refresh"); + + dialogProps.onChunkAcknowledged({ totalInserted: 250 }); + assert.strictEqual(documentRefreshed, 0, "Chunk 2 must not trigger refresh"); + assert.strictEqual(collectionsRefreshed, 0, "Chunk 2 must not trigger refresh"); + assert.strictEqual(importCommittedCountRef.current, 250); + + // 2. Trigger onTerminalState with "completed" + dialogProps.onTerminalState("op-success-1", "completed"); + + // 3. Verify exactly one document refresh and one collection refresh + assert.strictEqual( + documentRefreshed, + 1, + "Terminal completed must trigger exactly 1 document refresh", + ); + assert.strictEqual( + collectionsRefreshed, + 1, + "Terminal completed must trigger exactly 1 collection refresh", + ); + assert.strictEqual( + importCommittedCountRef.current, + 0, + "Committed count ref must be reset to 0", + ); + + // 4. Simulate user clicking Finish on the summary screen and dialog closing + simulateDialogFinish(); + assert.strictEqual(isBulkImportOpen, false, "Dialog must close on Finish"); + + // 5. Verify NO second refresh occurs + assert.strictEqual( + documentRefreshed, + 1, + "Clicking Finish must not trigger a second document refresh", + ); + assert.strictEqual( + collectionsRefreshed, + 1, + "Clicking Finish must not trigger a second collection refresh", + ); + + // 6. Verify cancellation with committed chunks refreshes once and resets ref + isBulkImportOpen = true; + dialogProps.onChunkAcknowledged({ totalInserted: 80 }); + dialogProps.onTerminalState("op-cancel-1", "cancelled"); + assert.strictEqual( + documentRefreshed, + 2, + "Cancelled import with committed docs must trigger 1 refresh", + ); + assert.strictEqual(collectionsRefreshed, 2); + assert.strictEqual( + importCommittedCountRef.current, + 0, + "Ref must reset to 0 after cancellation refresh", + ); + + // 7. Verify failure with committed chunks refreshes once and resets ref + dialogProps.onChunkAcknowledged({ totalInserted: 40 }); + dialogProps.onTerminalState("op-fail-1", "failed"); + assert.strictEqual( + documentRefreshed, + 3, + "Failed import with committed docs must trigger 1 refresh", + ); + assert.strictEqual(collectionsRefreshed, 3); + assert.strictEqual( + importCommittedCountRef.current, + 0, + "Ref must reset to 0 after failure refresh", + ); + + // 8. Verify failure with zero committed documents does not refresh + dialogProps.onChunkAcknowledged({ totalInserted: 0 }); + dialogProps.onTerminalState("op-fail-zero", "failed"); + assert.strictEqual( + documentRefreshed, + 3, + "Failed import with 0 committed docs must NOT refresh", + ); + assert.strictEqual(collectionsRefreshed, 3); + + // 9. Verify cancellation with zero committed documents does not refresh + dialogProps.onTerminalState("op-cancel-zero", "cancelled"); + assert.strictEqual( + documentRefreshed, + 3, + "Cancelled import with 0 committed docs must NOT refresh", + ); + assert.strictEqual(collectionsRefreshed, 3); + + // 10. Source verification confirms onImportComplete is omitted from MongoBulkImportDialog + const dataViewPanelPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/MongoDataViewPanel.tsx", + ); + const content = fs.readFileSync(dataViewPanelPath, "utf-8"); + assert.ok( + !content.includes("onImportComplete="), + "MongoDataViewPanel must not pass onImportComplete to MongoBulkImportDialog", + ); + }); + + // Scenario 17: Export does not invalidate document data + it("17. Pure export operations never trigger document, collection, or schema cache invalidation", () => { + const invalidatedKeys: string[][] = []; + const mockQueryClient = { + invalidateQueries: ({ queryKey }: { queryKey: any[] }) => { + invalidatedKeys.push(queryKey); + }, + }; + + const store = createMongoBulkOperationsStore(); + const targetDbId = "db-export-test"; + + // Setup listener from DatabaseDetails.tsx + let prevOperationsByDb = store.getSnapshot().operationsByDb; + const unsubscribe = store.subscribe(() => { + const currentOperationsByDb = store.getSnapshot().operationsByDb; + for (const [dbIdKey, ops] of Object.entries(currentOperationsByDb)) { + const prevOps = prevOperationsByDb[dbIdKey] || []; + for (const op of ops) { + if (op.kind === "sync") { + const prev = prevOps.find((p) => p.kind === "sync" && p.id === op.id); + if ( + prev && + prev.status === "running" && + (op.status === "completed" || op.status === "partial_success") + ) { + mockQueryClient.invalidateQueries({ queryKey: ["mongoDocuments", op.dbId] }); + } + } else if (op.kind === "async" && op.type === "import") { + const prev = prevOps.find( + (p) => p.kind === "async" && p.operationId === op.operationId, + ); + const wasActive = prev && (prev.status === "pending" || prev.status === "in_progress"); + const isTerminal = + op.status === "completed" || op.status === "failed" || op.status === "cancelled"; + if (wasActive && isTerminal && op.successfulDocuments > 0) { + mockQueryClient.invalidateQueries({ queryKey: ["mongoDocuments", op.dbId] }); + } + } + } + } + prevOperationsByDb = currentOperationsByDb; + }); + + // Register and complete an export operation + const now = new Date().toISOString(); + store.getSnapshot().registerAsyncOperation(targetDbId, { + kind: "async", + operationId: "exp-1", + dbId: targetDbId, + collection: "products", + type: "export", + status: "in_progress", + processedDocuments: 500, + successfulDocuments: 500, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: new Date(Date.now() + 60000).toISOString(), + }); + + store.getSnapshot().updateAsyncOperation("exp-1", { + status: "completed", + }); + + assert.strictEqual( + invalidatedKeys.length, + 0, + "Export operations must never trigger cache invalidations", + ); + + unsubscribe(); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoBulkOperationsHistory.test.ts b/src/features/mongodb/__tests__/MongoBulkOperationsHistory.test.ts new file mode 100644 index 00000000..6da510f5 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoBulkOperationsHistory.test.ts @@ -0,0 +1,320 @@ +/** + * MongoDB Milestone 7 Phase 4 Stage 5 — History Table Component Tests + * + * Exercises: + * - Search filtering (by collection name and operationId) + * - Type dropdown filtering (all, import, export, insert, update, delete) + * - Status dropdown filtering (all, active, completed, failed, cancelled, stale) + * - Action buttons: + * - onCancel(id) for active operations + * - onCleanup(id) for export operations + * - onRetry(id) for unreachable async operations + * - onRemove(id) + * - onClearExpired() and disabled state when expiredCount === 0 + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { + MongoBulkOperationsHistory, + type MongoBulkOperationsHistoryProps, +} from "../components/bulk/MongoBulkOperationsHistory"; +import type { BulkHistoryEntry } from "../types/bulk"; + +function renderWithFilters( + props: MongoBulkOperationsHistoryProps, + filters: { search?: string; type?: string; status?: string } = {}, +): string { + const internals = (React as any).__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + function TestWrapper() { + const origUseState = internals.H.useState; + let stateIndex = 0; + internals.H.useState = (initial: any) => { + stateIndex++; + if (stateIndex === 1) return [filters.search ?? initial, () => {}]; + if (stateIndex === 2) return [filters.type ?? initial, () => {}]; + if (stateIndex === 3) return [filters.status ?? initial, () => {}]; + return [initial, () => {}]; + }; + try { + return MongoBulkOperationsHistory(props); + } finally { + internals.H.useState = origUseState; + } + } + + return ReactDOMServer.renderToStaticMarkup(React.createElement(TestWrapper)); +} + +function findVdomNodes(node: any, predicate: (node: any) => boolean): any[] { + const matches: any[] = []; + if (!node || typeof node !== "object") return matches; + if (predicate(node)) { + matches.push(node); + } + if (Array.isArray(node.props?.children)) { + for (const child of node.props.children) { + matches.push(...findVdomNodes(child, predicate)); + } + } else if (node.props?.children) { + matches.push(...findVdomNodes(node.props.children, predicate)); + } + return matches; +} + +const sampleOperations: BulkHistoryEntry[] = [ + { + kind: "async", + operationId: "op-import-active", + dbId: "db-1", + collection: "orders", + type: "import", + status: "in_progress", + processedDocuments: 100, + successfulDocuments: 90, + failedDocuments: 10, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }, + { + kind: "async", + operationId: "op-export-done", + dbId: "db-1", + collection: "customers", + type: "export", + status: "completed", + processedDocuments: 500, + successfulDocuments: 500, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }, + { + kind: "async", + operationId: "op-unreachable-failed", + dbId: "db-1", + collection: "analytics", + type: "export", + status: "failed", + isUnreachable: true, + processedDocuments: 20, + successfulDocuments: 0, + failedDocuments: 20, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }, + { + kind: "sync", + id: "sync-mutation-1", + dbId: "db-1", + collection: "inventory", + type: "insert", + status: "completed", + timestamp: new Date().toISOString(), + insertedCount: 42, + }, +]; + +describe("MongoBulkOperationsHistory Component Tests", () => { + it("renders status badges and operation details", () => { + const html = renderWithFilters({ + operations: sampleOperations, + expiredCount: 0, + }); + + assert.ok(html.includes("orders")); + assert.ok(html.includes("customers")); + assert.ok(html.includes("in_progress")); + assert.ok(html.includes("completed")); + }); + + it("renders empty state message when operation list is empty", () => { + const html = renderWithFilters({ + operations: [], + expiredCount: 0, + }); + + assert.ok(html.includes("No bulk operations found")); + }); + + it("displays expired count in Clear Expired button", () => { + const html = renderWithFilters({ + operations: [], + onClearExpired: () => {}, + expiredCount: 3, + }); + + assert.ok(html.includes("Clear Expired (3)")); + }); + + it("search filtering filters rows by collection name and operationId", () => { + // Filter by collection name 'orders' + const htmlCol = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { search: "orders" }, + ); + assert.ok(htmlCol.includes("orders")); + assert.ok(!htmlCol.includes("customers")); + assert.ok(!htmlCol.includes("analytics")); + + // Filter by operationId 'op-export-done' + const htmlId = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { search: "op-export-done" }, + ); + assert.ok(htmlId.includes("customers")); + assert.ok(!htmlId.includes("orders")); + assert.ok(!htmlId.includes("analytics")); + }); + + it("type dropdown filtering filters rows by type", () => { + // Type: import + const htmlImport = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { type: "import" }, + ); + assert.ok(htmlImport.includes("orders")); + assert.ok(!htmlImport.includes("customers")); + assert.ok(!htmlImport.includes("inventory")); + + // Type: export + const htmlExport = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { type: "export" }, + ); + assert.ok(htmlExport.includes("customers")); + assert.ok(htmlExport.includes("analytics")); + assert.ok(!htmlExport.includes("orders")); + + // Type: insert + const htmlInsert = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { type: "insert" }, + ); + assert.ok(htmlInsert.includes("inventory")); + assert.ok(!htmlInsert.includes("orders")); + assert.ok(!htmlInsert.includes("customers")); + }); + + it("status dropdown filtering filters rows by status", () => { + // Status: active (matches pending, in_progress, running) + const htmlActive = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { status: "active" }, + ); + assert.ok(htmlActive.includes("orders")); // in_progress + assert.ok(!htmlActive.includes("customers")); // completed + assert.ok(!htmlActive.includes("analytics")); // failed + + // Status: completed + const htmlCompleted = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { status: "completed" }, + ); + assert.ok(htmlCompleted.includes("customers")); + assert.ok(!htmlCompleted.includes("orders")); + + // Status: failed + const htmlFailed = renderWithFilters( + { operations: sampleOperations, expiredCount: 0 }, + { status: "failed" }, + ); + assert.ok(htmlFailed.includes("analytics")); + assert.ok(!htmlFailed.includes("customers")); + }); + + it("action buttons invoke onCancel, onCleanup, onRetry, and onRemove callbacks", () => { + const cancelled: string[] = []; + const cleaned: string[] = []; + const retried: string[] = []; + const removed: string[] = []; + + let vdom: any; + function Wrapper() { + vdom = MongoBulkOperationsHistory({ + operations: sampleOperations, + onCancel: (id) => cancelled.push(id), + onCleanup: (id) => cleaned.push(id), + onRetry: (id) => retried.push(id), + onRemove: (id) => removed.push(id), + }); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(Wrapper)); + + // Find and invoke Cancel button (active operation: op-import-active) + const cancelBtns = findVdomNodes(vdom, (n) => n.props?.title === "Cancel"); + assert.ok(cancelBtns.length > 0); + cancelBtns[0].props.onClick(); + assert.deepEqual(cancelled, ["op-import-active"]); + + // Find and invoke Clean temporary cache button (export operations) + const cleanupBtns = findVdomNodes(vdom, (n) => n.props?.title === "Clean temporary cache"); + assert.ok(cleanupBtns.length > 0); + cleanupBtns[0].props.onClick(); + assert.deepEqual(cleaned, ["op-export-done"]); + + // Find and invoke Retry connection button (unreachable operation: op-unreachable-failed) + const retryBtns = findVdomNodes(vdom, (n) => n.props?.title === "Retry connection"); + assert.ok(retryBtns.length > 0); + retryBtns[0].props.onClick(); + assert.deepEqual(retried, ["op-unreachable-failed"]); + + // Find and invoke Remove buttons + const removeBtns = findVdomNodes(vdom, (n) => n.props?.title === "Remove from history"); + assert.ok(removeBtns.length > 0); + removeBtns[0].props.onClick(); + assert.equal(removed.length, 1); + }); + + it("clear expired button invokes onClearExpired and is disabled when expiredCount === 0", () => { + let clearExpiredCalled = false; + + // Test with expiredCount > 0 + let vdomEnabled: any; + function WrapperEnabled() { + vdomEnabled = MongoBulkOperationsHistory({ + operations: [], + expiredCount: 4, + onClearExpired: () => { + clearExpiredCalled = true; + }, + }); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(WrapperEnabled)); + + const clearBtn = findVdomNodes( + vdomEnabled, + (n) => n.props?.["data-testid"] === "clear-expired-button", + ); + assert.equal(clearBtn.length, 1); + assert.equal(clearBtn[0].props.disabled, false); + clearBtn[0].props.onClick(); + assert.equal(clearExpiredCalled, true); + + // Test with expiredCount === 0 + let vdomDisabled: any; + function WrapperDisabled() { + vdomDisabled = MongoBulkOperationsHistory({ + operations: [], + expiredCount: 0, + onClearExpired: () => {}, + }); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(WrapperDisabled)); + + const disabledBtn = findVdomNodes( + vdomDisabled, + (n) => n.props?.["data-testid"] === "clear-expired-button", + ); + assert.equal(disabledBtn.length, 1); + assert.equal(disabledBtn[0].props.disabled, true); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoBulkOperationsPanel.test.ts b/src/features/mongodb/__tests__/MongoBulkOperationsPanel.test.ts new file mode 100644 index 00000000..0e25faef --- /dev/null +++ b/src/features/mongodb/__tests__/MongoBulkOperationsPanel.test.ts @@ -0,0 +1,265 @@ +/** + * MongoDB Milestone 7 Phase 4 Stage 5 — Panel Component Tests + * + * Exercises: + * - Panel header, metric cards, and launch buttons + * - Modal triggers identifiers (panel-import-button, panel-export-button) + * - Metric calculations (Total, Active, Completed, Failed / Cancelled) + * - Active operations highlight card rendering and conditional visibility + * - Delegated modal prop verification (cancelOnUnmount: false, externalPolling: true) + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { MongoBulkOperationsPanel } from "../components/bulk/MongoBulkOperationsPanel"; +import { defaultMongoBulkOperationsStore } from "../hooks/useMongoBulkOperationsStore"; +import type { TrackedAsyncBulkOperation, TrackedSyncBulkMutation } from "../types/bulk"; + +function findVdomNodes(node: any, predicate: (node: any) => boolean): any[] { + const matches: any[] = []; + if (!node || typeof node !== "object") return matches; + if (predicate(node)) { + matches.push(node); + } + if (Array.isArray(node.props?.children)) { + for (const child of node.props.children) { + matches.push(...findVdomNodes(child, predicate)); + } + } else if (node.props?.children) { + matches.push(...findVdomNodes(node.props.children, predicate)); + } + return matches; +} + +describe("MongoBulkOperationsPanel Component Tests", () => { + it("renders panel header, metric cards, and launch buttons with proper data-testids", () => { + defaultMongoBulkOperationsStore.getSnapshot().resetStore(); + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoBulkOperationsPanel, { + dbId: "db-demo", + databaseName: "DemoDatabase", + activeCollection: "users", + }), + ); + + assert.ok(html.includes("Bulk Operations")); + assert.ok(html.includes("DemoDatabase")); + assert.ok(html.includes("Import Documents")); + assert.ok(html.includes("Export Documents")); + assert.ok(html.includes('data-testid="panel-import-button"')); + assert.ok(html.includes('data-testid="panel-export-button"')); + assert.ok(html.includes("Total Operations")); + assert.ok(html.includes("Active")); + assert.ok(html.includes("Completed")); + assert.ok(html.includes("Failed / Cancelled")); + }); + + it("calculates summary metric cards accurately from store operations", () => { + defaultMongoBulkOperationsStore.getSnapshot().resetStore(); + + const now = new Date().toISOString(); + const expiry = new Date(Date.now() + 3600000).toISOString(); + + // 1 pending async op + const op1: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-pending", + dbId: "db-calc", + collection: "col1", + type: "import", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: expiry, + }; + // 1 in_progress async op + const op2: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-active", + dbId: "db-calc", + collection: "col2", + type: "export", + status: "in_progress", + processedDocuments: 50, + successfulDocuments: 50, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: expiry, + }; + // 1 completed async op + const op3: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-completed", + dbId: "db-calc", + collection: "col3", + type: "export", + status: "completed", + processedDocuments: 100, + successfulDocuments: 100, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: expiry, + }; + // 1 failed async op + const op4: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-failed", + dbId: "db-calc", + collection: "col4", + type: "export", + status: "failed", + processedDocuments: 10, + successfulDocuments: 0, + failedDocuments: 10, + createdAt: now, + updatedAt: now, + expiresAt: expiry, + }; + // 1 cancelled async op + const op5: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-cancelled", + dbId: "db-calc", + collection: "col5", + type: "import", + status: "cancelled", + processedDocuments: 20, + successfulDocuments: 20, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: expiry, + }; + // 1 stale async op + const op6: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-stale", + dbId: "db-calc", + collection: "col6", + type: "export", + status: "stale", + processedDocuments: 5, + successfulDocuments: 5, + failedDocuments: 0, + createdAt: now, + updatedAt: now, + expiresAt: expiry, + }; + // 1 running sync mutation + const op7: TrackedSyncBulkMutation = { + kind: "sync", + id: "sync-running", + dbId: "db-calc", + collection: "col7", + type: "insert", + status: "running", + timestamp: now, + }; + + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-calc", op1); + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-calc", op2); + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-calc", op3); + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-calc", op4); + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-calc", op5); + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-calc", op6); + defaultMongoBulkOperationsStore.getSnapshot().registerSyncMutation("db-calc", op7); + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoBulkOperationsPanel, { + dbId: "db-calc", + databaseName: "CalcDatabase", + activeCollection: "col1", + }), + ); + + // Total = 7 + assert.ok(html.includes('data-testid="metric-total">7')); + // Active = op-pending + op-active + sync-running = 3 + assert.ok(html.includes('data-testid="metric-active">3')); + // Completed = op-completed = 1 + assert.ok(html.includes('data-testid="metric-completed">1')); + // Failed/Cancelled = op-failed + op-cancelled + op-stale = 3 + assert.ok(html.includes('data-testid="metric-failed">3')); + + defaultMongoBulkOperationsStore.getSnapshot().resetStore(); + }); + + it("renders active operations highlight when active operations exist and hides section when none", () => { + defaultMongoBulkOperationsStore.getSnapshot().resetStore(); + + // Case 1: No active operations + const htmlEmpty = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoBulkOperationsPanel, { + dbId: "db-highlight", + databaseName: "EmptyDatabase", + }), + ); + assert.ok(!htmlEmpty.includes('data-testid="active-operations-highlight"')); + + // Case 2: Active operation present + const activeOp: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-active-highlight", + dbId: "db-highlight", + collection: "orders", + type: "import", + status: "in_progress", + processedDocuments: 200, + successfulDocuments: 150, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + defaultMongoBulkOperationsStore.getSnapshot().registerAsyncOperation("db-highlight", activeOp); + + const htmlActive = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoBulkOperationsPanel, { + dbId: "db-highlight", + databaseName: "ActiveDatabase", + }), + ); + assert.ok(htmlActive.includes('data-testid="active-operations-highlight"')); + assert.ok(htmlActive.includes("Active Operations (1)")); + + defaultMongoBulkOperationsStore.getSnapshot().resetStore(); + }); + + it("passes correct Stage 5 delegated props to import and export dialogs", () => { + let capturedVdom: any; + function Wrapper() { + capturedVdom = MongoBulkOperationsPanel({ + dbId: "db-delegated-props", + databaseName: "PropDatabase", + activeCollection: "inventory", + }); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(Wrapper)); + + // Find import dialog and verify delegated cancelOnUnmount: false + const importDialogs = findVdomNodes( + capturedVdom, + (n) => n.type?.name === "MongoBulkImportDialog", + ); + assert.equal(importDialogs.length, 1); + assert.equal(importDialogs[0].props.cancelOnUnmount, false); + + // Find export dialog and verify delegated cancelOnUnmount: false and externalPolling: true + const exportDialogs = findVdomNodes( + capturedVdom, + (n) => n.type?.name === "MongoBulkExportDialog", + ); + assert.equal(exportDialogs.length, 1); + assert.equal(exportDialogs[0].props.cancelOnUnmount, false); + assert.equal(exportDialogs[0].props.externalPolling, true); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoDocumentCell.test.ts b/src/features/mongodb/__tests__/MongoDocumentCell.test.ts new file mode 100644 index 00000000..a986ac24 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoDocumentCell.test.ts @@ -0,0 +1,113 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { formatMongoValue } from "../components/MongoDocumentCell"; + +function expect(actual: any) { + return { + toBe(expected: any) { + assert.strictEqual(actual, expected); + }, + toEqual(expected: any) { + assert.deepStrictEqual(actual, expected); + }, + toBeNull() { + assert.strictEqual(actual, null); + }, + toBeUndefined() { + assert.strictEqual(actual, undefined); + }, + toBeDefined() { + assert.notStrictEqual(actual, undefined); + }, + toBeTruthy() { + assert.ok(actual); + }, + toBeFalsy() { + assert.ok(!actual); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && actual.includes(substr), + `Expected "${actual}" to contain "${substr}"`, + ); + }, + not: { + toBeNull() { + assert.notStrictEqual(actual, null); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && !actual.includes(substr), + `Expected "${actual}" not to contain "${substr}"`, + ); + }, + }, + }; +} + +describe("MongoDocumentCell formatting & BSON unwrapping", () => { + it("formats ObjectId into clean hex string without leaking $oid", () => { + const hex = "507f1f77bcf86cd799439011"; + const value = { $oid: hex }; + const res = formatMongoValue(value); + + expect(res.type).toBe("objectId"); + expect(res.display).toBe(hex); + expect(res.display).not.toContain("$oid"); + expect(res.badgeText).toBe("OID"); + }); + + it("formats Date ISO string without leaking $date", () => { + const iso = "2024-03-24T12:00:00.000Z"; + const value = { $date: iso }; + const res = formatMongoValue(value); + + expect(res.type).toBe("date"); + expect(res.display).not.toContain("$date"); + expect(res.badgeText).toBe("DATE"); + }); + + it("formats Date with $numberLong ms without leaking $date or $numberLong", () => { + const value = { $date: { $numberLong: "1711281600000" } }; + const res = formatMongoValue(value); + + expect(res.type).toBe("date"); + expect(res.display).not.toContain("$date"); + expect(res.display).not.toContain("$numberLong"); + }); + + it("formats $numberLong into digits without leaking $numberLong", () => { + const value = { $numberLong: "9007199254740991" }; + const res = formatMongoValue(value); + + expect(res.type).toBe("long"); + expect(res.display).toBe("9007199254740991"); + expect(res.display).not.toContain("$numberLong"); + expect(res.badgeText).toBe("LONG"); + }); + + it("formats $numberDecimal into digits without leaking $numberDecimal", () => { + const value = { $numberDecimal: "123.456" }; + const res = formatMongoValue(value); + + expect(res.type).toBe("decimal"); + expect(res.display).toBe("123.456"); + expect(res.display).not.toContain("$numberDecimal"); + }); + + it("handles null and boolean properly", () => { + expect(formatMongoValue(null).display).toBe("NULL"); + expect(formatMongoValue(true).display).toBe("true"); + expect(formatMongoValue(false).display).toBe("false"); + }); + + it("marks arrays and objects as expandable with item counts", () => { + const arr = formatMongoValue([1, 2, 3]); + expect(arr.isExpandable).toBe(true); + expect(arr.badgeText).toBe("[3]"); + + const obj = formatMongoValue({ a: 1, b: 2 }); + expect(obj.isExpandable).toBe(true); + expect(obj.badgeText).toBe("{2}"); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoFilterInput.test.ts b/src/features/mongodb/__tests__/MongoFilterInput.test.ts new file mode 100644 index 00000000..16f2210e --- /dev/null +++ b/src/features/mongodb/__tests__/MongoFilterInput.test.ts @@ -0,0 +1,79 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { validateMongoFilter } from "../components/MongoFilterInput"; + +function expect(actual: any) { + return { + toBe(expected: any) { + assert.strictEqual(actual, expected); + }, + toEqual(expected: any) { + assert.deepStrictEqual(actual, expected); + }, + toBeNull() { + assert.strictEqual(actual, null); + }, + toBeUndefined() { + assert.strictEqual(actual, undefined); + }, + toBeDefined() { + assert.notStrictEqual(actual, undefined); + }, + toBeTruthy() { + assert.ok(actual); + }, + toBeFalsy() { + assert.ok(!actual); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && actual.includes(substr), + `Expected "${actual}" to contain "${substr}"`, + ); + }, + not: { + toBeNull() { + assert.notStrictEqual(actual, null); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && !actual.includes(substr), + `Expected "${actual}" not to contain "${substr}"`, + ); + }, + }, + }; +} + +describe("MongoFilterInput EJSON validation", () => { + it("treats empty string as valid (no filter)", () => { + const res = validateMongoFilter(""); + expect(res.isValid).toBe(true); + expect(res.parsed).toBeUndefined(); + }); + + it("validates standard JSON filter object", () => { + const res = validateMongoFilter('{"status": "active", "count": 10}'); + expect(res.isValid).toBe(true); + expect(res.parsed).toEqual({ status: "active", count: 10 }); + }); + + it("validates Extended-JSON with $oid and $gte", () => { + const filter = '{"_id": {"$oid": "507f1f77bcf86cd799439011"}, "age": {"$gte": 21}}'; + const res = validateMongoFilter(filter); + expect(res.isValid).toBe(true); + expect(res.parsed?._id?.$oid).toBe("507f1f77bcf86cd799439011"); + }); + + it("rejects invalid JSON syntax", () => { + const res = validateMongoFilter('{status: "active"}'); + expect(res.isValid).toBe(false); + expect(res.error).toBeDefined(); + }); + + it("rejects JSON arrays because filters must be objects", () => { + const res = validateMongoFilter("[1, 2, 3]"); + expect(res.isValid).toBe(false); + expect(res.error).toContain("must be a JSON object"); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoIndexManagement.test.ts b/src/features/mongodb/__tests__/MongoIndexManagement.test.ts new file mode 100644 index 00000000..1f7f75df --- /dev/null +++ b/src/features/mongodb/__tests__/MongoIndexManagement.test.ts @@ -0,0 +1,434 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + formatBytes, + formatDirection, + generateIndexName, + validateIndexSpec, + IndexKeyField, +} from "../utils/indexHelpers"; +import { databaseBridge } from "@/services/bridge/databaseBridge"; +import { databaseService } from "@/services/bridge/database"; + +describe("MongoDB Index Management — Phase 2 Frontend Tests", () => { + // ========================================== + // 1. formatBytes tests + // ========================================== + describe("formatBytes", () => { + it("handles 0, null, undefined, NaN, and negative values cleanly", () => { + assert.strictEqual(formatBytes(0), "0 B"); + assert.strictEqual(formatBytes(undefined), "0 B"); + assert.strictEqual(formatBytes(null as any), "0 B"); + assert.strictEqual(formatBytes(NaN), "0 B"); + assert.strictEqual(formatBytes(-100), "0 B"); + }); + + it("formats small byte values without decimal places", () => { + assert.strictEqual(formatBytes(512), "512 B"); + assert.strictEqual(formatBytes(1023), "1023 B"); + }); + + it("formats kilobyte values with 1 decimal place", () => { + assert.strictEqual(formatBytes(1024), "1.0 KB"); + assert.strictEqual(formatBytes(4096), "4.0 KB"); + assert.strictEqual(formatBytes(16384), "16.0 KB"); + }); + + it("formats megabyte values correctly", () => { + assert.strictEqual(formatBytes(1048576), "1.0 MB"); + assert.strictEqual(formatBytes(1258291), "1.2 MB"); + assert.strictEqual(formatBytes(67108864), "64.0 MB"); + }); + + it("formats gigabyte and terabyte values correctly", () => { + assert.strictEqual(formatBytes(1073741824), "1.0 GB"); + assert.strictEqual(formatBytes(5368709120), "5.0 GB"); + assert.strictEqual(formatBytes(1099511627776), "1.0 TB"); + }); + }); + + // ========================================== + // 2. formatDirection tests + // ========================================== + describe("formatDirection", () => { + it("formats all supported MongoDB index directions with clear labels", () => { + assert.strictEqual(formatDirection(1), "Ascending (1)"); + assert.strictEqual(formatDirection(-1), "Descending (-1)"); + assert.strictEqual(formatDirection("text"), "Text"); + assert.strictEqual(formatDirection("2dsphere"), "2dsphere (Geo)"); + assert.strictEqual(formatDirection("hashed"), "Hashed"); + }); + + it("falls back to string representation for unexpected input", () => { + assert.strictEqual(formatDirection("custom" as any), "custom"); + }); + }); + + // ========================================== + // 3. generateIndexName tests + // ========================================== + describe("generateIndexName", () => { + it("generates correct canonical name for single-field ascending and descending keys", () => { + assert.strictEqual(generateIndexName([{ field: "email", direction: 1 }]), "email_1"); + assert.strictEqual( + generateIndexName([{ field: "createdAt", direction: -1 }]), + "createdAt_-1", + ); + }); + + it("generates correct canonical name for compound keys", () => { + const compoundFields: IndexKeyField[] = [ + { field: "tenantId", direction: 1 }, + { field: "status", direction: 1 }, + { field: "createdAt", direction: -1 }, + ]; + assert.strictEqual(generateIndexName(compoundFields), "tenantId_1_status_1_createdAt_-1"); + }); + + it("generates correct name for special types (text, 2dsphere, hashed)", () => { + assert.strictEqual( + generateIndexName([{ field: "description", direction: "text" }]), + "description_text", + ); + assert.strictEqual( + generateIndexName([{ field: "location", direction: "2dsphere" }]), + "location_2dsphere", + ); + assert.strictEqual( + generateIndexName([{ field: "userId", direction: "hashed" }]), + "userId_hashed", + ); + }); + + it("trims whitespace from field names", () => { + assert.strictEqual( + generateIndexName([ + { field: " username ", direction: 1 }, + { field: " score ", direction: -1 }, + ]), + "username_1_score_-1", + ); + }); + + it("ignores empty or whitespace-only fields and returns empty string if no valid fields", () => { + assert.strictEqual(generateIndexName([]), ""); + assert.strictEqual( + generateIndexName([ + { field: "", direction: 1 }, + { field: " ", direction: -1 }, + ]), + "", + ); + assert.strictEqual( + generateIndexName([ + { field: "", direction: 1 }, + { field: "validField", direction: 1 }, + ]), + "validField_1", + ); + }); + }); + + // ========================================== + // 4. validateIndexSpec tests + // ========================================== + describe("validateIndexSpec", () => { + it("accepts valid single-field and compound specifications", () => { + assert.strictEqual(validateIndexSpec([{ field: "username", direction: 1 }]).valid, true); + assert.strictEqual( + validateIndexSpec([ + { field: "orgId", direction: 1 }, + { field: "createdAt", direction: -1 }, + ]).valid, + true, + ); + }); + + it("rejects empty field array or missing input", () => { + assert.strictEqual(validateIndexSpec([]).valid, false); + assert.strictEqual(validateIndexSpec(null as any).valid, false); + }); + + it("rejects empty or whitespace-only field names", () => { + const res1 = validateIndexSpec([{ field: "", direction: 1 }]); + assert.strictEqual(res1.valid, false); + assert.strictEqual(res1.error, "All field names must be non-empty"); + + const res2 = validateIndexSpec([{ field: " ", direction: 1 }]); + assert.strictEqual(res2.valid, false); + assert.strictEqual(res2.error, "All field names must be non-empty"); + }); + + it("rejects duplicate field paths in compound keys", () => { + const res = validateIndexSpec([ + { field: "email", direction: 1 }, + { field: "email", direction: -1 }, + ]); + assert.strictEqual(res.valid, false); + assert.ok(res.error?.includes('Duplicate field path: "email"')); + }); + + it("rejects duplicate field paths with surrounding whitespace", () => { + const res = validateIndexSpec([ + { field: "userId", direction: 1 }, + { field: " userId ", direction: -1 }, + ]); + assert.strictEqual(res.valid, false); + assert.ok(res.error?.includes('Duplicate field path: "userId"')); + }); + + it("rejects unique constraint on hashed indexes", () => { + const res = validateIndexSpec([{ field: "userId", direction: "hashed" }], { unique: true }); + assert.strictEqual(res.valid, false); + assert.strictEqual(res.error, "Hashed indexes do not support unique constraints"); + }); + + it("rejects sparse option on hashed indexes", () => { + const res = validateIndexSpec([{ field: "userId", direction: "hashed" }], { sparse: true }); + assert.strictEqual(res.valid, false); + assert.strictEqual(res.error, "Hashed indexes do not support sparse option"); + }); + + it("rejects combining hashed field with other hashed, text, or geo fields", () => { + const multiHashed = validateIndexSpec([ + { field: "userA", direction: "hashed" }, + { field: "userB", direction: "hashed" }, + ]); + assert.strictEqual(multiHashed.valid, false); + assert.strictEqual( + multiHashed.error, + "Hashed index field cannot be combined with other hashed, text, or geospatial fields", + ); + + const hashedWithGeo = validateIndexSpec([ + { field: "userA", direction: "hashed" }, + { field: "loc", direction: "2dsphere" }, + ]); + assert.strictEqual(hashedWithGeo.valid, false); + + const hashedWithText = validateIndexSpec([ + { field: "userA", direction: "hashed" }, + { field: "bio", direction: "text" }, + ]); + assert.strictEqual(hashedWithText.valid, false); + }); + + it("rejects unique constraint on 2dsphere geospatial indexes", () => { + const res = validateIndexSpec([{ field: "location", direction: "2dsphere" }], { + unique: true, + }); + assert.strictEqual(res.valid, false); + assert.strictEqual(res.error, "2dsphere indexes do not support unique constraints"); + }); + + it("validates TTL expireAfterSeconds to be a non-negative integer", () => { + assert.strictEqual( + validateIndexSpec([{ field: "createdAt", direction: 1 }], { + expireAfterSeconds: -1, + }).valid, + false, + ); + + assert.strictEqual( + validateIndexSpec([{ field: "createdAt", direction: 1 }], { + expireAfterSeconds: 30.5, + }).valid, + false, + ); + + assert.strictEqual( + validateIndexSpec([{ field: "createdAt", direction: 1 }], { + expireAfterSeconds: 0, + }).valid, + true, + ); + + assert.strictEqual( + validateIndexSpec([{ field: "createdAt", direction: 1 }], { + expireAfterSeconds: 86400, + }).valid, + true, + ); + }); + + it("validates partialFilterExpression to be a valid JSON object", () => { + assert.strictEqual( + validateIndexSpec([{ field: "score", direction: 1 }], { + partialFilterExpression: { rating: { $gt: 5 } }, + }).valid, + true, + ); + + assert.strictEqual( + validateIndexSpec([{ field: "score", direction: 1 }], { + partialFilterExpression: [1, 2, 3] as any, + }).valid, + false, + ); + + assert.strictEqual( + validateIndexSpec([{ field: "score", direction: 1 }], { + partialFilterExpression: "invalid" as any, + }).valid, + false, + ); + }); + + it("validates collation locale when collation is provided", () => { + assert.strictEqual( + validateIndexSpec([{ field: "name", direction: 1 }], { + collation: { locale: "" }, + }).valid, + false, + ); + + assert.strictEqual( + validateIndexSpec([{ field: "name", direction: 1 }], { + collation: { locale: "en", strength: 2 }, + }).valid, + true, + ); + }); + }); + + // ========================================== + // 5. _id_ protection logic tests + // ========================================== + describe("_id_ Primary Key Protection", () => { + it("guards and rejects dropping the immutable _id_ index", () => { + const dropSafeguard = (indexName: string, isIdIndex?: boolean) => { + if (indexName === "_id_" || isIdIndex) { + throw new Error("The default _id_ index is required by MongoDB and cannot be dropped."); + } + return true; + }; + + assert.throws( + () => dropSafeguard("_id_"), + /The default _id_ index is required by MongoDB and cannot be dropped/, + ); + assert.throws( + () => dropSafeguard("custom_id_name", true), + /The default _id_ index is required by MongoDB and cannot be dropped/, + ); + assert.strictEqual(dropSafeguard("users_email_1"), true); + }); + + it("guards and rejects hiding the immutable _id_ index", () => { + const hideSafeguard = (indexName: string, isIdIndex?: boolean) => { + if (indexName === "_id_" || isIdIndex) { + throw new Error("The default _id_ index cannot be hidden."); + } + return true; + }; + + assert.throws(() => hideSafeguard("_id_"), /The default _id_ index cannot be hidden/); + assert.throws( + () => hideSafeguard("my_id_idx", true), + /The default _id_ index cannot be hidden/, + ); + assert.strictEqual(hideSafeguard("users_email_1"), true); + }); + }); + + // ========================================== + // 6. Redundancy advisory presentation logic + // ========================================== + describe("Redundancy Advisory Presentation", () => { + it("formats redundancy advisory with required non-destructive disclaimer", () => { + const disclaimer = "Advisory only; RelWave never drops indexes automatically."; + const sampleRedundantIndex = { + name: "user_id_1", + key: { user_id: 1 as const }, + sizeBytes: 8192, + unique: false, + sparse: false, + isIdIndex: false, + isRedundant: true, + redundantReason: 'Prefix is completely covered by compound index "user_id_1_createdAt_-1".', + }; + + assert.strictEqual(sampleRedundantIndex.isRedundant, true); + assert.ok(sampleRedundantIndex.redundantReason.includes("Prefix is completely covered")); + assert.strictEqual(disclaimer, "Advisory only; RelWave never drops indexes automatically."); + }); + }); + + // ========================================== + // 7. Bridge Service Contract Tests + // ========================================== + describe("Bridge Service Contract Signatures", () => { + it("verifies DatabaseBridge exports all 5 index methods", () => { + assert.strictEqual(typeof databaseBridge.listIndexes, "function"); + assert.strictEqual(typeof databaseBridge.createIndex, "function"); + assert.strictEqual(typeof databaseBridge.dropIndex, "function"); + assert.strictEqual(typeof databaseBridge.getIndexStats, "function"); + assert.strictEqual(typeof databaseBridge.hideIndex, "function"); + }); + + it("verifies databaseService exports all 5 mongo index methods", () => { + assert.strictEqual(typeof databaseService.listMongoIndexes, "function"); + assert.strictEqual(typeof databaseService.createMongoIndex, "function"); + assert.strictEqual(typeof databaseService.dropMongoIndex, "function"); + assert.strictEqual(typeof databaseService.getMongoIndexStats, "function"); + assert.strictEqual(typeof databaseService.hideMongoIndex, "function"); + }); + + it("delegates databaseService calls to databaseBridge properly", async () => { + const originalList = databaseBridge.listIndexes; + const originalCreate = databaseBridge.createIndex; + const originalDrop = databaseBridge.dropIndex; + const originalStats = databaseBridge.getIndexStats; + const originalHide = databaseBridge.hideIndex; + + let calledMethod = ""; + try { + databaseBridge.listIndexes = async (dbId, col) => { + calledMethod = `listIndexes:${dbId}:${col}`; + return { collection: col, indexes: [], totalSizeBytes: 0 }; + }; + const listRes = await databaseService.listMongoIndexes("db1", "users"); + assert.strictEqual(calledMethod, "listIndexes:db1:users"); + assert.strictEqual(listRes.collection, "users"); + + databaseBridge.createIndex = async (dbId, col, key, opts) => { + calledMethod = `createIndex:${dbId}:${col}:${JSON.stringify(key)}:${JSON.stringify(opts)}`; + return { success: true, indexName: "idx_1", executionTimeMs: 12 }; + }; + const createRes = await databaseService.createMongoIndex("db1", "users", { email: 1 }); + assert.ok(calledMethod.startsWith("createIndex:db1:users:")); + assert.strictEqual(createRes.indexName, "idx_1"); + + databaseBridge.dropIndex = async (dbId, col, name) => { + calledMethod = `dropIndex:${dbId}:${col}:${name}`; + return { success: true, droppedIndexName: name, executionTimeMs: 5 }; + }; + const dropRes = await databaseService.dropMongoIndex("db1", "users", "idx_1"); + assert.strictEqual(calledMethod, "dropIndex:db1:users:idx_1"); + assert.strictEqual(dropRes.droppedIndexName, "idx_1"); + + databaseBridge.getIndexStats = async (dbId, col) => { + calledMethod = `getIndexStats:${dbId}:${col}`; + return { collection: col, stats: {} }; + }; + const statsRes = await databaseService.getMongoIndexStats("db1", "users"); + assert.strictEqual(calledMethod, "getIndexStats:db1:users"); + assert.strictEqual(statsRes.collection, "users"); + + databaseBridge.hideIndex = async (dbId, col, name, hidden) => { + calledMethod = `hideIndex:${dbId}:${col}:${name}:${hidden}`; + return { success: true, indexName: name, hidden }; + }; + const hideRes = await databaseService.hideMongoIndex("db1", "users", "idx_1", true); + assert.strictEqual(calledMethod, "hideIndex:db1:users:idx_1:true"); + assert.strictEqual(hideRes.hidden, true); + } finally { + databaseBridge.listIndexes = originalList; + databaseBridge.createIndex = originalCreate; + databaseBridge.dropIndex = originalDrop; + databaseBridge.getIndexStats = originalStats; + databaseBridge.hideIndex = originalHide; + } + }); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoMigrationCard.test.ts b/src/features/mongodb/__tests__/MongoMigrationCard.test.ts new file mode 100644 index 00000000..d328965e --- /dev/null +++ b/src/features/mongodb/__tests__/MongoMigrationCard.test.ts @@ -0,0 +1,178 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { MongoMigrationCard, MigrationCardData } from "../components/migrations/MongoMigrationCard"; + +test("MongoMigrationCard - renders amber 'Drift Detected' badge with CHECKSUM_MISMATCH explanation and checksum hashes", () => { + const migration: MigrationCardData = { + id: "20261031120000", + name: "Drifted User Schema", + checksum: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + status: "APPLIED", + driftStatus: "DRIFTED", + drifted: true, + driftDetails: { + status: "DRIFTED", + reason: "CHECKSUM_MISMATCH", + expectedChecksum: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + actualChecksum: "f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5", + message: "On-disk manifest content has been altered since application", + }, + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoMigrationCard, { migration }), + ); + + assert.ok(html.includes('data-testid="migration-drift-badge"'), "Must render drift badge"); + assert.ok(html.includes("Drift Detected"), "Badge label must say 'Drift Detected'"); + assert.ok(html.includes("text-amber-500"), "Must have amber styling"); + assert.ok( + html.includes("Manifest Checksum Mismatch"), + "Tooltip title must be Manifest Checksum Mismatch", + ); + assert.ok(html.includes("a1b2c3d4e5"), "Tooltip must include applied checksum prefix"); + assert.ok(html.includes("f6e5d4c3b2"), "Tooltip must include on-disk checksum prefix"); + assert.ok(html.includes("APPLIED"), "Execution status APPLIED badge must remain visible"); +}); + +test("MongoMigrationCard - renders neutral 'Unverifiable' badge with MANIFEST_MISSING explanation", () => { + const migration: MigrationCardData = { + id: "20261031120000", + name: "Deleted Manifest", + checksum: "a1b2c3d4e5f6", + status: "APPLIED", + driftStatus: "UNVERIFIABLE", + drifted: false, + driftDetails: { + status: "UNVERIFIABLE", + reason: "MANIFEST_MISSING", + message: "Manifest file not found in migrations directory", + }, + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoMigrationCard, { migration }), + ); + + assert.ok( + html.includes('data-testid="migration-unverifiable-badge"'), + "Must render unverifiable badge", + ); + assert.ok(html.includes("Unverifiable"), "Badge label must say 'Unverifiable'"); + assert.ok(html.includes("text-muted-foreground"), "Must have neutral styling"); + assert.ok( + html.includes("Manifest File Missing"), + "Tooltip title must be 'Manifest File Missing'", + ); + assert.ok(html.includes("not found on disk"), "Tooltip description must explain file absence"); + assert.ok(html.includes("APPLIED"), "Execution status APPLIED badge must remain visible"); +}); + +test("MongoMigrationCard - renders neutral 'Unverifiable' badge with RECORDED_CHECKSUM_MISSING explanation", () => { + const migration: MigrationCardData = { + id: "20261031120000", + name: "Legacy Migration", + checksum: "", + status: "APPLIED", + driftStatus: "UNVERIFIABLE", + drifted: false, + driftDetails: { + status: "UNVERIFIABLE", + reason: "RECORDED_CHECKSUM_MISSING", + message: "Recorded migration summary contains no baseline checksum", + }, + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoMigrationCard, { migration }), + ); + + assert.ok( + html.includes('data-testid="migration-unverifiable-badge"'), + "Must render unverifiable badge", + ); + assert.ok(html.includes("Unverifiable"), "Badge label must say 'Unverifiable'"); + assert.ok(html.includes("No Recorded Checksum"), "Tooltip title must be 'No Recorded Checksum'"); + assert.ok(html.includes("legacy migration"), "Tooltip description must explain legacy migration"); + assert.ok(html.includes("APPLIED"), "Execution status APPLIED badge must remain visible"); +}); + +test("MongoMigrationCard - renders neutral 'Unverifiable' badge with MANIFEST_PARSE_ERROR explanation", () => { + const migration: MigrationCardData = { + id: "20261031120000", + name: "Corrupt Manifest", + checksum: "abc123456789", + status: "APPLIED", + driftStatus: "UNVERIFIABLE", + drifted: false, + driftDetails: { + status: "UNVERIFIABLE", + reason: "MANIFEST_PARSE_ERROR", + message: "Manifest file exceeds 1MB limit (1500000 bytes)", + }, + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoMigrationCard, { migration }), + ); + + assert.ok( + html.includes('data-testid="migration-unverifiable-badge"'), + "Must render unverifiable badge", + ); + assert.ok(html.includes("Manifest Parse Error"), "Tooltip title must be 'Manifest Parse Error'"); + assert.ok(html.includes("exceeds 1MB limit"), "Tooltip description must include error message"); + assert.ok(html.includes("APPLIED"), "Execution status APPLIED badge must remain visible"); +}); + +test("MongoMigrationCard - suppresses drift and unverifiable badges when driftStatus is VERIFIED", () => { + const migration: MigrationCardData = { + id: "20261031120000", + name: "Healthy Verified Migration", + checksum: "validhash123", + status: "APPLIED", + driftStatus: "VERIFIED", + drifted: false, + driftDetails: { + status: "VERIFIED", + reason: "CHECKSUM_MATCH", + expectedChecksum: "validhash123", + actualChecksum: "validhash123", + }, + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoMigrationCard, { migration }), + ); + + assert.strictEqual( + html.includes('data-testid="migration-drift-badge"'), + false, + "Must NOT render drift badge when VERIFIED", + ); + assert.strictEqual( + html.includes('data-testid="migration-unverifiable-badge"'), + false, + "Must NOT render unverifiable badge when VERIFIED", + ); + assert.ok(html.includes("APPLIED"), "Execution status APPLIED badge must remain visible"); +}); + +test("MongoMigrationCard - preserves canonical status badge (APPLIED, PENDING) unchanged without implying verification", () => { + const pendingMigration: MigrationCardData = { + id: "20261031130000", + name: "Unapplied Manifest", + checksum: "pendinghash123", + status: "PENDING", + }; + + const html = ReactDOMServer.renderToStaticMarkup( + React.createElement(MongoMigrationCard, { migration: pendingMigration }), + ); + + assert.ok(html.includes("PENDING"), "Must render PENDING badge"); + assert.strictEqual(html.includes('data-testid="migration-drift-badge"'), false); + assert.strictEqual(html.includes('data-testid="migration-unverifiable-badge"'), false); +}); diff --git a/src/features/mongodb/__tests__/MongoMigrationsPanel.test.ts b/src/features/mongodb/__tests__/MongoMigrationsPanel.test.ts new file mode 100644 index 00000000..8b12a6c0 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoMigrationsPanel.test.ts @@ -0,0 +1,332 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { getMigrationStatusBadgeConfig } from "../components/migrations/MongoMigrationCard"; +import { + groupAndSortMigrationEvents, + getEventTypeBadgeConfig, +} from "../components/migrations/MongoMigrationDetailDrawer"; +import { STATIC_PREFLIGHT_ADVISORY } from "../components/migrations/MongoMigrationApplyDialog"; +import { IRREVERSIBLE_ROLLBACK_ACKNOWLEDGMENT } from "../components/migrations/MongoMigrationRollbackDialog"; +import type { + MongoMigrationStatus, + MongoMigrationEvent, + MongoMigrationEventType, +} from "../types/migration"; + +test("MongoMigrations - All 9 Canonical Statuses are mapped correctly", () => { + const canonicalStatuses: MongoMigrationStatus[] = [ + "PENDING", + "ACQUIRING_LOCK", + "APPLYING", + "FAILED_PARTIAL", + "CRASHED_UNCERTAIN", + "ROLLING_BACK", + "ROLLED_BACK", + "APPLIED", + "FAILED_MANUAL_INTERVENTION_REQUIRED", + ]; + + assert.strictEqual( + canonicalStatuses.length, + 9, + "Must strictly preserve exactly 9 canonical states", + ); + + for (const status of canonicalStatuses) { + const config = getMigrationStatusBadgeConfig(status); + assert.strictEqual(config.label, status, `Badge label must match canonical status "${status}"`); + assert.ok(config.className, `Badge must define a styling className for "${status}"`); + assert.ok(config.description, `Badge must define an explanatory description for "${status}"`); + } + + // Check specific UX requirements + const pending = getMigrationStatusBadgeConfig("PENDING"); + assert.strictEqual(pending.variant, "secondary"); + + const acquiring = getMigrationStatusBadgeConfig("ACQUIRING_LOCK"); + assert.ok(acquiring.className.includes("animate-pulse"), "ACQUIRING_LOCK must pulse"); + assert.ok(acquiring.className.includes("blue"), "ACQUIRING_LOCK must be blue"); + + const applying = getMigrationStatusBadgeConfig("APPLYING"); + assert.ok(applying.className.includes("animate-pulse"), "APPLYING must pulse"); + assert.ok(applying.className.includes("amber"), "APPLYING must be amber"); + + const failedPartial = getMigrationStatusBadgeConfig("FAILED_PARTIAL"); + assert.strictEqual(failedPartial.variant, "destructive"); + + const crashedUncertain = getMigrationStatusBadgeConfig("CRASHED_UNCERTAIN"); + assert.strictEqual(crashedUncertain.variant, "outline"); + assert.ok( + crashedUncertain.className.includes("destructive"), + "CRASHED_UNCERTAIN must highlight destructive", + ); + + const rollingBack = getMigrationStatusBadgeConfig("ROLLING_BACK"); + assert.ok(rollingBack.className.includes("animate-pulse"), "ROLLING_BACK must pulse"); + assert.ok(rollingBack.className.includes("purple"), "ROLLING_BACK must be purple"); + + const rolledBack = getMigrationStatusBadgeConfig("ROLLED_BACK"); + assert.strictEqual(rolledBack.variant, "outline"); + + const applied = getMigrationStatusBadgeConfig("APPLIED"); + assert.ok(applied.className.includes("emerald"), "APPLIED must be emerald green"); + + const failedManual = getMigrationStatusBadgeConfig("FAILED_MANUAL_INTERVENTION_REQUIRED"); + assert.strictEqual(failedManual.variant, "destructive"); + assert.ok( + failedManual.className.includes("font-bold"), + "FAILED_MANUAL_INTERVENTION_REQUIRED must be bold", + ); +}); + +test("MongoMigrations - Event Audit History Grouping & Sequence Ordering", () => { + // Empty event stream test + assert.deepStrictEqual(groupAndSortMigrationEvents([]), []); + + // Jumbled events across two distinct attempts + const rawEvents: MongoMigrationEvent[] = [ + { + _id: "ev-5", + eventId: "e5", + migrationId: "20261031_001", + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 2, + stepIndex: 1, + stepType: "createIndex", + eventType: "STEP_COMPLETE", + timestamp: "2026-10-31T12:00:15Z", + lockOwnerId: "worker-b", + fencingToken: 12, + }, + { + _id: "ev-2", + eventId: "e2", + migrationId: "20261031_001", + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 2, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_COMPLETE", + timestamp: "2026-10-31T11:00:05Z", + lockOwnerId: "worker-a", + fencingToken: 11, + }, + { + _id: "ev-4", + eventId: "e4", + migrationId: "20261031_001", + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 1, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_START", + timestamp: "2026-10-31T12:00:00Z", + lockOwnerId: "worker-b", + fencingToken: 12, + }, + { + _id: "ev-1", + eventId: "e1", + migrationId: "20261031_001", + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 1, + stepIndex: 0, + stepType: "createCollection", + eventType: "STEP_START", + timestamp: "2026-10-31T11:00:00Z", + lockOwnerId: "worker-a", + fencingToken: 11, + }, + { + _id: "ev-3", + eventId: "e3", + migrationId: "20261031_001", + attemptId: "att-1", + attemptNumber: 1, + sequenceNumber: 3, + stepIndex: 1, + stepType: "createIndex", + eventType: "STEP_FAILED", + timestamp: "2026-10-31T11:00:10Z", + lockOwnerId: "worker-a", + fencingToken: 11, + metadata: { error: "Index key violation" }, + }, + { + _id: "ev-6", + eventId: "e6", + migrationId: "20261031_001", + attemptId: "att-2", + attemptNumber: 2, + sequenceNumber: 3, + eventType: "MIGRATION_APPLIED", + timestamp: "2026-10-31T12:00:20Z", + lockOwnerId: "worker-b", + fencingToken: 12, + }, + ]; + + const grouped = groupAndSortMigrationEvents(rawEvents); + + // Must have exactly two attempt groups + assert.strictEqual(grouped.length, 2); + + // Groups ordered by attemptNumber ascending + assert.strictEqual(grouped[0].attemptNumber, 1); + assert.strictEqual(grouped[0].attemptId, "att-1"); + assert.strictEqual(grouped[1].attemptNumber, 2); + assert.strictEqual(grouped[1].attemptId, "att-2"); + + // Attempt 1 events strictly ordered by sequenceNumber ascending: 1, 2, 3 + assert.strictEqual(grouped[0].events.length, 3); + assert.deepStrictEqual( + grouped[0].events.map((e) => e.sequenceNumber), + [1, 2, 3], + ); + assert.strictEqual(grouped[0].events[0].eventId, "e1"); + assert.strictEqual(grouped[0].events[1].eventId, "e2"); + assert.strictEqual(grouped[0].events[2].eventId, "e3"); + + // Attempt 2 events strictly ordered by sequenceNumber ascending: 1, 2, 3 + assert.strictEqual(grouped[1].events.length, 3); + assert.deepStrictEqual( + grouped[1].events.map((e) => e.sequenceNumber), + [1, 2, 3], + ); + assert.strictEqual(grouped[1].events[0].eventId, "e4"); + assert.strictEqual(grouped[1].events[1].eventId, "e5"); + assert.strictEqual(grouped[1].events[2].eventId, "e6"); +}); + +test("MongoMigrations - Static Preflight Mandatory Warning String", () => { + const expectedAdvisory = + "Validates manifest structure, expression safety, and syntax; does not test runtime constraint satisfaction on live data."; + + assert.strictEqual( + STATIC_PREFLIGHT_ADVISORY, + expectedAdvisory, + "Dry Run advisory text must match exact specification requirement", + ); +}); + +test("MongoMigrations - Rollback Irreversibility Disclosure & Confirmation", () => { + const expectedAcknowledgment = + "I acknowledge that data destroyed by dropCollection cannot be restored and attempt partial compensation of other steps."; + + assert.strictEqual( + IRREVERSIBLE_ROLLBACK_ACKNOWLEDGMENT, + expectedAcknowledgment, + "Rollback irreversibility acknowledgment text must match exact specification requirement", + ); +}); + +test("MongoMigrations - Read-Only Connection Protection Disables Mutating Actions", () => { + // Simulate read-only detection logic + const checkReadOnly = (dbConn: any): boolean => { + return Boolean(dbConn?.readOnly || dbConn?.readonly); + }; + + assert.strictEqual(checkReadOnly({ readOnly: true }), true); + assert.strictEqual(checkReadOnly({ readonly: true }), true); + assert.strictEqual(checkReadOnly({ readOnly: false }), false); + assert.strictEqual(checkReadOnly({}), false); + assert.strictEqual(checkReadOnly(null), false); + + // Helper verifying action guard logic + const isActionAllowed = (isReadOnly: boolean, hasPendingItems: boolean): boolean => { + if (isReadOnly) return false; + return hasPendingItems; + }; + + assert.strictEqual( + isActionAllowed(true, true), + false, + "Read-only prevents apply even if pending items exist", + ); + assert.strictEqual(isActionAllowed(true, false), false, "Read-only prevents apply when empty"); + assert.strictEqual( + isActionAllowed(false, true), + true, + "Read-write allows apply when items exist", + ); + assert.strictEqual( + isActionAllowed(false, false), + false, + "Read-write blocks apply when no items exist", + ); +}); + +test("MongoMigrations - Event Type Badges for all 14 Event Types", () => { + const allEventTypes: MongoMigrationEventType[] = [ + "VALIDATION_SUCCESS", + "LOCK_ACQUIRED", + "MIGRATION_START", + "STEP_START", + "STEP_COMPLETE", + "STEP_FAILED", + "STEP_UNCERTAIN", + "COMPENSATION_START", + "COMPENSATION_COMPLETE", + "COMPENSATION_FAILED", + "MIGRATION_APPLIED", + "MIGRATION_ROLLED_BACK", + "LOCK_RELEASED", + "MANUAL_INTERVENTION_REQUIRED", + ]; + + assert.strictEqual(allEventTypes.length, 14, "Must support all 14 migration event types"); + + for (const et of allEventTypes) { + const badge = getEventTypeBadgeConfig(et); + assert.strictEqual(badge.label, et, `Event badge label must match ${et}`); + assert.ok(badge.className, `Event badge className must be defined for ${et}`); + } +}); + +test("MongoMigrations - Unknown Event Type & Status Fallbacks", () => { + const unknownEvent = getEventTypeBadgeConfig("UNKNOWN_CUSTOM_EVENT" as any); + assert.strictEqual(unknownEvent.label, "UNKNOWN_CUSTOM_EVENT"); + assert.ok(unknownEvent.className.includes("bg-muted")); + + const unknownStatus = getMigrationStatusBadgeConfig("UNKNOWN_STATUS" as any); + assert.strictEqual(unknownStatus.label, "UNKNOWN_STATUS" as any); + assert.ok(unknownStatus.className.includes("bg-muted")); +}); + +test("MongoMigrations - Uncertain & Manual Intervention Action Restrictions", () => { + // Batch apply guard logic: pauses when any applied migration is uncertain or requires manual intervention + const canApplyAll = ( + isReadOnly: boolean, + pendingCount: number, + appliedSummaries: { status: MongoMigrationStatus }[], + ): boolean => { + if (isReadOnly) return false; + if (pendingCount === 0) return false; + const hasUncertainOrManualIntervention = appliedSummaries.some( + (m) => m.status === "CRASHED_UNCERTAIN" || m.status === "FAILED_MANUAL_INTERVENTION_REQUIRED", + ); + return !hasUncertainOrManualIntervention; + }; + + assert.strictEqual(canApplyAll(false, 3, []), true, "Clean state allows Apply All"); + assert.strictEqual( + canApplyAll(false, 3, [{ status: "APPLIED" }]), + true, + "Applied migrations allow Apply All", + ); + assert.strictEqual( + canApplyAll(false, 3, [{ status: "CRASHED_UNCERTAIN" }]), + false, + "CRASHED_UNCERTAIN halts automated batch apply", + ); + assert.strictEqual( + canApplyAll(false, 3, [{ status: "FAILED_MANUAL_INTERVENTION_REQUIRED" }]), + false, + "FAILED_MANUAL_INTERVENTION_REQUIRED halts automated batch apply", + ); + assert.strictEqual(canApplyAll(true, 3, []), false, "Read-only halts automated batch apply"); +}); diff --git a/src/features/mongodb/__tests__/MongoMqlEditor.test.ts b/src/features/mongodb/__tests__/MongoMqlEditor.test.ts new file mode 100644 index 00000000..5cce2933 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoMqlEditor.test.ts @@ -0,0 +1,91 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + MONGO_COMPLETION_OPERATORS, + FORBIDDEN_AUTOCOMPLETE_OPERATORS, + FORBIDDEN_CONSTRUCTOR_NAMES, + getMongoCompletions, + lightTheme, + darkThemeExtension, +} from "../components/editor/MongoMqlEditor"; + +test("MongoMqlEditor - Autocomplete Security Invariant", () => { + const completions = getMongoCompletions(); + const completionLabels = new Set(completions.map((c) => c.label)); + + // 1. Must include allowed query operators + assert.ok(completionLabels.has("$match"), "Must contain $match"); + assert.ok(completionLabels.has("$project"), "Must contain $project"); + assert.ok(completionLabels.has("$group"), "Must contain $group"); + assert.ok(completionLabels.has("$sort"), "Must contain $sort"); + assert.ok(completionLabels.has("$limit"), "Must contain $limit"); + assert.ok(completionLabels.has("$skip"), "Must contain $skip"); + assert.ok(completionLabels.has("$unwind"), "Must contain $unwind"); + assert.ok(completionLabels.has("$lookup"), "Must contain $lookup"); + assert.ok(completionLabels.has("$addFields"), "Must contain $addFields"); + assert.ok(completionLabels.has("$gt"), "Must contain $gt"); + assert.ok(completionLabels.has("$gte"), "Must contain $gte"); + assert.ok(completionLabels.has("$eq"), "Must contain $eq"); + assert.ok(completionLabels.has("$ne"), "Must contain $ne"); + assert.ok(completionLabels.has("$in"), "Must contain $in"); + assert.ok(completionLabels.has("$exists"), "Must contain $exists"); + + // 2. Extended JSON v2 valid wrappers must be present + assert.ok(completionLabels.has("$oid"), "Must contain EJSON v2 $oid"); + assert.ok(completionLabels.has("$date"), "Must contain EJSON v2 $date"); + assert.ok(completionLabels.has("$numberLong"), "Must contain EJSON v2 $numberLong"); + assert.ok(completionLabels.has("$numberDecimal"), "Must contain EJSON v2 $numberDecimal"); + assert.ok(completionLabels.has("$binary"), "Must contain EJSON v2 $binary"); + + // 3. MANDATORY SECURITY INVARIANT: Must strictly blacklist forbidden operators + for (const forbidden of FORBIDDEN_AUTOCOMPLETE_OPERATORS) { + assert.strictEqual( + completionLabels.has(forbidden), + false, + `SECURITY VIOLATION: Autocomplete must never contain ${forbidden}`, + ); + } + + // Explicitly assert every single forbidden operator + assert.strictEqual(completionLabels.has("$where"), false, "Must not contain $where"); + assert.strictEqual(completionLabels.has("$function"), false, "Must not contain $function"); + assert.strictEqual(completionLabels.has("$accumulator"), false, "Must not contain $accumulator"); + assert.strictEqual(completionLabels.has("$out"), false, "Must not contain $out"); + assert.strictEqual(completionLabels.has("$merge"), false, "Must not contain $merge"); +}); + +test("MongoMqlEditor - JSON-Only Contract & Constructor Absence", () => { + const completions = getMongoCompletions(); + const completionLabels = new Set(completions.map((c) => c.label)); + + // Mongosh constructors must NOT be present in JSON editor autocomplete + for (const constructorName of FORBIDDEN_CONSTRUCTOR_NAMES) { + assert.strictEqual( + completionLabels.has(constructorName), + false, + `JSON VIOLATION: Autocomplete must not suggest mongosh constructor ${constructorName}() in JSON editor`, + ); + } + assert.strictEqual(completionLabels.has("ObjectId"), false); + assert.strictEqual(completionLabels.has("ISODate"), false); + assert.strictEqual(completionLabels.has("NumberLong"), false); + assert.strictEqual(completionLabels.has("NumberDecimal"), false); + assert.strictEqual(completionLabels.has("UUID"), false); + + // All completion labels must begin with '$' to conform to valid JSON keys + for (const label of completionLabels) { + assert.ok( + label.startsWith("$"), + `Completion token "${label}" must start with '$' for JSON validity`, + ); + } +}); + +test("MongoMqlEditor - Theme and Extension Configurations", () => { + assert.ok(lightTheme, "lightTheme extension should be defined"); + assert.ok(darkThemeExtension, "darkThemeExtension should be defined"); + assert.strictEqual(typeof lightTheme, "object"); + assert.strictEqual(typeof darkThemeExtension, "object"); + assert.strictEqual(Array.isArray(MONGO_COMPLETION_OPERATORS), true); + assert.ok(MONGO_COMPLETION_OPERATORS.length > 30, "Should have rich completion list"); +}); diff --git a/src/features/mongodb/__tests__/MongoMqlParser.test.ts b/src/features/mongodb/__tests__/MongoMqlParser.test.ts new file mode 100644 index 00000000..efec553d --- /dev/null +++ b/src/features/mongodb/__tests__/MongoMqlParser.test.ts @@ -0,0 +1,626 @@ +import { test, describe } from "node:test"; +import assert from "node:assert"; +import { parseMongoshQuery } from "../parser/mongoshParser"; +import { MongoParserError } from "../parser/errors"; + +describe("Mongosh Parser - Collection Syntax & Naming Rules", () => { + test("parses standard identifier collection reference", () => { + const res = parseMongoshQuery("db.users.find({})"); + assert.strictEqual(res.collection, "users"); + assert.strictEqual(res.op.operation, "find"); + }); + + test("parses db.getCollection('audit.events') with dotted collection name", () => { + const res = parseMongoshQuery("db.getCollection('audit.events').find({})"); + assert.strictEqual(res.collection, "audit.events"); + assert.strictEqual(res.op.operation, "find"); + }); + + test("parses db.collection('audit.events') with dotted collection name", () => { + const res = parseMongoshQuery('db.collection("audit.events").find({})'); + assert.strictEqual(res.collection, "audit.events"); + assert.strictEqual(res.op.operation, "find"); + }); + + test("parses bracket notation with double and single quotes", () => { + const res1 = parseMongoshQuery('db["audit.events"].find({})'); + assert.strictEqual(res1.collection, "audit.events"); + + const res2 = parseMongoshQuery("db['analytics.2026'].find({})"); + assert.strictEqual(res2.collection, "analytics.2026"); + }); + + test("explicitly rejects unbracketed dotted namespace db.audit.events.find()", () => { + assert.throws( + () => parseMongoshQuery("db.audit.events.find()"), + (err: any) => { + assert(err instanceof MongoParserError); + assert.strictEqual(err.code, "INVALID_COLLECTION_NAME"); + assert(err.message.includes("Dotted collection names must use explicit collection API")); + return true; + }, + ); + }); + + test("rejects empty, null byte, $, and system.* collection names", () => { + assert.throws( + () => parseMongoshQuery('db.getCollection("").find({})'), + (err: any) => err.code === "INVALID_COLLECTION_NAME", + ); + assert.throws( + () => parseMongoshQuery('db.getCollection("users\\0admin").find({})'), + (err: any) => err.code === "INVALID_COLLECTION_NAME", + ); + assert.throws( + () => parseMongoshQuery('db.getCollection("users$admin").find({})'), + (err: any) => err.code === "INVALID_COLLECTION_NAME", + ); + assert.throws( + () => parseMongoshQuery('db.getCollection("system.users").find({})'), + (err: any) => err.code === "INVALID_COLLECTION_NAME", + ); + }); + + test("rejects collection names exceeding 120 UTF-8 bytes", () => { + const longName = "a".repeat(121); + assert.throws( + () => parseMongoshQuery(`db.getCollection("${longName}").find({})`), + (err: any) => err.code === "INVALID_COLLECTION_NAME", + ); + }); +}); + +describe("Mongosh Parser - Root Operations", () => { + test("parses find with default empty filter and projection", () => { + const res1 = parseMongoshQuery("db.users.find()"); + assert.strictEqual(res1.op.operation, "find"); + assert.deepStrictEqual(res1.op.payload, { filter: {}, projection: undefined }); + + const res2 = parseMongoshQuery('db.users.find({ status: "active" }, { name: 1, email: 1 })'); + assert.strictEqual(res2.op.operation, "find"); + assert.deepStrictEqual(res2.op.payload, { + filter: { status: "active" }, + projection: { name: 1, email: 1 }, + }); + }); + + test("parses aggregate with empty pipeline and stages", () => { + const res1 = parseMongoshQuery("db.users.aggregate([])"); + assert.strictEqual(res1.op.operation, "aggregate"); + assert.deepStrictEqual(res1.op.payload, { pipeline: [], allowDiskUse: undefined }); + + const res2 = parseMongoshQuery( + "db.users.aggregate([{ $match: { active: true } }], { allowDiskUse: true })", + ); + assert.strictEqual(res2.op.operation, "aggregate"); + assert.deepStrictEqual(res2.op.payload, { + pipeline: [{ $match: { active: true } }], + allowDiskUse: true, + }); + }); + + test("parses countDocuments with and without filter and options", () => { + const res1 = parseMongoshQuery("db.users.countDocuments()"); + assert.strictEqual(res1.op.operation, "countDocuments"); + assert.deepStrictEqual(res1.op.payload, { filter: {}, skip: undefined, limit: undefined }); + + const res2 = parseMongoshQuery( + 'db.users.countDocuments({ status: "pending" }, { limit: 100 })', + ); + assert.strictEqual(res2.op.operation, "countDocuments"); + assert.deepStrictEqual(res2.op.payload, { + filter: { status: "pending" }, + skip: undefined, + limit: 100, + }); + }); + + test("parses estimatedDocumentCount with empty or no argument", () => { + const res1 = parseMongoshQuery("db.users.estimatedDocumentCount()"); + assert.strictEqual(res1.op.operation, "estimatedDocumentCount"); + assert.strictEqual(res1.op.payload, undefined); + + const res2 = parseMongoshQuery("db.users.estimatedDocumentCount({})"); + assert.strictEqual(res2.op.operation, "estimatedDocumentCount"); + assert.strictEqual(res2.op.payload, undefined); + + // Reject filter in estimatedDocumentCount + assert.throws( + () => parseMongoshQuery('db.users.estimatedDocumentCount({ status: "active" })'), + (err: any) => err.code === "INVALID_SYNTAX", + ); + }); + + test("parses distinct with field and filter", () => { + const res1 = parseMongoshQuery('db.users.distinct("role")'); + assert.strictEqual(res1.op.operation, "distinct"); + assert.deepStrictEqual(res1.op.payload, { field: "role", filter: undefined }); + + const res2 = parseMongoshQuery('db.users.distinct("role", { active: true })'); + assert.strictEqual(res2.op.operation, "distinct"); + assert.deepStrictEqual(res2.op.payload, { + field: "role", + filter: { active: true }, + }); + + // Rejects empty or non-string field + assert.throws( + () => parseMongoshQuery('db.users.distinct("")'), + (err: any) => err.code === "INVALID_SYNTAX", + ); + }); +}); + +describe("Mongosh Parser - Chained Modifiers", () => { + test("parses .sort(), .skip(), .limit(), and .project() on find()", () => { + const query = + "db.users.find({}).sort({ createdAt: -1, name: 1 }).skip(10).limit(50).project({ email: 1 })"; + const res = parseMongoshQuery(query); + assert.strictEqual(res.op.operation, "find"); + assert.deepStrictEqual(res.op.payload, { + filter: {}, + sort: { createdAt: -1, name: 1 }, + skip: 10, + limit: 50, + projection: { email: 1 }, + }); + }); + + test("parses .allowDiskUse() on aggregate()", () => { + const res = parseMongoshQuery( + 'db.users.aggregate([{ $group: { _id: "$city" } }]).allowDiskUse(true)', + ); + assert.strictEqual(res.op.operation, "aggregate"); + assert.strictEqual((res.op.payload as any).allowDiskUse, true); + }); + + test("rejects modifiers incompatible with operation", () => { + // .sort() on aggregate() is invalid + assert.throws( + () => parseMongoshQuery("db.users.aggregate([]).sort({ x: 1 })"), + (err: any) => { + assert(err instanceof MongoParserError); + assert.strictEqual(err.code, "UNSUPPORTED_METHOD"); + assert(err.message.includes("not supported on aggregate()")); + return true; + }, + ); + + // .limit() on distinct() is invalid + assert.throws( + () => parseMongoshQuery('db.users.distinct("role").limit(10)'), + (err: any) => err.code === "UNSUPPORTED_METHOD", + ); + + // .allowDiskUse() on find() is invalid + assert.throws( + () => parseMongoshQuery("db.users.find({}).allowDiskUse(true)"), + (err: any) => err.code === "UNSUPPORTED_METHOD", + ); + }); + + test("rejects duplicate modifiers", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({}).limit(10).limit(20)"), + (err: any) => { + assert.strictEqual(err.code, "INVALID_SYNTAX"); + assert(err.message.includes("Duplicate modifier '.limit()'")); + return true; + }, + ); + }); + + test("rejects fractional or negative integer modifier arguments", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({}).skip(-5)"), + (err: any) => err.code === "INVALID_SYNTAX", + ); + assert.throws( + () => parseMongoshQuery("db.users.find({}).limit(10.5)"), + (err: any) => err.code === "INVALID_SYNTAX", + ); + assert.throws( + () => parseMongoshQuery("db.users.find({}).limit(0)"), + (err: any) => err.code === "INVALID_SYNTAX", + ); + }); + + test("rejects unsupported chained methods (toArray, forEach, map)", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({}).toArray()"), + (err: any) => { + assert.strictEqual(err.code, "UNSUPPORTED_METHOD"); + assert(err.message.includes("toArray")); + return true; + }, + ); + assert.throws( + () => parseMongoshQuery("db.users.find({}).forEach()"), + (err: any) => err.code === "UNSUPPORTED_METHOD", + ); + assert.throws( + () => parseMongoshQuery("db.users.find({}).map()"), + (err: any) => err.code === "UNSUPPORTED_METHOD", + ); + }); + + test("rejects trailing tokens after complete query", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({}) extra"), + (err: any) => { + assert.strictEqual(err.code, "INVALID_SYNTAX"); + assert(err.message.includes("Unexpected token 'extra'")); + return true; + }, + ); + }); +}); + +describe("Mongosh Parser - BSON Constructors & Extended JSON v2", () => { + test("converts ObjectId('24hex') into { $oid: '24hex' }", () => { + const res = parseMongoshQuery('db.users.find({ _id: ObjectId("507f1f77bcf86cd799439011") })'); + assert.deepStrictEqual((res.op.payload as any).filter, { + _id: { $oid: "507f1f77bcf86cd799439011" }, + }); + }); + + test("rejects malformed ObjectId values", () => { + assert.throws( + () => parseMongoshQuery('db.users.find({ _id: ObjectId("tooShort") })'), + (err: any) => { + assert.strictEqual(err.code, "INVALID_BSON_CONSTRUCTOR"); + assert(err.message.includes("exactly 24 hexadecimal characters")); + return true; + }, + ); + assert.throws( + () => parseMongoshQuery('db.users.find({ _id: ObjectId("507f1f77bcf86cd79943901g") })'), + (err: any) => err.code === "INVALID_BSON_CONSTRUCTOR", + ); + }); + + test("converts ISODate and new Date into { $date: '' }", () => { + const res1 = parseMongoshQuery( + 'db.users.find({ createdAt: ISODate("2026-10-30T10:00:00.000Z") })', + ); + assert.deepStrictEqual((res1.op.payload as any).filter, { + createdAt: { $date: "2026-10-30T10:00:00.000Z" }, + }); + + const res2 = parseMongoshQuery( + 'db.users.find({ createdAt: new Date("2026-10-30T10:00:00.000Z") })', + ); + assert.deepStrictEqual((res2.op.payload as any).filter, { + createdAt: { $date: "2026-10-30T10:00:00.000Z" }, + }); + }); + + test("rejects invalid date strings and prevents rollover (e.g. February 30)", () => { + // Non-existent calendar days must NOT silently normalize to the next month + assert.throws( + () => parseMongoshQuery('db.users.find({ d: ISODate("2026-02-30T10:00:00Z") })'), + (err: any) => { + assert.strictEqual(err.code, "INVALID_BSON_CONSTRUCTOR"); + assert(err.message.includes("Invalid day 30 for month 2")); + return true; + }, + ); + + // Non-leap year Feb 29 rejected + assert.throws( + () => parseMongoshQuery('db.users.find({ d: ISODate("2026-02-29T00:00:00Z") })'), + (err: any) => { + assert.strictEqual(err.code, "INVALID_BSON_CONSTRUCTOR"); + assert(err.message.includes("Invalid day 29 for month 2")); + return true; + }, + ); + + // Leap year Feb 29 accepted + const leapRes = parseMongoshQuery('db.users.find({ d: ISODate("2024-02-29T00:00:00Z") })'); + assert.strictEqual((leapRes.op.payload as any).filter.d.$date, "2024-02-29T00:00:00.000Z"); + + // Invalid month, time, or timezone + assert.throws( + () => parseMongoshQuery('db.users.find({ d: ISODate("2026-13-01T00:00:00Z") })'), + (err: any) => err.code === "INVALID_BSON_CONSTRUCTOR", + ); + assert.throws( + () => parseMongoshQuery('db.users.find({ d: ISODate("2026-05-10T25:00:00Z") })'), + (err: any) => err.code === "INVALID_BSON_CONSTRUCTOR", + ); + assert.throws( + () => parseMongoshQuery('db.users.find({ d: ISODate("2026-05-10T12:00:00+25:00") })'), + (err: any) => err.code === "INVALID_BSON_CONSTRUCTOR", + ); + }); + + test("converts NumberLong preserving precision without JS Number loss", () => { + // 9007199254740993 is greater than Number.MAX_SAFE_INTEGER (9007199254740991) + const res = parseMongoshQuery( + 'db.users.find({ counter: NumberLong("9007199254740993"), id: NumberLong(9007199254740993) })', + ); + assert.deepStrictEqual((res.op.payload as any).filter, { + counter: { $numberLong: "9007199254740993" }, + id: { $numberLong: "9007199254740993" }, + }); + }); + + test("validates signed 64-bit integer range for NumberLong", () => { + // Valid boundary values + const res = parseMongoshQuery( + 'db.users.find({ min: NumberLong("-9223372036854775808"), max: NumberLong("9223372036854775807") })', + ); + assert.strictEqual((res.op.payload as any).filter.min.$numberLong, "-9223372036854775808"); + assert.strictEqual((res.op.payload as any).filter.max.$numberLong, "9223372036854775807"); + + // Overflow boundary value (9223372036854775808 exceeds max) + assert.throws( + () => parseMongoshQuery('db.users.find({ bad: NumberLong("9223372036854775808") })'), + (err: any) => { + assert.strictEqual(err.code, "INVALID_BSON_CONSTRUCTOR"); + assert(err.message.includes("signed 64-bit integer range")); + return true; + }, + ); + }); + + test("converts NumberDecimal preserving exact decimal precision and supports NaN and Infinity", () => { + const res = parseMongoshQuery( + 'db.users.find({ balance: NumberDecimal("123.456789012345678901234567890"), sci: NumberDecimal("1.23e-4"), nan: NumberDecimal("NaN"), inf: NumberDecimal("Infinity"), ninf: NumberDecimal("-Infinity") })', + ); + assert.deepStrictEqual((res.op.payload as any).filter, { + balance: { $numberDecimal: "123.456789012345678901234567890" }, + sci: { $numberDecimal: "1.23e-4" }, + nan: { $numberDecimal: "NaN" }, + inf: { $numberDecimal: "Infinity" }, + ninf: { $numberDecimal: "-Infinity" }, + }); + + assert.throws( + () => parseMongoshQuery('db.users.find({ balance: NumberDecimal("not-a-decimal") })'), + (err: any) => err.code === "INVALID_BSON_CONSTRUCTOR", + ); + }); + + test("converts UUID into 16-byte Base64 representation with subtype 04 matching known fixture", () => { + const res = parseMongoshQuery( + 'db.users.find({ uuid: UUID("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") })', + ); + const filter = (res.op.payload as any).filter; + assert.strictEqual(filter.uuid.$binary.subType, "04"); + assert.strictEqual(filter.uuid.$binary.base64, "oO68mZwLTvi7bWu5vTgKEQ=="); + + // Reject non-36-char malformed UUID + assert.throws( + () => parseMongoshQuery('db.users.find({ uuid: UUID("550e8400-e29b") })'), + (err: any) => err.code === "INVALID_BSON_CONSTRUCTOR", + ); + }); + + test("rejects unsupported constructors (RegExp, Function, Symbol)", () => { + assert.throws( + () => parseMongoshQuery('db.users.find({ x: Function("return 1") })'), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery('db.users.find({ x: RegExp("abc") })'), + (err: any) => err.code === "UNSUPPORTED_METHOD", + ); + }); + + test("handles escape sequences in strings including quotes and unicode", () => { + const res = parseMongoshQuery( + 'db.users.find({ note: "Line 1\\nLine 2\\tTabbed \\"quoted\\" \\u0041" })', + ); + assert.strictEqual((res.op.payload as any).filter.note, 'Line 1\nLine 2\tTabbed "quoted" A'); + }); + + test("rejects malformed syntax like unclosed brackets, braces, and missing colons", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({ a: 1"), + (err: any) => err.code === "INVALID_SYNTAX", + ); + assert.throws( + () => parseMongoshQuery("db.users.find([ 1, 2"), + (err: any) => err.code === "INVALID_SYNTAX", + ); + assert.throws( + () => parseMongoshQuery("db.users.find({ a 1 })"), + (err: any) => err.code === "INVALID_SYNTAX", + ); + assert.throws( + () => parseMongoshQuery("db.users.find("), + (err: any) => err.code === "INVALID_SYNTAX", + ); + }); + + test("rejects duplicate projection when specified in both find() and .project()", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({}, { name: 1 }).project({ email: 1 })"), + (err: any) => { + assert.strictEqual(err.code, "INVALID_SYNTAX"); + assert(err.message.includes("Duplicate projection")); + return true; + }, + ); + }); + + test("parses new Date() without arguments as valid current date", () => { + const res = parseMongoshQuery("db.users.find({ time: new Date() })"); + const dStr = (res.op.payload as any).filter.time.$date; + assert(typeof dStr === "string"); + assert(!isNaN(new Date(dStr).getTime())); + }); +}); + +describe("Mongosh Parser - Security & Token-Aware Invariants", () => { + test("token-aware: string literals containing security keywords pass safely", () => { + // Semicolons inside strings are valid and must NOT trigger compound statement rejection + const res1 = parseMongoshQuery( + 'db.users.find({ text: "hello; world", query: "SELECT * FROM users; DROP TABLE;" })', + ); + assert.strictEqual((res1.op.payload as any).filter.text, "hello; world"); + assert.strictEqual((res1.op.payload as any).filter.query, "SELECT * FROM users; DROP TABLE;"); + + // Keywords inside string literals are valid and must NOT trigger security violations + const res2 = parseMongoshQuery( + 'db.users.find({ code: "function() { eval(); }", tag: "constructor", key: "__proto__" })', + ); + assert.strictEqual((res2.op.payload as any).filter.code, "function() { eval(); }"); + assert.strictEqual((res2.op.payload as any).filter.tag, "constructor"); + assert.strictEqual((res2.op.payload as any).filter.key, "__proto__"); + }); + + test("rejects unquoted compound statements with semicolons", () => { + assert.throws( + () => parseMongoshQuery("db.users.find(); db.users.drop()"), + (err: any) => { + assert(err instanceof MongoParserError); + assert.strictEqual(err.code, "COMPOUND_STATEMENT"); + assert(err.message.includes("Compound statements are not supported")); + return true; + }, + ); + + assert.throws( + () => parseMongoshQuery("db.users.find({}).limit(10); db.admin.dropDatabase()"), + (err: any) => err.code === "COMPOUND_STATEMENT", + ); + }); + + test("rejects arbitrary JavaScript declarations and statements outside strings", () => { + assert.throws( + () => parseMongoshQuery("var x = 1; db.users.find()"), + (err: any) => err.code === "COMPOUND_STATEMENT" || err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("const x = 10"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("function run() { db.users.find() }"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("for (let i = 0; i < 10; i++) {}"), + (err: any) => err.code === "COMPOUND_STATEMENT" || err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("while (true) {}"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("() => db.users.find()"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + }); + + test("rejects prototype-related property keys in object literals at any depth", () => { + // Top-level forbidden key + assert.throws( + () => parseMongoshQuery('db.users.find({ "__proto__": { admin: true } })'), + (err: any) => { + assert.strictEqual(err.code, "SECURITY_VIOLATION"); + assert(err.message.includes("__proto__")); + return true; + }, + ); + + // Nested forbidden key + assert.throws( + () => parseMongoshQuery('db.users.find({ nested: { constructor: "exploit" } })'), + (err: any) => { + assert.strictEqual(err.code, "SECURITY_VIOLATION"); + assert(err.message.includes("constructor")); + return true; + }, + ); + + assert.throws( + () => parseMongoshQuery('db.users.find({ nested: { prototype: "exploit" } })'), + (err: any) => { + assert.strictEqual(err.code, "SECURITY_VIOLATION"); + assert(err.message.includes("prototype")); + return true; + }, + ); + + // Unicode-escaped dangerous keys must be rejected after string decoding + assert.throws( + () => parseMongoshQuery('db.users.find({ "\\u005f\\u005fproto\\u005f\\u005f": "exploit" })'), + (err: any) => { + assert.strictEqual(err.code, "SECURITY_VIOLATION"); + assert(err.message.includes("__proto__")); + return true; + }, + ); + + assert.throws( + () => parseMongoshQuery('db.users.find({ "\\u0063onstructor": "exploit" })'), + (err: any) => { + assert.strictEqual(err.code, "SECURITY_VIOLATION"); + assert(err.message.includes("constructor")); + return true; + }, + ); + }); + + test("handles line and block comments without allowing security bypasses", () => { + // Valid query with comments + const res = parseMongoshQuery(` + // Fetch active users + db.users.find({ + /* Status check */ + status: "active" + }) // end of find + `); + assert.strictEqual(res.collection, "users"); + assert.deepStrictEqual((res.op.payload as any).filter, { status: "active" }); + + // Comments cannot hide compound statement injection + assert.throws( + () => parseMongoshQuery("db.users.find({}); /* comment */ db.users.drop()"), + (err: any) => err.code === "COMPOUND_STATEMENT", + ); + }); + + test("rejects prototype-related property access on query chains", () => { + assert.throws( + () => parseMongoshQuery("db.users.find({}).constructor"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("db.users.find({}).__proto__"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + assert.throws( + () => parseMongoshQuery("db.users.find({}).prototype"), + (err: any) => err.code === "SECURITY_VIOLATION", + ); + }); + + test("enforces maximum nesting depth of 15", () => { + // Build 15 levels of nesting: should succeed + let nested15 = '{"val": 1}'; + for (let i = 0; i < 14; i++) { + nested15 = `{"level${i}": ${nested15}}`; + } + const res = parseMongoshQuery(`db.users.find(${nested15})`); + assert.strictEqual(res.op.operation, "find"); + + // Build 16 levels of nesting: should throw DEPTH_LIMIT_EXCEEDED + let nested16 = '{"val": 1}'; + for (let i = 0; i < 15; i++) { + nested16 = `{"level${i}": ${nested16}}`; + } + assert.throws( + () => parseMongoshQuery(`db.users.find(${nested16})`), + (err: any) => { + assert.strictEqual(err.code, "DEPTH_LIMIT_EXCEEDED"); + assert(err.message.includes("depth of 15")); + return true; + }, + ); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoMutationValidation.test.ts b/src/features/mongodb/__tests__/MongoMutationValidation.test.ts new file mode 100644 index 00000000..008db9f3 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoMutationValidation.test.ts @@ -0,0 +1,86 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + validateCollectionName, + validateCappedOptions, + validateJsonDocument, +} from "../utils/validation"; + +describe("validateCollectionName", () => { + it("accepts valid collection names", () => { + assert.equal(validateCollectionName("users").valid, true); + assert.equal(validateCollectionName("user_accounts").valid, true); + assert.equal(validateCollectionName("orders-2026").valid, true); + assert.equal(validateCollectionName("_audit_log").valid, true); + assert.equal(validateCollectionName("a".repeat(120)).valid, true); + }); + + it("rejects empty or whitespace-only names", () => { + assert.equal(validateCollectionName("").valid, false); + assert.equal(validateCollectionName(" ").valid, false); + }); + + it("rejects names longer than 120 chars", () => { + assert.equal(validateCollectionName("a".repeat(121)).valid, false); + }); + + it("rejects names containing '$' or null bytes", () => { + assert.equal(validateCollectionName("my$coll").valid, false); + assert.equal(validateCollectionName("coll\0test").valid, false); + }); + + it("rejects names starting with 'system.'", () => { + assert.equal(validateCollectionName("system.views").valid, false); + assert.equal(validateCollectionName("system.profile").valid, false); + }); + + it("rejects invalid characters and leading numbers", () => { + assert.equal(validateCollectionName("123orders").valid, false); + assert.equal(validateCollectionName("-orders").valid, false); + assert.equal(validateCollectionName("my.coll").valid, false); + assert.equal(validateCollectionName("my coll").valid, false); + }); +}); + +describe("validateCappedOptions", () => { + it("passes when not capped", () => { + assert.equal(validateCappedOptions(undefined).valid, true); + assert.equal(validateCappedOptions({ capped: false }).valid, true); + }); + + it("requires size >= 4096 integer when capped", () => { + assert.equal(validateCappedOptions({ capped: true } as any).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 4095 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 5000.5 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 4096 }).valid, true); + assert.equal(validateCappedOptions({ capped: true, size: 8192 }).valid, true); + }); + + it("requires max to be a positive integer if provided", () => { + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: 0 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: -1 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: 10.5 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: 100 }).valid, true); + }); +}); + +describe("validateJsonDocument", () => { + it("accepts valid JSON objects", () => { + const res = validateJsonDocument('{"name": "Alice", "age": 30}'); + assert.equal(res.valid, true); + assert.deepEqual(res.doc, { name: "Alice", age: 30 }); + }); + + it("rejects invalid JSON syntax", () => { + const res = validateJsonDocument('{name: "Alice"}'); + assert.equal(res.valid, false); + assert.match(res.error || "", /JSON|position/i); + }); + + it("rejects JSON arrays or primitives", () => { + assert.equal(validateJsonDocument("[1, 2, 3]").valid, false); + assert.equal(validateJsonDocument('"hello"').valid, false); + assert.equal(validateJsonDocument("123").valid, false); + assert.equal(validateJsonDocument("null").valid, false); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoNavigationRouting.test.ts b/src/features/mongodb/__tests__/MongoNavigationRouting.test.ts new file mode 100644 index 00000000..4946837f --- /dev/null +++ b/src/features/mongodb/__tests__/MongoNavigationRouting.test.ts @@ -0,0 +1,169 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { getDatabasePanelItems } from "../../../components/layout/panelItems"; +import { + OPERATION_TEMPLATES, + isExplainSupported, + getExplainDisabledReason, +} from "../components/workspace/MongoWorkspacePanel"; + +test("Navigation Routing - MongoDB returns dedicated panel items", () => { + const mongoItems = getDatabasePanelItems("mongodb"); + const panelKeys = mongoItems.map((i) => i.panel); + + assert.deepStrictEqual(panelKeys, [ + "data", + "mongo-workspace", + "mongo-aggregation", + "mongo-migrations", + "mongo-relationships", + "mongo-bulk", + "git-status", + ]); + assert.strictEqual(mongoItems[0].label, "Data View"); + assert.strictEqual(mongoItems[1].label, "Query Workspace"); + assert.strictEqual(mongoItems[2].label, "Aggregation Builder"); + assert.strictEqual(mongoItems[3].label, "Migrations"); + assert.strictEqual(mongoItems[4].label, "Relationships"); + assert.strictEqual(mongoItems[5].label, "Bulk Operations"); + assert.strictEqual(mongoItems[6].label, "Git Status"); + assert.ok(mongoItems[4].icon, "Relationships item must have an icon defined"); + assert.ok(mongoItems[5].icon, "Bulk Operations item must have an icon defined"); + assert.ok(mongoItems[6].icon, "Git Status item must have an icon defined"); + + // Case-insensitive test + const mongoUpperItems = getDatabasePanelItems("MongoDB"); + assert.deepStrictEqual( + mongoUpperItems.map((i) => i.panel), + [ + "data", + "mongo-workspace", + "mongo-aggregation", + "mongo-migrations", + "mongo-relationships", + "mongo-bulk", + "git-status", + ], + ); +}); + +test("Navigation Routing - Relational databases retain full SQL suite without regression", () => { + const relationalDbs = [ + "postgres", + "postgresql", + "mysql", + "mariadb", + "sqlite", + "clickhouse", + "duckdb", + ]; + + for (const db of relationalDbs) { + const items = getDatabasePanelItems(db); + const panelKeys = items.map((i) => i.panel); + + assert.ok(panelKeys.includes("data"), `${db} must have data panel`); + assert.ok(panelKeys.includes("sql-workspace"), `${db} must retain sql-workspace`); + assert.ok(panelKeys.includes("query-builder"), `${db} must retain query-builder`); + assert.ok(panelKeys.includes("schema-explorer"), `${db} must retain schema-explorer`); + assert.ok(panelKeys.includes("er-diagram"), `${db} must retain er-diagram`); + assert.ok(panelKeys.includes("migrations"), `${db} must retain migrations`); + assert.ok(panelKeys.includes("git-status"), `${db} must retain git-status`); + assert.ok(!panelKeys.includes("mongo-workspace"), `${db} must not contain mongo-workspace`); + assert.ok(!panelKeys.includes("mongo-aggregation"), `${db} must not contain mongo-aggregation`); + assert.ok(!panelKeys.includes("mongo-bulk"), `${db} must not contain mongo-bulk`); + } +}); + +test("MongoWorkspace - Operation Templates validation", () => { + assert.strictEqual(typeof OPERATION_TEMPLATES.find, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.aggregate, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.countDocuments, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.estimatedDocumentCount, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.distinct, "string"); + + // All templates must parse into valid JSON + const findJson = JSON.parse(OPERATION_TEMPLATES.find); + const aggJson = JSON.parse(OPERATION_TEMPLATES.aggregate); + const countJson = JSON.parse(OPERATION_TEMPLATES.countDocuments); + const estJson = JSON.parse(OPERATION_TEMPLATES.estimatedDocumentCount); + const distJson = JSON.parse(OPERATION_TEMPLATES.distinct); + + assert.ok(typeof findJson === "object" && findJson !== null); + assert.ok(Array.isArray(aggJson), "Aggregate template must be an array of stages"); + assert.ok(aggJson.length > 0 && aggJson[0].$match !== undefined, "Aggregate template has $match"); + assert.ok(typeof countJson === "object" && countJson !== null); + assert.deepStrictEqual(estJson, {}, "estimatedDocumentCount template must be empty object"); + assert.ok(typeof distJson.field === "string", "distinct template must have field property"); +}); + +test("MongoWorkspace - Explain restriction to supported operations (find & aggregate only)", () => { + // Supported + assert.strictEqual(isExplainSupported("find"), true); + assert.strictEqual(isExplainSupported("aggregate"), true); + assert.strictEqual(getExplainDisabledReason("find"), null); + assert.strictEqual(getExplainDisabledReason("aggregate"), null); + + // Unsupported operations must be explicitly rejected + assert.strictEqual(isExplainSupported("countDocuments"), false); + assert.strictEqual(isExplainSupported("estimatedDocumentCount"), false); + assert.strictEqual(isExplainSupported("distinct"), false); + + assert.ok( + getExplainDisabledReason("countDocuments")?.includes( + "only supported for find() and aggregate()", + ), + ); + assert.ok( + getExplainDisabledReason("estimatedDocumentCount")?.includes( + "only supported for find() and aggregate()", + ), + ); + assert.ok( + getExplainDisabledReason("distinct")?.includes("only supported for find() and aggregate()"), + ); +}); + +test("Isolation Guarantee - Zero SQL execution services or hooks in MongoDB panels", () => { + const workspacePath = path.resolve( + process.cwd(), + "src/features/mongodb/components/workspace/MongoWorkspacePanel.tsx", + ); + const aggregationPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/aggregation/MongoAggregationPanel.tsx", + ); + + const workspaceContent = fs.readFileSync(workspacePath, "utf-8"); + const aggregationContent = fs.readFileSync(aggregationPath, "utf-8"); + + // Assert no SQL query execution or SQL workspace components are referenced + assert.strictEqual( + workspaceContent.includes("runQuery"), + false, + "MongoWorkspacePanel must not call SQL runQuery", + ); + assert.strictEqual( + workspaceContent.includes("SQLWorkspacePanel"), + false, + "MongoWorkspacePanel must not import SQLWorkspacePanel", + ); + assert.strictEqual( + workspaceContent.includes("useDatabaseDetails"), + false, + "MongoWorkspacePanel must not use SQL useDatabaseDetails", + ); + + assert.strictEqual( + aggregationContent.includes("runQuery"), + false, + "MongoAggregationPanel must not call SQL runQuery", + ); + assert.strictEqual( + aggregationContent.includes("QueryBuilderPanel"), + false, + "MongoAggregationPanel must not import QueryBuilderPanel", + ); +}); diff --git a/src/features/mongodb/__tests__/MongoPaginationPerformance.test.ts b/src/features/mongodb/__tests__/MongoPaginationPerformance.test.ts new file mode 100644 index 00000000..8bcb631e --- /dev/null +++ b/src/features/mongodb/__tests__/MongoPaginationPerformance.test.ts @@ -0,0 +1,228 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + canonicalizeValue, + generateFilterKey, + validateAndNormalizeEjsonBinary, +} from "../hooks/useMongoDatabaseDetails.ts"; + +describe("MongoPaginationPerformance & AST Canonicalizer", () => { + describe("Primitive & Special Number Canonicalization", () => { + it("canonicalizes primitives accurately", () => { + assert.deepEqual(canonicalizeValue(null), ["null", null]); + assert.deepEqual(canonicalizeValue(undefined), ["undefined", null]); + assert.deepEqual(canonicalizeValue(true), ["bool", true]); + assert.deepEqual(canonicalizeValue(false), ["bool", false]); + assert.deepEqual(canonicalizeValue("hello"), ["str", "hello"]); + assert.deepEqual(canonicalizeValue(42), ["num", 42]); + assert.deepEqual(canonicalizeValue(3.14159), ["num", 3.14159]); + }); + + it("canonicalizes BigInt to ['bigint', string]", () => { + assert.deepEqual(canonicalizeValue(9223372036854775807n), ["bigint", "9223372036854775807"]); + assert.deepEqual(canonicalizeValue(-9223372036854775808n), [ + "bigint", + "-9223372036854775808", + ]); + }); + + it("canonicalizes NaN and +/- Infinity", () => { + assert.deepEqual(canonicalizeValue(NaN), ["nan", "NaN"]); + assert.deepEqual(canonicalizeValue(Infinity), ["inf", "+Inf"]); + assert.deepEqual(canonicalizeValue(-Infinity), ["ninf", "-Inf"]); + }); + + it("canonicalizes Date and RegExp", () => { + const d = new Date("2026-11-10T12:00:00.000Z"); + assert.deepEqual(canonicalizeValue(d), ["date", "2026-11-10T12:00:00.000Z"]); + + const invalidDate = new Date("invalid"); + assert.deepEqual(canonicalizeValue(invalidDate), ["date_invalid", "Invalid Date"]); + + const r = /abc/gi; + assert.deepEqual(canonicalizeValue(r), ["regex", { p: "abc", f: "gi" }]); + }); + }); + + describe("Recursion-Stack AST: Shared DAG References vs True Cycles", () => { + it("permits shared subtrees in distinct DAG branches", () => { + const sharedNode = { status: "active", version: 1 }; + const filter = { + leftBranch: sharedNode, + rightBranch: sharedNode, + }; + + assert.doesNotThrow(() => canonicalizeValue(filter)); + const ast = canonicalizeValue(filter); + assert.equal(ast[0], "obj"); + }); + + it("rejects true circular references with MONGO_CYCLIC_FILTER", () => { + const cyclicObj: any = { name: "cycle" }; + cyclicObj.self = cyclicObj; + + assert.throws(() => canonicalizeValue(cyclicObj), /MONGO_CYCLIC_FILTER/); + }); + + it("rejects deep indirect circular references with MONGO_CYCLIC_FILTER", () => { + const a: any = { id: 1 }; + const b: any = { id: 2, parent: a }; + a.child = b; + + assert.throws(() => canonicalizeValue(a), /MONGO_CYCLIC_FILTER/); + }); + }); + + describe("Symbol-Keyed Own Property Rejection & Prototype Safety", () => { + it("rejects objects with symbol own properties with MONGO_INVALID_FILTER", () => { + const sym = Symbol("privateField"); + const obj = { [sym]: "val", publicField: 123 }; + + assert.throws(() => canonicalizeValue(obj), /MONGO_INVALID_FILTER/); + }); + + it("encodes __proto__, constructor, and prototype as safe tuples without pollution", () => { + const obj = JSON.parse('{"__proto__": "safe", "constructor": "safe", "prototype": "safe"}'); + const ast = canonicalizeValue(obj); + assert.equal(ast[0], "obj"); + const entries = ast[1] as Array<[string, any]>; + const keys = entries.map(([k]) => k).sort(); + assert.deepEqual(keys, ["__proto__", "constructor", "prototype"].sort()); + }); + }); + + describe("Unsupported Value & Class Rejection", () => { + it("rejects functions and symbols as values", () => { + assert.throws(() => canonicalizeValue(() => {}), /MONGO_INVALID_FILTER/); + assert.throws(() => canonicalizeValue(Symbol("foo")), /MONGO_INVALID_FILTER/); + }); + + it("rejects Map, Set, and custom class instances", () => { + assert.throws(() => canonicalizeValue(new Map()), /MONGO_INVALID_FILTER/); + assert.throws(() => canonicalizeValue(new Set()), /MONGO_INVALID_FILTER/); + + class CustomFilterClass { + name = "custom"; + } + assert.throws(() => canonicalizeValue(new CustomFilterClass()), /MONGO_INVALID_FILTER/); + }); + }); + + describe("Phase 2 Canonical Binary Validation", () => { + it("validates and normalizes valid canonical base64 and subtypes", () => { + const bin1 = { base64: "AQIDBA==", subType: 0 }; + assert.deepEqual(validateAndNormalizeEjsonBinary(bin1), { + b: "AQIDBA==", + s: "00", + }); + + const bin2 = { $base64: "////", $type: "04" }; + assert.deepEqual(validateAndNormalizeEjsonBinary(bin2), { + b: "////", + s: "04", + }); + }); + + it("rejects unexpected nested fields in $binary", () => { + const invalid = { base64: "AQID", subType: 0, unexpectedField: "danger" }; + assert.throws(() => validateAndNormalizeEjsonBinary(invalid), /unexpected nested field/); + }); + + it("rejects non-canonical base64 with non-zero pad bits", () => { + const nonCanonical = { base64: "AR==", subType: 0 }; + assert.throws(() => validateAndNormalizeEjsonBinary(nonCanonical), /non-canonical base64/); + }); + + it("rejects invalid subtype values", () => { + assert.throws(() => validateAndNormalizeEjsonBinary({ base64: "AQIDBA==", subType: 256 })); + assert.throws(() => validateAndNormalizeEjsonBinary({ base64: "AQIDBA==", subType: -1 })); + assert.throws(() => + validateAndNormalizeEjsonBinary({ base64: "AQIDBA==", subType: "invalidHex" }), + ); + }); + }); + + describe("EJSON Numeric Lexical & Range Validation", () => { + it("canonicalizes valid $numberInt within 32-bit range", () => { + assert.deepEqual(canonicalizeValue({ $numberInt: "2147483647" }), [ + "ejson_int", + "2147483647", + ]); + assert.deepEqual(canonicalizeValue({ $numberInt: "-2147483648" }), [ + "ejson_int", + "-2147483648", + ]); + }); + + it("rejects out-of-range or malformed $numberInt", () => { + assert.throws(() => canonicalizeValue({ $numberInt: "2147483648" })); + assert.throws(() => canonicalizeValue({ $numberInt: "123.45" })); + }); + + it("canonicalizes valid $numberLong within 64-bit range", () => { + assert.deepEqual(canonicalizeValue({ $numberLong: "9223372036854775807" }), [ + "ejson_long", + "9223372036854775807", + ]); + }); + + it("rejects out-of-range or malformed $numberLong", () => { + assert.throws(() => canonicalizeValue({ $numberLong: "9223372036854775808" })); + assert.throws(() => canonicalizeValue({ $numberLong: "abc" })); + }); + + it("canonicalizes valid $numberDecimal", () => { + assert.deepEqual(canonicalizeValue({ $numberDecimal: "123.456" }), [ + "ejson_decimal", + "123.456", + ]); + assert.deepEqual(canonicalizeValue({ $numberDecimal: "NaN" }), ["ejson_decimal", "NaN"]); + assert.deepEqual(canonicalizeValue({ $numberDecimal: "Infinity" }), [ + "ejson_decimal", + "Infinity", + ]); + }); + + it("encodes objects with extra fields alongside markers as ordinary objects", () => { + const objWithExtra = { $numberInt: "100", label: "test" }; + const ast = canonicalizeValue(objWithExtra); + assert.equal(ast[0], "obj"); + }); + }); + + describe("generateFilterKey and Option A Navigation", () => { + it("generates deterministic keys invariant to key order", () => { + const key1 = generateFilterKey("db1", "users", { age: 25, active: true }); + const key2 = generateFilterKey("db1", "users", { active: true, age: 25 }); + assert.equal(key1, key2); + }); + + it("handles explicit empty filter", () => { + assert.equal(generateFilterKey("db1", "users", {}), "db1::users::{}"); + assert.equal(generateFilterKey("db1", "users", undefined), "db1::users::{}"); + assert.equal(generateFilterKey("db1", "users", null), "db1::users::{}"); + }); + + it("derives Option A pagination using pageSize + 1 strategy with zero countDocuments", () => { + let countDocumentsCalls = 0; + const _mockCountDocuments = () => { + countDocumentsCalls++; + return 1000; + }; + + const pageSize = 50; + // Fetch pageSize + 1 records from database + const fetchedRawDocs = Array.from({ length: pageSize + 1 }, (_, i) => ({ _id: `doc-${i}` })); + + // Option A derivation: extra record determines hasMore, sliced to pageSize + const hasMore = fetchedRawDocs.length > pageSize; + const displayedDocs = hasMore ? fetchedRawDocs.slice(0, pageSize) : fetchedRawDocs; + const countOmitted = true; + + assert.equal(hasMore, true); + assert.equal(displayedDocs.length, 50); + assert.equal(countOmitted, true); + assert.equal(countDocumentsCalls, 0); // Proves zero count queries performed + }); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoQueryHistory.test.ts b/src/features/mongodb/__tests__/MongoQueryHistory.test.ts new file mode 100644 index 00000000..463224c1 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoQueryHistory.test.ts @@ -0,0 +1,467 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + getHistory, + addHistoryEntry, + clearHistory, + getHistorySettings, + updateHistorySettings, + setStorageAdapterForTesting, + validateDbId, + isValidHistoryEntry, + StorageAdapter, +} from "../history/historyStorage"; +import { redactQueryText, isRedactedQuery, sanitizeErrorMessage } from "../history/redaction"; +import { + MAX_HISTORY_ENTRIES, + CURRENT_HISTORY_SCHEMA_VERSION, + REDACTED_TEMPLATE_ADVISORY, +} from "../history/types"; + +// In-memory test storage adapter +function createMockStorage(): StorageAdapter & { + store: Map; + throwOnSet?: boolean; +} { + const store = new Map(); + return { + store, + throwOnSet: false, + getItem: (key: string) => store.get(key) ?? null, + setItem: function (key: string, value: string) { + if (this.throwOnSet) { + throw new Error("QuotaExceededError: LocalStorage quota exceeded"); + } + store.set(key, value); + }, + removeItem: (key: string) => { + store.delete(key); + }, + }; +} + +describe("MongoDB Query History & Privacy Storage", () => { + let mockStorage: ReturnType; + + beforeEach(() => { + mockStorage = createMockStorage(); + setStorageAdapterForTesting(mockStorage); + }); + + describe("Token-Aware Literal Redaction", () => { + it("redacts string literals in filter value position while preserving field names", () => { + const query = `{ email: "alice@company.com", status: "ACTIVE" }`; + const redacted = redactQueryText(query); + + assert.strictEqual(redacted, `{ email: "", status: "" }`); + assert.ok(!redacted.includes("alice@company.com")); + assert.ok(!redacted.includes("ACTIVE")); + assert.ok(redacted.includes("email:")); + assert.ok(redacted.includes("status:")); + }); + + it("redacts numeric and boolean literals while preserving operators", () => { + const query = `{ age: { $gte: 21 }, score: 99.5, isVerified: true, isBanned: false }`; + const redacted = redactQueryText(query); + + assert.strictEqual( + redacted, + `{ age: { $gte: }, score: , isVerified: , isBanned: }`, + ); + assert.ok(!redacted.includes("21")); + assert.ok(!redacted.includes("99.5")); + assert.ok(redacted.includes("$gte:")); + }); + + it("redacts mongosh BSON constructors (ISODate, Date, NumberLong, ObjectId, UUID)", () => { + const query = `db.orders.find({ + _id: ObjectId("507f1f77bcf86cd799439011"), + createdAt: ISODate("2026-10-30T10:00:00.000Z"), + updatedAt: new Date("2026-10-30T12:00:00.000Z"), + amount: NumberLong("9007199254740993"), + uuid: UUID("550e8400-e29b-41d4-a716-446655440000") +})`; + const redacted = redactQueryText(query); + + assert.ok(redacted.includes(`ObjectId("")`)); + assert.ok(redacted.includes(`ISODate("")`)); + assert.ok(redacted.includes(`new Date("")`)); + assert.ok(redacted.includes(`NumberLong("")`)); + assert.ok(redacted.includes(`UUID("")`)); + + // Assert literal secrets are absent + assert.ok(!redacted.includes("507f1f77bcf86cd799439011")); + assert.ok(!redacted.includes("2026-10-30T10:00:00.000Z")); + assert.ok(!redacted.includes("9007199254740993")); + assert.ok(!redacted.includes("550e8400-e29b-41d4-a716-446655440000")); + }); + + it("redacts EJSON v2 extended objects ($date, $oid, $numberLong)", () => { + const query = JSON.stringify( + { + _id: { $oid: "507f1f77bcf86cd799439011" }, + created: { $date: "2026-10-30T10:00:00.000Z" }, + bigInt: { $numberLong: "9999999999" }, + }, + null, + 2, + ); + const redacted = redactQueryText(query); + + assert.ok(redacted.includes(`"$oid": ""`)); + assert.ok(redacted.includes(`"$date": ""`)); + assert.ok(redacted.includes(`"$numberLong": ""`)); + assert.ok(!redacted.includes("507f1f77bcf86cd799439011")); + assert.ok(!redacted.includes("2026-10-30T10:00:00.000Z")); + assert.ok(!redacted.includes("9999999999")); + }); + + it("preserves collection name and method calls in mongosh syntax", () => { + const query = `db.getCollection("sensitive_customers").find({ token: "xyz-secret" })`; + const redacted = redactQueryText(query); + + assert.ok(redacted.includes(`db.getCollection("sensitive_customers").find`)); + assert.ok(redacted.includes(`token: ""`)); + assert.ok(!redacted.includes("xyz-secret")); + }); + + it("preserves distinct field name in distinct() operations", () => { + const query = `db.users.distinct("department", { active: true })`; + const redacted = redactQueryText(query); + + assert.ok(redacted.includes(`db.users.distinct("department"`)); + assert.ok(redacted.includes(`active: `)); + }); + + it("handles escaped string literals properly without escaping corruption", () => { + const query = `{ text: "He said \\"secret value\\"" }`; + const redacted = redactQueryText(query); + + assert.strictEqual(redacted, `{ text: "" }`); + assert.ok(!redacted.includes("secret value")); + }); + + it("accurately identifies redacted queries via isRedactedQuery", () => { + assert.strictEqual(isRedactedQuery(`{ email: "" }`), true); + assert.strictEqual(isRedactedQuery(`{ age: }`), true); + assert.strictEqual(isRedactedQuery(`{ date: "" }`), true); + assert.strictEqual(isRedactedQuery(`{ active: }`), true); + + assert.strictEqual(isRedactedQuery(`{ email: "john@doe.com" }`), false); + assert.strictEqual(isRedactedQuery(`{ age: 25 }`), false); + assert.strictEqual(isRedactedQuery(""), false); + }); + + it("sanitizes error messages containing database credentials and URIs", () => { + const rawError = + "MongoServerSelectionError: connect to mongodb://admin:SuperSecretPass123!@cluster0.abc.mongodb.net:27017/admin failed"; + const sanitized = sanitizeErrorMessage(rawError); + + assert.ok(sanitized); + assert.ok(!sanitized.includes("SuperSecretPass123!")); + assert.ok(sanitized.includes("")); + assert.ok(sanitized.includes("cluster0.abc.mongodb.net")); + }); + }); + + describe("History Storage Architecture & Invariants", () => { + it("maintains strict connection isolation between databases", () => { + const db1 = "conn-atlas-prod"; + const db2 = "conn-local-dev"; + + addHistoryEntry(db1, { + collection: "users", + operation: "find", + queryText: `{ email: "prod@company.com" }`, + durationMs: 12, + status: "success", + }); + + addHistoryEntry(db2, { + collection: "orders", + operation: "aggregate", + queryText: `[{ $match: { total: 100 } }]`, + durationMs: 25, + status: "success", + }); + + const history1 = getHistory(db1); + const history2 = getHistory(db2); + + assert.strictEqual(history1.length, 1); + assert.strictEqual(history1[0].collection, "users"); + + assert.strictEqual(history2.length, 1); + assert.strictEqual(history2[0].collection, "orders"); + + // Verify localStorage isolation keys + assert.ok(mockStorage.store.has(`relwave_mql_history_${db1}`)); + assert.ok(mockStorage.store.has(`relwave_mql_history_${db2}`)); + }); + + it("enforces strict 50-entry FIFO eviction limit", () => { + const dbId = "fifo-test-db"; + + // Add 55 entries sequentially + for (let i = 1; i <= 55; i++) { + addHistoryEntry(dbId, { + collection: "items", + operation: "find", + queryText: `{ index: ${i} }`, + durationMs: i, + status: "success", + id: `entry-${i}`, + timestamp: new Date(Date.now() + i * 1000).toISOString(), + }); + } + + const history = getHistory(dbId); + + // Exactly 50 entries + assert.strictEqual(history.length, MAX_HISTORY_ENTRIES); + assert.strictEqual(history.length, 50); + + // Newest entry is first (entry-55) + assert.strictEqual(history[0].id, "entry-55"); + + // 51st oldest entries (entry-1 through entry-5) must be evicted + const entryIds = new Set(history.map((e) => e.id)); + assert.ok(!entryIds.has("entry-1"), "entry-1 should be evicted"); + assert.ok(!entryIds.has("entry-2"), "entry-2 should be evicted"); + assert.ok(!entryIds.has("entry-5"), "entry-5 should be evicted"); + assert.ok(entryIds.has("entry-6"), "entry-6 should be retained"); + assert.ok(entryIds.has("entry-55"), "entry-55 should be retained"); + }); + + it("clears connection history without affecting other connections", () => { + const db1 = "clear-target-db"; + const db2 = "clear-preserve-db"; + + addHistoryEntry(db1, { + collection: "col1", + operation: "find", + queryText: "{}", + durationMs: 5, + status: "success", + }); + + addHistoryEntry(db2, { + collection: "col2", + operation: "find", + queryText: "{}", + durationMs: 5, + status: "success", + }); + + clearHistory(db1); + + assert.strictEqual(getHistory(db1).length, 0); + assert.strictEqual(getHistory(db2).length, 1); + assert.strictEqual(mockStorage.store.has(`relwave_mql_history_${db1}`), false); + assert.strictEqual(mockStorage.store.has(`relwave_mql_history_${db2}`), true); + }); + + it("recovers gracefully from malformed or corrupted JSON in localStorage", () => { + const dbId = "corrupt-json-db"; + mockStorage.store.set(`relwave_mql_history_${dbId}`, "INVALID_MALFORMED_JSON{{{"); + + const history = getHistory(dbId); + assert.deepStrictEqual(history, []); + }); + + it("filters out corrupted entries without crashing", () => { + const dbId = "corrupt-entry-db"; + const malformedPayload = { + version: CURRENT_HISTORY_SCHEMA_VERSION, + dbId, + entries: [ + { + id: "valid-1", + timestamp: "2026-10-30T10:00:00Z", + collection: "c", + operation: "find", + redactedQueryText: "{}", + durationMs: 1, + status: "success", + isRedacted: true, + }, + { id: 12345, missingFields: true }, // Corrupted + null, + { + id: "valid-2", + timestamp: "2026-10-30T10:00:01Z", + collection: "c", + operation: "find", + redactedQueryText: "{}", + durationMs: 2, + status: "success", + isRedacted: true, + }, + ], + }; + mockStorage.store.set(`relwave_mql_history_${dbId}`, JSON.stringify(malformedPayload)); + + const history = getHistory(dbId); + assert.strictEqual(history.length, 2); + assert.strictEqual(history[0].id, "valid-1"); + assert.strictEqual(history[1].id, "valid-2"); + }); + + it("handles localStorage quota failures gracefully", () => { + const dbId = "quota-db"; + mockStorage.throwOnSet = true; + + // Should not throw even when storage quota is exceeded + assert.doesNotThrow(() => { + addHistoryEntry(dbId, { + collection: "orders", + operation: "find", + queryText: `{ user: "alice" }`, + durationMs: 10, + status: "success", + }); + }); + }); + + it("validates dbId and safely rejects empty or whitespace-only keys", () => { + assert.strictEqual(validateDbId(""), false); + assert.strictEqual(validateDbId(" "), false); + assert.strictEqual(validateDbId(null as any), false); + assert.strictEqual(validateDbId(undefined as any), false); + assert.strictEqual(validateDbId("conn_valid_123"), true); + + assert.deepStrictEqual(getHistory(""), []); + assert.doesNotThrow(() => clearHistory("")); + }); + }); + + describe("Privacy Invariants & Raw-Query Opt-In", () => { + it("strictly never persists query result documents or output arrays", () => { + const dbId = "privacy-result-db"; + + const entry = addHistoryEntry(dbId, { + collection: "secrets", + operation: "find", + queryText: `{ ssn: "000-11-2222" }`, + durationMs: 15, + status: "success", + resultCount: 42, + }); + + // Verify returned entry has no result payload + assert.strictEqual((entry as any).documents, undefined); + assert.strictEqual((entry as any).results, undefined); + assert.strictEqual((entry as any).data, undefined); + + // Verify raw storage payload contains no result documents + const rawStored = mockStorage.store.get(`relwave_mql_history_${dbId}`); + assert.ok(rawStored); + assert.ok(!rawStored.includes("000-11-2222")); + assert.ok(!rawStored.includes("documents")); + }); + + it("enforces raw query storage is disabled by default", () => { + const dbId = "opt-in-default-db"; + const settings = getHistorySettings(dbId); + + assert.strictEqual(settings.storeRawQueries, false); + + const entry = addHistoryEntry(dbId, { + collection: "users", + operation: "find", + queryText: `{ password: "MySecretPassword123!" }`, + durationMs: 5, + status: "success", + }); + + // Assert entry is marked as redacted template + assert.strictEqual(entry.isRedacted, true); + assert.strictEqual(entry.rawQueryText, undefined); + assert.strictEqual(entry.redactedQueryText, `{ password: "" }`); + + // Assert raw storage does not contain the password + const stored = mockStorage.store.get(`relwave_mql_history_${dbId}`); + assert.ok(!stored?.includes("MySecretPassword123!")); + }); + + it("stores raw queries only after explicit user opt-in", () => { + const dbId = "opt-in-active-db"; + + // Explicit opt-in + updateHistorySettings(dbId, { storeRawQueries: true }); + const settings = getHistorySettings(dbId); + assert.strictEqual(settings.storeRawQueries, true); + + const entry = addHistoryEntry(dbId, { + collection: "users", + operation: "find", + queryText: `{ email: "test@example.com" }`, + durationMs: 8, + status: "success", + }); + + assert.strictEqual(entry.isRedacted, false); + assert.strictEqual(entry.rawQueryText, `{ email: "test@example.com" }`); + assert.strictEqual(entry.redactedQueryText, `{ email: "" }`); + }); + + it("disabling raw queries affects future entries without corrupting past ones", () => { + const dbId = "opt-in-toggle-db"; + + // 1. Opt in and write entry 1 + updateHistorySettings(dbId, { storeRawQueries: true }); + addHistoryEntry(dbId, { + collection: "c1", + operation: "find", + queryText: `{ val: "raw-val-1" }`, + durationMs: 1, + status: "success", + }); + + // 2. Opt out and write entry 2 + updateHistorySettings(dbId, { storeRawQueries: false }); + addHistoryEntry(dbId, { + collection: "c2", + operation: "find", + queryText: `{ val: "secret-val-2" }`, + durationMs: 2, + status: "success", + }); + + const history = getHistory(dbId); + assert.strictEqual(history.length, 2); + + // Entry 2 (newest) must be redacted + assert.strictEqual(history[0].isRedacted, true); + assert.strictEqual(history[0].rawQueryText, undefined); + assert.strictEqual(history[0].redactedQueryText, `{ val: "" }`); + + // Entry 1 (oldest) retains raw text from when setting was active + assert.strictEqual(history[1].isRedacted, false); + assert.strictEqual(history[1].rawQueryText, `{ val: "raw-val-1" }`); + }); + }); + + describe("Replay Safety & Template Advisory Invariants", () => { + it("exposes the exact mandatory advisory string required by M5 specification", () => { + assert.strictEqual( + REDACTED_TEMPLATE_ADVISORY, + "Loaded query template from history with redacted placeholders. Replace or placeholders with actual query values before executing.", + ); + }); + + it("marks redacted history entries as templates with isRedacted: true", () => { + const dbId = "template-mark-db"; + const entry = addHistoryEntry(dbId, { + collection: "orders", + operation: "find", + queryText: `{ customerId: "cust_123" }`, + durationMs: 4, + status: "success", + }); + + assert.strictEqual(entry.isRedacted, true); + assert.strictEqual(isValidHistoryEntry(entry), true); + }); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoReadOnlyGuards.test.ts b/src/features/mongodb/__tests__/MongoReadOnlyGuards.test.ts new file mode 100644 index 00000000..c09fbbc2 --- /dev/null +++ b/src/features/mongodb/__tests__/MongoReadOnlyGuards.test.ts @@ -0,0 +1,343 @@ +/** + * MongoDB Stage 8 Phase 1 — Frontend Read-Only Affordances Test Suite + * + * Validates AUD-02 read-only UI restrictions across: + * 1. MongoDocumentViewer: + * - Toolbar "Insert Document" disabled state, tooltip, and Tailwind classes + * - DataTable suppression of row-level inline edit and delete handlers + * 2. MongoCollectionsSidebar: + * - "Create new collection" button disabled state, title, and Tailwind classes + * - Suppression of row-level "Drop Collection" action button + * 3. MongoDataViewPanel: + * - Propagation of effective read-only state to MongoCollectionsSidebar and MongoDocumentViewer + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { MongoDocumentViewer } from "../components/MongoDocumentViewer"; +import { MongoCollectionsSidebar } from "../components/MongoCollectionsSidebar"; +import { DataTable } from "@/components/shared/DataTable"; + +function findVdomNodes(node: any, predicate: (node: any) => boolean): any[] { + const matches: any[] = []; + if (!node || typeof node !== "object") return matches; + if (predicate(node)) { + matches.push(node); + } + if (Array.isArray(node.props?.children)) { + for (const child of node.props.children) { + matches.push(...findVdomNodes(child, predicate)); + } + } else if (node.props?.children) { + matches.push(...findVdomNodes(node.props.children, predicate)); + } + return matches; +} + +function renderViewerVdom(props: any): any { + let capturedVdom: any; + function Wrapper() { + capturedVdom = MongoDocumentViewer(props); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(Wrapper)); + return capturedVdom; +} + +function renderSidebarVdom(props: any): any { + let capturedVdom: any; + function Wrapper() { + capturedVdom = MongoCollectionsSidebar(props); + return null; + } + ReactDOMServer.renderToStaticMarkup(React.createElement(Wrapper)); + return capturedVdom; +} + +describe("MongoDB Stage 8 Phase 1: Frontend Read-Only Affordances", () => { + const defaultViewerProps = { + collection: "customers", + documents: [{ _id: "cust-1", name: "Acme Corp" }], + totalDocuments: 1, + page: 1, + pageSize: 50, + totalPages: 1, + isLoading: false, + sortField: null, + sortDirection: null, + onSort: () => {}, + onPageChange: () => {}, + onPageSizeChange: () => {}, + filterInput: "", + onFilterInputChange: () => {}, + onApplyFilter: () => {}, + onResetFilter: () => {}, + isFilterApplied: false, + onRefresh: () => {}, + onInsertDocument: () => {}, + onEditDocument: () => {}, + onDeleteDocument: () => {}, + onImportClick: () => {}, + onExportClick: () => {}, + onBulkMutationClick: () => {}, + }; + + describe("MongoDocumentViewer Read-Only Affordances", () => { + it("disables 'Insert Document' button with tooltip and classes when isReadOnly is true", () => { + const vdom = renderViewerVdom({ + ...defaultViewerProps, + isReadOnly: true, + }); + + const insertButtons = findVdomNodes(vdom, (n) => { + const text = Array.isArray(n.props?.children) + ? n.props.children.join(" ") + : String(n.props?.children || ""); + return text.includes("Insert Document"); + }); + + assert.strictEqual(insertButtons.length, 1, "Must render exactly 1 Insert Document button"); + const [insertBtn] = insertButtons; + + assert.strictEqual( + insertBtn.props.disabled, + true, + "Insert Document must be disabled in read-only mode", + ); + assert.strictEqual( + insertBtn.props.title, + "Database is read-only", + "Insert Document must display 'Database is read-only' title", + ); + assert.ok( + insertBtn.props.className?.includes("disabled:opacity-50") && + insertBtn.props.className?.includes("disabled:cursor-not-allowed"), + `Insert Document button must include disabled styling classes, got: ${insertBtn.props.className}`, + ); + }); + + it("enables 'Insert Document' button when isReadOnly is false and not system collection", () => { + const vdom = renderViewerVdom({ + ...defaultViewerProps, + isReadOnly: false, + }); + + const insertButtons = findVdomNodes(vdom, (n) => { + const text = Array.isArray(n.props?.children) + ? n.props.children.join(" ") + : String(n.props?.children || ""); + return text.includes("Insert Document"); + }); + + assert.strictEqual(insertButtons.length, 1); + const [insertBtn] = insertButtons; + + assert.strictEqual( + insertBtn.props.disabled, + false, + "Insert Document must be enabled when writable", + ); + assert.strictEqual( + insertBtn.props.title, + undefined, + "Insert Document should have no read-only title", + ); + }); + + it("disables 'Insert Document' button on system collection regardless of isReadOnly", () => { + const vdom = renderViewerVdom({ + ...defaultViewerProps, + collection: "system.profile", + isReadOnly: false, + }); + + const insertButtons = findVdomNodes(vdom, (n) => { + const text = Array.isArray(n.props?.children) + ? n.props.children.join(" ") + : String(n.props?.children || ""); + return text.includes("Insert Document"); + }); + + assert.strictEqual(insertButtons.length, 1); + const [insertBtn] = insertButtons; + + assert.strictEqual( + insertBtn.props.disabled, + true, + "Insert Document must be disabled on system collections", + ); + }); + + it("suppresses row-level inline Edit and Delete action handlers in DataTable when isReadOnly is true", () => { + const vdom = renderViewerVdom({ + ...defaultViewerProps, + isReadOnly: true, + }); + + const dataTables = findVdomNodes(vdom, (n) => n.type === DataTable); + assert.strictEqual(dataTables.length, 1, "Must render DataTable"); + const [table] = dataTables; + + assert.strictEqual( + table.props.onEditRow, + undefined, + "DataTable must receive onEditRow=undefined when isReadOnly is true", + ); + assert.strictEqual( + table.props.onDeleteRow, + undefined, + "DataTable must receive onDeleteRow=undefined when isReadOnly is true", + ); + }); + + it("provides row-level inline Edit and Delete action handlers in DataTable when isReadOnly is false", () => { + const vdom = renderViewerVdom({ + ...defaultViewerProps, + isReadOnly: false, + }); + + const dataTables = findVdomNodes(vdom, (n) => n.type === DataTable); + assert.strictEqual(dataTables.length, 1, "Must render DataTable"); + const [table] = dataTables; + + assert.strictEqual( + typeof table.props.onEditRow, + "function", + "DataTable must receive onEditRow function when writable", + ); + assert.strictEqual( + typeof table.props.onDeleteRow, + "function", + "DataTable must receive onDeleteRow function when writable", + ); + }); + }); + + describe("MongoCollectionsSidebar Read-Only Affordances", () => { + const defaultSidebarProps = { + collections: [ + { name: "users", type: "collection" as const }, + { name: "logs", type: "collection" as const }, + { name: "system.views", type: "collection" as const }, + ], + selectedCollection: "users", + onSelectCollection: () => {}, + isLoading: false, + onRefresh: () => {}, + onCreateCollection: () => {}, + onDropCollection: () => {}, + }; + + it("disables 'Create new collection' button with title and classes when isReadOnly is true", () => { + const vdom = renderSidebarVdom({ + ...defaultSidebarProps, + isReadOnly: true, + }); + + const createButtons = findVdomNodes(vdom, (n) => n.props?.title === "Database is read-only"); + assert.strictEqual( + createButtons.length, + 1, + "Must find Create Collection button with read-only title", + ); + const [btn] = createButtons; + + assert.strictEqual(btn.props.disabled, true, "Create Collection button must be disabled"); + assert.ok( + btn.props.className?.includes("disabled:opacity-40") && + btn.props.className?.includes("disabled:cursor-not-allowed"), + `Create Collection button must have disabled classes, got: ${btn.props.className}`, + ); + }); + + it("enables 'Create new collection' button when isReadOnly is false", () => { + const vdom = renderSidebarVdom({ + ...defaultSidebarProps, + isReadOnly: false, + }); + + const createButtons = findVdomNodes(vdom, (n) => n.props?.title === "Create new collection"); + assert.strictEqual( + createButtons.length, + 1, + "Must find Create Collection button with active title", + ); + const [btn] = createButtons; + + assert.strictEqual( + btn.props.disabled, + false, + "Create Collection button must not be disabled", + ); + }); + + it("suppresses 'Drop Collection' context action buttons when isReadOnly is true", () => { + const vdom = renderSidebarVdom({ + ...defaultSidebarProps, + isReadOnly: true, + }); + + const dropButtons = findVdomNodes( + vdom, + (n) => typeof n.props?.title === "string" && n.props.title.startsWith("Drop "), + ); + assert.strictEqual( + dropButtons.length, + 0, + "Drop Collection buttons must be completely omitted from the DOM when isReadOnly is true", + ); + }); + + it("renders 'Drop Collection' context buttons only for non-system collections when writable", () => { + const vdom = renderSidebarVdom({ + ...defaultSidebarProps, + isReadOnly: false, + }); + + const dropButtons = findVdomNodes( + vdom, + (n) => typeof n.props?.title === "string" && n.props.title.startsWith("Drop "), + ); + // 'users' and 'logs' should have drop buttons, 'system.views' should NOT + assert.strictEqual( + dropButtons.length, + 2, + "Must render drop buttons for the 2 non-system collections", + ); + const titles = dropButtons.map((b) => b.props.title); + assert.ok(titles.includes("Drop users")); + assert.ok(titles.includes("Drop logs")); + assert.ok( + !titles.includes("Drop system.views"), + "System collection must never render drop button", + ); + }); + }); + + describe("MongoDataViewPanel Read-Only Propagation", () => { + it("propagates isEffectiveReadOnly down to MongoCollectionsSidebar and MongoDocumentViewer", () => { + const dataViewPanelPath = path.resolve( + process.cwd(), + "src/features/mongodb/components/MongoDataViewPanel.tsx", + ); + const content = fs.readFileSync(dataViewPanelPath, "utf-8"); + + assert.ok( + content.includes(" { + assert.strictEqual(formatMongoCardinality("one-to-one"), "1:1"); + assert.strictEqual(formatMongoCardinality("one-to-many"), "1:N"); + assert.strictEqual(formatMongoCardinality("many-to-one"), "N:1"); + assert.strictEqual(formatMongoCardinality("many-to-many"), "N:M"); + assert.strictEqual(formatMongoCardinality("insufficient-evidence"), "low-conf"); + assert.strictEqual(formatMongoCardinality("unknown"), "unknown"); +}); + +test("MongoErAdapter - Converts collections and relationships into ReactFlow nodes and styled edges", () => { + const confirmedRel: MongoRelationship = { + id: "orders.customer_id->customers._id", + sourceCollection: "orders", + sourceField: "customer_id", + targetCollection: "customers", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const inferredRel: MongoInferredRelationship = { + id: "orders.product_id->products._id", + sourceCollection: "orders", + sourceField: "product_id", + targetCollection: "products", + targetField: "_id", + cardinality: "many-to-one", + confidence: 0.88, + status: "inferred", + inferredAt: "2026-11-01T00:00:00Z", + evidence: { + sourceCollection: "orders", + sourceField: "product_id", + targetCollection: "products", + targetField: "_id", + sampleCount: 50, + hitCount: 48, + hitRatio: 0.96, + hasNamePatternMatch: true, + isBsonTypeMatch: true, + multipleSourcesPerTarget: true, + isArrayReference: false, + hasUniqueIndexOnSource: false, + sampleMatches: [{ sourceValue: "prod_1", targetId: "prod_1" }], + }, + }; + + const manualRel: MongoRelationship = { + id: "orders.created_by->users._id", + sourceCollection: "orders", + sourceField: "created_by", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + isManual: true, + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const { nodes, edges } = transformMongoToER({ + collections: ["orders", "customers", "products", "users"], + confirmed: [confirmedRel], + inferred: [inferredRel], + manual: [manualRel], + useDagreLayout: true, + }); + + // Assert nodes created + assert.strictEqual(nodes.length, 4); + const nodeIds = nodes.map((n) => n.id); + assert.ok(nodeIds.includes("orders")); + assert.ok(nodeIds.includes("customers")); + assert.ok(nodeIds.includes("products")); + assert.ok(nodeIds.includes("users")); + + // Assert 3 edges created + assert.strictEqual(edges.length, 3); + + // Confirmed edge styling + const confirmedEdge = edges.find((e) => e.target === "customers"); + assert.ok(confirmedEdge); + assert.strictEqual(confirmedEdge.style?.stroke, CONFIRMED_COLOR); + assert.strictEqual(confirmedEdge.style?.strokeDasharray, undefined); + assert.strictEqual(confirmedEdge.label, "N:1"); + + // Inferred edge styling + const inferredEdge = edges.find((e) => e.target === "products"); + assert.ok(inferredEdge); + assert.strictEqual(inferredEdge.style?.stroke, INFERRED_COLOR); + assert.strictEqual(inferredEdge.style?.strokeDasharray, "5,5"); + assert.strictEqual(inferredEdge.label, "N:1 (inferred)"); + + // Manual edge styling + const manualEdge = edges.find((e) => e.target === "users"); + assert.ok(manualEdge); + assert.strictEqual(manualEdge.style?.stroke, MANUAL_COLOR); + assert.strictEqual(manualEdge.label, "N:1 (manual)"); +}); + +test("MongoErAdapter - Generates foreignKeys and uniqueConstraints on TableNode data", () => { + const rel1: MongoRelationship = { + id: "orders.customer_id->customers._id", + sourceCollection: "orders", + sourceField: "customer_id", + targetCollection: "customers", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const relNonIdTarget: MongoRelationship = { + id: "orders.coupon_code->discounts.code", + sourceCollection: "orders", + sourceField: "coupon_code", + targetCollection: "discounts", + targetField: "code", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const { nodes } = transformMongoToER({ + collections: ["orders", "customers", "discounts"], + confirmed: [rel1, relNonIdTarget], + }); + + const ordersNode = nodes.find((n) => n.id === "orders"); + assert.ok(ordersNode); + assert.ok(Array.isArray(ordersNode.data.foreignKeys)); + assert.strictEqual(ordersNode.data.foreignKeys.length, 2); + + // Check foreign key fields for source handles + const fkCustomer = ordersNode.data.foreignKeys.find( + (fk: any) => fk.source_column === "customer_id", + ); + assert.ok(fkCustomer); + assert.strictEqual(fkCustomer.target_table, "customers"); + assert.strictEqual(fkCustomer.target_column, "_id"); + + // Check unique constraints on target discounts node for non-id target handle + const discountsNode = nodes.find((n) => n.id === "discounts"); + assert.ok(discountsNode); + assert.ok(Array.isArray(discountsNode.data.uniqueConstraints)); + assert.strictEqual(discountsNode.data.uniqueConstraints.length, 1); + assert.strictEqual(discountsNode.data.uniqueConstraints[0].column_name, "code"); + + // Customers target is _id, so it does not need non-id unique constraint + const customersNode = nodes.find((n) => n.id === "customers"); + assert.ok(customersNode); + assert.strictEqual(customersNode.data.uniqueConstraints.length, 0); +}); + +test("MongoErAdapter - Prevents duplicate edges between the same two fields", () => { + const confirmedRel: MongoRelationship = { + id: "orders.user_id->users._id", + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const duplicateInferredRel: MongoInferredRelationship = { + id: "orders.user_id->users._id", + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + confidence: 0.9, + status: "inferred", + inferredAt: "2026-11-01T00:00:00Z", + evidence: { + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + sampleCount: 50, + hitCount: 50, + hitRatio: 1.0, + hasNamePatternMatch: true, + isBsonTypeMatch: true, + multipleSourcesPerTarget: true, + isArrayReference: false, + hasUniqueIndexOnSource: false, + sampleMatches: [], + }, + }; + + const { edges } = transformMongoToER({ + collections: ["orders", "users"], + confirmed: [confirmedRel], + inferred: [duplicateInferredRel], + }); + + // Must only produce 1 edge, with confirmed taking precedence + assert.strictEqual(edges.length, 1); + assert.strictEqual(edges[0].style?.stroke, CONFIRMED_COLOR); +}); + +test("MongoErAdapter - Handles self-referencing relationships without throwing", () => { + const selfRel: MongoRelationship = { + id: "categories.parent_id->categories._id", + sourceCollection: "categories", + sourceField: "parent_id", + targetCollection: "categories", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + assert.doesNotThrow(() => { + const { nodes, edges } = transformMongoToER({ + collections: ["categories"], + confirmed: [selfRel], + useDagreLayout: true, + }); + assert.strictEqual(nodes.length, 1); + assert.strictEqual(edges.length, 1); + assert.strictEqual(edges[0].source, "categories"); + assert.strictEqual(edges[0].target, "categories"); + assert.strictEqual(edges[0].type, "smoothstep"); + }); +}); + +test("MongoErAdapter - Handles multiple distinct relationships between the same two collections", () => { + const rel1: MongoRelationship = { + id: "posts.author_id->users._id", + sourceCollection: "posts", + sourceField: "author_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const rel2: MongoRelationship = { + id: "posts.editor_id->users._id", + sourceCollection: "posts", + sourceField: "editor_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const { edges } = transformMongoToER({ + collections: ["posts", "users"], + confirmed: [rel1, rel2], + }); + + assert.strictEqual(edges.length, 2); + assert.strictEqual(edges[0].sourceHandle, "posts-author_id"); + assert.strictEqual(edges[1].sourceHandle, "posts-editor_id"); +}); + +test("MongoERDiagramView - Position preservation logic across dynamic node updates", () => { + const existingNodes = [ + { id: "orders", position: { x: 500, y: 350 }, data: { label: "orders" } }, + { id: "customers", position: { x: 100, y: 150 }, data: { label: "customers" } }, + ]; + + const incomingNodes = [ + { id: "orders", position: { x: 0, y: 0 }, data: { label: "orders" } }, + { id: "customers", position: { x: 0, y: 0 }, data: { label: "customers" } }, + { id: "products", position: { x: 300, y: 200 }, data: { label: "products" } }, + ]; + + // Apply position preservation mapping + const posMap = new Map(existingNodes.map((n) => [n.id, n.position])); + const synchronizedNodes = incomingNodes.map((node) => { + const existingPos = posMap.get(node.id); + return existingPos ? { ...node, position: existingPos } : node; + }); + + assert.strictEqual(synchronizedNodes.length, 3); + // Preserved user-adjusted positions + assert.deepStrictEqual(synchronizedNodes.find((n) => n.id === "orders")?.position, { + x: 500, + y: 350, + }); + assert.deepStrictEqual(synchronizedNodes.find((n) => n.id === "customers")?.position, { + x: 100, + y: 150, + }); + // New node keeps initial layout position + assert.deepStrictEqual(synchronizedNodes.find((n) => n.id === "products")?.position, { + x: 300, + y: 200, + }); +}); + +test("MongoCollectionRelationshipsTab - Excludes rejected relationships from incoming and outgoing lists", () => { + const rels: Array = [ + { + id: "orders.customer_id->customers._id", + sourceCollection: "orders", + sourceField: "customer_id", + targetCollection: "customers", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "", + updatedAt: "", + }, + { + id: "orders.rejected_id->users._id", + sourceCollection: "orders", + sourceField: "rejected_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + status: "rejected", + createdAt: "", + updatedAt: "", + } as any, + { + id: "items.order_id->orders._id", + sourceCollection: "items", + sourceField: "order_id", + targetCollection: "orders", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "", + updatedAt: "", + }, + ]; + + const rejectedIds = ["orders.rejected_id->users._id"]; + const collection = "orders"; + + const rejectedSet = new Set(rejectedIds); + const seen = new Set(); + const outgoing: any[] = []; + const incoming: any[] = []; + + for (const r of rels) { + if (rejectedSet.has(r.id)) continue; + if ("status" in r && (r as any).status === "rejected") continue; + if (seen.has(r.id)) continue; + seen.add(r.id); + + if (r.sourceCollection === collection) outgoing.push(r); + if (r.targetCollection === collection) incoming.push(r); + } + + assert.strictEqual(outgoing.length, 1); + assert.strictEqual(outgoing[0].id, "orders.customer_id->customers._id"); + assert.strictEqual(incoming.length, 1); + assert.strictEqual(incoming[0].id, "items.order_id->orders._id"); +}); + +test("MongoRelationshipDetailDrawer - Diagnostic candidate confirmation gating logic", () => { + const isCandidateInsufficient = (cardinality: string, confidence?: number) => { + return ( + cardinality === "insufficient-evidence" || + (typeof confidence === "number" && confidence < 0.65) + ); + }; + + // High confidence candidate -> not insufficient + assert.strictEqual(isCandidateInsufficient("many-to-one", 0.85), false); + + // Confidence < 0.65 -> insufficient + assert.strictEqual(isCandidateInsufficient("many-to-one", 0.55), true); + + // Insufficient-evidence cardinality -> insufficient + assert.strictEqual(isCandidateInsufficient("insufficient-evidence", 0.7), true); + + // Confirmation gating check + const canConfirmDirectly = (insufficient: boolean, acknowledged: boolean) => { + if (!insufficient) return true; + return acknowledged; + }; + + assert.strictEqual(canConfirmDirectly(false, false), true); + assert.strictEqual(canConfirmDirectly(true, false), false); + assert.strictEqual(canConfirmDirectly(true, true), true); +}); + +test("useMongoRelationships - React Query cache key consistency", () => { + const buildCacheKey = (dbId?: string, projectId?: string | null) => { + return ["mongoRelationships", dbId, projectId ?? undefined] as const; + }; + + const keyNull = buildCacheKey("conn_1", null); + const keyUndefined = buildCacheKey("conn_1", undefined); + const keyWithProject = buildCacheKey("conn_1", "proj_abc"); + + assert.deepStrictEqual(keyNull, keyUndefined); + assert.strictEqual(keyNull[2], undefined); + assert.strictEqual(keyWithProject[2], "proj_abc"); +}); + +test("MongoManualRelationshipDialog - Self-referencing field validation logic", () => { + const validateRelationship = ( + srcColl: string, + srcField: string, + tgtColl: string, + tgtField: string, + ): string | null => { + const sField = srcField.trim(); + const tField = tgtField.trim(); + if (!srcColl) return "Source collection is required."; + if (!sField) return "Source field is required."; + if (!tgtColl) return "Target collection is required."; + if (!tField) return "Target field is required."; + if (srcColl === tgtColl && sField === tField) { + return "Source and target cannot be the exact same field on the same collection."; + } + return null; + }; + + // Identical field on same collection fails + assert.strictEqual( + validateRelationship("users", "id", "users", "id"), + "Source and target cannot be the exact same field on the same collection.", + ); + + // Different field on same collection (e.g. parent_id -> _id) succeeds + assert.strictEqual(validateRelationship("categories", "parent_id", "categories", "_id"), null); + + // Different collection succeeds + assert.strictEqual(validateRelationship("orders", "customer_id", "customers", "_id"), null); +}); + +test("SQL ER Diagram - Zero regression on relational database panel items and schemas", () => { + const sqlItems = getDatabasePanelItems("postgres"); + const panels = sqlItems.map((i: any) => i.panel); + + assert.ok(panels.includes("er-diagram")); + assert.ok(!panels.includes("mongo-relationships")); +}); + +test("MongoErAdapter - transformMongoToER uses saved positions and skips Dagre when savedLayout is provided", () => { + const savedLayout: ERNode[] = [ + { tableId: "users", x: 150, y: 250, width: 300, height: 180 }, + { tableId: "orders", x: 600, y: 400, width: 280, height: 220 }, + ]; + + const confirmedRel: MongoRelationship = { + id: "orders.user_id->users._id", + sourceCollection: "orders", + sourceField: "user_id", + targetCollection: "users", + targetField: "_id", + cardinality: "many-to-one", + createdAt: "2026-11-01T00:00:00Z", + updatedAt: "2026-11-01T00:00:00Z", + }; + + const { nodes } = transformMongoToER({ + collections: ["users", "orders"], + confirmed: [confirmedRel], + useDagreLayout: true, + savedLayout, + }); + + const usersNode = nodes.find((n) => n.id === "users"); + const ordersNode = nodes.find((n) => n.id === "orders"); + + assert.ok(usersNode); + assert.ok(ordersNode); + + assert.strictEqual(usersNode.position.x, 150); + assert.strictEqual(usersNode.position.y, 250); + assert.strictEqual(usersNode.width, 300); + assert.strictEqual(usersNode.height, 180); + + assert.strictEqual(ordersNode.position.x, 600); + assert.strictEqual(ordersNode.position.y, 400); + assert.strictEqual(ordersNode.width, 280); + assert.strictEqual(ordersNode.height, 220); +}); diff --git a/src/features/mongodb/__tests__/MongoResultsViewer.test.ts b/src/features/mongodb/__tests__/MongoResultsViewer.test.ts new file mode 100644 index 00000000..ea27d01e --- /dev/null +++ b/src/features/mongodb/__tests__/MongoResultsViewer.test.ts @@ -0,0 +1,362 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + isBsonObject, + countExpandableNodes, + getAllExpandablePaths, + LARGE_TREE_NODE_LIMIT, + LARGE_TREE_DEPTH_LIMIT, +} from "../components/results/MongoTreeViewer"; +import { analyzeExplainPlan } from "../components/results/MongoExplainView"; +import { isExplainResult } from "../components/results/MongoResultsViewer"; +import { + MongoExplainResult, + MongoFindMqlResult, + MongoAggregateMqlResult, + MongoCountMqlResult, + MongoDistinctMqlResult, +} from "../types"; + +describe("MongoDB Results Viewer Unit Tests", () => { + describe("MongoTreeViewer — BSON and Hierarchy Analysis", () => { + it("differentiates BSON objects from ordinary nested objects", () => { + // BSON Extended JSON v2 objects + assert.strictEqual(isBsonObject({ $oid: "507f1f77bcf86cd799439011" }), true); + assert.strictEqual(isBsonObject({ $date: "2026-10-30T10:00:00.000Z" }), true); + assert.strictEqual(isBsonObject({ $date: { $numberLong: "1730282400000" } }), true); + assert.strictEqual(isBsonObject({ $numberLong: "9007199254740993" }), true); + assert.strictEqual(isBsonObject({ $numberDecimal: "123.456" }), true); + assert.strictEqual(isBsonObject({ $numberInt: "42" }), true); + assert.strictEqual(isBsonObject({ $numberDouble: "42.5" }), true); + assert.strictEqual(isBsonObject({ $binary: { base64: "SGVsbG8=", subType: "00" } }), true); + assert.strictEqual(isBsonObject({ $timestamp: { t: 1, i: 2 } }), true); + assert.strictEqual(isBsonObject({ $regex: "pattern", $options: "i" }), true); + assert.strictEqual(isBsonObject({ $undefined: true }), true); + + // Ordinary objects + assert.strictEqual(isBsonObject({ name: "Alice" }), false); + assert.strictEqual(isBsonObject({ user: { email: "a@b.com" } }), false); + assert.strictEqual(isBsonObject({ $where: "invalid" }), false); + assert.strictEqual(isBsonObject({}), false); + assert.strictEqual(isBsonObject([]), false); + assert.strictEqual(isBsonObject(null), false); + assert.strictEqual(isBsonObject(undefined), false); + assert.strictEqual(isBsonObject("507f1f77bcf86cd799439011"), false); + assert.strictEqual(isBsonObject(12345), false); + }); + + it("counts expandable nodes while treating BSON objects as terminals", () => { + const doc = { + _id: { $oid: "507f1f77bcf86cd799439011" }, // BSON leaf (not expandable) + user: { + name: "John", + createdAt: { $date: "2026-10-30T10:00:00.000Z" }, // BSON leaf + profile: { + bio: "Developer", + location: { + city: "San Francisco", + state: "CA", + }, + }, + }, + tags: ["mongodb", "database", "mql"], + }; + + // Root object (1), user (1), profile (1), location (1), tags (1) = 5 expandable nodes + const count = countExpandableNodes(doc); + assert.strictEqual(count, 5); + }); + + it("collects hierarchical paths with stable dot and bracket notation", () => { + const doc = { + title: "RelWave", + items: [ + { id: 1, val: "A" }, + { id: 2, val: "B" }, + ], + }; + + const paths = getAllExpandablePaths(doc, "$", 0, 10); + assert.deepStrictEqual(paths, ["$", "$.items", "$.items[0]", "$.items[1]"]); + }); + + it("respects large tree safeguard limits when expanding nodes", () => { + // Build a deeply nested structure exceeding limits + const hugeData: any = { level0: {} }; + let current = hugeData.level0; + for (let i = 1; i <= 10; i++) { + current[`level${i}`] = {}; + current = current[`level${i}`]; + } + + // Max depth limit + const pathsCappedByDepth = getAllExpandablePaths(hugeData, "$", 0, 3, 100); + assert.ok(pathsCappedByDepth.length <= 4); + + // Max node limit + const pathsCappedByNodes = getAllExpandablePaths(hugeData, "$", 0, 15, 2); + assert.strictEqual(pathsCappedByNodes.length, 2); + + assert.strictEqual(LARGE_TREE_NODE_LIMIT, 500); + assert.strictEqual(LARGE_TREE_DEPTH_LIMIT, 5); + }); + }); + + describe("MongoExplainView — Plan & Performance Analysis", () => { + it("accurately detects COLLSCAN and generates unindexed warning flag", () => { + const collScanExplain: MongoExplainResult = { + collection: "orders", + operation: "find", + executionTimeMs: 42, + queryPlanner: { + plannerVersion: 1, + namespace: "test.orders", + indexFilterSet: false, + winningPlan: { + stage: "COLLSCAN", + direction: "forward", + }, + }, + executionStats: { + executionSuccess: true, + nReturned: 10, + executionTimeMillis: 42, + totalKeysExamined: 0, + totalDocsExamined: 50000, + executionStages: { + stage: "COLLSCAN", + nReturned: 10, + docsExamined: 50000, + }, + }, + rawExplain: { explainVersion: "1" }, + }; + + const analysis = analyzeExplainPlan(collScanExplain); + + assert.strictEqual(analysis.winningStage, "COLLSCAN"); + assert.strictEqual(analysis.isCollScan, true); + assert.strictEqual(analysis.isIxScan, false); + assert.strictEqual(analysis.indexName, undefined); + assert.strictEqual(analysis.totalDocsExamined, 50000); + assert.strictEqual(analysis.totalKeysExamined, 0); + assert.strictEqual(analysis.nReturned, 10); + assert.strictEqual(analysis.executionTimeMs, 42); + assert.strictEqual(analysis.hasExecutionStats, true); + assert.strictEqual(analysis.highScanRatio, true); + assert.strictEqual(analysis.namespace, "test.orders"); + }); + + it("accurately detects nested IXSCAN inside FETCH stages and extracts index name", () => { + const ixScanExplain: MongoExplainResult = { + collection: "users", + operation: "find", + executionTimeMs: 3, + queryPlanner: { + plannerVersion: 1, + namespace: "test.users", + indexFilterSet: false, + winningPlan: { + stage: "FETCH", + inputStage: { + stage: "IXSCAN", + indexName: "email_unique_idx", + direction: "forward", + keyPattern: { email: 1 }, + }, + }, + }, + executionStats: { + executionSuccess: true, + nReturned: 1, + executionTimeMillis: 3, + totalKeysExamined: 1, + totalDocsExamined: 1, + executionStages: { + stage: "FETCH", + nReturned: 1, + docsExamined: 1, + inputStage: { + stage: "IXSCAN", + indexName: "email_unique_idx", + keysExamined: 1, + }, + }, + }, + rawExplain: { serverInfo: { version: "7.0.0" } }, + }; + + const analysis = analyzeExplainPlan(ixScanExplain); + + assert.strictEqual(analysis.winningStage, "FETCH"); + assert.strictEqual(analysis.isCollScan, false); + assert.strictEqual(analysis.isIxScan, true); + assert.strictEqual(analysis.indexName, "email_unique_idx"); + assert.strictEqual(analysis.totalDocsExamined, 1); + assert.strictEqual(analysis.totalKeysExamined, 1); + assert.strictEqual(analysis.nReturned, 1); + assert.strictEqual(analysis.executionTimeMs, 3); + assert.strictEqual(analysis.highScanRatio, false); + }); + + it("handles explain plans without executionStats (queryPlanner verbosity)", () => { + const plannerOnlyExplain: MongoExplainResult = { + collection: "audit_logs", + operation: "aggregate", + executionTimeMs: 5, + queryPlanner: { + plannerVersion: 1, + namespace: "test.audit_logs", + indexFilterSet: false, + winningPlan: { + stage: "COLLSCAN", + }, + rejectedPlans: [{ stage: "COLLSCAN" }], + }, + rawExplain: {}, + }; + + const analysis = analyzeExplainPlan(plannerOnlyExplain); + + assert.strictEqual(analysis.winningStage, "COLLSCAN"); + assert.strictEqual(analysis.isCollScan, true); + assert.strictEqual(analysis.hasExecutionStats, false); + assert.strictEqual(analysis.totalDocsExamined, undefined); + assert.strictEqual(analysis.nReturned, undefined); + assert.strictEqual(analysis.rejectedPlansCount, 1); + assert.strictEqual(analysis.executionTimeMs, 5); + }); + + it("flags high scan-to-return ratio when examined docs far exceed returned", () => { + const highRatioExplain: MongoExplainResult = { + collection: "events", + operation: "find", + executionTimeMs: 120, + queryPlanner: { + plannerVersion: 1, + namespace: "test.events", + indexFilterSet: false, + winningPlan: { + stage: "FETCH", + inputStage: { + stage: "IXSCAN", + indexName: "category_idx", + }, + }, + }, + executionStats: { + executionSuccess: true, + nReturned: 5, + executionTimeMillis: 120, + totalKeysExamined: 5000, + totalDocsExamined: 5000, + executionStages: { + stage: "FETCH", + nReturned: 5, + docsExamined: 5000, + }, + }, + rawExplain: {}, + }; + + const analysis = analyzeExplainPlan(highRatioExplain); + + assert.strictEqual(analysis.isIxScan, true); + assert.strictEqual(analysis.highScanRatio, true); + assert.strictEqual(analysis.efficiencyRatio, 1000); // 5000 / 5 + }); + }); + + describe("MongoResultsViewer — Result Type Guards & Contracts", () => { + it("accurately identifies explain results via isExplainResult", () => { + const explainRes: MongoExplainResult = { + collection: "coll", + operation: "find", + executionTimeMs: 10, + queryPlanner: { + plannerVersion: 1, + namespace: "db.coll", + indexFilterSet: false, + winningPlan: { stage: "COLLSCAN" }, + }, + rawExplain: {}, + }; + + assert.strictEqual(isExplainResult(explainRes), true); + + // Not explain results + const findRes: MongoFindMqlResult = { + operation: "find", + collection: "coll", + executionTimeMs: 15, + documents: [{ _id: 1 }], + totalReturned: 1, + }; + assert.strictEqual(isExplainResult(findRes), false); + + const aggRes: MongoAggregateMqlResult = { + operation: "aggregate", + collection: "coll", + executionTimeMs: 20, + documents: [], + stageCount: 2, + }; + assert.strictEqual(isExplainResult(aggRes), false); + + const countRes: MongoCountMqlResult = { + operation: "countDocuments", + collection: "coll", + executionTimeMs: 8, + count: 100, + }; + assert.strictEqual(isExplainResult(countRes), false); + + const distinctRes: MongoDistinctMqlResult = { + operation: "distinct", + collection: "coll", + executionTimeMs: 12, + field: "status", + values: ["active", "pending"], + count: 2, + }; + assert.strictEqual(isExplainResult(distinctRes), false); + + assert.strictEqual(isExplainResult(null), false); + assert.strictEqual(isExplainResult(undefined), false); + assert.strictEqual(isExplainResult(""), false); + assert.strictEqual(isExplainResult({}), false); + }); + + it("validates all 5 MQL execution result structures", () => { + const findResult: MongoFindMqlResult = { + operation: "find", + collection: "users", + executionTimeMs: 12, + documents: [{ _id: { $oid: "507f1f77bcf86cd799439011" }, name: "John" }], + totalReturned: 1, + }; + assert.strictEqual(findResult.operation, "find"); + assert.strictEqual(findResult.documents.length, 1); + + const countResult: MongoCountMqlResult = { + operation: "estimatedDocumentCount", + collection: "orders", + executionTimeMs: 1, + count: 15420, + }; + assert.strictEqual(countResult.operation, "estimatedDocumentCount"); + assert.strictEqual(countResult.count, 15420); + + const distinctResult: MongoDistinctMqlResult = { + operation: "distinct", + collection: "products", + executionTimeMs: 5, + field: "category", + values: ["electronics", "apparel", "books"], + count: 3, + }; + assert.strictEqual(distinctResult.operation, "distinct"); + assert.strictEqual(distinctResult.values.length, 3); + }); + }); +}); diff --git a/src/features/mongodb/__tests__/MongoWorkspaceCancellation.test.ts b/src/features/mongodb/__tests__/MongoWorkspaceCancellation.test.ts new file mode 100644 index 00000000..db804cfd --- /dev/null +++ b/src/features/mongodb/__tests__/MongoWorkspaceCancellation.test.ts @@ -0,0 +1,65 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { queryService } from "../../../services/bridge/query.ts"; + +describe("MongoWorkspaceCancellation Wiring", () => { + it("invokes queryService.cancelQuery with active sessionId when cancelled", async () => { + const originalCancel = queryService.cancelQuery; + let passedSessionId = ""; + queryService.cancelQuery = async (sid: string) => { + passedSessionId = sid; + return { cancelled: true }; + }; + + try { + const testSessionId = "test-session-123"; + const res = await queryService.cancelQuery(testSessionId); + + assert.equal(passedSessionId, testSessionId); + assert.deepEqual(res, { cancelled: true }); + } finally { + queryService.cancelQuery = originalCancel; + } + }); + + it("aborts AbortController signal on cancellation", () => { + const controller = new AbortController(); + assert.equal(controller.signal.aborted, false); + + controller.abort(); + assert.equal(controller.signal.aborted, true); + }); + + it("handles cancel when sessionId is missing by throwing error", async () => { + await assert.rejects(async () => { + await queryService.cancelQuery(""); + }, /sessionId is required/); + }); + + it("proves explain-view cancellation reaches active session and aborts correctly", async () => { + const originalCancel = queryService.cancelQuery; + let cancelCalledWith = ""; + queryService.cancelQuery = async (sid: string) => { + cancelCalledWith = sid; + return { cancelled: true }; + }; + + const explainAbortController = new AbortController(); + const activeExplainSessionId = "explain-session-uuid-456"; + + // Simulate explain cancellation trigger from MongoExplainView + const cancelExplain = async () => { + explainAbortController.abort(); + await queryService.cancelQuery(activeExplainSessionId); + }; + + try { + assert.equal(explainAbortController.signal.aborted, false); + await cancelExplain(); + assert.equal(explainAbortController.signal.aborted, true); + assert.equal(cancelCalledWith, activeExplainSessionId); + } finally { + queryService.cancelQuery = originalCancel; + } + }); +}); diff --git a/src/features/mongodb/__tests__/MongoWorkspaceIntegration.test.ts b/src/features/mongodb/__tests__/MongoWorkspaceIntegration.test.ts new file mode 100644 index 00000000..d63e0d9d --- /dev/null +++ b/src/features/mongodb/__tests__/MongoWorkspaceIntegration.test.ts @@ -0,0 +1,383 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + OPERATION_TEMPLATES, + MONGOSH_DEFAULT_TEMPLATE, + isExplainSupported, + getExplainDisabledReason, + EXPLAIN_SUPPORTED_OPERATIONS, + MongoWorkspaceOpType, +} from "../components/workspace/MongoWorkspacePanel"; +import { parseMongoshQuery } from "../parser"; +import { isExplainResult, analyzeExplainPlan } from "../components/results"; +import { + getHistory, + addHistoryEntry, + clearHistory, + getHistorySettings, + updateHistorySettings, + setStorageAdapterForTesting, + StorageAdapter, +} from "../history"; +import { + MongoFindMqlResult, + MongoAggregateMqlResult, + MongoCountMqlResult, + MongoDistinctMqlResult, + MongoExplainResult, +} from "../types"; + +function createMockStorage(): StorageAdapter { + const store = new Map(); + return { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => { + store.set(key, value); + }, + removeItem: (key: string) => { + store.delete(key); + }, + }; +} + +describe("MongoWorkspacePanel — Step 4 Integration Tests", () => { + let mockStorage: StorageAdapter; + + beforeEach(() => { + mockStorage = createMockStorage(); + setStorageAdapterForTesting(mockStorage); + }); + + describe("1. Input Mode Integration (JSON vs mongosh)", () => { + it("maintains operational templates for all 5 JSON operations", () => { + assert.strictEqual(typeof OPERATION_TEMPLATES.find, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.aggregate, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.countDocuments, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.estimatedDocumentCount, "string"); + assert.strictEqual(typeof OPERATION_TEMPLATES.distinct, "string"); + + const parsedFind = JSON.parse(OPERATION_TEMPLATES.find); + assert.deepStrictEqual(parsedFind, { filter: {} }); + + const parsedAgg = JSON.parse(OPERATION_TEMPLATES.aggregate); + assert.ok(Array.isArray(parsedAgg)); + assert.strictEqual(parsedAgg.length, 1); + assert.deepStrictEqual(parsedAgg[0], { $match: {} }); + }); + + it("provides idiomatic mongosh default template", () => { + assert.strictEqual(MONGOSH_DEFAULT_TEMPLATE, "db.users.find({})"); + + // Verify mongosh starter parses cleanly with the deterministic parser + const parsed = parseMongoshQuery(MONGOSH_DEFAULT_TEMPLATE); + assert.strictEqual(parsed.collection, "users"); + assert.strictEqual(parsed.op.operation, "find"); + assert.deepStrictEqual(parsed.op.payload.filter, {}); + }); + + it("verifies explain support restrictions across operations", () => { + assert.strictEqual(isExplainSupported("find"), true); + assert.strictEqual(isExplainSupported("aggregate"), true); + assert.strictEqual(isExplainSupported("countDocuments"), false); + assert.strictEqual(isExplainSupported("estimatedDocumentCount"), false); + assert.strictEqual(isExplainSupported("distinct"), false); + + assert.strictEqual(getExplainDisabledReason("find"), null); + assert.strictEqual(getExplainDisabledReason("aggregate"), null); + assert.ok( + getExplainDisabledReason("countDocuments")?.includes( + "only supported for find() and aggregate()", + ), + ); + assert.ok( + getExplainDisabledReason("distinct")?.includes("only supported for find() and aggregate()"), + ); + + assert.strictEqual(EXPLAIN_SUPPORTED_OPERATIONS.size, 2); + }); + }); + + describe("2. Mongosh Parser-to-RPC Pipeline Integration", () => { + it("parses valid mongosh query into strongly typed MQL operation", () => { + const query = `db.orders.find({ status: "PAID", total: { $gte: 100 } }).sort({ createdAt: -1 }).limit(25)`; + const parsed = parseMongoshQuery(query); + + assert.strictEqual(parsed.collection, "orders"); + assert.strictEqual(parsed.op.operation, "find"); + assert.deepStrictEqual(parsed.op.payload.filter, { + status: "PAID", + total: { $gte: 100 }, + }); + assert.deepStrictEqual(parsed.op.payload.sort, { createdAt: -1 }); + assert.strictEqual(parsed.op.payload.limit, 25); + }); + + it("rejects invalid mongosh query without invoking bridge RPC", () => { + const invalidQuery = `db.orders.find({ unclosedBrace: 123`; + assert.throws(() => { + parseMongoshQuery(invalidQuery); + }); + }); + + it("rejects unsafe JavaScript constructs before dispatch", () => { + const unsafeQuery = `db.users.find(); var x = 1;`; + assert.throws(() => { + parseMongoshQuery(unsafeQuery); + }); + }); + + it("enforces explain restrictions on mongosh operations", () => { + // 1. Valid find is explainable + const findQuery = `db.users.find({ active: true })`; + const findParsed = parseMongoshQuery(findQuery); + assert.strictEqual(isExplainSupported(findParsed.op.operation as MongoWorkspaceOpType), true); + + // 2. Valid aggregate is explainable + const aggQuery = `db.orders.aggregate([{ $match: { total: { $gt: 50 } } }])`; + const aggParsed = parseMongoshQuery(aggQuery); + assert.strictEqual(isExplainSupported(aggParsed.op.operation as MongoWorkspaceOpType), true); + + // 3. Count in mongosh is NOT explainable in M5 + const countQuery = `db.users.countDocuments({ active: true })`; + const countParsed = parseMongoshQuery(countQuery); + assert.strictEqual( + isExplainSupported(countParsed.op.operation as MongoWorkspaceOpType), + false, + ); + assert.ok( + getExplainDisabledReason(countParsed.op.operation as MongoWorkspaceOpType)?.includes( + "only supported for find()", + ), + ); + + // 4. Distinct in mongosh is NOT explainable in M5 + const distinctQuery = `db.users.distinct("email")`; + const distinctParsed = parseMongoshQuery(distinctQuery); + assert.strictEqual( + isExplainSupported(distinctParsed.op.operation as MongoWorkspaceOpType), + false, + ); + }); + }); + + describe("3. Results Viewer Integration Contracts", () => { + it("handles find and aggregate document array results", () => { + const findRes: MongoFindMqlResult = { + operation: "find", + collection: "users", + executionTimeMs: 14, + documents: [ + { _id: { $oid: "507f1f77bcf86cd799439011" }, name: "Alice", active: true }, + { _id: { $oid: "507f1f77bcf86cd799439012" }, name: "Bob", active: false }, + ], + totalReturned: 2, + }; + + assert.strictEqual(isExplainResult(findRes), false); + assert.strictEqual(findRes.documents.length, 2); + + const aggRes: MongoAggregateMqlResult = { + operation: "aggregate", + collection: "orders", + executionTimeMs: 32, + documents: [{ _id: "PAID", count: 120 }], + stageCount: 2, + }; + + assert.strictEqual(isExplainResult(aggRes), false); + assert.strictEqual(aggRes.documents.length, 1); + }); + + it("handles countDocuments and estimatedDocumentCount primitive metric results", () => { + const countRes: MongoCountMqlResult = { + operation: "countDocuments", + collection: "events", + executionTimeMs: 8, + count: 4210, + }; + + assert.strictEqual(isExplainResult(countRes), false); + assert.strictEqual(countRes.count, 4210); + + const estRes: MongoCountMqlResult = { + operation: "estimatedDocumentCount", + collection: "events", + executionTimeMs: 1, + count: 500000, + }; + + assert.strictEqual(isExplainResult(estRes), false); + assert.strictEqual(estRes.count, 500000); + }); + + it("handles distinct result structures", () => { + const distinctRes: MongoDistinctMqlResult = { + operation: "distinct", + collection: "products", + executionTimeMs: 9, + field: "brand", + values: ["Apple", "Samsung", "Sony"], + count: 3, + }; + + assert.strictEqual(isExplainResult(distinctRes), false); + assert.strictEqual(distinctRes.values.length, 3); + assert.strictEqual(distinctRes.field, "brand"); + }); + + it("handles structured explain result and plan analysis", () => { + const explainRes: MongoExplainResult = { + collection: "users", + operation: "find", + executionTimeMs: 6, + queryPlanner: { + plannerVersion: 1, + namespace: "test.users", + indexFilterSet: false, + winningPlan: { + stage: "FETCH", + inputStage: { + stage: "IXSCAN", + indexName: "email_1", + }, + }, + }, + executionStats: { + executionSuccess: true, + nReturned: 1, + executionTimeMillis: 6, + totalKeysExamined: 1, + totalDocsExamined: 1, + executionStages: { + stage: "FETCH", + docsExamined: 1, + inputStage: { stage: "IXSCAN", keysExamined: 1 }, + }, + }, + rawExplain: {}, + }; + + assert.strictEqual(isExplainResult(explainRes), true); + const analysis = analyzeExplainPlan(explainRes); + assert.strictEqual(analysis.winningStage, "FETCH"); + assert.strictEqual(analysis.isIxScan, true); + assert.strictEqual(analysis.indexName, "email_1"); + }); + }); + + describe("4. Query History Workspace Integration", () => { + it("persists execution metadata to history scoped to connection dbId", () => { + const dbId = "workspace-test-db"; + + // 1. Record success + addHistoryEntry(dbId, { + collection: "users", + operation: "find", + queryText: `db.users.find({ email: "secret@company.com" })`, + durationMs: 18, + status: "success", + resultCount: 1, + }); + + const history = getHistory(dbId); + assert.strictEqual(history.length, 1); + assert.strictEqual(history[0].collection, "users"); + assert.strictEqual(history[0].status, "success"); + assert.strictEqual(history[0].isRedacted, true); + assert.strictEqual(history[0].redactedQueryText, `db.users.find({ email: "" })`); + assert.ok(!history[0].redactedQueryText.includes("secret@company.com")); + }); + + it("persists failed execution metadata without sensitive information", () => { + const dbId = "workspace-fail-db"; + + addHistoryEntry(dbId, { + collection: "accounts", + operation: "find", + queryText: `db.accounts.find({ pin: 1234 })`, + durationMs: 0, + status: "error", + errorMessage: "Auth failed on mongodb://admin:secretPass@localhost:27017", + }); + + const history = getHistory(dbId); + assert.strictEqual(history.length, 1); + assert.strictEqual(history[0].status, "error"); + assert.ok(!history[0].errorMessage?.includes("secretPass")); + assert.ok(history[0].errorMessage?.includes("")); + assert.strictEqual(history[0].redactedQueryText, `db.accounts.find({ pin: })`); + }); + + it("respects raw query opt-in preference when user explicitly toggles", () => { + const dbId = "raw-optin-db"; + + // Disabled by default + assert.strictEqual(getHistorySettings(dbId).storeRawQueries, false); + + // Explicit opt-in + updateHistorySettings(dbId, { storeRawQueries: true }); + assert.strictEqual(getHistorySettings(dbId).storeRawQueries, true); + + addHistoryEntry(dbId, { + collection: "users", + operation: "find", + queryText: `db.users.find({ role: "ADMIN" })`, + durationMs: 10, + status: "success", + }); + + const history = getHistory(dbId); + assert.strictEqual(history[0].isRedacted, false); + assert.strictEqual(history[0].rawQueryText, `db.users.find({ role: "ADMIN" })`); + }); + + it("clears connection history completely", () => { + const dbId = "clear-db"; + addHistoryEntry(dbId, { + collection: "c", + operation: "find", + queryText: "{}", + durationMs: 1, + status: "success", + }); + + assert.strictEqual(getHistory(dbId).length, 1); + clearHistory(dbId); + assert.strictEqual(getHistory(dbId).length, 0); + }); + }); + + describe("5. Connection Switching & Isolation Guarantees", () => { + it("guarantees independent query history between distinct connections", () => { + const dbA = "conn-cluster-primary"; + const dbB = "conn-cluster-analytics"; + + addHistoryEntry(dbA, { + collection: "orders", + operation: "find", + queryText: "db.orders.find({})", + durationMs: 5, + status: "success", + }); + + addHistoryEntry(dbB, { + collection: "metrics", + operation: "aggregate", + queryText: "db.metrics.aggregate([])", + durationMs: 20, + status: "success", + }); + + assert.strictEqual(getHistory(dbA).length, 1); + assert.strictEqual(getHistory(dbA)[0].collection, "orders"); + + assert.strictEqual(getHistory(dbB).length, 1); + assert.strictEqual(getHistory(dbB)[0].collection, "metrics"); + + // Clear dbA must not affect dbB + clearHistory(dbA); + assert.strictEqual(getHistory(dbA).length, 0); + assert.strictEqual(getHistory(dbB).length, 1); + }); + }); +}); diff --git a/src/features/mongodb/__tests__/useDocumentDiff.test.ts b/src/features/mongodb/__tests__/useDocumentDiff.test.ts new file mode 100644 index 00000000..bad45f79 --- /dev/null +++ b/src/features/mongodb/__tests__/useDocumentDiff.test.ts @@ -0,0 +1,212 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { computeDocumentDiff } from "../hooks/useDocumentDiff"; + +describe("computeDocumentDiff (M4 Spec §4.2.1)", () => { + it("detects no-op when documents are identical", () => { + const doc = { _id: { $oid: "507f1f77bcf86cd799439011" }, name: "Acme", count: 10 }; + const res = computeDocumentDiff(doc, { ...doc }); + assert.equal(res.hasChanges, false); + assert.equal(res.updateDoc, undefined); + }); + + it("handles top-level field addition and modification", () => { + const orig = { _id: "1", name: "Alice" }; + const edited = { _id: "1", name: "Alice Smith", role: "admin" }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { name: "Alice Smith", role: "admin" }, + }); + }); + + it("emits recursive dot-notation for nested property changes", () => { + const orig = { + _id: "1", + address: { city: "Seattle", zip: "98101", country: "USA" }, + }; + const edited = { + _id: "1", + address: { city: "Boston", zip: "98101", country: "USA" }, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { "address.city": "Boston" }, + }); + }); + + it("emits dot-notation unset for nested deletions", () => { + const orig = { + _id: "1", + address: { city: "Seattle", zip: "98101" }, + }; + const edited = { + _id: "1", + address: { city: "Seattle" }, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $unset: { "address.zip": "" }, + }); + }); + + it("emits single parent unset on complete nested-object removal", () => { + const orig = { + _id: "1", + address: { city: "Seattle", zip: "98101" }, + active: true, + }; + const edited = { + _id: "1", + active: true, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $unset: { address: "" }, + }); + }); + + it("handles object-to-scalar transition without child unsets", () => { + const orig = { + _id: "1", + metadata: { author: "Alice", revision: 1 }, + }; + const edited = { + _id: "1", + metadata: "archived", + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { metadata: "archived" }, + }); + }); + + it("handles scalar-to-object transition", () => { + const orig = { + _id: "1", + metadata: "none", + }; + const edited = { + _id: "1", + metadata: { author: "Alice", revision: 1 }, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { metadata: { author: "Alice", revision: 1 } }, + }); + }); + + it("handles object-to-array transition", () => { + const orig = { + _id: "1", + items: { a: 1, b: 2 }, + }; + const edited = { + _id: "1", + items: [1, 2, 3], + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { items: [1, 2, 3] }, + }); + }); + + it("treats arrays as atomic units on modification", () => { + const orig = { + _id: "1", + tags: ["alpha", "beta", "gamma"], + }; + const edited = { + _id: "1", + tags: ["alpha", "delta"], + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { tags: ["alpha", "delta"] }, + }); + }); + + it("distinguishes explicit null ($set) from omitted key ($unset)", () => { + const orig = { + _id: "1", + fieldA: "valA", + fieldB: "valB", + }; + const edited = { + _id: "1", + fieldA: null, // explicit null + // fieldB omitted + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { fieldA: null }, + $unset: { fieldB: "" }, + }); + }); + + it("preserves empty object {} as $set", () => { + const orig = { + _id: "1", + metadata: { temp: 1 }, + }; + const edited = { + _id: "1", + metadata: {}, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { metadata: {} }, + }); + }); + + it("preserves BSON/EJSON tagged scalar types as atomic units", () => { + const orig = { + _id: { $oid: "507f1f77bcf86cd799439011" }, + createdAt: { $date: "2026-01-01T00:00:00.000Z" }, + amount: { $numberDecimal: "99.95" }, + }; + const edited = { + _id: { $oid: "507f1f77bcf86cd799439011" }, + createdAt: { $date: "2026-02-01T00:00:00.000Z" }, + amount: { $numberDecimal: "99.95" }, // unchanged + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { createdAt: { $date: "2026-02-01T00:00:00.000Z" } }, + }); + }); + + it("throws when _id is modified", () => { + const orig = { _id: { $oid: "507f1f77bcf86cd799439011" }, name: "A" }; + const edited = { _id: { $oid: "507f1f77bcf86cd799439022" }, name: "A" }; + assert.throws(() => computeDocumentDiff(orig, edited), /ID_MODIFICATION_PROHIBITED/); + }); + + it("throws when _id is deleted", () => { + const orig = { _id: "1", name: "A" }; + const edited = { name: "A" }; + assert.throws(() => computeDocumentDiff(orig, edited), /ID_MODIFICATION_PROHIBITED/); + }); + + it("rejects keys containing '$'", () => { + const orig = { _id: "1" }; + const edited = { _id: "1", $badKey: 123 }; + assert.throws(() => computeDocumentDiff(orig, edited), /INVALID_FIELD_PATH.*'\$badKey'/); + }); + + it("rejects keys containing invalid dot notation", () => { + const orig = { _id: "1" }; + const edited = { _id: "1", ".badKey": 123 }; + assert.throws(() => computeDocumentDiff(orig, edited), /INVALID_FIELD_PATH.*'\.badKey'/); + }); +}); diff --git a/src/features/mongodb/__tests__/useMongoBulkExport.test.ts b/src/features/mongodb/__tests__/useMongoBulkExport.test.ts new file mode 100644 index 00000000..dd543ede --- /dev/null +++ b/src/features/mongodb/__tests__/useMongoBulkExport.test.ts @@ -0,0 +1,792 @@ +/** + * MongoDB Milestone 7 Phase 4 — Bulk Export Hook Unit Tests + * + * Exercises the complete client-side export lifecycle: + * - Successful export and dynamic polling to completion + * - Export initialization failure + * - Polling interval progression and in-flight guard + * - Cancellation during export stream and polling sleep + * - Bridge MONGO_NOT_FOUND error handling + * - Stale response rejection + * - Max polling timeout / deadline enforcement + * - Duplicate export prevention + * - Duplicate cancellation deduplication + * - Save-to execution with saveMongoBulkExportTo + * - Save-to MONGO_FILE_EXISTS error and overwrite: true retry + * - Generic save failure preserving source on bridge + * - Unmount cleanup (clearing poller and dispatching cancel) + * - Resetting and cleaning up bridge operation + * - Stage 5 externalPolling suppression of internal timer + * - Stage 5 centralized store persistence across unmount (cancelOnUnmount: false) + * - Stage 5 single poller verification in delegated mode + * - Stage 5 terminal completion poller teardown + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { + useMongoBulkExport, + type BulkExportServiceContract, + type UseMongoBulkExportReturn, + INITIAL_EXPORT_PROGRESS, +} from "../hooks/useMongoBulkExport"; +import { createMongoBulkOperationsStore } from "../hooks/useMongoBulkOperationsStore"; +import type { + MongoBulkExportStartRequest, + MongoBulkGetOperationStatusRequest, + MongoBulkOperationStatusResult, + MongoBulkCancelRequest, + MongoBulkCleanupRequest, + MongoBulkExportSaveToRequest, + TrackedAsyncBulkOperation, +} from "../types/bulk"; + +function createMockExportService( + overrides?: Partial, +): BulkExportServiceContract & { + calls: { + start: MongoBulkExportStartRequest[]; + status: MongoBulkGetOperationStatusRequest[]; + cancel: MongoBulkCancelRequest[]; + cleanup: MongoBulkCleanupRequest[]; + save: MongoBulkExportSaveToRequest[]; + }; +} { + const calls = { + start: [] as MongoBulkExportStartRequest[], + status: [] as MongoBulkGetOperationStatusRequest[], + cancel: [] as MongoBulkCancelRequest[], + cleanup: [] as MongoBulkCleanupRequest[], + save: [] as MongoBulkExportSaveToRequest[], + }; + + const service: BulkExportServiceContract = { + async startMongoBulkExport(req) { + calls.start.push(req); + if (overrides?.startMongoBulkExport) { + return overrides.startMongoBulkExport(req); + } + return { + operationId: "mock-export-op-1", + status: "in_progress", + collection: req.collection, + format: req.format, + }; + }, + async getMongoBulkOperationStatus(req) { + calls.status.push(req); + if (overrides?.getMongoBulkOperationStatus) { + return overrides.getMongoBulkOperationStatus(req); + } + return { + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 100, + successfulDocuments: 100, + failedDocuments: 0, + fileSizeBytes: 2048, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + async cancelMongoBulkOperation(req) { + calls.cancel.push(req); + if (overrides?.cancelMongoBulkOperation) { + return overrides.cancelMongoBulkOperation(req); + } + return { + operationId: req.operationId, + status: "cancelled", + message: "Operation cancelled successfully", + }; + }, + async cleanupMongoBulkOperation(req) { + calls.cleanup.push(req); + if (overrides?.cleanupMongoBulkOperation) { + return overrides.cleanupMongoBulkOperation(req); + } + return { + operationId: req.operationId, + success: true, + }; + }, + async saveMongoBulkExportTo(req) { + calls.save.push(req); + if (overrides?.saveMongoBulkExportTo) { + return overrides.saveMongoBulkExportTo(req); + } + return { + operationId: req.operationId, + destinationPath: req.destinationPath, + bytesWritten: 2048, + success: true, + }; + }, + }; + + return Object.assign(service, { calls }); +} + +describe("useMongoBulkExport Hook & Polling Engine", () => { + it("initializes in idle state with initial progress values", () => { + let hook!: UseMongoBulkExportReturn; + function TestComp() { + hook = useMongoBulkExport("db-1", createMockExportService()); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + assert.equal(hook.state, "idle"); + assert.equal(hook.saveStatus, "not_saved"); + assert.equal(hook.operationId, null); + assert.equal(hook.error, null); + assert.equal(hook.warning, null); + assert.deepEqual(hook.progress, INITIAL_EXPORT_PROGRESS); + }); + + it("successfully starts export, polls bridge status, and transitions to ready-to-save upon completion", async () => { + let hook!: UseMongoBulkExportReturn; + let pollCount = 0; + + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => { + pollCount++; + if (pollCount === 1) { + return { + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "users", + format: "json", + processedDocuments: 50, + successfulDocuments: 50, + failedDocuments: 0, + fileSizeBytes: 1024, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + } + return { + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 100, + successfulDocuments: 100, + failedDocuments: 0, + fileSizeBytes: 2048, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 10, maxIntervalMs: 50 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + + assert.equal(mockService.calls.start.length, 1); + assert.equal(mockService.calls.start[0].collection, "users"); + + await new Promise((resolve) => setTimeout(resolve, 80)); + + assert.equal(hook.state, "ready-to-save"); + assert.equal(hook.progress.successfulDocuments, 100); + assert.equal(hook.progress.fileSizeBytes, 2048); + }); + + it("handles startMongoBulkExport failure and transitions to failed state", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + startMongoBulkExport: async () => { + throw new Error("Target collection is locked or inaccessible"); + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await assert.rejects(async () => { + await hook.startExport({ collection: "locked", format: "ndjson" }); + }, /Target collection is locked or inaccessible/); + + assert.equal(hook.state, "failed"); + assert.equal(hook.error, "Target collection is locked or inaccessible"); + }); + + it("prevents overlapping status requests using an in-flight guard", async () => { + let hook!: UseMongoBulkExportReturn; + let inFlightMax = 0; + let currentlyInFlight = 0; + + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => { + currentlyInFlight++; + if (currentlyInFlight > inFlightMax) { + inFlightMax = currentlyInFlight; + } + await new Promise((resolve) => setTimeout(resolve, 30)); + currentlyInFlight--; + return { + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "users", + format: "json", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 5, maxIntervalMs: 10 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 80)); + await hook.cancelExport(); + + assert.equal(inFlightMax, 1); + }); + + it("cancels active export and stops poller immediately", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => { + return { + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "orders", + format: "csv", + processedDocuments: 500, + successfulDocuments: 500, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 20 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "orders", format: "csv" }); + await hook.cancelExport(); + + assert.equal(hook.state, "cancelled"); + assert.equal(mockService.calls.cancel.length, 1); + }); + + it("handles MONGO_NOT_FOUND error by stopping poller and marking state as failed", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async () => { + const err = new Error("Operation expired or not found"); + (err as unknown as Record).code = "MONGO_NOT_FOUND"; + throw err; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 10 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "items", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + + assert.equal(hook.state, "failed"); + assert.ok(hook.error?.includes("no longer available")); + }); + + it("transitions to failed state if polling deadline timeout is exceeded", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "items", + format: "json", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }), + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { + pollingTimeoutMs: 25, + initialIntervalMs: 10, + maxIntervalMs: 15, + }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "items", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 60)); + + assert.equal(hook.state, "failed"); + assert.ok(hook.error?.includes("timed out")); + }); + + it("prevents duplicate concurrent exports", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + startMongoBulkExport: async (req) => { + await new Promise((resolve) => setTimeout(resolve, 40)); + return { + operationId: "mock-export-op-1", + status: "in_progress", + collection: req.collection, + format: req.format, + }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + const p1 = hook.startExport({ collection: "users", format: "json" }); + + await assert.rejects(async () => { + await hook.startExport({ collection: "users", format: "json" }); + }, /An export operation is already in progress/); + + await p1; + }); + + it("deduplicates concurrent cancelExport calls", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + cancelMongoBulkOperation: async (req) => { + await new Promise((resolve) => setTimeout(resolve, 30)); + return { operationId: req.operationId, status: "cancelled", message: "Cancelled" }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 50 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + + const c1 = hook.cancelExport(); + const c2 = hook.cancelExport(); + + assert.strictEqual(c1, c2); + await Promise.all([c1, c2]); + assert.equal(mockService.calls.cancel.length, 1); + }); + + it("executes saveToFile successfully and records destination and bytes written", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 200, + successfulDocuments: 200, + failedDocuments: 0, + fileSizeBytes: 4096, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }), + saveMongoBulkExportTo: async (req) => ({ + operationId: req.operationId, + destinationPath: req.destinationPath, + bytesWritten: 4096, + success: true, + warning: "Temporary staging cleanup queued for retry", + }), + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 10 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + + assert.equal(hook.state, "ready-to-save"); + + const saveResult = await hook.saveToFile("/exports/users.json", false); + + assert.equal(saveResult.success, true); + assert.equal(saveResult.bytesWritten, 4096); + assert.equal(hook.state, "completed"); + assert.equal(hook.saveStatus, "saved"); + assert.equal(hook.destinationPath, "/exports/users.json"); + assert.equal(hook.warning, "Temporary staging cleanup queued for retry"); + }); + + it("handles MONGO_FILE_EXISTS error and permits subsequent overwrite: true save", async () => { + let hook!: UseMongoBulkExportReturn; + let callCount = 0; + + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 200, + successfulDocuments: 200, + failedDocuments: 0, + fileSizeBytes: 4096, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }), + saveMongoBulkExportTo: async (req) => { + callCount++; + if (callCount === 1) { + const err = new Error("Destination file already exists"); + (err as unknown as Record).code = "MONGO_FILE_EXISTS"; + throw err; + } + return { + operationId: req.operationId, + destinationPath: req.destinationPath, + bytesWritten: 4096, + success: true, + }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 10 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + + await assert.rejects(async () => { + await hook.saveToFile("/exports/existing.json", false); + }, /Destination file already exists/); + + assert.equal(hook.state, "ready-to-save"); + assert.equal(hook.saveStatus, "not_saved"); + + const res = await hook.saveToFile("/exports/existing.json", true); + assert.equal(res.success, true); + assert.equal(hook.state, "completed"); + assert.equal(hook.saveStatus, "saved"); + assert.equal(mockService.calls.save[1].options?.overwrite, true); + }); + + it("handles generic save failure by preserving source export on bridge for retry", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 200, + successfulDocuments: 200, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }), + saveMongoBulkExportTo: async () => { + const err = new Error("Permission denied or disk full"); + (err as unknown as Record).code = "EACCES"; + throw err; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 10 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + + await assert.rejects(async () => { + await hook.saveToFile("/restricted/file.json", false); + }, /Permission denied or disk full/); + + assert.equal(hook.state, "failed"); + assert.equal(hook.saveStatus, "save_failed"); + assert.equal(mockService.calls.cleanup.length, 0); + }); + + it("cleans up bridge operation and resets state upon reset()", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }), + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 10 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + + await hook.reset(); + + assert.equal(hook.state, "idle"); + assert.equal(hook.operationId, null); + assert.equal(mockService.calls.cleanup.length, 1); + }); + + it("dispatches cancellation and clears timers upon component unmount while export is active", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "orders", + format: "csv", + processedDocuments: 250, + successfulDocuments: 250, + failedDocuments: 0, + fileSizeBytes: 5120, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }), + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 15 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "orders", format: "csv" }); + assert.equal(hook.state, "exporting"); + assert.equal(mockService.calls.cancel.length, 0); + + hook.unmount(); + + assert.equal(mockService.calls.cancel.length, 1); + assert.equal(mockService.calls.cancel[0].operationId, "mock-export-op-1"); + + const cancelCountBefore = mockService.calls.cancel.length; + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal(mockService.calls.cancel.length, cancelCountBefore); + }); + + it("drops stale polling responses and prevents overwriting newer state or progress", async () => { + let hook!: UseMongoBulkExportReturn; + let resolveStaleStatus!: (val: MongoBulkOperationStatusResult) => void; + const stalePromise = new Promise((res) => { + resolveStaleStatus = res; + }); + + let pollCount = 0; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => { + pollCount++; + if (pollCount === 1) { + return stalePromise; + } + return { + operationId: req.operationId, + type: "export", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 500, + successfulDocuments: 500, + failedDocuments: 0, + fileSizeBytes: 10240, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { initialIntervalMs: 5 }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + + await hook.cancelExport(); + assert.equal(hook.state, "cancelled"); + + resolveStaleStatus({ + operationId: "mock-export-op-1", + type: "export", + status: "in_progress", + collection: "users", + format: "json", + processedDocuments: 50, + successfulDocuments: 50, + failedDocuments: 0, + fileSizeBytes: 512, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal(hook.state, "cancelled"); + }); + + it("externalPolling: true suppresses internal hook polling timer", async () => { + let hook!: UseMongoBulkExportReturn; + const mockService = createMockExportService(); + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { + externalPolling: true, + initialIntervalMs: 10, + }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "users", format: "json" }); + assert.equal(hook.state, "exporting"); + assert.equal(mockService.calls.start.length, 1); + + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal( + mockService.calls.status.length, + 0, + "Internal hook poller must remain suppressed when externalPolling is true", + ); + }); + + it("centralized store continues polling after export dialog unmounts (cancelOnUnmount: false)", async () => { + let statusPollCount = 0; + const mockService = createMockExportService({ + getMongoBulkOperationStatus: async (req) => { + statusPollCount++; + return { + operationId: req.operationId, + type: "export", + status: statusPollCount >= 2 ? "completed" : "in_progress", + collection: "orders", + format: "json", + processedDocuments: statusPollCount * 50, + successfulDocuments: statusPollCount * 50, + failedDocuments: 0, + fileSizeBytes: 2048, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + const store = createMongoBulkOperationsStore(mockService); + let hook!: UseMongoBulkExportReturn; + + function TestComp() { + hook = useMongoBulkExport("db-1", mockService, { + cancelOnUnmount: false, + externalPolling: true, + onOperationStart: (op) => { + store.getSnapshot().registerAsyncOperation("db-1", op); + }, + }); + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + await hook.startExport({ collection: "orders", format: "json" }); + const opId = hook.operationId!; + assert.ok(opId); + + // Exactly one centralized poller exists in delegated mode + assert.equal(store._activePollers.size, 1); + assert.ok(store._activePollers.has(opId)); + + // Unmount export hook with cancelOnUnmount: false + hook.unmount(); + + // Operation is NOT cancelled on unmount + assert.equal(mockService.calls.cancel.length, 0); + + // Centralized poller continues running + await new Promise((resolve) => setTimeout(resolve, 1600)); + assert.ok(statusPollCount >= 2, "Centralized store must continue polling after hook unmount"); + + // Terminal completion stops centralized polling + const opRecord = store.getSnapshot().operationsByDb["db-1"][0] as TrackedAsyncBulkOperation; + assert.equal(opRecord.status, "completed"); + assert.equal( + store._activePollers.has(opId), + false, + "Centralized poller must halt upon terminal status", + ); + + store.getSnapshot().resetStore(); + }); +}); diff --git a/src/features/mongodb/__tests__/useMongoBulkImport.test.ts b/src/features/mongodb/__tests__/useMongoBulkImport.test.ts new file mode 100644 index 00000000..51b6e004 --- /dev/null +++ b/src/features/mongodb/__tests__/useMongoBulkImport.test.ts @@ -0,0 +1,1089 @@ +/** + * MongoDB Milestone 7 Phase 4 — Bulk Import Hook and Engine Unit Tests + * + * Exercises the complete client-side import lifecycle: + * - Pre-validation and file size limits + * - Zero-document rejection before bridge handshake + * - Handshake RPC and sequential chunk upload + * - Progress tracking accuracy + * - Transient retry mechanism with identical payloads + * - Terminal MONGO_NOT_FOUND and MONGO_VALIDATION_ERROR halt without retry + * - In-flight chunk awaiting and reconciliation on cancellation + * - Handshake cancellation safety (no orphaned operations) + * - Final operation completion and status reconciliation + * - Hook-level duplicate import prevention and cancellation deduplication + * - Stage 5 cancellation between chunks (halts reader, 0 subsequent chunks, 1 cancel RPC) + * - Stage 5 in-flight chunk success/failure progress reconciliation + * - Stage 5 delegated unmount (cancelOnUnmount: false) vs standalone unmount (cancelOnUnmount: true) + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import React from "react"; +import ReactDOMServer from "react-dom/server"; +import { + executeMongoBulkImport, + useMongoBulkImport, + type BulkImportServiceContract, + type ImportState, + type ImportProgress, + type UseMongoBulkImportReturn, + INITIAL_IMPORT_PROGRESS, +} from "../hooks/useMongoBulkImport"; +import { + ZeroDocumentImportError, + ImportLimitExceededError, + MAX_FILE_SIZE_BYTES, +} from "../utils/payloadAccounting"; +import type { + MongoBulkImportStartRequest, + MongoBulkImportChunkRequest, + MongoBulkCancelRequest, + MongoBulkCancelResult, + MongoBulkCleanupRequest, + MongoBulkGetOperationStatusRequest, +} from "../types/bulk"; + +function createMockService( + overrides?: Partial, +): BulkImportServiceContract & { + calls: { + start: MongoBulkImportStartRequest[]; + chunk: MongoBulkImportChunkRequest[]; + cancel: MongoBulkCancelRequest[]; + cleanup: MongoBulkCleanupRequest[]; + status: MongoBulkGetOperationStatusRequest[]; + }; +} { + const calls = { + start: [] as MongoBulkImportStartRequest[], + chunk: [] as MongoBulkImportChunkRequest[], + cancel: [] as MongoBulkCancelRequest[], + cleanup: [] as MongoBulkCleanupRequest[], + status: [] as MongoBulkGetOperationStatusRequest[], + }; + + const service: BulkImportServiceContract = { + async startMongoBulkImport(req) { + calls.start.push(req); + if (overrides?.startMongoBulkImport) { + return overrides.startMongoBulkImport(req); + } + return { + operationId: "mock-op-123", + status: "in_progress", + createdAt: new Date().toISOString(), + format: req.format, + collection: req.collection, + }; + }, + async importMongoBulkChunk(req) { + calls.chunk.push(req); + if (overrides?.importMongoBulkChunk) { + return overrides.importMongoBulkChunk(req); + } + return { + operationId: req.operationId, + chunkIndex: req.chunkIndex, + insertedCount: 10, + failedCount: 0, + totalInserted: 10, + totalFailed: 0, + status: req.isLastChunk ? "completed" : "in_progress", + isComplete: req.isLastChunk ?? false, + }; + }, + async cancelMongoBulkOperation(req) { + calls.cancel.push(req); + if (overrides?.cancelMongoBulkOperation) { + return overrides.cancelMongoBulkOperation(req); + } + return { + operationId: req.operationId, + status: "cancelled", + message: "Operation cancelled successfully", + cancelledAt: new Date().toISOString(), + }; + }, + async cleanupMongoBulkOperation(req) { + calls.cleanup.push(req); + if (overrides?.cleanupMongoBulkOperation) { + return overrides.cleanupMongoBulkOperation(req); + } + return { + operationId: req.operationId, + success: true, + }; + }, + async getMongoBulkOperationStatus(req) { + calls.status.push(req); + if (overrides?.getMongoBulkOperationStatus) { + return overrides.getMongoBulkOperationStatus(req); + } + return { + operationId: req.operationId, + type: "import", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date().toISOString(), + }; + }, + }; + + return Object.assign(service, { calls }); +} + +function createCallbackTracker() { + const stateTransitions: ImportState[] = []; + let latestProgress: ImportProgress = { ...INITIAL_IMPORT_PROGRESS }; + let latestError: string | null = null; + let latestOpId: string | null = null; + + return { + stateTransitions, + get progress() { + return latestProgress; + }, + get error() { + return latestError; + }, + get operationId() { + return latestOpId; + }, + callbacks: { + onStateChange: (s: ImportState) => { + stateTransitions.push(s); + }, + onProgressChange: (updater: (prev: ImportProgress) => ImportProgress) => { + latestProgress = updater(latestProgress); + }, + onErrorChange: (err: string | null) => { + latestError = err; + }, + onOperationIdChange: (opId: string | null) => { + latestOpId = opId; + }, + }, + }; +} + +describe("useMongoBulkImport / executeMongoBulkImport Workflow", () => { + it("successfully completes import for a valid JSON array", async () => { + const jsonContent = JSON.stringify([ + { name: "Alice", age: 30 }, + { name: "Bob", age: 25 }, + ]); + const file = new File([jsonContent], "users.json", { type: "application/json" }); + const mockService = createMockService({ + importMongoBulkChunk: async (req) => ({ + operationId: req.operationId, + chunkIndex: req.chunkIndex, + insertedCount: 2, + failedCount: 0, + totalInserted: 2, + totalFailed: 0, + status: "completed", + isComplete: true, + }), + }); + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + // Verify state progression + assert.deepEqual(tracker.stateTransitions, [ + "validating", + "starting", + "uploading", + "completed", + ]); + assert.equal(tracker.error, null); + assert.equal(tracker.operationId, "mock-op-123"); + + // Verify service calls + assert.equal(mockService.calls.start.length, 1); + assert.equal(mockService.calls.start[0].dbId, "db-1"); + assert.equal(mockService.calls.start[0].collection, "users"); + assert.equal(mockService.calls.start[0].format, "json"); + + assert.equal(mockService.calls.chunk.length, 1); + assert.equal(mockService.calls.chunk[0].chunkIndex, 0); + assert.equal(mockService.calls.chunk[0].isLastChunk, true); + + // Verify progress tracking + assert.equal(tracker.progress.documentsParsed, 2); + assert.equal(tracker.progress.documentsAcknowledged, 2); + assert.equal(tracker.progress.totalInserted, 2); + assert.equal(tracker.progress.totalFailed, 0); + }); + + it("successfully handles multi-chunk sequential uploads with correct chunkIndex and isLastChunk", async () => { + // Generate 1500 documents so reader creates 2 chunks (MAX_IMPORT_CHUNK_DOCS = 1000) + const docs = Array.from({ length: 1500 }, (_, i) => ({ id: i, name: `User_${i}` })); + const file = new File([JSON.stringify(docs)], "users_large.json", { type: "application/json" }); + + let runningInserted = 0; + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + const chunkDocs = JSON.parse(req.payload as string) as unknown[]; + runningInserted += chunkDocs.length; + return { + operationId: req.operationId, + chunkIndex: req.chunkIndex, + insertedCount: chunkDocs.length, + failedCount: 0, + totalInserted: runningInserted, + totalFailed: 0, + status: req.isLastChunk ? "completed" : "in_progress", + isComplete: req.isLastChunk ?? false, + }; + }, + }); + + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + assert.deepEqual(tracker.stateTransitions, [ + "validating", + "starting", + "uploading", + "completed", + ]); + assert.equal(mockService.calls.chunk.length, 2); + + // Chunk 0 + assert.equal(mockService.calls.chunk[0].chunkIndex, 0); + assert.equal(mockService.calls.chunk[0].isLastChunk, false); + + // Chunk 1 + assert.equal(mockService.calls.chunk[1].chunkIndex, 1); + assert.equal(mockService.calls.chunk[1].isLastChunk, true); + + // Final progress + assert.equal(tracker.progress.documentsParsed, 1500); + assert.equal(tracker.progress.documentsAcknowledged, 1500); + assert.equal(tracker.progress.totalInserted, 1500); + }); + + it("rejects files exceeding MAX_FILE_SIZE_BYTES (32 MB) before reader or handshake", async () => { + const oversizedFile = { + name: "giant.json", + size: MAX_FILE_SIZE_BYTES + 1024, + slice: () => new Blob(), + } as unknown as File; + + const mockService = createMockService(); + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects( + async () => { + await executeMongoBulkImport( + "db-1", + oversizedFile, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, + (err: Error) => { + assert.ok(err instanceof ImportLimitExceededError); + assert.ok(err.message.includes("exceeds maximum allowable size (32 MB)")); + return true; + }, + ); + + assert.deepEqual(tracker.stateTransitions, ["validating", "failed"]); + assert.equal(mockService.calls.start.length, 0); + assert.equal(mockService.calls.chunk.length, 0); + }); + + it("rejects 0-byte file before startMongoBulkImport handshake is called", async () => { + const emptyFile = new File([], "empty.json", { type: "application/json" }); + const mockService = createMockService(); + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects(async () => { + await executeMongoBulkImport( + "db-1", + emptyFile, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, ZeroDocumentImportError); + + assert.equal(mockService.calls.start.length, 0); + assert.equal(mockService.calls.chunk.length, 0); + assert.deepEqual(tracker.stateTransitions, ["validating", "failed"]); + }); + + it("rejects empty JSON array [] before startMongoBulkImport handshake is called", async () => { + const emptyArrayFile = new File(["[]"], "empty_array.json", { type: "application/json" }); + const mockService = createMockService(); + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects(async () => { + await executeMongoBulkImport( + "db-1", + emptyArrayFile, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, ZeroDocumentImportError); + + assert.equal(mockService.calls.start.length, 0); + assert.deepEqual(tracker.stateTransitions, ["validating", "failed"]); + }); + + it("rejects header-only CSV before startMongoBulkImport handshake is called", async () => { + const headerOnlyCsv = new File(["id,name,role\n"], "empty.csv", { type: "text/csv" }); + const mockService = createMockService(); + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects(async () => { + await executeMongoBulkImport( + "db-1", + headerOnlyCsv, + "users", + "csv", + abortController, + tracker.callbacks, + mockService, + ); + }, ZeroDocumentImportError); + + assert.equal(mockService.calls.start.length, 0); + assert.deepEqual(tracker.stateTransitions, ["validating", "failed"]); + }); + + it("retries transient transport errors (e.g. ECONNRESET, socket hang up) up to 2 times and succeeds", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "single.json", { + type: "application/json", + }); + + let attempts = 0; + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + attempts++; + if (attempts === 1) { + const err = new Error("socket hang up"); + (err as unknown as Record).code = "ECONNRESET"; + throw err; + } + return { + operationId: req.operationId, + chunkIndex: req.chunkIndex, + insertedCount: 1, + failedCount: 0, + totalInserted: 1, + totalFailed: 0, + status: "completed", + isComplete: true, + }; + }, + }); + + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + assert.equal(attempts, 2); + assert.equal(mockService.calls.chunk.length, 2); + assert.equal(mockService.calls.chunk[0].chunkIndex, 0); + assert.equal(mockService.calls.chunk[1].chunkIndex, 0); + assert.equal(mockService.calls.chunk[0].payload, mockService.calls.chunk[1].payload); + + assert.deepEqual(tracker.stateTransitions, [ + "validating", + "starting", + "uploading", + "completed", + ]); + assert.equal(tracker.progress.totalInserted, 1); + }); + + it("halts immediately on terminal non-transient errors (e.g. MONGO_VALIDATION_ERROR) without retrying", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "single.json", { + type: "application/json", + }); + + let attempts = 0; + const mockService = createMockService({ + importMongoBulkChunk: async () => { + attempts++; + const error = new Error("Document validation failed"); + (error as unknown as Record).code = "MONGO_VALIDATION_ERROR"; + throw error; + }, + }); + + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects( + async () => { + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, + (err: Error) => { + assert.ok(err.message.includes("Document validation failed")); + return true; + }, + ); + + assert.equal(attempts, 1); + assert.deepEqual(tracker.stateTransitions, ["validating", "starting", "uploading", "failed"]); + }); + + it("halts immediately on MONGO_NOT_FOUND without retrying", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "single.json", { + type: "application/json", + }); + + let attempts = 0; + const mockService = createMockService({ + importMongoBulkChunk: async () => { + attempts++; + const error = new Error( + "MONGO_NOT_FOUND: Operation session is no longer available in bridge memory", + ); + (error as unknown as Record).code = "MONGO_NOT_FOUND"; + throw error; + }, + }); + + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects( + async () => { + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, + (err: Error) => { + assert.ok(err.message.includes("MONGO_NOT_FOUND")); + return true; + }, + ); + + assert.equal(attempts, 1); + assert.deepEqual(tracker.stateTransitions, ["validating", "starting", "uploading", "failed"]); + }); + + it("awaits in-flight chunk when cancelled during upload and reconciles acknowledged progress", async () => { + const docs = Array.from({ length: 2500 }, (_, i) => ({ id: i })); + const file = new File([JSON.stringify(docs)], "large.json", { type: "application/json" }); + const abortController = new AbortController(); + + let inFlightChunkCompleted = false; + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + if (req.chunkIndex === 0) { + abortController.abort(); + await new Promise((resolve) => setTimeout(resolve, 20)); + inFlightChunkCompleted = true; + } + return { + operationId: req.operationId, + chunkIndex: req.chunkIndex, + insertedCount: 1000, + failedCount: 0, + totalInserted: 1000, + totalFailed: 0, + status: "in_progress", + isComplete: false, + }; + }, + }); + + const tracker = createCallbackTracker(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + assert.equal(inFlightChunkCompleted, true); + assert.equal(tracker.progress.documentsAcknowledged, 1000); + assert.equal(tracker.progress.totalInserted, 1000); + + assert.equal(mockService.calls.cancel.length, 1); + assert.equal(mockService.calls.cancel[0].operationId, "mock-op-123"); + assert.ok(tracker.stateTransitions.includes("cancelled")); + assert.equal(mockService.calls.chunk.length, 1); + }); + + it("cancellation between chunks halts reader, submits zero subsequent chunks, and dispatches 1 cancel RPC", async () => { + // 3000 documents -> 3 chunks (chunk 0, 1, 2) + const docs = Array.from({ length: 3000 }, (_, i) => ({ id: i })); + const file = new File([JSON.stringify(docs)], "multi_cancel.json", { + type: "application/json", + }); + const abortController = new AbortController(); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + return { + operationId: req.operationId, + chunkIndex: req.chunkIndex, + insertedCount: 1000, + failedCount: 0, + totalInserted: (req.chunkIndex + 1) * 1000, + totalFailed: 0, + status: "in_progress", + isComplete: false, + }; + }, + }); + + const tracker = createCallbackTracker(); + + // Trigger abort right after chunk 0 acknowledges + const origOnProgress = tracker.callbacks.onProgressChange; + tracker.callbacks.onProgressChange = (updater) => { + origOnProgress(updater); + if (tracker.progress.documentsAcknowledged === 1000) { + abortController.abort(); + } + }; + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + // Only chunk 0 was submitted; zero subsequent chunks (chunk 1 and 2 halted) + assert.equal(mockService.calls.chunk.length, 1); + assert.equal(mockService.calls.cancel.length, 1); + assert.equal(tracker.progress.documentsAcknowledged, 1000); + assert.equal(tracker.stateTransitions[tracker.stateTransitions.length - 1], "cancelled"); + }); + + it("failed in-flight chunk preserves previous acknowledged counts and transitions to cancelled", async () => { + const docs = Array.from({ length: 2500 }, (_, i) => ({ id: i })); + const file = new File([JSON.stringify(docs)], "chunk_fail_cancel.json", { + type: "application/json", + }); + const abortController = new AbortController(); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + if (req.chunkIndex === 0) { + return { + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 1000, + failedCount: 0, + totalInserted: 1000, + totalFailed: 0, + status: "in_progress", + isComplete: false, + }; + } + // Chunk 1 is aborted while in flight, then fails + abortController.abort(); + await new Promise((resolve) => setTimeout(resolve, 15)); + const err = new Error("Transport failure while aborting"); + (err as unknown as Record).code = "ECONNRESET"; + throw err; + }, + }); + + const tracker = createCallbackTracker(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + // Acknowledged count strictly preserved from successful chunk 0 + assert.equal(tracker.progress.documentsAcknowledged, 1000); + assert.equal(tracker.progress.totalInserted, 1000); + assert.equal(mockService.calls.cancel.length, 1); + assert.equal(tracker.stateTransitions[tracker.stateTransitions.length - 1], "cancelled"); + }); + + it("cancels bridge operation immediately if cancelled during startMongoBulkImport handshake", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "test.json", { type: "application/json" }); + const abortController = new AbortController(); + + const mockService = createMockService({ + startMongoBulkImport: async (req) => { + abortController.abort(); + return { + operationId: "handshake-cancelled-op-456", + status: "in_progress", + createdAt: new Date().toISOString(), + format: req.format, + collection: req.collection, + }; + }, + }); + + const tracker = createCallbackTracker(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + assert.equal(mockService.calls.cancel.length, 1); + assert.equal(mockService.calls.cancel[0].operationId, "handshake-cancelled-op-456"); + assert.equal(tracker.stateTransitions[tracker.stateTransitions.length - 1], "cancelled"); + assert.equal(mockService.calls.chunk.length, 0); + }); + + it("surfaces error and marks failed if bridge cancel returns non-cancelled status", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "test.json", { type: "application/json" }); + const abortController = new AbortController(); + + const mockService = createMockService({ + importMongoBulkChunk: async () => { + abortController.abort(); + return { + operationId: "mock-op-123", + chunkIndex: 0, + insertedCount: 1, + failedCount: 0, + totalInserted: 1, + totalFailed: 0, + status: "completed", + isComplete: true, + }; + }, + cancelMongoBulkOperation: async () => { + return { + operationId: "mock-op-123", + status: "failed", + message: "Internal error preventing cancellation", + } as unknown as MongoBulkCancelResult; + }, + }); + + const tracker = createCallbackTracker(); + + await assert.rejects(async () => { + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, /Bridge failed to cancel operation/); + + assert.equal(tracker.stateTransitions[tracker.stateTransitions.length - 1], "failed"); + }); + + it("reconciles completion via getMongoBulkOperationStatus when last chunk returns in_progress", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "test.json", { type: "application/json" }); + const abortController = new AbortController(); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => ({ + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 1, + failedCount: 0, + totalInserted: 1, + totalFailed: 0, + status: "in_progress", + isComplete: false, + }), + getMongoBulkOperationStatus: async (req) => ({ + operationId: req.operationId, + type: "import", + status: "completed", + collection: "users", + format: "json", + processedDocuments: 1, + successfulDocuments: 1, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date().toISOString(), + }), + }); + + const tracker = createCallbackTracker(); + + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + + assert.equal(mockService.calls.status.length, 1); + assert.deepEqual(tracker.stateTransitions, [ + "validating", + "starting", + "uploading", + "completed", + ]); + }); + + it("marks state as failed if final chunk reports status failed", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "test.json", { type: "application/json" }); + const abortController = new AbortController(); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => ({ + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 0, + failedCount: 1, + totalInserted: 0, + totalFailed: 1, + status: "failed", + isComplete: true, + }), + }); + + const tracker = createCallbackTracker(); + + await assert.rejects(async () => { + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, /Bulk import failed on final chunk/); + + assert.equal(tracker.stateTransitions[tracker.stateTransitions.length - 1], "failed"); + }); + + it("cleans up operation on bridge when import fails after retries exhausted (settled failure)", async () => { + const file = new File([JSON.stringify([{ id: 1 }])], "fail.json", { type: "application/json" }); + + let attempts = 0; + const mockService = createMockService({ + importMongoBulkChunk: async () => { + attempts++; + const err = new Error("Connection reset by peer"); + (err as unknown as Record).code = "ECONNRESET"; + throw err; + }, + }); + + const tracker = createCallbackTracker(); + const abortController = new AbortController(); + + await assert.rejects( + async () => { + await executeMongoBulkImport( + "db-1", + file, + "users", + "json", + abortController, + tracker.callbacks, + mockService, + ); + }, + (err: Error) => { + assert.equal(err.message, "Connection reset by peer"); + return true; + }, + ); + + assert.equal(attempts, 3); + assert.deepEqual(tracker.stateTransitions, ["validating", "starting", "uploading", "failed"]); + assert.equal(mockService.calls.cleanup.length, 1); + assert.equal(mockService.calls.cleanup[0].operationId, "mock-op-123"); + }); + + it("useMongoBulkImport hook prevents duplicate imports while an import is in progress", async () => { + let hookInstance!: UseMongoBulkImportReturn; + const file = new File([JSON.stringify([{ id: 1 }])], "test.json", { type: "application/json" }); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 1, + failedCount: 0, + totalInserted: 1, + totalFailed: 0, + status: "completed", + isComplete: true, + }; + }, + }); + + function TestComp() { + hookInstance = useMongoBulkImport("db-1", mockService); + return null; + } + + ReactDOMServer.renderToString(React.createElement(TestComp)); + + const firstPromise = hookInstance.startImport(file, "users", "json"); + + await assert.rejects(async () => { + await hookInstance.startImport(file, "users", "json"); + }, /An import operation is already in progress/); + + await firstPromise; + }); + + it("useMongoBulkImport hook deduplicates concurrent cancelImport calls", async () => { + let hookInstance!: UseMongoBulkImportReturn; + const file = new File([JSON.stringify([{ id: 1 }, { id: 2 }])], "test.json", { + type: "application/json", + }); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 2, + failedCount: 0, + totalInserted: 2, + totalFailed: 0, + status: "in_progress", + isComplete: false, + }; + }, + }); + + function TestComp() { + hookInstance = useMongoBulkImport("db-1", mockService); + return null; + } + + ReactDOMServer.renderToString(React.createElement(TestComp)); + + const importPromise = hookInstance.startImport(file, "users", "json"); + + await new Promise((resolve) => setTimeout(resolve, 10)); + + const cancelP1 = hookInstance.cancelImport(); + const cancelP2 = hookInstance.cancelImport(); + + assert.strictEqual(cancelP1, cancelP2); + + await Promise.all([cancelP1, cancelP2]); + try { + await importPromise; + } catch { + // ignore abort error + } + + assert.equal(mockService.calls.cancel.length, 1); + }); + + it("delegated unmount (cancelOnUnmount: false) does not abort or cancel background import", async () => { + let cleanupFn: (() => void) | undefined; + const internals = (React as any) + .__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + + let hookInstance!: UseMongoBulkImportReturn; + const file = new File([JSON.stringify([{ id: 1 }])], "delegated.json", { + type: "application/json", + }); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return { + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 1, + failedCount: 0, + totalInserted: 1, + totalFailed: 0, + status: "completed", + isComplete: true, + }; + }, + }); + + function TestComp() { + const origUseEffect = internals.H.useEffect; + internals.H.useEffect = (cb: any) => { + const res = cb(); + if (typeof res === "function") cleanupFn = res; + }; + try { + hookInstance = useMongoBulkImport("db-1", mockService, { cancelOnUnmount: false }); + } finally { + internals.H.useEffect = origUseEffect; + } + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + const importPromise = hookInstance.startImport(file, "users", "json"); + + // Unmount hook while import is running + assert.ok(cleanupFn, "Cleanup function must be captured"); + cleanupFn!(); + + await importPromise; + + // Zero cancel calls dispatched because cancelOnUnmount was false + assert.equal( + mockService.calls.cancel.length, + 0, + "No cancel RPC should be sent when cancelOnUnmount is false", + ); + }); + + it("standalone unmount (default cancelOnUnmount: true) dispatches cancel RPC to bridge", async () => { + let cleanupFn: (() => void) | undefined; + const internals = (React as any) + .__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + + let hookInstance!: UseMongoBulkImportReturn; + const file = new File([JSON.stringify([{ id: 1 }])], "standalone.json", { + type: "application/json", + }); + + const mockService = createMockService({ + importMongoBulkChunk: async (req) => { + await new Promise((resolve) => setTimeout(resolve, 40)); + return { + operationId: req.operationId, + chunkIndex: 0, + insertedCount: 1, + failedCount: 0, + totalInserted: 1, + totalFailed: 0, + status: "completed", + isComplete: true, + }; + }, + }); + + function TestComp() { + const origUseEffect = internals.H.useEffect; + internals.H.useEffect = (cb: any) => { + const res = cb(); + if (typeof res === "function") cleanupFn = res; + }; + try { + hookInstance = useMongoBulkImport("db-1", mockService); // default cancelOnUnmount: true + } finally { + internals.H.useEffect = origUseEffect; + } + return null; + } + ReactDOMServer.renderToString(React.createElement(TestComp)); + + const importPromise = hookInstance.startImport(file, "users", "json"); + + // Wait until start handshake completes and operationId is set + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Unmount hook + assert.ok(cleanupFn, "Cleanup function must be captured"); + cleanupFn!(); + + try { + await importPromise; + } catch { + // ignore + } + + // Cancel RPC was dispatched to bridge + assert.ok( + mockService.calls.cancel.length >= 1, + "Cancel RPC must be sent upon unmount when cancelOnUnmount is true", + ); + assert.equal(mockService.calls.cancel[0].operationId, "mock-op-123"); + }); +}); diff --git a/src/features/mongodb/__tests__/useMongoBulkMutations.test.ts b/src/features/mongodb/__tests__/useMongoBulkMutations.test.ts new file mode 100644 index 00000000..a97c3eea --- /dev/null +++ b/src/features/mongodb/__tests__/useMongoBulkMutations.test.ts @@ -0,0 +1,510 @@ +/** + * MongoDB Milestone 7 Phase 4 Stage 6 — Bulk Mutations Hook & Validation Tests + * + * Exercises the 14 test cases defined in implementation_plan.md §9.1: + * 1. Top-Level Empty Filter Distinction + * 2. Update Operator Validation + * 3. RPC Argument Integrity (Update) + * 4. RPC Argument Integrity (Delete) + * 5. Partial Bulk Insert Outcome + * 6. Defensive Error Sanitization + * 7. Defensive Response Handling + * 8. Bulk Insert Count Boundary Enforcement + * 9. Bulk Insert Size Boundary Enforcement + * 10. Prototype Pollution Rejection on Insert + * 11. Bridge Advisory Warning Capture + * 12. Bridge Error Handling + * 13. Poller Exclusion Invariant + * 14. Database Isolation & Duplicate Prevention + */ + +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + parseAndValidateFilter, + isTopLevelEmptyFilter, + parseAndValidateUpdate, + sanitizeErrorMessage, + sanitizeMutationErrors, +} from "../utils/mongoSemanticEmpty"; +import { + useMongoBulkMutations, + type BulkMutationsServiceContract, +} from "../hooks/useMongoBulkMutations"; +import { createMongoBulkOperationsStore } from "../hooks/useMongoBulkOperationsStore"; +import type { TrackedSyncBulkMutation } from "../types/bulk"; + +describe("useMongoBulkMutations and mongoSemanticEmpty Tests", () => { + let mockStore: ReturnType; + + beforeEach(() => { + mockStore = createMongoBulkOperationsStore(); + }); + + // 1. Top-Level Empty Filter Distinction + it("1. distinguishes top-level empty filter from nested empty objects and whitespace values", () => { + // Top-level empty + const resEmpty = parseAndValidateFilter("{}"); + assert.equal(resEmpty.valid, true); + assert.equal(resEmpty.isUnrestricted, true); + assert.equal(isTopLevelEmptyFilter(resEmpty.filter), true); + + // Nested empty object: restricted + const resNested = parseAndValidateFilter('{"profile": {}}'); + assert.equal(resNested.valid, true); + assert.equal(resNested.isUnrestricted, false); + assert.equal(isTopLevelEmptyFilter(resNested.filter), false); + + // Whitespace string value: restricted + const resWhitespace = parseAndValidateFilter('{"name": " "}'); + assert.equal(resWhitespace.valid, true); + assert.equal(resWhitespace.isUnrestricted, false); + + // Empty or whitespace-only inputs: validation error + const resBlank = parseAndValidateFilter(""); + assert.equal(resBlank.valid, false); + assert.ok(resBlank.error?.includes("required")); + + const resSpaces = parseAndValidateFilter(" "); + assert.equal(resSpaces.valid, false); + assert.ok(resSpaces.error?.includes("required")); + + // Malformed JSON: syntax error + const resMalformed = parseAndValidateFilter("{ invalid"); + assert.equal(resMalformed.valid, false); + assert.ok(resMalformed.error?.includes("Invalid JSON syntax")); + + // Non-object JSON: validation error + const resArray = parseAndValidateFilter("[1, 2]"); + assert.equal(resArray.valid, false); + assert.ok(resArray.error?.includes("valid JSON object")); + }); + + // 2. Update Operator Validation + it("2. validates update operators and rejects non-operator keys, empty objects, and prototype pollution", () => { + // Valid update operators + const resValid = parseAndValidateUpdate( + '{"$set": {"status": "active"}, "$inc": {"score": 1}, "$unset": {"temp": ""}}', + ); + assert.equal(resValid.valid, true); + assert.deepEqual(Object.keys(resValid.update!), ["$set", "$inc", "$unset"]); + + // Empty update document: rejected + const resEmpty = parseAndValidateUpdate("{}"); + assert.equal(resEmpty.valid, false); + assert.ok(resEmpty.error?.includes("No update operators provided")); + + // Non-operator direct replacement: rejected + const resReplacement = parseAndValidateUpdate('{"name": "Bob", "age": 30}'); + assert.equal(resReplacement.valid, false); + assert.ok(resReplacement.error?.includes("not an update operator")); + + // Prototype pollution in update: rejected + const resProto = parseAndValidateUpdate('{"$set": {"__proto__": {"admin": true}}}'); + assert.equal(resProto.valid, false); + assert.ok(resProto.error?.includes("Security violation")); + }); + + // 3. RPC Argument Integrity (Update) + it("3. verifies update RPC argument forwarding and conditional allowUnrestricted dispatch", async () => { + let capturedReq: any; + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkUpdateMongoDocuments: async (req) => { + capturedReq = req; + return { + collection: req.collection, + matchedCount: 5, + modifiedCount: 5, + }; + }, + }; + + const hook = useMongoBulkMutations("db-update", mockService); + + // Call 1: Restricted filter + await hook.executeBulkUpdate({ + collection: "users", + filter: { active: false }, + update: { $set: { active: true } }, + }); + assert.equal(capturedReq.dbId, "db-update"); + assert.equal(capturedReq.collection, "users"); + assert.deepEqual(capturedReq.filter, { active: false }); + assert.equal(capturedReq.options?.allowUnrestricted, undefined); + + // Call 2: Unrestricted filter with confirmed option + await hook.executeBulkUpdate({ + collection: "users", + filter: {}, + update: { $set: { version: 2 } }, + options: { allowUnrestricted: true }, + }); + assert.equal(capturedReq.options?.allowUnrestricted, true); + }); + + // 4. RPC Argument Integrity (Delete) + it("4. verifies delete RPC argument forwarding and conditional allowUnrestricted dispatch", async () => { + let capturedReq: any; + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkDeleteMongoDocuments: async (req) => { + capturedReq = req; + return { + collection: req.collection, + deletedCount: 12, + }; + }, + }; + + const hook = useMongoBulkMutations("db-delete", mockService); + + // Restricted delete + await hook.executeBulkDelete({ + collection: "logs", + filter: { level: "debug" }, + }); + assert.equal(capturedReq.collection, "logs"); + assert.deepEqual(capturedReq.filter, { level: "debug" }); + assert.equal(capturedReq.options?.allowUnrestricted, undefined); + + // Unrestricted delete + await hook.executeBulkDelete({ + collection: "logs", + filter: {}, + options: { allowUnrestricted: true }, + }); + assert.equal(capturedReq.options?.allowUnrestricted, true); + }); + + // 5. Partial Bulk Insert Outcome + it("5. classifies bulk insert outcomes accurately into completed, partial_success, and failed", async () => { + let returnOutcome: any; + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkInsertMongoDocuments: async () => returnOutcome, + }; + + const hook = useMongoBulkMutations("db-insert", mockService); + + // Outcome A: 100% completed + returnOutcome = { collection: "items", insertedCount: 2, failedCount: 0 }; + const resA = await hook.executeBulkInsert({ + collection: "items", + documents: [{ a: 1 }, { a: 2 }], + }); + assert.equal(resA?.status, "completed"); + + // Outcome B: Partial success (1 inserted, 1 failed) + returnOutcome = { + collection: "items", + insertedCount: 1, + failedCount: 1, + errors: [{ index: 1, code: "MONGO_DUPLICATE_KEY", message: "E11000 duplicate key error" }], + }; + const resB = await hook.executeBulkInsert({ + collection: "items", + documents: [{ a: 1 }, { a: 2 }], + }); + assert.equal(resB?.status, "partial_success"); + + // Outcome C: 100% failed (0 inserted, 2 failed) + returnOutcome = { + collection: "items", + insertedCount: 0, + failedCount: 2, + errors: [{ index: 0, code: "MONGO_WRITE_ERROR", message: "Write error" }], + }; + const resC = await hook.executeBulkInsert({ + collection: "items", + documents: [{ a: 1 }, { a: 2 }], + }); + assert.equal(resC?.status, "failed"); + }); + + // 6. Defensive Error Sanitization + it("6. bounds retained errors to 10 items, truncates messages, strips control characters, and redacts credentials", () => { + // Credential redaction + const uriMsg = + "Connection failed: mongodb://admin:secretPass123@cluster0.mongodb.net/test?ssl=true"; + const sanitizedUri = sanitizeErrorMessage(uriMsg); + assert.ok(!sanitizedUri.includes("secretPass123")); + assert.ok(sanitizedUri.includes("@")); + + // Control characters stripped + const ctrlMsg = "Error\x00with\x08null\x1Fbytes"; + assert.equal(sanitizeErrorMessage(ctrlMsg), "Errorwithnullbytes"); + + // Message length capped at 200 + const longMsg = "A".repeat(300); + const sanitizedLong = sanitizeErrorMessage(longMsg); + assert.equal(sanitizedLong.length, 200); + assert.ok(sanitizedLong.endsWith("...")); + + // Bounds error array to 10 entries while preserving totalErrors count + const manyErrors = Array.from({ length: 25 }, (_, idx) => ({ + index: idx, + code: "WRITE_ERR", + message: `Error number ${idx}`, + })); + const { sanitizedErrors, totalErrors } = sanitizeMutationErrors(manyErrors, 10); + assert.equal(totalErrors, 25); + assert.equal(sanitizedErrors.length, 10); + assert.equal(sanitizedErrors[0].index, 0); + assert.equal(sanitizedErrors[9].index, 9); + }); + + // 7. Defensive Response Handling + it("7. defensively handles malformed or inconsistent bridge responses", async () => { + let mockResponse: any; + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkUpdateMongoDocuments: async () => mockResponse, + bulkInsertMongoDocuments: async () => mockResponse, + }; + + const hook = useMongoBulkMutations("db-defensive", mockService); + + // Malformed non-object update response + mockResponse = null; + const resUpdateNull = await hook.executeBulkUpdate({ + collection: "c", + filter: { id: 1 }, + update: { $set: { a: 1 } }, + }); + assert.equal(resUpdateNull?.status, "failed"); + + // Negative count in update response + mockResponse = { matchedCount: -1, modifiedCount: 0 }; + const resUpdateNegative = await hook.executeBulkUpdate({ + collection: "c", + filter: { id: 1 }, + update: { $set: { a: 1 } }, + }); + assert.equal(resUpdateNegative?.status, "failed"); + + // 0/0 insert for non-empty array + mockResponse = { insertedCount: 0, failedCount: 0 }; + const resInsertZero = await hook.executeBulkInsert({ + collection: "c", + documents: [{ test: 1 }], + }); + assert.equal(resInsertZero?.status, "failed"); + + // Count mismatch (1 + 0 !== 2) + mockResponse = { insertedCount: 1, failedCount: 0 }; + const resInsertMismatch = await hook.executeBulkInsert({ + collection: "c", + documents: [{ test: 1 }, { test: 2 }], + }); + assert.equal(resInsertMismatch?.status, "failed"); + }); + + // 8. Bulk Insert Count Boundary Enforcement + it("8. enforces document count boundaries (0 rejected, 5,000 accepted, 5,001 rejected)", async () => { + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkInsertMongoDocuments: async () => ({ + collection: "boundary", + insertedCount: 5000, + failedCount: 0, + }), + }; + const hook = useMongoBulkMutations("db-boundary", mockService); + + // Empty array rejected before bridge call + await assert.rejects( + async () => hook.executeBulkInsert({ collection: "boundary", documents: [] }), + /cannot be empty/, + ); + + // Exactly 5,000 documents accepted + const docs5000 = Array.from({ length: 5000 }, (_, i) => ({ i })); + const res5000 = await hook.executeBulkInsert({ collection: "boundary", documents: docs5000 }); + assert.equal(res5000?.status, "completed"); + + // 5,001 documents rejected client-side + const docs5001 = Array.from({ length: 5001 }, (_, i) => ({ i })); + await assert.rejects( + async () => hook.executeBulkInsert({ collection: "boundary", documents: docs5001 }), + /exceeds maximum limit of 5,000 documents/, + ); + }); + + // 9. Bulk Insert Size Boundary Enforcement + it("9. enforces 16 MB byte payload boundaries client-side", async () => { + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkInsertMongoDocuments: async () => ({ + collection: "size", + insertedCount: 1, + failedCount: 0, + }), + }; + const hook = useMongoBulkMutations("db-size", mockService); + + // Exceeding 16 MB rejected client-side + // Create an object with a string property that pushes payload over 16MB + const bigString = "x".repeat(16 * 1024 * 1024 + 10); + const hugeDoc = [{ bigString }]; + + await assert.rejects( + async () => hook.executeBulkInsert({ collection: "size", documents: hugeDoc }), + /exceeds 16 MB limit/, + ); + }); + + // 10. Prototype Pollution Rejection on Insert + it("10. rejects documents containing prototype pollution keys before RPC dispatch", async () => { + let rpcCalled = false; + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkInsertMongoDocuments: async () => { + rpcCalled = true; + return { collection: "sec", insertedCount: 1, failedCount: 0 }; + }, + }; + const hook = useMongoBulkMutations("db-sec", mockService); + + const pollutedDoc = JSON.parse('{"name": "Alice", "__proto__": {"isAdmin": true}}'); + await assert.rejects( + async () => hook.executeBulkInsert({ collection: "sec", documents: [pollutedDoc] }), + /Security violation/, + ); + assert.equal(rpcCalled, false); + }); + + // 11. Bridge Advisory Warning Capture + it("11. captures bridge advisory warnings for unrestricted update and broad delete", async () => { + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkUpdateMongoDocuments: async () => ({ + collection: "p", + matchedCount: 200, + modifiedCount: 200, + advisoryWarning: "Unrestricted bulk update performed across entire collection", + }), + bulkDeleteMongoDocuments: async () => ({ + collection: "p", + deletedCount: 150, + advisoryWarning: "Broad delete operation affected more than 100 documents (150 deleted)", + }), + }; + const hook = useMongoBulkMutations("db-advisory", mockService); + + // Update advisory warning + const resUpdate = await hook.executeBulkUpdate({ + collection: "p", + filter: {}, + update: { $set: { touched: true } }, + options: { allowUnrestricted: true }, + }); + assert.equal(resUpdate?.status, "completed"); + const opsUpdate = mockStore.getSnapshot().operationsByDb["db-advisory"] || []; + assert.equal( + (opsUpdate[0] as TrackedSyncBulkMutation).advisoryWarning, + "Unrestricted bulk update performed across entire collection", + ); + + // Delete advisory warning + const resDelete = await hook.executeBulkDelete({ + collection: "p", + filter: {}, + options: { allowUnrestricted: true }, + }); + assert.equal(resDelete?.status, "completed"); + const opsDelete = mockStore.getSnapshot().operationsByDb["db-advisory"] || []; + assert.equal( + (opsDelete[0] as TrackedSyncBulkMutation).advisoryWarning, + "Broad delete operation affected more than 100 documents (150 deleted)", + ); + }); + + // 12. Bridge Error Handling + it("12. handles bridge rejection and records sanitized error in store", async () => { + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkUpdateMongoDocuments: async () => { + throw new Error("MONGO_READ_ONLY: Database connection is read-only"); + }, + }; + const hook = useMongoBulkMutations("db-err", mockService); + + const res = await hook.executeBulkUpdate({ + collection: "c", + filter: { a: 1 }, + update: { $set: { b: 2 } }, + }); + assert.equal(res?.status, "failed"); + assert.ok(res.error?.includes("MONGO_READ_ONLY")); + + const ops = mockStore.getSnapshot().operationsByDb["db-err"] || []; + assert.equal(ops[0].status, "failed"); + assert.ok(ops[0].error?.includes("MONGO_READ_ONLY")); + }); + + // 13. Poller Exclusion Invariant + it("13. ensures synchronous mutations are never registered in activePollers map", async () => { + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkUpdateMongoDocuments: async () => ({ + collection: "t", + matchedCount: 1, + modifiedCount: 1, + }), + }; + const hook = useMongoBulkMutations("db-poller-test", mockService); + + await hook.executeBulkUpdate({ + collection: "t", + filter: { a: 1 }, + update: { $set: { a: 2 } }, + }); + + const activePollers = (mockStore as any)._activePollers; + assert.equal(activePollers.size, 0); + }); + + // 14. Database Isolation & Duplicate Prevention + it("14. enforces database isolation and synchronously drops duplicate concurrent invocations", async () => { + let resolver: (val: any) => void; + const delayedPromise = new Promise((resolve) => { + resolver = resolve; + }); + + const mockService: BulkMutationsServiceContract = { + bulkStore: mockStore, + bulkUpdateMongoDocuments: async () => { + await delayedPromise; + return { collection: "users", matchedCount: 1, modifiedCount: 1 }; + }, + }; + + const hookDb1 = useMongoBulkMutations("db-iso-1", mockService); + + // Call 1 starts + const p1 = hookDb1.executeBulkUpdate({ + collection: "users", + filter: { id: 1 }, + update: { $set: { status: "A" } }, + }); + + // Call 2 concurrent invocation (should be dropped synchronously) + const p2 = hookDb1.executeBulkUpdate({ + collection: "users", + filter: { id: 1 }, + update: { $set: { status: "B" } }, + }); + assert.equal(await p2, undefined); + + // Release promise + resolver!({ collection: "users", matchedCount: 1, modifiedCount: 1 }); + await p1; + + // Database isolation: operations are strictly recorded under db-iso-1 + const db1Ops = mockStore.getSnapshot().operationsByDb["db-iso-1"] || []; + const db2Ops = mockStore.getSnapshot().operationsByDb["db-iso-2"] || []; + assert.equal(db1Ops.length, 1); + assert.equal(db2Ops.length, 0); + }); +}); diff --git a/src/features/mongodb/__tests__/useMongoBulkOperationsStore.test.ts b/src/features/mongodb/__tests__/useMongoBulkOperationsStore.test.ts new file mode 100644 index 00000000..ab1e7435 --- /dev/null +++ b/src/features/mongodb/__tests__/useMongoBulkOperationsStore.test.ts @@ -0,0 +1,903 @@ +/** + * MongoDB Milestone 7 Phase 4 Stage 5 — Store & Poller Unit Tests + * + * Exercises: + * - Immediate registration and fallback expiresAt + * - In-flight poller guard & request deduplication + * - Authoritative expiresAt reconciliation & immutability + * - Status transitions (pending -> in_progress -> completed) + * - Transient backoff (2s -> 4s -> 8s -> 10s) + * - 5+ consecutive errors transition to isUnreachable & 10s probing + * - Recovery from unreachable state on probe success + * - Manual retryPolling during in-flight (no duplicate RPC) + * - Bridge MONGO_NOT_FOUND stale transition and informative reason + * - Strictly sequenced import cancellation (shared AbortController, chunk settlement, single bridge RPC) + * - Single-flight concurrent cancellation promise deduplication + * - Expiration pruning (active operations immune, terminal pruned) + * - Database switching isolation + * - Store reset and beforeunload teardown + * - Deduplicated single poller per operationId + * - Multi-database independent polling isolation + * - Delayed poll response dropped after terminal/cancelled transition + */ + +import { describe, it, mock } from "node:test"; +import assert from "node:assert/strict"; +import { + createMongoBulkOperationsStore, + type BulkOperationsStoreServiceContract, +} from "../hooks/useMongoBulkOperationsStore"; +import type { TrackedAsyncBulkOperation, MongoBulkOperationStatusResult } from "../types/bulk"; + +function createMockService( + overrides?: Partial, +): BulkOperationsStoreServiceContract & { + calls: { + status: string[]; + cancel: string[]; + cleanup: string[]; + }; +} { + const calls = { + status: [] as string[], + cancel: [] as string[], + cleanup: [] as string[], + }; + + const service: BulkOperationsStoreServiceContract = { + async getMongoBulkOperationStatus(req) { + calls.status.push(req.operationId); + if (overrides?.getMongoBulkOperationStatus) { + return overrides.getMongoBulkOperationStatus(req); + } + return { + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "orders", + format: "json", + processedDocuments: 50, + successfulDocuments: 50, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: "2026-11-04T23:59:00.000Z", + }; + }, + async cancelMongoBulkOperation(req) { + calls.cancel.push(req.operationId); + if (overrides?.cancelMongoBulkOperation) { + return overrides.cancelMongoBulkOperation(req); + } + return { + operationId: req.operationId, + status: "cancelled", + message: "Operation cancelled", + }; + }, + async cleanupMongoBulkOperation(req) { + calls.cleanup.push(req.operationId); + if (overrides?.cleanupMongoBulkOperation) { + return overrides.cleanupMongoBulkOperation(req); + } + return { + operationId: req.operationId, + success: true, + }; + }, + }; + + return Object.assign(service, { calls }); +} + +describe("useMongoBulkOperationsStore Unit Tests", { concurrency: 1 }, () => { + it("registers operation with initial fallback expiresAt (+1 hr) and status pending", () => { + const service = createMockService(); + const store = createMongoBulkOperationsStore(service); + + const now = Date.now(); + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-test-1", + dbId: "db-alpha", + collection: "users", + type: "import", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: "", + }; + + store.getSnapshot().registerAsyncOperation("db-alpha", op); + + const registered = store.getSnapshot().operationsByDb[ + "db-alpha" + ][0] as TrackedAsyncBulkOperation; + assert.equal(registered.operationId, "op-test-1"); + assert.equal(registered.status, "pending"); + assert.equal(registered.hasAuthoritativeExpiresAt, false); + + const expiryTime = new Date(registered.expiresAt).getTime(); + assert.ok(expiryTime >= now + 3500000 && expiryTime <= now + 3700000); + }); + + it("authoritative expiresAt reconciles and locks bridge value on first status RPC", () => { + const service = createMockService(); + const store = createMongoBulkOperationsStore(service); + + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-test-2", + dbId: "db-alpha", + collection: "users", + type: "export", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + hasAuthoritativeExpiresAt: false, + }; + + store.getSnapshot().registerAsyncOperation("db-alpha", op); + + // Update with authoritative bridge response + const bridgeExpiresAt = "2026-11-04T23:45:00.000Z"; + store.getSnapshot().updateAsyncOperation("op-test-2", { + status: "in_progress", + expiresAt: bridgeExpiresAt, + hasAuthoritativeExpiresAt: true, + }); + + let current = store.getSnapshot().operationsByDb["db-alpha"][0] as TrackedAsyncBulkOperation; + assert.equal(current.expiresAt, bridgeExpiresAt); + assert.equal(current.hasAuthoritativeExpiresAt, true); + + // Immutability test: subsequent estimated patch must not overwrite authoritative bridge timestamp + store.getSnapshot().updateAsyncOperation("op-test-2", { + expiresAt: "2026-11-05T01:00:00.000Z", + hasAuthoritativeExpiresAt: false, + }); + + current = store.getSnapshot().operationsByDb["db-alpha"][0] as TrackedAsyncBulkOperation; + assert.equal(current.expiresAt, bridgeExpiresAt); + }); + + it("transitions status to stale on bridge MONGO_NOT_FOUND and does not remove record", () => { + const service = createMockService(); + const store = createMongoBulkOperationsStore(service); + + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-test-stale", + dbId: "db-alpha", + collection: "items", + type: "export", + status: "in_progress", + processedDocuments: 200, + successfulDocuments: 200, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-alpha", op); + store + .getSnapshot() + .markOperationStale("op-test-stale", "Operation session is no longer available."); + + const entry = store.getSnapshot().operationsByDb["db-alpha"][0] as TrackedAsyncBulkOperation; + assert.equal(entry.status, "stale"); + assert.ok(entry.error?.includes("no longer available")); + }); + + it("pruning with clearExpiredOperations keeps active operations strictly immune", () => { + const service = createMockService(); + const store = createMongoBulkOperationsStore(service); + + const pastDate = new Date(Date.now() - 10000).toISOString(); + + // Active operation with expired timestamp (due to clock skew) -> MUST NOT be pruned + const activeOp: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-active-clock-skew", + dbId: "db-alpha", + collection: "live_events", + type: "import", + status: "in_progress", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: pastDate, + updatedAt: pastDate, + expiresAt: pastDate, + }; + + // Terminal expired operation -> MUST be pruned + const terminalOp: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-terminal-expired", + dbId: "db-alpha", + collection: "logs", + type: "export", + status: "completed", + processedDocuments: 500, + successfulDocuments: 500, + failedDocuments: 0, + createdAt: pastDate, + updatedAt: pastDate, + expiresAt: pastDate, + }; + + // Terminal NOT expired operation -> MUST NOT be pruned + const futureDate = new Date(Date.now() + 60000).toISOString(); + const validTerminalOp: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-terminal-valid", + dbId: "db-alpha", + collection: "logs", + type: "export", + status: "completed", + processedDocuments: 500, + successfulDocuments: 500, + failedDocuments: 0, + createdAt: pastDate, + updatedAt: pastDate, + expiresAt: futureDate, + }; + + store.getSnapshot().registerAsyncOperation("db-alpha", activeOp); + store.getSnapshot().registerAsyncOperation("db-alpha", terminalOp); + store.getSnapshot().registerAsyncOperation("db-alpha", validTerminalOp); + + store.getSnapshot().clearExpiredOperations("db-alpha"); + + const remaining = store + .getSnapshot() + .operationsByDb["db-alpha"].map((o) => (o as TrackedAsyncBulkOperation).operationId); + assert.ok( + remaining.includes("op-active-clock-skew"), + "Active operation must be immune from pruning", + ); + assert.ok( + remaining.includes("op-terminal-valid"), + "Non-expired terminal operation must remain", + ); + assert.ok( + !remaining.includes("op-terminal-expired"), + "Expired terminal operation must be pruned", + ); + }); + + it("strictly sequenced import cancellation coordinates chunk settlement and single bridge RPC", async () => { + let cancelCalls = 0; + const service = createMockService({ + cancelMongoBulkOperation: async (req) => { + cancelCalls++; + return { operationId: req.operationId, status: "cancelled", message: "ok" }; + }, + }); + + const store = createMongoBulkOperationsStore(service); + const abortController = new AbortController(); + + let inFlightChunkFinished = false; + let cancelFnInvoked = false; + + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "import-op-cancel-seq", + dbId: "db-alpha", + collection: "users", + type: "import", + status: "in_progress", + processedDocuments: 50, + successfulDocuments: 50, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + const cancelFn = async () => { + cancelFnInvoked = true; + // Simulate awaiting in-flight chunk settlement + await new Promise((resolve) => setTimeout(resolve, 20)); + inFlightChunkFinished = true; + await service.cancelMongoBulkOperation({ operationId: op.operationId }); + }; + + store.getSnapshot().registerAsyncOperation("db-alpha", op, { + abortController, + cancelFn, + }); + + // Concurrent double cancel invocation + const p1 = store.getSnapshot().cancelOperation("import-op-cancel-seq"); + const p2 = store.getSnapshot().cancelOperation("import-op-cancel-seq"); + + assert.equal( + p1, + p2, + "Concurrent cancel requests must share the identical single-flight promise", + ); + + await p1; + + assert.equal( + abortController.signal.aborted, + true, + "AbortController must be signaled immediately", + ); + assert.equal(cancelFnInvoked, true); + assert.equal(inFlightChunkFinished, true); + assert.equal(cancelCalls, 1, "cancelMongoBulkOperation must be dispatched exactly once"); + + const finalOp = store.getSnapshot().operationsByDb["db-alpha"][0] as TrackedAsyncBulkOperation; + assert.equal(finalOp.status, "cancelled"); + }); + + it("creates only one poller per operationId even across repeated registrations", () => { + const service = createMockService(); + const store = createMongoBulkOperationsStore(service); + try { + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-dedupe-1", + dbId: "db-1", + collection: "users", + type: "export", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-1", op); + assert.equal(store._activePollers.size, 1); + const pollerControl = store._activePollers.get("op-dedupe-1"); + assert.ok(pollerControl); + + // Call startPoller multiple times + store._startPoller("op-dedupe-1"); + store._startPoller("op-dedupe-1"); + // Register the same operation again + store.getSnapshot().registerAsyncOperation("db-1", op); + + assert.equal(store._activePollers.size, 1); + assert.strictEqual(store._activePollers.get("op-dedupe-1"), pollerControl); + } finally { + store.getSnapshot().resetStore(); + } + }); + + it("in-flight guard blocks overlapping status RPC requests during execution", async () => { + let resolveStatus!: (val: MongoBulkOperationStatusResult) => void; + let statusCallCount = 0; + + const service = createMockService({ + getMongoBulkOperationStatus: async (_req) => { + statusCallCount++; + return new Promise((res) => { + resolveStatus = res; + }); + }, + }); + + const store = createMongoBulkOperationsStore(service); + try { + store._startPoller("op-inflight-guard"); + const control = store._activePollers.get("op-inflight-guard")!; + + // Trigger first poll via retryPolling + store.getSnapshot().retryPolling("op-inflight-guard"); + assert.equal(control.inFlight, true); + assert.equal(statusCallCount, 1); + + // Now call retryPolling again while inFlight is true + store.getSnapshot().retryPolling("op-inflight-guard"); + assert.equal( + statusCallCount, + 1, + "Duplicate RPC must not be dispatched while request is in flight", + ); + + // Resolve the pending status + resolveStatus({ + operationId: "op-inflight-guard", + type: "export", + status: "in_progress", + collection: "users", + format: "json", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(control.inFlight, false); + } finally { + store.getSnapshot().resetStore(); + } + }); + + it("backoff progression (2s, 4s, 8s, 10s), marks unreachable on 5th error, and probe restores polling", async () => { + mock.timers.enable({ apis: ["setTimeout"] }); + let pollAttempts = 0; + let failPolling = true; + + const service = createMockService({ + getMongoBulkOperationStatus: async (req) => { + pollAttempts++; + if (failPolling) { + throw new Error("Temporary network glitch"); + } + return { + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "orders", + format: "json", + processedDocuments: 200, + successfulDocuments: 200, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + const store = createMongoBulkOperationsStore(service); + try { + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-backoff-test", + dbId: "db-1", + collection: "orders", + type: "export", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-1", op); + const control = store._activePollers.get("op-backoff-test")!; + + const advance = async (ms: number) => { + mock.timers.tick(ms); + await new Promise((r) => setImmediate(r)); + }; + + // Initial delay is 500ms + await advance(500); + + // 1st error occurred -> consecutiveErrors = 1. Backoff delay should be 2000ms. + assert.equal(pollAttempts, 1); + assert.equal(control.consecutiveErrors, 1); + + await advance(1999); + assert.equal(pollAttempts, 1, "Should not trigger before 2000ms"); + + await advance(1); + // 2nd error -> consecutiveErrors = 2. Backoff delay should be 4000ms. + assert.equal(pollAttempts, 2); + assert.equal(control.consecutiveErrors, 2); + + await advance(4000); + // 3rd error -> consecutiveErrors = 3. Backoff delay should be 8000ms. + assert.equal(pollAttempts, 3); + assert.equal(control.consecutiveErrors, 3); + + await advance(8000); + // 4th error -> consecutiveErrors = 4. Backoff delay should be 10000ms. + assert.equal(pollAttempts, 4); + assert.equal(control.consecutiveErrors, 4); + + let opRecord = store.getSnapshot().operationsByDb["db-1"][0] as TrackedAsyncBulkOperation; + assert.equal(opRecord.isUnreachable ?? false, false); + + await advance(10000); + // 5th consecutive failure -> marks isUnreachable: true and schedules 10s probe + assert.equal(pollAttempts, 5); + assert.equal(control.consecutiveErrors, 5); + opRecord = store.getSnapshot().operationsByDb["db-1"][0] as TrackedAsyncBulkOperation; + assert.equal(opRecord.isUnreachable, true, "5th error must set isUnreachable to true"); + + // Unreachable probe interval is 10s: advancing by 9999ms should not trigger poll + await advance(9999); + assert.equal(pollAttempts, 5); + + // Make next probe succeed + failPolling = false; + await advance(1); + + assert.equal(pollAttempts, 6); + assert.equal(control.consecutiveErrors, 0, "Successful probe resets consecutive errors"); + opRecord = store.getSnapshot().operationsByDb["db-1"][0] as TrackedAsyncBulkOperation; + assert.equal(opRecord.isUnreachable, false, "Successful probe clears isUnreachable"); + + // Restores normal 1s polling interval + await advance(1000); + assert.equal(pollAttempts, 7, "Normal 1s dynamic polling restored after successful probe"); + } finally { + mock.timers.reset(); + store.getSnapshot().resetStore(); + } + }); + + it("terminal status transitions stop poller and clear its timers", () => { + const service = createMockService(); + const store = createMongoBulkOperationsStore(service); + + try { + const statuses: Array<"completed" | "failed" | "cancelled" | "stale"> = [ + "completed", + "failed", + "cancelled", + "stale", + ]; + + for (const termStatus of statuses) { + const opId = `op-term-${termStatus}`; + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: opId, + dbId: "db-1", + collection: "items", + type: "export", + status: "in_progress", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-1", op); + store._startPoller(opId); + assert.ok(store._activePollers.has(opId)); + + const control = store._activePollers.get(opId)!; + assert.ok( + control.timerId !== null, + `Timer must be active for ${opId} prior to terminal transition`, + ); + assert.equal( + control.isCancelled, + false, + `Poller must not be cancelled for ${opId} prior to transition`, + ); + + if (termStatus === "stale") { + store.getSnapshot().markOperationStale(opId, "session expired"); + } else { + store.getSnapshot().updateAsyncOperation(opId, { status: termStatus }); + } + + assert.equal( + control.timerId, + null, + `Timer must be cleared for ${opId} upon transition to ${termStatus}`, + ); + assert.equal( + control.isCancelled, + true, + `Poller must be marked cancelled for ${opId} upon transition to ${termStatus}`, + ); + assert.equal( + store._activePollers.has(opId), + false, + `Poller for ${opId} must be stopped and removed upon transition to ${termStatus}`, + ); + } + } finally { + store.getSnapshot().resetStore(); + } + }); + + it("bridge MONGO_NOT_FOUND during polling transitions operation to stale and clears its poller and timer", async () => { + mock.timers.enable({ apis: ["setTimeout"] }); + const service = createMockService({ + getMongoBulkOperationStatus: async () => { + const error = new Error("Operation session not found"); + (error as any).code = "MONGO_NOT_FOUND"; + throw error; + }, + }); + + const store = createMongoBulkOperationsStore(service); + try { + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-not-found-poll", + dbId: "db-1", + collection: "items", + type: "export", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-1", op); + const control = store._activePollers.get("op-not-found-poll")!; + assert.ok(control); + assert.ok(control.timerId !== null, "Timer must be scheduled initially"); + assert.equal(control.isCancelled, false); + + // Advance initial delay (500ms) + mock.timers.tick(500); + await new Promise((r) => setImmediate(r)); + + // Operation should be marked stale + const updatedOp = store.getSnapshot().operationsByDb["db-1"][0] as TrackedAsyncBulkOperation; + assert.equal(updatedOp.status, "stale"); + assert.ok(updatedOp.error?.includes("no longer available")); + + // Poller and timer must be cleared + assert.equal( + control.timerId, + null, + "Timer must be cleared upon MONGO_NOT_FOUND stale transition", + ); + assert.equal(control.isCancelled, true, "Poller control must be marked cancelled"); + assert.equal( + store._activePollers.has("op-not-found-poll"), + false, + "Poller must be removed from active pollers map", + ); + } finally { + mock.timers.reset(); + store.getSnapshot().resetStore(); + } + }); + + it("operations across multiple databases are polled independently and isolated", async () => { + mock.timers.enable({ apis: ["setTimeout"] }); + const statusCalls: string[] = []; + const service = createMockService({ + getMongoBulkOperationStatus: async (req) => { + statusCalls.push(req.operationId); + return { + operationId: req.operationId, + type: "export", + status: "in_progress", + collection: "data", + format: "json", + processedDocuments: 100, + successfulDocuments: 100, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + }, + }); + + const store = createMongoBulkOperationsStore(service); + try { + const advance = async (ms: number) => { + mock.timers.tick(ms); + await new Promise((r) => setImmediate(r)); + }; + + // 1. Initial active database context is db-1 + let activeDbId = "db-1"; + + const op1: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-db1-1", + dbId: "db-1", + collection: "users", + type: "export", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation(activeDbId, op1); + assert.ok(store._activePollers.has("op-db1-1")); + + // Execute initial poll tick (500ms) for db-1 operation + await advance(500); + const db1CallsBeforeSwitch = statusCalls.filter((id) => id === "op-db1-1").length; + assert.equal(db1CallsBeforeSwitch, 1, "db-1 operation must execute its initial poll"); + + // 2. Active database changes from db-1 to db-2 while db-1 operation is actively polling + activeDbId = "db-2"; + + const op2: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-db2-1", + dbId: "db-2", + collection: "logs", + type: "export", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation(activeDbId, op2); + assert.ok(store._activePollers.has("op-db2-1")); + + // Advance time by 1000ms: db-1 operation's recurring tick fires while active database is db-2 + await advance(1000); + + const db1CallsAfterSwitch = statusCalls.filter((id) => id === "op-db1-1").length; + assert.ok( + db1CallsAfterSwitch > db1CallsBeforeSwitch, + "db-1 polling must continue uninterrupted even when active database is changed to db-2", + ); + + // Verify isolated state snapshots + const snapshot = store.getSnapshot(); + assert.equal(snapshot.operationsByDb["db-1"].length, 1); + assert.equal(snapshot.operationsByDb["db-2"].length, 1); + assert.equal( + (snapshot.operationsByDb["db-1"][0] as TrackedAsyncBulkOperation).processedDocuments, + 100, + ); + + // Complete db-1 op + store.getSnapshot().updateAsyncOperation("op-db1-1", { status: "completed" }); + + // db-1 poller is stopped, but db-2 poller remains active and polling + assert.equal(store._activePollers.has("op-db1-1"), false); + assert.equal(store._activePollers.has("op-db2-1"), true); + } finally { + mock.timers.reset(); + store.getSnapshot().resetStore(); + } + }); + + it("delayed poll response cannot overwrite terminal or cancelled status", async () => { + let resolveDelayedStatus!: (val: MongoBulkOperationStatusResult) => void; + const delayedPromise = new Promise((res) => { + resolveDelayedStatus = res; + }); + + const service = createMockService({ + getMongoBulkOperationStatus: async () => delayedPromise, + }); + + const store = createMongoBulkOperationsStore(service); + try { + const op: TrackedAsyncBulkOperation = { + kind: "async", + operationId: "op-delayed-res", + dbId: "db-1", + collection: "users", + type: "export", + status: "in_progress", + processedDocuments: 10, + successfulDocuments: 10, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-1", op); + // Trigger in-flight poll via retryPolling + store.getSnapshot().retryPolling("op-delayed-res"); + + const control = store._activePollers.get("op-delayed-res"); + assert.ok(control); + assert.equal(control.inFlight, true); + + // Cancel operation while poll request is in flight + await store.getSnapshot().cancelOperation("op-delayed-res"); + + const opAfterCancel = store.getSnapshot().operationsByDb[ + "db-1" + ][0] as TrackedAsyncBulkOperation; + assert.equal(opAfterCancel.status, "cancelled"); + + // Now resolve the delayed status with an older 'in_progress' result + resolveDelayedStatus({ + operationId: "op-delayed-res", + type: "export", + status: "in_progress", + collection: "users", + format: "json", + processedDocuments: 50, + successfulDocuments: 50, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }); + + await Promise.resolve(); + await Promise.resolve(); + + const opFinal = store.getSnapshot().operationsByDb["db-1"][0] as TrackedAsyncBulkOperation; + assert.equal( + opFinal.status, + "cancelled", + "Delayed poll response must be discarded and cannot overwrite cancelled state", + ); + } finally { + store.getSnapshot().resetStore(); + } + }); + + it("maintains stable cached snapshot identity across repeated getSnapshot calls to avoid React re-render loops", () => { + const mockService = createMockService(); + const store = createMongoBulkOperationsStore(mockService); + + // Initial snapshot should be reference-equal on repeated calls + const snapshot1 = store.getSnapshot(); + const snapshot2 = store.getSnapshot(); + assert.strictEqual( + snapshot1, + snapshot2, + "getSnapshot must return cached reference when store has not mutated", + ); + + // Mutation triggers a snapshot reference change + const op: TrackedAsyncBulkOperation = { + kind: "async", + type: "export", + operationId: "op-snapshot-cache-1", + dbId: "db-snap", + collection: "users", + format: "json", + status: "pending", + processedDocuments: 0, + successfulDocuments: 0, + failedDocuments: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 3600000).toISOString(), + }; + + store.getSnapshot().registerAsyncOperation("db-snap", op); + + const snapshot3 = store.getSnapshot(); + assert.notStrictEqual( + snapshot3, + snapshot1, + "getSnapshot must return a new snapshot reference upon state mutation", + ); + + // After mutation, repeated calls return the new stable cached reference + const snapshot4 = store.getSnapshot(); + assert.strictEqual( + snapshot3, + snapshot4, + "getSnapshot must return identical reference on subsequent reads after mutation", + ); + + store.getSnapshot().resetStore(); + }); +}); diff --git a/src/features/mongodb/adapters/mongoErAdapter.ts b/src/features/mongodb/adapters/mongoErAdapter.ts new file mode 100644 index 00000000..ba03ce0c --- /dev/null +++ b/src/features/mongodb/adapters/mongoErAdapter.ts @@ -0,0 +1,294 @@ +import { MarkerType, Node, Edge } from "reactflow"; +import dagre from "dagre"; +import type { ERNode } from "@/features/project/types"; +import type { + MongoCardinality, + MongoInferredRelationship, + MongoRelationship, +} from "../types/relationship"; + +export interface MongoCollectionNodeItem { + name: string; + type?: "collection" | "view"; + fields?: Array<{ + path: string; + bsonType?: string; + }>; +} + +export interface MongoErAdapterOptions { + collections: MongoCollectionNodeItem[] | string[]; + confirmed?: MongoRelationship[]; + inferred?: MongoInferredRelationship[]; + manual?: MongoRelationship[]; + useDagreLayout?: boolean; + savedLayout?: ERNode[] | null; +} + +export function formatMongoCardinality(cardinality: MongoCardinality): string { + switch (cardinality) { + case "one-to-one": + return "1:1"; + case "one-to-many": + return "1:N"; + case "many-to-one": + return "N:1"; + case "many-to-many": + return "N:M"; + case "insufficient-evidence": + return "low-conf"; + default: + return "unknown"; + } +} + +export const CONFIRMED_COLOR = "#10b981"; +export const INFERRED_COLOR = "#06b6d4"; +export const MANUAL_COLOR = "#8b5cf6"; + +export function transformMongoToER({ + collections, + confirmed = [], + inferred = [], + manual = [], + useDagreLayout = true, + savedLayout = null, +}: MongoErAdapterOptions) { + const normCollections: MongoCollectionNodeItem[] = collections.map((c) => + typeof c === "string" ? { name: c } : c, + ); + + const collectionFields = new Map>(); + normCollections.forEach((c) => { + const fields = new Set(); + fields.add("_id"); + if (c.fields) { + c.fields.forEach((f) => fields.add(f.path)); + } + collectionFields.set(c.name, fields); + }); + + const registerField = (coll: string, field: string) => { + let fields = collectionFields.get(coll); + if (!fields) { + fields = new Set(["_id"]); + collectionFields.set(coll, fields); + } + if (field) fields.add(field); + }; + + confirmed.forEach((r) => { + registerField(r.sourceCollection, r.sourceField); + registerField(r.targetCollection, r.targetField); + }); + manual.forEach((r) => { + registerField(r.sourceCollection, r.sourceField); + registerField(r.targetCollection, r.targetField); + }); + inferred.forEach((r) => { + registerField(r.sourceCollection, r.sourceField); + registerField(r.targetCollection, r.targetField); + }); + + // Collect all relationships with confirmed/manual precedence + const allRels: Array<{ + rel: MongoRelationship | MongoInferredRelationship; + kind: "manual" | "confirmed" | "inferred"; + }> = [ + ...confirmed.map((r) => ({ rel: r, kind: "confirmed" as const })), + ...manual.map((r) => ({ rel: r, kind: "manual" as const })), + ...inferred.map((r) => ({ rel: r, kind: "inferred" as const })), + ]; + + // Deduplicate field-to-field relationship pairs + const dedupedRels: Array = []; + const seenFieldPairs = new Set(); + + for (const item of allRels) { + const pairKey = `${item.rel.sourceCollection}.${item.rel.sourceField}->${item.rel.targetCollection}.${item.rel.targetField}`; + if (!seenFieldPairs.has(pairKey)) { + seenFieldPairs.add(pairKey); + dedupedRels.push(item.rel); + } + } + + const layoutMap = new Map(); + if (savedLayout) { + savedLayout.forEach((n) => layoutMap.set(n.tableId, n)); + } + + const nodes: Node[] = []; + let index = 0; + + collectionFields.forEach((fieldSet, collName) => { + const columns = Array.from(fieldSet).map((f) => { + const isPk = f === "_id"; + return { + name: f, + type: isPk ? "objectId" : "string", + nullable: !isPk, + isPrimaryKey: isPk, + isForeignKey: !isPk, + }; + }); + + // Populate foreignKeys for outgoing connections so TableNode creates source handles + const outgoingRels = dedupedRels.filter((r) => r.sourceCollection === collName); + const foreignKeys = outgoingRels.map((r, idx) => ({ + constraint_name: r.id, + source_schema: "collection", + source_table: r.sourceCollection, + source_column: r.sourceField, + target_schema: "collection", + target_table: r.targetCollection, + target_column: r.targetField, + update_rule: "NO ACTION", + delete_rule: "NO ACTION", + ordinal_position: idx + 1, + })); + + // Populate uniqueConstraints for incoming connections targeting non-_id fields so TableNode creates target handles + const incomingRels = dedupedRels.filter( + (r) => r.targetCollection === collName && r.targetField !== "_id", + ); + const uniqueConstraints = incomingRels.map((r) => ({ + column_name: r.targetField, + constraint_name: r.id, + })); + + const savedNode = layoutMap.get(collName); + const width = savedNode?.width || 240; + const height = savedNode?.height || 40 + columns.length * 28 + 20; + const x = savedNode ? savedNode.x : (index % 3) * 320; + const y = savedNode ? savedNode.y : Math.floor(index / 3) * 260; + + nodes.push({ + id: collName, + type: "table", + position: { x, y }, + width, + height, + data: { + label: collName, + schema: "collection", + columns, + foreignKeys, + uniqueConstraints, + isHighlighted: false, + }, + }); + index++; + }); + + const edges: Edge[] = []; + const seenEdges = new Set(); + + for (const item of allRels) { + const r = item.rel; + const kind = item.kind; + const pairKey = `${r.sourceCollection}.${r.sourceField}->${r.targetCollection}.${r.targetField}`; + + if (seenEdges.has(pairKey)) { + continue; + } + seenEdges.add(pairKey); + + const sourceHandleId = `${r.sourceCollection}-${r.sourceField}`; + const targetHandleId = `${r.targetCollection}-${r.targetField}`; + const edgeId = `edge-${r.sourceCollection}.${r.sourceField}->${r.targetCollection}.${r.targetField}`; + + const cardLabel = formatMongoCardinality(r.cardinality); + let strokeColor = CONFIRMED_COLOR; + let strokeDasharray: string | undefined = undefined; + let label = `${cardLabel}`; + + if (kind === "confirmed") { + strokeColor = CONFIRMED_COLOR; + label = `${cardLabel}`; + } else if (kind === "inferred") { + strokeColor = INFERRED_COLOR; + strokeDasharray = "5,5"; + label = `${cardLabel} (inferred)`; + } else if (kind === "manual") { + strokeColor = MANUAL_COLOR; + label = `${cardLabel} (manual)`; + } + + edges.push({ + id: edgeId, + source: r.sourceCollection, + target: r.targetCollection, + sourceHandle: sourceHandleId, + targetHandle: targetHandleId, + animated: kind === "inferred", + style: { + stroke: strokeColor, + strokeWidth: 2, + strokeDasharray, + }, + markerEnd: { + type: MarkerType.ArrowClosed, + color: strokeColor, + width: 14, + height: 14, + }, + label, + labelStyle: { + fontSize: 10, + fontWeight: 600, + fill: strokeColor, + }, + labelBgStyle: { + fill: "rgba(15, 23, 42, 0.85)", + fillOpacity: 0.9, + }, + labelBgPadding: [4, 4], + labelBgBorderRadius: 4, + type: r.sourceCollection === r.targetCollection ? "smoothstep" : "default", + data: { + relationship: r, + kind, + }, + }); + } + + const hasSavedPositions = Boolean(savedLayout && savedLayout.length > 0); + if (useDagreLayout && nodes.length > 0 && !hasSavedPositions) { + const g = new dagre.graphlib.Graph(); + g.setDefaultEdgeLabel(() => ({})); + g.setGraph({ + rankdir: "LR", + nodesep: 80, + ranksep: 160, + marginx: 40, + marginy: 40, + }); + + nodes.forEach((n) => { + g.setNode(n.id, { + width: n.width || 240, + height: n.height || 200, + }); + }); + + edges.forEach((e) => { + if (e.source !== e.target) { + g.setEdge(e.source, e.target); + } + }); + + dagre.layout(g); + + nodes.forEach((n) => { + const pos = g.node(n.id); + if (pos) { + n.position = { + x: pos.x - (n.width || 240) / 2, + y: pos.y - (n.height || 200) / 2, + }; + } + }); + } + + return { nodes, edges }; +} diff --git a/src/features/mongodb/components/MongoCollectionsSidebar.tsx b/src/features/mongodb/components/MongoCollectionsSidebar.tsx new file mode 100644 index 00000000..228c324c --- /dev/null +++ b/src/features/mongodb/components/MongoCollectionsSidebar.tsx @@ -0,0 +1,142 @@ +import React, { useState, useMemo } from "react"; +import { MongoCollectionInfo } from "../types"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Layers, Search, RefreshCw, Eye, Plus, Trash2 } from "lucide-react"; + +export function MongoCollectionsSidebar({ + collections, + selectedCollection, + onSelectCollection, + isLoading, + onRefresh, + onCreateCollection, + onDropCollection, + isReadOnly, +}: { + collections: MongoCollectionInfo[]; + selectedCollection: string | null; + onSelectCollection: (name: string) => void; + isLoading: boolean; + onRefresh: () => void; + onCreateCollection?: () => void; + onDropCollection?: (name: string) => void; + isReadOnly?: boolean; +}) { + const [search, setSearch] = useState(""); + + const filtered = useMemo(() => { + return collections.filter((col) => + col.name.toLowerCase().includes(search.toLowerCase().trim()), + ); + }, [collections, search]); + + return ( +
+
+
+ + + Collections ({collections.length}) + +
+
+ {onCreateCollection && ( + + )} + +
+
+ +
+
+ + setSearch(e.target.value)} + placeholder="Search collections..." + className="pl-8 text-xs h-8 bg-background/50 border-border/40" + /> +
+
+ + +
+ {isLoading && collections.length === 0 ? ( +
+ Loading collections... +
+ ) : filtered.length === 0 ? ( +
+ No collections found +
+ ) : ( + filtered.map((col) => { + const isSelected = col.name === selectedCollection; + const isSystem = col.name.startsWith("system."); + return ( +
+ + +
+ {col.type === "view" && ( + + view + + )} + + {!isSystem && !isReadOnly && onDropCollection && ( + + )} +
+
+ ); + }) + )} +
+
+
+ ); +} diff --git a/src/features/mongodb/components/MongoCreateCollectionDialog.tsx b/src/features/mongodb/components/MongoCreateCollectionDialog.tsx new file mode 100644 index 00000000..fffcb089 --- /dev/null +++ b/src/features/mongodb/components/MongoCreateCollectionDialog.tsx @@ -0,0 +1,222 @@ +import React, { useState, useEffect } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Loader2, Plus, AlertCircle } from "lucide-react"; +import { toast } from "sonner"; +import { useCreateMongoCollection } from "../hooks/useMongoMutations"; +import { validateCollectionName, validateCappedOptions } from "../utils/validation"; + +interface MongoCreateCollectionDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + onCollectionCreated: (name: string) => void; +} + +export function MongoCreateCollectionDialog({ + open, + onOpenChange, + dbId, + onCollectionCreated, +}: MongoCreateCollectionDialogProps) { + const [collectionName, setCollectionName] = useState(""); + const [isCapped, setIsCapped] = useState(false); + const [cappedSize, setCappedSize] = useState("8192"); + const [cappedMax, setCappedMax] = useState(""); + const [error, setError] = useState(null); + + const createMutation = useCreateMongoCollection(dbId); + + useEffect(() => { + if (open) { + setCollectionName(""); + setIsCapped(false); + setCappedSize("8192"); + setCappedMax(""); + setError(null); + } + }, [open]); + + const handleNameChange = (val: string) => { + setCollectionName(val); + const res = validateCollectionName(val); + if (!res.valid) { + setError(res.error || "Invalid collection name"); + } else { + setError(null); + } + }; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + const nameCheck = validateCollectionName(collectionName); + if (!nameCheck.valid) { + setError(nameCheck.error || "Invalid collection name"); + return; + } + + let options: { capped?: boolean; size?: number; max?: number } | undefined = undefined; + if (isCapped) { + const sizeNum = parseInt(cappedSize, 10); + const maxNum = cappedMax.trim() ? parseInt(cappedMax, 10) : undefined; + options = { + capped: true, + size: sizeNum, + max: maxNum, + }; + const cappedCheck = validateCappedOptions(options); + if (!cappedCheck.valid) { + setError(cappedCheck.error || "Invalid capped options"); + return; + } + } + + try { + await createMutation.mutateAsync({ + collection: collectionName.trim(), + options, + }); + toast.success(`Collection "${collectionName.trim()}" created successfully`); + onCollectionCreated(collectionName.trim()); + onOpenChange(false); + } catch (err: any) { + setError(err.message || "Failed to create collection"); + toast.error("Collection creation failed", { description: err.message }); + } + }; + + return ( + + +
+ + + + Create Collection + + + Create a new MongoDB collection with optional capped storage parameters. + + + +
+ {/* Collection Name */} +
+ + handleNameChange(e.target.value)} + placeholder="e.g. orders, user_events" + className="font-mono text-xs h-9" + autoFocus + /> +

+ Letters, numbers, underscores, and hyphens (max 120 chars). +

+
+ + {/* Capped Collection Toggle */} +
+
+ +

+ Fixed-size collection that automatically overwrites oldest documents. +

+
+ +
+ + {/* Capped Options */} + {isCapped && ( +
+
+ + setCappedSize(e.target.value)} + className="font-mono text-xs h-8" + /> +

+ Must be at least 4096 bytes (4 KB). +

+
+ +
+ + setCappedMax(e.target.value)} + placeholder="Unlimited" + className="font-mono text-xs h-8" + /> +

+ Positive integer cap on document count. +

+
+
+ )} + + {error && ( +
+ + {error} +
+ )} +
+ + + + + +
+
+
+ ); +} diff --git a/src/features/mongodb/components/MongoDataViewPanel.tsx b/src/features/mongodb/components/MongoDataViewPanel.tsx new file mode 100644 index 00000000..1ec28bb2 --- /dev/null +++ b/src/features/mongodb/components/MongoDataViewPanel.tsx @@ -0,0 +1,289 @@ +import React, { useState, useRef } from "react"; +import { useMongoDatabaseDetails } from "../hooks/useMongoDatabaseDetails"; +import { useDatabase } from "@/features/project/hooks/useDbQueries"; +import { MongoCollectionsSidebar } from "./MongoCollectionsSidebar"; +import { MongoDocumentViewer } from "./MongoDocumentViewer"; +import { MongoSchemaPanel } from "./MongoSchemaPanel"; +import { MongoIndexesPanel } from "./indexes/MongoIndexesPanel"; +import { MongoInsertDocumentDialog } from "./MongoInsertDocumentDialog"; +import { MongoEditDocumentDialog } from "./MongoEditDocumentDialog"; +import { MongoDeleteDocumentDialog } from "./MongoDeleteDocumentDialog"; +import { MongoCreateCollectionDialog } from "./MongoCreateCollectionDialog"; +import { MongoDropCollectionDialog } from "./MongoDropCollectionDialog"; +import { MongoCollectionRelationshipsTab } from "./relationships"; +import { MongoBulkImportDialog, MongoBulkExportDialog, MongoBulkMutationDialog } from "./bulk"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { FileText, GitBranch, Database, Layers, Network } from "lucide-react"; + +export function MongoDataViewPanel({ + dbId, + databaseName, + projectId, + isReadOnly: isReadOnlyProp = false, +}: { + dbId: string; + databaseName: string; + projectId?: string | null; + isReadOnly?: boolean; +}) { + // Query database connection to resolve effective read-only mode if not explicitly passed + const { data: dbConn } = useDatabase(dbId); + const isEffectiveReadOnly = + isReadOnlyProp || Boolean((dbConn as any)?.readOnly || (dbConn as any)?.readonly); + + const [isBulkImportOpen, setIsBulkImportOpen] = useState(false); + const [isBulkExportOpen, setIsBulkExportOpen] = useState(false); + const [isBulkMutationOpen, setIsBulkMutationOpen] = useState(false); + const importCommittedCountRef = useRef(0); + + const { + collections, + collectionsLoading, + selectedCollection, + setSelectedCollection, + documents, + totalDocuments, + totalPages, + page, + setPage, + pageSize, + setPageSize, + sortField, + sortDirection, + handleSort, + filterInput, + setFilterInput, + appliedFilter, + applyFilter, + resetFilter, + activeTab, + setActiveTab, + isLoadingDocuments, + refetchDocuments, + refetchCollections, + isInsertDialogOpen, + setIsInsertDialogOpen, + editingDocument, + setEditingDocument, + deletingDocument, + setDeletingDocument, + isCreateCollectionOpen, + setIsCreateCollectionOpen, + droppingCollection, + setDroppingCollection, + handleCollectionCreated, + handleCollectionDropped, + } = useMongoDatabaseDetails(dbId); + + return ( +
+ {/* Top Banner / Collection Header */} +
+
+
+
+ +
+ {databaseName} +
+ + {selectedCollection && ( + <> + / + + {selectedCollection} + + + )} +
+ + {selectedCollection && ( + + setActiveTab(val as "documents" | "schema" | "indexes" | "relationships") + } + > + + + + Documents + + + + Inferred Schema + + + + Indexes + + + + Relationships + + + + )} +
+ + {/* Main split: Sidebar + Content */} +
+ setIsCreateCollectionOpen(true)} + onDropCollection={(name) => setDroppingCollection(name)} + isReadOnly={isEffectiveReadOnly} + /> + +
+ {!selectedCollection ? ( +
+ Select a collection from the sidebar to view documents. +
+ ) : activeTab === "documents" ? ( + setIsInsertDialogOpen(true)} + onEditDocument={(doc) => setEditingDocument(doc)} + onDeleteDocument={(doc) => setDeletingDocument(doc)} + onImportClick={() => setIsBulkImportOpen(true)} + onExportClick={() => setIsBulkExportOpen(true)} + onBulkMutationClick={() => setIsBulkMutationOpen(true)} + isReadOnly={isEffectiveReadOnly} + /> + ) : activeTab === "schema" ? ( + + ) : activeTab === "relationships" ? ( + + ) : ( + + )} +
+
+ + {/* M4 Mutation Modals */} + {selectedCollection && ( + <> + + !open && setEditingDocument(null)} + dbId={dbId} + collection={selectedCollection} + document={editingDocument} + onRefreshView={refetchDocuments} + /> + !open && setDeletingDocument(null)} + dbId={dbId} + collection={selectedCollection} + document={deletingDocument} + onRefreshView={refetchDocuments} + /> + + )} + + {/* Bulk Operations Dialogs */} + {selectedCollection && ( + <> + { + importCommittedCountRef.current = progress.totalInserted; + }} + onTerminalState={(_opId, status) => { + if (status === "completed" || importCommittedCountRef.current > 0) { + refetchDocuments(); + refetchCollections(); + importCommittedCountRef.current = 0; + } + }} + /> + + c.name)} + isReadOnly={isEffectiveReadOnly} + /> + + )} + + + + {droppingCollection && ( + !open && setDroppingCollection(null)} + dbId={dbId} + collection={droppingCollection} + onCollectionDropped={handleCollectionDropped} + /> + )} +
+ ); +} diff --git a/src/features/mongodb/components/MongoDeleteDocumentDialog.tsx b/src/features/mongodb/components/MongoDeleteDocumentDialog.tsx new file mode 100644 index 00000000..db5ca44b --- /dev/null +++ b/src/features/mongodb/components/MongoDeleteDocumentDialog.tsx @@ -0,0 +1,150 @@ +import React, { useMemo } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Loader2, Trash2, AlertTriangle } from "lucide-react"; +import { toast } from "sonner"; +import { EJSONDocument } from "../types"; +import { useDeleteMongoDocument } from "../hooks/useMongoMutations"; + +interface MongoDeleteDocumentDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + collection: string; + document: EJSONDocument | null; + onRefreshView: () => void; +} + +export function MongoDeleteDocumentDialog({ + open, + onOpenChange, + dbId, + collection, + document, + onRefreshView, +}: MongoDeleteDocumentDialogProps) { + const deleteMutation = useDeleteMongoDocument(dbId); + + const idFormatted = useMemo(() => { + if (!document?._id) return "unknown"; + if (typeof document._id === "object" && "$oid" in document._id) { + return document._id.$oid; + } + return String(document._id); + }, [document]); + + const previewFields = useMemo(() => { + if (!document) return []; + return Object.entries(document) + .filter(([k]) => k !== "_id") + .slice(0, 4) + .map(([k, v]) => ({ + key: k, + val: typeof v === "object" ? JSON.stringify(v) : String(v), + })); + }, [document]); + + const handleDelete = async () => { + if (!document?._id) return; + try { + const res = await deleteMutation.mutateAsync({ + collection, + filter: { _id: document._id }, + options: { multi: false }, + }); + + if (res.deletedCount === 0) { + toast.warning("Document not found", { + description: "The document may have already been deleted concurrently.", + }); + } else { + toast.success("Document deleted successfully"); + } + + onRefreshView(); + onOpenChange(false); + } catch (err: any) { + toast.error("Delete failed", { description: err.message }); + } + }; + + return ( + + + + + + Delete Document + + + Are you sure you want to delete this document from{" "} + {collection}? This + action cannot be undone. + + + +
+ {/* Target details */} +
+
+ _id: + {idFormatted} +
+ + {previewFields.length > 0 && ( +
+ {previewFields.map(({ key, val }) => ( +
+ {key}: + {val} +
+ ))} +
+ )} +
+ +
+ Filter: {"{ _id: " + idFormatted + " }"} +
+
+ + + + + +
+
+ ); +} diff --git a/src/features/mongodb/components/MongoDocumentCell.tsx b/src/features/mongodb/components/MongoDocumentCell.tsx new file mode 100644 index 00000000..c15a5798 --- /dev/null +++ b/src/features/mongodb/components/MongoDocumentCell.tsx @@ -0,0 +1,305 @@ +import React, { useState } from "react"; +import { formatTimestamp } from "@/lib/utils"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Copy, Check, ChevronRight } from "lucide-react"; +import { toast } from "sonner"; + +export interface MongoFormattedValue { + type: string; + display: string; + raw: any; + badgeText?: string; + badgeColor?: string; + isExpandable?: boolean; +} + +export function formatMongoValue(value: any, _column?: string): MongoFormattedValue { + if (value === null || value === undefined) { + return { type: "null", display: "NULL", raw: value, badgeText: "NULL" }; + } + + if (typeof value === "boolean") { + return { + type: "bool", + display: value ? "true" : "false", + raw: value, + badgeText: "BOOL", + }; + } + + if (typeof value === "number") { + return { + type: "number", + display: value.toLocaleString(), + raw: value, + badgeText: "NUM", + }; + } + + if (typeof value === "string") { + return { type: "string", display: value, raw: value }; + } + + if (typeof value === "object") { + if ("$oid" in value) { + return { + type: "objectId", + display: String(value.$oid), + raw: value, + badgeText: "OID", + badgeColor: "bg-teal-500/10 text-teal-600 dark:text-teal-400 border-teal-500/20", + }; + } + + if ("$date" in value) { + let isoStr = ""; + if (typeof value.$date === "string") { + isoStr = value.$date; + } else if (value.$date && typeof value.$date === "object" && "$numberLong" in value.$date) { + const ms = parseInt(value.$date.$numberLong, 10); + isoStr = !isNaN(ms) ? new Date(ms).toISOString() : String(value.$date.$numberLong); + } else { + isoStr = String(value.$date); + } + return { + type: "date", + display: formatTimestamp(isoStr) || isoStr, + raw: value, + badgeText: "DATE", + badgeColor: "bg-purple-500/10 text-purple-600 dark:text-purple-400 border-purple-500/20", + }; + } + + if ("$numberLong" in value) { + return { + type: "long", + display: String(value.$numberLong), + raw: value, + badgeText: "LONG", + badgeColor: "bg-blue-500/10 text-blue-600 dark:text-blue-400 border-blue-500/20", + }; + } + + if ("$numberDecimal" in value) { + return { + type: "decimal", + display: String(value.$numberDecimal), + raw: value, + badgeText: "DEC", + badgeColor: "bg-cyan-500/10 text-cyan-600 dark:text-cyan-400 border-cyan-500/20", + }; + } + + if ("$numberInt" in value) { + return { + type: "number", + display: String(value.$numberInt), + raw: value, + badgeText: "INT", + }; + } + + if ("$numberDouble" in value) { + return { + type: "number", + display: String(value.$numberDouble), + raw: value, + badgeText: "DBL", + }; + } + + if ("$binary" in value) { + const b64 = value.$binary.base64 || value.$binary.$base64 || ""; + return { + type: "binary", + display: ``, + raw: value, + badgeText: "BIN", + badgeColor: "bg-orange-500/10 text-orange-600 dark:text-orange-400 border-orange-500/20", + }; + } + + if ("$timestamp" in value) { + const t = value.$timestamp.t ?? value.$timestamp.$t ?? 0; + const i = value.$timestamp.i ?? value.$timestamp.$i ?? 0; + return { + type: "timestamp", + display: `Timestamp(${t}, ${i})`, + raw: value, + badgeText: "TS", + }; + } + + if ("$regex" in value) { + return { + type: "regex", + display: `/${value.$regex}/${value.$options || ""}`, + raw: value, + badgeText: "REGEX", + }; + } + + if ("$undefined" in value) { + return { type: "undefined", display: "undefined", raw: value, badgeText: "UNDEF" }; + } + + if (Array.isArray(value)) { + return { + type: "array", + display: `Array(${value.length})`, + raw: value, + badgeText: `[${value.length}]`, + badgeColor: + "bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20", + isExpandable: true, + }; + } + + const keys = Object.keys(value); + return { + type: "object", + display: `{${keys.length} fields}`, + raw: value, + badgeText: `{${keys.length}}`, + badgeColor: "bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/20", + isExpandable: true, + }; + } + + return { type: "unknown", display: String(value), raw: value }; +} + +export function MongoDocumentCell({ + value, + column, + row: _row, +}: { + value: any; + column: string; + row: Record; +}) { + const [dialogOpen, setDialogOpen] = useState(false); + const [copied, setCopied] = useState(false); + + const formatted = formatMongoValue(value, column); + + const handleCopyJson = (e: React.MouseEvent) => { + e.stopPropagation(); + const str = JSON.stringify(value, null, 2); + navigator.clipboard.writeText(str); + setCopied(true); + toast.success("JSON copied to clipboard", { duration: 1500 }); + setTimeout(() => setCopied(false), 1500); + }; + + if (formatted.type === "null") { + return ( + + NULL + + ); + } + + if (formatted.type === "bool") { + return ( + + {formatted.display} + + ); + } + + if (formatted.isExpandable) { + const jsonStr = JSON.stringify(value); + const truncatedPreview = jsonStr.length > 50 ? `${jsonStr.slice(0, 50)}…` : jsonStr; + + return ( + <> +
+ + + {truncatedPreview} + +
+ + + + +
+
+ + {column} + + {formatted.type} + + + + Document field inspection + +
+ +
+
+
+
+                {JSON.stringify(value, null, 2)}
+              
+
+
+
+ + ); + } + + return ( +
+ {formatted.badgeText && ( + + {formatted.badgeText} + + )} + {formatted.display} +
+ ); +} diff --git a/src/features/mongodb/components/MongoDocumentViewer.tsx b/src/features/mongodb/components/MongoDocumentViewer.tsx new file mode 100644 index 00000000..174e0daa --- /dev/null +++ b/src/features/mongodb/components/MongoDocumentViewer.tsx @@ -0,0 +1,327 @@ +import React, { useMemo, useState } from "react"; +import { DataTable } from "@/components/shared/DataTable"; +import { MongoDocumentCell } from "./MongoDocumentCell"; +import { MongoFilterInput } from "./MongoFilterInput"; +import { EJSONDocument } from "../types"; +import { Button } from "@/components/ui/button"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Copy, + Check, + ChevronLeft, + ChevronRight, + ChevronsLeft, + ChevronsRight, + RefreshCw, + Database, + Plus, + Upload, + Download, + FileEdit, +} from "lucide-react"; +import { toast } from "sonner"; + +export function MongoDocumentViewer({ + collection, + documents, + totalDocuments, + page, + pageSize, + totalPages, + isLoading, + sortField, + sortDirection, + onSort, + onPageChange, + onPageSizeChange, + filterInput, + onFilterInputChange, + onApplyFilter, + onResetFilter, + isFilterApplied, + onRefresh, + onInsertDocument, + onEditDocument, + onDeleteDocument, + onImportClick, + onExportClick, + onBulkMutationClick, + isReadOnly, +}: { + collection: string; + documents: EJSONDocument[]; + totalDocuments: number; + page: number; + pageSize: number; + totalPages: number; + isLoading: boolean; + sortField: string | null; + sortDirection: 1 | -1 | null; + onSort: (field: string) => void; + onPageChange: (newPage: number) => void; + onPageSizeChange: (newSize: number) => void; + filterInput: string; + onFilterInputChange: (text: string) => void; + onApplyFilter: (filter: EJSONDocument | undefined) => void; + onResetFilter: () => void; + isFilterApplied: boolean; + onRefresh: () => void; + onInsertDocument?: () => void; + onEditDocument?: (doc: EJSONDocument) => void; + onDeleteDocument?: (doc: EJSONDocument) => void; + onImportClick?: () => void; + onExportClick?: () => void; + onBulkMutationClick?: () => void; + isReadOnly?: boolean; +}) { + const [copiedAll, setCopiedAll] = useState(false); + + // Derive column list, pinning _id to the first position + const columns = useMemo(() => { + if (!documents || documents.length === 0) return []; + const keySet = new Set(); + documents.forEach((doc) => { + Object.keys(doc).forEach((k) => keySet.add(k)); + }); + keySet.delete("_id"); + const otherCols = Array.from(keySet).sort(); + return ["_id", ...otherCols]; + }, [documents]); + + const handleCopyAll = () => { + if (!documents || documents.length === 0) return; + const str = JSON.stringify(documents, null, 2); + navigator.clipboard.writeText(str); + setCopiedAll(true); + toast.success(`Copied ${documents.length} visible documents to clipboard`); + setTimeout(() => setCopiedAll(false), 1500); + }; + + const startDoc = (page - 1) * pageSize + 1; + const endDoc = Math.min(page * pageSize, totalDocuments); + const isSystemCollection = collection ? collection.startsWith("system.") : false; + + return ( +
+ {/* Top action toolbar */} +
+ + +
+ {onInsertDocument && ( + + )} + + {onImportClick && ( + + )} + + {onExportClick && ( + + )} + + {onBulkMutationClick && ( + + )} + + + + +
+
+ + {/* Main table area */} +
+ {isLoading && documents.length === 0 ? ( +
+ +

Loading documents...

+
+ ) : documents.length === 0 ? ( +
+
+ +
+

No documents found

+

+ {isFilterApplied + ? "No documents matched your active filter." + : `Collection "${collection}" has no documents.`} +

+
+ ) : ( + onEditDocument(row) : undefined} + onDeleteRow={ + !isReadOnly && onDeleteDocument ? (row) => onDeleteDocument(row) : undefined + } + renderCell={(val, col, row) => } + /> + )} +
+ + {/* Pagination Footer */} +
+
+ + {totalDocuments > 0 ? ( + <> + Showing {startDoc.toLocaleString()} –{" "} + {endDoc.toLocaleString()} of{" "} + {totalDocuments.toLocaleString()}{" "} + documents + + ) : ( + "0 documents" + )} + + +
+ Per page: + +
+
+ +
+ + + + Page {page} of {totalPages || 1} + + + +
+
+
+ ); +} diff --git a/src/features/mongodb/components/MongoDropCollectionDialog.tsx b/src/features/mongodb/components/MongoDropCollectionDialog.tsx new file mode 100644 index 00000000..5be9708a --- /dev/null +++ b/src/features/mongodb/components/MongoDropCollectionDialog.tsx @@ -0,0 +1,139 @@ +import React, { useState, useEffect } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Loader2, Trash2, AlertTriangle, ShieldAlert } from "lucide-react"; +import { toast } from "sonner"; +import { useDropMongoCollection } from "../hooks/useMongoMutations"; + +interface MongoDropCollectionDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + collection: string; + onCollectionDropped: (name: string) => void; +} + +export function MongoDropCollectionDialog({ + open, + onOpenChange, + dbId, + collection, + onCollectionDropped, +}: MongoDropCollectionDialogProps) { + const [confirmText, setConfirmText] = useState(""); + const dropMutation = useDropMongoCollection(dbId); + + useEffect(() => { + if (open) { + setConfirmText(""); + } + }, [open, collection]); + + const isSystemCollection = collection.startsWith("system."); + const isConfirmed = confirmText === collection && !isSystemCollection; + + const handleDrop = async () => { + if (!isConfirmed) return; + try { + await dropMutation.mutateAsync({ collection }); + toast.success(`Collection "${collection}" dropped successfully`); + onCollectionDropped(collection); + onOpenChange(false); + } catch (err: any) { + toast.error("Drop collection failed", { description: err.message }); + } + }; + + return ( + + + + + + Drop Collection: {collection} + + + This action cannot be undone. All documents, indexes, and metadata in{" "} + {collection} will be + permanently destroyed. + + + +
+ {isSystemCollection ? ( +
+ +
+ Protected System Collection: Collections starting with{" "} + system. cannot be dropped. +
+
+ ) : ( + <> + {/* Type to confirm */} +
+ + setConfirmText(e.target.value)} + className="font-mono text-xs h-9" + autoFocus + /> +
+ + {/* Warning box */} +
+ ⚠️ Warning: This operation drops the collection immediately on the + MongoDB server. +
+ + )} +
+ + + + + +
+
+ ); +} diff --git a/src/features/mongodb/components/MongoEditDocumentDialog.tsx b/src/features/mongodb/components/MongoEditDocumentDialog.tsx new file mode 100644 index 00000000..8b537d72 --- /dev/null +++ b/src/features/mongodb/components/MongoEditDocumentDialog.tsx @@ -0,0 +1,318 @@ +import React, { useState, useEffect, useMemo } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Textarea } from "@/components/ui/textarea"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Loader2, Edit, FileCode, Check, AlertCircle, AlertTriangle, Lock } from "lucide-react"; +import { toast } from "sonner"; +import { EJSONDocument } from "../types"; +import { useUpdateMongoDocument, useReplaceMongoDocument } from "../hooks/useMongoMutations"; +import { computeDocumentDiff } from "../hooks/useDocumentDiff"; +import { validateJsonDocument } from "../utils/validation"; + +interface MongoEditDocumentDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + collection: string; + document: EJSONDocument | null; + onRefreshView: () => void; +} + +export function MongoEditDocumentDialog({ + open, + onOpenChange, + dbId, + collection, + document, + onRefreshView, +}: MongoEditDocumentDialogProps) { + const [mode, setMode] = useState<"patch" | "replace">("patch"); + const [jsonText, setJsonText] = useState(""); + const [error, setError] = useState(null); + + const updateMutation = useUpdateMongoDocument(dbId); + const replaceMutation = useReplaceMongoDocument(dbId); + + useEffect(() => { + if (open && document) { + setJsonText(JSON.stringify(document, null, 2)); + setError(null); + setMode("patch"); + } + }, [open, document]); + + const idFormatted = useMemo(() => { + if (!document?._id) return "unknown"; + if (typeof document._id === "object" && "$oid" in document._id) { + return document._id.$oid; + } + return String(document._id); + }, [document]); + + const parsedDoc = useMemo(() => { + try { + const p = JSON.parse(jsonText); + return typeof p === "object" && p !== null && !Array.isArray(p) ? p : null; + } catch { + return null; + } + }, [jsonText]); + + const diffResult = useMemo(() => { + if (!document || !parsedDoc || mode !== "patch") return null; + try { + return computeDocumentDiff(document, parsedDoc); + } catch (err: any) { + return { error: err.message }; + } + }, [document, parsedDoc, mode]); + + const handleTextChange = (text: string) => { + setJsonText(text); + const val = validateJsonDocument(text); + if (!val.valid) { + setError(val.error || "Invalid JSON"); + return; + } + if (document?._id !== undefined) { + const parsed = val.doc; + if (parsed && JSON.stringify(parsed._id) !== JSON.stringify(document._id)) { + setError("_id is immutable and cannot be modified"); + return; + } + } + setError(null); + }; + + const handleFormat = () => { + try { + const parsed = JSON.parse(jsonText); + setJsonText(JSON.stringify(parsed, null, 2)); + setError(null); + } catch (e: any) { + setError(e.message || "Cannot format invalid JSON"); + } + }; + + const isSaving = updateMutation.isPending || replaceMutation.isPending; + + const handleSubmit = async () => { + if (!document) return; + const val = validateJsonDocument(jsonText); + if (!val.valid || !val.doc) { + setError(val.error || "Please enter valid JSON"); + return; + } + + const editedDoc = val.doc; + + try { + if (mode === "patch") { + const diff = computeDocumentDiff(document, editedDoc); + if (!diff.hasChanges || !diff.updateDoc) { + toast.info("No changes made", { + description: "The document already contained the specified values.", + }); + onOpenChange(false); + return; + } + + const res = await updateMutation.mutateAsync({ + collection, + filter: { _id: document._id }, + update: diff.updateDoc, + }); + + if (res.matchedCount === 0) { + toast.error("Document not found", { + description: + "The target document no longer exists. It may have been deleted concurrently.", + }); + onRefreshView(); + onOpenChange(false); + return; + } + + if (res.modifiedCount === 0) { + toast.info("No changes made", { + description: "The document already contained the specified values.", + }); + } else { + toast.success("Document updated successfully"); + } + onOpenChange(false); + } else { + // Mode B: Full Document Replacement + const res = await replaceMutation.mutateAsync({ + collection, + filter: { _id: document._id }, + replacement: editedDoc, + }); + + if (res.matchedCount === 0) { + toast.error("Document not found", { + description: + "The target document no longer exists. It may have been deleted concurrently.", + }); + onRefreshView(); + onOpenChange(false); + return; + } + + if (res.modifiedCount === 0) { + toast.info("No changes made", { + description: "The replacement was identical to current document.", + }); + } else { + toast.success("Document replaced successfully"); + } + onOpenChange(false); + } + } catch (err: any) { + setError(err.message || "Failed to update document"); + toast.error("Update failed", { description: err.message }); + } + }; + + return ( + + + +
+ + + Edit Document in {collection} + +
+ + + _id: + + {idFormatted} + + +
+ + {/* Mode Selector and Controls */} +
+ setMode(val as "patch" | "replace")}> + + + Field Patch Mode (Safe) + + + Full Replacement + + + + + +
+ + {/* Warning banner in Replacement Mode */} + {mode === "replace" && ( +
+ + + Warning: Full Replacement mode replaces the entire document. Fields + omitted from this editor will be permanently deleted. + +
+ )} + + {/* Editor Area */} +
+