From e303c3dbb26227004fbd12b2025d701cff04930d Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 25 Oct 2026 20:13:30 +0530 Subject: [PATCH 01/34] feat(mongodb): implement M1 connector foundation --- bridge/__tests__/connectors/mongodb.test.ts | 142 +++++++++ .../__tests__/databaseService.mongo.test.ts | 90 ++++++ bridge/__tests__/mongoBson.test.ts | 106 +++++++ bridge/__tests__/mongoUri.test.ts | 112 +++++++ bridge/docker-compose.test.yml | 13 + bridge/package.json | 1 + bridge/pnpm-lock.yaml | 110 +++++++ bridge/scripts/seed-mongodb.cjs | 127 ++++++++ bridge/src/connectors/mongo.ts | 276 ++++++++++++++++++ bridge/src/connectors/mongoBson.ts | 143 +++++++++ bridge/src/handlers/migrationHandlers.ts | 18 ++ bridge/src/services/connectionBuilder.ts | 19 +- bridge/src/services/connectorKind.ts | 26 ++ bridge/src/services/connectorRegistry.ts | 5 +- bridge/src/services/databaseService.ts | 45 ++- bridge/src/types/index.ts | 2 + bridge/src/types/mongo.ts | 96 ++++++ bridge/src/utils/dbTypeDetector.ts | 1 + bridge/src/utils/mongoUri.ts | 218 ++++++++++++++ 19 files changed, 1547 insertions(+), 3 deletions(-) create mode 100644 bridge/__tests__/connectors/mongodb.test.ts create mode 100644 bridge/__tests__/databaseService.mongo.test.ts create mode 100644 bridge/__tests__/mongoBson.test.ts create mode 100644 bridge/__tests__/mongoUri.test.ts create mode 100644 bridge/scripts/seed-mongodb.cjs create mode 100644 bridge/src/connectors/mongo.ts create mode 100644 bridge/src/connectors/mongoBson.ts create mode 100644 bridge/src/services/connectorKind.ts create mode 100644 bridge/src/types/mongo.ts create mode 100644 bridge/src/utils/mongoUri.ts diff --git a/bridge/__tests__/connectors/mongodb.test.ts b/bridge/__tests__/connectors/mongodb.test.ts new file mode 100644 index 00000000..633c8d19 --- /dev/null +++ b/bridge/__tests__/connectors/mongodb.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, test, beforeAll, afterAll, jest } from "@jest/globals"; +import { MongoClient, ObjectId } from "mongodb"; +import * as mongoConnector from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const MONGO_HOST = process.env.REAL_MONGODB_HOST || "127.0.0.1"; +const MONGO_PORT = process.env.REAL_MONGODB_PORT || "27017"; +const MONGO_DB = process.env.REAL_MONGODB_DATABASE || "testdb"; + +// Credential-free URI: the test mongo service runs without auth. +const validConfig: MongoConfig = { + uri: `mongodb://${MONGO_HOST}:${MONGO_PORT}/${MONGO_DB}`, + database: MONGO_DB, +}; + +// A closed local port with a short selection timeout → fast, deterministic failure. +const invalidConfig: MongoConfig = { + uri: "mongodb://127.0.0.1:1/testdb?serverSelectionTimeoutMS=5000", + database: "testdb", +}; + +let seedClient: MongoClient | null = null; + +async function seedFixtures(): Promise { + seedClient = new MongoClient(validConfig.uri as string); + await seedClient.connect(); + const db = seedClient.db(MONGO_DB); + + for (const name of ["users", "orders", "active_users_view"]) { + try { + await db.dropCollection(name); + } catch { + // may not exist + } + } + + const userIds: ObjectId[] = []; + const userInsert = await db.collection("users").insertMany([ + { name: "John Doe", email: "john@example.com", age: 30, active: true, createdAt: new Date() }, + { name: "Jane Smith", email: "jane@example.com", age: 27, active: true, createdAt: new Date() }, + { name: "Bob Lee", email: "bob@example.com", age: 45, active: false, createdAt: new Date() }, + ]); + for (const id of Object.values(userInsert.insertedIds)) userIds.push(id); + + await db.collection("orders").insertMany([ + { + userId: userIds[0], + items: [ + { sku: "ABC-1", qty: 2, price: 9.99 }, + { sku: "XYZ-9", qty: 1, price: 19.5 }, + ], + total: 39.48, + status: "placed", + orderedAt: new Date(), + }, + { + userId: userIds[1], + items: [{ sku: "ABC-1", qty: 5, price: 9.99 }], + total: 49.95, + status: "shipped", + orderedAt: new Date(), + }, + ]); + + await db.command({ + create: "active_users_view", + viewOn: "users", + pipeline: [{ $match: { active: true } }], + }); +} + +describe("MongoDB Connector", () => { + jest.setTimeout(30000); + + beforeAll(async () => { + await seedFixtures(); + }); + + afterAll(async () => { + await mongoConnector.closeAllMongoClients(); + if (seedClient) { + await seedClient.close(); + seedClient = null; + } + }); + + test("fails to connect with an invalid config", async () => { + const result = await mongoConnector.testConnection(invalidConfig); + expect(result.ok).toBe(false); + expect(result.status).toBe("disconnected"); + }); + + test("connects with a valid config", async () => { + const result = await mongoConnector.testConnection(validConfig); + expect(result).toStrictEqual({ + message: "Connection successful", + status: "connected", + ok: true, + }); + }); + + test("lists collections (including a view)", async () => { + const collections = await mongoConnector.listCollections(validConfig); + expect(Array.isArray(collections)).toBe(true); + const byName = new Map(collections.map((c) => [c.name, c.type])); + expect(byName.get("users")).toBe("collection"); + expect(byName.get("orders")).toBe("collection"); + expect(byName.get("active_users_view")).toBe("view"); + }); + + test("lists tables as the relational TableInfo adapter", async () => { + const tables = await mongoConnector.listTables(validConfig); + expect(Array.isArray(tables)).toBe(true); + const byName = new Map(tables.map((t) => [t.name, t.type])); + expect(byName.get("users")).toBe("BASE TABLE"); + expect(byName.get("active_users_view")).toBe("VIEW"); + expect(tables.every((t) => t.schema === MONGO_DB)).toBe(true); + }); + + test("returns database statistics", async () => { + const stats = await mongoConnector.getDBStats(validConfig); + expect(stats).toHaveProperty("total_tables"); + expect(stats).toHaveProperty("total_rows"); + expect(stats).toHaveProperty("total_db_size_mb"); + expect(stats.total_tables).toBeGreaterThanOrEqual(2); + expect(stats.total_rows).toBeGreaterThanOrEqual(5); + }); + + test("returns a single pseudo-schema name", async () => { + const names = await mongoConnector.listSchemaNames(validConfig); + expect(Array.isArray(names)).toBe(true); + expect(names).toContain(MONGO_DB); + }); + + test("connectToDatabase returns collections as pseudo-schemas with no migrations", async () => { + const result = await mongoConnector.connectToDatabase(validConfig, "mongo-integration-id"); + expect(result.baselined).toBe(false); + expect(result.schema.map((s) => s.name)).toContain("users"); + expect(result.migrations.local).toEqual([]); + expect(result.migrations.applied).toEqual([]); + }); +}); diff --git a/bridge/__tests__/databaseService.mongo.test.ts b/bridge/__tests__/databaseService.mongo.test.ts new file mode 100644 index 00000000..ad46e424 --- /dev/null +++ b/bridge/__tests__/databaseService.mongo.test.ts @@ -0,0 +1,90 @@ +import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; +import fs from "fs/promises"; +import fsSync from "fs"; +import os from "os"; +import path from "path"; + +// RELWAVE_HOME must be set BEFORE the databaseService (and its dbStore/config +// singletons) are loaded, so point it at an isolated temp directory. +const TEST_RELWAVE_HOME = path.join(os.tmpdir(), `mongo-db-service-test-${Date.now()}`); +process.env.RELWAVE_HOME = TEST_RELWAVE_HOME; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { DatabaseService } = require("../src/services/databaseService"); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { dbStoreInstance } = require("../src/services/dbStore"); + +const SECRET = "sup3r-s3cret-pw"; +const createdIds: string[] = []; + +describe("DatabaseService MongoDB credential handling", () => { + const dbService = new DatabaseService(); + + beforeAll(async () => { + if (!fsSync.existsSync(TEST_RELWAVE_HOME)) { + await fs.mkdir(TEST_RELWAVE_HOME, { recursive: true }); + } + }); + + afterAll(async () => { + for (const id of createdIds) { + try { + await dbService.deleteDatabase(id); + } catch { + // ignore + } + } + if (fsSync.existsSync(TEST_RELWAVE_HOME)) { + await fs.rm(TEST_RELWAVE_HOME, { recursive: true, force: true }); + } + }); + + test("strips the embedded password from the persisted URI and AES-stores it", async () => { + const url = `mongodb://testuser:${SECRET}@localhost:27017/testdb`; + const res = await dbService.addDatabase({ name: "MongoCredDB", type: "mongodb", url }); + createdIds.push(res.id); + + expect(res.id).toBeDefined(); + + const meta = await dbService.getDatabase(res.id); + expect(meta?.type).toBe("mongodb"); + // The persisted URI is sanitized — no plaintext password. + expect(meta?.url).toBe("mongodb://testuser@localhost:27017/testdb"); + expect(meta?.url).not.toContain(SECRET); + expect(meta?.credentialId).toBeDefined(); + + // The password is retrievable through the encrypted credential store. + const pwd = meta ? await dbStoreInstance.getPasswordFor(meta) : null; + expect(pwd).toBe(SECRET); + + // Byte-level scan of the persisted metadata finds no plaintext password. + const raw = await fs.readFile(path.join(TEST_RELWAVE_HOME, "databases.json"), "utf-8"); + expect(raw).not.toContain(SECRET); + }); + + test("reconstructs the authenticated URI in memory at runtime, never persisting it", async () => { + const url = `mongodb://alice:${SECRET}@localhost:27017/proddb`; + const res = await dbService.addDatabase({ name: "MongoRuntimeDB", type: "mongodb", url }); + createdIds.push(res.id); + + const { conn, dbType } = await dbService.getDatabaseConnection(res.id); + expect(dbType).toBe("mongodb"); + // Runtime config carries the password for the driver. + expect((conn as { uri?: string }).uri).toContain(SECRET); + expect((conn as { uri?: string }).uri).toContain("alice"); + + // But the persisted metadata keeps the sanitized URI. + const meta = await dbService.getDatabase(res.id); + expect(meta?.url).toBe("mongodb://alice@localhost:27017/proddb"); + expect(meta?.url).not.toContain(SECRET); + + const raw = await fs.readFile(path.join(TEST_RELWAVE_HOME, "databases.json"), "utf-8"); + expect(raw).not.toContain(SECRET); + }); + + test("requires name and type for a MongoDB connection", async () => { + await expect( + dbService.addDatabase({ type: "mongodb", url: "mongodb://localhost/db" }), + ).rejects.toThrow("Missing required field: name"); + }); +}); diff --git a/bridge/__tests__/mongoBson.test.ts b/bridge/__tests__/mongoBson.test.ts new file mode 100644 index 00000000..e6231670 --- /dev/null +++ b/bridge/__tests__/mongoBson.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "@jest/globals"; +import { ObjectId, Decimal128, Long, Timestamp, Int32, Binary } from "mongodb"; +import { serializeMongoValue, serializeMongoDocument } from "../src/connectors/mongoBson"; + +describe("serializeMongoValue", () => { + test("passes primitives through unchanged", () => { + expect(serializeMongoValue(null)).toBeNull(); + expect(serializeMongoValue(true)).toBe(true); + expect(serializeMongoValue(42)).toBe(42); + expect(serializeMongoValue("hi")).toBe("hi"); + }); + + test("drops non-finite numbers to stay JSON-safe", () => { + expect(serializeMongoValue(Number.NaN)).toBeNull(); + expect(serializeMongoValue(Number.POSITIVE_INFINITY)).toBeNull(); + }); + + test("maps undefined to the wire convention", () => { + expect(serializeMongoValue(undefined)).toEqual({ $undefined: true }); + }); + + test("recurses into arrays", () => { + expect(serializeMongoValue([1, "two", null])).toEqual([1, "two", null]); + }); + + test("serializes ObjectId, Date, Decimal128, Long, Timestamp", () => { + const oid = new ObjectId("507f1f77bcf86cd799439011"); + expect(serializeMongoValue(oid)).toEqual({ $oid: "507f1f77bcf86cd799439011" }); + + const date = new Date("2020-01-02T03:04:05.000Z"); + expect(serializeMongoValue(date)).toEqual({ $date: "2020-01-02T03:04:05.000Z" }); + + expect(serializeMongoValue(new Decimal128("123.45"))).toEqual({ $numberDecimal: "123.45" }); + + expect(serializeMongoValue(Long.fromNumber(42))).toEqual({ $numberLong: "42" }); + + const ts = new Timestamp({ t: 1234567890, i: 1 }); + expect(serializeMongoValue(ts)).toEqual({ $timestamp: { t: 1234567890, i: 1 } }); + }); + + test("flattens Int32 to a plain number", () => { + expect(serializeMongoValue(new Int32(7))).toBe(7); + expect(serializeMongoValue(new Int32(7)).constructor).toBe(Number); + }); + + test("serializes Binary and raw buffers to $binary", () => { + const bin = new Binary(Buffer.from("hello")); + const out = serializeMongoValue(bin) as { $binary: { base64: string; subtype: string } }; + expect(out.$binary.base64).toBe(Buffer.from("hello").toString("base64")); + expect(out.$binary.subtype).toBe("00"); + + const bufOut = serializeMongoValue(Buffer.from("hi")) as { + $binary: { base64: string; subtype: string }; + }; + expect(bufOut.$binary.base64).toBe(Buffer.from("hi").toString("base64")); + + const u8Out = serializeMongoValue(new Uint8Array([1, 2, 3])) as { + $binary: { base64: string; subtype: string }; + }; + expect(u8Out.$binary.base64).toBe(Buffer.from([1, 2, 3]).toString("base64")); + }); + + test("recurses through nested documents and arrays", () => { + const value = serializeMongoValue({ + _id: new ObjectId("507f1f77bcf86cd799439011"), + nested: { when: new Date("2021-06-07T08:09:10.000Z"), tags: ["a", "b"] }, + list: [1, { deep: true }], + }); + expect(value).toEqual({ + _id: { $oid: "507f1f77bcf86cd799439011" }, + nested: { when: { $date: "2021-06-07T08:09:10.000Z" }, tags: ["a", "b"] }, + list: [1, { deep: true }], + }); + }); + + test("output is always JSON.stringify-safe", () => { + const value = serializeMongoValue({ + _id: new ObjectId(), + when: new Date(), + big: Long.fromNumber(9007199254740993), + dec: new Decimal128("3.14"), + raw: Buffer.from("bytes"), + missing: undefined, + deep: { arr: [new ObjectId(), new Date()] }, + }); + expect(() => JSON.stringify(value)).not.toThrow(); + expect(JSON.parse(JSON.stringify(value))).toBeDefined(); + }); +}); + +describe("serializeMongoDocument", () => { + test("serializes a document and omits the internal _bsontype marker", () => { + const doc = { + _id: new ObjectId("507f1f77bcf86cd799439011"), + name: "test", + createdAt: new Date("2022-03-04T05:06:07.000Z"), + }; + const out = serializeMongoDocument(doc); + expect(out).toEqual({ + _id: { $oid: "507f1f77bcf86cd799439011" }, + name: "test", + createdAt: { $date: "2022-03-04T05:06:07.000Z" }, + }); + expect(Object.keys(out)).not.toContain("_bsontype"); + }); +}); diff --git a/bridge/__tests__/mongoUri.test.ts b/bridge/__tests__/mongoUri.test.ts new file mode 100644 index 00000000..0b99e3fa --- /dev/null +++ b/bridge/__tests__/mongoUri.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, test } from "@jest/globals"; +import { + isMongoUri, + extractMongoCredentials, + sanitizeMongoUri, + injectMongoPassword, + normalizeMongoUriMeta, + splitMongoUriCredentials, + buildMongoConnectionUri, +} from "../src/utils/mongoUri"; + +describe("mongoUri", () => { + test("isMongoUri recognizes mongodb and mongodb+srv schemes", () => { + expect(isMongoUri("mongodb://localhost:27017")).toBe(true); + expect(isMongoUri("mongodb+srv://cluster/db")).toBe(true); + expect(isMongoUri("postgres://localhost")).toBe(false); + expect(isMongoUri(undefined)).toBe(false); + expect(isMongoUri("")).toBe(false); + }); + + test("extractMongoCredentials decodes percent-encoded credentials", () => { + const creds = extractMongoCredentials("mongodb://user:p%40ss%23word@localhost:27017/testdb"); + expect(creds.user).toBe("user"); + expect(creds.password).toBe("p@ss#word"); + }); + + test("sanitizeMongoUri removes the password but keeps user and options", () => { + const sanitized = sanitizeMongoUri( + "mongodb://user:secret@localhost:27017/testdb?authSource=admin&replicaSet=rs0", + ); + expect(sanitized).toBe("mongodb://user@localhost:27017/testdb?authSource=admin&replicaSet=rs0"); + expect(sanitized).not.toContain("secret"); + }); + + test("sanitizeMongoUri is idempotent and safe for SRV URIs", () => { + const srv = "mongodb+srv://alice:p%40ss%23word@cluster0.example.net/prod"; + const s1 = sanitizeMongoUri(srv); + expect(s1).toBe("mongodb+srv://alice@cluster0.example.net/prod"); + expect(sanitizeMongoUri(s1)).toBe(s1); + }); + + test("injectMongoPassword re-injects and round-trips to the original", () => { + const original = "mongodb://user:p%40ss@localhost:27017/testdb"; + const sanitized = sanitizeMongoUri(original); + const reconstructed = injectMongoPassword(sanitized, "p@ss"); + // decoded password is re-encoded identically + expect(decodeURIComponent(reconstructed)).toBe(decodeURIComponent(original)); + expect(reconstructed).toContain("p%40ss"); + }); + + test("injectMongoPassword without a username returns the URI unchanged", () => { + const uri = "mongodb://localhost:27017/testdb"; + expect(injectMongoPassword(uri, "secret")).toBe(uri); + }); + + test("normalizeMongoUriMeta derives relational-meta fields", () => { + const meta = normalizeMongoUriMeta( + "mongodb://user:p%40ss%23word@example.net:27018/proddb?authSource=admin&replicaSet=rs0", + ); + expect(meta.host).toBe("example.net"); + expect(meta.port).toBe(27018); + expect(meta.user).toBe("user"); + expect(meta.database).toBe("proddb"); + expect(meta.authSource).toBe("admin"); + expect(meta.replicaSet).toBe("rs0"); + }); + + test("normalizeMongoUriMeta defaults port to 27017", () => { + const meta = normalizeMongoUriMeta("mongodb://localhost/testdb"); + expect(meta.port).toBe(27017); + expect(meta.database).toBe("testdb"); + }); + + test("splitMongoUriCredentials separates sanitized URI from the password", () => { + const parts = splitMongoUriCredentials("mongodb://u:p%40ss%23word@h:27017/db"); + expect(parts.isMongoUri).toBe(true); + expect(parts.hasCredentials).toBe(true); + expect(parts.sanitizedUri).toBe("mongodb://u@h:27017/db"); + expect(parts.password).toBe("p@ss#word"); + }); + + test("splitMongoUriCredentials passes non-URI values through", () => { + const parts = splitMongoUriCredentials("not-a-uri"); + expect(parts.isMongoUri).toBe(false); + expect(parts.hasCredentials).toBe(false); + expect(parts.sanitizedUri).toBe("not-a-uri"); + }); + + test("buildMongoConnectionUri re-injects runtime password into a sanitized URI", () => { + const sanitized = "mongodb://testuser@127.0.0.1:27017/testdb"; + const uri = buildMongoConnectionUri({ url: sanitized }, "runtime-pw"); + expect(uri).toContain("testuser"); + expect(decodeURIComponent(uri)).toContain("runtime-pw"); + }); + + test("buildMongoConnectionUri leaves a passwordless sanitized URI untouched", () => { + const sanitized = "mongodb://127.0.0.1:27017/testdb"; + expect(buildMongoConnectionUri({ url: sanitized }, null)).toBe(sanitized); + }); + + test("buildMongoConnectionUri composes from host/port/user/password", () => { + const uri = buildMongoConnectionUri( + { host: "localhost", port: 27017, user: "u", database: "db" }, + "p@ss", + ); + expect(uri).toBe("mongodb://u:p%40ss@localhost:27017/db"); + }); + + test("buildMongoConnectionUri throws when neither uri nor host is present", () => { + expect(() => buildMongoConnectionUri({}, "pw")).toThrow(/requires a url/i); + }); +}); diff --git a/bridge/docker-compose.test.yml b/bridge/docker-compose.test.yml index 190aab41..0b63e4c6 100644 --- a/bridge/docker-compose.test.yml +++ b/bridge/docker-compose.test.yml @@ -67,6 +67,19 @@ services: timeout: 5s retries: 5 + mongodb: + image: mongo:7 + environment: + # Runs without auth so the credential-free integration test can connect. + MONGO_INITDB_DATABASE: testdb + ports: + - "27017:27017" + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping')"] + interval: 10s + timeout: 5s + retries: 5 + ssh-server: image: linuxserver/openssh-server environment: diff --git a/bridge/package.json b/bridge/package.json index c2b36d48..05f2d7c9 100644 --- a/bridge/package.json +++ b/bridge/package.json @@ -30,6 +30,7 @@ "better-sqlite3": "^11.9.0", "dotenv": "^17.2.3", "groq-sdk": "^1.2.1", + "mongodb": "^6.21.0", "mysql2": "^3.15.3", "ollama": "^0.6.3", "openai": "^6.41.0", diff --git a/bridge/pnpm-lock.yaml b/bridge/pnpm-lock.yaml index 47fc76ed..aec0cbca 100644 --- a/bridge/pnpm-lock.yaml +++ b/bridge/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: groq-sdk: specifier: ^1.2.1 version: 1.2.1 + mongodb: + specifier: ^6.21.0 + version: 6.21.0 mysql2: specifier: ^3.15.3 version: 3.15.3 @@ -629,6 +632,9 @@ packages: '@mistralai/mistralai@2.2.5': resolution: {integrity: sha512-ATbWzKkNzNAZ+gtw9MI/c/ULTMG80tKUiRNIbQFfg4OP0uEZZpTfXZeBCNfs5Dq0uqMQ/tQWc4o6RRJQtMrpDA==} + '@mongodb-js/saslprep@1.6.1': + resolution: {integrity: sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg==} + '@napi-rs/keyring-darwin-arm64@1.2.0': resolution: {integrity: sha512-CA83rDeyONDADO25JLZsh3eHY8yTEtm/RS6ecPsY+1v+dSawzT9GywBMu2r6uOp1IEhQs/xAfxgybGAFr17lSA==} engines: {node: '>= 10'} @@ -799,6 +805,12 @@ packages: '@types/strip-json-comments@0.0.30': resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==} + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@11.0.5': + resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} + '@types/yargs-parser@21.0.3': resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -1056,6 +1068,10 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + bson@6.10.4: + resolution: {integrity: sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==} + engines: {node: '>=16.20.1'} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -1665,6 +1681,9 @@ packages: makeerror@1.0.12: resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==} + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -1702,6 +1721,36 @@ packages: engines: {node: '>=10'} hasBin: true + mongodb-connection-string-url@3.0.2: + resolution: {integrity: sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==} + + mongodb@6.21.0: + resolution: {integrity: sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 || ^2.0.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.3.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1944,6 +1993,10 @@ packages: pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + pure-rand@7.0.1: resolution: {integrity: sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==} @@ -2057,6 +2110,9 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -2172,6 +2228,10 @@ packages: tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -2301,9 +2361,17 @@ packages: webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + whatwg-fetch@3.6.20: resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -2959,6 +3027,10 @@ snapshots: - bufferutil - utf-8-validate + '@mongodb-js/saslprep@1.6.1': + dependencies: + sparse-bitfield: 3.0.3 + '@napi-rs/keyring-darwin-arm64@1.2.0': optional: true @@ -3113,6 +3185,12 @@ snapshots: '@types/strip-json-comments@0.0.30': {} + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@11.0.5': + dependencies: + '@types/webidl-conversions': 7.0.3 + '@types/yargs-parser@21.0.3': {} '@types/yargs@17.0.35': @@ -3369,6 +3447,8 @@ snapshots: dependencies: node-int64: 0.4.0 + bson@6.10.4: {} + buffer-from@1.1.2: {} buffer@5.7.1: @@ -4136,6 +4216,8 @@ snapshots: dependencies: tmpl: 1.0.5 + memory-pager@1.5.0: {} + merge-stream@2.0.0: {} micromatch@4.0.8: @@ -4163,6 +4245,17 @@ snapshots: mkdirp@1.0.4: {} + mongodb-connection-string-url@3.0.2: + dependencies: + '@types/whatwg-url': 11.0.5 + whatwg-url: 14.2.0 + + mongodb@6.21.0: + dependencies: + '@mongodb-js/saslprep': 1.6.1 + bson: 6.10.4 + mongodb-connection-string-url: 3.0.2 + ms@2.1.3: {} multistream@4.1.0: @@ -4392,6 +4485,8 @@ snapshots: end-of-stream: 1.4.5 once: 1.4.0 + punycode@2.3.1: {} + pure-rand@7.0.1: {} quick-format-unescaped@4.0.4: {} @@ -4496,6 +4591,10 @@ snapshots: source-map@0.6.1: {} + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + split2@4.2.0: {} sprintf-js@1.0.3: {} @@ -4618,6 +4717,10 @@ snapshots: tr46@0.0.3: {} + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} ts-algebra@2.0.0: {} @@ -4758,8 +4861,15 @@ snapshots: webidl-conversions@3.0.1: {} + webidl-conversions@7.0.0: {} + whatwg-fetch@3.6.20: {} + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + whatwg-url@5.0.0: dependencies: tr46: 0.0.3 diff --git a/bridge/scripts/seed-mongodb.cjs b/bridge/scripts/seed-mongodb.cjs new file mode 100644 index 00000000..a2c5f4b0 --- /dev/null +++ b/bridge/scripts/seed-mongodb.cjs @@ -0,0 +1,127 @@ +/* + * Standalone MongoDB seed script (manual tool). + * + * Seeds the test database with a couple of collections plus a view so the + * MongoDB connector's listing/stats can be exercised: + * - users (flat documents) + * - orders (nested documents with an ObjectId reference + array + Date) + * - active_users_view (a view over users, to exercise VIEW detection) + * + * Run with: node scripts/seed-mongodb.cjs + * Reads REAL_MONGODB_* from .env (loaded via dotenv). + */ + +const fs = require("dotenv"); +fs.config({ path: require("path").join(__dirname, "..", ".env") }); + +const { MongoClient, ObjectId } = require("mongodb"); + +function wait(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function seed() { + const host = process.env.REAL_MONGODB_HOST || "127.0.0.1"; + const port = process.env.REAL_MONGODB_PORT || "27017"; + const user = process.env.REAL_MONGODB_USER || "testuser"; + const password = process.env.REAL_MONGODB_PASSWORD || "testpass"; + const database = process.env.REAL_MONGODB_DATABASE || "testdb"; + const uri = + process.env.REAL_MONGODB_URI || + `mongodb://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:${port}/?authSource=admin`; + + const client = new MongoClient(uri); + + let connected = false; + for (let attempt = 1; attempt <= 15; attempt++) { + try { + await client.connect(); + await client.db(database).command({ ping: 1 }); + connected = true; + break; + } catch { + await wait(1000); + } + } + + if (!connected) { + throw new Error(`MongoDB at ${host}:${port} was not reachable after 15 attempts.`); + } + + const db = client.db(database); + + // Idempotent: drop existing fixtures first. + for (const name of ["users", "orders", "active_users_view"]) { + try { + await db.dropCollection(name); + } catch { + // may not exist + } + } + + const userIds = []; + const users = [ + { + _id: new ObjectId(), + name: "John Doe", + email: "john@example.com", + age: 30, + active: true, + createdAt: new Date(), + }, + { + _id: new ObjectId(), + name: "Jane Smith", + email: "jane@example.com", + age: 27, + active: true, + createdAt: new Date(), + }, + { + _id: new ObjectId(), + name: "Bob Lee", + email: "bob@example.com", + age: 45, + active: false, + createdAt: new Date(), + }, + ]; + const userInsert = await db.collection("users").insertMany(users); + for (const doc of userInsert.insertedDocs.values()) { + userIds.push(doc._id); + } + + const orders = [ + { + _id: new ObjectId(), + userId: userIds[0], + items: [ + { sku: "ABC-1", qty: 2, price: 9.99 }, + { sku: "XYZ-9", qty: 1, price: 19.5 }, + ], + total: 39.48, + status: "placed", + orderedAt: new Date(), + }, + { + _id: new ObjectId(), + userId: userIds[1], + items: [{ sku: "ABC-1", qty: 5, price: 9.99 }], + total: 49.95, + status: "shipped", + orderedAt: new Date(), + }, + ]; + await db.collection("orders").insertMany(orders); + + // A view to exercise VIEW detection in listTables/listCollections. + await db.createView("active_users_view", "users", [{ $match: { active: true } }]); + + await client.close(); + console.log("MongoDB test database seeded successfully."); +} + +seed().catch((err) => { + console.error("Failed to seed MongoDB:", err); + process.exit(1); +}); diff --git a/bridge/src/connectors/mongo.ts b/bridge/src/connectors/mongo.ts new file mode 100644 index 00000000..59eeee7a --- /dev/null +++ b/bridge/src/connectors/mongo.ts @@ -0,0 +1,276 @@ +/** + * MongoDB connector (Milestone 1 foundation). + * + * Implements the connection / listing / statistics subset required to wire + * MongoDB into the existing structural connector registry: + * - client pool (per uri+database) + * - testConnection + * - listCollections (native, document-oriented) + * - listTables (relational compatibility adapter → TableInfo[]) + * - getDBStats (→ DBStats) + * - listSchemaNames (single pseudo-schema) + * - connectToDatabase (minimal; no baseline, no schema_migrations) + * - mongodbCache (satisfies the `${dbType}Cache` naming convention) + * + * Document-native operations (findDocuments / insertDocument / updateDocument / + * deleteDocument / runAggregation / inferCollectionSchema) and the row/table + * compatibility adapters land in a later milestone. They are intentionally NOT + * stubbed here. + * + * Credentials never appear in logs or error messages: connection errors are + * sanitized and the raw config is never logged. + */ + +import { MongoClient } from "mongodb"; +import { MongoConfig, MongoCollectionInfo } from "../types/mongo"; +import { TableInfo, DBStats, SchemaInfo } from "../types/common"; +import { CacheEntry } from "../types/cache"; +import logger from "../services/logger"; + +// --------------------------------------------------------------------------- +// Error sanitization — decision #8: never expose credentials in errors/logs. +// --------------------------------------------------------------------------- + +/** Redact any embedded credentials if a message ever echoes the connection URI. */ +function safeMongoError(err: unknown): string { + const base = err instanceof Error ? err.message : String(err); + return base.replace(/(mongodb\+?srv?:\/\/)[^@\s]+@/gi, "$1@"); +} + +// --------------------------------------------------------------------------- +// CACHING +// --------------------------------------------------------------------------- + +const CACHE_TTL = 30_000; +const STATS_CACHE_TTL = 10_000; + +class MongoCacheManager { + private tableListCache = new Map>(); + private dbStatsCache = new Map>(); + + private getConfigKey(cfg: MongoConfig): string { + return `${cfg.uri || ""}:${cfg.database || ""}`; + } + + private isValid(entry: CacheEntry | undefined): boolean { + return !!entry && Date.now() - entry.timestamp < entry.ttl; + } + + getTableList(cfg: MongoConfig): TableInfo[] | null { + const entry = this.tableListCache.get(this.getConfigKey(cfg)); + return this.isValid(entry) ? entry!.data : null; + } + setTableList(cfg: MongoConfig, data: TableInfo[]): void { + this.tableListCache.set(this.getConfigKey(cfg), { + data, + timestamp: Date.now(), + ttl: CACHE_TTL, + }); + } + + getDBStats(cfg: MongoConfig): DBStats | null { + const entry = this.dbStatsCache.get(this.getConfigKey(cfg)); + return this.isValid(entry) ? entry!.data : null; + } + setDBStats(cfg: MongoConfig, data: DBStats): void { + this.dbStatsCache.set(this.getConfigKey(cfg), { + data, + timestamp: Date.now(), + ttl: STATS_CACHE_TTL, + }); + } + + clearForConnection(cfg: MongoConfig): void { + const key = this.getConfigKey(cfg); + for (const [k] of this.tableListCache) { + if (k.startsWith(key)) this.tableListCache.delete(k); + } + this.dbStatsCache.delete(key); + } + + clearAll(): void { + this.tableListCache.clear(); + this.dbStatsCache.clear(); + } +} + +export const mongodbCache = new MongoCacheManager(); + +// --------------------------------------------------------------------------- +// CLIENT CONNECTION POOL +// --------------------------------------------------------------------------- + +const clientPool = new Map(); + +function getClientKey(cfg: MongoConfig): string { + return `${cfg.uri || ""}:${cfg.database || ""}`; +} + +export function getMongoClient(cfg: MongoConfig): MongoClient { + const key = getClientKey(cfg); + let client = clientPool.get(key); + if (!client) { + if (!cfg.uri) { + throw new Error("MongoDB connection requires a connection URI"); + } + client = new MongoClient(cfg.uri); + clientPool.set(key, client); + } + return client; +} + +export async function closeMongoClient(cfg: MongoConfig): Promise { + const key = getClientKey(cfg); + const client = clientPool.get(key); + if (client) { + try { + await client.close(); + } catch { + // ignore + } + clientPool.delete(key); + } +} + +export async function closeAllMongoClients(): Promise { + for (const client of clientPool.values()) { + try { + await client.close(); + } catch { + // ignore + } + } + clientPool.clear(); +} + +/** Select the target database, defaulting to the configured one. */ +function selectDb(cfg: MongoConfig): ReturnType { + const client = getMongoClient(cfg); + return cfg.database ? client.db(cfg.database) : client.db(); +} + +// --------------------------------------------------------------------------- +// CORE CONNECTOR OPERATIONS (M1 subset) +// --------------------------------------------------------------------------- + +export async function testConnection( + cfg: MongoConfig, +): Promise<{ ok: boolean; message?: string; status: "connected" | "disconnected" }> { + if (!cfg.uri) { + return { ok: false, status: "disconnected", message: "Missing connection URI" }; + } + + const tempClient = new MongoClient(cfg.uri); + try { + await tempClient.db().admin().ping(); + return { ok: true, status: "connected", message: "Connection successful" }; + } catch (err) { + return { ok: false, status: "disconnected", message: safeMongoError(err) }; + } finally { + try { + await tempClient.close(); + } catch { + // ignore + } + } +} + +/** Native: list collections (and views) in the configured database. */ +export async function listCollections(cfg: MongoConfig): Promise { + const cached = mongodbCache.getTableList(cfg); + if (cached) { + // Derive collection info from the cached relational adapter view. + return cached.map((t) => ({ name: t.name, type: t.type === "VIEW" ? "view" : "collection" })); + } + + const db = selectDb(cfg); + const infos = await db.listCollections({}, { nameOnly: true }).toArray(); + + const result: MongoCollectionInfo[] = infos.map((c) => { + const isView = c.type === "view"; + return { name: c.name, type: isView ? "view" : "collection" }; + }); + + // Prime the table-list cache from the native result. + mongodbCache.setTableList( + cfg, + result.map((c) => ({ + schema: cfg.database || "default", + name: c.name, + type: c.type === "view" ? "VIEW" : "BASE TABLE", + })), + ); + return result; +} + +/** + * Relational compatibility adapter: map collections → `TableInfo[]` so the + * existing `db.listTables` RPC and schema tree keep working. The relational + * `schema` filter is ignored (Mongo is not schema-namespaced). + */ +export async function listTables(cfg: MongoConfig, _schema?: string): Promise { + const cached = mongodbCache.getTableList(cfg); + if (cached) return cached; + + const collections = await listCollections(cfg); + const tables: TableInfo[] = collections.map((c) => ({ + schema: cfg.database || "default", + name: c.name, + type: c.type === "view" ? "VIEW" : "BASE TABLE", + })); + mongodbCache.setTableList(cfg, tables); + return tables; +} + +/** Basic database statistics mapped onto the shared `DBStats` shape. */ +export async function getDBStats(cfg: MongoConfig): Promise { + const cached = mongodbCache.getDBStats(cfg); + if (cached) return cached; + + const db = selectDb(cfg); + const stats = await db.stats(); + + let total_tables = 0; + try { + const collInfos = await db.listCollections({}, { nameOnly: true }).toArray(); + total_tables = collInfos.length; + } catch (err) { + logger.warn({ err: safeMongoError(err) }, "MongoDB countCollections failed"); + } + + const total_rows = typeof stats.objects === "number" ? stats.objects : 0; + const total_db_size_mb = + typeof stats.size === "number" ? Math.round((stats.size / (1024 * 1024)) * 100) / 100 : 0; + + const result: DBStats = { total_tables, total_rows, total_db_size_mb }; + mongodbCache.setDBStats(cfg, result); + return result; +} + +/** Single pseudo-schema (the database name) for the existing `db.listSchemas`. */ +export async function listSchemaNames(cfg: MongoConfig): Promise { + return [cfg.database || "default"]; +} + +/** + * Minimal connect bootstrap. Returns collections as pseudo-schemas and empty + * migration lists. No baseline and no `schema_migrations` — MongoDB migrations + * are out of scope for this milestone. + */ +export async function connectToDatabase( + cfg: MongoConfig, + _connectionId: string, + _options?: { readOnly?: boolean }, +): Promise<{ + baselined: boolean; + schema: SchemaInfo[]; + migrations: { local: unknown[]; applied: unknown[] }; +}> { + const collections = await listCollections(cfg); + const schema: SchemaInfo[] = collections.map((c) => ({ name: c.name })); + return { + baselined: false, + schema, + migrations: { local: [], applied: [] }, + }; +} diff --git a/bridge/src/connectors/mongoBson.ts b/bridge/src/connectors/mongoBson.ts new file mode 100644 index 00000000..a59a24f9 --- /dev/null +++ b/bridge/src/connectors/mongoBson.ts @@ -0,0 +1,143 @@ +/** + * BSON-to-wire serializer for MongoDB documents. + * + * Converts a driver document into a lossless, JSON-safe `SerializedMongoDocument` + * that can travel over the newline-delimited JSON-RPC stdio channel. ObjectId, + * Decimal128, Long, Timestamp, Binary, and `undefined` are not JSON-representable + * and are mapped to MongoDB Extended JSON forms; plain primitives pass through. + * + * Type detection is by BSON `_bsontype` (stable across driver/bson versions) plus + * `instanceof Date` / `Buffer.isBuffer`, so this module is pure and unit-testable + * without a live driver. No value it returns fails `JSON.stringify`. + */ + +import type { Document } from "mongodb"; +import type { + SerializedMongoDocument, + MongoWireValue, + MongoWireObject, + MongoWireBinary, +} from "../types/mongo"; + +type WithBsontype = { _bsontype?: string }; + +/** Minimal structural views so we never touch `any`. */ +interface ObjectIdLike { + _bsontype: "ObjectId"; + toHexString(): string; +} +interface Decimal128Like { + _bsontype: "Decimal128"; + toString(): string; +} +interface LongLike { + _bsontype: "Long"; + toString(): string; +} +interface TimestampLike { + _bsontype: "Timestamp"; + t: number; + i: number; +} +interface IntLike { + _bsontype: "Int32" | "Double"; + valueOf(): number; +} +interface BinaryLike { + _bsontype: "Binary"; + buffer: Uint8Array; + sub_type?: number; +} + +function toFiniteNumber(n: number): number | null { + return Number.isFinite(n) ? n : null; +} + +function serializeBinary(value: BinaryLike): MongoWireBinary { + const buf = Buffer.isBuffer(value.buffer) ? value.buffer : Buffer.from(value.buffer); + const subtype = typeof value.sub_type === "number" ? value.sub_type : 0; + return { + $binary: { + base64: buf.toString("base64"), + subtype: subtype.toString(16).padStart(2, "0"), + }, + }; +} + +/** + * Serialize a single BSON value into a JSON-safe wire value. Arrays and nested + * documents recurse. + */ +export function serializeMongoValue(value: unknown): MongoWireValue { + if (value === null) return null; + + const kind = typeof value; + if (kind === "string" || kind === "boolean") { + return value as MongoWireValue; + } + if (kind === "number") { + return toFiniteNumber(value as number) as MongoWireValue; + } + if (kind === "undefined") { + return { $undefined: true }; + } + if (Array.isArray(value)) { + return value.map((item) => serializeMongoValue(item)); + } + + const bsontype = (value as WithBsontype)._bsontype; + if (bsontype) { + switch (bsontype) { + case "ObjectId": + return { $oid: (value as ObjectIdLike).toHexString() }; + case "Decimal128": + return { $numberDecimal: (value as Decimal128Like).toString() }; + case "Long": + return { $numberLong: (value as LongLike).toString() }; + case "Timestamp": { + const ts = value as TimestampLike; + return { $timestamp: { t: ts.t | 0, i: ts.i | 0 } }; + } + case "Int32": + case "Double": + return toFiniteNumber((value as IntLike).valueOf()) as MongoWireValue; + case "Binary": + return serializeBinary(value as BinaryLike); + default: + // Unknown BSON subtype (e.g. DBRef) falls through to object serialization. + break; + } + } + + if (value instanceof Date) { + return { $date: value.toISOString() }; + } + if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + return serializeBinary({ _bsontype: "Binary", buffer: value, sub_type: 0 }); + } + + if (kind === "object") { + const out: MongoWireObject = {}; + for (const [key, item] of Object.entries(value as Record)) { + if (key === "_bsontype") continue; + out[key] = serializeMongoValue(item); + } + return out; + } + + // functions / symbols / bigint are not representable on the wire. + return null; +} + +/** + * Serialize a whole document. Own enumerable keys are serialized; the internal + * `_bsontype` marker is dropped. + */ +export function serializeMongoDocument(doc: Document): SerializedMongoDocument { + const out: SerializedMongoDocument = {}; + for (const [key, item] of Object.entries(doc as Record)) { + if (key === "_bsontype") continue; + out[key] = serializeMongoValue(item); + } + return out; +} diff --git a/bridge/src/handlers/migrationHandlers.ts b/bridge/src/handlers/migrationHandlers.ts index e893ee89..9f0d65d6 100644 --- a/bridge/src/handlers/migrationHandlers.ts +++ b/bridge/src/handlers/migrationHandlers.ts @@ -27,6 +27,12 @@ export class MigrationHandlers { } const { dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Migrations are not supported for MongoDB databases", + }); + } const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); // Generate migration file @@ -68,6 +74,12 @@ export class MigrationHandlers { } const { dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Migrations are not supported for MongoDB databases", + }); + } const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); // Generate migration file @@ -115,6 +127,12 @@ export class MigrationHandlers { } const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + if (dbType === "mongodb") { + return this.rpc.sendError(id, { + code: "BAD_REQUEST", + message: "Migrations are not supported for MongoDB databases", + }); + } const migrationsDir = await projectStoreInstance.resolveMigrationsDir(dbId); let columns = passedColumns; diff --git a/bridge/src/services/connectionBuilder.ts b/bridge/src/services/connectionBuilder.ts index 3e91e98d..1f57a77b 100644 --- a/bridge/src/services/connectionBuilder.ts +++ b/bridge/src/services/connectionBuilder.ts @@ -1,11 +1,13 @@ import { DBType, DatabaseConfig } from "../types"; import { SQLiteConfig } from "../types/sqlite"; +import { MongoConfig } from "../types/mongo"; import { isWindowsDriveRootPath, normalizeSQLitePath } from "../utils/sqlitePath"; import { parseClickHouseUrl } from "../utils/clickhouseUrl"; +import { buildMongoConnectionUri } from "../utils/mongoUri"; import { sshTunnelServiceInstance, TunnelInfo } from "./sshTunnelService"; export interface BuildResult { - config: DatabaseConfig | SQLiteConfig; + config: DatabaseConfig | SQLiteConfig | MongoConfig; tunnel?: TunnelInfo; } @@ -52,6 +54,21 @@ export class ConnectionBuilder { }; } + if (dbType === DBType.MONGODB) { + // The persisted URI is credential-free; the runtime password is re-injected + // here (in memory only) via the sanitized URI + decrypted pwd. + const uri = buildMongoConnectionUri(db, pwd); + const config: MongoConfig = { + uri, + database: db.database, + authSource: db.authSource, + replicaSet: db.replicaSet, + tls: db.tls, + directConnection: db.directConnection, + }; + return { config }; + } + const defaultPort = dbType === DBType.MYSQL || dbType === DBType.MARIADB ? 3306 : 5432; const base: DatabaseConfig = { diff --git a/bridge/src/services/connectorKind.ts b/bridge/src/services/connectorKind.ts new file mode 100644 index 00000000..37db5830 --- /dev/null +++ b/bridge/src/services/connectorKind.ts @@ -0,0 +1,26 @@ +/** + * Database kind discriminator. + * + * A single switch key for "relational vs document" behavior, replacing scattered + * per-engine checks. M1 only defines it; later milestones consume it to gate + * SQL-only features (migrations, relational schema, SQL editor/builder, ER + * auto-edges, monitoring whitelists, NL-to-SQL). + */ + +import { DBType } from "../types"; + +export type DatabaseKind = "sql" | "nosql"; + +const KIND: Record = { + [DBType.POSTGRES]: "sql", + [DBType.MYSQL]: "sql", + [DBType.MARIADB]: "sql", + [DBType.SQLITE]: "sql", + [DBType.DUCKDB]: "sql", + [DBType.CLICKHOUSE]: "sql", + [DBType.MONGODB]: "nosql", +}; + +export function connectorKind(dbType: DBType): DatabaseKind { + return KIND[dbType]; +} diff --git a/bridge/src/services/connectorRegistry.ts b/bridge/src/services/connectorRegistry.ts index fe84ca45..7f60b577 100644 --- a/bridge/src/services/connectorRegistry.ts +++ b/bridge/src/services/connectorRegistry.ts @@ -13,6 +13,7 @@ import * as mariadbConnector from "../connectors/mariadb"; import * as sqliteConnector from "../connectors/sqlite"; import * as duckdbConnector from "../connectors/duckdb"; import * as clickhouseConnector from "../connectors/clickhouse"; +import * as mongoConnector from "../connectors/mongo"; export type Connector = | typeof postgresConnector @@ -20,7 +21,8 @@ export type Connector = | typeof mariadbConnector | typeof sqliteConnector | typeof duckdbConnector - | typeof clickhouseConnector; + | typeof clickhouseConnector + | typeof mongoConnector; const registry: Record = { [DBType.POSTGRES]: postgresConnector, @@ -29,6 +31,7 @@ const registry: Record = { [DBType.SQLITE]: sqliteConnector, [DBType.DUCKDB]: duckdbConnector, [DBType.CLICKHOUSE]: clickhouseConnector, + [DBType.MONGODB]: mongoConnector, }; /** diff --git a/bridge/src/services/databaseService.ts b/bridge/src/services/databaseService.ts index e23bf6ba..4f46fa39 100644 --- a/bridge/src/services/databaseService.ts +++ b/bridge/src/services/databaseService.ts @@ -1,6 +1,12 @@ import { ConnectionBuilder } from "./connectionBuilder"; import { DBTypeDetector } from "../utils/dbTypeDetector"; import { DBType } from "../types"; +import { + isMongoUri, + extractMongoCredentials, + sanitizeMongoUri, + normalizeMongoUriMeta, +} from "../utils/mongoUri"; import { dbStoreInstance } from "./dbStore"; // always use the singleton import { connectionPool } from "./connectionPool"; import { keyringServiceInstance } from "./keyringService"; @@ -63,9 +69,10 @@ export class DatabaseService { (payload.type as string | undefined)?.toLowerCase() || "", ); const isClickHouse = (payload.type as string | undefined)?.toLowerCase() === "clickhouse"; + const isMongo = (payload.type as string | undefined)?.toLowerCase() === "mongodb"; const required = isFileBasedDB ? ["name", "database", "type"] - : isClickHouse + : isClickHouse || isMongo ? ["name", "type"] : ["name", "host", "port", "user", "database", "type"]; for (const field of required) { @@ -102,6 +109,32 @@ export class DatabaseService { database: payload.database || parsed.database || "default", ssl: payload.ssl !== undefined ? payload.ssl : parsed.ssl, }; + } else if (isMongo) { + if (!payload.url && !payload.host) { + throw new Error("Missing required field: url"); + } + if (payload.url && isMongoUri(payload.url)) { + // Extract any embedded credentials into the AES store and persist a + // credential-free (sanitized) URI instead. dbStore.addDB encrypts + // payload.password and stores the sanitized url at rest. + const creds = extractMongoCredentials(payload.url); + const meta = normalizeMongoUriMeta(payload.url); + payload = { + ...payload, + url: sanitizeMongoUri(payload.url), + host: payload.host || meta.host, + port: payload.port || meta.port, + user: payload.user || meta.user, + database: payload.database || meta.database, + authSource: payload.authSource ?? meta.authSource, + replicaSet: payload.replicaSet ?? meta.replicaSet, + }; + if (creds.password) { + payload = { ...payload, password: creds.password }; + } + } else { + payload = { ...payload, port: payload.port || 27017, database: payload.database || "" }; + } } const res = await dbStoreInstance.addDB(payload as Parameters[0]); @@ -123,6 +156,7 @@ export class DatabaseService { (payload.type as string | undefined)?.toLowerCase() || "", ); const isClickHouse = (payload.type as string | undefined)?.toLowerCase() === "clickhouse"; + const isMongo = (payload.type as string | undefined)?.toLowerCase() === "mongodb"; // Extract SSH credentials const sshPwd = payload.ssh?.password; @@ -144,6 +178,15 @@ export class DatabaseService { ssl: payload.ssl !== undefined ? payload.ssl : parsed.ssl, }; } + if (isMongo && payload.url && isMongoUri(payload.url)) { + // Keep persisted mongo URIs credential-free on update too. An explicitly + // supplied password wins over one embedded in the URI. + const creds = extractMongoCredentials(payload.url); + payload = { ...payload, url: sanitizeMongoUri(payload.url) }; + if (creds.password && payload.password === undefined) { + payload = { ...payload, password: creds.password }; + } + } if (payload.ssh) { delete payload.ssh.password; delete payload.ssh.passphrase; diff --git a/bridge/src/types/index.ts b/bridge/src/types/index.ts index b05ea952..52d2576f 100644 --- a/bridge/src/types/index.ts +++ b/bridge/src/types/index.ts @@ -16,6 +16,7 @@ export * from "./postgres"; export * from "./sqlite"; export * from "./duckdb"; export * from "./clickhouse"; +export * from "./mongo"; export * from "./ai"; export { SSHConfig } from "./common"; @@ -26,6 +27,7 @@ export enum DBType { SQLITE = "sqlite", DUCKDB = "duckdb", CLICKHOUSE = "clickhouse", + MONGODB = "mongodb", } export type Rpc = { diff --git a/bridge/src/types/mongo.ts b/bridge/src/types/mongo.ts new file mode 100644 index 00000000..a1fb5748 --- /dev/null +++ b/bridge/src/types/mongo.ts @@ -0,0 +1,96 @@ +/** + * MongoDB-specific Types + * + * Single source of Mongo types for the bridge. Kept deliberately separate from + * `common.ts` (which models the relational catalog) so that "row / table / + * primary key" is never the conceptual MongoDB model. The connector reuses the + * shared `TableInfo` / `DBStats` shapes only as compatibility adapters for the + * existing relational RPC surface. + */ + +/** + * Connection configuration handed to the connector by ConnectionBuilder. + * `uri` is the authoritative field: it is the (runtime-reconstructed) + * authenticated `mongodb://` / `mongodb+srv://` connection string. The other + * fields are carried for completeness / composed-form connections. + */ +export type MongoConfig = { + uri?: string; // full authenticated connection string (preferred) + host?: string; // composed-form fallback + port?: number; + user?: string; + password?: string; // present only transiently when composing a URI; never persisted + database?: string; // default/auth database + authSource?: string; + replicaSet?: string; + tls?: boolean; + directConnection?: boolean; +}; + +// --------------------------------------------------------------------------- +// Typed BSON-to-wire representation (see connectors/mongoBson.ts). +// Every value here is safe to pass through JSON.stringify on the stdio channel. +// --------------------------------------------------------------------------- + +export type MongoWireObject = { [key: string]: MongoWireValue }; +export type MongoWireArray = MongoWireValue[]; + +export type MongoWireObjectId = { $oid: string }; +export type MongoWireDate = { $date: string }; // ISO-8601 UTC +export type MongoWireBinary = { $binary: { base64: string; subtype: string } }; +export type MongoWireDecimal = { $numberDecimal: string }; +export type MongoWireLong = { $numberLong: string }; +export type MongoWireTimestamp = { $timestamp: { t: number; i: number } }; +export type MongoWireUndefined = { $undefined: true }; + +export type MongoWireValue = + | null + | boolean + | number + | string + | MongoWireArray + | MongoWireObject + | MongoWireObjectId + | MongoWireDate + | MongoWireBinary + | MongoWireDecimal + | MongoWireLong + | MongoWireTimestamp + | MongoWireUndefined; + +export type SerializedMongoDocument = { [key: string]: MongoWireValue }; + +// --------------------------------------------------------------------------- +// Native document-oriented result shapes. +// `MongoCollectionInfo` is used in M1 (collection listing). The inferred-schema +// shapes are declared now so the connector contract is complete and typed; they +// are consumed starting in a later milestone. +// --------------------------------------------------------------------------- + +export type MongoCollectionInfo = { name: string; type: "collection" | "view" }; + +export type MongoInferredField = { + path: string; + bsonType: string; + presentInPct: number; + sampleCount: number; + nested?: MongoInferredField[]; +}; + +export type MongoInferredSchema = { + collection: string; + fields: MongoInferredField[]; + sampled: number; + totalDocs?: number; +}; + +/** + * Result of parsing a MongoDB URI that may embed credentials. + * `sanitizedUri` never contains a password and is what gets persisted at rest. + */ +export type MongoUriParts = { + sanitizedUri: string; + username?: string; + password?: string; + hasCredentials: boolean; +}; diff --git a/bridge/src/utils/dbTypeDetector.ts b/bridge/src/utils/dbTypeDetector.ts index 9df8046c..fd97c444 100644 --- a/bridge/src/utils/dbTypeDetector.ts +++ b/bridge/src/utils/dbTypeDetector.ts @@ -12,6 +12,7 @@ export class DBTypeDetector { if (normalized.includes("sqlite")) return DBType.SQLITE; if (normalized.includes("duckdb")) return DBType.DUCKDB; if (normalized.includes("clickhouse")) return DBType.CLICKHOUSE; + if (normalized.includes("mongo")) return DBType.MONGODB; } return DBType.POSTGRES; // default } diff --git a/bridge/src/utils/mongoUri.ts b/bridge/src/utils/mongoUri.ts new file mode 100644 index 00000000..e7485430 --- /dev/null +++ b/bridge/src/utils/mongoUri.ts @@ -0,0 +1,218 @@ +/** + * MongoDB URI utilities. + * + * Implements the decision-#11 credential handling: a MongoDB connection URI that + * embeds credentials is parsed, the password is extracted (to be stored through + * the existing dbStore AES path), and a *sanitized* URI (credentials removed) is + * what gets persisted in databases.json. The authenticated URI is reconstructed + * only at runtime from the sanitized URI + the decrypted password, and is never + * written back. + * + * WHATWG `URL` parses `mongodb://` and `mongodb+srv://` authorities correctly + * (credentials, host, port, path, and query options), so it is used as the + * parser. All returned strings are JSON-safe and free of plaintext passwords + * unless explicitly re-injecting at runtime. + */ + +import type { MongoUriParts } from "../types/mongo"; + +const MONGO_URI_RE = /^mongodb(\+srv)?:\/\//i; + +/** True when the string is a MongoDB connection URI. */ +export function isMongoUri(raw?: string): boolean { + return typeof raw === "string" && MONGO_URI_RE.test(raw.trim()); +} + +type MongoUriComponents = { + scheme: "mongodb" | "mongodb+srv"; + host: string; + port?: number; + user?: string; + password?: string; // decoded + database?: string; + options: string; // raw query string, without leading '?' +}; + +/** Parse a MongoDB URI into its components (password decoded). */ +function parseMongoUriComponents(raw: string): MongoUriComponents { + const trimmed = raw.trim(); + const u = new URL(trimmed); + const scheme: "mongodb" | "mongodb+srv" = trimmed.toLowerCase().startsWith("mongodb+srv://") + ? "mongodb+srv" + : "mongodb"; + const user = u.username ? decodeURIComponent(u.username) : undefined; + const password = u.password ? decodeURIComponent(u.password) : undefined; + const host = u.hostname; + const port = u.port ? Number(u.port) : undefined; + const database = + u.pathname && u.pathname !== "/" + ? decodeURIComponent(u.pathname.replace(/^\//, "")) + : undefined; + const options = u.search ? u.search.slice(1) : ""; + return { scheme, host, port, user, password, database, options }; +} + +/** Rebuild a MongoDB URI string from components. */ +function buildMongoUriString(c: MongoUriComponents, includePassword: boolean): string { + let auth = ""; + if (c.user) auth += encodeURIComponent(c.user); + if (includePassword && c.password) auth += ":" + encodeURIComponent(c.password); + if (auth) auth += "@"; + const portPart = c.port ? `:${c.port}` : ""; + const path = c.database ? `/${encodeURIComponent(c.database)}` : ""; + const qs = c.options ? `?${c.options}` : ""; + return `${c.scheme}://${auth}${c.host}${portPart}${path}${qs}`; +} + +type MongoUriOptions = { authSource?: string; replicaSet?: string }; + +function parseMongoOptions(rawOptions: string): MongoUriOptions { + const out: MongoUriOptions = {}; + if (!rawOptions) return out; + for (const pair of rawOptions.split("&")) { + const idx = pair.indexOf("="); + if (idx === -1) continue; + const key = pair.slice(0, idx); + const value = decodeURIComponent(pair.slice(idx + 1)); + if (key === "authSource") out.authSource = value; + else if (key === "replicaSet") out.replicaSet = value; + } + return out; +} + +/** + * Extract credentials from a MongoDB URI. The password returned here is meant to + * be stored through the dbStore AES credential path — never persisted in the URI. + */ +export function extractMongoCredentials(uri: string): { user?: string; password?: string } { + const c = parseMongoUriComponents(uri); + return { user: c.user, password: c.password }; +} + +/** + * Return a copy of the URI with any embedded password removed. The username and + * all query options are preserved (only the secret is stripped). + */ +export function sanitizeMongoUri(uri: string): string { + return buildMongoUriString(parseMongoUriComponents(uri), false); +} + +/** + * Reconstruct the authenticated URI at runtime by re-injecting the password into + * a sanitized URI. If the URI has no username there is nowhere to attach the + * password, so the URI is returned unchanged. + */ +export function injectMongoPassword(sanitizedUri: string, password: string): string { + const c = parseMongoUriComponents(sanitizedUri); + if (!c.user) return sanitizedUri; + return buildMongoUriString({ ...c, password }, true); +} + +/** + * Split a URI (if any) into a persisted-safe shape: the sanitized URI plus the + * extracted password (to be AES-stored). When the input is not a MongoDB URI the + * original value is passed through untouched with no credentials. + */ +export function splitMongoUriCredentials(uri: string): MongoUriParts & { isMongoUri: boolean } { + if (!isMongoUri(uri)) { + return { sanitizedUri: uri, hasCredentials: false, isMongoUri: false }; + } + const c = parseMongoUriComponents(uri); + const sanitizedUri = buildMongoUriString(c, false); + const hasCredentials = Boolean(c.password); + return { + sanitizedUri, + username: c.user, + password: c.password, + hasCredentials, + isMongoUri: true, + }; +} + +/** + * Derive the relational-meta fields (host/port/user/database/options) from a + * MongoDB URI so dbStore's required fields are populated. Options such as + * authSource / replicaSet remain in the (sanitized) URI query and are therefore + * preserved on the wire; they are also surfaced here for convenience. + */ +export function normalizeMongoUriMeta(uri: string): { + host: string; + port: number; + user: string; + database: string; + authSource?: string; + replicaSet?: string; +} { + const c = parseMongoUriComponents(uri); + const opts = parseMongoOptions(c.options); + return { + host: c.host, + port: c.port || 27017, + user: c.user || "", + database: c.database || "", + authSource: opts.authSource, + replicaSet: opts.replicaSet, + }; +} + +/** + * Build the authenticated connection URI handed to the driver. + * - If a MongoDB URI is present it is treated as the persisted (sanitized) form + * and the runtime password is re-injected when both a username and a password + * exist. + * - Otherwise a URI is composed from host/port/user/password. + */ +export function buildMongoConnectionUri( + db: { + url?: string; + uri?: string; + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + authSource?: string; + replicaSet?: string; + tls?: boolean; + directConnection?: boolean; + }, + pwd: string | null, +): string { + const rawUri = db.url || db.uri; + + if (rawUri && isMongoUri(rawUri)) { + const c = parseMongoUriComponents(rawUri); + const pass = pwd ?? undefined; + // Only attach the runtime password when there is a username to attach it to. + const finalPassword = c.user && pass ? pass : c.password; + return buildMongoUriString({ ...c, password: finalPassword }, true); + } + + const host = db.host; + if (!host) { + throw new Error("MongoDB connection requires a url (mongodb://...) or host"); + } + const port = db.port || 27017; + const user = db.user; + const pass = pwd ?? db.password; + + let auth = ""; + if (user) { + auth += encodeURIComponent(user); + if (pass) auth += ":" + encodeURIComponent(pass); + auth += "@"; + } + + let uri = `mongodb://${auth}${host}:${port}`; + if (db.database) uri += `/${encodeURIComponent(db.database)}`; + + const params = new URLSearchParams(); + if (db.authSource) params.set("authSource", db.authSource); + if (db.replicaSet) params.set("replicaSet", db.replicaSet); + if (db.tls) params.set("tls", "true"); + if (db.directConnection) params.set("directConnection", "true"); + const qs = params.toString(); + if (qs) uri += `?${qs}`; + + return uri; +} From 6429f3baccfdc3b0ddda6d1f3c52649e86728c39 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 26 Oct 2026 23:28:08 +0530 Subject: [PATCH 02/34] feat(mongodb): implement M2 document engine --- .../connectors/mongoDocuments.test.ts | 290 ++++++++ .../mongoDocuments.integration.test.ts | 349 ++++++++++ bridge/src/connectors/mongo.ts | 631 +++++++++++++++++- bridge/src/connectors/mongoBson.ts | 223 +++++++ bridge/src/handlers/queryHandlers.ts | 116 ++++ bridge/src/jsonRpcHandler.ts | 6 + bridge/src/services/queryExecutor.ts | 86 +++ bridge/src/types/mongo.ts | 167 +++++ 8 files changed, 1866 insertions(+), 2 deletions(-) create mode 100644 bridge/__tests__/connectors/mongoDocuments.test.ts create mode 100644 bridge/__tests__/integration/mongoDocuments.integration.test.ts diff --git a/bridge/__tests__/connectors/mongoDocuments.test.ts b/bridge/__tests__/connectors/mongoDocuments.test.ts new file mode 100644 index 00000000..0538ac1c --- /dev/null +++ b/bridge/__tests__/connectors/mongoDocuments.test.ts @@ -0,0 +1,290 @@ +import { describe, expect, test } from "@jest/globals"; +import { ObjectId, Int32, Long } from "mongodb"; +import { + mongoEJSONtoBSON, + mongoBSONtoEJSON, + serializeMongoUpdateOperand, + serializeMongoPipelineStage, +} from "../../src/connectors/mongoBson"; +import { + mongoConnector, + mongodbCache, + MongoApiError, + mapMongoError, + withMongoTimeout, +} from "../../src/connectors/mongo"; +import type { + MongoConfig, + MongoFindRequest, + MongoSortSpec, + EJSONDocument, +} from "../../src/types/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; + +// --------------------------------------------------------------------------- +// M2 unit tests — driver-free. Covers the EJSON<->BSON converters, the request +// validation / limit guards (which reject before any driver call), the error +// mapper, the client-side timeout wrapper, and the collection-scoped cache. +// (docs/MONGODB_M2_SPEC.md §16.1) +// --------------------------------------------------------------------------- + +describe("mongoBson EJSON round-trip", () => { + const cases: unknown[] = [ + { $oid: "507f1f77bcf86cd799439011" }, + { $date: "2024-01-01T00:00:00.000Z" }, + { $numberInt: "123" }, + { $numberLong: "9007199254740993" }, + { $numberDecimal: "1.50" }, + { $binary: { $type: "00", $base64: "aGVsbG8=" } }, + { $regex: "ab+c", $options: "i" }, + { $timestamp: { $t: "1", $i: "2" } }, + { + name: "x", + tags: ["a", "b"], + nested: { + n: { $numberInt: "5" }, + id: { $oid: "507f1f77bcf86cd799439011" }, + }, + }, + "plain string", + 42, + 3.14, + true, + false, + null, + ]; + + test("mongoBSONtoEJSON(mongoEJSONtoBSON(x)) is structurally equal to x", () => { + for (const c of cases) { + expect(mongoBSONtoEJSON(mongoEJSONtoBSON(c as EJSONDocument))).toEqual(c); + } + }); + + test("a bare hex string stays a string; only $oid yields an ObjectId", () => { + const hex = "507f1f77bcf86cd799439011"; + expect(mongoEJSONtoBSON({ v: hex }).v).toBe(hex); + expect(mongoEJSONtoBSON({ v: { $oid: hex } }).v).toBeInstanceOf(ObjectId); + }); + + test("unknown $-wrappers pass through unchanged", () => { + const input: EJSONDocument = { $unknownThing: "x" }; + expect(mongoBSONtoEJSON(mongoEJSONtoBSON(input))).toEqual(input); + }); + + test("serializeMongoUpdateOperand preserves operator keys and resolves leaf values", () => { + const op = serializeMongoUpdateOperand({ + $set: { name: "a", age: { $numberInt: "5" } }, + $inc: { count: { $numberLong: "1" } }, + }); + expect(Object.keys(op).sort()).toEqual(["$inc", "$set"]); + const set = op.$set as Record; + const inc = op.$inc as Record; + expect(set.name).toBe("a"); + expect(set.age).toBeInstanceOf(Int32); + expect(inc.count).toBeInstanceOf(Long); + }); + + test("serializeMongoPipelineStage resolves EJSON but does not inject $literal", () => { + const stage = serializeMongoPipelineStage({ + $group: { _id: "$dept", avg: { $avg: "$salary" } }, + }); + expect(stage).toEqual({ $group: { _id: "$dept", avg: { $avg: "$salary" } } }); + }); +}); + +describe("M2 request validation (rejects before touching the driver)", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("findDocuments rejects a page beyond the maximum", async () => { + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", page: 99 }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("findDocuments rejects an oversized filter", async () => { + const big: EJSONDocument = { big: "x".repeat(9000) }; + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", filter: big }), + ).rejects.toMatchObject({ code: "MONGO_LIMIT_EXCEEDED" }); + }); + + test("findDocuments rejects an invalid sort direction", async () => { + const req: MongoFindRequest = { + collection: "c", + sort: { a: "SIDEWAYS" } as unknown as MongoSortSpec, + }; + await expect(mongoConnector.findDocuments(dummy, req)).rejects.toMatchObject({ + code: "MONGO_INVALID_FILTER", + }); + }); + + test("findDocuments rejects a projection mixing inclusion and exclusion", async () => { + await expect( + mongoConnector.findDocuments(dummy, { collection: "c", projection: { a: 1, b: 0 } }), + ).rejects.toMatchObject({ code: "MONGO_INVALID_FILTER" }); + }); + + test("updateDocument rejects an empty filter (bulk-op guard)", async () => { + await expect( + mongoConnector.updateDocument(dummy, { + collection: "c", + filter: {}, + update: { $set: { a: 1 } }, + }), + ).rejects.toMatchObject({ code: "MONGO_BULK_OP_FORBIDDEN" }); + }); + + test("updateDocument rejects a non-operator (replacement) document", async () => { + await expect( + mongoConnector.updateDocument(dummy, { + collection: "c", + filter: { x: 1 }, + update: { name: "a" }, + }), + ).rejects.toMatchObject({ code: "MONGO_INVALID_UPDATE_OPERAND" }); + }); + + test("deleteDocument rejects an empty filter (bulk-op guard)", async () => { + await expect( + mongoConnector.deleteDocument(dummy, { collection: "c", filter: {} }), + ).rejects.toMatchObject({ code: "MONGO_BULK_OP_FORBIDDEN" }); + }); + + test("runAggregation rejects an empty pipeline", async () => { + await expect( + mongoConnector.runAggregation(dummy, { collection: "c", pipeline: [] }), + ).rejects.toMatchObject({ code: "MONGO_INVALID_PIPELINE" }); + }); + + test("runAggregation rejects a write stage ($out)", async () => { + await expect( + mongoConnector.runAggregation(dummy, { + collection: "c", + pipeline: [{ $out: { coll: "x" } }], + }), + ).rejects.toMatchObject({ code: "MONGO_WRITE_STAGE_FORBIDDEN" }); + }); + + test("runAggregation rejects a pipeline over the stage cap", async () => { + const pipeline = Array.from({ length: 65 }, (): EJSONDocument => ({ $match: {} })); + await expect( + mongoConnector.runAggregation(dummy, { collection: "c", pipeline }), + ).rejects.toMatchObject({ code: "MONGO_LIMIT_EXCEEDED" }); + }); + + test("insertDocument rejects a non-object document", async () => { + await expect( + mongoConnector.insertDocument(dummy, { + collection: "c", + document: [1, 2, 3] as unknown as EJSONDocument, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); +}); + +describe("M2 error mapping", () => { + test("maps common MongoServerError shapes to stable codes", () => { + expect(mapMongoError({ name: "MongoServerError", codeName: "NamespaceNotFound" }).code).toBe( + "MONGO_NS_NOT_FOUND", + ); + expect(mapMongoError({ name: "MongoServerError", code: 11000 }).code).toBe( + "MONGO_DUPLICATE_KEY", + ); + expect(mapMongoError({ name: "MongoServerError", codeName: "AuthenticationFailed" }).code).toBe( + "MONGO_AUTH_ERROR", + ); + expect(mapMongoError({ name: "MongoServerError", codeName: "MaxTimeMSExpired" }).code).toBe( + "MONGO_OPERATION_TIMEOUT", + ); + }); + + test("maps a generic error to a connection error and passes through MongoApiError", () => { + expect(mapMongoError(new Error("boom")).code).toBe("MONGO_CONNECTION_ERROR"); + const api = new MongoApiError("MONGO_BULK_OP_FORBIDDEN", "nope"); + expect(mapMongoError(api)).toBe(api); + }); +}); + +describe("M2 client-side timeout wrapper", () => { + test("passes through a fast result", async () => { + await expect(withMongoTimeout(1000, async () => 42)).resolves.toBe(42); + }); + + test("propagates a real error from the operation", async () => { + await expect( + withMongoTimeout(1000, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + }); + + test("rejects with MONGO_OPERATION_TIMEOUT when the operation hangs", async () => { + const start = Date.now(); + await expect(withMongoTimeout(30, () => new Promise(() => {}))).rejects.toMatchObject({ + code: "MONGO_OPERATION_TIMEOUT", + name: "MongoApiError", + }); + expect(Date.now() - start).toBeLessThan(1000); + }); +}); + +describe("M2 collection-scoped cache", () => { + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/cachetest", database: "cachetest" }; + + test("stores and reads collection stats and document pages, then evicts on clearCollectionCache", () => { + mongodbCache.setCollectionStats(cfg, "c", { + collection: "c", + count: 5, + sizeBytes: 100, + avgObjSize: 20, + }); + expect(mongodbCache.getCollectionStats(cfg, "c")).toEqual({ + collection: "c", + count: 5, + sizeBytes: 100, + avgObjSize: 20, + }); + expect(mongodbCache.getCollectionStats(cfg, "other")).toBeNull(); + + mongodbCache.setDocumentPage(cfg, "key1", [{ a: 1 }]); + expect(mongodbCache.getDocumentPage(cfg, "key1")).toEqual([{ a: 1 }]); + + mongodbCache.clearCollectionCache(cfg, "c"); + expect(mongodbCache.getCollectionStats(cfg, "c")).toBeNull(); + expect(mongodbCache.getDocumentPage(cfg, "key1")).toBeNull(); + }); + + test("isolation: a different connection does not see another connection's pages", () => { + const other: MongoConfig = { uri: "mongodb://localhost:27017/other", database: "other" }; + mongodbCache.setDocumentPage(cfg, "shared", [{ a: 1 }]); + expect(mongodbCache.getDocumentPage(other, "shared")).toBeNull(); + expect(mongodbCache.getDocumentPage(cfg, "shared")).toEqual([{ a: 1 }]); + }); +}); + +describe("M2 executor document-op routing (spec §12)", () => { + const executor = new QueryExecutor(); + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects document operations on a non-MongoDB connection with MONGO_UNSUPPORTED_OPERATION", async () => { + await expect( + executor.findDocuments(cfg, DBType.POSTGRES, { collection: "c" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.insertDocument(cfg, DBType.MYSQL, { collection: "c", document: { a: 1 } }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.updateDocument(cfg, DBType.POSTGRES, { collection: "c", filter: {}, update: {} }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.deleteDocument(cfg, DBType.POSTGRES, { collection: "c", filter: {} }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + await expect( + executor.runAggregation(cfg, DBType.POSTGRES, { + collection: "c", + pipeline: [{ $match: {} }], + }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); +}); diff --git a/bridge/__tests__/integration/mongoDocuments.integration.test.ts b/bridge/__tests__/integration/mongoDocuments.integration.test.ts new file mode 100644 index 00000000..1a9017b8 --- /dev/null +++ b/bridge/__tests__/integration/mongoDocuments.integration.test.ts @@ -0,0 +1,349 @@ +import { describe, expect, test, beforeAll, afterAll, jest } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { mongoConnector, collectionExists, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +// --------------------------------------------------------------------------- +// M2 integration tests — require a live MongoDB. Uses a dedicated database so +// it never collides with the M1 `testdb` fixtures. Skips cleanly when the +// server is unreachable (spec §16.2). +// --------------------------------------------------------------------------- + +const MONGO_HOST = process.env.REAL_MONGODB_HOST || "127.0.0.1"; +const MONGO_PORT = process.env.REAL_MONGODB_PORT || "27017"; +const MONGO_DB = process.env.REAL_MONGODB_DATABASE_M2 || "relwave_m2_it"; + +const config: MongoConfig = { + uri: `mongodb://${MONGO_HOST}:${MONGO_PORT}/${MONGO_DB}`, + database: MONGO_DB, +}; + +let seedClient: MongoClient | null = null; +let connected = false; + +beforeAll(async () => { + jest.setTimeout(60000); + try { + seedClient = new MongoClient(config.uri as string); + await seedClient.connect(); + try { + await seedClient.db(MONGO_DB).dropDatabase(); + } catch { + // ignore — may lack dropDatabase privilege; collections are dropped per-test + } + connected = true; + } catch { + connected = false; + console.warn( + `[mongoDocuments.integration] MongoDB not reachable at ${MONGO_HOST}:${MONGO_PORT}; skipping M2 integration tests.`, + ); + } +}, 60000); + +afterAll(async () => { + if (seedClient) { + try { + await seedClient.close(); + } catch { + // ignore + } + seedClient = null; + } + await closeAllMongoClients(); +}); + +describe("M2 findDocuments", () => { + test("round-trips ObjectId / int32 / int64 / decimal / date / nested types", async () => { + if (!connected) return; + const oid = new (await import("mongodb")).ObjectId(); + const now = new Date("2024-01-01T00:00:00.000Z"); + const res = await mongoConnector.insertDocument(config, { + collection: "rt", + document: { + _id: { $oid: oid.toHexString() }, + name: "Alice", + age: { $numberInt: "30" }, + score: { $numberDecimal: "9.50" }, + big: { $numberLong: "9007199254740993" }, + createdAt: { $date: now.toISOString() }, + tags: ["a", "b"], + meta: { nested: true }, + }, + }); + expect(res.insertedId).toEqual({ $oid: oid.toHexString() }); + expect(res.document._id).toEqual({ $oid: oid.toHexString() }); + + const found = await mongoConnector.findDocuments(config, { + collection: "rt", + filter: { _id: { $oid: oid.toHexString() } }, + }); + expect(found.documents).toHaveLength(1); + const d = found.documents[0]; + expect(d.name).toBe("Alice"); + expect(d.age).toEqual({ $numberInt: "30" }); + expect(d.score).toEqual({ $numberDecimal: "9.50" }); + expect(d.big).toEqual({ $numberLong: "9007199254740993" }); + expect(d.createdAt).toEqual({ $date: now.toISOString() }); + expect(d.tags).toEqual(["a", "b"]); + expect(d.meta).toEqual({ nested: true }); + }); + + test("filters with $in, $gt (typed), $regex and _id $oid", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "filt", + document: { tag: "red", n: { $numberInt: "10" }, name: "Alice" }, + }); + await mongoConnector.insertDocument(config, { + collection: "filt", + document: { tag: "blue", n: { $numberInt: "20" }, name: "Bob" }, + }); + await mongoConnector.insertDocument(config, { + collection: "filt", + document: { tag: "red", n: { $numberInt: "30" }, name: "Carol" }, + }); + + const byIn = await mongoConnector.findDocuments(config, { + collection: "filt", + filter: { tag: { $in: ["red", "green"] } }, + }); + expect(byIn.total).toBe(2); + + const byGt = await mongoConnector.findDocuments(config, { + collection: "filt", + filter: { n: { $gt: { $numberInt: "15" } } }, + }); + expect(byGt.total).toBe(2); + + const byRegex = await mongoConnector.findDocuments(config, { + collection: "filt", + filter: { name: { $regex: "^A" } }, + }); + expect(byRegex.documents).toHaveLength(1); + expect(byRegex.documents[0].name).toBe("Alice"); + }); + + test("paginates with total / totalPages / hasMore", async () => { + if (!connected) return; + for (let i = 0; i < 25; i++) { + await mongoConnector.insertDocument(config, { + collection: "page", + document: { idx: { $numberInt: String(i) } }, + }); + } + const p1 = await mongoConnector.findDocuments(config, { + collection: "page", + pageSize: 10, + page: 1, + sort: { idx: 1 }, + }); + expect(p1.total).toBe(25); + expect(p1.documents).toHaveLength(10); + expect(p1.totalPages).toBe(3); + expect(p1.hasMore).toBe(true); + expect(p1.documents[0].idx).toEqual({ $numberInt: "0" }); + + const p3 = await mongoConnector.findDocuments(config, { + collection: "page", + pageSize: 10, + page: 3, + sort: { idx: 1 }, + }); + expect(p3.documents).toHaveLength(5); + expect(p3.hasMore).toBe(false); + }); + + test("applies sort and projection", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "sp", + document: { k: "a", secret: "s1", val: { $numberInt: "3" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "sp", + document: { k: "b", secret: "s2", val: { $numberInt: "1" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "sp", + document: { k: "c", secret: "s3", val: { $numberInt: "2" } }, + }); + + const res = await mongoConnector.findDocuments(config, { + collection: "sp", + sort: { val: 1 }, + projection: { k: 1, _id: 0 }, + }); + expect(res.documents.map((d) => d.k)).toEqual(["b", "c", "a"]); + expect(res.documents[0]).not.toHaveProperty("secret"); + expect(res.documents[0]).not.toHaveProperty("_id"); + }); +}); + +describe("M2 updateDocument", () => { + test("applies $set + $inc and returns the after-document", async () => { + if (!connected) return; + const ins = await mongoConnector.insertDocument(config, { + collection: "upd", + document: { name: "Bob", count: { $numberInt: "1" } }, + }); + const res = await mongoConnector.updateDocument(config, { + collection: "upd", + filter: { _id: ins.insertedId }, + update: { $set: { name: "Bobby" }, $inc: { count: { $numberInt: "5" } } }, + options: { returnDocument: "after" }, + }); + expect(res.matchedCount).toBe(1); + expect(res.modifiedCount).toBe(1); + expect(res.updatedDocument?.name).toBe("Bobby"); + expect(res.updatedDocument?.count).toEqual({ $numberInt: "6" }); + }); + + test("returnDocument=before returns the pre-update document and persists the change", async () => { + if (!connected) return; + const ins = await mongoConnector.insertDocument(config, { + collection: "updb", + document: { v: { $numberInt: "1" } }, + }); + const res = await mongoConnector.updateDocument(config, { + collection: "updb", + filter: { _id: ins.insertedId }, + update: { $set: { v: { $numberInt: "2" } } }, + options: { returnDocument: "before" }, + }); + expect(res.updatedDocument?.v).toEqual({ $numberInt: "1" }); + const found = await mongoConnector.findDocuments(config, { + collection: "updb", + filter: { _id: ins.insertedId }, + }); + expect(found.documents[0].v).toEqual({ $numberInt: "2" }); + }); + + test("upsert creates a document and reports upsertedId", async () => { + if (!connected) return; + const res = await mongoConnector.updateDocument(config, { + collection: "ups", + filter: { code: "XYZ" }, + update: { $set: { val: 1 } }, + options: { upsert: true }, + }); + expect(res.matchedCount).toBe(0); + expect(res.upsertedId).toBeDefined(); + const found = await mongoConnector.findDocuments(config, { + collection: "ups", + filter: { code: "XYZ" }, + }); + expect(found.documents).toHaveLength(1); + }); + + test("multi updates every match", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "multiu", + document: { g: "a", n: { $numberInt: "0" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "multiu", + document: { g: "a", n: { $numberInt: "0" } }, + }); + const res = await mongoConnector.updateDocument(config, { + collection: "multiu", + filter: { g: "a" }, + update: { $set: { n: { $numberInt: "9" } } }, + options: { multi: true }, + }); + expect(res.matchedCount).toBe(2); + expect(res.modifiedCount).toBe(2); + }); +}); + +describe("M2 deleteDocument", () => { + test("removes a single document and reports deletedCount", async () => { + if (!connected) return; + const ins = await mongoConnector.insertDocument(config, { + collection: "dels", + document: { x: 1 }, + }); + const res = await mongoConnector.deleteDocument(config, { + collection: "dels", + filter: { _id: ins.insertedId }, + }); + expect(res.deletedCount).toBe(1); + const found = await mongoConnector.findDocuments(config, { + collection: "dels", + filter: { _id: ins.insertedId }, + }); + expect(found.documents).toHaveLength(0); + }); + + test("multi removes every match", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { collection: "delm", document: { g: "z" } }); + await mongoConnector.insertDocument(config, { collection: "delm", document: { g: "z" } }); + const res = await mongoConnector.deleteDocument(config, { + collection: "delm", + filter: { g: "z" }, + options: { multi: true }, + }); + expect(res.deletedCount).toBe(2); + }); +}); + +describe("M2 runAggregation", () => { + test("groups by a field and computes an aggregate", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "agg", + document: { dept: "eng", salary: { $numberInt: "100" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "agg", + document: { dept: "eng", salary: { $numberInt: "200" } }, + }); + await mongoConnector.insertDocument(config, { + collection: "agg", + document: { dept: "sales", salary: { $numberInt: "150" } }, + }); + + const res = await mongoConnector.runAggregation(config, { + collection: "agg", + pipeline: [{ $group: { _id: "$dept", avg: { $avg: "$salary" } } }, { $sort: { _id: 1 } }], + }); + expect(res.stageCount).toBe(2); + expect(res.documents).toHaveLength(2); + expect(res.documents[0]._id).toBe("eng"); + expect(Number(res.documents[0].avg)).toBe(150); + }); +}); + +describe("M2 cache + existence", () => { + test("document-page cache reflects a mutation after invalidation", async () => { + if (!connected) return; + const cacheConfig: MongoConfig = { ...config, cacheDocuments: true }; + const ins = await mongoConnector.insertDocument(cacheConfig, { + collection: "cache", + document: { k: "v1" }, + }); + const q = { collection: "cache", filter: { _id: ins.insertedId } }; + const first = await mongoConnector.findDocuments(cacheConfig, q); + expect(first.documents[0].k).toBe("v1"); + + // A mutation clears the collection cache, so the next read reflects v2. + await mongoConnector.updateDocument(cacheConfig, { + collection: "cache", + filter: { _id: ins.insertedId }, + update: { $set: { k: "v2" } }, + }); + const second = await mongoConnector.findDocuments(cacheConfig, q); + expect(second.documents[0].k).toBe("v2"); + }); + + test("collectionExists reports presence and absence", async () => { + if (!connected) return; + await mongoConnector.insertDocument(config, { + collection: "exists_check", + document: { a: 1 }, + }); + expect(await collectionExists(config, "exists_check")).toBe(true); + expect(await collectionExists(config, "nonexistent_xyz_123")).toBe(false); + }); +}); diff --git a/bridge/src/connectors/mongo.ts b/bridge/src/connectors/mongo.ts index 59eeee7a..e55180c2 100644 --- a/bridge/src/connectors/mongo.ts +++ b/bridge/src/connectors/mongo.ts @@ -21,11 +21,50 @@ * sanitized and the raw config is never logged. */ -import { MongoClient } from "mongodb"; -import { MongoConfig, MongoCollectionInfo } from "../types/mongo"; +import { MongoClient, Collection, FindOptions, AggregateOptions } from "mongodb"; +import { createHash } from "crypto"; +import { + MongoConfig, + MongoCollectionInfo, + MongoCollectionStats, + EJSONDocument, + EJsonValue, + MongoFindRequest, + MongoFindResult, + MongoInsertRequest, + MongoInsertResult, + MongoUpdateRequest, + MongoUpdateResult, + MongoDeleteRequest, + MongoDeleteResult, + MongoAggregationRequest, + MongoAggregationResult, + MongoSortSpec, + MongoProjection, + MongoPageCacheKey, + MAX_PAGE_SIZE, + MAX_PAGE, + DEFAULT_PAGE_SIZE, + MAX_FILTER_SIZE_BYTES, + MAX_SORT_FIELDS, + MAX_PROJECTION_FIELDS, + MAX_AGG_STAGES, + MAX_AGG_STAGE_SIZE_BYTES, + MAX_AGG_RESULTS, + MAX_DOCUMENT_SIZE_BYTES, + DEFAULT_OP_TIMEOUT_MS, + MAX_OP_TIMEOUT_MS, +} from "../types/mongo"; import { TableInfo, DBStats, SchemaInfo } from "../types/common"; import { CacheEntry } from "../types/cache"; import logger from "../services/logger"; +import { + serializeMongoEJSONDocument, + serializeMongoFilter, + serializeMongoUpdateOperand, + serializeMongoPipelineStage, + mongoBSONtoEJSON, +} from "./mongoBson"; // --------------------------------------------------------------------------- // Error sanitization — decision #8: never expose credentials in errors/logs. @@ -43,15 +82,26 @@ function safeMongoError(err: unknown): string { const CACHE_TTL = 30_000; const STATS_CACHE_TTL = 10_000; +const DOC_PAGE_TTL = 5_000; // M2 opt-in document-page cache (spec §14) class MongoCacheManager { private tableListCache = new Map>(); private dbStatsCache = new Map>(); + // M2 collection-scoped caches (spec §14) + private collectionStatsCache = new Map>(); + private docPageCache = new Map>(); private getConfigKey(cfg: MongoConfig): string { return `${cfg.uri || ""}:${cfg.database || ""}`; } + /** Credential-free connection identifier for M2 query-result cache keys (spec §15). */ + private getConnHash(cfg: MongoConfig): string { + return createHash("sha256") + .update(cfg.uri || "") + .digest("hex"); + } + private isValid(entry: CacheEntry | undefined): boolean { return !!entry && Date.now() - entry.timestamp < entry.ttl; } @@ -80,17 +130,61 @@ class MongoCacheManager { }); } + // ---- M2: collection-scoped caches (spec §14 / D-M2-5) ---- + + getCollectionStats(cfg: MongoConfig, collection: string): MongoCollectionStats | null { + const entry = this.collectionStatsCache.get(`${this.getConfigKey(cfg)}:${collection}`); + return this.isValid(entry) ? entry!.data : null; + } + setCollectionStats(cfg: MongoConfig, collection: string, data: MongoCollectionStats): void { + this.collectionStatsCache.set(`${this.getConfigKey(cfg)}:${collection}`, { + data, + timestamp: Date.now(), + ttl: STATS_CACHE_TTL, + }); + } + + getDocumentPage(cfg: MongoConfig, key: MongoPageCacheKey): EJSONDocument[] | null { + const entry = this.docPageCache.get(`${this.getConnHash(cfg)}:${key}`); + return this.isValid(entry) ? entry!.data : null; + } + setDocumentPage(cfg: MongoConfig, key: MongoPageCacheKey, docs: EJSONDocument[]): void { + this.docPageCache.set(`${this.getConnHash(cfg)}:${key}`, { + data: docs, + timestamp: Date.now(), + ttl: DOC_PAGE_TTL, + }); + } + + /** Evict a collection's stats + all its cached document pages (called on mutation). */ + clearCollectionCache(cfg: MongoConfig, collection: string): void { + this.collectionStatsCache.delete(`${this.getConfigKey(cfg)}:${collection}`); + const connHash = this.getConnHash(cfg); + for (const k of Array.from(this.docPageCache.keys())) { + if (k.startsWith(`${connHash}:`)) this.docPageCache.delete(k); + } + } + clearForConnection(cfg: MongoConfig): void { const key = this.getConfigKey(cfg); for (const [k] of this.tableListCache) { if (k.startsWith(key)) this.tableListCache.delete(k); } this.dbStatsCache.delete(key); + for (const k of Array.from(this.collectionStatsCache.keys())) { + if (k.startsWith(key)) this.collectionStatsCache.delete(k); + } + const connHash = this.getConnHash(cfg); + for (const k of Array.from(this.docPageCache.keys())) { + if (k.startsWith(`${connHash}:`)) this.docPageCache.delete(k); + } } clearAll(): void { this.tableListCache.clear(); this.dbStatsCache.clear(); + this.collectionStatsCache.clear(); + this.docPageCache.clear(); } } @@ -274,3 +368,536 @@ export async function connectToDatabase( migrations: { local: [], applied: [] }, }; } + +// =========================================================================== +// M2 — MongoDB Document Engine (findDocuments / insertDocument / updateDocument +// / deleteDocument / runAggregation). See docs/MONGODB_M2_SPEC.md. +// +// Document-native operations, distinct from the M1 row/table compatibility +// adapters above. They run through a small MongoConnector class so the +// executor/handler layer has one typed surface to call (spec §13). Errors map to +// the stable MongoApiError codes (spec §11) and never carry credentials (§15). +// =========================================================================== + +/** Stable JSON encoding with sorted keys, for size measurement + cache keys. */ +function canonicalJSON(value: unknown): string { + const sortKeys = (v: unknown): unknown => { + if (v && typeof v === "object" && !Array.isArray(v)) { + const obj = v as Record; + const sorted: Record = {}; + for (const k of Object.keys(obj).sort()) sorted[k] = sortKeys(obj[k]); + return sorted; + } + if (Array.isArray(v)) return v.map(sortKeys); + return v; + }; + return JSON.stringify(sortKeys(value)); +} + +function jsonByteSize(value: unknown): number { + return Buffer.byteLength(canonicalJSON(value), "utf8"); +} + +/** Stable, credential-free error carrying a M2 error code (spec §11). */ +export class MongoApiError extends Error { + constructor( + public readonly code: string, + message: string, + ) { + super(message); + this.name = "MongoApiError"; + } +} + +function assertCollectionName(collection: unknown): asserts collection is string { + if (typeof collection !== "string" || collection.length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection must be a non-empty string"); + } + if (collection.includes(".")) { + throw new MongoApiError("MONGO_INVALID_FILTER", "collection name cannot contain '.'"); + } +} + +function assertSize(value: unknown, max: number, what: string): void { + if (jsonByteSize(value) > max) { + throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `${what} exceeds the ${max}-byte limit`); + } +} + +function validateSort(sort: MongoSortSpec | undefined): Record | undefined { + if (sort === undefined || sort === null) return undefined; + if (typeof sort !== "object" || Array.isArray(sort)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "sort must be an object"); + } + const entries = Object.entries(sort); + if (entries.length > MAX_SORT_FIELDS) { + throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `sort exceeds ${MAX_SORT_FIELDS} fields`); + } + const out: Record = {}; + for (const [field, dir] of entries) { + if (field.includes("$") || field.startsWith(".")) { + throw new MongoApiError("MONGO_INVALID_FILTER", `invalid sort field '${field}'`); + } + if (dir === 1 || dir === "ASC") out[field] = 1; + else if (dir === -1 || dir === "DESC") out[field] = -1; + else throw new MongoApiError("MONGO_INVALID_FILTER", `invalid sort direction for '${field}'`); + } + return out; +} + +function validateProjection( + projection: MongoProjection | undefined, +): Record | undefined { + if (projection === undefined || projection === null) return undefined; + if (typeof projection !== "object" || Array.isArray(projection)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "projection must be an object"); + } + const entries = Object.entries(projection); + if (entries.length > MAX_PROJECTION_FIELDS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `projection exceeds ${MAX_PROJECTION_FIELDS} fields`, + ); + } + const includes: string[] = []; + const excludes: string[] = []; + const out: Record = {}; + for (const [field, mode] of entries) { + if (mode === 1 || mode === true) { + out[field] = 1; + includes.push(field); + } else if (mode === 0 || mode === false) { + out[field] = 0; + excludes.push(field); + } else { + throw new MongoApiError("MONGO_INVALID_FILTER", `invalid projection mode for '${field}'`); + } + } + if (includes.length > 0 && excludes.filter((f) => f !== "_id").length > 0) { + throw new MongoApiError( + "MONGO_INVALID_FILTER", + "cannot mix inclusion and exclusion (except _id)", + ); + } + return out; +} + +function assertNonEmptyFilter(filter: EJSONDocument): void { + if (typeof filter !== "object" || filter === null || Array.isArray(filter)) { + throw new MongoApiError("MONGO_INVALID_FILTER", "filter must be an object"); + } + if (Object.keys(filter).length === 0) { + throw new MongoApiError( + "MONGO_BULK_OP_FORBIDDEN", + "update/delete requires a non-empty filter to prevent accidental whole-collection operations", + ); + } +} + +function assertOperatorDocument(update: EJSONDocument): void { + if (typeof update !== "object" || update === null || Array.isArray(update)) { + throw new MongoApiError("MONGO_INVALID_UPDATE_OPERAND", "update must be an operator document"); + } + const keys = Object.keys(update); + if (keys.length === 0) { + throw new MongoApiError("MONGO_INVALID_UPDATE_OPERAND", "No update operators provided"); + } + for (const key of keys) { + if (!key.startsWith("$")) { + throw new MongoApiError( + "MONGO_INVALID_UPDATE_OPERAND", + `'${key}' is not an update operator (full-document replacement is not supported)`, + ); + } + } +} + +function validatePipeline(pipeline: EJSONDocument[]): EJSONDocument[] { + if (!Array.isArray(pipeline) || pipeline.length === 0) { + throw new MongoApiError("MONGO_INVALID_PIPELINE", "pipeline must be a non-empty array"); + } + if (pipeline.length > MAX_AGG_STAGES) { + throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `pipeline exceeds ${MAX_AGG_STAGES} stages`); + } + for (const stage of pipeline) { + if (typeof stage !== "object" || stage === null || Array.isArray(stage)) { + throw new MongoApiError("MONGO_INVALID_PIPELINE", "each pipeline stage must be an object"); + } + const keys = Object.keys(stage); + if (keys.length !== 1) { + throw new MongoApiError( + "MONGO_INVALID_PIPELINE", + "each pipeline stage must have exactly one key", + ); + } + const name = keys[0]; + if (!name.startsWith("$")) { + throw new MongoApiError("MONGO_INVALID_PIPELINE", `invalid stage name '${name}'`); + } + if (name === "$out" || name === "$merge") { + throw new MongoApiError( + "MONGO_WRITE_STAGE_FORBIDDEN", + `write stage '${name}' is not allowed (read-only engine)`, + ); + } + assertSize(stage, MAX_AGG_STAGE_SIZE_BYTES, "pipeline stage"); + } + return pipeline; +} + +function clampPageSize(pageSize: number | undefined): number { + const n = + typeof pageSize === "number" && Number.isFinite(pageSize) + ? Math.floor(pageSize) + : DEFAULT_PAGE_SIZE; + if (n < 1) return 1; + if (n > MAX_PAGE_SIZE) return MAX_PAGE_SIZE; + return n; +} + +function clampPage(page: number | undefined): number { + const n = typeof page === "number" && Number.isFinite(page) ? Math.floor(page) : 1; + return n < 1 ? 1 : n; +} + +function clampTimeout(ms?: number): number { + const base = typeof ms === "number" && ms > 0 ? ms : DEFAULT_OP_TIMEOUT_MS; + return Math.min(base, MAX_OP_TIMEOUT_MS); +} + +function isEmptyObject(obj: unknown): boolean { + return ( + typeof obj === "object" && obj !== null && !Array.isArray(obj) && Object.keys(obj).length === 0 + ); +} + +/** + * Run a driver operation under a client-side timeout. On expiry the in-flight + * driver promise is aborted and a MONGO_OPERATION_TIMEOUT error is thrown + * (spec §12). Mirrors the AbortController pattern of the SQL streamQueryCancelable. + */ +export function withMongoTimeout( + ms: number, + fn: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(); + reject( + new MongoApiError("MONGO_OPERATION_TIMEOUT", `MongoDB operation timed out after ${ms}ms`), + ); + }, ms); + }); + const run = (async () => { + try { + return await fn(controller.signal); + } finally { + if (timer) clearTimeout(timer); + } + })(); + run.catch(() => {}); // avoid an unhandled rejection if the timeout wins the race + return Promise.race([run, timeout]); +} + +/** Defensive: strip any credential fragments from an error message (spec §11/§15). */ +function stripCredentialTokens(message: string): string { + return message + .replace(/password[=:]\S+/gi, "password=") + .replace(/\bpasswd\b[=:]\S+/gi, "passwd="); +} + +/** Map a driver error to a stable, credential-free M2 error (spec §11). */ +export function mapMongoError(err: unknown): MongoApiError { + if (err instanceof MongoApiError) return err; + const anyErr = err as { codeName?: string; code?: number; name?: string; message?: string }; + const codeName = anyErr?.codeName; + const code = anyErr?.code; + let mapped: { code: string; message: string }; + if (anyErr?.name === "MongoServerError") { + if (codeName === "NamespaceNotFound" || code === 26) { + mapped = { code: "MONGO_NS_NOT_FOUND", message: "Namespace not found" }; + } else if (code === 11000 || code === 11001) { + mapped = { code: "MONGO_DUPLICATE_KEY", message: "Duplicate key" }; + } else if (codeName === "AuthenticationFailed" || code === 18) { + mapped = { code: "MONGO_AUTH_ERROR", message: "Authentication failed" }; + } else if (codeName === "MaxTimeMSExpired" || code === 50) { + mapped = { code: "MONGO_OPERATION_TIMEOUT", message: "Operation timed out" }; + } else { + mapped = { code: "MONGO_UNKNOWN_ERROR", message: safeMongoError(anyErr?.message) }; + } + } else { + mapped = { code: "MONGO_CONNECTION_ERROR", message: safeMongoError(err) }; + } + return new MongoApiError(mapped.code, stripCredentialTokens(mapped.message)); +} + +function selectCollection(config: MongoConfig, collection: string): Collection { + return selectDb(config).collection(collection); +} + +/** Whether a collection (or view) exists in the configured database. */ +export async function collectionExists(config: MongoConfig, collection: string): Promise { + const db = selectDb(config); + const infos = await db.listCollections({ name: collection }, { nameOnly: true }).toArray(); + return infos.length > 0; +} + +/** Credential-free cache key for a document page (spec §15). */ +function pageCacheKey( + collection: string, + filter: EJSONDocument, + sort: Record | undefined, + projection: Record | undefined, + page: number, + pageSize: number, +): string { + const canonical = JSON.stringify([ + collection, + filter, + sort ?? null, + projection ?? null, + page, + pageSize, + ]); + return createHash("sha256").update(canonical).digest("hex"); +} + +function buildFindResult( + collection: string, + documents: EJSONDocument[], + total: number, + page: number, + pageSize: number, +): MongoFindResult { + const totalPages = total === 0 ? 0 : Math.ceil(total / pageSize); + return { + collection, + documents, + total, + page, + pageSize, + totalPages, + hasMore: page * pageSize < total, + }; +} + +/** + * BSON deserialization options that keep numeric/binary values in their typed + * form (int32 -> Int32, int64 -> Long, double -> Double, binary -> Binary) + * instead of promoting them to plain JS numbers. Without this the driver + * collapses every numeric BSON value to a JS number, so the EJSON converter + * could not emit $numberInt / $numberLong / $numberDouble and the documented + * round-trip (§7.1) would silently lose type information. + */ +const PRESERVE_BSON_TYPES = { + promoteValues: false, + promoteLongs: false, + promoteBuffers: false, +} as const; + +export class MongoConnector { + async findDocuments(config: MongoConfig, req: MongoFindRequest): Promise { + assertCollectionName(req.collection); + const pageSize = clampPageSize(req.pageSize); + const page = clampPage(req.page); + if (page > MAX_PAGE) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `page ${page} exceeds the maximum of ${MAX_PAGE} (use sort by _id + range for deeper pagination)`, + ); + } + const filter: EJSONDocument = req.filter ?? {}; + assertSize(filter, MAX_FILTER_SIZE_BYTES, "filter"); + const sort = validateSort(req.sort); + const projection = validateProjection(req.projection); + const bsonFilter = serializeMongoFilter(filter); + const skip = Math.max(0, (page - 1) * pageSize); + + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const nativeDocs = await withMongoTimeout(timeout, async (signal) => { + const options: FindOptions & { signal?: AbortSignal } = { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }; + if (sort) options.sort = sort; + if (projection) options.projection = projection; + const cursor = coll.find(bsonFilter, options).skip(skip).limit(pageSize); + return (await cursor.toArray()) as unknown[]; + }); + + const total = isEmptyObject(filter) + ? await coll.estimatedDocumentCount({ maxTimeMS: timeout }) + : await coll.countDocuments(bsonFilter, { maxTimeMS: timeout }); + + if (config.cacheDocuments) { + const key = pageCacheKey(req.collection, filter, sort, projection, page, pageSize); + const cached = mongodbCache.getDocumentPage(config, key); + if (cached) return buildFindResult(req.collection, cached, total, page, pageSize); + } + + const documents = nativeDocs.map((d) => mongoBSONtoEJSON(d) as EJSONDocument); + + if (config.cacheDocuments) { + const key = pageCacheKey(req.collection, filter, sort, projection, page, pageSize); + mongodbCache.setDocumentPage(config, key, documents); + } + + return buildFindResult(req.collection, documents, total, page, pageSize); + } + + async insertDocument(config: MongoConfig, req: MongoInsertRequest): Promise { + assertCollectionName(req.collection); + if (typeof req.document !== "object" || req.document === null || Array.isArray(req.document)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "document must be an object"); + } + assertSize(req.document, MAX_DOCUMENT_SIZE_BYTES, "document"); + const doc = serializeMongoEJSONDocument(req.document); + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const res = await withMongoTimeout(timeout, () => coll.insertOne(doc, { maxTimeMS: timeout })); + + const insertedId = mongoBSONtoEJSON(res.insertedId); + const echoed = mongoBSONtoEJSON({ ...doc, _id: res.insertedId }) as EJSONDocument; + mongodbCache.clearCollectionCache(config, req.collection); + return { collection: req.collection, insertedId, document: echoed }; + } + + async updateDocument(config: MongoConfig, req: MongoUpdateRequest): Promise { + assertCollectionName(req.collection); + assertNonEmptyFilter(req.filter); + assertOperatorDocument(req.update); + const bsonFilter = serializeMongoFilter(req.filter); + const bsonUpdate = serializeMongoUpdateOperand(req.update); + const multi = req.options?.multi ?? false; + const returnDocument = req.options?.returnDocument ?? "after"; + const upsert = req.options?.upsert ?? false; + const arrayFilters = req.options?.arrayFilters + ? req.options.arrayFilters.map((f) => serializeMongoFilter(f)) + : undefined; + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const result = await withMongoTimeout(timeout, async (signal) => { + let updatedDocument: EJSONDocument | undefined; + if (returnDocument === "before") { + const before = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = before ? (mongoBSONtoEJSON(before) as EJSONDocument) : undefined; + } + const updateOptions = { + maxTimeMS: timeout, + upsert, + arrayFilters, + }; + const r = multi + ? await coll.updateMany(bsonFilter, bsonUpdate, updateOptions) + : await coll.updateOne(bsonFilter, bsonUpdate, updateOptions); + if (returnDocument !== "before") { + // Best-effort per spec §7.2: if the follow-up read fails, omit + // updatedDocument but still return the driver counts. + try { + const after = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = after ? (mongoBSONtoEJSON(after) as EJSONDocument) : undefined; + } catch (readErr) { + logger.warn( + { collection: req.collection, err: readErr }, + "mongo updateDocument follow-up read failed; returning counts only", + ); + } + } + return { r, updatedDocument }; + }); + + const upsertedRaw = result.r.upsertedId; + const upsertedId = + upsertedRaw && Object.keys(upsertedRaw).length > 0 + ? (mongoBSONtoEJSON(Object.values(upsertedRaw)[0]) as EJsonValue) + : undefined; + + mongodbCache.clearCollectionCache(config, req.collection); + return { + collection: req.collection, + matchedCount: result.r.matchedCount, + modifiedCount: result.r.modifiedCount, + upsertedId, + updatedDocument: result.updatedDocument, + }; + } + + async deleteDocument(config: MongoConfig, req: MongoDeleteRequest): Promise { + assertCollectionName(req.collection); + assertNonEmptyFilter(req.filter); + const bsonFilter = serializeMongoFilter(req.filter); + const multi = req.options?.multi ?? false; + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const res = await withMongoTimeout(timeout, () => + multi + ? coll.deleteMany(bsonFilter, { maxTimeMS: timeout }) + : coll.deleteOne(bsonFilter, { maxTimeMS: timeout }), + ); + + logger.debug( + { collection: req.collection, deletedCount: res.deletedCount }, + "mongo deleteDocument", + ); + mongodbCache.clearCollectionCache(config, req.collection); + return { collection: req.collection, deletedCount: res.deletedCount }; + } + + async runAggregation( + config: MongoConfig, + req: MongoAggregationRequest, + ): Promise { + assertCollectionName(req.collection); + const stages = validatePipeline(req.pipeline).map((stage) => + serializeMongoPipelineStage(stage), + ); + const timeout = clampTimeout(req.options?.maxTimeMS); + const allowDiskUse = req.options?.allowDiskUse ?? false; + const coll = selectCollection(config, req.collection); + + const started = Date.now(); + const nativeDocs = await withMongoTimeout(timeout, (signal) => { + const options: AggregateOptions & { signal?: AbortSignal } = { + signal, + maxTimeMS: timeout, + allowDiskUse, + ...PRESERVE_BSON_TYPES, + }; + if (typeof req.options?.batchSize === "number") options.batchSize = req.options.batchSize; + return coll.aggregate(stages, options).toArray(); + }); + + if (nativeDocs.length > MAX_AGG_RESULTS) { + throw new MongoApiError( + "MONGO_LIMIT_EXCEEDED", + `aggregation returned ${nativeDocs.length} documents (max ${MAX_AGG_RESULTS}); add a $limit stage`, + ); + } + + const documents = nativeDocs.map((d) => mongoBSONtoEJSON(d) as EJSONDocument); + return { + collection: req.collection, + documents, + stageCount: stages.length, + executionTimeMs: Date.now() - started, + }; + } +} + +export const mongoConnector = new MongoConnector(); diff --git a/bridge/src/connectors/mongoBson.ts b/bridge/src/connectors/mongoBson.ts index a59a24f9..0605193e 100644 --- a/bridge/src/connectors/mongoBson.ts +++ b/bridge/src/connectors/mongoBson.ts @@ -11,12 +11,15 @@ * without a live driver. No value it returns fails `JSON.stringify`. */ +import { ObjectId, Int32, Long, Decimal128, Binary, Timestamp } from "mongodb"; import type { Document } from "mongodb"; import type { SerializedMongoDocument, MongoWireValue, MongoWireObject, MongoWireBinary, + EJsonValue, + EJSONDocument, } from "../types/mongo"; type WithBsontype = { _bsontype?: string }; @@ -141,3 +144,223 @@ export function serializeMongoDocument(doc: Document): SerializedMongoDocument { } return out; } + +// =========================================================================== +// M2 — Extended JSON (EJSON) <-> driver-native conversion. +// +// serializeMongoValue/serializeMongoDocument above go driver-native -> wire/EJSON +// using the driver's wire shapes for the JSON-RPC response channel. The M2 +// document engine needs the canonical EJSON v2 round-trip in BOTH directions +// (docs/MONGODB_M2_SPEC.md §4.3): mongoEJSONtoBSON parses request inputs; +// mongoBSONtoEJSON renders results so that mongoBSONtoEJSON(mongoEJSONtoBSON(x)) +// is structurally equal to x for the supported type set. Pure and unit-testable. +// =========================================================================== + +const EJSON_WRAPPER_KEYS = new Set([ + "$oid", + "$date", + "$numberInt", + "$numberLong", + "$numberDouble", + "$numberDecimal", + "$binary", + "$regex", + "$timestamp", + "$undefined", +]); + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` is an EJSON object with exactly one known wrapper key. */ +function isEJSONWrapper(value: unknown): value is Record { + if (!isPlainRecord(value)) return false; + const keys = Object.keys(value); + return keys.length === 1 && EJSON_WRAPPER_KEYS.has(keys[0]); +} + +/** + * Recursively convert an EJSON value to driver-native BSON types. Resolves the + * EJSON v2 wrappers; plain JSON passes through; a bare 24-hex string stays a + * string (only $oid yields an ObjectId — spec §4.2 / D-M2-6); unknown $-wrappers + * are returned unchanged for the driver to handle. + */ +export function mongoEJSONtoBSON(doc: EJSONDocument): Record; +export function mongoEJSONtoBSON(value: EJsonValue): unknown; +export function mongoEJSONtoBSON(value: EJsonValue): unknown { + if (value === null || value === undefined) return value; + + const kind = typeof value; + if (kind === "string" || kind === "boolean") { + return value; + } + if (kind === "number") { + return value; + } + if (Array.isArray(value)) { + return value.map((item) => mongoEJSONtoBSON(item)); + } + if (kind === "bigint") { + return (value as unknown as bigint).toString(); + } + + if (isEJSONWrapper(value)) { + const key = Object.keys(value)[0]; + const v = value as Record; + switch (key) { + case "$oid": + return new ObjectId(v["$oid"] as string); + case "$date": { + const d = v["$date"]; + if (isPlainRecord(d) && typeof d["$numberLong"] === "string") { + return new Date(Long.fromString(d["$numberLong"]).toNumber()); + } + return new Date(d as string); + } + case "$numberInt": + return new Int32(Number(v["$numberInt"])); + case "$numberLong": + return Long.fromString(v["$numberLong"] as string); + case "$numberDouble": + return Number(v["$numberDouble"] as string); + case "$numberDecimal": + return new Decimal128(v["$numberDecimal"] as string); + case "$binary": { + const b = v["$binary"] as { $type?: string; $base64?: string }; + const subtype = parseInt(b["$type"] || "0", 16) | 0; + return new Binary(Buffer.from(b["$base64"] || "", "base64"), subtype); + } + case "$regex": + return new RegExp(v["$regex"] as string, (v["$options"] as string) || ""); + case "$timestamp": { + const t = v["$timestamp"] as { $t?: string; $i?: string }; + return new Timestamp({ t: Number(t["$t"] || 0), i: Number(t["$i"] || 0) }); + } + case "$undefined": + return undefined; + default: + return value; // unreachable: isEJSONWrapper matched a known key + } + } + + if (kind === "object") { + const out: Record = {}; + for (const [k, item] of Object.entries(value as unknown as Record)) { + out[k] = mongoEJSONtoBSON(item as EJsonValue); + } + return out; + } + return value; +} + +/** + * Convert a driver-native value to canonical EJSON v2 for the JSON-RPC response. + * Int32 -> $numberInt and Long -> $numberLong are preserved (so int types + * round-trip); Double -> plain JSON number (the accepted double normalization). + */ +export function mongoBSONtoEJSON(value: unknown): EJsonValue { + if (value === null || value === undefined) return null; + + const kind = typeof value; + if (kind === "string" || kind === "boolean") { + return value as EJsonValue; + } + if (kind === "number") { + return toFiniteNumber(value as number) as EJsonValue; + } + if (kind === "bigint") { + return (value as bigint).toString() as EJsonValue; + } + if (Array.isArray(value)) { + return value.map((item) => mongoBSONtoEJSON(item)); + } + + if (value instanceof Date) { + return { $date: value.toISOString() }; + } + if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + const buf = Buffer.isBuffer(value) ? value : Buffer.from(value); + return { $binary: { $type: "00", $base64: buf.toString("base64") } }; + } + + const bsontype = (value as WithBsontype)._bsontype; + if (bsontype) { + switch (bsontype) { + case "ObjectId": + return { $oid: (value as ObjectIdLike).toHexString() }; + case "Decimal128": + return { $numberDecimal: (value as Decimal128Like).toString() }; + case "Long": + return { $numberLong: (value as LongLike).toString() }; + case "Timestamp": { + const ts = value as TimestampLike; + return { $timestamp: { $t: String(ts.t), $i: String(ts.i) } }; + } + case "Int32": + return { $numberInt: String((value as IntLike).valueOf()) }; + case "Double": + return toFiniteNumber((value as IntLike).valueOf()) as EJsonValue; + case "Binary": { + const b = value as BinaryLike; + const buf = Buffer.isBuffer(b.buffer) ? b.buffer : Buffer.from(b.buffer); + const subtype = typeof b.sub_type === "number" ? b.sub_type : 0; + return { + $binary: { + $type: subtype.toString(16).padStart(2, "0"), + $base64: buf.toString("base64"), + }, + }; + } + default: + break; + } + } + + if (kind === "object") { + const out: EJSONDocument = {}; + for (const [key, item] of Object.entries(value as Record)) { + if (key === "_bsontype") continue; + out[key] = mongoBSONtoEJSON(item); + } + return out; + } + return null; +} + +/** + * Convert an insert document (EJSON) to driver-native form. Named distinctly + * from serializeMongoDocument (which goes the opposite direction) to avoid the + * name collision flagged during design. + */ +export function serializeMongoEJSONDocument(doc: EJSONDocument): Record { + return mongoEJSONtoBSON(doc); +} + +/** Convert a query filter (EJSON) to driver-native form. */ +export function serializeMongoFilter(filter: EJSONDocument): Record { + return mongoEJSONtoBSON(filter); +} + +/** + * Convert an update operator document (e.g. { $set: {...}, $inc: {...} }) to + * driver-native form. Operator KEYS are preserved verbatim; their leaf VALUES + * are resolved via mongoEJSONtoBSON. Never promotes a plain-field object to an + * operator document (the connector rejects those). + */ +export function serializeMongoUpdateOperand(update: EJSONDocument): Record { + const out: Record = {}; + for (const [op, body] of Object.entries(update)) { + out[op] = mongoEJSONtoBSON(body as EJsonValue); + } + return out; +} + +/** + * Convert one aggregation pipeline stage to driver-native form. Resolves EJSON + * wrappers but does NOT auto-$literal-wrap plain-object values — the caller is + * responsible for MQL expression structure (spec §5.6 / D-M2-4). + */ +export function serializeMongoPipelineStage(stage: EJSONDocument): Record { + return mongoEJSONtoBSON(stage); +} diff --git a/bridge/src/handlers/queryHandlers.ts b/bridge/src/handlers/queryHandlers.ts index 9252370a..dcb3f9d1 100644 --- a/bridge/src/handlers/queryHandlers.ts +++ b/bridge/src/handlers/queryHandlers.ts @@ -3,6 +3,7 @@ import { DatabaseService } from "../services/databaseService"; import { QueryExecutor } from "../services/queryExecutor"; import { SessionManager } from "../sessionManager"; import { getConnector } from "../services/connectorRegistry"; +import { MongoApiError, mapMongoError } from "../connectors/mongo"; import { Logger } from "pino"; export class QueryHandlers { @@ -336,4 +337,119 @@ export class QueryHandlers { this.rpc.sendError(id, { code: "IO_ERROR", message: String(e) }); } } + + // ---- M2: MongoDB document engine handlers (docs/MONGODB_M2_SPEC.md §13) ---- + + /** Map a MongoApiError (or driver error) to the stable M2 error envelope. */ + private sendMongoError(id: number | string, e: unknown): void { + const mapped = + e instanceof MongoApiError ? { code: e.code, message: e.message } : mapMongoError(e); + this.logger.error({ err: e, code: mapped.code }, "MongoDB document operation failed"); + this.rpc.sendError(id, { code: mapped.code, message: mapped.message }); + } + + async handleFetchDocuments(params: any, id: number | string) { + try { + const { dbId, collection, filter, sort, projection, page, pageSize } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.findDocuments(conn, dbType, { + collection, + filter, + sort, + projection, + page, + pageSize, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleInsertDocument(params: any, id: number | string) { + try { + const { dbId, collection, document } = params || {}; + if (!dbId || !collection || !document) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or document", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.insertDocument(conn, dbType, { collection, document }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleUpdateDocument(params: any, id: number | string) { + try { + const { dbId, collection, filter, update, options } = params || {}; + if (!dbId || !collection || !filter || !update) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, filter, or update", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.updateDocument(conn, dbType, { + collection, + filter, + update, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleDeleteDocument(params: any, id: number | string) { + try { + const { dbId, collection, filter, options } = params || {}; + if (!dbId || !collection || !filter) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or filter", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.deleteDocument(conn, dbType, { + collection, + filter, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleRunAggregation(params: any, id: number | string) { + try { + const { dbId, collection, pipeline, options } = params || {}; + if (!dbId || !collection || !pipeline) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, or pipeline", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.runAggregation(conn, dbType, { + collection, + pipeline, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } } diff --git a/bridge/src/jsonRpcHandler.ts b/bridge/src/jsonRpcHandler.ts index dfcdb8fc..27cfb454 100644 --- a/bridge/src/jsonRpcHandler.ts +++ b/bridge/src/jsonRpcHandler.ts @@ -70,6 +70,12 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "query.updateRow", (p, id) => queryHandlers.handleUpdateRow(p, id)); rpcRegister(rpc, "query.deleteRow", (p, id) => queryHandlers.handleDeleteRow(p, id)); rpcRegister(rpc, "query.searchTable", (p, id) => queryHandlers.handleSearchTable(p, id)); + // M2: MongoDB document engine (docs/MONGODB_M2_SPEC.md §13) + rpcRegister(rpc, "query.fetchDocuments", (p, id) => queryHandlers.handleFetchDocuments(p, id)); + rpcRegister(rpc, "query.insertDocument", (p, id) => queryHandlers.handleInsertDocument(p, id)); + rpcRegister(rpc, "query.updateDocument", (p, id) => queryHandlers.handleUpdateDocument(p, id)); + rpcRegister(rpc, "query.deleteDocument", (p, id) => queryHandlers.handleDeleteDocument(p, id)); + rpcRegister(rpc, "query.runAggregation", (p, id) => queryHandlers.handleRunAggregation(p, id)); // ========================================== // DATABASE CRUD HANDLERS diff --git a/bridge/src/services/queryExecutor.ts b/bridge/src/services/queryExecutor.ts index 5a625ef7..83ed93e8 100644 --- a/bridge/src/services/queryExecutor.ts +++ b/bridge/src/services/queryExecutor.ts @@ -1,6 +1,20 @@ import { DBType, Rpc, QueryParams } from "../types"; import { getConnector } from "./connectorRegistry"; import logger from "./logger"; +import { mongoConnector, MongoApiError } from "../connectors/mongo"; +import { + MongoConfig, + MongoFindRequest, + MongoFindResult, + MongoInsertRequest, + MongoInsertResult, + MongoUpdateRequest, + MongoUpdateResult, + MongoDeleteRequest, + MongoDeleteResult, + MongoAggregationRequest, + MongoAggregationResult, +} from "../types/mongo"; // Concurrency limit for parallel processing const PARALLEL_LIMIT = 5; @@ -169,4 +183,76 @@ export class QueryExecutor { return { name: dbName, schemas: finalSchemas.filter(Boolean) }; } + + // ---- M2: MongoDB document engine (docs/MONGODB_M2_SPEC.md §13) ---- + + async findDocuments( + conn: MongoConfig, + dbType: DBType, + req: MongoFindRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.findDocuments(conn, req); + } + + async insertDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoInsertRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.insertDocument(conn, req); + } + + async updateDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoUpdateRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.updateDocument(conn, req); + } + + async deleteDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoDeleteRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.deleteDocument(conn, req); + } + + async runAggregation( + conn: MongoConfig, + dbType: DBType, + req: MongoAggregationRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "runAggregation requires a MongoDB connection", + ); + } + return mongoConnector.runAggregation(conn, req); + } } diff --git a/bridge/src/types/mongo.ts b/bridge/src/types/mongo.ts index a1fb5748..ae32ebbe 100644 --- a/bridge/src/types/mongo.ts +++ b/bridge/src/types/mongo.ts @@ -25,6 +25,11 @@ export type MongoConfig = { replicaSet?: string; tls?: boolean; directConnection?: boolean; + /** + * M2 (opt-in): when true, findDocuments caches paginated pages briefly. + * Off by default to avoid stale reads (spec §14 / D-M2-5). + */ + cacheDocuments?: boolean; }; // --------------------------------------------------------------------------- @@ -94,3 +99,165 @@ export type MongoUriParts = { password?: string; hasCredentials: boolean; }; + +// =========================================================================== +// M2 — MongoDB Document Engine (see docs/MONGODB_M2_SPEC.md) +// Extended-JSON document model, operation request/response types, and the +// safety/limit constants enforced by the connector. Everything below this line +// is new in M2; nothing above it changes. +// =========================================================================== + +/** + * MongoDB Extended JSON v2 type wrappers. A value crossing the JSON-RPC boundary + * uses these to represent BSON types plain JSON cannot carry. A bare 24-char hex + * string is a STRING, not an ObjectId — ObjectId must use $oid (spec §4.2 / D-M2-6). + */ +export type EJSONTypeWrapper = + | { $oid: string } + | { $date: string } + | { $date: { $numberLong: string } } + | { $numberInt: string } + | { $numberLong: string } + | { $numberDouble: string } + | { $numberDecimal: string } + | { $binary: { $type: string; $base64: string } } + | { $regex: string; $options?: string } + | { $timestamp: { $t: string; $i: string } } + | { $undefined: Record }; + +/** A JSON-RPC-safe value: plain JSON plus EJSON type wrappers. */ +export type EJsonValue = + string | number | boolean | null | EJsonValue[] | EJSONDocument | EJSONTypeWrapper; + +/** A JSON-RPC-safe document (string keys → EJSON values). */ +export type EJSONDocument = { [key: string]: EJsonValue }; + +/** Stable, credential-free key identifying a cached document page (spec §14). */ +export type MongoPageCacheKey = string; + +/** Per-collection statistics shape used by the M2 cache (spec §14). */ +export type MongoCollectionStats = { + collection: string; + count: number; + sizeBytes: number; + avgObjSize: number; +}; + +// --- Shared value objects (spec §5.1) --- + +export type MongoSortDirection = 1 | -1 | "ASC" | "DESC"; +export type MongoSortSpec = { [field: string]: MongoSortDirection }; +export type MongoProjection = { [field: string]: 0 | 1 | false | true }; + +// --- findDocuments (spec §5.2) --- + +export interface MongoFindRequest { + collection: string; + filter?: EJSONDocument; + sort?: MongoSortSpec; + projection?: MongoProjection; + page?: number; + pageSize?: number; +} + +export interface MongoFindResult { + collection: string; + documents: EJSONDocument[]; + total: number; + page: number; + pageSize: number; + totalPages: number; + hasMore: boolean; +} + +// --- insertDocument (spec §5.3) --- + +export interface MongoInsertRequest { + collection: string; + document: EJSONDocument; +} + +export interface MongoInsertResult { + collection: string; + insertedId: EJsonValue; + document: EJSONDocument; +} + +// --- updateDocument (spec §5.4) --- + +export interface MongoUpdateOptions { + upsert?: boolean; + multi?: boolean; + arrayFilters?: EJSONDocument[]; + returnDocument?: "before" | "after"; +} + +export interface MongoUpdateRequest { + collection: string; + filter: EJSONDocument; + update: EJSONDocument; + options?: MongoUpdateOptions; +} + +export interface MongoUpdateResult { + collection: string; + matchedCount: number; + modifiedCount: number; + upsertedId?: EJsonValue; + updatedDocument?: EJSONDocument; +} + +// --- deleteDocument (spec §5.5) --- + +export interface MongoDeleteOptions { + multi?: boolean; +} + +export interface MongoDeleteRequest { + collection: string; + filter: EJSONDocument; + options?: MongoDeleteOptions; +} + +export interface MongoDeleteResult { + collection: string; + deletedCount: number; +} + +// --- runAggregation (spec §5.6) --- + +export interface MongoAggregationOptions { + allowDiskUse?: boolean; + maxTimeMS?: number; + batchSize?: number; +} + +export interface MongoAggregationRequest { + collection: string; + pipeline: EJSONDocument[]; + options?: MongoAggregationOptions; +} + +export interface MongoAggregationResult { + collection: string; + documents: EJSONDocument[]; + stageCount: number; + executionTimeMs?: number; +} + +// --- Safety / limit constants (spec §10) --- + +export const MAX_PAGE_SIZE = 1000; +export const MAX_PAGE = 10; +export const MAX_SKIP = MAX_PAGE_SIZE * MAX_PAGE; // 10000 +export const DEFAULT_PAGE_SIZE = 100; +export const MAX_FILTER_SIZE_BYTES = 8 * 1024; +export const MAX_SORT_FIELDS = 32; +export const MAX_PROJECTION_FIELDS = 256; +export const MAX_AGG_STAGES = 64; +export const MAX_AGG_STAGE_SIZE_BYTES = 16 * 1024; +export const MAX_AGG_RESULTS = 10000; +export const MAX_DOCUMENT_SIZE_BYTES = 16 * 1024 * 1024; +export const DEFAULT_OP_TIMEOUT_MS = 30000; +export const MAX_OP_TIMEOUT_MS = 120000; +export const MAX_INSERT_DOCUMENTS = 1; From 42612fbdb135baf0eeb8e1150b4e97eba8618c34 Mon Sep 17 00:00:00 2001 From: Yash Date: Thu, 29 Oct 2026 16:19:23 +0530 Subject: [PATCH 03/34] feat(mongodb): implement M3 read experience --- .../connectors/mongoInferSchema.test.ts | 231 ++++++++++++ .../mongoInferSchema.integration.test.ts | 119 +++++++ bridge/__tests__/mongoBson.test.ts | 2 +- bridge/__tests__/mongoUri.test.ts | 39 +++ bridge/src/connectors/mongo.ts | 328 ++++++++++++++++++ bridge/src/connectors/mongoBson.ts | 62 ++++ bridge/src/handlers/queryHandlers.ts | 38 ++ bridge/src/jsonRpcHandler.ts | 3 + bridge/src/services/queryExecutor.ts | 32 ++ bridge/src/types/mongo.ts | 69 +++- bridge/src/utils/mongoUri.ts | 133 +++++-- .../parseConnectionUrl.mongo.test.ts | 130 +++++++ src/components/layout/VerticalIconBar.tsx | 5 + src/components/shared/DataTable.tsx | 6 +- .../database/hooks/useDatabaseDetails.ts | 11 +- src/features/database/types.ts | 4 +- .../home/components/AddConnectionDialog.tsx | 15 +- .../components/DiscoveredDatabasesCard.tsx | 5 + src/features/home/hooks/useIndexPage.ts | 13 +- src/features/home/types.ts | 1 + .../__tests__/MongoDocumentCell.test.ts | 113 ++++++ .../__tests__/MongoFilterInput.test.ts | 79 +++++ .../components/MongoCollectionsSidebar.tsx | 101 ++++++ .../mongodb/components/MongoDataViewPanel.tsx | 127 +++++++ .../mongodb/components/MongoDocumentCell.tsx | 305 ++++++++++++++++ .../components/MongoDocumentViewer.tsx | 244 +++++++++++++ .../mongodb/components/MongoFilterInput.tsx | 129 +++++++ .../mongodb/components/MongoSchemaPanel.tsx | 223 ++++++++++++ .../mongodb/hooks/useMongoDatabaseDetails.ts | 117 +++++++ src/features/mongodb/types.ts | 79 +++++ src/features/project/hooks/useDbQueries.ts | 106 +++++- src/lib/databases.tsx | 70 +++- src/lib/parseConnectionUrl.ts | 113 +++++- src/pages/DatabaseDetails.tsx | 14 +- src/services/bridge/database.ts | 44 ++- src/services/bridge/query.ts | 17 + 36 files changed, 3071 insertions(+), 56 deletions(-) create mode 100644 bridge/__tests__/connectors/mongoInferSchema.test.ts create mode 100644 bridge/__tests__/integration/mongoInferSchema.integration.test.ts create mode 100644 src/__tests__/parseConnectionUrl.mongo.test.ts create mode 100644 src/features/mongodb/__tests__/MongoDocumentCell.test.ts create mode 100644 src/features/mongodb/__tests__/MongoFilterInput.test.ts create mode 100644 src/features/mongodb/components/MongoCollectionsSidebar.tsx create mode 100644 src/features/mongodb/components/MongoDataViewPanel.tsx create mode 100644 src/features/mongodb/components/MongoDocumentCell.tsx create mode 100644 src/features/mongodb/components/MongoDocumentViewer.tsx create mode 100644 src/features/mongodb/components/MongoFilterInput.tsx create mode 100644 src/features/mongodb/components/MongoSchemaPanel.tsx create mode 100644 src/features/mongodb/hooks/useMongoDatabaseDetails.ts create mode 100644 src/features/mongodb/types.ts diff --git a/bridge/__tests__/connectors/mongoInferSchema.test.ts b/bridge/__tests__/connectors/mongoInferSchema.test.ts new file mode 100644 index 00000000..839286c4 --- /dev/null +++ b/bridge/__tests__/connectors/mongoInferSchema.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, test } from "@jest/globals"; +import { ObjectId, Int32, Long, Decimal128, Binary, Timestamp } from "mongodb"; +import { bsonTypeOf, bsonNumericKind } from "../../src/connectors/mongoBson"; +import { + FieldAccumulator, + clampInferSampleSize, + clampInferMaxDepth, + mongodbCache, +} from "../../src/connectors/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; +import type { MongoConfig } from "../../src/types/mongo"; + +describe("M3 pure BSON type inference (bsonTypeOf / bsonNumericKind)", () => { + test("bsonTypeOf detects primitives, BSON types, and composites correctly", () => { + expect(bsonTypeOf(null)).toBe("null"); + expect(bsonTypeOf(undefined)).toBe("undefined"); + expect(bsonTypeOf("test")).toBe("string"); + expect(bsonTypeOf(true)).toBe("bool"); + expect(bsonTypeOf(false)).toBe("bool"); + expect(bsonTypeOf(42)).toBe("number"); + expect(bsonTypeOf(3.14)).toBe("number"); + expect(bsonTypeOf(new Int32(100))).toBe("number"); + expect(bsonTypeOf(new Long(9007199254740993n))).toBe("long"); + expect(bsonTypeOf(new Decimal128("12.34"))).toBe("decimal"); + expect(bsonTypeOf(new Timestamp({ t: 1, i: 2 }))).toBe("timestamp"); + expect(bsonTypeOf(new Binary(Buffer.from("abc")))).toBe("binary"); + expect(bsonTypeOf(Buffer.from("abc"))).toBe("binary"); + expect(bsonTypeOf(new Uint8Array([1, 2, 3]))).toBe("binary"); + expect(bsonTypeOf(new Date("2026-01-01T00:00:00.000Z"))).toBe("date"); + expect(bsonTypeOf(new ObjectId("507f1f77bcf86cd799439011"))).toBe("objectId"); + expect(bsonTypeOf([1, 2, 3])).toBe("array"); + expect(bsonTypeOf({ a: 1, b: "hello" })).toBe("object"); + expect(bsonTypeOf({ _bsontype: "UnknownSubtype" })).toBe("object"); + }); + + test("bsonNumericKind identifies int and double accurately", () => { + expect(bsonNumericKind(new Int32(5))).toBe("int"); + expect(bsonNumericKind(42)).toBe("int"); + expect(bsonNumericKind(3.14)).toBe("double"); + expect(bsonNumericKind({ _bsontype: "Double", valueOf: () => 5.5 })).toBe("double"); + expect(bsonNumericKind("42")).toBeUndefined(); + expect(bsonNumericKind(null)).toBeUndefined(); + expect(bsonNumericKind(undefined)).toBeUndefined(); + expect(bsonNumericKind(new Long(100n))).toBeUndefined(); + }); +}); + +describe("M3 clamping helpers", () => { + test("clampInferSampleSize clamps to [1, 1000] and defaults to 100", () => { + expect(clampInferSampleSize(undefined)).toBe(100); + expect(clampInferSampleSize(NaN)).toBe(100); + expect(clampInferSampleSize(0)).toBe(1); + expect(clampInferSampleSize(-10)).toBe(1); + expect(clampInferSampleSize(50)).toBe(50); + expect(clampInferSampleSize(2000)).toBe(1000); + }); + + test("clampInferMaxDepth clamps to [1, 16] and defaults to 8", () => { + expect(clampInferMaxDepth(undefined)).toBe(8); + expect(clampInferMaxDepth(NaN)).toBe(8); + expect(clampInferMaxDepth(0)).toBe(1); + expect(clampInferMaxDepth(-5)).toBe(1); + expect(clampInferMaxDepth(4)).toBe(4); + expect(clampInferMaxDepth(25)).toBe(16); + }); +}); + +describe("FieldAccumulator (pure schema inference)", () => { + test("infers uniform flat documents correctly", () => { + const acc = new FieldAccumulator(2, 8); + acc.ingest({ _id: new ObjectId(), name: "Alice", age: 30 }); + acc.ingest({ _id: new ObjectId(), name: "Bob", age: 25 }); + const fields = acc.toFields(); + + expect(fields).toHaveLength(3); + const idField = fields.find((f) => f.path === "_id")!; + expect(idField.bsonType).toBe("objectId"); + expect(idField.presencePct).toBe(100); + expect(idField.nullable).toBe(false); + expect(idField.optional).toBe(false); + expect(idField.depth).toBe(0); + expect(idField.typeConflicts).toHaveLength(0); + + const nameField = fields.find((f) => f.path === "name")!; + expect(nameField.bsonType).toBe("string"); + expect(nameField.presencePct).toBe(100); + + const ageField = fields.find((f) => f.path === "age")!; + expect(ageField.bsonType).toBe("number"); + expect(ageField.numericKinds).toEqual(["int"]); + }); + + test("tracks nested paths, depths, parentPath, and maxChildDepth", () => { + const acc = new FieldAccumulator(1, 8); + acc.ingest({ + user: { + profile: { + bio: "Engineer", + }, + }, + }); + const fields = acc.toFields(); + + const user = fields.find((f) => f.path === "user")!; + expect(user.depth).toBe(0); + expect(user.isNested).toBe(true); + expect(user.parentPath).toBeUndefined(); + expect(user.maxChildDepth).toBe(2); + + const profile = fields.find((f) => f.path === "user.profile")!; + expect(profile.depth).toBe(1); + expect(profile.isNested).toBe(true); + expect(profile.parentPath).toBe("user"); + expect(profile.maxChildDepth).toBe(2); + + const bio = fields.find((f) => f.path === "user.profile.bio")!; + expect(bio.depth).toBe(2); + expect(bio.isNested).toBe(false); + expect(bio.parentPath).toBe("user.profile"); + expect(bio.bsonType).toBe("string"); + }); + + test("handles arrays and distinct arrayElementTypes without emitting indexed subpaths", () => { + const acc = new FieldAccumulator(2, 8); + acc.ingest({ tags: ["red", "green"] }); + acc.ingest({ tags: ["blue", 123] }); + const fields = acc.toFields(); + + expect(fields.some((f) => f.path.startsWith("tags."))).toBe(false); + + const tagsField = fields.find((f) => f.path === "tags")!; + expect(tagsField.isArray).toBe(true); + expect(tagsField.bsonType).toBe("array"); + expect(tagsField.arrayElementTypes?.sort()).toEqual(["number", "string"]); + }); + + test("tracks mixed type conflicts with sorted counts and nullable/optional flags", () => { + const acc = new FieldAccumulator(4, 8); + acc.ingest({ score: 100 }); + acc.ingest({ score: "N/A" }); + acc.ingest({ score: "exempt" }); + acc.ingest({ score: null }); + + const fields = acc.toFields(); + const score = fields.find((f) => f.path === "score")!; + + expect(score.nullable).toBe(true); + expect(score.nullCount).toBe(1); + expect(score.count).toBe(4); + expect(score.optional).toBe(false); + expect(score.presencePct).toBe(100); + + expect(score.bsonType).toBe("string"); + expect(score.typeConflicts.length).toBeGreaterThan(1); + expect(score.typeConflicts[0].bsonType).toBe("string"); + expect(score.typeConflicts[0].count).toBe(2); + }); + + test("respects maxDepth and marks objects as truncated", () => { + const acc = new FieldAccumulator(1, 1); + acc.ingest({ + level0: { + level1: { + level2: "deep", + }, + }, + }); + + const fields = acc.toFields(); + expect(fields.find((f) => f.path === "level0.level1.level2")).toBeUndefined(); + const l1 = fields.find((f) => f.path === "level0.level1")!; + expect(l1.depth).toBe(1); + expect(l1.truncated).toBe(true); + }); +}); + +describe("M3 Cache Manager inferred schema integration", () => { + const dummyCfg: MongoConfig = { + uri: "mongodb://127.0.0.1:27017/test_db", + database: "test_db", + }; + + test("stores, retrieves, and invalidates inferred schema", () => { + const schemaPayload: any = { + collection: "users", + sampleSize: 10, + totalDocs: 100, + inferredAt: new Date().toISOString(), + fieldCount: 1, + nestedFieldCount: 0, + arrayFieldCount: 0, + fields: [], + sampleDocuments: [], + }; + + mongodbCache.setInferredSchema(dummyCfg, "users", 100, 8, schemaPayload); + const cached = mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8); + expect(cached).toEqual(schemaPayload); + + // Eviction on clearCollectionCache + mongodbCache.clearCollectionCache(dummyCfg, "users"); + expect(mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8)).toBeNull(); + + // Re-set and clearForConnection + mongodbCache.setInferredSchema(dummyCfg, "users", 100, 8, schemaPayload); + mongodbCache.clearForConnection(dummyCfg); + expect(mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8)).toBeNull(); + + // Re-set and clearAll + mongodbCache.setInferredSchema(dummyCfg, "users", 100, 8, schemaPayload); + mongodbCache.clearAll(); + expect(mongodbCache.getInferredSchema(dummyCfg, "users", 100, 8)).toBeNull(); + }); +}); + +describe("QueryExecutor M3 MongoDB guards", () => { + const executor = new QueryExecutor(); + + test("listCollections rejects non-MongoDB dbType", async () => { + await expect(executor.listCollections({} as any, DBType.POSTGRES)).rejects.toThrow( + "listCollections requires a MongoDB connection", + ); + }); + + test("inferCollectionSchema rejects non-MongoDB dbType", async () => { + await expect( + executor.inferCollectionSchema({} as any, DBType.MYSQL, { collection: "users" }), + ).rejects.toThrow("inferCollectionSchema requires a MongoDB connection"); + }); +}); diff --git a/bridge/__tests__/integration/mongoInferSchema.integration.test.ts b/bridge/__tests__/integration/mongoInferSchema.integration.test.ts new file mode 100644 index 00000000..9b182f8e --- /dev/null +++ b/bridge/__tests__/integration/mongoInferSchema.integration.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, test, beforeAll, afterAll } from "@jest/globals"; +import { MongoClient } from "mongodb"; +import { mongoConnector, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const TEST_URI = process.env.MONGO_TEST_URI; +const itLive = TEST_URI ? test : test.skip; + +describe("M3 MongoDB Schema Inference (Integration)", () => { + let client: MongoClient | null = null; + const dbName = "relwave_m3_infer_test"; + let config: MongoConfig; + + beforeAll(async () => { + if (!TEST_URI) return; + client = new MongoClient(TEST_URI); + await client.connect(); + const db = client.db(dbName); + try { + await db.dropDatabase(); + } catch { + // ignore drop errors + } + + config = { + uri: TEST_URI, + database: dbName, + cacheDocuments: true, + }; + + // Seed flat collection + await db.collection("flat").insertMany([ + { name: "Alpha", active: true, score: 10 }, + { name: "Beta", active: false, score: 20 }, + ]); + + // Seed sparse and mixed collection + await db.collection("sparse_mixed").insertMany([ + { title: "Item 1", meta: { color: "blue" }, tags: ["a", "b"] }, + { title: "Item 2", meta: { color: "red", extra: 123 }, tags: ["c"], nullableField: null }, + { title: 3, tags: [true, 1] }, + ]); + + // Seed empty collection + await db.createCollection("empty_col"); + }, 30000); + + afterAll(async () => { + if (client) { + try { + await client.db(dbName).dropDatabase(); + await client.close(); + } catch { + // ignore + } + } + await closeAllMongoClients(); + }); + + itLive("listCollections returns seeded collections", async () => { + const collections = await mongoConnector.listCollections(config); + const names = collections.map((c) => c.name); + expect(names).toContain("flat"); + expect(names).toContain("sparse_mixed"); + expect(names).toContain("empty_col"); + }); + + itLive("inferCollectionSchema infers flat uniform collection accurately", async () => { + const schema = await mongoConnector.inferCollectionSchema(config, { + collection: "flat", + sampleSize: 100, + }); + + expect(schema.collection).toBe("flat"); + expect(schema.sampleSize).toBe(2); + expect(schema.totalDocs).toBe(2); + expect(schema.sampleDocuments.length).toBe(2); + + const nameField = schema.fields.find((f) => f.path === "name")!; + expect(nameField.bsonType).toBe("string"); + expect(nameField.presencePct).toBe(100); + expect(nameField.optional).toBe(false); + }); + + itLive("inferCollectionSchema infers sparse, mixed, and nested schemas", async () => { + const schema = await mongoConnector.inferCollectionSchema(config, { + collection: "sparse_mixed", + sampleSize: 100, + }); + + expect(schema.collection).toBe("sparse_mixed"); + expect(schema.sampleSize).toBe(3); + + const titleField = schema.fields.find((f) => f.path === "title")!; + expect(titleField.typeConflicts.length).toBeGreaterThan(0); + + const tagsField = schema.fields.find((f) => f.path === "tags")!; + expect(tagsField.isArray).toBe(true); + expect(tagsField.arrayElementTypes?.length).toBeGreaterThanOrEqual(2); + + const metaExtra = schema.fields.find((f) => f.path === "meta.extra")!; + expect(metaExtra.optional).toBe(true); + expect(metaExtra.depth).toBe(1); + expect(metaExtra.parentPath).toBe("meta"); + }); + + itLive("inferCollectionSchema succeeds on empty collection", async () => { + const schema = await mongoConnector.inferCollectionSchema(config, { + collection: "empty_col", + sampleSize: 50, + }); + + expect(schema.collection).toBe("empty_col"); + expect(schema.sampleSize).toBe(0); + expect(schema.totalDocs).toBe(0); + expect(schema.fields).toHaveLength(0); + expect(schema.sampleDocuments).toHaveLength(0); + }); +}); diff --git a/bridge/__tests__/mongoBson.test.ts b/bridge/__tests__/mongoBson.test.ts index e6231670..4514aebe 100644 --- a/bridge/__tests__/mongoBson.test.ts +++ b/bridge/__tests__/mongoBson.test.ts @@ -77,7 +77,7 @@ describe("serializeMongoValue", () => { const value = serializeMongoValue({ _id: new ObjectId(), when: new Date(), - big: Long.fromNumber(9007199254740993), + big: Long.fromString("9007199254740993"), dec: new Decimal128("3.14"), raw: Buffer.from("bytes"), missing: undefined, diff --git a/bridge/__tests__/mongoUri.test.ts b/bridge/__tests__/mongoUri.test.ts index 0b99e3fa..03a8c9c0 100644 --- a/bridge/__tests__/mongoUri.test.ts +++ b/bridge/__tests__/mongoUri.test.ts @@ -109,4 +109,43 @@ describe("mongoUri", () => { test("buildMongoConnectionUri throws when neither uri nor host is present", () => { expect(() => buildMongoConnectionUri({}, "pw")).toThrow(/requires a url/i); }); + + test("parses multi-host replica set URI without throwing ERR_INVALID_URL", () => { + const uri = + "mongodb://user:secret@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin"; + expect(() => extractMongoCredentials(uri)).not.toThrow(); + const meta = normalizeMongoUriMeta(uri); + expect(meta.host).toBe("host1"); + expect(meta.port).toBe(27017); + expect(meta.user).toBe("user"); + expect(meta.database).toBe("testdb"); + expect(meta.replicaSet).toBe("rs0"); + expect(meta.authSource).toBe("admin"); + }); + + test("multi-host credentials extraction and sanitization", () => { + const uri = + "mongodb://user:secret@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin"; + const creds = extractMongoCredentials(uri); + expect(creds.user).toBe("user"); + expect(creds.password).toBe("secret"); + + const sanitized = sanitizeMongoUri(uri); + expect(sanitized).toBe( + "mongodb://user@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin", + ); + expect(sanitized).not.toContain("secret"); + expect(sanitized).toContain("host1:27017,host2:27017,host3:27017"); + expect(sanitized).toContain("replicaSet=rs0"); + expect(sanitized).toContain("authSource=admin"); + }); + + test("re-injects password into sanitized multi-host URI with buildMongoConnectionUri", () => { + const sanitized = + "mongodb://user@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin"; + const reconstructed = buildMongoConnectionUri({ url: sanitized }, "secretPass"); + expect(reconstructed).toBe( + "mongodb://user:secretPass@host1:27017,host2:27017,host3:27017/testdb?replicaSet=rs0&authSource=admin", + ); + }); }); diff --git a/bridge/src/connectors/mongo.ts b/bridge/src/connectors/mongo.ts index e55180c2..1391eec2 100644 --- a/bridge/src/connectors/mongo.ts +++ b/bridge/src/connectors/mongo.ts @@ -27,6 +27,10 @@ import { MongoConfig, MongoCollectionInfo, MongoCollectionStats, + MongoBsonType, + MongoInferSchemaRequest, + MongoCollectionSchema, + MongoInferredField, EJSONDocument, EJsonValue, MongoFindRequest, @@ -54,16 +58,25 @@ import { MAX_DOCUMENT_SIZE_BYTES, DEFAULT_OP_TIMEOUT_MS, MAX_OP_TIMEOUT_MS, + MAX_SAMPLE_SIZE, + MAX_INFERENCE_DEPTH, + MAX_INFERRED_FIELDS, + DEFAULT_SAMPLE_SIZE, + DEFAULT_MAX_DEPTH, } from "../types/mongo"; import { TableInfo, DBStats, SchemaInfo } from "../types/common"; import { CacheEntry } from "../types/cache"; import logger from "../services/logger"; import { + serializeMongoDocument, + serializeMongoValue, serializeMongoEJSONDocument, serializeMongoFilter, serializeMongoUpdateOperand, serializeMongoPipelineStage, mongoBSONtoEJSON, + bsonTypeOf, + bsonNumericKind, } from "./mongoBson"; // --------------------------------------------------------------------------- @@ -83,6 +96,7 @@ function safeMongoError(err: unknown): string { const CACHE_TTL = 30_000; const STATS_CACHE_TTL = 10_000; const DOC_PAGE_TTL = 5_000; // M2 opt-in document-page cache (spec §14) +const SCHEMA_CACHE_TTL = 300_000; // M3 inferred schema cache (5 min, spec §4.4, §9) class MongoCacheManager { private tableListCache = new Map>(); @@ -90,6 +104,11 @@ class MongoCacheManager { // M2 collection-scoped caches (spec §14) private collectionStatsCache = new Map>(); private docPageCache = new Map>(); + // M3 inferred schema cache + private inferredSchemaCache = new Map< + string, + CacheEntry & { connHash: string; collection: string } + >(); private getConfigKey(cfg: MongoConfig): string { return `${cfg.uri || ""}:${cfg.database || ""}`; @@ -102,6 +121,19 @@ class MongoCacheManager { .digest("hex"); } + private getSchemaCacheKey( + cfg: MongoConfig, + collection: string, + sampleSize: number, + maxDepth: number, + ): string { + return createHash("sha256") + .update( + `${this.getConnHash(cfg)}:${cfg.database || ""}:${collection}:${sampleSize}:${maxDepth}`, + ) + .digest("hex"); + } + private isValid(entry: CacheEntry | undefined): boolean { return !!entry && Date.now() - entry.timestamp < entry.ttl; } @@ -156,6 +188,34 @@ class MongoCacheManager { }); } + getInferredSchema( + cfg: MongoConfig, + collection: string, + sampleSize: number, + maxDepth: number, + ): MongoCollectionSchema | null { + const key = this.getSchemaCacheKey(cfg, collection, sampleSize, maxDepth); + const entry = this.inferredSchemaCache.get(key); + return this.isValid(entry) ? entry!.data : null; + } + + setInferredSchema( + cfg: MongoConfig, + collection: string, + sampleSize: number, + maxDepth: number, + data: MongoCollectionSchema, + ): void { + const key = this.getSchemaCacheKey(cfg, collection, sampleSize, maxDepth); + this.inferredSchemaCache.set(key, { + data, + timestamp: Date.now(), + ttl: SCHEMA_CACHE_TTL, + connHash: this.getConnHash(cfg), + collection, + }); + } + /** Evict a collection's stats + all its cached document pages (called on mutation). */ clearCollectionCache(cfg: MongoConfig, collection: string): void { this.collectionStatsCache.delete(`${this.getConfigKey(cfg)}:${collection}`); @@ -163,6 +223,11 @@ class MongoCacheManager { for (const k of Array.from(this.docPageCache.keys())) { if (k.startsWith(`${connHash}:`)) this.docPageCache.delete(k); } + for (const [k, v] of Array.from(this.inferredSchemaCache.entries())) { + if (v.connHash === connHash && v.collection === collection) { + this.inferredSchemaCache.delete(k); + } + } } clearForConnection(cfg: MongoConfig): void { @@ -178,6 +243,11 @@ class MongoCacheManager { for (const k of Array.from(this.docPageCache.keys())) { if (k.startsWith(`${connHash}:`)) this.docPageCache.delete(k); } + for (const [k, v] of Array.from(this.inferredSchemaCache.entries())) { + if (v.connHash === connHash) { + this.inferredSchemaCache.delete(k); + } + } } clearAll(): void { @@ -185,6 +255,7 @@ class MongoCacheManager { this.dbStatsCache.clear(); this.collectionStatsCache.clear(); this.docPageCache.clear(); + this.inferredSchemaCache.clear(); } } @@ -565,6 +636,196 @@ function clampTimeout(ms?: number): number { return Math.min(base, MAX_OP_TIMEOUT_MS); } +// ---- M3 Clamping Helpers (docs/MONGODB_M3_SPEC.md §4.4, §4.6) ---- + +export function clampInferSampleSize(size?: number): number { + if (typeof size !== "number" || isNaN(size)) return DEFAULT_SAMPLE_SIZE; + return Math.min(Math.max(1, Math.floor(size)), MAX_SAMPLE_SIZE); +} + +export function clampInferMaxDepth(depth?: number): number { + if (typeof depth !== "number" || isNaN(depth)) return DEFAULT_MAX_DEPTH; + return Math.min(Math.max(1, Math.floor(depth)), MAX_INFERENCE_DEPTH); +} + +function isPlainRecord(v: unknown): v is Record { + return ( + typeof v === "object" && + v !== null && + !Array.isArray(v) && + !(v instanceof Date) && + !Buffer.isBuffer(v) && + !(v instanceof Uint8Array) && + !(v as any)._bsontype + ); +} + +interface FieldAccumulatorStats { + path: string; + depth: number; + parentPath?: string; + isArray: boolean; + isNested: boolean; + count: number; + nullCount: number; + typeCounts: Map; + numericKinds: Set<"int" | "double">; + arrayElementTypes: Set; + sampleValue?: EJSONDocument | any; + maxChildDepth?: number; + truncated?: boolean; +} + +/** + * Pure statistical accumulator over driver-native documents for schema inference. + * Never makes network or driver calls (docs/MONGODB_M3_SPEC.md §4.4). + */ +export class FieldAccumulator { + private fieldMap = new Map(); + + constructor( + public readonly sampleSize: number, + public readonly maxDepth: number, + ) {} + + ingest(doc: Document | Record, parentPath = "", depth = 0): void { + if (!doc || typeof doc !== "object" || Array.isArray(doc)) return; + + for (const [key, value] of Object.entries(doc)) { + if (key === "_bsontype") continue; + + const path = parentPath ? `${parentPath}.${key}` : key; + + if (!this.fieldMap.has(path)) { + if (this.fieldMap.size >= MAX_INFERRED_FIELDS) { + if (parentPath && this.fieldMap.has(parentPath)) { + this.fieldMap.get(parentPath)!.truncated = true; + } + continue; + } + this.fieldMap.set(path, { + path, + depth, + parentPath: parentPath || undefined, + isArray: false, + isNested: false, + count: 0, + nullCount: 0, + typeCounts: new Map(), + numericKinds: new Set<"int" | "double">(), + arrayElementTypes: new Set(), + }); + } + + const stats = this.fieldMap.get(path)!; + + if (parentPath) { + let p: string | undefined = parentPath; + while (p) { + const ancestor = this.fieldMap.get(p); + if (ancestor) { + ancestor.maxChildDepth = Math.max(ancestor.maxChildDepth ?? 0, depth); + p = ancestor.parentPath; + } else { + break; + } + } + } + + const bType = bsonTypeOf(value); + stats.typeCounts.set(bType, (stats.typeCounts.get(bType) || 0) + 1); + + if (value !== undefined) { + stats.count++; + } + if (value === null) { + stats.nullCount++; + } + + if (stats.sampleValue === undefined && value !== undefined && value !== null) { + stats.sampleValue = serializeMongoValue(value); + } + + if (bType === "number") { + const numKind = bsonNumericKind(value); + if (numKind) { + stats.numericKinds.add(numKind); + } + } + + if (Array.isArray(value)) { + stats.isArray = true; + for (const item of value) { + stats.arrayElementTypes.add(bsonTypeOf(item)); + } + } else if (isPlainRecord(value)) { + stats.isNested = true; + if (depth < this.maxDepth) { + this.ingest(value, path, depth + 1); + } else { + stats.truncated = true; + } + } + } + } + + toFields(): MongoInferredField[] { + const fields: MongoInferredField[] = []; + + for (const stats of this.fieldMap.values()) { + const sortedTypes = Array.from(stats.typeCounts.entries()).sort((a, b) => { + if (b[1] !== a[1]) return b[1] - a[1]; + if (a[0] === "null" || a[0] === "undefined") return 1; + if (b[0] === "null" || b[0] === "undefined") return -1; + return a[0].localeCompare(b[0]); + }); + + const dominantType = sortedTypes.length > 0 ? sortedTypes[0][0] : "null"; + const types = sortedTypes.map(([t]) => t); + + const typeConflicts = + types.length > 1 ? sortedTypes.map(([bsonType, count]) => ({ bsonType, count })) : []; + + const presencePct = + this.sampleSize > 0 ? Math.round((stats.count / this.sampleSize) * 1000) / 10 : 0; + + const field: MongoInferredField = { + path: stats.path, + bsonType: dominantType, + types, + numericKinds: + stats.numericKinds.size > 0 + ? (Array.from(stats.numericKinds).sort() as Array<"int" | "double">) + : undefined, + depth: stats.depth, + parentPath: stats.parentPath, + isArray: stats.isArray, + isNested: stats.isNested, + count: stats.count, + nullCount: stats.nullCount, + sampleSize: this.sampleSize, + presencePct, + nullable: stats.nullCount > 0, + optional: stats.count < this.sampleSize, + typeConflicts, + arrayElementTypes: stats.isArray ? Array.from(stats.arrayElementTypes).sort() : undefined, + sampleValue: stats.sampleValue, + maxChildDepth: stats.maxChildDepth, + truncated: stats.truncated ? true : undefined, + }; + + fields.push(field); + } + + fields.sort((a, b) => { + if (a.depth !== b.depth) return a.depth - b.depth; + return a.path.localeCompare(b.path); + }); + + return fields; + } +} + function isEmptyObject(obj: unknown): boolean { return ( typeof obj === "object" && obj !== null && !Array.isArray(obj) && Object.keys(obj).length === 0 @@ -898,6 +1159,73 @@ export class MongoConnector { executionTimeMs: Date.now() - started, }; } + + async listCollections(config: MongoConfig): Promise { + return listCollections(config); + } + + async inferCollectionSchema( + config: MongoConfig, + req: MongoInferSchemaRequest, + ): Promise { + assertCollectionName(req.collection); + const sampleSize = clampInferSampleSize(req.sampleSize); + const maxDepth = clampInferMaxDepth(req.maxDepth); + + const cached = mongodbCache.getInferredSchema(config, req.collection, sampleSize, maxDepth); + if (cached) { + return cached; + } + + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const nativeDocs = (await withMongoTimeout(timeout, (signal) => + coll + .find({}, { signal, maxTimeMS: timeout, limit: sampleSize, ...PRESERVE_BSON_TYPES }) + .toArray(), + )) as Array>; + + const actualSampleSize = nativeDocs.length; + const acc = new FieldAccumulator(actualSampleSize, maxDepth); + for (const doc of nativeDocs) { + acc.ingest(doc, "", 0); + } + const fields = acc.toFields(); + + let totalDocs = 0; + try { + totalDocs = await coll.estimatedDocumentCount({ maxTimeMS: timeout }); + } catch { + try { + totalDocs = await coll.countDocuments({}, { maxTimeMS: timeout }); + } catch { + totalDocs = actualSampleSize; + } + } + + const sampleDocuments = nativeDocs + .slice(0, 5) + .map((d) => serializeMongoDocument(d as Document)); + + const schema: MongoCollectionSchema = { + collection: req.collection, + sampleSize: actualSampleSize, + totalDocs, + inferredAt: new Date().toISOString(), + fieldCount: fields.length, + nestedFieldCount: fields.filter((f) => f.depth > 0).length, + arrayFieldCount: fields.filter((f) => f.isArray).length, + fields, + sampleDocuments, + }; + + if (config.cacheDocuments !== false) { + mongodbCache.setInferredSchema(config, req.collection, sampleSize, maxDepth, schema); + } + + return schema; + } } export const mongoConnector = new MongoConnector(); diff --git a/bridge/src/connectors/mongoBson.ts b/bridge/src/connectors/mongoBson.ts index 0605193e..95ed6b74 100644 --- a/bridge/src/connectors/mongoBson.ts +++ b/bridge/src/connectors/mongoBson.ts @@ -20,6 +20,7 @@ import type { MongoWireBinary, EJsonValue, EJSONDocument, + MongoBsonType, } from "../types/mongo"; type WithBsontype = { _bsontype?: string }; @@ -145,6 +146,67 @@ export function serializeMongoDocument(doc: Document): SerializedMongoDocument { return out; } +// =========================================================================== +// M3 — BSON Type Inference helpers (pure, docs/MONGODB_M3_SPEC.md §4.2) +// =========================================================================== + +/** + * Determine the canonical BSON type of a value without requiring a live driver. + * Never throws; unknown BSON types fall back to "object". + */ +export function bsonTypeOf(value: unknown): MongoBsonType { + if (value === null) return "null"; + if (value === undefined) return "undefined"; + + const kind = typeof value; + if (kind === "string") return "string"; + if (kind === "boolean") return "bool"; + if (kind === "number") return "number"; + + if (Array.isArray(value)) return "array"; + + const bsontype = (value as WithBsontype)?._bsontype; + if (bsontype) { + switch (bsontype) { + case "ObjectId": + return "objectId"; + case "Int32": + case "Double": + return "number"; + case "Long": + return "long"; + case "Decimal128": + return "decimal"; + case "Timestamp": + return "timestamp"; + case "Binary": + return "binary"; + default: + break; + } + } + + if (value instanceof Date) return "date"; + if (Buffer.isBuffer(value) || value instanceof Uint8Array) return "binary"; + + if (kind === "object") return "object"; + + return "object"; +} + +/** + * Determine the numeric kind ("int" | "double") for numeric values. + */ +export function bsonNumericKind(value: unknown): "int" | "double" | undefined { + const bsontype = (value as WithBsontype)?._bsontype; + if (bsontype === "Int32") return "int"; + if (bsontype === "Double") return "double"; + if (typeof value === "number") { + return Number.isInteger(value) ? "int" : "double"; + } + return undefined; +} + // =========================================================================== // M2 — Extended JSON (EJSON) <-> driver-native conversion. // diff --git a/bridge/src/handlers/queryHandlers.ts b/bridge/src/handlers/queryHandlers.ts index dcb3f9d1..5f684160 100644 --- a/bridge/src/handlers/queryHandlers.ts +++ b/bridge/src/handlers/queryHandlers.ts @@ -452,4 +452,42 @@ export class QueryHandlers { this.sendMongoError(id, e); } } + + async handleListCollections(params: any, id: number | string) { + try { + const { dbId } = params || {}; + if (!dbId) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.listCollections(conn, dbType, params); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleGetInferredSchema(params: any, id: number | string) { + try { + const { dbId, collection, sampleSize, maxDepth } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.inferCollectionSchema(conn, dbType, { + collection, + sampleSize, + maxDepth, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } } diff --git a/bridge/src/jsonRpcHandler.ts b/bridge/src/jsonRpcHandler.ts index 27cfb454..40009142 100644 --- a/bridge/src/jsonRpcHandler.ts +++ b/bridge/src/jsonRpcHandler.ts @@ -72,6 +72,7 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "query.searchTable", (p, id) => queryHandlers.handleSearchTable(p, id)); // M2: MongoDB document engine (docs/MONGODB_M2_SPEC.md §13) rpcRegister(rpc, "query.fetchDocuments", (p, id) => queryHandlers.handleFetchDocuments(p, id)); + rpcRegister(rpc, "query.findDocuments", (p, id) => queryHandlers.handleFetchDocuments(p, id)); rpcRegister(rpc, "query.insertDocument", (p, id) => queryHandlers.handleInsertDocument(p, id)); rpcRegister(rpc, "query.updateDocument", (p, id) => queryHandlers.handleUpdateDocument(p, id)); rpcRegister(rpc, "query.deleteDocument", (p, id) => queryHandlers.handleDeleteDocument(p, id)); @@ -93,6 +94,8 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "db.listTables", (p, id) => databaseHandlers.handleListTables(p, id)); rpcRegister(rpc, "db.getSchema", (p, id) => databaseHandlers.handleGetSchema(p, id)); rpcRegister(rpc, "db.listSchemas", (p, id) => databaseHandlers.handleListSchemas(p, id)); + rpcRegister(rpc, "db.listCollections", (p, id) => queryHandlers.handleListCollections(p, id)); + rpcRegister(rpc, "db.getInferredSchema", (p, id) => queryHandlers.handleGetInferredSchema(p, id)); // ========================================== // MIGRATION HANDLERS diff --git a/bridge/src/services/queryExecutor.ts b/bridge/src/services/queryExecutor.ts index 83ed93e8..7e7530ea 100644 --- a/bridge/src/services/queryExecutor.ts +++ b/bridge/src/services/queryExecutor.ts @@ -4,6 +4,9 @@ import logger from "./logger"; import { mongoConnector, MongoApiError } from "../connectors/mongo"; import { MongoConfig, + MongoCollectionInfo, + MongoInferSchemaRequest, + MongoCollectionSchema, MongoFindRequest, MongoFindResult, MongoInsertRequest, @@ -255,4 +258,33 @@ export class QueryExecutor { } return mongoConnector.runAggregation(conn, req); } + + async listCollections( + conn: MongoConfig, + dbType: DBType, + _req?: { collectionFilter?: string }, + ): Promise<{ collections: MongoCollectionInfo[] }> { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "listCollections requires a MongoDB connection", + ); + } + const collections = await mongoConnector.listCollections(conn); + return { collections }; + } + + async inferCollectionSchema( + conn: MongoConfig, + dbType: DBType, + req: MongoInferSchemaRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "inferCollectionSchema requires a MongoDB connection", + ); + } + return mongoConnector.inferCollectionSchema(conn, req); + } } diff --git a/bridge/src/types/mongo.ts b/bridge/src/types/mongo.ts index ae32ebbe..73140679 100644 --- a/bridge/src/types/mongo.ts +++ b/bridge/src/types/mongo.ts @@ -74,20 +74,62 @@ export type SerializedMongoDocument = { [key: string]: MongoWireValue }; export type MongoCollectionInfo = { name: string; type: "collection" | "view" }; -export type MongoInferredField = { +export type MongoBsonType = + | "objectId" + | "string" + | "number" + | "long" + | "decimal" + | "timestamp" + | "binary" + | "date" + | "bool" + | "null" + | "array" + | "object" + | "undefined"; + +export interface MongoInferSchemaRequest { + collection: string; + sampleSize?: number; // default 100, clamped [1, 1000] + maxDepth?: number; // default 8, clamped [1, 16] +} + +export interface MongoInferredField { path: string; - bsonType: string; - presentInPct: number; - sampleCount: number; - nested?: MongoInferredField[]; -}; + bsonType: MongoBsonType; + types: MongoBsonType[]; + numericKinds?: Array<"int" | "double">; + depth: number; + parentPath?: string; + isArray: boolean; + isNested: boolean; + count: number; + nullCount: number; + sampleSize: number; + presencePct: number; + nullable: boolean; + optional: boolean; + typeConflicts: Array<{ bsonType: MongoBsonType; count: number }>; + arrayElementTypes?: MongoBsonType[]; + sampleValue?: EJSONDocument | MongoWireValue; + maxChildDepth?: number; + truncated?: boolean; +} -export type MongoInferredSchema = { +export interface MongoCollectionSchema { collection: string; + sampleSize: number; + totalDocs: number; + inferredAt: string; + fieldCount: number; + nestedFieldCount: number; + arrayFieldCount: number; fields: MongoInferredField[]; - sampled: number; - totalDocs?: number; -}; + sampleDocuments: SerializedMongoDocument[]; +} +export type MongoInferredSchema = MongoCollectionSchema; +export type MongoCollectionListItem = MongoCollectionInfo; /** * Result of parsing a MongoDB URI that may embed credentials. @@ -261,3 +303,10 @@ export const MAX_DOCUMENT_SIZE_BYTES = 16 * 1024 * 1024; export const DEFAULT_OP_TIMEOUT_MS = 30000; export const MAX_OP_TIMEOUT_MS = 120000; export const MAX_INSERT_DOCUMENTS = 1; + +// M3 schema inference limits (docs/MONGODB_M3_SPEC.md §4.6) +export const MAX_SAMPLE_SIZE = 1000; +export const MAX_INFERENCE_DEPTH = 16; +export const MAX_INFERRED_FIELDS = 5000; +export const DEFAULT_SAMPLE_SIZE = 100; +export const DEFAULT_MAX_DEPTH = 8; diff --git a/bridge/src/utils/mongoUri.ts b/bridge/src/utils/mongoUri.ts index e7485430..224f2acd 100644 --- a/bridge/src/utils/mongoUri.ts +++ b/bridge/src/utils/mongoUri.ts @@ -8,9 +8,9 @@ * only at runtime from the sanitized URI + the decrypted password, and is never * written back. * - * WHATWG `URL` parses `mongodb://` and `mongodb+srv://` authorities correctly - * (credentials, host, port, path, and query options), so it is used as the - * parser. All returned strings are JSON-safe and free of plaintext passwords + * Uses a zero-dependency parser that safely handles single-host, mongodb+srv://, + * and multi-host comma-separated seedlists (which WHATWG URL rejects with ERR_INVALID_URL). + * All returned strings are JSON-safe and free of plaintext passwords * unless explicitly re-injecting at runtime. */ @@ -25,6 +25,7 @@ export function isMongoUri(raw?: string): boolean { type MongoUriComponents = { scheme: "mongodb" | "mongodb+srv"; + hosts: string; host: string; port?: number; user?: string; @@ -36,32 +37,120 @@ type MongoUriComponents = { /** Parse a MongoDB URI into its components (password decoded). */ function parseMongoUriComponents(raw: string): MongoUriComponents { const trimmed = raw.trim(); - const u = new URL(trimmed); - const scheme: "mongodb" | "mongodb+srv" = trimmed.toLowerCase().startsWith("mongodb+srv://") - ? "mongodb+srv" - : "mongodb"; - const user = u.username ? decodeURIComponent(u.username) : undefined; - const password = u.password ? decodeURIComponent(u.password) : undefined; - const host = u.hostname; - const port = u.port ? Number(u.port) : undefined; - const database = - u.pathname && u.pathname !== "/" - ? decodeURIComponent(u.pathname.replace(/^\//, "")) - : undefined; - const options = u.search ? u.search.slice(1) : ""; - return { scheme, host, port, user, password, database, options }; + const isSrv = trimmed.toLowerCase().startsWith("mongodb+srv://"); + const scheme: "mongodb" | "mongodb+srv" = isSrv ? "mongodb+srv" : "mongodb"; + const prefix = isSrv ? "mongodb+srv://" : "mongodb://"; + const rest = trimmed.toLowerCase().startsWith(prefix) + ? trimmed.slice(prefix.length) + : trimmed.replace(/^mongodb(\+srv)?:\/\//i, ""); + + const queryIndex = rest.indexOf("?"); + const hashIndex = rest.indexOf("#"); + let endOfPathIndex = rest.length; + if (queryIndex !== -1) endOfPathIndex = queryIndex; + if (hashIndex !== -1 && (queryIndex === -1 || hashIndex < queryIndex)) { + endOfPathIndex = hashIndex; + } + + const authorityAndPath = rest.slice(0, endOfPathIndex); + let options = ""; + if (queryIndex !== -1) { + if (hashIndex !== -1 && hashIndex > queryIndex) { + options = rest.slice(queryIndex + 1, hashIndex); + } else { + options = rest.slice(queryIndex + 1); + } + } + + const slashIndex = authorityAndPath.indexOf("/"); + const authority = slashIndex !== -1 ? authorityAndPath.slice(0, slashIndex) : authorityAndPath; + const rawDb = slashIndex !== -1 ? authorityAndPath.slice(slashIndex + 1) : ""; + let database: string | undefined = undefined; + if (rawDb) { + try { + database = decodeURIComponent(rawDb); + } catch { + database = rawDb; + } + } + + let user: string | undefined; + let password: string | undefined; + let hosts = authority; + + const atIndex = authority.lastIndexOf("@"); + if (atIndex !== -1) { + const creds = authority.slice(0, atIndex); + hosts = authority.slice(atIndex + 1); + const colonIndex = creds.indexOf(":"); + if (colonIndex !== -1) { + const rawUser = creds.slice(0, colonIndex); + const rawPass = creds.slice(colonIndex + 1); + try { + user = decodeURIComponent(rawUser); + } catch { + user = rawUser; + } + try { + password = decodeURIComponent(rawPass); + } catch { + password = rawPass; + } + } else { + try { + user = decodeURIComponent(creds); + } catch { + user = creds; + } + } + if (user === "") user = undefined; + if (password === "") password = undefined; + } + + let host = ""; + let port: number | undefined; + + const firstHost = hosts.split(",")[0].trim(); + if (firstHost.startsWith("[")) { + const closeBracket = firstHost.indexOf("]"); + if (closeBracket !== -1) { + host = firstHost.slice(1, closeBracket); + const after = firstHost.slice(closeBracket + 1); + if (after.startsWith(":")) { + const p = Number(after.slice(1)); + if (!isNaN(p)) port = p; + } + } else { + host = firstHost; + } + } else { + const colonIndex = firstHost.lastIndexOf(":"); + if (colonIndex !== -1) { + host = firstHost.slice(0, colonIndex); + const p = Number(firstHost.slice(colonIndex + 1)); + if (!isNaN(p)) port = p; + } else { + host = firstHost; + } + } + + return { scheme, hosts, host, port, user, password, database, options }; } /** Rebuild a MongoDB URI string from components. */ function buildMongoUriString(c: MongoUriComponents, includePassword: boolean): string { let auth = ""; - if (c.user) auth += encodeURIComponent(c.user); - if (includePassword && c.password) auth += ":" + encodeURIComponent(c.password); - if (auth) auth += "@"; - const portPart = c.port ? `:${c.port}` : ""; + if (c.user) { + auth += encodeURIComponent(c.user); + if (includePassword && c.password) { + auth += ":" + encodeURIComponent(c.password); + } + auth += "@"; + } + const hosts = c.hosts || (c.host ? (c.port ? `${c.host}:${c.port}` : c.host) : ""); const path = c.database ? `/${encodeURIComponent(c.database)}` : ""; const qs = c.options ? `?${c.options}` : ""; - return `${c.scheme}://${auth}${c.host}${portPart}${path}${qs}`; + return `${c.scheme}://${auth}${hosts}${path}${qs}`; } type MongoUriOptions = { authSource?: string; replicaSet?: string }; diff --git a/src/__tests__/parseConnectionUrl.mongo.test.ts b/src/__tests__/parseConnectionUrl.mongo.test.ts new file mode 100644 index 00000000..5f539ea5 --- /dev/null +++ b/src/__tests__/parseConnectionUrl.mongo.test.ts @@ -0,0 +1,130 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { parseConnectionUrl, buildConnectionUrl, parseMongoUrl } from "@/lib/parseConnectionUrl"; +import { MONGODB_REQUIRED_FIELDS } from "@/features/home/types"; + +function expect(actual: any) { + return { + toBe(expected: any) { + assert.strictEqual(actual, expected); + }, + toEqual(expected: any) { + assert.deepStrictEqual(actual, expected); + }, + toBeNull() { + assert.strictEqual(actual, null); + }, + toBeUndefined() { + assert.strictEqual(actual, undefined); + }, + toBeDefined() { + assert.notStrictEqual(actual, undefined); + }, + toBeTruthy() { + assert.ok(actual); + }, + toBeFalsy() { + assert.ok(!actual); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && actual.includes(substr), + `Expected "${actual}" to contain "${substr}"`, + ); + }, + not: { + toBeNull() { + assert.notStrictEqual(actual, null); + }, + toContain(substr: string) { + assert.ok( + typeof actual === "string" && !actual.includes(substr), + `Expected "${actual}" not to contain "${substr}"`, + ); + }, + }, + }; +} + +describe("MongoDB Connection URL Parsing & Round-Tripping", () => { + it("parses plain mongodb:// single-host URI and retains originalUri", () => { + const uri = "mongodb://localhost:27017/analytics"; + const parsed = parseConnectionUrl(uri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.host).toBe("localhost"); + expect(parsed!.port).toBe("27017"); + expect(parsed!.database).toBe("analytics"); + expect(parsed!.ssl).toBe(false); + expect(parsed!.originalUri).toBe(uri); + }); + + it("parses mongodb:// with authentication and query params", () => { + const uri = "mongodb://admin:secretPass@127.0.0.1:27017/authDb?authSource=admin&tls=true"; + const parsed = parseConnectionUrl(uri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.user).toBe("admin"); + expect(parsed!.password).toBe("secretPass"); + expect(parsed!.host).toBe("127.0.0.1"); + expect(parsed!.port).toBe("27017"); + expect(parsed!.database).toBe("authDb"); + expect(parsed!.ssl).toBe(true); + expect(parsed!.originalUri).toBe(uri); + }); + + it("CRITICAL SRV INVARIANT: parses mongodb+srv:// and preserves originalUri verbatim", () => { + const srvUri = + "mongodb+srv://user1:pwd%40123@cluster0.abcde.mongodb.net/production?retryWrites=true&w=majority"; + const parsed = parseConnectionUrl(srvUri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.ssl).toBe(true); + expect(parsed!.originalUri).toBe(srvUri); + expect(parsed!.originalUri).toBe(srvUri.trim()); + + // Proves buildConnectionUrl preserves originalUri verbatim + const rebuilt = buildConnectionUrl(parsed!); + expect(rebuilt).toBe(srvUri); + }); + + it("preserves replica-set seed list verbatim via originalUri", () => { + const rsUri = + "mongodb://node1:27017,node2:27017,node3:27017/replicaDb?replicaSet=myRs&readPreference=secondaryPreferred"; + const parsed = parseConnectionUrl(rsUri); + + expect(parsed).not.toBeNull(); + expect(parsed!.type).toBe("mongodb"); + expect(parsed!.originalUri).toBe(rsUri); + + const rebuilt = buildConnectionUrl(parsed!); + expect(rebuilt).toBe(rsUri); + }); + + it("MONGODB_REQUIRED_FIELDS matches ['name', 'type', 'host', 'port']", () => { + expect(MONGODB_REQUIRED_FIELDS).toEqual(["name", "type", "host", "port"]); + }); + + it("parseMongoUrl retains originalUri verbatim for single-host, multi-host, and mongodb+srv://", () => { + const singleHost = "mongodb://localhost:27017/mydb"; + const multiHost = + "mongodb://user:pass@host1:27017,host2:27017,host3:27017/mydb?replicaSet=rs0&authSource=admin"; + const srv = + "mongodb+srv://user:pass@cluster0.abcde.mongodb.net/production?retryWrites=true&w=majority"; + + const parsedSingle = parseMongoUrl(singleHost); + expect(parsedSingle).not.toBeNull(); + expect(parsedSingle!.originalUri).toBe(singleHost); + + const parsedMulti = parseMongoUrl(multiHost); + expect(parsedMulti).not.toBeNull(); + expect(parsedMulti!.originalUri).toBe(multiHost); + + const parsedSrv = parseMongoUrl(srv); + expect(parsedSrv).not.toBeNull(); + expect(parsedSrv!.originalUri).toBe(srv); + }); +}); diff --git a/src/components/layout/VerticalIconBar.tsx b/src/components/layout/VerticalIconBar.tsx index 84ac2aed..a59552d7 100644 --- a/src/components/layout/VerticalIconBar.tsx +++ b/src/components/layout/VerticalIconBar.tsx @@ -16,6 +16,7 @@ import { Link, useLocation } from "react-router-dom"; import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { SettingsDialog } from "@/features/settings/components"; +import { getDatabaseKind } from "@/lib/databases"; export type PanelType = | "data" @@ -53,6 +54,10 @@ export function getDatabasePanelItems(databaseType?: string): Array<{ label: string; panel: PanelType; }> { + if (getDatabaseKind(databaseType) === "nosql") { + return [{ icon: Layers, label: "Data View", panel: "data" }]; + } + return [ { icon: Layers, label: "Data View", panel: "data" }, { icon: Terminal, label: "SQL Workspace", panel: "sql-workspace" }, diff --git a/src/components/shared/DataTable.tsx b/src/components/shared/DataTable.tsx index 9be2e3e6..9c7abbf2 100644 --- a/src/components/shared/DataTable.tsx +++ b/src/components/shared/DataTable.tsx @@ -38,6 +38,7 @@ interface DataTableProps { sortColumn?: string; sortDirection?: "asc" | "desc"; onSort?: (column: string) => void; + renderCell?: (value: any, column: string, row: Record) => React.ReactNode; } export const DataTable = ({ @@ -51,6 +52,7 @@ export const DataTable = ({ sortColumn, sortDirection, onSort, + renderCell, }: DataTableProps) => { const [copiedCellKey, setCopiedCellKey] = useState(null); @@ -199,7 +201,9 @@ export const DataTable = ({ } >
-
{formatCellValue(val, column)}
+
+ {renderCell ? renderCell(val, column, row) : formatCellValue(val, column)} +
{hasValue && ( +
+ +
+
+ + setSearch(e.target.value)} + placeholder="Search collections..." + className="pl-8 text-xs h-8 bg-background/50 border-border/40" + /> +
+
+ + +
+ {isLoading && collections.length === 0 ? ( +
+ Loading collections... +
+ ) : filtered.length === 0 ? ( +
+ No collections found +
+ ) : ( + filtered.map((col) => { + const isSelected = col.name === selectedCollection; + return ( + + ); + }) + )} +
+
+ + ); +} diff --git a/src/features/mongodb/components/MongoDataViewPanel.tsx b/src/features/mongodb/components/MongoDataViewPanel.tsx new file mode 100644 index 00000000..b6256ce5 --- /dev/null +++ b/src/features/mongodb/components/MongoDataViewPanel.tsx @@ -0,0 +1,127 @@ +import React from "react"; +import { useMongoDatabaseDetails } from "../hooks/useMongoDatabaseDetails"; +import { MongoCollectionsSidebar } from "./MongoCollectionsSidebar"; +import { MongoDocumentViewer } from "./MongoDocumentViewer"; +import { MongoSchemaPanel } from "./MongoSchemaPanel"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { FileText, GitBranch, Database } from "lucide-react"; + +export function MongoDataViewPanel({ dbId, databaseName }: { dbId: string; databaseName: string }) { + const { + collections, + collectionsLoading, + selectedCollection, + setSelectedCollection, + documents, + totalDocuments, + totalPages, + page, + setPage, + pageSize, + setPageSize, + sortField, + sortDirection, + handleSort, + filterInput, + setFilterInput, + appliedFilter, + applyFilter, + resetFilter, + activeTab, + setActiveTab, + isLoadingDocuments, + refetchDocuments, + refetchCollections, + } = useMongoDatabaseDetails(dbId); + + return ( +
+ {/* Top Banner / Collection Header */} +
+
+
+
+ +
+ {databaseName} +
+ + {selectedCollection && ( + <> + / + + {selectedCollection} + + + )} +
+ + {selectedCollection && ( + setActiveTab(val as "documents" | "schema")} + > + + + + Documents + + + + Inferred Schema + + + + )} +
+ + {/* Main split: Sidebar + Content */} +
+ + +
+ {!selectedCollection ? ( +
+ Select a collection from the sidebar to view documents. +
+ ) : activeTab === "documents" ? ( + + ) : ( + + )} +
+
+
+ ); +} diff --git a/src/features/mongodb/components/MongoDocumentCell.tsx b/src/features/mongodb/components/MongoDocumentCell.tsx new file mode 100644 index 00000000..c15a5798 --- /dev/null +++ b/src/features/mongodb/components/MongoDocumentCell.tsx @@ -0,0 +1,305 @@ +import React, { useState } from "react"; +import { formatTimestamp } from "@/lib/utils"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Copy, Check, ChevronRight } from "lucide-react"; +import { toast } from "sonner"; + +export interface MongoFormattedValue { + type: string; + display: string; + raw: any; + badgeText?: string; + badgeColor?: string; + isExpandable?: boolean; +} + +export function formatMongoValue(value: any, _column?: string): MongoFormattedValue { + if (value === null || value === undefined) { + return { type: "null", display: "NULL", raw: value, badgeText: "NULL" }; + } + + if (typeof value === "boolean") { + return { + type: "bool", + display: value ? "true" : "false", + raw: value, + badgeText: "BOOL", + }; + } + + if (typeof value === "number") { + return { + type: "number", + display: value.toLocaleString(), + raw: value, + badgeText: "NUM", + }; + } + + if (typeof value === "string") { + return { type: "string", display: value, raw: value }; + } + + if (typeof value === "object") { + if ("$oid" in value) { + return { + type: "objectId", + display: String(value.$oid), + raw: value, + badgeText: "OID", + badgeColor: "bg-teal-500/10 text-teal-600 dark:text-teal-400 border-teal-500/20", + }; + } + + if ("$date" in value) { + let isoStr = ""; + if (typeof value.$date === "string") { + isoStr = value.$date; + } else if (value.$date && typeof value.$date === "object" && "$numberLong" in value.$date) { + const ms = parseInt(value.$date.$numberLong, 10); + isoStr = !isNaN(ms) ? new Date(ms).toISOString() : String(value.$date.$numberLong); + } else { + isoStr = String(value.$date); + } + return { + type: "date", + display: formatTimestamp(isoStr) || isoStr, + raw: value, + badgeText: "DATE", + badgeColor: "bg-purple-500/10 text-purple-600 dark:text-purple-400 border-purple-500/20", + }; + } + + if ("$numberLong" in value) { + return { + type: "long", + display: String(value.$numberLong), + raw: value, + badgeText: "LONG", + badgeColor: "bg-blue-500/10 text-blue-600 dark:text-blue-400 border-blue-500/20", + }; + } + + if ("$numberDecimal" in value) { + return { + type: "decimal", + display: String(value.$numberDecimal), + raw: value, + badgeText: "DEC", + badgeColor: "bg-cyan-500/10 text-cyan-600 dark:text-cyan-400 border-cyan-500/20", + }; + } + + if ("$numberInt" in value) { + return { + type: "number", + display: String(value.$numberInt), + raw: value, + badgeText: "INT", + }; + } + + if ("$numberDouble" in value) { + return { + type: "number", + display: String(value.$numberDouble), + raw: value, + badgeText: "DBL", + }; + } + + if ("$binary" in value) { + const b64 = value.$binary.base64 || value.$binary.$base64 || ""; + return { + type: "binary", + display: ``, + raw: value, + badgeText: "BIN", + badgeColor: "bg-orange-500/10 text-orange-600 dark:text-orange-400 border-orange-500/20", + }; + } + + if ("$timestamp" in value) { + const t = value.$timestamp.t ?? value.$timestamp.$t ?? 0; + const i = value.$timestamp.i ?? value.$timestamp.$i ?? 0; + return { + type: "timestamp", + display: `Timestamp(${t}, ${i})`, + raw: value, + badgeText: "TS", + }; + } + + if ("$regex" in value) { + return { + type: "regex", + display: `/${value.$regex}/${value.$options || ""}`, + raw: value, + badgeText: "REGEX", + }; + } + + if ("$undefined" in value) { + return { type: "undefined", display: "undefined", raw: value, badgeText: "UNDEF" }; + } + + if (Array.isArray(value)) { + return { + type: "array", + display: `Array(${value.length})`, + raw: value, + badgeText: `[${value.length}]`, + badgeColor: + "bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/20", + isExpandable: true, + }; + } + + const keys = Object.keys(value); + return { + type: "object", + display: `{${keys.length} fields}`, + raw: value, + badgeText: `{${keys.length}}`, + badgeColor: "bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/20", + isExpandable: true, + }; + } + + return { type: "unknown", display: String(value), raw: value }; +} + +export function MongoDocumentCell({ + value, + column, + row: _row, +}: { + value: any; + column: string; + row: Record; +}) { + const [dialogOpen, setDialogOpen] = useState(false); + const [copied, setCopied] = useState(false); + + const formatted = formatMongoValue(value, column); + + const handleCopyJson = (e: React.MouseEvent) => { + e.stopPropagation(); + const str = JSON.stringify(value, null, 2); + navigator.clipboard.writeText(str); + setCopied(true); + toast.success("JSON copied to clipboard", { duration: 1500 }); + setTimeout(() => setCopied(false), 1500); + }; + + if (formatted.type === "null") { + return ( + + NULL + + ); + } + + if (formatted.type === "bool") { + return ( + + {formatted.display} + + ); + } + + if (formatted.isExpandable) { + const jsonStr = JSON.stringify(value); + const truncatedPreview = jsonStr.length > 50 ? `${jsonStr.slice(0, 50)}…` : jsonStr; + + return ( + <> +
+ + + {truncatedPreview} + +
+ + + + +
+
+ + {column} + + {formatted.type} + + + + Document field inspection + +
+ +
+
+
+
+                {JSON.stringify(value, null, 2)}
+              
+
+
+
+ + ); + } + + return ( +
+ {formatted.badgeText && ( + + {formatted.badgeText} + + )} + {formatted.display} +
+ ); +} diff --git a/src/features/mongodb/components/MongoDocumentViewer.tsx b/src/features/mongodb/components/MongoDocumentViewer.tsx new file mode 100644 index 00000000..a50785fe --- /dev/null +++ b/src/features/mongodb/components/MongoDocumentViewer.tsx @@ -0,0 +1,244 @@ +import React, { useMemo, useState } from "react"; +import { DataTable } from "@/components/shared/DataTable"; +import { MongoDocumentCell } from "./MongoDocumentCell"; +import { MongoFilterInput } from "./MongoFilterInput"; +import { EJSONDocument } from "../types"; +import { Button } from "@/components/ui/button"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Copy, + Check, + ChevronLeft, + ChevronRight, + ChevronsLeft, + ChevronsRight, + RefreshCw, + Database, +} from "lucide-react"; +import { toast } from "sonner"; + +export function MongoDocumentViewer({ + collection, + documents, + totalDocuments, + page, + pageSize, + totalPages, + isLoading, + sortField, + sortDirection, + onSort, + onPageChange, + onPageSizeChange, + filterInput, + onFilterInputChange, + onApplyFilter, + onResetFilter, + isFilterApplied, + onRefresh, +}: { + collection: string; + documents: EJSONDocument[]; + totalDocuments: number; + page: number; + pageSize: number; + totalPages: number; + isLoading: boolean; + sortField: string | null; + sortDirection: 1 | -1 | null; + onSort: (field: string) => void; + onPageChange: (newPage: number) => void; + onPageSizeChange: (newSize: number) => void; + filterInput: string; + onFilterInputChange: (text: string) => void; + onApplyFilter: (filter: EJSONDocument | undefined) => void; + onResetFilter: () => void; + isFilterApplied: boolean; + onRefresh: () => void; +}) { + const [copiedAll, setCopiedAll] = useState(false); + + // Derive column list, pinning _id to the first position + const columns = useMemo(() => { + if (!documents || documents.length === 0) return []; + const keySet = new Set(); + documents.forEach((doc) => { + Object.keys(doc).forEach((k) => keySet.add(k)); + }); + keySet.delete("_id"); + const otherCols = Array.from(keySet).sort(); + return ["_id", ...otherCols]; + }, [documents]); + + const handleCopyAll = () => { + if (!documents || documents.length === 0) return; + const str = JSON.stringify(documents, null, 2); + navigator.clipboard.writeText(str); + setCopiedAll(true); + toast.success(`Copied ${documents.length} visible documents to clipboard`); + setTimeout(() => setCopiedAll(false), 1500); + }; + + const startDoc = (page - 1) * pageSize + 1; + const endDoc = Math.min(page * pageSize, totalDocuments); + + return ( +
+ {/* Top action toolbar */} +
+ + +
+ + + +
+
+ + {/* Main table area */} +
+ {isLoading && documents.length === 0 ? ( +
+ +

Loading documents...

+
+ ) : documents.length === 0 ? ( +
+
+ +
+

No documents found

+

+ {isFilterApplied + ? "No documents matched your active filter." + : `Collection "${collection}" has no documents.`} +

+
+ ) : ( + } + /> + )} +
+ + {/* Pagination Footer */} +
+
+ + {totalDocuments > 0 ? ( + <> + Showing {startDoc.toLocaleString()} –{" "} + {endDoc.toLocaleString()} of{" "} + {totalDocuments.toLocaleString()}{" "} + documents + + ) : ( + "0 documents" + )} + + +
+ Per page: + +
+
+ +
+ + + + Page {page} of {totalPages || 1} + + + +
+
+
+ ); +} diff --git a/src/features/mongodb/components/MongoFilterInput.tsx b/src/features/mongodb/components/MongoFilterInput.tsx new file mode 100644 index 00000000..ccd1e6c1 --- /dev/null +++ b/src/features/mongodb/components/MongoFilterInput.tsx @@ -0,0 +1,129 @@ +import React, { useState, useEffect } from "react"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Filter, X, CheckCircle2, AlertCircle } from "lucide-react"; +import { EJSONDocument } from "../types"; + +export interface MongoFilterValidation { + isValid: boolean; + error?: string; + parsed?: EJSONDocument; +} + +export function validateMongoFilter(text: string): MongoFilterValidation { + const trimmed = text.trim(); + if (!trimmed) { + return { isValid: true, parsed: undefined }; + } + try { + const parsed = JSON.parse(trimmed); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return { + isValid: false, + error: 'Filter must be a JSON object, e.g. {"status": "active"}', + }; + } + return { isValid: true, parsed }; + } catch (err: any) { + return { isValid: false, error: err.message || "Invalid JSON syntax" }; + } +} + +export function MongoFilterInput({ + filterInput, + onFilterInputChange, + onApply, + onReset, + isApplied, +}: { + filterInput: string; + onFilterInputChange: (text: string) => void; + onApply: (filter: EJSONDocument | undefined) => void; + onReset: () => void; + isApplied: boolean; +}) { + const [validation, setValidation] = useState(() => + validateMongoFilter(filterInput), + ); + + useEffect(() => { + setValidation(validateMongoFilter(filterInput)); + }, [filterInput]); + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && validation.isValid) { + e.preventDefault(); + onApply(validation.parsed); + } + }; + + const handleApply = () => { + if (validation.isValid) { + onApply(validation.parsed); + } + }; + + return ( +
+
+
+ + onFilterInputChange(e.target.value)} + onKeyDown={handleKeyDown} + placeholder='EJSON Filter: {"status": "active"} or {"_id": {"$oid": "..."}}' + className="pl-9 pr-8 font-mono text-xs h-9 bg-background/50 border-border/50 focus-visible:ring-emerald-500/30" + /> + {filterInput && ( + + )} +
+ + + + {isApplied && ( + + )} +
+ +
+ {filterInput.trim() ? ( + validation.isValid ? ( + + Valid EJSON filter + + ) : ( + + {validation.error} + + ) + ) : ( + + Enter an Extended-JSON object to filter documents + + )} +
+
+ ); +} diff --git a/src/features/mongodb/components/MongoSchemaPanel.tsx b/src/features/mongodb/components/MongoSchemaPanel.tsx new file mode 100644 index 00000000..e7809bba --- /dev/null +++ b/src/features/mongodb/components/MongoSchemaPanel.tsx @@ -0,0 +1,223 @@ +import React, { useState } from "react"; +import { useMongoSchema } from "@/features/project/hooks/useDbQueries"; +import { MongoBsonType, MongoInferredField } from "../types"; +import { Button } from "@/components/ui/button"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Progress } from "@/components/ui/progress"; +import { RefreshCw, GitBranch, AlertTriangle, Database } from "lucide-react"; +import { formatTimestamp } from "@/lib/utils"; + +function getBsonTypeBadge(type: MongoBsonType) { + switch (type) { + case "objectId": + return "bg-teal-500/10 text-teal-600 dark:text-teal-400 border-teal-500/30"; + case "string": + return "bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/30"; + case "number": + return "bg-indigo-500/10 text-indigo-600 dark:text-indigo-400 border-indigo-500/30"; + case "long": + return "bg-blue-500/10 text-blue-600 dark:text-blue-400 border-blue-500/30"; + case "decimal": + return "bg-cyan-500/10 text-cyan-600 dark:text-cyan-400 border-cyan-500/30"; + case "date": + return "bg-purple-500/10 text-purple-600 dark:text-purple-400 border-purple-500/30"; + case "bool": + return "bg-emerald-500/10 text-emerald-600 dark:text-emerald-400 border-emerald-500/30"; + case "array": + return "bg-rose-500/10 text-rose-600 dark:text-rose-400 border-rose-500/30"; + case "object": + return "bg-orange-500/10 text-orange-600 dark:text-orange-400 border-orange-500/30"; + case "binary": + return "bg-yellow-500/10 text-yellow-600 dark:text-yellow-400 border-yellow-500/30"; + default: + return "bg-muted text-muted-foreground border-border/40"; + } +} + +export function MongoSchemaPanel({ dbId, collection }: { dbId: string; collection: string }) { + const [sampleSize, setSampleSize] = useState(100); + + const { + data: schema, + isLoading, + refetch, + isFetching, + } = useMongoSchema(dbId, collection, sampleSize); + + return ( +
+ {/* Header toolbar */} +
+
+
+ + + Statistical Schema Inference + +
+ + {schema && ( + + Scanned {schema.sampleSize} of ~ + {schema.totalDocs.toLocaleString()} docs • Discovered{" "} + {schema.fieldCount} fields ({schema.nestedFieldCount} nested,{" "} + {schema.arrayFieldCount} arrays) • {formatTimestamp(schema.inferredAt)} + + )} +
+ +
+
+ Sample size: + +
+ + +
+
+ + {/* Main Field Tree Area */} +
+ {isLoading ? ( +
+ +

Sampling documents and inferring schema...

+
+ ) : !schema || schema.fields.length === 0 ? ( +
+
+ +
+

No schema inferred

+

+ Collection "{collection}" has no documents to sample. +

+
+ ) : ( +
+ + + + + + + + + + + {schema.fields.map((field: MongoInferredField) => { + const hasConflict = field.typeConflicts && field.typeConflicts.length > 1; + return ( + + + + + + + + + + ); + })} + +
Field PathTypePresenceAttributes & Conflicts
+
+ {field.depth > 0 && ( + ↳ + )} + + {field.path} + +
+
+ + {field.bsonType} + + +
+ + + {field.presencePct}% ({field.count}/{field.sampleSize}) + +
+
+
+ {field.optional && ( + + optional + + )} + + {field.nullable && ( + + nullable ({field.nullCount}) + + )} + + {field.isArray && field.arrayElementTypes && ( + + items: [{field.arrayElementTypes.join(", ")}] + + )} + + {hasConflict && ( + + + conflict:{" "} + {field.typeConflicts + .map((c) => `${c.bsonType} (${c.count})`) + .join(", ")} + + )} + + {field.truncated && ( + + depth limit reached + + )} +
+
+
+ )} +
+
+ ); +} diff --git a/src/features/mongodb/hooks/useMongoDatabaseDetails.ts b/src/features/mongodb/hooks/useMongoDatabaseDetails.ts new file mode 100644 index 00000000..aafa6b8a --- /dev/null +++ b/src/features/mongodb/hooks/useMongoDatabaseDetails.ts @@ -0,0 +1,117 @@ +import { useState, useEffect, useMemo, useCallback } from "react"; +import { useMongoCollections, useMongoDocuments } from "@/features/project/hooks/useDbQueries"; +import { EJSONDocument, MongoFindRequest } from "../types"; + +export function useMongoDatabaseDetails(dbId: string) { + const [selectedCollection, setSelectedCollection] = useState(null); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(50); + const [sortField, setSortField] = useState(null); + const [sortDirection, setSortDirection] = useState<1 | -1 | null>(null); + const [filterInput, setFilterInput] = useState(""); + const [appliedFilter, setAppliedFilter] = useState(undefined); + const [activeTab, setActiveTab] = useState<"documents" | "schema">("documents"); + + const { + data: collectionsData, + isLoading: collectionsLoading, + error: collectionsError, + refetch: refetchCollections, + } = useMongoCollections(dbId); + + const collections = useMemo(() => collectionsData?.collections || [], [collectionsData]); + + useEffect(() => { + if (collections.length > 0 && !selectedCollection) { + setSelectedCollection(collections[0].name); + } + }, [collections, selectedCollection]); + + const handleSelectCollection = useCallback((name: string) => { + setSelectedCollection(name); + setPage(1); + setFilterInput(""); + setAppliedFilter(undefined); + setSortField(null); + setSortDirection(null); + }, []); + + const handleSort = useCallback((field: string) => { + setSortField((currentField) => { + if (currentField === field) { + setSortDirection((currentDir) => (currentDir === 1 ? -1 : currentDir === -1 ? null : 1)); + return field; + } + setSortDirection(1); + return field; + }); + setPage(1); + }, []); + + const sortSpec = useMemo(() => { + if (!sortField || !sortDirection) return undefined; + return { [sortField]: sortDirection }; + }, [sortField, sortDirection]); + + const findRequest: MongoFindRequest | null = useMemo(() => { + if (!dbId || !selectedCollection) return null; + return { + dbId, + collection: selectedCollection, + page, + pageSize, + filter: appliedFilter, + sort: sortSpec, + }; + }, [dbId, selectedCollection, page, pageSize, appliedFilter, sortSpec]); + + const { + data: docResult, + isLoading: isLoadingDocuments, + isFetching: isFetchingDocuments, + error: documentsError, + refetch: refetchDocuments, + } = useMongoDocuments(findRequest, !!selectedCollection); + + const applyFilter = useCallback((filterObj: EJSONDocument | undefined) => { + setAppliedFilter(filterObj); + setPage(1); + }, []); + + const resetFilter = useCallback(() => { + setFilterInput(""); + setAppliedFilter(undefined); + setPage(1); + }, []); + + return { + collections, + collectionsLoading, + collectionsError, + selectedCollection, + setSelectedCollection: handleSelectCollection, + documents: docResult?.documents || [], + totalDocuments: docResult?.total || 0, + totalPages: docResult?.totalPages || 1, + hasMore: docResult?.hasMore || false, + page, + setPage, + pageSize, + setPageSize, + sortField, + sortDirection, + handleSort, + filterInput, + setFilterInput, + appliedFilter, + applyFilter, + resetFilter, + activeTab, + setActiveTab, + isLoadingDocuments, + isFetchingDocuments, + documentsError, + refetchDocuments, + refetchCollections, + }; +} diff --git a/src/features/mongodb/types.ts b/src/features/mongodb/types.ts new file mode 100644 index 00000000..c1016cd4 --- /dev/null +++ b/src/features/mongodb/types.ts @@ -0,0 +1,79 @@ +export type MongoBsonType = + | "objectId" + | "string" + | "number" + | "long" + | "decimal" + | "timestamp" + | "binary" + | "date" + | "bool" + | "null" + | "array" + | "object" + | "undefined"; + +export interface MongoCollectionInfo { + name: string; + type: "collection" | "view"; +} + +export interface MongoInferredField { + path: string; + bsonType: MongoBsonType; + types: MongoBsonType[]; + numericKinds?: Array<"int" | "double">; + depth: number; + parentPath?: string; + isArray: boolean; + isNested: boolean; + count: number; + nullCount: number; + sampleSize: number; + presencePct: number; + nullable: boolean; + optional: boolean; + typeConflicts: Array<{ bsonType: MongoBsonType; count: number }>; + arrayElementTypes?: MongoBsonType[]; + sampleValue?: any; + maxChildDepth?: number; + truncated?: boolean; +} + +export interface MongoCollectionSchema { + collection: string; + sampleSize: number; + totalDocs: number; + inferredAt: string; + fieldCount: number; + nestedFieldCount: number; + arrayFieldCount: number; + fields: MongoInferredField[]; + sampleDocuments: Array>; +} + +export type MongoSortDirection = 1 | -1 | "ASC" | "DESC"; +export type MongoSortSpec = { [field: string]: MongoSortDirection }; +export type MongoProjection = { [field: string]: 0 | 1 | false | true }; + +export type EJSONDocument = Record; + +export interface MongoFindRequest { + dbId: string; + collection: string; + filter?: EJSONDocument; + sort?: MongoSortSpec; + projection?: MongoProjection; + page?: number; + pageSize?: number; +} + +export interface MongoFindResult { + collection: string; + documents: EJSONDocument[]; + total: number; + page: number; + pageSize: number; + totalPages: number; + hasMore: boolean; +} diff --git a/src/features/project/hooks/useDbQueries.ts b/src/features/project/hooks/useDbQueries.ts index a1b1a3e2..fe4b2786 100644 --- a/src/features/project/hooks/useDbQueries.ts +++ b/src/features/project/hooks/useDbQueries.ts @@ -1,5 +1,6 @@ import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; import { TableInfo } from "@/features/database/types"; +import type { MongoFindRequest } from "@/features/mongodb/types"; import { isBridgeReady } from "@/services/bridge/bridgeClient"; import { databaseService } from "@/services/bridge/database"; import { queryService } from "@/services/bridge/query"; @@ -24,6 +25,21 @@ export const queryKeys = { ["tableDetails", dbId, schema, table] as const, primaryKeys: (dbId: string, schema: string, table: string) => ["primaryKeys", dbId, schema, table] as const, + + // MongoDB collections, documents, schema + mongoCollections: (dbId: string) => ["mongoCollections", dbId] as const, + mongoDocuments: (req: MongoFindRequest) => + [ + "mongoDocuments", + req.dbId, + req.collection, + req.page, + req.pageSize, + req.filter, + req.sort, + ] as const, + mongoSchema: (dbId: string, col: string, sampleSize?: number, maxDepth?: number) => + ["mongoSchema", dbId, col, sampleSize ?? 100, maxDepth ?? 8] as const, }; // ============================================ @@ -36,6 +52,9 @@ const STALE_TIMES = { stats: 30 * 1000, // 30 seconds - database stats schemas: 5 * 60 * 1000, // 5 minutes - schemas rarely change tableDetails: 60 * 1000, // 1 minute - column info + mongoCollections: 60 * 1000, + mongoDocuments: 30 * 1000, + mongoSchema: 5 * 60 * 1000, }; // ============================================ @@ -94,7 +113,7 @@ export function useMigrations(dbId: string | undefined) { * - Returns cached data instantly if available * - Background refetch if stale */ -export function useTables(dbId: string | undefined, schema?: string) { +export function useTables(dbId: string | undefined, schema?: string, enabled = true) { return useQuery({ queryKey: queryKeys.tables(dbId!, schema), queryFn: async () => { @@ -105,7 +124,7 @@ export function useTables(dbId: string | undefined, schema?: string) { type: item.type || "table", })); }, - enabled: !!dbId, + enabled: !!dbId && enabled, staleTime: STALE_TIMES.tables, gcTime: 5 * 60 * 1000, }); @@ -114,11 +133,11 @@ export function useTables(dbId: string | undefined, schema?: string) { /** * Fetch schema names for a database */ -export function useSchemaNames(dbId: string | undefined) { +export function useSchemaNames(dbId: string | undefined, enabled = true) { return useQuery({ queryKey: ["schemaNames", dbId] as const, queryFn: () => databaseService.listSchemas(dbId!), - enabled: !!dbId, + enabled: !!dbId && enabled, staleTime: STALE_TIMES.schemas, }); } @@ -144,6 +163,43 @@ export function useTableData( }); } +// ============================================ +// MongoDB Collection, Document, & Schema Hooks +// ============================================ + +export function useMongoCollections(dbId: string | undefined) { + return useQuery({ + queryKey: queryKeys.mongoCollections(dbId!), + queryFn: () => databaseService.listMongoCollections(dbId!), + enabled: !!dbId, + staleTime: STALE_TIMES.mongoCollections, + }); +} + +export function useMongoDocuments(req: MongoFindRequest | null, enabled = true) { + return useQuery({ + queryKey: req ? queryKeys.mongoDocuments(req) : ["mongoDocuments", "none"], + queryFn: () => queryService.fetchMongoDocuments(req!), + enabled: !!req && !!req.dbId && !!req.collection && enabled, + staleTime: STALE_TIMES.mongoDocuments, + }); +} + +export function useMongoSchema( + dbId: string | undefined, + collection: string | undefined, + sampleSize?: number, + maxDepth?: number, + enabled = true, +) { + return useQuery({ + queryKey: queryKeys.mongoSchema(dbId!, collection!, sampleSize, maxDepth), + queryFn: () => databaseService.getMongoInferredSchema(dbId!, collection!, sampleSize, maxDepth), + enabled: !!dbId && !!collection && enabled, + staleTime: STALE_TIMES.mongoSchema, + }); +} + // ============================================ // Database Metadata Hooks // ============================================ @@ -339,6 +395,19 @@ export function usePrefetch() { staleTime: STALE_TIMES.schemas, }); }, + + prefetchMongoNextPage: (req: MongoFindRequest) => { + if (!req.dbId || !req.collection) return; + const nextPageReq: MongoFindRequest = { + ...req, + page: (req.page || 1) + 1, + }; + queryClient.prefetchQuery({ + queryKey: queryKeys.mongoDocuments(nextPageReq), + queryFn: () => queryService.fetchMongoDocuments(nextPageReq), + staleTime: STALE_TIMES.mongoDocuments, + }); + }, }; } @@ -364,6 +433,9 @@ export function useInvalidateCache() { queryClient.invalidateQueries({ queryKey: ["migrations", dbId] }), // Also invalidate migrations queryClient.invalidateQueries({ queryKey: ["tableData", dbId] }), queryClient.invalidateQueries({ queryKey: ["projects"] }), + queryClient.invalidateQueries({ queryKey: ["mongoCollections", dbId] }), + queryClient.invalidateQueries({ queryKey: ["mongoDocuments", dbId] }), + queryClient.invalidateQueries({ queryKey: ["mongoSchema", dbId] }), ]); }, @@ -381,6 +453,32 @@ export function useInvalidateCache() { }); }, + invalidateMongoCollections: (dbId: string) => { + return queryClient.invalidateQueries({ queryKey: ["mongoCollections", dbId] }); + }, + invalidateMongoDocuments: (dbId: string, collection?: string) => { + return queryClient.invalidateQueries({ + predicate: (query) => { + const key = query.queryKey; + if (key[0] !== "mongoDocuments") return false; + if (key[1] !== dbId) return false; + if (collection && key[2] !== collection) return false; + return true; + }, + }); + }, + invalidateMongoSchema: (dbId: string, collection?: string) => { + return queryClient.invalidateQueries({ + predicate: (query) => { + const key = query.queryKey; + if (key[0] !== "mongoSchema") return false; + if (key[1] !== dbId) return false; + if (collection && key[2] !== collection) return false; + return true; + }, + }); + }, + /** Force refresh all data */ invalidateAll: () => { queryClient.invalidateQueries(); diff --git a/src/lib/databases.tsx b/src/lib/databases.tsx index 81c1e704..de0c369e 100644 --- a/src/lib/databases.tsx +++ b/src/lib/databases.tsx @@ -5,6 +5,8 @@ import { Database } from "lucide-react"; export type SupportedDatabaseType = DatabaseType | "mariadb"; +export type DatabaseKind = "sql" | "nosql"; + export interface DatatypeCategory { category: string; types: string[]; @@ -21,7 +23,8 @@ export interface DatabaseConfig { name: string; icon: ReactNode; deviconClass?: string; - category: "Relational" | "Embedded" | "OLAP"; + kind?: DatabaseKind; + category: "Relational" | "Embedded" | "OLAP" | "Document"; description: string; defaultPort?: string; defaultUser?: string; @@ -280,6 +283,61 @@ export const databases: DatabaseConfig[] = [ ), }, + { + id: "mongodb", + type: "mongodb", + name: "MongoDB", + kind: "nosql", + category: "Document", + description: "Scalable document (NoSQL) database", + defaultPort: "27017", + defaultUser: "", + defaultDatabase: "", + placeholderUrl: "mongodb://localhost:27017/mydb", + supportsSsl: true, + supportsSsh: true, + deviconClass: "devicon-mongodb-plain colored", + foreignKeySupport: { + supported: false, + notes: + "Document store; relationships are embedded or referenced in application logic, not enforced by the database.", + }, + acceptedDatatypes: [ + { + category: "BSON Types", + types: [ + "String", + "Int32", + "Int64/Long", + "Double", + "Decimal128", + "Bool", + "Date", + "ObjectId", + "Binary", + "Array", + "Object", + "Null", + ], + }, + ], + icon: ( + + + + + + ), + }, ]; export const SUPPORTED_DATABASES = databases; @@ -334,8 +392,18 @@ export function getAllAcceptedDatatypes(typeOrName?: string | null): string[] { return categories.flatMap((cat) => cat.types); } +export function isDocumentDatabase(typeOrName?: string | null): boolean { + const normalized = typeOrName?.toLowerCase().trim(); + return normalized === "mongodb"; +} + +export function getDatabaseKind(typeOrName?: string | null): DatabaseKind { + return isDocumentDatabase(typeOrName) ? "nosql" : "sql"; +} + export function getDefaultColumnType(typeOrName?: string | null): string { const normalized = typeOrName?.toLowerCase().trim(); + if (normalized === "mongodb") return "string"; if (normalized === "clickhouse") return "String"; if (normalized === "sqlite") return "INTEGER"; return "VARCHAR"; diff --git a/src/lib/parseConnectionUrl.ts b/src/lib/parseConnectionUrl.ts index 54d951e5..fc202bc8 100644 --- a/src/lib/parseConnectionUrl.ts +++ b/src/lib/parseConnectionUrl.ts @@ -15,6 +15,82 @@ export interface ParsedConnectionUrl { database: string; ssl: boolean; sslmode: string; + originalUri?: string; +} + +export function parseMongoUrl(url: string): ParsedConnectionUrl | null { + const originalUri = url.trim(); + const isSrv = originalUri.startsWith("mongodb+srv://"); + const prefix = isSrv ? "mongodb+srv://" : "mongodb://"; + const rest = originalUri.slice(prefix.length); + + const queryIndex = rest.indexOf("?"); + const hashIndex = rest.indexOf("#"); + let endOfPathIndex = rest.length; + if (queryIndex !== -1) endOfPathIndex = queryIndex; + if (hashIndex !== -1 && (queryIndex === -1 || hashIndex < queryIndex)) endOfPathIndex = hashIndex; + + const authorityAndPath = rest.slice(0, endOfPathIndex); + const queryString = queryIndex !== -1 ? rest.slice(queryIndex) : ""; + + const slashIndex = authorityAndPath.indexOf("/"); + let authority = slashIndex !== -1 ? authorityAndPath.slice(0, slashIndex) : authorityAndPath; + let database = slashIndex !== -1 ? authorityAndPath.slice(slashIndex + 1) : ""; + try { + database = decodeURIComponent(database); + } catch {} + + let user = ""; + let password = ""; + const atIndex = authority.lastIndexOf("@"); + if (atIndex !== -1) { + const userInfo = authority.slice(0, atIndex); + authority = authority.slice(atIndex + 1); + const colonIndex = userInfo.indexOf(":"); + if (colonIndex !== -1) { + user = userInfo.slice(0, colonIndex); + password = userInfo.slice(colonIndex + 1); + } else { + user = userInfo; + } + try { + user = decodeURIComponent(user); + } catch {} + try { + password = decodeURIComponent(password); + } catch {} + } + + let host = authority; + let port = isSrv ? "" : "27017"; + + if (!isSrv && !authority.includes(",")) { + const colonIndex = authority.lastIndexOf(":"); + if (colonIndex !== -1) { + host = authority.slice(0, colonIndex); + port = authority.slice(colonIndex + 1) || "27017"; + } + } else if (!isSrv && authority.includes(",")) { + const firstHost = authority.split(",")[0]; + const colonIndex = firstHost.lastIndexOf(":"); + if (colonIndex !== -1) { + port = firstHost.slice(colonIndex + 1) || "27017"; + } + } + + const ssl = isSrv || /[?&](?:ssl|tls)=true/i.test(queryString); + + return { + type: "mongodb", + host, + port: port || (isSrv ? "27017" : "27017"), + user, + password, + database, + ssl, + sslmode: "", + originalUri, + }; } function normalizeSqlitePathFromUrl(parsed: URL): string { @@ -121,13 +197,24 @@ export function parseClickHouseUrl(raw?: string): { export function parseConnectionUrl(url: string): ParsedConnectionUrl | null { try { // Trim whitespace - url = url.trim(); + const trimmed = url.trim(); // Handle empty string - if (!url) return null; + if (!trimmed) return null; + + if (trimmed.startsWith("mongodb://") || trimmed.startsWith("mongodb+srv://")) { + return parseMongoUrl(trimmed); + } + + const isExplicitOther = + trimmed.startsWith("postgres://") || + trimmed.startsWith("postgresql://") || + trimmed.startsWith("mysql://") || + trimmed.startsWith("sqlite://") || + trimmed.startsWith("duckdb://"); // Parse the URL - const parsed = new URL(cleanClickHouseUrl(url)); + const parsed = new URL(isExplicitOther ? trimmed : cleanClickHouseUrl(trimmed)); // Determine database type from protocol let type = ""; @@ -170,6 +257,7 @@ export function parseConnectionUrl(url: string): ParsedConnectionUrl | null { database: dbPath, ssl: false, sslmode: "", + originalUri: trimmed, }; } @@ -205,6 +293,7 @@ export function parseConnectionUrl(url: string): ParsedConnectionUrl | null { database, ssl, sslmode, + originalUri: trimmed, }; } catch { // Invalid URL format @@ -223,7 +312,25 @@ export function buildConnectionUrl(params: { password: string; database: string; sslmode?: string; + ssl?: boolean; + originalUri?: string; }): string { + if (params.type === "mongodb") { + if (params.originalUri) { + return params.originalUri; + } + const auth = params.user + ? params.password + ? `${encodeURIComponent(params.user)}:${encodeURIComponent(params.password)}@` + : `${encodeURIComponent(params.user)}@` + : ""; + const host = params.host || "localhost"; + const port = params.port ? `:${params.port}` : ":27017"; + const db = params.database ? `/${encodeURIComponent(params.database)}` : ""; + const sslQuery = params.ssl ? "?tls=true" : ""; + return `mongodb://${auth}${host}${port}${db}${sslQuery}`; + } + if (params.type === "sqlite" || params.type === "duckdb") { return `${params.type}:///${encodeSqlitePathSegments(params.database)}`; } diff --git a/src/pages/DatabaseDetails.tsx b/src/pages/DatabaseDetails.tsx index 86afc2e5..95a5ea27 100644 --- a/src/pages/DatabaseDetails.tsx +++ b/src/pages/DatabaseDetails.tsx @@ -31,6 +31,8 @@ import AIWorkspacePanel from "@/features/database/components/AIWorkspacePanel"; import { ShortcutsHelp } from "@/components/shared/ShortcutsHelp"; import { ShortcutsTrigger } from "@/components/shared/ShortcutsTrigger"; import { MigrationSyncDialog } from "@/features/project/components/MigrationSyncDialog"; +import { getDatabaseKind } from "@/lib/databases"; +import { MongoDataViewPanel } from "@/features/mongodb/components/MongoDataViewPanel"; import { useState, useEffect } from "react"; const DatabaseDetail = () => { @@ -89,15 +91,17 @@ const DatabaseDetail = () => { databaseName: databaseName || "database", }); + const isNosql = getDatabaseKind(databaseType) === "nosql"; + // Migrations - const { data: migrationsResponse } = useMigrations(dbId); + const { data: migrationsResponse } = useMigrations(isNosql ? undefined : dbId); const migrationsData = migrationsResponse?.migrations || { local: [], applied: [] }; const baselined = migrationsResponse?.baselined || false; // Project sync - const { data: schemaData, refetch: refetchSchema } = useFullSchema(dbId); + const { data: schemaData, refetch: refetchSchema } = useFullSchema(isNosql ? undefined : dbId); const { projectId, importAnalysis, importAnalysisLoading, refetchImportAnalysis } = - useProjectSync(dbId, schemaData ?? undefined); + useProjectSync(isNosql ? undefined : dbId, isNosql ? undefined : (schemaData ?? undefined)); const { data: projectDir } = useProjectDir(projectId); const [migrationSyncDismissed, setMigrationSyncDismissed] = useState(false); @@ -139,6 +143,10 @@ const DatabaseDetail = () => { // ---- Panel router ---- const renderPanel = () => { + if (isNosql) { + return ; + } + switch (activePanel) { case "sql-workspace": return ; diff --git a/src/services/bridge/database.ts b/src/services/bridge/database.ts index f75088be..b47e4c65 100644 --- a/src/services/bridge/database.ts +++ b/src/services/bridge/database.ts @@ -10,6 +10,7 @@ import { SSHConfig, UpdateDatabaseParams, } from "@/features/database/types"; +import { MongoCollectionInfo, MongoCollectionSchema } from "@/features/mongodb/types"; import { bridgeRequest } from "./bridgeClient"; class DatabaseService { @@ -67,11 +68,14 @@ class DatabaseService { // Validate required fields const isFileBasedDB = params.type === "sqlite" || params.type === "duckdb"; const isClickHouse = params.type === "clickhouse"; + const isMongo = params.type === "mongodb"; const required = isFileBasedDB ? ["name", "type", "database"] : isClickHouse ? ["name", "type"] - : ["name", "type", "host", "port", "user", "database"]; + : isMongo + ? (params.url ? ["name", "type"] : ["name", "type", "host", "port"]) + : ["name", "type", "host", "port", "user", "database"]; for (const field of required) { if (!params[field as keyof AddDatabaseParams]) { throw new Error(`Missing required field: ${field}`); @@ -594,6 +598,44 @@ class DatabaseService { return []; // Return empty array on error, don't throw } } + + /** + * List collections in a MongoDB database (db.listCollections) + */ + async listMongoCollections(dbId: string): Promise<{ collections: MongoCollectionInfo[] }> { + try { + if (!dbId) throw new Error("Database ID is required"); + const result = await bridgeRequest("db.listCollections", { dbId }); + return result?.data || { collections: [] }; + } catch (error: any) { + console.error("Failed to list MongoDB collections:", error); + throw new Error(`Failed to list collections: ${error.message}`); + } + } + + /** + * Get inferred schema for a MongoDB collection (db.getInferredSchema) + */ + async getMongoInferredSchema( + dbId: string, + collection: string, + sampleSize?: number, + maxDepth?: number, + ): Promise { + try { + if (!dbId || !collection) throw new Error("Database ID and collection name are required"); + const result = await bridgeRequest("db.getInferredSchema", { + dbId, + collection, + sampleSize, + maxDepth, + }); + return result?.data; + } catch (error: any) { + console.error("Failed to get inferred MongoDB schema:", error); + throw new Error(`Failed to get inferred schema: ${error.message}`); + } + } } export const databaseService = new DatabaseService(); diff --git a/src/services/bridge/query.ts b/src/services/bridge/query.ts index b6643ec4..8c41cbad 100644 --- a/src/services/bridge/query.ts +++ b/src/services/bridge/query.ts @@ -1,4 +1,5 @@ import { RunQueryParams, TableRow } from "@/features/database/types"; +import { MongoFindRequest, MongoFindResult } from "@/features/mongodb/types"; import { bridgeRequest } from "./bridgeClient"; class QueryService { @@ -59,6 +60,22 @@ class QueryService { throw new Error(`Failed to fetch table data: ${error.message}`); } } + + /** + * Fetches MongoDB documents with pagination, sorting, and filtering support (query.findDocuments). + */ + async fetchMongoDocuments(req: MongoFindRequest): Promise { + try { + if (!req.dbId || !req.collection) { + throw new Error("Database ID and collection name are required."); + } + const result = await bridgeRequest("query.findDocuments", req); + return result?.data; + } catch (error: any) { + console.error("Failed to fetch MongoDB documents:", error); + throw new Error(`Failed to fetch MongoDB documents: ${error.message}`); + } + } } export const queryService = new QueryService(); From 8b8309d68c98fd0800501267cddff86e1e11c762 Mon Sep 17 00:00:00 2001 From: Yash Date: Thu, 29 Oct 2026 21:14:24 +0530 Subject: [PATCH 04/34] feat(mongodb): implement M4 mutation experience --- .../connectors/mongoMutations.test.ts | 269 +++++++++++++++ .../mongoMutations.integration.test.ts | 143 ++++++++ bridge/src/connectors/mongo.ts | 186 ++++++++++ bridge/src/handlers/queryHandlers.ts | 61 ++++ bridge/src/jsonRpcHandler.ts | 3 + bridge/src/services/queryExecutor.ts | 48 +++ bridge/src/types/mongo.ts | 49 +++ .../__tests__/MongoMutationValidation.test.ts | 86 +++++ .../mongodb/__tests__/useDocumentDiff.test.ts | 224 ++++++++++++ .../components/MongoCollectionsSidebar.tsx | 78 +++-- .../MongoCreateCollectionDialog.tsx | 226 ++++++++++++ .../mongodb/components/MongoDataViewPanel.tsx | 67 ++++ .../components/MongoDeleteDocumentDialog.tsx | 150 ++++++++ .../components/MongoDocumentViewer.tsx | 21 ++ .../components/MongoDropCollectionDialog.tsx | 139 ++++++++ .../components/MongoEditDocumentDialog.tsx | 321 ++++++++++++++++++ .../components/MongoInsertDocumentDialog.tsx | 247 ++++++++++++++ src/features/mongodb/hooks/useDocumentDiff.ts | 235 +++++++++++++ .../mongodb/hooks/useMongoDatabaseDetails.ts | 34 ++ .../mongodb/hooks/useMongoMutations.ts | 128 +++++++ src/features/mongodb/types.ts | 102 ++++++ src/features/mongodb/utils/validation.ts | 72 ++++ src/services/bridge/database.ts | 35 +- src/services/bridge/query.ts | 77 ++++- 24 files changed, 2979 insertions(+), 22 deletions(-) create mode 100644 bridge/__tests__/connectors/mongoMutations.test.ts create mode 100644 bridge/__tests__/integration/mongoMutations.integration.test.ts create mode 100644 src/features/mongodb/__tests__/MongoMutationValidation.test.ts create mode 100644 src/features/mongodb/__tests__/useDocumentDiff.test.ts create mode 100644 src/features/mongodb/components/MongoCreateCollectionDialog.tsx create mode 100644 src/features/mongodb/components/MongoDeleteDocumentDialog.tsx create mode 100644 src/features/mongodb/components/MongoDropCollectionDialog.tsx create mode 100644 src/features/mongodb/components/MongoEditDocumentDialog.tsx create mode 100644 src/features/mongodb/components/MongoInsertDocumentDialog.tsx create mode 100644 src/features/mongodb/hooks/useDocumentDiff.ts create mode 100644 src/features/mongodb/hooks/useMongoMutations.ts create mode 100644 src/features/mongodb/utils/validation.ts diff --git a/bridge/__tests__/connectors/mongoMutations.test.ts b/bridge/__tests__/connectors/mongoMutations.test.ts new file mode 100644 index 00000000..3c9e4114 --- /dev/null +++ b/bridge/__tests__/connectors/mongoMutations.test.ts @@ -0,0 +1,269 @@ +import { describe, expect, test } from "@jest/globals"; +import { + assertNewCollectionName, + assertNoOperatorKeys, + mongoConnector, + mongodbCache, + MongoApiError, +} from "../../src/connectors/mongo"; +import type { + MongoConfig, + EJSONDocument, +} from "../../src/types/mongo"; +import { QueryExecutor } from "../../src/services/queryExecutor"; +import { DBType } from "../../src/types"; + +describe("assertNewCollectionName", () => { + test("accepts valid collection names", () => { + expect(() => assertNewCollectionName("users")).not.toThrow(); + expect(() => assertNewCollectionName("user_accounts")).not.toThrow(); + expect(() => assertNewCollectionName("orders-2026")).not.toThrow(); + expect(() => assertNewCollectionName("_audit_log")).not.toThrow(); + expect(() => assertNewCollectionName("A")).not.toThrow(); + expect(() => assertNewCollectionName("a".repeat(120))).not.toThrow(); + }); + + test("rejects non-string or empty name", () => { + expect(() => assertNewCollectionName("")).toThrow(MongoApiError); + expect(() => assertNewCollectionName(null)).toThrow(MongoApiError); + expect(() => assertNewCollectionName(undefined)).toThrow(MongoApiError); + expect(() => assertNewCollectionName(123)).toThrow(MongoApiError); + }); + + test("rejects names longer than 120 chars", () => { + expect(() => assertNewCollectionName("a".repeat(121))).toThrow(MongoApiError); + }); + + test("rejects names with '$' or null bytes", () => { + expect(() => assertNewCollectionName("my$coll")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("coll\0name")).toThrow(MongoApiError); + }); + + test("rejects names starting with 'system.'", () => { + expect(() => assertNewCollectionName("system.views")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("system.profile")).toThrow(MongoApiError); + }); + + test("rejects names starting with numbers or containing invalid characters", () => { + expect(() => assertNewCollectionName("123users")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("-users")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("my.collection")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("my collection")).toThrow(MongoApiError); + expect(() => assertNewCollectionName("coll@test")).toThrow(MongoApiError); + }); +}); + +describe("assertNoOperatorKeys", () => { + test("allows documents without top-level operator keys", () => { + expect(() => assertNoOperatorKeys({ name: "Alice", age: 30 })).not.toThrow(); + expect(() => assertNoOperatorKeys({ _id: { $oid: "507f1f77bcf86cd799439011" } })).not.toThrow(); + expect(() => assertNoOperatorKeys({})).not.toThrow(); + }); + + test("throws MONGO_VALIDATION_ERROR if any top-level key starts with '$'", () => { + expect(() => assertNoOperatorKeys({ $set: { name: "Bob" } })).toThrow( + "replacement document must not contain top-level operator keys", + ); + expect(() => assertNoOperatorKeys({ name: "Bob", $inc: { count: 1 } })).toThrow( + "replacement document must not contain top-level operator keys", + ); + }); +}); + +describe("M4 replaceDocument validation (rejects before driver)", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects empty filter", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: {}, + replacement: { name: "test" }, + }), + ).rejects.toMatchObject({ code: "MONGO_BULK_OP_FORBIDDEN" }); + }); + + test("rejects non-object replacement", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: "123" }, + replacement: [1, 2, 3] as unknown as EJSONDocument, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects replacement with operator keys", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: "123" }, + replacement: { $set: { a: 1 } }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects changing _id between filter and replacement", async () => { + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: { $oid: "507f1f77bcf86cd799439011" } }, + replacement: { _id: { $oid: "507f1f77bcf86cd799439022" }, name: "New" }, + }), + ).rejects.toMatchObject({ + code: "MONGO_VALIDATION_ERROR", + message: "Cannot change _id of a document", + }); + }); + + test("rejects oversized replacement", async () => { + const huge = "x".repeat(17 * 1024 * 1024); + await expect( + mongoConnector.replaceDocument(dummy, { + collection: "users", + filter: { _id: "123" }, + replacement: { data: huge }, + }), + ).rejects.toMatchObject({ code: "MONGO_LIMIT_EXCEEDED" }); + }); +}); + +describe("M4 createCollection validation", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects invalid collection name", async () => { + await expect( + mongoConnector.createCollection(dummy, { collection: "system.mycoll" }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { collection: "invalid.name" }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects capped collection without size or size < 4096", async () => { + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true } as any, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 2048 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 5000.5 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); + + test("rejects capped collection with max <= 0 or non-integer", async () => { + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 8192, max: 0 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 8192, max: -5 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + + await expect( + mongoConnector.createCollection(dummy, { + collection: "logs", + options: { capped: true, size: 8192, max: 10.5 }, + }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); +}); + +describe("M4 dropCollection validation", () => { + const dummy: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects dropping system collection", async () => { + await expect( + mongoConnector.dropCollection(dummy, { collection: "system.views" }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: "Cannot drop system collection", + }); + + await expect( + mongoConnector.dropCollection(dummy, { collection: "system.profile" }), + ).rejects.toMatchObject({ + code: "MONGO_FORBIDDEN", + message: "Cannot drop system collection", + }); + }); + + test("rejects invalid collection name", async () => { + await expect( + mongoConnector.dropCollection(dummy, { collection: "" }), + ).rejects.toMatchObject({ code: "MONGO_VALIDATION_ERROR" }); + }); +}); + +describe("M4 cache invalidation", () => { + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/cachedb", database: "cachedb" }; + + test("clearCollectionCache and flushConnection invalidate collection metadata and page cache", () => { + mongodbCache.setCollectionStats(cfg, "coll1", { + collection: "coll1", + count: 10, + sizeBytes: 200, + avgObjSize: 20, + }); + mongodbCache.setDocumentPage(cfg, "p1", [{ doc: 1 }]); + mongodbCache.setTableList(cfg, [{ name: "coll1", type: "TABLE", schema: "cachedb" }]); + + expect(mongodbCache.getCollectionStats(cfg, "coll1")).toBeDefined(); + expect(mongodbCache.getDocumentPage(cfg, "p1")).toBeDefined(); + expect(mongodbCache.getTableList(cfg)).toBeDefined(); + + mongodbCache.clearCollectionCache(cfg, "coll1"); + expect(mongodbCache.getCollectionStats(cfg, "coll1")).toBeNull(); + expect(mongodbCache.getDocumentPage(cfg, "p1")).toBeNull(); + expect(mongodbCache.getTableList(cfg)).toBeDefined(); + + mongodbCache.flushConnection(cfg); + expect(mongodbCache.getTableList(cfg)).toBeNull(); + }); +}); + +describe("M4 QueryExecutor routing", () => { + const executor = new QueryExecutor(); + const cfg: MongoConfig = { uri: "mongodb://localhost:27017/dummy", database: "dummy" }; + + test("rejects replaceDocument on non-MongoDB connection", async () => { + await expect( + executor.replaceDocument(cfg, DBType.POSTGRES, { + collection: "c", + filter: { a: 1 }, + replacement: { a: 2 }, + }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); + + test("rejects createCollection on non-MongoDB connection", async () => { + await expect( + executor.createCollection(cfg, DBType.POSTGRES, { collection: "c" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); + + test("rejects dropCollection on non-MongoDB connection", async () => { + await expect( + executor.dropCollection(cfg, DBType.POSTGRES, { collection: "c" }), + ).rejects.toMatchObject({ code: "MONGO_UNSUPPORTED_OPERATION" }); + }); +}); diff --git a/bridge/__tests__/integration/mongoMutations.integration.test.ts b/bridge/__tests__/integration/mongoMutations.integration.test.ts new file mode 100644 index 00000000..826eeaef --- /dev/null +++ b/bridge/__tests__/integration/mongoMutations.integration.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test, beforeAll, afterAll, jest } from "@jest/globals"; +import { MongoClient, ObjectId } from "mongodb"; +import { mongoConnector, closeAllMongoClients } from "../../src/connectors/mongo"; +import type { MongoConfig } from "../../src/types/mongo"; + +const TEST_URI = process.env.MONGO_TEST_URI; +const TEST_DB = "relwave_m4_it"; + +const config: MongoConfig = { + uri: TEST_URI ? `${TEST_URI.replace(/\/$/, "")}/${TEST_DB}` : undefined, + database: TEST_DB, +}; + +let client: MongoClient | null = null; +let connected = false; + +beforeAll(async () => { + jest.setTimeout(60000); + if (!TEST_URI) { + console.warn( + "[mongoMutations.integration] MONGO_TEST_URI not set; skipping M4 integration tests.", + ); + return; + } + try { + client = new MongoClient(config.uri as string); + await client.connect(); + try { + await client.db(TEST_DB).dropDatabase(); + } catch { + // ignore + } + connected = true; + } catch { + connected = false; + console.warn( + `[mongoMutations.integration] MongoDB not reachable at ${TEST_URI}; skipping M4 integration tests.`, + ); + } +}, 60000); + +afterAll(async () => { + if (client) { + try { + await client.db(TEST_DB).dropDatabase(); + await client.close(); + } catch { + // ignore + } + client = null; + } + await closeAllMongoClients(); +}); + +describe("M4 Mutations Lifecycle (Integration)", () => { + const collection = "test_m4_mutations"; + + test("creates a new collection explicitly", async () => { + if (!connected) return; + const res = await mongoConnector.createCollection(config, { collection }); + expect(res.ok).toBe(true); + expect(res.collection).toBe(collection); + + const list = await mongoConnector.listCollections(config); + expect(list.some((c) => c.name === collection)).toBe(true); + }); + + test("creates a capped collection with validated size", async () => { + if (!connected) return; + const cappedColl = "test_m4_capped"; + const res = await mongoConnector.createCollection(config, { + collection: cappedColl, + options: { capped: true, size: 8192, max: 100 }, + }); + expect(res.ok).toBe(true); + expect(res.collection).toBe(cappedColl); + + await mongoConnector.dropCollection(config, { collection: cappedColl }); + }); + + test("replaces a document and updates cache", async () => { + if (!connected) return; + const oid = new ObjectId(); + const originalDoc = { + _id: { $oid: oid.toHexString() }, + title: "Initial Title", + version: { $numberInt: "1" }, + tags: ["alpha", "beta"], + }; + + const inserted = await mongoConnector.insertDocument(config, { + collection, + document: originalDoc, + }); + expect(inserted.insertedId).toEqual({ $oid: oid.toHexString() }); + + const replacement = { + _id: { $oid: oid.toHexString() }, + title: "Replaced Title", + author: "Alice", + active: true, + }; + + const replaceRes = await mongoConnector.replaceDocument(config, { + collection, + filter: { _id: { $oid: oid.toHexString() } }, + replacement, + options: { returnDocument: "after" }, + }); + + expect(replaceRes.matchedCount).toBe(1); + expect(replaceRes.modifiedCount).toBe(1); + expect(replaceRes.updatedDocument?.title).toBe("Replaced Title"); + expect(replaceRes.updatedDocument?.author).toBe("Alice"); + expect(replaceRes.updatedDocument?.version).toBeUndefined(); + expect(replaceRes.updatedDocument?.tags).toBeUndefined(); + + const found = await mongoConnector.findDocuments(config, { + collection, + filter: { _id: { $oid: oid.toHexString() } }, + }); + expect(found.documents).toHaveLength(1); + expect(found.documents[0].title).toBe("Replaced Title"); + expect(found.documents[0].version).toBeUndefined(); + }); + + test("rejects dropping system collection", async () => { + if (!connected) return; + await expect( + mongoConnector.dropCollection(config, { collection: "system.views" }), + ).rejects.toMatchObject({ code: "MONGO_FORBIDDEN" }); + }); + + test("drops collection successfully", async () => { + if (!connected) return; + const dropRes = await mongoConnector.dropCollection(config, { collection }); + expect(dropRes.ok).toBe(true); + expect(dropRes.collection).toBe(collection); + + const list = await mongoConnector.listCollections(config); + expect(list.some((c) => c.name === collection)).toBe(false); + }); +}); diff --git a/bridge/src/connectors/mongo.ts b/bridge/src/connectors/mongo.ts index 1391eec2..5505f532 100644 --- a/bridge/src/connectors/mongo.ts +++ b/bridge/src/connectors/mongo.ts @@ -41,6 +41,12 @@ import { MongoUpdateResult, MongoDeleteRequest, MongoDeleteResult, + MongoReplaceRequest, + MongoReplaceResult, + MongoCreateCollectionRequest, + MongoCreateCollectionResult, + MongoDropCollectionRequest, + MongoDropCollectionResult, MongoAggregationRequest, MongoAggregationResult, MongoSortSpec, @@ -257,6 +263,10 @@ class MongoCacheManager { this.docPageCache.clear(); this.inferredSchemaCache.clear(); } + + flushConnection(cfg: MongoConfig): void { + this.clearForConnection(cfg); + } } export const mongodbCache = new MongoCacheManager(); @@ -489,6 +499,48 @@ function assertCollectionName(collection: unknown): asserts collection is string } } +export function assertNewCollectionName(collection: unknown): asserts collection is string { + if (typeof collection !== "string" || collection.length === 0) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "collection must be a non-empty string"); + } + if (collection.length > 120) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not exceed 120 characters", + ); + } + if (collection.includes("$") || collection.includes("\0")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not contain '$' or null bytes", + ); + } + if (collection.startsWith("system.")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "collection name must not start with 'system.'", + ); + } + const VALID_NAME_REGEX = /^[a-zA-Z_][a-zA-Z0-9_-]*$/; + if (!VALID_NAME_REGEX.test(collection)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + `collection name '${collection}' is invalid; must start with a letter or underscore and contain only alphanumeric, underscores, or hyphens`, + ); + } +} + +export function assertNoOperatorKeys(doc: Record): void { + for (const key of Object.keys(doc)) { + if (key.startsWith("$")) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "replacement document must not contain top-level operator keys", + ); + } + } +} + function assertSize(value: unknown, max: number, what: string): void { if (jsonByteSize(value) > max) { throw new MongoApiError("MONGO_LIMIT_EXCEEDED", `${what} exceeds the ${max}-byte limit`); @@ -1098,6 +1150,86 @@ export class MongoConnector { }; } + async replaceDocument( + config: MongoConfig, + req: MongoReplaceRequest, + ): Promise { + assertCollectionName(req.collection); + assertNonEmptyFilter(req.filter); + if ( + typeof req.replacement !== "object" || + req.replacement === null || + Array.isArray(req.replacement) + ) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "replacement document must be an object"); + } + assertNoOperatorKeys(req.replacement); + assertSize(req.replacement, MAX_DOCUMENT_SIZE_BYTES, "replacement"); + + if (req.replacement._id !== undefined && req.filter._id !== undefined) { + if (canonicalJSON(req.replacement._id) !== canonicalJSON(req.filter._id)) { + throw new MongoApiError("MONGO_VALIDATION_ERROR", "Cannot change _id of a document"); + } + } + + const bsonFilter = serializeMongoFilter(req.filter); + const doc = serializeMongoEJSONDocument(req.replacement); + const upsert = req.options?.upsert ?? false; + const returnDocument = req.options?.returnDocument ?? "after"; + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + + const result = await withMongoTimeout(timeout, async (signal) => { + let updatedDocument: EJSONDocument | undefined; + if (returnDocument === "before") { + const before = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = before ? (mongoBSONtoEJSON(before) as EJSONDocument) : undefined; + } + const replaceOptions = { + maxTimeMS: timeout, + upsert, + }; + const r = await coll.replaceOne(bsonFilter, doc, replaceOptions); + if (returnDocument !== "before") { + try { + const after = await coll.findOne(bsonFilter, { + signal, + maxTimeMS: timeout, + ...PRESERVE_BSON_TYPES, + }); + updatedDocument = after ? (mongoBSONtoEJSON(after) as EJSONDocument) : undefined; + } catch (readErr) { + logger.warn( + { collection: req.collection, err: readErr }, + "mongo replaceDocument follow-up read failed; returning counts only", + ); + } + } + return { r, updatedDocument }; + }); + + const upsertedRaw = result.r.upsertedId; + const upsertedId = + upsertedRaw && Object.keys(upsertedRaw).length > 0 + ? (mongoBSONtoEJSON(Object.values(upsertedRaw)[0]) as EJsonValue) + : upsertedRaw + ? (mongoBSONtoEJSON(upsertedRaw) as EJsonValue) + : undefined; + + mongodbCache.clearCollectionCache(config, req.collection); + return { + collection: req.collection, + matchedCount: result.r.matchedCount, + modifiedCount: result.r.modifiedCount, + upsertedId, + updatedDocument: result.updatedDocument, + }; + } + async deleteDocument(config: MongoConfig, req: MongoDeleteRequest): Promise { assertCollectionName(req.collection); assertNonEmptyFilter(req.filter); @@ -1226,6 +1358,60 @@ export class MongoConnector { return schema; } + + async createCollection( + config: MongoConfig, + req: MongoCreateCollectionRequest, + ): Promise { + assertNewCollectionName(req.collection); + if (req.options?.capped) { + const { size, max } = req.options; + if (typeof size !== "number" || !Number.isInteger(size) || size < 4096) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Capped collection size must be an integer >= 4096 bytes", + ); + } + if (max !== undefined && (typeof max !== "number" || !Number.isInteger(max) || max <= 0)) { + throw new MongoApiError( + "MONGO_VALIDATION_ERROR", + "Capped collection max must be a positive integer", + ); + } + } + const db = selectDb(config); + const timeout = clampTimeout(); + await withMongoTimeout(timeout, () => db.createCollection(req.collection, req.options as any)); + mongodbCache.flushConnection(config); + return { ok: true, collection: req.collection }; + } + + async dropCollection( + config: MongoConfig, + req: MongoDropCollectionRequest, + ): Promise { + if (typeof req.collection === "string" && req.collection.startsWith("system.")) { + throw new MongoApiError("MONGO_FORBIDDEN", "Cannot drop system collection"); + } + assertCollectionName(req.collection); + const coll = selectCollection(config, req.collection); + const timeout = clampTimeout(); + let dropped: boolean = true; + try { + dropped = await withMongoTimeout(timeout, () => coll.drop()); + } catch (err: any) { + if (err?.codeName === "NamespaceNotFound" || err?.code === 26) { + throw new MongoApiError("MONGO_NOT_FOUND", `Collection '${req.collection}' does not exist`); + } + throw err; + } + if (dropped === false) { + throw new MongoApiError("MONGO_NOT_FOUND", `Collection '${req.collection}' does not exist`); + } + mongodbCache.clearCollectionCache(config, req.collection); + mongodbCache.flushConnection(config); + return { ok: true, collection: req.collection }; + } } export const mongoConnector = new MongoConnector(); diff --git a/bridge/src/handlers/queryHandlers.ts b/bridge/src/handlers/queryHandlers.ts index 5f684160..a4296fcb 100644 --- a/bridge/src/handlers/queryHandlers.ts +++ b/bridge/src/handlers/queryHandlers.ts @@ -490,4 +490,65 @@ export class QueryHandlers { this.sendMongoError(id, e); } } + + async handleReplaceDocument(params: any, id: number | string) { + try { + const { dbId, collection, filter, replacement, options } = params || {}; + if (!dbId || !collection || !filter || !replacement) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId, collection, filter, or replacement", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.replaceDocument(conn, dbType, { + collection, + filter, + replacement, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleCreateCollection(params: any, id: number | string) { + try { + const { dbId, collection, options } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.createCollection(conn, dbType, { + collection, + options, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } + + async handleDropCollection(params: any, id: number | string) { + try { + const { dbId, collection } = params || {}; + if (!dbId || !collection) { + return this.rpc.sendError(id, { + code: "MONGO_VALIDATION_ERROR", + message: "Missing dbId or collection", + }); + } + const { conn, dbType } = await this.dbService.getDatabaseConnection(dbId); + const data = await this.queryExecutor.dropCollection(conn, dbType, { + collection, + }); + this.rpc.sendResponse(id, { ok: true, data }); + } catch (e) { + this.sendMongoError(id, e); + } + } } diff --git a/bridge/src/jsonRpcHandler.ts b/bridge/src/jsonRpcHandler.ts index 40009142..71cee12d 100644 --- a/bridge/src/jsonRpcHandler.ts +++ b/bridge/src/jsonRpcHandler.ts @@ -75,6 +75,7 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "query.findDocuments", (p, id) => queryHandlers.handleFetchDocuments(p, id)); rpcRegister(rpc, "query.insertDocument", (p, id) => queryHandlers.handleInsertDocument(p, id)); rpcRegister(rpc, "query.updateDocument", (p, id) => queryHandlers.handleUpdateDocument(p, id)); + rpcRegister(rpc, "query.replaceDocument", (p, id) => queryHandlers.handleReplaceDocument(p, id)); rpcRegister(rpc, "query.deleteDocument", (p, id) => queryHandlers.handleDeleteDocument(p, id)); rpcRegister(rpc, "query.runAggregation", (p, id) => queryHandlers.handleRunAggregation(p, id)); @@ -96,6 +97,8 @@ export function registerDbHandlers(rpc: Rpc, logger: Logger, sessions: SessionMa rpcRegister(rpc, "db.listSchemas", (p, id) => databaseHandlers.handleListSchemas(p, id)); rpcRegister(rpc, "db.listCollections", (p, id) => queryHandlers.handleListCollections(p, id)); rpcRegister(rpc, "db.getInferredSchema", (p, id) => queryHandlers.handleGetInferredSchema(p, id)); + rpcRegister(rpc, "db.createCollection", (p, id) => queryHandlers.handleCreateCollection(p, id)); + rpcRegister(rpc, "db.dropCollection", (p, id) => queryHandlers.handleDropCollection(p, id)); // ========================================== // MIGRATION HANDLERS diff --git a/bridge/src/services/queryExecutor.ts b/bridge/src/services/queryExecutor.ts index 7e7530ea..705d8df6 100644 --- a/bridge/src/services/queryExecutor.ts +++ b/bridge/src/services/queryExecutor.ts @@ -15,6 +15,12 @@ import { MongoUpdateResult, MongoDeleteRequest, MongoDeleteResult, + MongoReplaceRequest, + MongoReplaceResult, + MongoCreateCollectionRequest, + MongoCreateCollectionResult, + MongoDropCollectionRequest, + MongoDropCollectionResult, MongoAggregationRequest, MongoAggregationResult, } from "../types/mongo"; @@ -231,6 +237,48 @@ export class QueryExecutor { return mongoConnector.updateDocument(conn, req); } + async replaceDocument( + conn: MongoConfig, + dbType: DBType, + req: MongoReplaceRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Document operations require a MongoDB connection", + ); + } + return mongoConnector.replaceDocument(conn, req); + } + + async createCollection( + conn: MongoConfig, + dbType: DBType, + req: MongoCreateCollectionRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Collection operations require a MongoDB connection", + ); + } + return mongoConnector.createCollection(conn, req); + } + + async dropCollection( + conn: MongoConfig, + dbType: DBType, + req: MongoDropCollectionRequest, + ): Promise { + if (dbType !== DBType.MONGODB) { + throw new MongoApiError( + "MONGO_UNSUPPORTED_OPERATION", + "Collection operations require a MongoDB connection", + ); + } + return mongoConnector.dropCollection(conn, req); + } + async deleteDocument( conn: MongoConfig, dbType: DBType, diff --git a/bridge/src/types/mongo.ts b/bridge/src/types/mongo.ts index 73140679..73be1089 100644 --- a/bridge/src/types/mongo.ts +++ b/bridge/src/types/mongo.ts @@ -266,6 +266,55 @@ export interface MongoDeleteResult { deletedCount: number; } +// --- replaceDocument (spec §5.2.1) --- + +export interface MongoReplaceOptions { + upsert?: boolean; + returnDocument?: "before" | "after"; +} + +export interface MongoReplaceRequest { + collection: string; + filter: EJSONDocument; + replacement: EJSONDocument; + options?: MongoReplaceOptions; +} + +export interface MongoReplaceResult { + collection: string; + matchedCount: number; + modifiedCount: number; + upsertedId?: EJsonValue; + updatedDocument?: EJSONDocument; +} + +// --- collection lifecycle (spec §5.2.2 / §5.2.3) --- + +export interface MongoCreateCollectionOptions { + capped?: boolean; + size?: number; // bytes (min 4096) + max?: number; // max documents (positive integer) +} + +export interface MongoCreateCollectionRequest { + collection: string; + options?: MongoCreateCollectionOptions; +} + +export interface MongoCreateCollectionResult { + ok: boolean; + collection: string; +} + +export interface MongoDropCollectionRequest { + collection: string; +} + +export interface MongoDropCollectionResult { + ok: boolean; + collection: string; +} + // --- runAggregation (spec §5.6) --- export interface MongoAggregationOptions { diff --git a/src/features/mongodb/__tests__/MongoMutationValidation.test.ts b/src/features/mongodb/__tests__/MongoMutationValidation.test.ts new file mode 100644 index 00000000..887f269f --- /dev/null +++ b/src/features/mongodb/__tests__/MongoMutationValidation.test.ts @@ -0,0 +1,86 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + validateCollectionName, + validateCappedOptions, + validateJsonDocument, +} from "../utils/validation"; + +describe("validateCollectionName", () => { + it("accepts valid collection names", () => { + assert.equal(validateCollectionName("users").valid, true); + assert.equal(validateCollectionName("user_accounts").valid, true); + assert.equal(validateCollectionName("orders-2026").valid, true); + assert.equal(validateCollectionName("_audit_log").valid, true); + assert.equal(validateCollectionName("a".repeat(120)).valid, true); + }); + + it("rejects empty or whitespace-only names", () => { + assert.equal(validateCollectionName("").valid, false); + assert.equal(validateCollectionName(" ").valid, false); + }); + + it("rejects names longer than 120 chars", () => { + assert.equal(validateCollectionName("a".repeat(121)).valid, false); + }); + + it("rejects names containing '$' or null bytes", () => { + assert.equal(validateCollectionName("my$coll").valid, false); + assert.equal(validateCollectionName("coll\0test").valid, false); + }); + + it("rejects names starting with 'system.'", () => { + assert.equal(validateCollectionName("system.views").valid, false); + assert.equal(validateCollectionName("system.profile").valid, false); + }); + + it("rejects invalid characters and leading numbers", () => { + assert.equal(validateCollectionName("123orders").valid, false); + assert.equal(validateCollectionName("-orders").valid, false); + assert.equal(validateCollectionName("my.coll").valid, false); + assert.equal(validateCollectionName("my coll").valid, false); + }); +}); + +describe("validateCappedOptions", () => { + it("passes when not capped", () => { + assert.equal(validateCappedOptions(undefined).valid, true); + assert.equal(validateCappedOptions({ capped: false }).valid, true); + }); + + it("requires size >= 4096 integer when capped", () => { + assert.equal(validateCappedOptions({ capped: true } as any).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 4095 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 5000.5 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 4096 }).valid, true); + assert.equal(validateCappedOptions({ capped: true, size: 8192 }).valid, true); + }); + + it("requires max to be a positive integer if provided", () => { + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: 0 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: -1 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: 10.5 }).valid, false); + assert.equal(validateCappedOptions({ capped: true, size: 8192, max: 100 }).valid, true); + }); +}); + +describe("validateJsonDocument", () => { + it("accepts valid JSON objects", () => { + const res = validateJsonDocument('{"name": "Alice", "age": 30}'); + assert.equal(res.valid, true); + assert.deepEqual(res.doc, { name: "Alice", age: 30 }); + }); + + it("rejects invalid JSON syntax", () => { + const res = validateJsonDocument('{name: "Alice"}'); + assert.equal(res.valid, false); + assert.match(res.error || "", /JSON|position/i); + }); + + it("rejects JSON arrays or primitives", () => { + assert.equal(validateJsonDocument('[1, 2, 3]').valid, false); + assert.equal(validateJsonDocument('"hello"').valid, false); + assert.equal(validateJsonDocument('123').valid, false); + assert.equal(validateJsonDocument('null').valid, false); + }); +}); diff --git a/src/features/mongodb/__tests__/useDocumentDiff.test.ts b/src/features/mongodb/__tests__/useDocumentDiff.test.ts new file mode 100644 index 00000000..283ffb2d --- /dev/null +++ b/src/features/mongodb/__tests__/useDocumentDiff.test.ts @@ -0,0 +1,224 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { computeDocumentDiff } from "../hooks/useDocumentDiff"; + +describe("computeDocumentDiff (M4 Spec §4.2.1)", () => { + it("detects no-op when documents are identical", () => { + const doc = { _id: { $oid: "507f1f77bcf86cd799439011" }, name: "Acme", count: 10 }; + const res = computeDocumentDiff(doc, { ...doc }); + assert.equal(res.hasChanges, false); + assert.equal(res.updateDoc, undefined); + }); + + it("handles top-level field addition and modification", () => { + const orig = { _id: "1", name: "Alice" }; + const edited = { _id: "1", name: "Alice Smith", role: "admin" }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { name: "Alice Smith", role: "admin" }, + }); + }); + + it("emits recursive dot-notation for nested property changes", () => { + const orig = { + _id: "1", + address: { city: "Seattle", zip: "98101", country: "USA" }, + }; + const edited = { + _id: "1", + address: { city: "Boston", zip: "98101", country: "USA" }, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { "address.city": "Boston" }, + }); + }); + + it("emits dot-notation unset for nested deletions", () => { + const orig = { + _id: "1", + address: { city: "Seattle", zip: "98101" }, + }; + const edited = { + _id: "1", + address: { city: "Seattle" }, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $unset: { "address.zip": "" }, + }); + }); + + it("emits single parent unset on complete nested-object removal", () => { + const orig = { + _id: "1", + address: { city: "Seattle", zip: "98101" }, + active: true, + }; + const edited = { + _id: "1", + active: true, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $unset: { address: "" }, + }); + }); + + it("handles object-to-scalar transition without child unsets", () => { + const orig = { + _id: "1", + metadata: { author: "Alice", revision: 1 }, + }; + const edited = { + _id: "1", + metadata: "archived", + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { metadata: "archived" }, + }); + }); + + it("handles scalar-to-object transition", () => { + const orig = { + _id: "1", + metadata: "none", + }; + const edited = { + _id: "1", + metadata: { author: "Alice", revision: 1 }, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { metadata: { author: "Alice", revision: 1 } }, + }); + }); + + it("handles object-to-array transition", () => { + const orig = { + _id: "1", + items: { a: 1, b: 2 }, + }; + const edited = { + _id: "1", + items: [1, 2, 3], + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { items: [1, 2, 3] }, + }); + }); + + it("treats arrays as atomic units on modification", () => { + const orig = { + _id: "1", + tags: ["alpha", "beta", "gamma"], + }; + const edited = { + _id: "1", + tags: ["alpha", "delta"], + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { tags: ["alpha", "delta"] }, + }); + }); + + it("distinguishes explicit null ($set) from omitted key ($unset)", () => { + const orig = { + _id: "1", + fieldA: "valA", + fieldB: "valB", + }; + const edited = { + _id: "1", + fieldA: null, // explicit null + // fieldB omitted + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { fieldA: null }, + $unset: { fieldB: "" }, + }); + }); + + it("preserves empty object {} as $set", () => { + const orig = { + _id: "1", + metadata: { temp: 1 }, + }; + const edited = { + _id: "1", + metadata: {}, + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { metadata: {} }, + }); + }); + + it("preserves BSON/EJSON tagged scalar types as atomic units", () => { + const orig = { + _id: { $oid: "507f1f77bcf86cd799439011" }, + createdAt: { $date: "2026-01-01T00:00:00.000Z" }, + amount: { $numberDecimal: "99.95" }, + }; + const edited = { + _id: { $oid: "507f1f77bcf86cd799439011" }, + createdAt: { $date: "2026-02-01T00:00:00.000Z" }, + amount: { $numberDecimal: "99.95" }, // unchanged + }; + const res = computeDocumentDiff(orig, edited); + assert.equal(res.hasChanges, true); + assert.deepEqual(res.updateDoc, { + $set: { createdAt: { $date: "2026-02-01T00:00:00.000Z" } }, + }); + }); + + it("throws when _id is modified", () => { + const orig = { _id: { $oid: "507f1f77bcf86cd799439011" }, name: "A" }; + const edited = { _id: { $oid: "507f1f77bcf86cd799439022" }, name: "A" }; + assert.throws( + () => computeDocumentDiff(orig, edited), + /ID_MODIFICATION_PROHIBITED/, + ); + }); + + it("throws when _id is deleted", () => { + const orig = { _id: "1", name: "A" }; + const edited = { name: "A" }; + assert.throws( + () => computeDocumentDiff(orig, edited), + /ID_MODIFICATION_PROHIBITED/, + ); + }); + + it("rejects keys containing '$'", () => { + const orig = { _id: "1" }; + const edited = { _id: "1", "$badKey": 123 }; + assert.throws( + () => computeDocumentDiff(orig, edited), + /INVALID_FIELD_PATH.*'\$badKey'/, + ); + }); + + it("rejects keys containing invalid dot notation", () => { + const orig = { _id: "1" }; + const edited = { _id: "1", ".badKey": 123 }; + assert.throws( + () => computeDocumentDiff(orig, edited), + /INVALID_FIELD_PATH.*'\.badKey'/, + ); + }); +}); diff --git a/src/features/mongodb/components/MongoCollectionsSidebar.tsx b/src/features/mongodb/components/MongoCollectionsSidebar.tsx index f2e77a77..6ab4bc0a 100644 --- a/src/features/mongodb/components/MongoCollectionsSidebar.tsx +++ b/src/features/mongodb/components/MongoCollectionsSidebar.tsx @@ -3,7 +3,7 @@ import { MongoCollectionInfo } from "../types"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; -import { Layers, Search, RefreshCw, Eye } from "lucide-react"; +import { Layers, Search, RefreshCw, Eye, Plus, Trash2 } from "lucide-react"; export function MongoCollectionsSidebar({ collections, @@ -11,12 +11,16 @@ export function MongoCollectionsSidebar({ onSelectCollection, isLoading, onRefresh, + onCreateCollection, + onDropCollection, }: { collections: MongoCollectionInfo[]; selectedCollection: string | null; onSelectCollection: (name: string) => void; isLoading: boolean; onRefresh: () => void; + onCreateCollection?: () => void; + onDropCollection?: (name: string) => void; }) { const [search, setSearch] = useState(""); @@ -35,15 +39,28 @@ export function MongoCollectionsSidebar({ Collections ({collections.length}) - +
+ {onCreateCollection && ( + + )} + +
@@ -71,26 +88,47 @@ export function MongoCollectionsSidebar({ ) : ( filtered.map((col) => { const isSelected = col.name === selectedCollection; + const isSystem = col.name.startsWith("system."); return ( - + +
+ {col.type === "view" && ( + + view + + )} + + {!isSystem && onDropCollection && ( + + )}
- {col.type === "view" && ( - - view - - )} - +
); }) )} diff --git a/src/features/mongodb/components/MongoCreateCollectionDialog.tsx b/src/features/mongodb/components/MongoCreateCollectionDialog.tsx new file mode 100644 index 00000000..cafd30e1 --- /dev/null +++ b/src/features/mongodb/components/MongoCreateCollectionDialog.tsx @@ -0,0 +1,226 @@ +import React, { useState, useEffect } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Loader2, Plus, AlertCircle } from "lucide-react"; +import { toast } from "sonner"; +import { useCreateMongoCollection } from "../hooks/useMongoMutations"; +import { validateCollectionName, validateCappedOptions } from "../utils/validation"; + +interface MongoCreateCollectionDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + onCollectionCreated: (name: string) => void; +} + +export function MongoCreateCollectionDialog({ + open, + onOpenChange, + dbId, + onCollectionCreated, +}: MongoCreateCollectionDialogProps) { + const [collectionName, setCollectionName] = useState(""); + const [isCapped, setIsCapped] = useState(false); + const [cappedSize, setCappedSize] = useState("8192"); + const [cappedMax, setCappedMax] = useState(""); + const [error, setError] = useState(null); + + const createMutation = useCreateMongoCollection(dbId); + + useEffect(() => { + if (open) { + setCollectionName(""); + setIsCapped(false); + setCappedSize("8192"); + setCappedMax(""); + setError(null); + } + }, [open]); + + const handleNameChange = (val: string) => { + setCollectionName(val); + const res = validateCollectionName(val); + if (!res.valid) { + setError(res.error || "Invalid collection name"); + } else { + setError(null); + } + }; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + const nameCheck = validateCollectionName(collectionName); + if (!nameCheck.valid) { + setError(nameCheck.error || "Invalid collection name"); + return; + } + + let options: { capped?: boolean; size?: number; max?: number } | undefined = undefined; + if (isCapped) { + const sizeNum = parseInt(cappedSize, 10); + const maxNum = cappedMax.trim() ? parseInt(cappedMax, 10) : undefined; + options = { + capped: true, + size: sizeNum, + max: maxNum, + }; + const cappedCheck = validateCappedOptions(options); + if (!cappedCheck.valid) { + setError(cappedCheck.error || "Invalid capped options"); + return; + } + } + + try { + await createMutation.mutateAsync({ + collection: collectionName.trim(), + options, + }); + toast.success(`Collection "${collectionName.trim()}" created successfully`); + onCollectionCreated(collectionName.trim()); + onOpenChange(false); + } catch (err: any) { + setError(err.message || "Failed to create collection"); + toast.error("Collection creation failed", { description: err.message }); + } + }; + + return ( + + +
+ + + + Create Collection + + + Create a new MongoDB collection with optional capped storage parameters. + + + +
+ {/* Collection Name */} +
+ + handleNameChange(e.target.value)} + placeholder="e.g. orders, user_events" + className="font-mono text-xs h-9" + autoFocus + /> +

+ Letters, numbers, underscores, and hyphens (max 120 chars). +

+
+ + {/* Capped Collection Toggle */} +
+
+ +

+ Fixed-size collection that automatically overwrites oldest documents. +

+
+ +
+ + {/* Capped Options */} + {isCapped && ( +
+
+ + setCappedSize(e.target.value)} + className="font-mono text-xs h-8" + /> +

+ Must be at least 4096 bytes (4 KB). +

+
+ +
+ + setCappedMax(e.target.value)} + placeholder="Unlimited" + className="font-mono text-xs h-8" + /> +

+ Positive integer cap on document count. +

+
+
+ )} + + {error && ( +
+ + {error} +
+ )} +
+ + + + + +
+
+
+ ); +} diff --git a/src/features/mongodb/components/MongoDataViewPanel.tsx b/src/features/mongodb/components/MongoDataViewPanel.tsx index b6256ce5..8982f2b8 100644 --- a/src/features/mongodb/components/MongoDataViewPanel.tsx +++ b/src/features/mongodb/components/MongoDataViewPanel.tsx @@ -3,6 +3,11 @@ import { useMongoDatabaseDetails } from "../hooks/useMongoDatabaseDetails"; import { MongoCollectionsSidebar } from "./MongoCollectionsSidebar"; import { MongoDocumentViewer } from "./MongoDocumentViewer"; import { MongoSchemaPanel } from "./MongoSchemaPanel"; +import { MongoInsertDocumentDialog } from "./MongoInsertDocumentDialog"; +import { MongoEditDocumentDialog } from "./MongoEditDocumentDialog"; +import { MongoDeleteDocumentDialog } from "./MongoDeleteDocumentDialog"; +import { MongoCreateCollectionDialog } from "./MongoCreateCollectionDialog"; +import { MongoDropCollectionDialog } from "./MongoDropCollectionDialog"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { FileText, GitBranch, Database } from "lucide-react"; @@ -32,6 +37,18 @@ export function MongoDataViewPanel({ dbId, databaseName }: { dbId: string; datab isLoadingDocuments, refetchDocuments, refetchCollections, + isInsertDialogOpen, + setIsInsertDialogOpen, + editingDocument, + setEditingDocument, + deletingDocument, + setDeletingDocument, + isCreateCollectionOpen, + setIsCreateCollectionOpen, + droppingCollection, + setDroppingCollection, + handleCollectionCreated, + handleCollectionDropped, } = useMongoDatabaseDetails(dbId); return ( @@ -89,6 +106,8 @@ export function MongoDataViewPanel({ dbId, databaseName }: { dbId: string; datab onSelectCollection={setSelectedCollection} isLoading={collectionsLoading} onRefresh={refetchCollections} + onCreateCollection={() => setIsCreateCollectionOpen(true)} + onDropCollection={(name) => setDroppingCollection(name)} />
@@ -116,12 +135,60 @@ export function MongoDataViewPanel({ dbId, databaseName }: { dbId: string; datab onResetFilter={resetFilter} isFilterApplied={!!appliedFilter} onRefresh={refetchDocuments} + onInsertDocument={() => setIsInsertDialogOpen(true)} + onEditDocument={(doc) => setEditingDocument(doc)} + onDeleteDocument={(doc) => setDeletingDocument(doc)} /> ) : ( )}
+ + {/* M4 Mutation Modals */} + {selectedCollection && ( + <> + + !open && setEditingDocument(null)} + dbId={dbId} + collection={selectedCollection} + document={editingDocument} + onRefreshView={refetchDocuments} + /> + !open && setDeletingDocument(null)} + dbId={dbId} + collection={selectedCollection} + document={deletingDocument} + onRefreshView={refetchDocuments} + /> + + )} + + + + {droppingCollection && ( + !open && setDroppingCollection(null)} + dbId={dbId} + collection={droppingCollection} + onCollectionDropped={handleCollectionDropped} + /> + )} ); } diff --git a/src/features/mongodb/components/MongoDeleteDocumentDialog.tsx b/src/features/mongodb/components/MongoDeleteDocumentDialog.tsx new file mode 100644 index 00000000..89b2b247 --- /dev/null +++ b/src/features/mongodb/components/MongoDeleteDocumentDialog.tsx @@ -0,0 +1,150 @@ +import React, { useMemo } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Loader2, Trash2, AlertTriangle } from "lucide-react"; +import { toast } from "sonner"; +import { EJSONDocument } from "../types"; +import { useDeleteMongoDocument } from "../hooks/useMongoMutations"; + +interface MongoDeleteDocumentDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + collection: string; + document: EJSONDocument | null; + onRefreshView: () => void; +} + +export function MongoDeleteDocumentDialog({ + open, + onOpenChange, + dbId, + collection, + document, + onRefreshView, +}: MongoDeleteDocumentDialogProps) { + const deleteMutation = useDeleteMongoDocument(dbId); + + const idFormatted = useMemo(() => { + if (!document?._id) return "unknown"; + if (typeof document._id === "object" && "$oid" in document._id) { + return document._id.$oid; + } + return String(document._id); + }, [document]); + + const previewFields = useMemo(() => { + if (!document) return []; + return Object.entries(document) + .filter(([k]) => k !== "_id") + .slice(0, 4) + .map(([k, v]) => ({ + key: k, + val: typeof v === "object" ? JSON.stringify(v) : String(v), + })); + }, [document]); + + const handleDelete = async () => { + if (!document?._id) return; + try { + const res = await deleteMutation.mutateAsync({ + collection, + filter: { _id: document._id }, + options: { multi: false }, + }); + + if (res.deletedCount === 0) { + toast.warning("Document not found", { + description: "The document may have already been deleted concurrently.", + }); + } else { + toast.success("Document deleted successfully"); + } + + onRefreshView(); + onOpenChange(false); + } catch (err: any) { + toast.error("Delete failed", { description: err.message }); + } + }; + + return ( + + + + + + Delete Document + + + Are you sure you want to delete this document from{" "} + {collection}? + This action cannot be undone. + + + +
+ {/* Target details */} +
+
+ _id: + {idFormatted} +
+ + {previewFields.length > 0 && ( +
+ {previewFields.map(({ key, val }) => ( +
+ {key}: + {val} +
+ ))} +
+ )} +
+ +
+ Filter: {"{ _id: " + idFormatted + " }"} +
+
+ + + + + +
+
+ ); +} diff --git a/src/features/mongodb/components/MongoDocumentViewer.tsx b/src/features/mongodb/components/MongoDocumentViewer.tsx index a50785fe..5843f30f 100644 --- a/src/features/mongodb/components/MongoDocumentViewer.tsx +++ b/src/features/mongodb/components/MongoDocumentViewer.tsx @@ -20,6 +20,7 @@ import { ChevronsRight, RefreshCw, Database, + Plus, } from "lucide-react"; import { toast } from "sonner"; @@ -42,6 +43,9 @@ export function MongoDocumentViewer({ onResetFilter, isFilterApplied, onRefresh, + onInsertDocument, + onEditDocument, + onDeleteDocument, }: { collection: string; documents: EJSONDocument[]; @@ -61,6 +65,9 @@ export function MongoDocumentViewer({ onResetFilter: () => void; isFilterApplied: boolean; onRefresh: () => void; + onInsertDocument?: () => void; + onEditDocument?: (doc: EJSONDocument) => void; + onDeleteDocument?: (doc: EJSONDocument) => void; }) { const [copiedAll, setCopiedAll] = useState(false); @@ -101,6 +108,18 @@ export function MongoDocumentViewer({ />
+ {onInsertDocument && ( + + )} + + + + + + ); +} diff --git a/src/features/mongodb/components/MongoEditDocumentDialog.tsx b/src/features/mongodb/components/MongoEditDocumentDialog.tsx new file mode 100644 index 00000000..1da43340 --- /dev/null +++ b/src/features/mongodb/components/MongoEditDocumentDialog.tsx @@ -0,0 +1,321 @@ +import React, { useState, useEffect, useMemo } from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Textarea } from "@/components/ui/textarea"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + Loader2, + Edit, + FileCode, + Check, + AlertCircle, + AlertTriangle, + Lock, +} from "lucide-react"; +import { toast } from "sonner"; +import { EJSONDocument } from "../types"; +import { useUpdateMongoDocument, useReplaceMongoDocument } from "../hooks/useMongoMutations"; +import { computeDocumentDiff } from "../hooks/useDocumentDiff"; +import { validateJsonDocument } from "../utils/validation"; + +interface MongoEditDocumentDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + dbId: string; + collection: string; + document: EJSONDocument | null; + onRefreshView: () => void; +} + +export function MongoEditDocumentDialog({ + open, + onOpenChange, + dbId, + collection, + document, + onRefreshView, +}: MongoEditDocumentDialogProps) { + const [mode, setMode] = useState<"patch" | "replace">("patch"); + const [jsonText, setJsonText] = useState(""); + const [error, setError] = useState(null); + + const updateMutation = useUpdateMongoDocument(dbId); + const replaceMutation = useReplaceMongoDocument(dbId); + + useEffect(() => { + if (open && document) { + setJsonText(JSON.stringify(document, null, 2)); + setError(null); + setMode("patch"); + } + }, [open, document]); + + const idFormatted = useMemo(() => { + if (!document?._id) return "unknown"; + if (typeof document._id === "object" && "$oid" in document._id) { + return document._id.$oid; + } + return String(document._id); + }, [document]); + + const parsedDoc = useMemo(() => { + try { + const p = JSON.parse(jsonText); + return typeof p === "object" && p !== null && !Array.isArray(p) ? p : null; + } catch { + return null; + } + }, [jsonText]); + + const diffResult = useMemo(() => { + if (!document || !parsedDoc || mode !== "patch") return null; + try { + return computeDocumentDiff(document, parsedDoc); + } catch (err: any) { + return { error: err.message }; + } + }, [document, parsedDoc, mode]); + + const handleTextChange = (text: string) => { + setJsonText(text); + const val = validateJsonDocument(text); + if (!val.valid) { + setError(val.error || "Invalid JSON"); + return; + } + if (document?._id !== undefined) { + const parsed = val.doc; + if (parsed && JSON.stringify(parsed._id) !== JSON.stringify(document._id)) { + setError("_id is immutable and cannot be modified"); + return; + } + } + setError(null); + }; + + const handleFormat = () => { + try { + const parsed = JSON.parse(jsonText); + setJsonText(JSON.stringify(parsed, null, 2)); + setError(null); + } catch (e: any) { + setError(e.message || "Cannot format invalid JSON"); + } + }; + + const isSaving = updateMutation.isPending || replaceMutation.isPending; + + const handleSubmit = async () => { + if (!document) return; + const val = validateJsonDocument(jsonText); + if (!val.valid || !val.doc) { + setError(val.error || "Please enter valid JSON"); + return; + } + + const editedDoc = val.doc; + + try { + if (mode === "patch") { + const diff = computeDocumentDiff(document, editedDoc); + if (!diff.hasChanges || !diff.updateDoc) { + toast.info("No changes made", { description: "The document already contained the specified values." }); + onOpenChange(false); + return; + } + + const res = await updateMutation.mutateAsync({ + collection, + filter: { _id: document._id }, + update: diff.updateDoc, + }); + + if (res.matchedCount === 0) { + toast.error("Document not found", { + description: "The target document no longer exists. It may have been deleted concurrently.", + }); + onRefreshView(); + onOpenChange(false); + return; + } + + if (res.modifiedCount === 0) { + toast.info("No changes made", { + description: "The document already contained the specified values.", + }); + } else { + toast.success("Document updated successfully"); + } + onOpenChange(false); + } else { + // Mode B: Full Document Replacement + const res = await replaceMutation.mutateAsync({ + collection, + filter: { _id: document._id }, + replacement: editedDoc, + }); + + if (res.matchedCount === 0) { + toast.error("Document not found", { + description: "The target document no longer exists. It may have been deleted concurrently.", + }); + onRefreshView(); + onOpenChange(false); + return; + } + + if (res.modifiedCount === 0) { + toast.info("No changes made", { + description: "The replacement was identical to current document.", + }); + } else { + toast.success("Document replaced successfully"); + } + onOpenChange(false); + } + } catch (err: any) { + setError(err.message || "Failed to update document"); + toast.error("Update failed", { description: err.message }); + } + }; + + return ( + + + +
+ + + Edit Document in {collection} + +
+ + + _id: + + {idFormatted} + + +
+ + {/* Mode Selector and Controls */} +
+ setMode(val as "patch" | "replace")}> + + + Field Patch Mode (Safe) + + + Full Replacement + + + + + +
+ + {/* Warning banner in Replacement Mode */} + {mode === "replace" && ( +
+ + + Warning: Full Replacement mode replaces the entire document. + Fields omitted from this editor will be permanently deleted. + +
+ )} + + {/* Editor Area */} +
+