diff --git a/README.md b/README.md index 6a20267e..10b055a2 100644 --- a/README.md +++ b/README.md @@ -404,13 +404,16 @@ memories. Prefixes stay opt-in by default. Opt-in background pass that finds **related** memories (via `dreaming.relatedThreshold`, default `0.75`, independent of `deduplicationSimilarityThreshold`), asks the configured -`opencodeProvider` / `opencodeModel` to `merge`, `update`, `supersede`, `keep`, or `flag` +`opencodeProvider` / `opencodeModel` (with `memoryModel` / `memoryApiUrl` as fallback, +same channel order as auto-capture) to `merge`, `update`, `supersede`, `keep`, or `flag` contradictions, and soft-deletes superseded rows into recoverable trash. Pinned memories are never touched. Set `dreaming.crossProject: true` (or `memory.defaultScope: "all-projects"`) to group across project tags; user-scope stays separate. Configure under -`dreaming` in `opencode-mem.jsonc` (`enabled` default `false`). The web UI exposes -**Trash** (restore) and **Dream** (manual run). Soft-deleted rows are hard-purged after -`dreaming.trashRetentionDays` during normal cleanup. +`dreaming` in `opencode-mem.jsonc`. `dreaming.enabled` (default `false`) gates only the +scheduled background run — the web UI **Dream** button and `POST /api/dreaming/run` still +work when it is false (manual-only mode). Soft-deleted rows are hard-purged after +`dreaming.trashRetentionDays` during normal cleanup. The web UI also exposes **Trash** +(restore). ### Choosing / configuring embeddings diff --git a/src/config.ts b/src/config.ts index 40280806..12ab4de2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -257,6 +257,8 @@ interface OpenCodeMemConfig { /** * Opt-in background memory consolidation ("dreaming"): merge near-duplicate * memories via the configured LLM and soft-delete superseded rows into trash. + * `enabled` controls only the scheduled background pass; the web UI / API + * manual trigger still runs when false. */ dreaming?: { enabled?: boolean; @@ -941,6 +943,8 @@ const CONFIG_TEMPLATE = `{ // Opt-in background consolidation: merge/update/supersede related memories // via the configured LLM and soft-delete superseded rows into trash. + // "enabled" gates only the scheduler; the Dream button / POST /api/dreaming/run + // still work when false (manual-only mode). "dreaming": { "enabled": false, "intervalHours": 24, diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 37800fdb..1e4ef7f8 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -1057,12 +1057,8 @@ export async function handleRunDreaming( > > { try { - if (!CONFIG.dreaming.enabled) { - return { - success: false, - error: "Dreaming is disabled. Set dreaming.enabled to true in opencode-mem.jsonc.", - }; - } + // Manual runs are always allowed. `dreaming.enabled` only gates the + // background scheduler (see DreamingService.shouldRun / maybeRun). const { dreamingService } = await import("./dreaming-service.js"); const result = await dreamingService.run(directory); return { success: true, data: result }; diff --git a/src/services/dreaming-service.ts b/src/services/dreaming-service.ts index aedce012..68d9731a 100644 --- a/src/services/dreaming-service.ts +++ b/src/services/dreaming-service.ts @@ -127,6 +127,172 @@ const dreamingActionSchema = z.object({ reason: z.string().optional(), }); +type DreamingAction = z.infer; +type DreamingChannel = "opencode" | "external"; + +const DREAMING_SYSTEM_PROMPT = + "You are a careful memory consolidation assistant. Return only structured JSON."; + +const dreamingToolSchema = { + type: "function" as const, + function: { + name: "consolidate_memories", + description: "Decide how to consolidate a related memory group", + parameters: { + type: "object", + properties: { + action: { + type: "string", + enum: ["merge", "update", "supersede", "keep", "flag"], + }, + content: { type: "string" }, + keepId: { type: "string" }, + dropIds: { type: "array", items: { type: "string" } }, + reason: { type: "string" }, + }, + required: ["action"], + }, + }, +}; + +function isOpencodeDreamingConfigured(): boolean { + return Boolean(CONFIG.opencodeProvider && CONFIG.opencodeModel && getV2Client()); +} + +function isExternalDreamingConfigured(): boolean { + return Boolean(CONFIG.memoryModel && CONFIG.memoryApiUrl); +} + +function isDreamingChannelConfigured(channel: DreamingChannel): boolean { + return channel === "opencode" ? isOpencodeDreamingConfigured() : isExternalDreamingConfigured(); +} + +function dreamingChannelOrder(): DreamingChannel[] { + const order = CONFIG.autoCaptureProviderOrder ?? "opencode-first"; + return order === "external-first" ? ["external", "opencode"] : ["opencode", "external"]; +} + +function isEmptyStructuredOutputError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /no structured output/i.test(message); +} + +function parseDreamingAction(data: unknown): DreamingAction { + const parsed = dreamingActionSchema.safeParse(data); + if (!parsed.success) { + throw new Error(`dreaming: invalid consolidation decision (${parsed.error.message})`); + } + return parsed.data; +} + +function channelLabel(channel: DreamingChannel): string { + if (channel === "opencode") { + const variant = CONFIG.opencodeVariant ? ` variant=${CONFIG.opencodeVariant}` : ""; + return `opencode ${CONFIG.opencodeProvider}/${CONFIG.opencodeModel}${variant}`; + } + return `external ${CONFIG.memoryProvider}/${CONFIG.memoryModel}`; +} + +async function decideViaOpencode( + group: DreamingNearDupGroup, + directory: string | undefined, + omitVariant: boolean +): Promise { + const { generateStructuredOutput } = await import("./ai/opencode-provider.js"); + const v2Client = getV2Client(); + if (!v2Client || !CONFIG.opencodeProvider || !CONFIG.opencodeModel) { + throw new Error("dreaming_provider_unavailable"); + } + + const useVariant = !omitVariant && Boolean(CONFIG.opencodeVariant); + return generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + ...(useVariant ? { variant: CONFIG.opencodeVariant } : {}), + systemPrompt: DREAMING_SYSTEM_PROMPT, + userPrompt: buildMergePrompt(group), + schema: dreamingActionSchema, + directory, + }); +} + +async function decideViaExternal(group: DreamingNearDupGroup): Promise { + if (!CONFIG.memoryModel || !CONFIG.memoryApiUrl) { + throw new Error("dreaming_provider_unavailable"); + } + + const { AIProviderFactory } = await import("./ai/ai-provider-factory.js"); + const { buildMemoryProviderConfig } = await import("./ai/provider-config.js"); + const providerConfig = buildMemoryProviderConfig(CONFIG, { + maxIterations: 1, + iterationTimeout: 60_000, + }); + const provider = AIProviderFactory.createProvider(CONFIG.memoryProvider, providerConfig); + const sessionId = `dreaming-${group.representative.id}`; + const result = await provider.executeToolCall( + DREAMING_SYSTEM_PROMPT, + `${buildMergePrompt(group)}\n\nCall the consolidate_memories tool with your decision.`, + dreamingToolSchema, + sessionId + ); + + if (!result.success || !result.data) { + throw new Error(result.error || "dreaming: external provider returned no decision"); + } + return parseDreamingAction(result.data); +} + +/** + * Resolve a consolidation decision for one group. Tries configured channels in + * auto-capture order. On empty OpenCode structured output, retries once without + * `opencodeVariant`, then falls through to the other channel when configured. + */ +export async function decideDreamingAction( + group: DreamingNearDupGroup, + directory?: string +): Promise { + const channels = dreamingChannelOrder(); + let lastError: unknown; + + for (let i = 0; i < channels.length; i++) { + const channel = channels[i]!; + if (!isDreamingChannelConfigured(channel)) continue; + + try { + if (channel === "opencode") { + try { + return await decideViaOpencode(group, directory, false); + } catch (error) { + if (CONFIG.opencodeVariant && isEmptyStructuredOutputError(error)) { + log("dreaming: empty structured output; retrying without variant", { + channel: channelLabel("opencode"), + error: error instanceof Error ? error.message : String(error), + }); + return await decideViaOpencode(group, directory, true); + } + throw error; + } + } + return await decideViaExternal(group); + } catch (error) { + lastError = error; + const nextConfigured = channels.slice(i + 1).find((c) => isDreamingChannelConfigured(c)); + if (!nextConfigured) break; + + const fallbackLabel = nextConfigured === "external" ? "external API" : "opencode provider"; + log(`dreaming: ${channel} provider failed, falling back to ${fallbackLabel}`, { + channel: channelLabel(channel), + fallback: channelLabel(nextConfigured), + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (lastError) throw lastError; + throw new Error("dreaming_provider_unavailable"); +} + function statePath(): string { return join(CONFIG.storagePath, "dreaming-state.json"); } @@ -327,28 +493,16 @@ export class DreamingService { return result; } - const v2Client = getV2Client(); - if (!v2Client || !CONFIG.opencodeProvider || !CONFIG.opencodeModel) { - log("dreaming: skipped (no opencode structured-output client)"); + if (!isOpencodeDreamingConfigured() && !isExternalDreamingConfigured()) { + log("dreaming: skipped (no opencode or external structured-output provider)"); throw new Error("dreaming_provider_unavailable"); } - const { generateStructuredOutput } = await import("./ai/opencode-provider.js"); let abortedForProvider = false; for (const group of batch) { try { - const decision = await generateStructuredOutput({ - client: v2Client, - providerID: CONFIG.opencodeProvider, - modelID: CONFIG.opencodeModel, - ...(CONFIG.opencodeVariant ? { variant: CONFIG.opencodeVariant } : {}), - systemPrompt: - "You are a careful memory consolidation assistant. Return only structured JSON.", - userPrompt: buildMergePrompt(group), - schema: dreamingActionSchema, - directory, - }); + const decision = await decideDreamingAction(group, directory); const memberIds = new Set([ group.representative.id, @@ -411,7 +565,16 @@ export class DreamingService { } catch (error) { result.groupsFailed++; const errMsg = error instanceof Error ? error.message : String(error); - log("dreaming: group failed", { error: errMsg }); + log("dreaming: group failed", { + error: errMsg, + opencode: CONFIG.opencodeProvider + ? `${CONFIG.opencodeProvider}/${CONFIG.opencodeModel}${CONFIG.opencodeVariant ? ` variant=${CONFIG.opencodeVariant}` : ""}` + : null, + external: + CONFIG.memoryModel && CONFIG.memoryApiUrl + ? `${CONFIG.memoryProvider}/${CONFIG.memoryModel}` + : null, + }); // Auth / provider outages will fail every remaining group the same way. if (/ProviderRequestError|API_KEY is not set|401|Unauthorized/i.test(errMsg)) { const processed = diff --git a/tests/dreaming-run.test.ts b/tests/dreaming-run.test.ts index fa89611f..c7bbc8e4 100644 --- a/tests/dreaming-run.test.ts +++ b/tests/dreaming-run.test.ts @@ -52,13 +52,21 @@ function runFixture(scriptBody: string): { } describe("dreaming run API mapping", () => { - function apiMappingFixture(runError: string): string { + function apiMappingFixture(opts: { + runError?: string; + runResult?: Record; + dreamingEnabled?: boolean; + }): string { const storageDir = mkdtempSync(join(tmpdir(), "dream-api-map-")); tempDirs.push(storageDir); const promptManagerUrl = new URL( "../src/services/user-prompt/user-prompt-manager.js", import.meta.url ).href; + const enabled = opts.dreamingEnabled !== false; + const runImpl = opts.runError + ? `async () => { throw new Error(${JSON.stringify(opts.runError)}); }` + : `async () => (${JSON.stringify(opts.runResult ?? { groupsConsidered: 0, groupsMerged: 0, groupsSkipped: 0, groupsFailed: 0, groupsFlagged: 0, memoriesSoftDeleted: 0 })})`; return ` import { mock } from "bun:test"; import { mkdirSync } from "node:fs"; @@ -67,7 +75,7 @@ const storagePath = ${JSON.stringify(storageDir)}; mkdirSync(storagePath, { recursive: true }); mock.module(${JSON.stringify(configUrl)}, () => ({ - CONFIG: { storagePath, dreaming: { enabled: true } }, + CONFIG: { storagePath, dreaming: { enabled: ${enabled} } }, })); mock.module(${JSON.stringify(loggerUrl)}, () => ({ log: () => {}, @@ -79,10 +87,8 @@ mock.module(${JSON.stringify(promptManagerUrl)}, () => ({ })); mock.module(${JSON.stringify(dreamingServiceUrl)}, () => ({ dreamingService: { - run: async () => { - throw new Error(${JSON.stringify(runError)}); - }, - getStatus: () => ({ isRunning: true }), + run: ${runImpl}, + getStatus: () => ({ isRunning: false, enabled: ${enabled} }), }, })); @@ -93,18 +99,47 @@ console.log(JSON.stringify(out)); } it("maps concurrent runs to already_running", () => { - const result = runFixture(apiMappingFixture("Dreaming already running")); + const result = runFixture(apiMappingFixture({ runError: "Dreaming already running" })); expect(result.stderr).toBe(""); expect(result.exitCode).toBe(0); expect(result.parsed).toEqual({ success: false, error: "already_running" }); }); it("maps missing provider to dreaming_provider_unavailable", () => { - const result = runFixture(apiMappingFixture("dreaming_provider_unavailable")); + const result = runFixture(apiMappingFixture({ runError: "dreaming_provider_unavailable" })); expect(result.stderr).toBe(""); expect(result.exitCode).toBe(0); expect(result.parsed).toEqual({ success: false, error: "dreaming_provider_unavailable" }); }); + + it("allows manual run when dreaming.enabled is false (issue #491)", () => { + const result = runFixture( + apiMappingFixture({ + dreamingEnabled: false, + runResult: { + groupsConsidered: 1, + groupsMerged: 1, + groupsSkipped: 0, + groupsFailed: 0, + groupsFlagged: 0, + memoriesSoftDeleted: 1, + }, + }) + ); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.parsed).toEqual({ + success: true, + data: { + groupsConsidered: 1, + groupsMerged: 1, + groupsSkipped: 0, + groupsFailed: 0, + groupsFlagged: 0, + memoriesSoftDeleted: 1, + }, + }); + }); }); describe("dreaming service groupsFailed", () => { @@ -234,6 +269,187 @@ console.log(JSON.stringify({ out, state })); expect(parsed.state).toBeNull(); }); + it("falls back to external API when opencode returns empty structured output (issue #493)", () => { + const storageDir = mkdtempSync(join(tmpdir(), "dream-fallback-")); + tempDirs.push(storageDir); + const factoryUrl = new URL("../src/services/ai/ai-provider-factory.js", import.meta.url).href; + const providerConfigUrl = new URL("../src/services/ai/provider-config.js", import.meta.url) + .href; + const result = runFixture(` +import { mock } from "bun:test"; + +mock.module(${JSON.stringify(configUrl)}, () => ({ + CONFIG: { + storagePath: ${JSON.stringify(storageDir)}, + opencodeProvider: "deepseek", + opencodeModel: "deepseek-flash", + opencodeVariant: "low", + memoryProvider: "openai-chat", + memoryModel: "deepseek-flash", + memoryApiUrl: "https://example.test/v1", + autoCaptureProviderOrder: "opencode-first", + dreaming: { + enabled: true, + maxGroupsPerRun: 5, + relatedThreshold: 0.5, + crossProject: false, + trashRetentionDays: 30, + intervalHours: 24, + }, + memory: { defaultScope: "project" }, + }, +})); +mock.module(${JSON.stringify(loggerUrl)}, () => ({ + log: () => {}, + safeLog: () => {}, + serializeError: (e) => ({ error: String(e) }), +})); + +const calls = []; +mock.module(${JSON.stringify(opencodeProviderUrl)}, () => ({ + getV2Client: () => ({ session: {} }), + generateStructuredOutput: async (opts) => { + calls.push({ channel: "opencode", variant: opts.variant ?? null }); + throw new Error( + "opencode-mem: opencode returned no structured output (info.structured_output/info.structured were empty)" + ); + }, +})); +mock.module(${JSON.stringify(providerConfigUrl)}, () => ({ + buildMemoryProviderConfig: () => ({ model: "deepseek-flash", apiUrl: "https://example.test/v1" }), +})); +mock.module(${JSON.stringify(factoryUrl)}, () => ({ + AIProviderFactory: { + createProvider: () => ({ + executeToolCall: async () => { + calls.push({ channel: "external" }); + return { + success: true, + data: { action: "keep", reason: "external ok" }, + }; + }, + }), + }, +})); + +const { decideDreamingAction } = await import(${JSON.stringify(dreamingServiceUrl)}); +const decision = await decideDreamingAction({ + representative: { + id: "a", + content: "alpha", + containerTag: "proj", + createdAt: 1, + isPinned: false, + scope: "project", + }, + duplicates: [ + { + id: "b", + content: "alpha again", + similarity: 0.95, + isPinned: false, + containerTag: "proj", + createdAt: 2, + }, + ], +}); +console.log(JSON.stringify({ decision, calls })); +`); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + const parsed = result.parsed as { + decision: { action: string }; + calls: Array<{ channel: string; variant?: string | null }>; + }; + expect(parsed.decision.action).toBe("keep"); + // Empty structured output → retry without variant, then external fallback. + expect(parsed.calls).toEqual([ + { channel: "opencode", variant: "low" }, + { channel: "opencode", variant: null }, + { channel: "external" }, + ]); + }); + + it("retries without variant then succeeds without needing external (issue #493)", () => { + const storageDir = mkdtempSync(join(tmpdir(), "dream-retry-")); + tempDirs.push(storageDir); + const result = runFixture(` +import { mock } from "bun:test"; + +mock.module(${JSON.stringify(configUrl)}, () => ({ + CONFIG: { + storagePath: ${JSON.stringify(storageDir)}, + opencodeProvider: "deepseek", + opencodeModel: "deepseek-flash", + opencodeVariant: "low", + memoryModel: "deepseek-flash", + memoryApiUrl: "https://example.test/v1", + memoryProvider: "openai-chat", + autoCaptureProviderOrder: "opencode-first", + dreaming: { + enabled: true, + maxGroupsPerRun: 5, + relatedThreshold: 0.5, + crossProject: false, + trashRetentionDays: 30, + intervalHours: 24, + }, + memory: { defaultScope: "project" }, + }, +})); +mock.module(${JSON.stringify(loggerUrl)}, () => ({ + log: () => {}, + safeLog: () => {}, + serializeError: (e) => ({ error: String(e) }), +})); + +const calls = []; +mock.module(${JSON.stringify(opencodeProviderUrl)}, () => ({ + getV2Client: () => ({ session: {} }), + generateStructuredOutput: async (opts) => { + calls.push({ variant: opts.variant ?? null }); + if (opts.variant) { + throw new Error( + "opencode-mem: opencode returned no structured output (info.structured_output/info.structured were empty)" + ); + } + return { action: "keep", reason: "retry ok" }; + }, +})); + +const { decideDreamingAction } = await import(${JSON.stringify(dreamingServiceUrl)}); +const decision = await decideDreamingAction({ + representative: { + id: "a", + content: "alpha", + containerTag: "proj", + createdAt: 1, + isPinned: false, + scope: "project", + }, + duplicates: [ + { + id: "b", + content: "alpha again", + similarity: 0.95, + isPinned: false, + containerTag: "proj", + createdAt: 2, + }, + ], +}); +console.log(JSON.stringify({ decision, calls })); +`); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + const parsed = result.parsed as { + decision: { action: string }; + calls: Array<{ variant: string | null }>; + }; + expect(parsed.decision.action).toBe("keep"); + expect(parsed.calls).toEqual([{ variant: "low" }, { variant: null }]); + }); + it("throws when already running", () => { const storageDir = mkdtempSync(join(tmpdir(), "dream-busy-")); tempDirs.push(storageDir);