feat(deepnote): load integrations from .deepnote.env.yaml and .env

cspell.json

@@ -98,6 +98,8 @@
         "Unconfigured",
         "unuse",
         "unittests",
+        "userpod",
+        "Userpod",
         "vegalite",
         "venv",
         "Venv",

package.json

@@ -1649,6 +1649,12 @@
                     "description": "When enabled, outputs are saved to separate snapshot files in a 'snapshots' folder instead of the main .deepnote file.",
                     "scope": "resource"
                 },
+                "deepnote.integrations.envFile.enabled": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "When enabled, integration credentials are also loaded from a '.deepnote.env.yaml' file (with 'env:' references resolved against '.env' and environment variables) next to the .deepnote file or at the workspace root.",
+                    "scope": "resource"
+                },
                 "deepnote.experiments.enabled": {
                     "type": "boolean",
                     "default": true,
@@ -2703,6 +2709,7 @@
         "clsx": "^2.1.1",
         "cross-fetch": "^3.1.5",
         "d3-format": "^3.1.0",
+        "dotenv": "^17.2.3",
         "encoding": "^0.1.13",
         "express": "^5.2.1",
         "fast-deep-equal": "^2.0.1",

src/kernels/deepnote/deepnoteLspClientManager.node.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as vscode from 'vscode';
 import { CancellationError } from 'vscode';
-import { inject, injectable } from 'inversify';
+import { inject, injectable, optional } from 'inversify';
 import type {
     LanguageClient as LanguageClientType,
     LanguageClientOptions,
@@ -26,7 +26,8 @@ import { noop } from '../../platform/common/utils/misc';
 import {
     IIntegrationStorage,
     IPlatformNotebookEditorProvider,
-    IPlatformDeepnoteNotebookManager
+    IPlatformDeepnoteNotebookManager,
+    ISqlIntegrationEnvVarsProvider
 } from '../../platform/notebooks/deepnote/types';
 import { ConfigurableDatabaseIntegrationConfig } from '../../platform/notebooks/deepnote/integrationTypes';
 import { SqlLspConnection, isSupportedBySqlLsp, convertToSqlLspConnection } from './sqlLspConnectionUtils';
@@ -64,7 +65,10 @@ export class DeepnoteLspClientManager
         @inject(IIntegrationStorage) private readonly integrationStorage: IIntegrationStorage,
         @inject(IPlatformNotebookEditorProvider)
         private readonly notebookEditorProvider: IPlatformNotebookEditorProvider,
-        @inject(IPlatformDeepnoteNotebookManager) private readonly notebookManager: IPlatformDeepnoteNotebookManager
+        @inject(IPlatformDeepnoteNotebookManager) private readonly notebookManager: IPlatformDeepnoteNotebookManager,
+        @inject(ISqlIntegrationEnvVarsProvider)
+        @optional()
+        private readonly sqlIntegrationEnvVars?: ISqlIntegrationEnvVarsProvider
     ) {
         this.disposables.push(this);
     }
@@ -623,13 +627,19 @@ export class DeepnoteLspClientManager
 
             logger.trace(`SQL LSP: Found ${projectIntegrations.length} integrations in project ${projectId}`);
 
-            const projectIntegrationConfigs = (
-                await Promise.all(
-                    projectIntegrations.map((integration) =>
-                        this.integrationStorage.getIntegrationConfig(integration.id)
-                    )
-                )
-            ).filter((config): config is ConfigurableDatabaseIntegrationConfig => config != null);
+            // Prefer the merged (SecretStorage + `.deepnote.env.yaml`) configs so file-configured databases also get
+            // LSP autocomplete/schema (F13); fall back to SecretStorage-only when the merged provider is unavailable (e.g. web).
+            const projectIntegrationConfigs = this.sqlIntegrationEnvVars
+                ? (await this.sqlIntegrationEnvVars.getMergedConfigs(notebookUri)).filter(
+                      (config): config is ConfigurableDatabaseIntegrationConfig => config.type !== 'pandas-dataframe'
+                  )
+                : (
+                      await Promise.all(
+                          projectIntegrations.map((integration) =>
+                              this.integrationStorage.getIntegrationConfig(integration.id)
+                          )
+                      )
+                  ).filter((config): config is ConfigurableDatabaseIntegrationConfig => config != null);
 
             const connections = projectIntegrationConfigs
                 .filter((config) => isSupportedBySqlLsp(config.type))

src/kernels/deepnote/deepnoteServerStarter.node.ts

@@ -19,9 +19,10 @@ import { IProcessServiceFactory } from '../../platform/common/process/types.node
 import { IAsyncDisposableRegistry, IDisposable, IOutputChannel } from '../../platform/common/types';
 import { sleep } from '../../platform/common/utils/async';
 import { generateUuid } from '../../platform/common/uuid';
+import { resolveProjectIdForFile } from '../../platform/deepnote/deepnoteProjectIdResolver';
 import { DeepnoteServerStartupError } from '../../platform/errors/deepnoteKernelErrors';
 import { logger } from '../../platform/logging';
-import { ISqlIntegrationEnvVarsProvider } from '../../platform/notebooks/deepnote/types';
+import { ISqlIntegrationEnvVarsProvider, IUserpodApiEndpoints } from '../../platform/notebooks/deepnote/types';
 import { PythonEnvironment } from '../../platform/pythonEnvironments/info';
 import * as path from '../../platform/vscode-path/path';
 import { DeepnoteServerInfo, IDeepnoteServerStarter, IDeepnoteToolkitInstaller } from './types';
@@ -77,7 +78,10 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
         @inject(IAsyncDisposableRegistry) asyncRegistry: IAsyncDisposableRegistry,
         @inject(ISqlIntegrationEnvVarsProvider)
         @optional()
-        private readonly sqlIntegrationEnvVars?: ISqlIntegrationEnvVarsProvider
+        private readonly sqlIntegrationEnvVars?: ISqlIntegrationEnvVarsProvider,
+        @inject(IUserpodApiEndpoints)
+        @optional()
+        private readonly userpodApiEndpoints?: IUserpodApiEndpoints
     ) {
         asyncRegistry.push(this);
     }
@@ -262,6 +266,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
         this.outputChannel.appendLine(l10n.t('Starting Deepnote server...'));
 
         const extraEnv = await this.gatherSqlIntegrationEnvVars(deepnoteFileUri, environmentId, token);
+        await this.applyIntegrationEndpointEnv(extraEnv, deepnoteFileUri);
 
         // Initialize output tracking for error reporting
         this.serverOutputByFile.set(fileKey, { stdout: '', stderr: '' });
@@ -397,6 +402,41 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
         return extraEnv;
     }
 
+    // Skipped unless the endpoint is up and the file has a project id — else the toolkit raises on an unreachable URL.
+    private async applyIntegrationEndpointEnv(extraEnv: Record<string, string>, deepnoteFileUri: Uri): Promise<void> {
+        const endpoint = this.userpodApiEndpoints;
+
+        if (!endpoint) {
+            return;
+        }
+
+        // Wait for the initial bind so a kernel starting before the loopback endpoint is listening still gets the env (F3).
+        await endpoint.ready;
+
+        const baseUrl = endpoint.baseUrl;
+        if (!baseUrl) {
+            logger.warn(
+                'DeepnoteServerStarter: integration endpoint is not listening; skipping live integration env injection.'
+            );
+
+            return;
+        }
+
+        const projectId = await resolveProjectIdForFile(deepnoteFileUri);
+
+        if (!projectId) {
+            return;
+        }
+
+        extraEnv['DEEPNOTE_RUNTIME__ENV_INTEGRATION_ENABLED'] = 'true';
+        extraEnv['DEEPNOTE_RUNTIME__RUNNING_IN_DETACHED_MODE'] = 'true';
+        extraEnv['DEEPNOTE_RUNTIME__WEBAPP_URL'] = baseUrl;
+        // 2.1.1 dereferences project_secret without a null-check in detached mode; also the endpoint's per-project bearer token.
+        extraEnv['DEEPNOTE_RUNTIME__PROJECT_SECRET'] = endpoint.getAuthToken(projectId);
+        // Legacy key (not __PROJECT_ID): also satisfies set_notebook_path's has_env check, avoiding a session-name parse.
+        extraEnv['DEEPNOTE_PROJECT_ID'] = projectId;
+    }
+
     /**
      * Stream stdout/stderr from the server process to the VSCode output channel.
      */

src/kernels/deepnote/deepnoteServerStarter.unit.test.ts

@@ -1,21 +1,24 @@
 import { assert } from 'chai';
 import * as fakeTimers from '@sinonjs/fake-timers';
 import * as sinon from 'sinon';
-import { anything, instance, mock, when } from 'ts-mockito';
+import { anything, instance, mock, verify, when } from 'ts-mockito';
 import { CancellationError, Uri } from 'vscode';
 
+import { serializeDeepnoteFile, type DeepnoteFile } from '@deepnote/blocks';
+
 import { DeepnoteAgentSkillsManager } from './deepnoteAgentSkillsManager.node';
 import { DeepnoteServerStarter } from './deepnoteServerStarter.node';
 import { IProcessServiceFactory } from '../../platform/common/process/types.node';
 import { IAsyncDisposableRegistry, IOutputChannel } from '../../platform/common/types';
 import { IDeepnoteToolkitInstaller } from './types';
-import { ISqlIntegrationEnvVarsProvider } from '../../platform/notebooks/deepnote/types';
+import { ISqlIntegrationEnvVarsProvider, IUserpodApiEndpoints } from '../../platform/notebooks/deepnote/types';
 import { PythonEnvironment } from '../../platform/pythonEnvironments/info';
 import {
     __getStartServerCalls,
     __getStopServerCalls,
     __resetRuntimeCoreMock
 } from '../../test/mocks/deepnoteRuntimeCore';
+import { mockedVSCodeNamespaces, resetVSCodeMocks } from '../../test/vscode-mock';
 
 /**
  * Unit tests for DeepnoteServerStarter.
@@ -264,4 +267,153 @@ suite('DeepnoteServerStarter', () => {
             });
         });
     });
+
+    /**
+     * Direct tests for the private `applyIntegrationEndpointEnv`, exercised via a cast so we mutate a
+     * plain env object rather than intercepting the third-party `startServer`. The four integration env
+     * vars are only injected when the loopback endpoint is listening AND the file resolves to a project
+     * id; every other path degrades gracefully to whatever SQL_* vars were already gathered.
+     */
+    suite('applyIntegrationEndpointEnv (integration endpoint env injection)', () => {
+        const projectFileUri = Uri.file('/workspace/project/notebook-a.deepnote');
+        const baseUrl = 'http://127.0.0.1:5555';
+        // A pre-seeded SQL_* var stands in for the env already gathered before the endpoint step runs.
+        const sqlEnvKey = 'SQL_DEEPNOTE_INTEGRATION_ABC';
+        const sqlEnvValue = 'postgres://localhost:5432/db';
+
+        type WithApplyIntegrationEndpointEnv = {
+            applyIntegrationEndpointEnv(extraEnv: Record<string, string>, deepnoteFileUri: Uri): Promise<void>;
+        };
+
+        // Starters built with an endpoint arg are tracked so each is disposed after its test.
+        let extraStarters: DeepnoteServerStarter[];
+
+        setup(() => {
+            resetVSCodeMocks();
+            extraStarters = [];
+        });
+
+        teardown(async () => {
+            await Promise.all(extraStarters.map((starter) => starter.dispose()));
+        });
+
+        /**
+         * Builds a starter wired with the integration endpoint (optional ctor arg at positional index 6),
+         * whose `baseUrl` getter yields `endpointBaseUrl`. Tracked for disposal.
+         */
+        function createStarterWithEndpoint(endpointBaseUrl: string | undefined): DeepnoteServerStarter {
+            const endpoint: IUserpodApiEndpoints = {
+                baseUrl: endpointBaseUrl,
+                ready: Promise.resolve(),
+                getAuthToken: () => 'endpoint-token'
+            };
+            const starter = new DeepnoteServerStarter(
+                instance(mockProcessServiceFactory),
+                instance(mockToolkitInstaller),
+                instance(mockAgentSkillsManager),
+                instance(mockOutputChannel),
+                instance(mockAsyncRegistry),
+                instance(mockSqlIntegrationEnvVars),
+                endpoint
+            );
+            extraStarters.push(starter);
+
+            return starter;
+        }
+
+        function applyIntegrationEndpointEnv(
+            starter: DeepnoteServerStarter,
+            extraEnv: Record<string, string>,
+            deepnoteFileUri: Uri
+        ): Promise<void> {
+            return (starter as unknown as WithApplyIntegrationEndpointEnv).applyIntegrationEndpointEnv(
+                extraEnv,
+                deepnoteFileUri
+            );
+        }
+
+        /**
+         * Stubs `workspace.fs.readFile` — the read behind `resolveProjectIdForFile` — to yield the
+         * serialized `.deepnote` bytes. Returns the mock so tests can assert whether the read happened.
+         */
+        function stubReadFile(fileContents: string): typeof import('vscode').workspace.fs {
+            const mockFs = mock<typeof import('vscode').workspace.fs>();
+
+            when(mockFs.readFile(anything())).thenReturn(Promise.resolve(new TextEncoder().encode(fileContents)));
+            when(mockedVSCodeNamespaces.workspace.fs).thenReturn(instance(mockFs));
+
+            return mockFs;
+        }
+
+        function serializeProjectFile(projectId: string): string {
+            const file: DeepnoteFile = {
+                metadata: {
+                    createdAt: '2023-01-01T00:00:00Z',
+                    modifiedAt: '2023-01-02T00:00:00Z'
+                },
+                project: {
+                    id: projectId,
+                    name: 'Project',
+                    notebooks: [{ id: 'notebook-1', name: 'Notebook One', blocks: [] }],
+                    settings: {}
+                },
+                version: '1.0.0'
+            };
+
+            return serializeDeepnoteFile(file);
+        }
+
+        test('injects all five integration env vars (preserving pre-seeded SQL_* keys) when the endpoint is listening and the file has a project id', async () => {
+            const mockFs = stubReadFile(serializeProjectFile('the-project-id'));
+            const starter = createStarterWithEndpoint(baseUrl);
+
+            const extraEnv: Record<string, string> = { [sqlEnvKey]: sqlEnvValue };
+            await applyIntegrationEndpointEnv(starter, extraEnv, projectFileUri);
+
+            assert.deepStrictEqual(extraEnv, {
+                [sqlEnvKey]: sqlEnvValue,
+                DEEPNOTE_RUNTIME__ENV_INTEGRATION_ENABLED: 'true',
+                DEEPNOTE_RUNTIME__RUNNING_IN_DETACHED_MODE: 'true',
+                DEEPNOTE_RUNTIME__WEBAPP_URL: baseUrl,
+                DEEPNOTE_RUNTIME__PROJECT_SECRET: 'endpoint-token',
+                DEEPNOTE_PROJECT_ID: 'the-project-id'
+            });
+            // The enabled path must resolve the project id from the file.
+            verify(mockFs.readFile(anything())).once();
+        });
+
+        test('injects nothing and does NOT read the file when the endpoint has no baseUrl', async () => {
+            const mockFs = stubReadFile(serializeProjectFile('the-project-id'));
+            const starter = createStarterWithEndpoint(undefined);
+
+            const extraEnv: Record<string, string> = { [sqlEnvKey]: sqlEnvValue };
+            await applyIntegrationEndpointEnv(starter, extraEnv, projectFileUri);
+
+            assert.deepStrictEqual(extraEnv, { [sqlEnvKey]: sqlEnvValue });
+            // A missing baseUrl short-circuits BEFORE resolving the project id — no file read.
+            verify(mockFs.readFile(anything())).never();
+        });
+
+        test('injects nothing and does NOT read the file when no endpoint was injected at all', async () => {
+            // The outer-suite `serverStarter` is constructed WITHOUT the optional endpoint arg.
+            const mockFs = stubReadFile(serializeProjectFile('the-project-id'));
+
+            const extraEnv: Record<string, string> = { [sqlEnvKey]: sqlEnvValue };
+            await applyIntegrationEndpointEnv(serverStarter, extraEnv, projectFileUri);
+
+            assert.deepStrictEqual(extraEnv, { [sqlEnvKey]: sqlEnvValue });
+            verify(mockFs.readFile(anything())).never();
+        });
+
+        test('injects nothing when the endpoint is listening but the file resolves to no project id', async () => {
+            // A schema-valid `.deepnote` whose project.id is empty — `resolveProjectIdForFile` yields a falsy id.
+            stubReadFile(serializeProjectFile(''));
+            const starter = createStarterWithEndpoint(baseUrl);
+
+            const extraEnv: Record<string, string> = { [sqlEnvKey]: sqlEnvValue };
+            await applyIntegrationEndpointEnv(starter, extraEnv, projectFileUri);
+
+            assert.deepStrictEqual(extraEnv, { [sqlEnvKey]: sqlEnvValue });
+        });
+    });
 });

src/notebooks/deepnote/integrations/federatedAuth/federatedAuthKernelRestartBridge.node.ts

@@ -10,7 +10,7 @@ import { IFederatedAuthTokenStorage } from '../types';
 
 /**
  * Node-only bridge that restarts kernels when a federated integration's token changes, clearing stale
- * `os.environ` mutations and kernel globals. Separate from {@link IntegrationKernelRestartHandler} because
+ * `os.environ` mutations and kernel globals. Separate from {@link IntegrationEnvRefreshHandler} because
  * {@link IFederatedAuthTokenStorage} is node-only.
  */
 @injectable()