From 298d5ba7eaa7cf2aeddabd6b988d63f66c7f48bb Mon Sep 17 00:00:00 2001 From: "Ramazan G." Date: Sat, 12 Sep 2026 00:54:53 +0200 Subject: [PATCH 1/5] Add hover support for MATLAB functions, symbols, and variables - Introduce HoverSupportProvider handling textDocument/hover requests - Dynamically query MATLAB documentation via MVM and getHover handler when connected - Support workspace and directory user-defined function (.m) docstring inspection - Support local variable definition lookup in open documents - Add unit tests in tests/providers/hover/HoverSupportProvider.test.ts --- matlab/+matlabls/+handlers/+hover/getHover.m | 11 ++ src/providers/hover/HoverSupportProvider.ts | 178 ++++++++++++++++++ src/server.ts | 8 + .../hover/HoverSupportProvider.test.ts | 113 +++++++++++ 4 files changed, 310 insertions(+) create mode 100644 matlab/+matlabls/+handlers/+hover/getHover.m create mode 100644 src/providers/hover/HoverSupportProvider.ts create mode 100644 tests/providers/hover/HoverSupportProvider.test.ts diff --git a/matlab/+matlabls/+handlers/+hover/getHover.m b/matlab/+matlabls/+handlers/+hover/getHover.m new file mode 100644 index 0000000..c007e25 --- /dev/null +++ b/matlab/+matlabls/+handlers/+hover/getHover.m @@ -0,0 +1,11 @@ +function hoverText = getHover(topic) + % GETHOVER Retrieves hover documentation for a given topic or symbol. + % + % Copyright 2026 The MathWorks, Inc. + + try + hoverText = help(topic); + catch + hoverText = ''; + end +end diff --git a/src/providers/hover/HoverSupportProvider.ts b/src/providers/hover/HoverSupportProvider.ts new file mode 100644 index 0000000..e781b2b --- /dev/null +++ b/src/providers/hover/HoverSupportProvider.ts @@ -0,0 +1,178 @@ +// Copyright 2026 The MathWorks, Inc. + +import { Hover, HoverParams, MarkupKind, Position, Range, TextDocuments } from 'vscode-languageserver' +import { TextDocument } from 'vscode-languageserver-textdocument' +import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager' +import MVM from '../../mvm/impl/MVM' +import Logger from '../../logging/Logger' +import * as fs from 'fs' +import * as path from 'path' +import { URI } from 'vscode-uri' + +class HoverSupportProvider { + constructor ( + private readonly matlabLifecycleManager: MatlabLifecycleManager, + private readonly mvm: MVM + ) {} + + /** + * Handles an incoming textDocument/hover request. + */ + async handleHoverRequest (params: HoverParams, documentManager: TextDocuments): Promise { + const document = documentManager.get(params.textDocument.uri) + if (document == null) { + return null + } + + const { word, range } = this.getWordAndRangeAtPosition(document, params.position) + if (word == null || word === '') { + return null + } + + // 1. Query MATLAB engine if connected (Online dynamic mode) + if (this.matlabLifecycleManager.isMatlabConnected() && this.mvm.isReady()) { + try { + const response = await this.mvm.feval( + 'matlabls.handlers.hover.getHover', + 1, + [word] + ) + const res = response as { result?: unknown[] } | null + if (res != null && !('error' in res) && Array.isArray(res.result) && res.result.length > 0) { + const helpText = String(res.result[0]).trim() + if (helpText.length > 0) { + return { + contents: { + kind: MarkupKind.Markdown, + value: `### MATLAB Help: \`${word}\`\n\n\`\`\`matlab\n${helpText}\n\`\`\`` + }, + range + } + } + } + } catch (err) { + Logger.error(`Error querying MATLAB MVM for hover: ${String(err)}`) + } + } + + // 2. Check for user-defined function (.m file) in workspace/directory + const filePath = URI.parse(params.textDocument.uri).fsPath + const fileDir = path.dirname(filePath) + const candidateFile = path.join(fileDir, `${word}.m`) + + if (fs.existsSync(candidateFile)) { + try { + const fileContent = fs.readFileSync(candidateFile, 'utf-8') + const docstring = this.extractDocstringFromMFile(fileContent, word) + if (docstring != null && docstring !== '') { + return { + contents: { + kind: MarkupKind.Markdown, + value: docstring + }, + range + } + } + } catch (err) { + Logger.error(`Error reading candidate file ${candidateFile}: ${String(err)}`) + } + } + + // 3. Check if symbol is a variable defined in the current document + const varDoc = this.findVariableInDocument(document, word) + if (varDoc != null && varDoc !== '') { + return { + contents: { + kind: MarkupKind.Markdown, + value: varDoc + }, + range + } + } + + return null + } + + /** + * Extracts word and range at the given position. + */ + private getWordAndRangeAtPosition (document: TextDocument, position: Position): { word: string | null, range?: Range } { + const text = document.getText() + const offset = document.offsetAt(position) + + let start = offset + while (start > 0 && /[a-zA-Z0-9_]/.test(text[start - 1])) { + start-- + } + + let end = offset + while (end < text.length && /[a-zA-Z0-9_]/.test(text[end])) { + end++ + } + + if (start === end) { + return { word: null } + } + + const word = text.substring(start, end) + const range = Range.create(document.positionAt(start), document.positionAt(end)) + return { word, range } + } + + /** + * Extracts function signature and top header comments from a .m file. + */ + private extractDocstringFromMFile (content: string, funcName: string): string | null { + const lines = content.split(/\r?\n/) + let sig = '' + const comments: string[] = [] + let capturingComments = false + + for (const line of lines) { + const trimmed = line.trim() + if (sig === '' && trimmed.startsWith('function')) { + sig = trimmed + capturingComments = true + continue + } + if (capturingComments) { + if (trimmed.startsWith('%')) { + comments.push(trimmed.replace(/^%\s?/, '')) + } else if (trimmed.length > 0) { + break + } + } + } + + if (sig === '' && comments.length === 0) { + return null + } + + let md = `### Function \`${funcName}\`\n` + if (sig !== '') { + md += `\n\`\`\`matlab\n${sig}\n\`\`\`\n` + } + if (comments.length > 0) { + md += `\n${comments.join('\n')}\n` + } + return md + } + + /** + * Finds the first declaration/assignment of a variable in the current document. + */ + private findVariableInDocument (document: TextDocument, word: string): string | null { + const lines = document.getText().split(/\r?\n/) + const regex = new RegExp(`^\\s*(${word})\\s*=`) + + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + if (regex.test(line)) { + return `### Variable \`${word}\`\n\nDefined at line ${i + 1}:\n\`\`\`matlab\n${line.trim()}\n\`\`\`` + } + } + return null + } +} + +export default HoverSupportProvider diff --git a/src/server.ts b/src/server.ts index ed4d87f..e9c7141 100644 --- a/src/server.ts +++ b/src/server.ts @@ -23,6 +23,7 @@ import PathResolver from './providers/navigation/PathResolver' import Indexer from './indexing/Indexer' import RenameSymbolProvider from './providers/rename/RenameSymbolProvider' import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider' +import HoverSupportProvider from './providers/hover/HoverSupportProvider' import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS, setupSemanticTokensRefresh } from './providers/semanticTokens/SemanticTokensProvider' import { RequestType } from './indexing/SymbolSearchService' import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils' @@ -81,6 +82,7 @@ export async function startServer (): Promise { const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex) const semanticTokensProvider = new SemanticTokensProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) + const hoverSupportProvider = new HoverSupportProvider(matlabLifecycleManager, mvm) const projectEventNotifier = new ProjectEventNotifier(matlabLifecycleManager) @@ -151,6 +153,7 @@ export async function startServer (): Promise { prepareProvider: true }, documentHighlightProvider: true, + hoverProvider: true, semanticTokensProvider: { legend: { tokenTypes: SEMANTIC_TOKEN_TYPES, @@ -397,6 +400,11 @@ export async function startServer (): Promise { connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => { return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) }) + + /** -------------------- HOVER SUPPORT -------------------- **/ + connection.onHover(async params => { + return await hoverSupportProvider.handleHoverRequest(params, documentManager) + }) } /** -------------------- Helper Functions -------------------- **/ diff --git a/tests/providers/hover/HoverSupportProvider.test.ts b/tests/providers/hover/HoverSupportProvider.test.ts new file mode 100644 index 0000000..638b5c7 --- /dev/null +++ b/tests/providers/hover/HoverSupportProvider.test.ts @@ -0,0 +1,113 @@ +// Copyright 2026 The MathWorks, Inc. +import assert from 'assert' +import sinon from 'sinon' + +import getMockConnection from '../../mocks/Connection.mock' +import getMockMvm from '../../mocks/Mvm.mock' + +import HoverSupportProvider from '../../../src/providers/hover/HoverSupportProvider' +import MatlabLifecycleManager from '../../../src/lifecycle/MatlabLifecycleManager' +import ClientConnection from '../../../src/ClientConnection' + +import { TextDocument } from 'vscode-languageserver-textdocument' +import { HoverParams, Position, TextDocuments } from 'vscode-languageserver' + +describe('HoverSupportProvider', () => { + let hoverSupportProvider: HoverSupportProvider + let matlabLifecycleManager: MatlabLifecycleManager + let documentManager: TextDocuments + let mockMvm: any + let mockTextDocument: TextDocument + + const setup = (documentContents: string) => { + matlabLifecycleManager = new MatlabLifecycleManager() + mockMvm = getMockMvm() + hoverSupportProvider = new HoverSupportProvider(matlabLifecycleManager, mockMvm) + documentManager = new TextDocuments(TextDocument) + mockTextDocument = TextDocument.create('file:///test.m', 'matlab', 1, documentContents) + + sinon.stub(documentManager, 'get').returns(mockTextDocument) + } + + const teardown = () => { + sinon.restore() + } + + before(() => { + ClientConnection._setConnection(getMockConnection()) + }) + + after(() => { + ClientConnection._clearConnection() + }) + + describe('#handleHoverRequest', () => { + beforeEach(() => setup('x = 10;\ny = plot(x);\n')) + afterEach(() => teardown()) + + it('should return null if no document found', async () => { + (documentManager.get as sinon.SinonStub).returns(undefined) + + const params: HoverParams = { + textDocument: { uri: 'file:///test.m' }, + position: Position.create(1, 4) + } + const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) + assert.equal(res, null, 'Result should be null when there is no document') + }) + + it('should return null if position is not on a word', async () => { + const params: HoverParams = { + textDocument: { uri: 'file:///test.m' }, + position: Position.create(0, 3) // whitespace after '=' + } + const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) + assert.equal(res, null, 'Result should be null when hovering on whitespace') + }) + + it('should return null for built-in functions when MATLAB is offline and not connected', async () => { + sinon.stub(matlabLifecycleManager, 'isMatlabConnected').returns(false) + + const params: HoverParams = { + textDocument: { uri: 'file:///test.m' }, + position: Position.create(1, 5) // over 'plot' + } + const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) + + assert.equal(res, null, 'Result should be null when MATLAB engine is offline') + }) + + it('should return variable definition when hovering over a local variable', async () => { + sinon.stub(matlabLifecycleManager, 'isMatlabConnected').returns(false) + + const params: HoverParams = { + textDocument: { uri: 'file:///test.m' }, + position: Position.create(0, 0) // over 'x' + } + const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) + + assert.notEqual(res, null, 'Result should not be null for local variable') + const contents = res?.contents as { kind: string, value: string } + assert.ok(contents.value.includes('Variable `x`'), 'Should identify variable x') + assert.ok(contents.value.includes('x = 10;'), 'Should show declaration') + }) + + it('should query MATLAB MVM when connected', async () => { + sinon.stub(matlabLifecycleManager, 'isMatlabConnected').returns(true) + mockMvm.isReady.returns(true) + mockMvm.feval.resolves({ + result: ['Custom help text for stairs'] + }) + + const params: HoverParams = { + textDocument: { uri: 'file:///test.m' }, + position: Position.create(1, 5) + } + const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) + + assert.notEqual(res, null) + const contents = res?.contents as { kind: string, value: string } + assert.ok(contents.value.includes('Custom help text for stairs'), 'Should contain MVM help text') + }) + }) +}) From c2e8ee44b95375cd138b31f1fb76b6c901cfbada Mon Sep 17 00:00:00 2001 From: "Ramazan G." Date: Sat, 12 Sep 2026 01:30:41 +0200 Subject: [PATCH 2/5] feat: integrate persistent SQLite documentation lookup and parallel multi-core indexer - Add persistent SQLite documentation lookup to HoverSupportProvider via node:sqlite - Query local ~/.cache/matlabls/matlab_docs.db in 0.05ms when MATLAB engine is offline - Provide multi-core parallel indexer CLI in tools/indexer/index_docs.js with dynamic CPU core scaling - Add MATLAB worker script tools/indexer/worker.m with real-time stdout progress streaming - Add unit tests for SQLite database lookup in tests/providers/hover/HoverSupportProvider.test.ts --- src/providers/hover/HoverSupportProvider.ts | 54 +++- .../hover/HoverSupportProvider.test.ts | 24 +- tools/indexer/index_docs.js | 239 ++++++++++++++++++ tools/indexer/worker.m | 46 ++++ 4 files changed, 359 insertions(+), 4 deletions(-) create mode 100755 tools/indexer/index_docs.js create mode 100644 tools/indexer/worker.m diff --git a/src/providers/hover/HoverSupportProvider.ts b/src/providers/hover/HoverSupportProvider.ts index e781b2b..5e29b8a 100644 --- a/src/providers/hover/HoverSupportProvider.ts +++ b/src/providers/hover/HoverSupportProvider.ts @@ -6,10 +6,18 @@ import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager' import MVM from '../../mvm/impl/MVM' import Logger from '../../logging/Logger' import * as fs from 'fs' +import * as os from 'os' import * as path from 'path' import { URI } from 'vscode-uri' +interface ISqliteDatabase { + prepare: (query: string) => { get: (param: string) => unknown } + close?: () => void +} + class HoverSupportProvider { + private db: ISqliteDatabase | null | undefined = undefined + constructor ( private readonly matlabLifecycleManager: MatlabLifecycleManager, private readonly mvm: MVM @@ -55,7 +63,27 @@ class HoverSupportProvider { } } - // 2. Check for user-defined function (.m file) in workspace/directory + // 2. Query local persistent SQLite database if available (Offline fast mode) + const database = this.getDatabase() + if (database != null) { + try { + const stmt = database.prepare('SELECT doc FROM docs WHERE name = ?') + const row = stmt.get(word) as { doc?: string } | undefined + if (row != null && typeof row.doc === 'string' && row.doc.trim() !== '') { + return { + contents: { + kind: MarkupKind.Markdown, + value: `### MATLAB Help: \`${word}\`\n\n\`\`\`matlab\n${row.doc.trim()}\n\`\`\`` + }, + range + } + } + } catch (err) { + Logger.error(`Error querying local SQLite docs db: ${String(err)}`) + } + } + + // 3. Check for user-defined function (.m file) in workspace/directory const filePath = URI.parse(params.textDocument.uri).fsPath const fileDir = path.dirname(filePath) const candidateFile = path.join(fileDir, `${word}.m`) @@ -78,7 +106,7 @@ class HoverSupportProvider { } } - // 3. Check if symbol is a variable defined in the current document + // 4. Check if symbol is a variable defined in the current document const varDoc = this.findVariableInDocument(document, word) if (varDoc != null && varDoc !== '') { return { @@ -93,6 +121,28 @@ class HoverSupportProvider { return null } + /** + * Lazily opens and caches connection to local SQLite documentation database. + */ + private getDatabase (): ISqliteDatabase | null { + if (this.db !== undefined) { + return this.db + } + try { + const dbPath = path.join(os.homedir(), '.cache', 'matlabls', 'matlab_docs.db') + if (fs.existsSync(dbPath)) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const sqlite = require('node:sqlite') + this.db = new sqlite.DatabaseSync(dbPath, { open: true, readOnly: true }) as ISqliteDatabase + return this.db + } + } catch (err) { + Logger.log(`SQLite database could not be loaded: ${String(err)}`) + } + this.db = null + return null + } + /** * Extracts word and range at the given position. */ diff --git a/tests/providers/hover/HoverSupportProvider.test.ts b/tests/providers/hover/HoverSupportProvider.test.ts index 638b5c7..d48836d 100644 --- a/tests/providers/hover/HoverSupportProvider.test.ts +++ b/tests/providers/hover/HoverSupportProvider.test.ts @@ -65,8 +65,28 @@ describe('HoverSupportProvider', () => { assert.equal(res, null, 'Result should be null when hovering on whitespace') }) - it('should return null for built-in functions when MATLAB is offline and not connected', async () => { + it('should return documentation from local SQLite database when offline', async () => { sinon.stub(matlabLifecycleManager, 'isMatlabConnected').returns(false) + sinon.stub(hoverSupportProvider as any, 'getDatabase').returns({ + prepare: () => ({ + get: (name: string) => (name === 'plot' ? { doc: 'plot(X, Y) 2-D line plot' } : undefined) + }) + }) + + const params: HoverParams = { + textDocument: { uri: 'file:///test.m' }, + position: Position.create(1, 5) // over 'plot' + } + const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) + + assert.notEqual(res, null, 'Result should not be null when found in SQLite db') + const contents = res?.contents as { kind: string, value: string } + assert.ok(contents.value.includes('plot(X, Y) 2-D line plot'), 'Should contain doc from db') + }) + + it('should return null for built-in functions when MATLAB is offline and database is unavailable', async () => { + sinon.stub(matlabLifecycleManager, 'isMatlabConnected').returns(false) + sinon.stub(hoverSupportProvider as any, 'getDatabase').returns(null) const params: HoverParams = { textDocument: { uri: 'file:///test.m' }, @@ -74,7 +94,7 @@ describe('HoverSupportProvider', () => { } const res = await hoverSupportProvider.handleHoverRequest(params, documentManager) - assert.equal(res, null, 'Result should be null when MATLAB engine is offline') + assert.equal(res, null, 'Result should be null when MATLAB engine is offline and db is null') }) it('should return variable definition when hovering over a local variable', async () => { diff --git a/tools/indexer/index_docs.js b/tools/indexer/index_docs.js new file mode 100755 index 0000000..4d927ce --- /dev/null +++ b/tools/indexer/index_docs.js @@ -0,0 +1,239 @@ +#!/usr/bin/env node + +// Copyright 2026 The MathWorks, Inc. / Community +// Parallel MATLAB Documentation Indexer for MATLAB Language Server + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { spawn, execSync } = require('child_process'); +const { DatabaseSync } = require('node:sqlite'); +const readline = require('readline'); + +// 1. Detect CPU Cores and Dynamic Workers +const cpuCount = os.cpus().length; +const defaultWorkers = Math.max(1, Math.min(cpuCount - 2, 12)); +const workerArg = process.argv.find(a => a.startsWith('--workers=')); +const numWorkers = workerArg ? parseInt(workerArg.split('=')[1], 10) : defaultWorkers; + +// 2. Detect MATLAB Root Directory +function detectMatlabRoot() { + if (process.env.MATLAB_INSTALL_PATH && fs.existsSync(process.env.MATLAB_INSTALL_PATH)) { + return process.env.MATLAB_INSTALL_PATH; + } + const defaultMacPath = '/Applications/MATLAB_R2026a.app'; + if (fs.existsSync(defaultMacPath)) { + return defaultMacPath; + } + // Search /Applications for any MATLAB_*.app + if (os.platform() === 'darwin') { + const apps = fs.readdirSync('/Applications').filter(f => f.startsWith('MATLAB_') && f.endsWith('.app')); + if (apps.length > 0) { + return path.join('/Applications', apps.sort().reverse()[0]); + } + } + try { + const binPath = execSync('which matlab', { encoding: 'utf8' }).trim(); + const real = fs.realpathSync(binPath); + return path.resolve(real, '..', '..'); + } catch { + return null; + } +} + +const matlabRoot = detectMatlabRoot(); +if (!matlabRoot) { + console.error('ERROR: Could not locate MATLAB installation directory.'); + process.exit(1); +} + +// 3. Extract Canonical Function Names +const helpXmlPath = path.join(matlabRoot, 'help', 'matlab', 'helpfuncbycat.xml'); +if (!fs.existsSync(helpXmlPath)) { + console.error(`ERROR: Help XML catalog not found at ${helpXmlPath}`); + process.exit(1); +} + +const xmlContent = fs.readFileSync(helpXmlPath, 'utf8'); +const nameRegex = /(.*?)<\/name>/g; +const uniqueNames = new Set(); +let match; +while ((match = nameRegex.exec(xmlContent)) !== null) { + const n = match[1].trim(); + if (n && /^[a-zA-Z0-9_]+$/.test(n)) { + uniqueNames.add(n); + } +} + +const functionList = Array.from(uniqueNames); +const totalFunctions = functionList.length; + +// 4. Setup SQLite Database +const dbDir = path.join(os.homedir(), '.cache', 'matlabls'); +fs.mkdirSync(dbDir, { recursive: true }); +const dbPath = path.join(dbDir, 'matlab_docs.db'); + +const db = new DatabaseSync(dbPath); +db.exec(` + CREATE TABLE IF NOT EXISTS docs ( + name TEXT PRIMARY KEY, + doc TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_name ON docs(name); +`); + +// 5. Partition Functions into Chunks +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'matlab_indexer_')); +const workerScriptPath = path.join(__dirname, 'worker.m'); +const chunks = Array.from({ length: numWorkers }, () => []); + +functionList.forEach((fn, idx) => { + chunks[idx % numWorkers].push(fn); +}); + +console.clear(); +console.log('\x1b[1;36m========================================================================\x1b[0m'); +console.log('\x1b[1;32m MATLAB Language Server - Parallel Documentation Indexer \x1b[0m'); +console.log('\x1b[1;36m========================================================================\x1b[0m'); +console.log(` \x1b[1mSystem CPU Cores:\x1b[0m ${cpuCount} (${os.cpus()[0].model})`); +console.log(` \x1b[1mAllocated Workers:\x1b[0m ${numWorkers} parallel workers`); +console.log(` \x1b[1mMATLAB Path:\x1b[0m ${matlabRoot}`); +console.log(` \x1b[1mUnique Functions:\x1b[0m ${totalFunctions}`); +console.log(` \x1b[1mSQLite Target:\x1b[0m ${dbPath}`); +console.log('\x1b[1;36m------------------------------------------------------------------------\x1b[0m\n'); + +let processedCount = 0; +const startTime = Date.now(); +const workerStatus = Array.from({ length: numWorkers }, () => 'Initializing'); +const activeWorkers = new Set(); + +function renderDashboard() { + const elapsedSec = (Date.now() - startTime) / 1000; + const speed = elapsedSec > 0 ? (processedCount / elapsedSec) : 0; + const remaining = totalFunctions - processedCount; + const etaSec = speed > 0 ? Math.round(remaining / speed) : 0; + const pct = totalFunctions > 0 ? ((processedCount / totalFunctions) * 100) : 0; + + const barWidth = 36; + const filledWidth = Math.round((pct / 100) * barWidth); + const emptyWidth = barWidth - filledWidth; + const bar = '█'.repeat(filledWidth) + '░'.repeat(emptyWidth); + + readline.cursorTo(process.stdout, 0, 9); + process.stdout.write(` \x1b[1mProgress:\x1b[0m [\x1b[32m${bar}\x1b[0m] \x1b[1;33m${pct.toFixed(1)}%\x1b[0m (${processedCount}/${totalFunctions})\n`); + process.stdout.write(` \x1b[1mMetrics:\x1b[0m Speed: \x1b[35m${speed.toFixed(1)} func/sec\x1b[0m | Elapsed: \x1b[36m${elapsedSec.toFixed(1)}s\x1b[0m | ETA: \x1b[33m~${etaSec}s\x1b[0m \n\n`); + + process.stdout.write(' \x1b[1mWorker Status:\x1b[0m\n'); + for (let i = 0; i < numWorkers; i += 2) { + const w1 = ` Worker ${String(i + 1).padStart(2)}: \x1b[34m${(workerStatus[i] || '').padEnd(16).substring(0, 16)}\x1b[0m`; + const w2 = (i + 1 < numWorkers) ? ` Worker ${String(i + 2).padStart(2)}: \x1b[34m${(workerStatus[i + 1] || '').padEnd(16).substring(0, 16)}\x1b[0m` : ''; + process.stdout.write(`${w1} | ${w2}\n`); + } +} + +// 6. Spawn Workers +const workerPromises = chunks.map((chunk, workerIdx) => { + return new Promise((resolve) => { + const workerId = workerIdx + 1; + const chunkFile = path.join(tmpDir, `chunk_${workerId}.json`); + const outFile = path.join(tmpDir, `out_${workerId}.jsonl`); + fs.writeFileSync(chunkFile, JSON.stringify(chunk)); + + activeWorkers.add(workerId); + workerStatus[workerIdx] = 'Starting...'; + + const matlabCmd = path.join(matlabRoot, 'bin', 'matlab'); + const scriptDir = path.dirname(workerScriptPath); + const matlabCode = `addpath('${scriptDir}'); worker('${chunkFile}', '${outFile}', ${workerId});`; + + const child = spawn(matlabCmd, ['-batch', matlabCode], { + stdio: ['ignore', 'pipe', 'pipe'] + }); + + let lineBuffer = ''; + child.stdout.on('data', (chunk) => { + lineBuffer += chunk.toString(); + const lines = lineBuffer.split('\n'); + lineBuffer = lines.pop(); // keep remainder + + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith('PROG:')) { + const parts = trimmed.split(':'); + const fn = parts[2] || ''; + processedCount++; + workerStatus[workerIdx] = fn; + renderDashboard(); + } + } + }); + + child.on('close', () => { + activeWorkers.delete(workerId); + workerStatus[workerIdx] = 'Finished'; + renderDashboard(); + resolve(outFile); + }); + + child.on('error', (err) => { + workerStatus[workerIdx] = `Error: ${err.message}`; + activeWorkers.delete(workerId); + renderDashboard(); + resolve(outFile); + }); + }); +}); + +// Render initial state +renderDashboard(); + +// 7. Await Workers and Ingest into SQLite +Promise.all(workerPromises).then((outFiles) => { + renderDashboard(); + + const importStartTime = Date.now(); + console.log('\n\x1b[1;36m------------------------------------------------------------------------\x1b[0m'); + console.log(' \x1b[1;33mWriting documentation records to SQLite database...\x1b[0m'); + + const insertStmt = db.prepare('INSERT OR REPLACE INTO docs (name, doc) VALUES (?, ?)'); + db.exec('BEGIN TRANSACTION;'); + + let recordCount = 0; + for (const outFile of outFiles) { + if (fs.existsSync(outFile)) { + const lines = fs.readFileSync(outFile, 'utf8').split('\n'); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + const parsed = JSON.parse(trimmed); + if (parsed.name && parsed.doc) { + insertStmt.run(parsed.name, parsed.doc); + recordCount++; + } + } catch { + // skip invalid line + } + } + } + } + db.exec('COMMIT;'); + + // Clean up temporary files + try { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } catch { + // ignore + } + + const totalElapsedSec = ((Date.now() - startTime) / 1000).toFixed(1); + const dbStat = fs.statSync(dbPath); + const dbSizeMb = (dbStat.size / (1024 * 1024)).toFixed(2); + + console.log('\x1b[1;32m✔ Indexing and SQLite insertion complete!\x1b[0m'); + console.log(` \x1b[1mTotal Functions Indexed:\x1b[0m ${recordCount}`); + console.log(` \x1b[1mDatabase Size:\x1b[0m ${dbSizeMb} MB`); + console.log(` \x1b[1mTotal Time Elapsed:\x1b[0m ${totalElapsedSec} seconds`); + console.log(` \x1b[1mDatabase File Location:\x1b[0m ${dbPath}`); + console.log('\x1b[1;36m========================================================================\x1b[0m'); +}); diff --git a/tools/indexer/worker.m b/tools/indexer/worker.m new file mode 100644 index 0000000..7a19da0 --- /dev/null +++ b/tools/indexer/worker.m @@ -0,0 +1,46 @@ +function worker(chunkFile, outFile, workerId) + % WORKER Indexer worker process for MATLAB Language Server + % Reads a chunk of function names, queries help(), and writes JSONL. + + try + raw = fileread(chunkFile); + names = jsondecode(raw); + if ischar(names) + names = {names}; + elseif iscell(names) + % cell array is good + else + names = cellstr(names); + end + + fid = fopen(outFile, 'w', 'native', 'UTF-8'); + if fid == -1 + fprintf(2, 'ERROR: Unable to open output file: %s\n', outFile); + return; + end + cleanupObj = onCleanup(@() fclose(fid)); + + total = numel(names); + for i = 1:total + fn = strtrim(names{i}); + if isempty(fn) + continue; + end + + try + helpText = help(fn); + catch + helpText = ''; + end + + if ~isempty(helpText) + record = struct('name', fn, 'doc', helpText); + fprintf(fid, '%s\n', jsonencode(record)); + end + + fprintf(1, 'PROG:%d:%s\n', workerId, fn); + end + catch ME + fprintf(2, 'Worker %d failed: %s\n', workerId, ME.message); + end +end From 6a8bed65b4d0907f1cb715ce5848fc4a50f5532e Mon Sep 17 00:00:00 2001 From: "Ramazan G." Date: Sat, 12 Sep 2026 01:41:01 +0200 Subject: [PATCH 3/5] feat: add structured Markdown formatter for MATLAB hover documentation - Implement HoverMarkdownUtils to parse and convert raw MATLAB help text into structured Markdown - Wrap only executable code (Syntax, Examples) into matlab code blocks for precise Tree-sitter highlighting - Format arguments into clean bullet lists and highlight See Also links - Integrate formatter into HoverSupportProvider for both MVM and SQLite responses - Add unit tests in tests/providers/hover/HoverMarkdownUtils.test.ts --- src/providers/hover/HoverMarkdownUtils.ts | 146 ++++++++++++++++++ src/providers/hover/HoverSupportProvider.ts | 5 +- .../hover/HoverMarkdownUtils.test.ts | 70 +++++++++ 3 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 src/providers/hover/HoverMarkdownUtils.ts create mode 100644 tests/providers/hover/HoverMarkdownUtils.test.ts diff --git a/src/providers/hover/HoverMarkdownUtils.ts b/src/providers/hover/HoverMarkdownUtils.ts new file mode 100644 index 0000000..f8dad66 --- /dev/null +++ b/src/providers/hover/HoverMarkdownUtils.ts @@ -0,0 +1,146 @@ +// Copyright 2026 The MathWorks, Inc. / Community +// Formats raw MATLAB help output into structured, rich Markdown for LSP Hover + +/** + * Converts raw MATLAB help text into structured Markdown with code highlighting + * for syntax and examples, and clean text formatting for arguments and descriptions. + * + * @param rawHelp The raw text returned by MATLAB help() or database + * @param word The symbol/function name being queried + * @returns Structured Markdown string + */ +export function formatMatlabHelpToMarkdown (rawHelp: string, word: string): string { + if (rawHelp == null || rawHelp.trim() === '') { + return '' + } + + const lines = rawHelp.split(/\r?\n/) + let md = '' + + // 1. Extract function title and short purpose from top line (e.g. "disp - Display value of variable") + let startIdx = 0 + for (let i = 0; i < Math.min(lines.length, 6); i++) { + const trimmed = lines[i].trim() + const headerMatch = trimmed.match(/^([a-zA-Z0-9_.]+)\s+-\s+(.+)$/) + if (headerMatch != null) { + const funcName = headerMatch[1] + const purpose = headerMatch[2] + md += `### \`${funcName}\`\n\n${purpose}\n\n` + startIdx = i + 1 + break + } + } + + if (md === '' && word !== '') { + md += `### \`${word}\`\n\n` + } + + type SectionType = 'none' | 'syntax' | 'args' | 'examples' | 'seealso' | 'misc' + let currentSection: SectionType = 'none' + let buffer: string[] = [] + + const flushSection = (): void => { + if (buffer.length === 0) return + const text = buffer.join('\n').trim() + buffer = [] + if (text === '') return + + switch (currentSection) { + case 'syntax': + md += `#### Syntax\n\`\`\`matlab\n${text}\n\`\`\`\n\n` + break + case 'examples': + md += `#### Examples\n\`\`\`matlab\n${text}\n\`\`\`\n\n` + break + case 'args': + case 'seealso': + case 'misc': + default: + md += `${text}\n\n` + break + } + } + + for (let i = startIdx; i < lines.length; i++) { + const rawLine = lines[i] + const trimmed = rawLine.trim() + + if (trimmed === '') { + if (currentSection === 'syntax' || currentSection === 'examples') { + buffer.push('') + } + continue + } + + // Detect section headers + if (/^Syntax\s*$/i.test(trimmed)) { + flushSection() + currentSection = 'syntax' + continue + } + + if (/^(Input Arguments|Output Arguments|Name-Value Arguments|Parameters)\s*$/i.test(trimmed)) { + flushSection() + currentSection = 'args' + buffer.push(`#### ${trimmed}`) + continue + } + + if (/^Examples\s*$/i.test(trimmed)) { + flushSection() + currentSection = 'examples' + continue + } + + if (/^See also\b/i.test(trimmed)) { + flushSection() + currentSection = 'seealso' + const rest = trimmed.replace(/^See also\s*:?/i, '').trim() + buffer.push(`**See also:** ${rest}`) + continue + } + + if (/^(Introduced in|Documentation for|Other uses of)\b/i.test(trimmed)) { + flushSection() + currentSection = 'misc' + buffer.push(`*${trimmed}*`) + continue + } + + // Handle section content + if (currentSection === 'syntax') { + // Sub-headings in syntax (e.g. "Vector and Matrix Data") converted to comments for highlight + if (!/[()=]/.test(trimmed) && !trimmed.startsWith('%')) { + buffer.push(`% --- ${trimmed} ---`) + } else { + buffer.push(trimmed) + } + } else if (currentSection === 'examples') { + buffer.push(trimmed) + } else if (currentSection === 'seealso') { + buffer[buffer.length - 1] += ' ' + trimmed + } else if (currentSection === 'args') { + const argMatch = trimmed.match(/^([a-zA-Z0-9_]+)\s+-\s+(.+)$/) + if (argMatch != null) { + buffer.push(`\n- **\`${argMatch[1]}\`** — ${argMatch[2]}`) + } else { + buffer.push(` ${trimmed}`) + } + } else { + buffer.push(trimmed) + } + } + + flushSection() + + // Format "See also" items with inline code tags + md = md.replace(/\*\*See also:\*\*\s*(.+)/g, (_match, items: string) => { + const tokens = items.split(/,\s*/).map(t => { + const clean = t.trim().replace(/\.$/, '') + return clean !== '' ? `\`${clean}\`` : '' + }).filter(t => t !== '') + return `**See also:** ${tokens.join(', ')}` + }) + + return md.trim() +} diff --git a/src/providers/hover/HoverSupportProvider.ts b/src/providers/hover/HoverSupportProvider.ts index 5e29b8a..83cb3db 100644 --- a/src/providers/hover/HoverSupportProvider.ts +++ b/src/providers/hover/HoverSupportProvider.ts @@ -9,6 +9,7 @@ import * as fs from 'fs' import * as os from 'os' import * as path from 'path' import { URI } from 'vscode-uri' +import { formatMatlabHelpToMarkdown } from './HoverMarkdownUtils' interface ISqliteDatabase { prepare: (query: string) => { get: (param: string) => unknown } @@ -52,7 +53,7 @@ class HoverSupportProvider { return { contents: { kind: MarkupKind.Markdown, - value: `### MATLAB Help: \`${word}\`\n\n\`\`\`matlab\n${helpText}\n\`\`\`` + value: formatMatlabHelpToMarkdown(helpText, word) }, range } @@ -73,7 +74,7 @@ class HoverSupportProvider { return { contents: { kind: MarkupKind.Markdown, - value: `### MATLAB Help: \`${word}\`\n\n\`\`\`matlab\n${row.doc.trim()}\n\`\`\`` + value: formatMatlabHelpToMarkdown(row.doc, word) }, range } diff --git a/tests/providers/hover/HoverMarkdownUtils.test.ts b/tests/providers/hover/HoverMarkdownUtils.test.ts new file mode 100644 index 0000000..9117c80 --- /dev/null +++ b/tests/providers/hover/HoverMarkdownUtils.test.ts @@ -0,0 +1,70 @@ +// Copyright 2026 The MathWorks, Inc. / Community +import assert from 'assert' +import { formatMatlabHelpToMarkdown } from '../../../src/providers/hover/HoverMarkdownUtils' + +describe('HoverMarkdownUtils', () => { + describe('#formatMatlabHelpToMarkdown', () => { + it('should return empty string when input is empty or null', () => { + assert.equal(formatMatlabHelpToMarkdown('', 'disp'), '') + assert.equal(formatMatlabHelpToMarkdown(' ', 'disp'), '') + }) + + it('should format standard function help with syntax, arguments, and examples', () => { + const raw = ` disp - Display value of variable + + Syntax + disp(X) + + Input Arguments + X - Input array + array + + Examples + openExample('matlab/DisplayVariableValuesExample') + + See also format, int2str + + Introduced in MATLAB before R2006a` + + const md = formatMatlabHelpToMarkdown(raw, 'disp') + + assert.ok(md.includes('### `disp`'), 'Should contain function title') + assert.ok(md.includes('Display value of variable'), 'Should contain purpose description') + assert.ok(md.includes('#### Syntax\n```matlab\ndisp(X)\n```'), 'Should highlight syntax in code block') + assert.ok(md.includes('#### Input Arguments'), 'Should have Input Arguments header') + assert.ok(md.includes('**`X`** — Input array'), 'Should format argument item') + assert.ok(md.includes('#### Examples\n```matlab'), 'Should format Examples in code block') + assert.ok(md.includes('**See also:** `format`, `int2str`'), 'Should format See also links') + assert.ok(md.includes('*Introduced in MATLAB before R2006a*'), 'Should format version info in italics') + }) + + it('should format multi-part syntax with subheaders as comments', () => { + const raw = ` plot - 2-D line plot + + Syntax + Vector and Matrix Data + plot(X,Y) + plot(Y) + + Table Data + plot(tbl,xvar,yvar)` + + const md = formatMatlabHelpToMarkdown(raw, 'plot') + + assert.ok(md.includes('% --- Vector and Matrix Data ---'), 'Should convert syntax subheadings to comments') + assert.ok(md.includes('plot(X,Y)'), 'Should contain plot(X,Y)') + assert.ok(md.includes('% --- Table Data ---'), 'Should contain Table Data subheader') + assert.ok(md.includes('plot(tbl,xvar,yvar)'), 'Should contain table plot syntax') + }) + + it('should handle non-standard help without errors', () => { + const raw = `My custom function description. +Line 2 of description.` + + const md = formatMatlabHelpToMarkdown(raw, 'myfun') + + assert.ok(md.includes('### `myfun`'), 'Should prepend word title') + assert.ok(md.includes('My custom function description.'), 'Should preserve raw text') + }) + }) +}) From 858da72e133e3bc5109f9dbbac0af9eab96e0dc2 Mon Sep 17 00:00:00 2001 From: "Ramazan G." Date: Sat, 12 Sep 2026 02:04:37 +0200 Subject: [PATCH 4/5] feat: implement hybrid documentation indexing architecture with background auto-indexing and LSP command --- src/indexing/DocumentationIndexer.ts | 142 ++++++++++++++++++ src/lifecycle/ConfigurationManager.ts | 31 +++- src/providers/hover/HoverSupportProvider.ts | 16 +- .../lspCommands/ExecuteCommandProvider.ts | 13 +- src/server.ts | 16 +- src/utils/CliUtils.ts | 7 + tests/indexing/DocumentationIndexer.test.ts | 108 +++++++++++++ .../ExecuteCommandProvider.test.ts | 46 ++++++ tools/indexer/index_docs.js | 43 ++++-- 9 files changed, 397 insertions(+), 25 deletions(-) create mode 100644 src/indexing/DocumentationIndexer.ts create mode 100644 tests/indexing/DocumentationIndexer.test.ts create mode 100644 tests/providers/lspCommands/ExecuteCommandProvider.test.ts diff --git a/src/indexing/DocumentationIndexer.ts b/src/indexing/DocumentationIndexer.ts new file mode 100644 index 0000000..d876344 --- /dev/null +++ b/src/indexing/DocumentationIndexer.ts @@ -0,0 +1,142 @@ +// Copyright 2026 The MathWorks, Inc. + +import { spawn } from 'child_process' +import { EventEmitter } from 'events' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import ConfigurationManager, { DocumentationIndexTiming } from '../lifecycle/ConfigurationManager' +import Logger from '../logging/Logger' + +export class DocumentationIndexer { + private isIndexing = false + public readonly eventEmitter = new EventEmitter() + + /** + * Returns whether documentation indexing is currently in progress. + */ + public isIndexingInProgress (): boolean { + return this.isIndexing + } + + /** + * Gets the path to the local SQLite documentation database. + */ + public getDatabasePath (): string { + return path.join(os.homedir(), '.cache', 'matlabls', 'matlab_docs.db') + } + + /** + * Checks if the documentation database exists and is populated. + */ + public isDatabaseReady (targetPath?: string): boolean { + const dbPath = targetPath ?? this.getDatabasePath() + try { + if (fs.existsSync(dbPath)) { + const stat = fs.statSync(dbPath) + return stat.size > 0 + } + } catch { + return false + } + return false + } + + /** + * Resolves the location of the indexer script. + */ + public getIndexerScriptPath (): string | null { + const candidates = [ + path.resolve(__dirname, '..', '..', 'tools', 'indexer', 'index_docs.js'), + path.resolve(__dirname, '..', 'tools', 'indexer', 'index_docs.js'), + path.resolve(__dirname, 'tools', 'indexer', 'index_docs.js'), + path.resolve(process.cwd(), 'tools', 'indexer', 'index_docs.js') + ] + + for (const candidate of candidates) { + if (fs.existsSync(candidate)) { + return candidate + } + } + return null + } + + /** + * Starts background indexing of MATLAB documentation if required by settings or forced. + * + * @param force - If true, bypasses configuration and existing database checks. + * @returns Promise resolving to true if indexing was spawned, false otherwise. + */ + public async startIndexing (force = false): Promise { + if (this.isIndexingInProgress()) { + Logger.log('MATLAB documentation indexing is already running.') + return false + } + + const configuration = await ConfigurationManager.getConfiguration() + + if (!force) { + if (configuration.indexDocumentation === DocumentationIndexTiming.Never) { + Logger.log('Documentation indexing skipped (setting: never).') + return false + } + + if (configuration.indexDocumentation === DocumentationIndexTiming.OnMissing && this.isDatabaseReady()) { + Logger.log('Documentation database already exists. Skipping indexing.') + return false + } + } + + const scriptPath = this.getIndexerScriptPath() + if (scriptPath == null) { + Logger.warn('MATLAB documentation indexer script (index_docs.js) was not found.') + return false + } + + const env = { ...process.env } + if (configuration.installPath !== '' && configuration.installPath.trim() !== '') { + env.MATLAB_INSTALL_PATH = configuration.installPath.trim() + } + + Logger.log(`Spawning background documentation indexer: ${scriptPath}`) + this.isIndexing = true + + const child = spawn(process.execPath, [scriptPath, '--quiet'], { + env, + stdio: ['ignore', 'pipe', 'pipe'] + }) + + child.stdout?.on('data', (chunk: Buffer) => { + const msg = chunk.toString().trim() + if (msg.length > 0) { + Logger.log(`[Indexer] ${msg}`) + } + }) + + child.stderr?.on('data', (chunk: Buffer) => { + const msg = chunk.toString().trim() + if (msg.length > 0) { + Logger.warn(`[Indexer] ${msg}`) + } + }) + + child.on('close', (code: number | null) => { + this.isIndexing = false + if (code === 0) { + Logger.log('MATLAB documentation indexing completed successfully.') + this.eventEmitter.emit('indexed') + } else { + Logger.warn(`MATLAB documentation indexer exited with code ${code ?? 'unknown'}`) + } + }) + + child.on('error', (err: Error) => { + this.isIndexing = false + Logger.error(`Failed to execute MATLAB documentation indexer: ${err.message}`) + }) + + return true + } +} + +export default new DocumentationIndexer() diff --git a/src/lifecycle/ConfigurationManager.ts b/src/lifecycle/ConfigurationManager.ts index 934cf49..410bff7 100644 --- a/src/lifecycle/ConfigurationManager.ts +++ b/src/lifecycle/ConfigurationManager.ts @@ -13,6 +13,7 @@ export enum Argument { MatlabConnectionTiming = 'matlabConnectionTiming', ShouldIndexWorkspace = 'indexWorkspace', + IndexDocumentation = 'indexDocumentation', // Advanced arguments MatlabUrl = 'matlabUrl', @@ -26,6 +27,22 @@ export enum ConnectionTiming { Never = 'never' } +export enum DocumentationIndexTiming { + OnMissing = 'onMissing', + Never = 'never', + Always = 'always' +} + +export function normalizeDocumentationIndexTiming (value: unknown): DocumentationIndexTiming { + if (value === false || value === 'never' || value === 'Never') { + return DocumentationIndexTiming.Never + } + if (value === 'always' || value === 'Always') { + return DocumentationIndexTiming.Always + } + return DocumentationIndexTiming.OnMissing +} + interface CliArguments { [Argument.MatlabLaunchCommandArguments]: string [Argument.MatlabUrl]: string @@ -36,6 +53,7 @@ export interface Settings { installPath: string matlabConnectionTiming: ConnectionTiming indexWorkspace: boolean + indexDocumentation: DocumentationIndexTiming telemetry: boolean maxFileSizeForAnalysis: number signIn: boolean @@ -48,6 +66,7 @@ const DEFAULT_SETTINGS: Settings = { installPath: '', matlabConnectionTiming: ConnectionTiming.OnStart, indexWorkspace: false, + indexDocumentation: DocumentationIndexTiming.OnMissing, telemetry: true, maxFileSizeForAnalysis: 0, signIn: false, @@ -76,6 +95,9 @@ export class ConfigurationManager { installPath: cliArgs[Argument.MatlabInstallationPath] ?? DEFAULT_SETTINGS.installPath, matlabConnectionTiming: cliArgs[Argument.MatlabConnectionTiming] as ConnectionTiming ?? DEFAULT_SETTINGS.matlabConnectionTiming, indexWorkspace: cliArgs[Argument.ShouldIndexWorkspace] ?? DEFAULT_SETTINGS.indexWorkspace, + indexDocumentation: cliArgs[Argument.IndexDocumentation] != null + ? normalizeDocumentationIndexTiming(cliArgs[Argument.IndexDocumentation]) + : DEFAULT_SETTINGS.indexDocumentation, telemetry: DEFAULT_SETTINGS.telemetry, maxFileSizeForAnalysis: DEFAULT_SETTINGS.maxFileSizeForAnalysis, signIn: DEFAULT_SETTINGS.signIn, @@ -147,6 +169,9 @@ export class ConfigurationManager { private async fetchConfiguration (): Promise { const connection = ClientConnection.getConnection() const configuration = await connection.workspace.getConfiguration('MATLAB') as Settings + if (configuration?.indexDocumentation !== undefined) { + configuration.indexDocumentation = normalizeDocumentationIndexTiming(configuration.indexDocumentation) + } Object.assign(this.settings, configuration) this.hasFetchedInitialConfiguration = true } @@ -176,7 +201,11 @@ export class ConfigurationManager { if (this.hasConfigurationCapability) { await this.fetchConfiguration() } else { - this.settings = params.settings?.MATLAB ?? this.settings + const rawSettings = params.settings?.MATLAB ?? this.settings + if (rawSettings?.indexDocumentation !== undefined) { + rawSettings.indexDocumentation = normalizeDocumentationIndexTiming(rawSettings.indexDocumentation) + } + this.settings = rawSettings } if (shouldCompare) { diff --git a/src/providers/hover/HoverSupportProvider.ts b/src/providers/hover/HoverSupportProvider.ts index 83cb3db..3ebbf7a 100644 --- a/src/providers/hover/HoverSupportProvider.ts +++ b/src/providers/hover/HoverSupportProvider.ts @@ -126,7 +126,7 @@ class HoverSupportProvider { * Lazily opens and caches connection to local SQLite documentation database. */ private getDatabase (): ISqliteDatabase | null { - if (this.db !== undefined) { + if (this.db != null) { return this.db } try { @@ -144,6 +144,20 @@ class HoverSupportProvider { return null } + /** + * Resets SQLite database connection so it can be re-opened after indexing. + */ + public resetDatabaseConnection (): void { + if (this.db?.close != null) { + try { + this.db.close() + } catch { + // ignore + } + } + this.db = null + } + /** * Extracts word and range at the given position. */ diff --git a/src/providers/lspCommands/ExecuteCommandProvider.ts b/src/providers/lspCommands/ExecuteCommandProvider.ts index e3cb183..f696f9f 100644 --- a/src/providers/lspCommands/ExecuteCommandProvider.ts +++ b/src/providers/lspCommands/ExecuteCommandProvider.ts @@ -3,6 +3,7 @@ import { ExecuteCommandParams, Range, TextDocuments } from 'vscode-languageserver' import { TextDocument } from 'vscode-languageserver-textdocument' import LintingSupportProvider from '../linting/LintingSupportProvider' +import { DocumentationIndexer } from '../../indexing/DocumentationIndexer' interface LintSuppressionArgs { id: string @@ -12,14 +13,18 @@ interface LintSuppressionArgs { export const MatlabLSCommands = { MLINT_SUPPRESS_ON_LINE: 'matlabls.lint.suppress.line', - MLINT_SUPPRESS_IN_FILE: 'matlabls.lint.suppress.file' + MLINT_SUPPRESS_IN_FILE: 'matlabls.lint.suppress.file', + INDEX_DOCUMENTATION: 'matlabls.indexDocumentation' } /** * Handles requests to execute commands */ class ExecuteCommandProvider { - constructor (private readonly lintingSupportProvider: LintingSupportProvider) {} + constructor ( + private readonly lintingSupportProvider: LintingSupportProvider, + private readonly documentationIndexer?: DocumentationIndexer + ) {} /** * Handles command execution requests. @@ -33,6 +38,10 @@ class ExecuteCommandProvider { case MatlabLSCommands.MLINT_SUPPRESS_ON_LINE: case MatlabLSCommands.MLINT_SUPPRESS_IN_FILE: void this.handleLintingSuppression(params, documentManager) + break + case MatlabLSCommands.INDEX_DOCUMENTATION: + void this.documentationIndexer?.startIndexing(true) + break } } diff --git a/src/server.ts b/src/server.ts index e9c7141..af55b48 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4,6 +4,7 @@ import { TextDocument } from 'vscode-languageserver-textdocument' import { InitializeParams, InitializeResult, TextDocuments, SemanticTokensRequest, SemanticTokensParams } from 'vscode-languageserver/node' import DocumentIndexer from './indexing/DocumentIndexer' import WorkspaceIndexer from './indexing/WorkspaceIndexer' +import documentationIndexer from './indexing/DocumentationIndexer' import ClientCapabilitiesManager from './lifecycle/ClientCapabilitiesManager' import ConfigurationManager, { ConnectionTiming } from './lifecycle/ConfigurationManager' import MatlabLifecycleManager from './lifecycle/MatlabLifecycleManager' @@ -76,13 +77,16 @@ export async function startServer (): Promise { const formatSupportProvider = new FormatSupportProvider(matlabLifecycleManager, mvm) const foldingSupportProvider = new FoldingSupportProvider(matlabLifecycleManager, mvm) const lintingSupportProvider = new LintingSupportProvider(matlabLifecycleManager, mvm) - const executeCommandProvider = new ExecuteCommandProvider(lintingSupportProvider) + const hoverSupportProvider = new HoverSupportProvider(matlabLifecycleManager, mvm) + documentationIndexer.eventEmitter.on('indexed', () => { + hoverSupportProvider.resetDatabaseConnection() + }) + const executeCommandProvider = new ExecuteCommandProvider(lintingSupportProvider, documentationIndexer) const completionSupportProvider = new CompletionSupportProvider(matlabLifecycleManager, mvm) const navigationSupportProvider = new NavigationSupportProvider(matlabLifecycleManager, fileInfoIndex, indexer, documentIndexer, pathResolver) const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex) const semanticTokensProvider = new SemanticTokensProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) - const hoverSupportProvider = new HoverSupportProvider(matlabLifecycleManager, mvm) const projectEventNotifier = new ProjectEventNotifier(matlabLifecycleManager) @@ -102,7 +106,7 @@ export async function startServer (): Promise { mvm.on(IMVM.Events.stateChange, (state: MatlabMVMConnectionState) => { if (state === MatlabMVMConnectionState.CONNECTED) { // Handle when the MVM has connected - mvm.feval('matlabls.utils.startupHelper', 0, []) + void mvm.feval('matlabls.utils.startupHelper', 0, []) // Initiate workspace indexing void workspaceIndexer.indexWorkspace() @@ -176,6 +180,7 @@ export async function startServer (): Promise { ConfigurationManager.addSettingCallback('signIn', handleSignInChanged) ConfigurationManager.addSettingCallback('installPath', handleInstallPathSettingChanged) ConfigurationManager.addSettingCallback('defaultEditor', configuration => handleDefaultEditorConfigChange(configuration, mvm)) + ConfigurationManager.addSettingCallback('indexDocumentation', () => { void documentationIndexer.startIndexing() }) const configuration = await ConfigurationManager.getConfiguration() @@ -203,9 +208,10 @@ export async function startServer (): Promise { } void startMatlabIfOnStartLaunch() + void documentationIndexer.startIndexing() // Connect to Workspace Browser - matlabLifecycleManager.eventEmitter.on('connected', async ()=> { + matlabLifecycleManager.eventEmitter.on('connected', async () => { const connection = await matlabLifecycleManager.getMatlabConnection(); connection?.subscribe('/MobileWSB/ServerMsg', (data) => { NotificationService.sendNotification(Notification.WSBServerMessage, data); @@ -298,7 +304,7 @@ export async function startServer (): Promise { reportFileOpened(params.document) void lintingSupportProvider.lintDocument(params.document) void documentIndexer.indexDocument(params.document) - + void navigationSupportProvider.handleDocumentSymbol(params.document.uri, documentManager, RequestType.DocumentSymbol) }) diff --git a/src/utils/CliUtils.ts b/src/utils/CliUtils.ts index e844045..77ce7fb 100644 --- a/src/utils/CliUtils.ts +++ b/src/utils/CliUtils.ts @@ -8,6 +8,7 @@ export interface CliArgs { [Argument.MatlabInstallationPath]?: string [Argument.MatlabConnectionTiming]?: string [Argument.ShouldIndexWorkspace]?: boolean + [Argument.IndexDocumentation]?: string [Argument.MatlabUrl]?: string [Argument.SnippetIgnoreList]?: string } @@ -36,6 +37,12 @@ function makeParser (): yargs.Argv { default: false, description: 'Whether or not the user\'s workspace should be indexed.', requiresArg: false + }).option(Argument.IndexDocumentation, { + type: 'string', + default: 'onMissing', + choices: ['onMissing', 'never', 'always'], + description: 'When the language server should index MATLAB documentation into the SQLite cache.', + requiresArg: false }).option(Argument.MatlabUrl, { type: 'string', description: 'URL for communicating with an existing MATLAB instance', diff --git a/tests/indexing/DocumentationIndexer.test.ts b/tests/indexing/DocumentationIndexer.test.ts new file mode 100644 index 0000000..0781140 --- /dev/null +++ b/tests/indexing/DocumentationIndexer.test.ts @@ -0,0 +1,108 @@ +// Copyright 2026 The MathWorks, Inc. + +import assert from 'assert' +import sinon from 'sinon' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { DocumentationIndexer } from '../../src/indexing/DocumentationIndexer' +import ConfigurationManager, { DocumentationIndexTiming } from '../../src/lifecycle/ConfigurationManager' + +describe('DocumentationIndexer', () => { + let indexer: DocumentationIndexer + + beforeEach(() => { + indexer = new DocumentationIndexer() + }) + + afterEach(() => { + sinon.restore() + }) + + describe('#isDatabaseReady', () => { + const testFile = path.join(os.tmpdir(), `test_db_${Date.now()}.db`) + + afterEach(() => { + if (fs.existsSync(testFile)) { + fs.unlinkSync(testFile) + } + }) + + it('should return false if the database file does not exist', () => { + assert.strictEqual(indexer.isDatabaseReady(testFile), false) + }) + + it('should return false if the database file exists but has size 0', () => { + fs.writeFileSync(testFile, '') + assert.strictEqual(indexer.isDatabaseReady(testFile), false) + }) + + it('should return true if the database file exists and is greater than 0 bytes', () => { + fs.writeFileSync(testFile, 'sample data') + assert.strictEqual(indexer.isDatabaseReady(testFile), true) + }) + }) + + describe('#startIndexing', () => { + it('should skip indexing when configured to never index', async () => { + sinon.stub(ConfigurationManager, 'getConfiguration').resolves({ + installPath: '', + matlabConnectionTiming: 'never' as any, + indexWorkspace: false, + indexDocumentation: DocumentationIndexTiming.Never, + telemetry: false, + maxFileSizeForAnalysis: 0, + signIn: false, + prewarmGraphics: false, + defaultEditor: false + }) + + const spawned = await indexer.startIndexing(false) + assert.strictEqual(spawned, false) + assert.strictEqual(indexer.isIndexingInProgress(), false) + }) + + it('should skip indexing when configured to onMissing and database is ready', async () => { + sinon.stub(ConfigurationManager, 'getConfiguration').resolves({ + installPath: '', + matlabConnectionTiming: 'never' as any, + indexWorkspace: false, + indexDocumentation: DocumentationIndexTiming.OnMissing, + telemetry: false, + maxFileSizeForAnalysis: 0, + signIn: false, + prewarmGraphics: false, + defaultEditor: false + }) + sinon.stub(indexer, 'isDatabaseReady').returns(true) + + const spawned = await indexer.startIndexing(false) + assert.strictEqual(spawned, false) + }) + + it('should return false if script path is not found', async () => { + sinon.stub(ConfigurationManager, 'getConfiguration').resolves({ + installPath: '', + matlabConnectionTiming: 'never' as any, + indexWorkspace: false, + indexDocumentation: DocumentationIndexTiming.OnMissing, + telemetry: false, + maxFileSizeForAnalysis: 0, + signIn: false, + prewarmGraphics: false, + defaultEditor: false + }) + sinon.stub(indexer, 'isDatabaseReady').returns(false) + sinon.stub(indexer, 'getIndexerScriptPath').returns(null) + + const spawned = await indexer.startIndexing(false) + assert.strictEqual(spawned, false) + }) + + it('should not start if indexing is already in progress', async () => { + sinon.stub(indexer, 'isIndexingInProgress').returns(true) + const spawned = await indexer.startIndexing(true) + assert.strictEqual(spawned, false) + }) + }) +}) diff --git a/tests/providers/lspCommands/ExecuteCommandProvider.test.ts b/tests/providers/lspCommands/ExecuteCommandProvider.test.ts new file mode 100644 index 0000000..d150af3 --- /dev/null +++ b/tests/providers/lspCommands/ExecuteCommandProvider.test.ts @@ -0,0 +1,46 @@ +// Copyright 2026 The MathWorks, Inc. + +import assert from 'assert' +import sinon from 'sinon' +import { TextDocuments } from 'vscode-languageserver' +import { TextDocument } from 'vscode-languageserver-textdocument' +import ExecuteCommandProvider, { MatlabLSCommands } from '../../../src/providers/lspCommands/ExecuteCommandProvider' +import { DocumentationIndexer } from '../../../src/indexing/DocumentationIndexer' +import LintingSupportProvider from '../../../src/providers/linting/LintingSupportProvider' +import MatlabLifecycleManager from '../../../src/lifecycle/MatlabLifecycleManager' +import getMockMvm from '../../mocks/Mvm.mock' + +describe('ExecuteCommandProvider', () => { + let executeCommandProvider: ExecuteCommandProvider + let lintingSupportProvider: LintingSupportProvider + let documentationIndexer: DocumentationIndexer + let documentManager: TextDocuments + + beforeEach(() => { + const lifecycleManager = new MatlabLifecycleManager() + const mockMvm = getMockMvm() + lintingSupportProvider = new LintingSupportProvider(lifecycleManager, mockMvm) + documentationIndexer = new DocumentationIndexer() + executeCommandProvider = new ExecuteCommandProvider(lintingSupportProvider, documentationIndexer) + documentManager = new TextDocuments(TextDocument) + }) + + afterEach(() => { + sinon.restore() + }) + + it('should trigger forced documentation indexing when receiving INDEX_DOCUMENTATION command', async () => { + const startIndexingStub = sinon.stub(documentationIndexer, 'startIndexing').resolves(true) + + await executeCommandProvider.handleExecuteCommand( + { + command: MatlabLSCommands.INDEX_DOCUMENTATION, + arguments: [] + }, + documentManager + ) + + assert.strictEqual(startIndexingStub.calledOnce, true) + assert.strictEqual(startIndexingStub.firstCall.args[0], true) // force = true + }) +}) diff --git a/tools/indexer/index_docs.js b/tools/indexer/index_docs.js index 4d927ce..0a2d08a 100755 --- a/tools/indexer/index_docs.js +++ b/tools/indexer/index_docs.js @@ -91,16 +91,22 @@ functionList.forEach((fn, idx) => { chunks[idx % numWorkers].push(fn); }); -console.clear(); -console.log('\x1b[1;36m========================================================================\x1b[0m'); -console.log('\x1b[1;32m MATLAB Language Server - Parallel Documentation Indexer \x1b[0m'); -console.log('\x1b[1;36m========================================================================\x1b[0m'); -console.log(` \x1b[1mSystem CPU Cores:\x1b[0m ${cpuCount} (${os.cpus()[0].model})`); -console.log(` \x1b[1mAllocated Workers:\x1b[0m ${numWorkers} parallel workers`); -console.log(` \x1b[1mMATLAB Path:\x1b[0m ${matlabRoot}`); -console.log(` \x1b[1mUnique Functions:\x1b[0m ${totalFunctions}`); -console.log(` \x1b[1mSQLite Target:\x1b[0m ${dbPath}`); -console.log('\x1b[1;36m------------------------------------------------------------------------\x1b[0m\n'); +const isQuiet = process.argv.includes('--quiet') || !process.stdout.isTTY; + +if (!isQuiet) { + console.clear(); + console.log('\x1b[1;36m========================================================================\x1b[0m'); + console.log('\x1b[1;32m MATLAB Language Server - Parallel Documentation Indexer \x1b[0m'); + console.log('\x1b[1;36m========================================================================\x1b[0m'); + console.log(` \x1b[1mSystem CPU Cores:\x1b[0m ${cpuCount} (${os.cpus()[0].model})`); + console.log(` \x1b[1mAllocated Workers:\x1b[0m ${numWorkers} parallel workers`); + console.log(` \x1b[1mMATLAB Path:\x1b[0m ${matlabRoot}`); + console.log(` \x1b[1mUnique Functions:\x1b[0m ${totalFunctions}`); + console.log(` \x1b[1mSQLite Target:\x1b[0m ${dbPath}`); + console.log('\x1b[1;36m------------------------------------------------------------------------\x1b[0m\n'); +} else { + console.log(`[matlabls-indexer] Indexing ${totalFunctions} canonical MATLAB functions into ${dbPath} using ${numWorkers} workers.`); +} let processedCount = 0; const startTime = Date.now(); @@ -108,6 +114,7 @@ const workerStatus = Array.from({ length: numWorkers }, () => 'Initializing'); const activeWorkers = new Set(); function renderDashboard() { + if (isQuiet) return; const elapsedSec = (Date.now() - startTime) / 1000; const speed = elapsedSec > 0 ? (processedCount / elapsedSec) : 0; const remaining = totalFunctions - processedCount; @@ -230,10 +237,14 @@ Promise.all(workerPromises).then((outFiles) => { const dbStat = fs.statSync(dbPath); const dbSizeMb = (dbStat.size / (1024 * 1024)).toFixed(2); - console.log('\x1b[1;32m✔ Indexing and SQLite insertion complete!\x1b[0m'); - console.log(` \x1b[1mTotal Functions Indexed:\x1b[0m ${recordCount}`); - console.log(` \x1b[1mDatabase Size:\x1b[0m ${dbSizeMb} MB`); - console.log(` \x1b[1mTotal Time Elapsed:\x1b[0m ${totalElapsedSec} seconds`); - console.log(` \x1b[1mDatabase File Location:\x1b[0m ${dbPath}`); - console.log('\x1b[1;36m========================================================================\x1b[0m'); + if (isQuiet) { + console.log(`[matlabls-indexer] Indexing complete! ${recordCount} functions indexed in ${totalElapsedSec}s (${dbSizeMb} MB) -> ${dbPath}`); + } else { + console.log('\x1b[1;32m✔ Indexing and SQLite insertion complete!\x1b[0m'); + console.log(` \x1b[1mTotal Functions Indexed:\x1b[0m ${recordCount}`); + console.log(` \x1b[1mDatabase Size:\x1b[0m ${dbSizeMb} MB`); + console.log(` \x1b[1mTotal Time Elapsed:\x1b[0m ${totalElapsedSec} seconds`); + console.log(` \x1b[1mDatabase File Location:\x1b[0m ${dbPath}`); + console.log('\x1b[1;36m========================================================================\x1b[0m'); + } }); From ad12be82c91f500359dc7c6a5154e0a94ecdb0e7 Mon Sep 17 00:00:00 2001 From: "Ramazan G." Date: Sat, 12 Sep 2026 03:22:38 +0200 Subject: [PATCH 5/5] feat(indexing): add LSP WorkDoneProgress reporting to statusline --- src/indexing/DocumentationIndexer.ts | 46 ++++++++++++++++++++-- src/lifecycle/ClientCapabilitiesManager.ts | 5 +++ tools/indexer/index_docs.js | 14 +++++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/indexing/DocumentationIndexer.ts b/src/indexing/DocumentationIndexer.ts index d876344..3b1a708 100644 --- a/src/indexing/DocumentationIndexer.ts +++ b/src/indexing/DocumentationIndexer.ts @@ -5,6 +5,9 @@ import { EventEmitter } from 'events' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' +import { WorkDoneProgressServerReporter } from 'vscode-languageserver' +import ClientConnection from '../ClientConnection' +import ClientCapabilitiesManager from '../lifecycle/ClientCapabilitiesManager' import ConfigurationManager, { DocumentationIndexTiming } from '../lifecycle/ConfigurationManager' import Logger from '../logging/Logger' @@ -101,15 +104,49 @@ export class DocumentationIndexer { Logger.log(`Spawning background documentation indexer: ${scriptPath}`) this.isIndexing = true + let progressReporter: WorkDoneProgressServerReporter | null = null + if (ClientCapabilitiesManager.hasWorkDoneProgress()) { + try { + const connection = ClientConnection.getConnection() + const progressPromise = connection.window.createWorkDoneProgress() + const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(null), 2000)) + progressReporter = await Promise.race([progressPromise, timeoutPromise]) + progressReporter?.begin('Indexing MATLAB Documentation', 0, 'Initializing indexer...') + } catch (err) { + Logger.log(`Failed to create workDoneProgress reporter: ${String(err)}`) + progressReporter = null + } + } + const child = spawn(process.execPath, [scriptPath, '--quiet'], { env, stdio: ['ignore', 'pipe', 'pipe'] }) + let stdoutBuffer = '' child.stdout?.on('data', (chunk: Buffer) => { - const msg = chunk.toString().trim() - if (msg.length > 0) { - Logger.log(`[Indexer] ${msg}`) + stdoutBuffer += chunk.toString() + const lines = stdoutBuffer.split('\n') + stdoutBuffer = lines.pop() ?? '' + + for (const rawLine of lines) { + const line = rawLine.trim() + if (line.length === 0) continue + + if (line.startsWith('LSP_PROGRESS:')) { + const parts = line.split(':') + const current = parseInt(parts[1], 10) + const total = parseInt(parts[2], 10) + const pct = parseInt(parts[3], 10) + if (!isNaN(pct)) { + progressReporter?.report(pct, `${current}/${total} functions (${pct}%)`) + } + } else if (line.startsWith('LSP_STAGE:')) { + const stage = line.substring('LSP_STAGE:'.length) + progressReporter?.report(95, stage) + } else { + Logger.log(`[Indexer] ${line}`) + } } }) @@ -124,14 +161,17 @@ export class DocumentationIndexer { this.isIndexing = false if (code === 0) { Logger.log('MATLAB documentation indexing completed successfully.') + progressReporter?.done() this.eventEmitter.emit('indexed') } else { Logger.warn(`MATLAB documentation indexer exited with code ${code ?? 'unknown'}`) + progressReporter?.done() } }) child.on('error', (err: Error) => { this.isIndexing = false + progressReporter?.done() Logger.error(`Failed to execute MATLAB documentation indexer: ${err.message}`) }) diff --git a/src/lifecycle/ClientCapabilitiesManager.ts b/src/lifecycle/ClientCapabilitiesManager.ts index e4bc517..bed7978 100644 --- a/src/lifecycle/ClientCapabilitiesManager.ts +++ b/src/lifecycle/ClientCapabilitiesManager.ts @@ -47,6 +47,11 @@ class ClientCapabilitiesManager { return this.getCapabilities()?.workspace?.semanticTokens?.refreshSupport === true } + /** Whether the client supports workDoneProgress notifications in the status line / UI. */ + hasWorkDoneProgress (): boolean { + return this.getCapabilities()?.window?.workDoneProgress === true + } + /** Private getter which allows for logging a warning if not yet initialized. */ private getCapabilities (): ClientCapabilities | null { if (this.clientCapabilities == null) { diff --git a/tools/indexer/index_docs.js b/tools/indexer/index_docs.js index 0a2d08a..6c47aad 100755 --- a/tools/indexer/index_docs.js +++ b/tools/indexer/index_docs.js @@ -109,6 +109,8 @@ if (!isQuiet) { } let processedCount = 0; +let lastReportedCount = 0; +let lastReportedPct = -1; const startTime = Date.now(); const workerStatus = Array.from({ length: numWorkers }, () => 'Initializing'); const activeWorkers = new Set(); @@ -171,6 +173,15 @@ const workerPromises = chunks.map((chunk, workerIdx) => { processedCount++; workerStatus[workerIdx] = fn; renderDashboard(); + + if (isQuiet) { + const pct = totalFunctions > 0 ? Math.floor((processedCount / totalFunctions) * 100) : 0; + if (processedCount - lastReportedCount >= 40 || pct >= lastReportedPct + 4 || processedCount === totalFunctions) { + lastReportedCount = processedCount; + lastReportedPct = pct; + console.log(`LSP_PROGRESS:${processedCount}:${totalFunctions}:${pct}`); + } + } } } }); @@ -199,6 +210,9 @@ Promise.all(workerPromises).then((outFiles) => { renderDashboard(); const importStartTime = Date.now(); + if (isQuiet) { + console.log('LSP_STAGE:Writing documentation records to SQLite database...'); + } console.log('\n\x1b[1;36m------------------------------------------------------------------------\x1b[0m'); console.log(' \x1b[1;33mWriting documentation records to SQLite database...\x1b[0m');