Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 81 additions & 0 deletions api/routers/agent.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
"""
Agent chat endpoint — runs an Ollama-powered tool-use loop against Modly's API.
"""
import json
import re
import uuid
import httpx
from fastapi import APIRouter
from pydantic import BaseModel

import services.generator_registry as reg_module

router = APIRouter(prefix="/agent", tags=["agent"])

MODLY_API = "http://localhost:8765"
Expand Down Expand Up @@ -408,6 +411,60 @@ def _extract_thinking(msg: dict) -> tuple[str, str | None]:
return content, thinking


def _read_glb_modly_extras(glb_path) -> dict | None:
"""Parse a GLB's leading JSON chunk and return asset.extras.modly, if present.
Used only as a fallback for assets that predate the .tags.json sidecar."""
try:
with open(glb_path, "rb") as f:
header = f.read(12)
if len(header) < 12 or header[0:4] != b"glTF":
return None
chunk_header = f.read(8)
if len(chunk_header) < 8 or chunk_header[4:8] != b"JSON":
return None
chunk_length = int.from_bytes(chunk_header[0:4], "little")
doc = json.loads(f.read(chunk_length))
return (doc.get("asset") or {}).get("extras", {}).get("modly")
except Exception:
return None


def _load_asset_meta(workspace_rel_path: str) -> dict | None:
"""Resolve name/project/tags/lineage for the model currently in the viewer.

Reads the .tags.json sidecar when present, falling back to the GLB's own
embedded extras.modly for name/project/tags. Lineage (derived_from) is only
ever written to the sidecar, so an asset with no sidecar has none to report.
"""
workspace_dir = reg_module.WORKSPACE_DIR.resolve()
abs_path = (workspace_dir / workspace_rel_path).resolve()
if not str(abs_path).startswith(str(workspace_dir)):
return None # escapes the workspace — refuse to read

sidecar_path = abs_path.with_suffix(".tags.json")
if sidecar_path.exists():
try:
data = json.loads(sidecar_path.read_text(encoding="utf-8"))
return {
"name": data.get("name"),
"project": data.get("project"),
"tags": data.get("tags") or [],
"derived_from": data.get("derived_from"),
}
except Exception:
pass # malformed sidecar — fall through to the GLB fallback

modly = _read_glb_modly_extras(abs_path)
if modly:
return {
"name": modly.get("name"),
"project": modly.get("project"),
"tags": modly.get("tags") or [],
"derived_from": None,
}
return None


@router.get("/models")
async def list_ollama_models(ollama_url: str = "http://localhost:11434"):
async with httpx.AsyncClient(timeout=5.0) as client:
Expand All @@ -431,6 +488,30 @@ async def agent_chat(request: AgentChatRequest):
ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}")
if request.context.get("meshTriangles"):
ctx_lines.append(f"Current mesh triangles: {request.context['meshTriangles']:,}")
if request.context.get("currentClipName"):
ctx_lines.append(f"Current animation clip: {request.context['currentClipName']}")
if request.context.get("availableClips"):
ctx_lines.append(f"Available animation clips: {', '.join(request.context['availableClips'])}")

mesh_path = request.context.get("currentMeshPath")
meta = _load_asset_meta(mesh_path) if mesh_path else None
if meta:
if meta.get("name"):
ctx_lines.append(f"Model name: {meta['name']}")
if meta.get("project"):
ctx_lines.append(f"Project: {meta['project']}")
if meta.get("tags"):
ctx_lines.append(f"Tags: {', '.join(meta['tags'])}")
derived = meta.get("derived_from")
if derived:
parent = derived.get("parent") or {}
root = derived.get("root") or {}
parent_label = parent.get("name") or parent.get("path")
root_label = root.get("name") or root.get("path")
if parent_label == root_label:
ctx_lines.append(f"Derived from: {parent_label}")
else:
ctx_lines.append(f"Derived from: {parent_label} (originally: {root_label})")
if ctx_lines:
messages.append({
"role": "system",
Expand Down
27 changes: 27 additions & 0 deletions electron/main/artifact-registry-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
normalizeWorkspaceAssetPath,
openWorkspaceAssetLibraryEntry,
readWorkspaceAssetLibraryEntry,
readWorkspaceAssetLibraryThumbnail,
registerWorkspaceAssetLibraryIpcHandlers,
} from './artifact-registry-service.ts'

Expand Down Expand Up @@ -198,6 +199,27 @@ test('fails closed for unsafe, self, missing, and mismatched sourceWorkspacePath
assert.equal(!mismatched.success && mismatched.error.code, 'not-openable')
}))

test('reads a sibling .thumb.png for GLB assets and stays silent when one is missing', () => withWorkspace(async (workspaceDir) => {
await mkdir(path.join(workspaceDir, 'Exports/props'), { recursive: true })
await writeFile(path.join(workspaceDir, 'Exports/props/hero.glb'), 'glb')
await writeFile(path.join(workspaceDir, 'Exports/props/hero.thumb.png'), 'fake-png-bytes')
await writeFile(path.join(workspaceDir, 'Exports/props/no-thumb.glb'), 'glb')
await writeFile(path.join(workspaceDir, 'Exports/static.ply'), 'ply')

const withThumb = await readWorkspaceAssetLibraryThumbnail({ workspaceDir, workspacePath: 'Exports/props/hero.glb' })
assert.equal(withThumb.success, true)
assert.equal(withThumb.success && withThumb.dataUrl, `data:image/png;base64,${Buffer.from('fake-png-bytes').toString('base64')}`)

const withoutThumb = await readWorkspaceAssetLibraryThumbnail({ workspaceDir, workspacePath: 'Exports/props/no-thumb.glb' })
assert.equal(withoutThumb.success, false)

const nonMesh = await readWorkspaceAssetLibraryThumbnail({ workspaceDir, workspacePath: 'Exports/static.ply' })
assert.equal(nonMesh.success, false)

const unsafe = await readWorkspaceAssetLibraryThumbnail({ workspaceDir, workspacePath: '../secret.glb' })
assert.equal(unsafe.success, false)
}))

test('IPC read and open handlers forward sourceWorkspacePath without trusting malformed payloads', async () => {
const handlers = new Map<string, (event: unknown, payload: unknown) => Promise<unknown>>()
registerWorkspaceAssetLibraryIpcHandlers({
Expand All @@ -224,7 +246,12 @@ test('registers workspace library IPC handlers with structured results', async (
assert.equal(typeof handlers.get('workspace:library:list'), 'function')
assert.equal(typeof handlers.get('workspace:library:read'), 'function')
assert.equal(typeof handlers.get('workspace:library:open'), 'function')
assert.equal(typeof handlers.get('workspace:library:thumbnail'), 'function')
const result = await handlers.get('workspace:library:read')?.({}, { workspacePath: '../escape.glb' })
assert.equal(typeof (result as { success?: unknown }).success, 'boolean')
assert.equal((result as { success: boolean, error?: { code: string } }).error?.code, 'unsafe-path')
const thumbnail = await handlers.get('workspace:library:thumbnail')?.({}, { workspacePath: '../escape.glb' })
assert.equal((thumbnail as { success: boolean }).success, false)
const missingPayload = await handlers.get('workspace:library:thumbnail')?.({}, {})
assert.equal((missingPayload as { success: boolean }).success, false)
})
25 changes: 25 additions & 0 deletions electron/main/artifact-registry-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type {
AssetLibraryPreviewPayload,
AssetLibraryReadResult,
AssetLibrarySourceScope,
AssetLibraryThumbnailResult,
} from '../../src/shared/types/assetLibrary'
import type { ArtifactProvenance } from '../../src/shared/types/artifacts'

Expand Down Expand Up @@ -51,6 +52,10 @@ export interface WorkspaceAssetLibraryReadRequest extends WorkspaceAssetLibraryR
sourceWorkspacePath?: string
}

export interface WorkspaceAssetLibraryThumbnailRequest extends WorkspaceAssetLibraryRequest {
workspacePath: string
}

interface AssetLibraryMetadata {
sourceWorkspacePath?: string
manifestWorkspacePath?: string
Expand Down Expand Up @@ -342,6 +347,21 @@ export async function openWorkspaceAssetLibraryEntry(request: WorkspaceAssetLibr
return { success: true, entry: read.entry }
}

// Thumbnails are pre-rendered beside their source mesh (see thumbs.py): a
// `<name>.glb` asset may have a sibling `<name>.thumb.png`. Missing files are
// a normal, silent condition, never surfaced as an AssetLibraryError.
export async function readWorkspaceAssetLibraryThumbnail(request: WorkspaceAssetLibraryThumbnailRequest): Promise<AssetLibraryThumbnailResult> {
try {
const normalized = normalizeWorkspaceAssetPath(request.workspaceDir, request.workspacePath)
if (!isGlbOrGltf(normalized.workspacePath)) return { success: false }
const thumbnailAbsolutePath = normalized.absolutePath.replace(/\.(glb|gltf)$/i, '.thumb.png')
const buffer = await readFile(thumbnailAbsolutePath)
return { success: true, dataUrl: `data:image/png;base64,${buffer.toString('base64')}` }
} catch {
return { success: false }
}
}

function readPayloadRequest(payload: unknown): { workspacePath?: string, sourceWorkspacePath?: string } {
if (typeof payload !== 'object' || payload === null) return {}
const values = payload as { workspacePath?: unknown, sourceWorkspacePath?: unknown }
Expand All @@ -363,4 +383,9 @@ export function registerWorkspaceAssetLibraryIpcHandlers(deps: WorkspaceAssetLib
if (!workspacePath) return { success: false, error: libraryError('invalid-request', 'workspacePath is required.') }
return openWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), workspacePath, sourceWorkspacePath })
})
deps.ipcMain.handle('workspace:library:thumbnail', async (_event, payload) => {
const { workspacePath } = readPayloadRequest(payload)
if (!workspacePath) return { success: false }
return readWorkspaceAssetLibraryThumbnail({ workspaceDir: deps.getWorkspaceDir(), workspacePath })
})
}
59 changes: 59 additions & 0 deletions electron/main/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { app, BrowserWindow, shell, session } from 'electron'
import { join } from 'path'
import { readFileSync, writeFileSync } from 'fs'
import { electronApp, optimizer, is } from '@electron-toolkit/utils'
import { setupIpcHandlers } from './ipc-handlers'
import { PythonBridge } from './python-bridge'
Expand All @@ -16,6 +17,37 @@ let pythonBridge: PythonBridge | null = null
process.stdout?.on('error', () => {})
process.stderr?.on('error', () => {})

// UI zoom: Ctrl/Cmd with + / - / 0 scales the whole window like a browser,
// clamped to a sane range and persisted across restarts. This is done at the
// Chromium level (not CSS) so pointer math in the 3D viewport stays correct.
const ZOOM_MIN = -2
const ZOOM_MAX = 4

function zoomFilePath(): string {
return join(app.getPath('userData'), 'ui-zoom.json')
}

function loadZoomLevel(): number {
try {
const saved = JSON.parse(readFileSync(zoomFilePath(), 'utf-8')) as { level?: number }
return typeof saved.level === 'number' ? saved.level : 0
} catch {
return 0
}
}

function saveZoomLevel(level: number): void {
try {
writeFileSync(zoomFilePath(), JSON.stringify({ level }), 'utf-8')
} catch (err) {
logger.error(`Failed to persist UI zoom level: ${err}`)
}
}

function clampZoomLevel(level: number): number {
return Math.min(ZOOM_MAX, Math.max(ZOOM_MIN, level))
}

function createWindow(): void {
mainWindow = new BrowserWindow({
width: 1280,
Expand Down Expand Up @@ -57,6 +89,33 @@ function createWindow(): void {
event.preventDefault()
app.quit()
}

const isZoomChord = input.type === 'keyDown' && (input.control || input.meta) && !input.alt
if (!isZoomChord) return

const wc = mainWindow?.webContents
if (!wc) return

if (input.key === '+' || input.key === '=') {
event.preventDefault()
const level = clampZoomLevel(wc.getZoomLevel() + 0.5)
wc.setZoomLevel(level)
saveZoomLevel(level)
} else if (input.key === '-' || input.key === '_') {
event.preventDefault()
const level = clampZoomLevel(wc.getZoomLevel() - 0.5)
wc.setZoomLevel(level)
saveZoomLevel(level)
} else if (input.key === '0') {
event.preventDefault()
wc.setZoomLevel(0)
saveZoomLevel(0)
}
})

mainWindow.webContents.on('did-finish-load', () => {
const level = loadZoomLevel()
if (level) mainWindow?.webContents.setZoomLevel(level)
})

mainWindow.webContents.setWindowOpenHandler((details) => {
Expand Down
2 changes: 1 addition & 1 deletion electron/main/ipc-handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1395,7 +1395,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe
})

// Run a process extension in an isolated worker thread
ipcMain.handle('extensions:runProcess', async (_, extensionId: string, input: { filePath?: string; text?: string; texts?: (string | undefined)[]; nodeId?: string }, params: Record<string, unknown>) => {
ipcMain.handle('extensions:runProcess', async (_, extensionId: string, input: { filePath?: string; text?: string; texts?: (string | undefined)[]; nodeId?: string; sourceAssetPath?: string }, params: Record<string, unknown>) => {
const userData = app.getPath('userData')
const { extensionsDir, workspaceDir } = getSettings(userData)

Expand Down
3 changes: 3 additions & 0 deletions electron/main/process-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ export interface ProcessInput {
/** Per-slot texts for multi-text-input nodes (index = target handle slot). */
texts?: (string | undefined)[]
nodeId?: string
/** Absolute path of the existing workspace asset this input was sourced from
* (if any) — threaded through the run so mesh-exporter can record lineage. */
sourceAssetPath?: string
}

export interface ProcessResult {
Expand Down
2 changes: 2 additions & 0 deletions electron/preload/artifact-registry-preload.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ test('preload exposes scoped workspace library list/read/open methods', async ()
workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json',
sourceWorkspacePath: 'Workflows/checkpoints/hero.glb',
})
await api.workspace.library.thumbnail({ workspacePath: 'Exports/hero.glb' })

assert.deepEqual(calls, [
{ channel: 'workspace:library:list', payload: undefined },
Expand All @@ -41,5 +42,6 @@ test('preload exposes scoped workspace library list/read/open methods', async ()
sourceWorkspacePath: 'Workflows/checkpoints/hero.glb',
},
},
{ channel: 'workspace:library:thumbnail', payload: { workspacePath: 'Exports/hero.glb' } },
])
})
5 changes: 4 additions & 1 deletion electron/preload/electron-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import type {
AssetLibraryOpenResult,
AssetLibraryReadRequest,
AssetLibraryReadResult,
AssetLibraryThumbnailRequest,
AssetLibraryThumbnailResult,
} from '../../src/shared/types/assetLibrary'

export interface IpcRendererLike {
Expand Down Expand Up @@ -189,6 +191,7 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra
list: (): Promise<AssetLibraryListResult> => ipcRenderer.invoke('workspace:library:list') as Promise<AssetLibraryListResult>,
read: (request: AssetLibraryReadRequest): Promise<AssetLibraryReadResult> => ipcRenderer.invoke('workspace:library:read', request) as Promise<AssetLibraryReadResult>,
open: (request: AssetLibraryOpenRequest): Promise<AssetLibraryOpenResult> => ipcRenderer.invoke('workspace:library:open', request) as Promise<AssetLibraryOpenResult>,
thumbnail: (request: AssetLibraryThumbnailRequest): Promise<AssetLibraryThumbnailResult> => ipcRenderer.invoke('workspace:library:thumbnail', request) as Promise<AssetLibraryThumbnailResult>,
},
},

Expand Down Expand Up @@ -230,7 +233,7 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra

runProcess: (
extensionId: string,
input: { filePath?: string; text?: string; texts?: (string | undefined)[]; nodeId?: string },
input: { filePath?: string; text?: string; texts?: (string | undefined)[]; nodeId?: string; sourceAssetPath?: string },
params: Record<string, unknown>,
): Promise<{ success: boolean; result?: { filePath?: string; text?: string }; error?: string }> =>
ipcRenderer.invoke('extensions:runProcess', extensionId, input, params) as Promise<{ success: boolean; result?: { filePath?: string; text?: string }; error?: string }>,
Expand Down
Loading