diff --git a/cmd/mcp/semaphore.go b/cmd/mcp/semaphore.go new file mode 100644 index 000000000..5f7bb1127 --- /dev/null +++ b/cmd/mcp/semaphore.go @@ -0,0 +1,33 @@ +package mcp + +import ( + "context" + "fmt" +) + +type opSemaphore struct { + slots chan struct{} +} + +func newOpSemaphore(max int) *opSemaphore { + if max <= 0 { + max = 1 + } + return &opSemaphore{slots: make(chan struct{}, max)} +} + +func (s *opSemaphore) acquire(ctx context.Context) (func(), error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("waiting for a free operation slot: %w", err) + } + select { + case s.slots <- struct{}{}: + if err := ctx.Err(); err != nil { + <-s.slots + return nil, fmt.Errorf("waiting for a free operation slot: %w", err) + } + return func() { <-s.slots }, nil + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for a free operation slot: %w", ctx.Err()) + } +} diff --git a/cmd/mcp/semaphore_test.go b/cmd/mcp/semaphore_test.go new file mode 100644 index 000000000..d45a51a12 --- /dev/null +++ b/cmd/mcp/semaphore_test.go @@ -0,0 +1,96 @@ +package mcp + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestOpSemaphore_LimitsConcurrency(t *testing.T) { + sem := newOpSemaphore(2) + var inFlight, maxInFlight atomic.Int32 + + track := func() { + cur := inFlight.Add(1) + for { + m := maxInFlight.Load() + if cur <= m || maxInFlight.CompareAndSwap(m, cur) { + break + } + } + time.Sleep(20 * time.Millisecond) + inFlight.Add(-1) + } + + done := make(chan struct{}, 5) + for range 5 { + go func() { + release, err := sem.acquire(context.Background()) + if err != nil { + t.Errorf("acquire failed: %v", err) + done <- struct{}{} + return + } + track() + release() + done <- struct{}{} + }() + } + for range 5 { + <-done + } + + if got := maxInFlight.Load(); got > 2 { + t.Fatalf("max concurrent = %d, want <= 2", got) + } +} + +func TestOpSemaphore_ReleaseAllowsNextAcquire(t *testing.T) { + sem := newOpSemaphore(1) + release1, err := sem.acquire(context.Background()) + if err != nil { + t.Fatalf("first acquire failed: %v", err) + } + + acquired := make(chan struct{}) + go func() { + release2, err := sem.acquire(context.Background()) + if err != nil { + t.Errorf("second acquire failed: %v", err) + return + } + close(acquired) + release2() + }() + + select { + case <-acquired: + t.Fatal("second acquire succeeded while first slot was held") + case <-time.After(50 * time.Millisecond): + } + + release1() + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("second acquire never succeeded after release") + } +} + +func TestOpSemaphore_AcquireRespectsContextCancel(t *testing.T) { + sem := newOpSemaphore(1) + release, err := sem.acquire(context.Background()) + if err != nil { + t.Fatalf("first acquire failed: %v", err) + } + defer release() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + _, err = sem.acquire(ctx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("acquire error = %v, want context deadline exceeded", err) + } +} diff --git a/cmd/mcp/serve.go b/cmd/mcp/serve.go index de5ca2ca1..1a623b22f 100644 --- a/cmd/mcp/serve.go +++ b/cmd/mcp/serve.go @@ -21,6 +21,9 @@ type ServeCmd struct { ExecTimeoutDefault time.Duration ExecTimeoutMax time.Duration ExecOutputCap int + MaxConcurrentOps int + + opSem *opSemaphore } // NewServeCmd builds the `serve` subcommand. @@ -54,14 +57,21 @@ func NewServeCmd(globalFlags *flags.GlobalFlags) *cobra.Command { 100*1024, "Per-stream byte cap for workspace_exec output; excess is replaced with a truncation marker", ), + cliflags.Int( + &cmd.MaxConcurrentOps, + names.MaxConcurrentOps, + 8, + "Maximum number of concurrent workspace_exec/workspace_create/workspace_start "+ + "operations; excess calls wait for a free slot", + ), ) return cobraCmd } // Run wires up the MCP server and serves over stdio until ctx is cancelled. func (cmd *ServeCmd) Run(ctx context.Context) error { - log.Debugf("starting MCP server (timeout default=%s max=%s cap=%dB)", - cmd.ExecTimeoutDefault, cmd.ExecTimeoutMax, cmd.ExecOutputCap) + log.Debugf("starting MCP server (timeout default=%s max=%s cap=%dB maxops=%d)", + cmd.ExecTimeoutDefault, cmd.ExecTimeoutMax, cmd.ExecOutputCap, cmd.MaxConcurrentOps) // Reserve real stdout for the JSON-RPC frame; redirect os.Stdout to stderr // so any stray write elsewhere in the process can't corrupt the transport. @@ -79,13 +89,14 @@ func (cmd *ServeCmd) Run(ctx context.Context) error { Version: version.GetVersion(), }, nil) - cmd.registerTools(server) + cmd.opSem = newOpSemaphore(cmd.MaxConcurrentOps) + cmd.registerTools(server, cmd.opSem) return server.Run(ctx, transport) } -func (cmd *ServeCmd) registerTools(s *sdkmcp.Server) { - registerWorkspaceTools(s, cmd.GlobalFlags) - registerExecTool(s, cmd) +func (cmd *ServeCmd) registerTools(s *sdkmcp.Server, sem *opSemaphore) { + registerWorkspaceTools(s, cmd.GlobalFlags, sem) + registerExecTool(s, cmd, sem) registerProviderTools(s, cmd.GlobalFlags) } diff --git a/cmd/mcp/serve_test.go b/cmd/mcp/serve_test.go index d7ce8388c..aee0ede1c 100644 --- a/cmd/mcp/serve_test.go +++ b/cmd/mcp/serve_test.go @@ -2,6 +2,7 @@ package mcp import ( "context" + "strings" "testing" "time" @@ -19,7 +20,7 @@ func TestServer_ListsAllTools(t *testing.T) { server := sdkmcp.NewServer(&sdkmcp.Implementation{Name: "devsy-test", Version: "test"}, nil) g := &flags.GlobalFlags{} serveCmd := &ServeCmd{GlobalFlags: g, ExecOutputCap: 1024} - serveCmd.registerTools(server) + serveCmd.registerTools(server, newOpSemaphore(8)) clientTransport, serverTransport := sdkmcp.NewInMemoryTransports() @@ -57,3 +58,82 @@ func TestServer_ListsAllTools(t *testing.T) { t.Errorf("expected %d tools, got %d: %+v", len(wantNames), len(tools.Tools), have) } } + +func TestServer_WorkspaceExecRespectsSemaphore(t *testing.T) { + home := t.TempDir() + t.Setenv("DEVSY_HOME", home) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + server := sdkmcp.NewServer(&sdkmcp.Implementation{Name: "devsy-test", Version: "test"}, nil) + g := &flags.GlobalFlags{} + serveCmd := &ServeCmd{GlobalFlags: g, ExecOutputCap: 1024, MaxConcurrentOps: 1} + sem := newOpSemaphore(serveCmd.MaxConcurrentOps) + serveCmd.registerTools(server, sem) + + clientTransport, serverTransport := sdkmcp.NewInMemoryTransports() + + serverErr := make(chan error, 1) + go func() { + serverErr <- server.Run(ctx, serverTransport) + }() + + client := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "test-client", Version: "0"}, nil) + session, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = session.Close() }) + + execArgs := map[string]any{ + "name": "some-workspace", + "command": []string{"echo", "hi"}, + } + + release, err := sem.acquire(context.Background()) + if err != nil { + t.Fatalf("acquire: %v", err) + } + + blockedCtx, blockedCancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer blockedCancel() + _, callErr := session.CallTool(blockedCtx, &sdkmcp.CallToolParams{ + Name: "workspace_exec", + Arguments: execArgs, + }) + if callErr == nil { + t.Fatal("expected workspace_exec call to fail while the only semaphore slot is held") + } + + release() + + res, callErr := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: "workspace_exec", + Arguments: execArgs, + }) + if callErr != nil { + t.Fatalf( + "expected workspace_exec call to reach the handler after release, got transport error: %v", + callErr, + ) + } + assertNotSemaphoreError(t, res) +} + +// A non-semaphore error (e.g. workspace not found) is expected here. +func assertNotSemaphoreError(t *testing.T, res *sdkmcp.CallToolResult) { + t.Helper() + if !res.IsError { + return + } + var msg string + for _, c := range res.Content { + if tc, ok := c.(*sdkmcp.TextContent); ok { + msg = tc.Text + } + } + if strings.Contains(msg, "waiting for a free operation slot") { + t.Fatalf("workspace_exec failed due to the semaphore even after release: %s", msg) + } +} diff --git a/cmd/mcp/tools_exec.go b/cmd/mcp/tools_exec.go index 4cac9d24c..2905880ae 100644 --- a/cmd/mcp/tools_exec.go +++ b/cmd/mcp/tools_exec.go @@ -42,7 +42,7 @@ type execOutput struct { Error *ErrorPayload `json:"error,omitempty"` } -func registerExecTool(s *sdkmcp.Server, cmd *ServeCmd) { +func registerExecTool(s *sdkmcp.Server, cmd *ServeCmd, sem *opSemaphore) { sdkmcp.AddTool(s, &sdkmcp.Tool{ Name: "workspace_exec", Description: "Run a one-shot command in a running workspace container. The " + @@ -59,6 +59,13 @@ func registerExecTool(s *sdkmcp.Server, cmd *ServeCmd) { if len(in.Command) == 0 { return errorResult(fmt.Errorf("command is required")), execOutput{}, nil } + + release, err := sem.acquire(ctx) + if err != nil { + return errorResult(err), execOutput{}, nil + } + defer release() + stdout := NewBoundedBuffer(cmd.ExecOutputCap) stderr := NewBoundedBuffer(cmd.ExecOutputCap) diff --git a/cmd/mcp/tools_workspace.go b/cmd/mcp/tools_workspace.go index 787540e07..f99b7add0 100644 --- a/cmd/mcp/tools_workspace.go +++ b/cmd/mcp/tools_workspace.go @@ -33,7 +33,7 @@ type workspaceStatusInput struct { Name string `json:"name" jsonschema:"required"` } -func registerWorkspaceTools(s *sdkmcp.Server, g *flags.GlobalFlags) { +func registerWorkspaceTools(s *sdkmcp.Server, g *flags.GlobalFlags, sem *opSemaphore) { sdkmcp.AddTool(s, &sdkmcp.Tool{ Name: "workspace_list", Description: "List all Devsy workspaces with their provider, IDE, and source. " + @@ -64,7 +64,7 @@ func registerWorkspaceTools(s *sdkmcp.Server, g *flags.GlobalFlags) { return nil, out, nil })) - registerWorkspaceLifecycleTools(s, g) + registerWorkspaceLifecycleTools(s, g, sem) } func handleWorkspaceList(ctx context.Context, g *flags.GlobalFlags) (workspaceListOutput, error) { @@ -138,7 +138,14 @@ type createInput struct { DevcontainerPath string `json:"devcontainer_path,omitempty"` } -func registerWorkspaceLifecycleTools(s *sdkmcp.Server, g *flags.GlobalFlags) { +func registerWorkspaceLifecycleTools(s *sdkmcp.Server, g *flags.GlobalFlags, sem *opSemaphore) { + registerWorkspaceStartTool(s, g, sem) + registerWorkspaceStopTool(s, g) + registerWorkspaceDeleteTool(s, g) + registerWorkspaceCreateTool(s, g, sem) +} + +func registerWorkspaceStartTool(s *sdkmcp.Server, g *flags.GlobalFlags, sem *opSemaphore) { sdkmcp.AddTool(s, &sdkmcp.Tool{ Name: "workspace_start", Description: "Start (or resume) an existing workspace by name. The name must " + @@ -150,13 +157,20 @@ func registerWorkspaceLifecycleTools(s *sdkmcp.Server, g *flags.GlobalFlags) { if in.Name == "" { return errorResult(fmt.Errorf("name is required")), opOK{}, nil } + release, err := sem.acquire(ctx) + if err != nil { + return errorResult(err), opOK{}, nil + } + defer release() return opResultHandler(func() error { return streamLogsToSession(ctx, req.Session, func() error { return startWorkspace(ctx, g, in.Name) }) }) })) +} +func registerWorkspaceStopTool(s *sdkmcp.Server, g *flags.GlobalFlags) { sdkmcp.AddTool(s, &sdkmcp.Tool{ Name: "workspace_stop", Description: "Stop a running workspace by name. The name must match a workspace from workspace_list.", @@ -168,7 +182,9 @@ func registerWorkspaceLifecycleTools(s *sdkmcp.Server, g *flags.GlobalFlags) { } return opResultHandler(func() error { return stopWorkspace(ctx, g, in.Name) }) })) +} +func registerWorkspaceDeleteTool(s *sdkmcp.Server, g *flags.GlobalFlags) { sdkmcp.AddTool(s, &sdkmcp.Tool{ Name: "workspace_delete", Description: "Delete a workspace by name (must match workspace_list). " + @@ -181,7 +197,9 @@ func registerWorkspaceLifecycleTools(s *sdkmcp.Server, g *flags.GlobalFlags) { } return opResultHandler(func() error { return deleteWorkspace(ctx, g, in.Name, in.Force) }) })) +} +func registerWorkspaceCreateTool(s *sdkmcp.Server, g *flags.GlobalFlags, sem *opSemaphore) { sdkmcp.AddTool(s, &sdkmcp.Tool{ Name: "workspace_create", Description: "Create and start a new workspace. May take several minutes on " + @@ -197,10 +215,16 @@ func registerWorkspaceLifecycleTools(s *sdkmcp.Server, g *flags.GlobalFlags) { }, safeHandler(func( ctx context.Context, req *sdkmcp.CallToolRequest, in createInput, ) (*sdkmcp.CallToolResult, any, error) { - var ( - out any - err error - ) + if in.Source == "" { + return errorResult(fmt.Errorf("source is required")), nil, nil + } + release, err := sem.acquire(ctx) + if err != nil { + return errorResult(err), nil, nil + } + defer release() + + var out any streamErr := streamLogsToSession(ctx, req.Session, func() error { out, err = createWorkspace(ctx, g, in) return err diff --git a/cmd/workspace/up/up_test.go b/cmd/workspace/up/up_test.go index dd97ceec8..f3e24d457 100644 --- a/cmd/workspace/up/up_test.go +++ b/cmd/workspace/up/up_test.go @@ -7,7 +7,9 @@ import ( "testing" "github.com/devsy-org/devsy/cmd/flags" + "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/ide/opener" "github.com/google/go-containerregistry/pkg/registry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -440,3 +442,36 @@ func TestBuildUpCmd_DoesNotMutateCallerGlobalFlags(t *testing.T) { "caller's ResultFormat must remain untouched even after copy defaulted it", ) } + +func TestUpCmd_IDELaunchSkipImpliesIDENoneWhenIDEUnset(t *testing.T) { + cmd := &UpCmd{GlobalFlags: &flags.GlobalFlags{}} + cmd.IDELaunch = opener.LaunchSkip + cmd.IDE = "" + + err := cmd.validate() + require.NoError(t, err) + assert.Equal(t, string(config.IDENone), cmd.IDE, + "ide-launch=skip with no explicit --ide should default IDE to none, "+ + "so the container never downloads an IDE server binary") +} + +func TestUpCmd_IDELaunchSkipRespectsExplicitIDE(t *testing.T) { + cmd := &UpCmd{GlobalFlags: &flags.GlobalFlags{}} + cmd.IDELaunch = opener.LaunchSkip + cmd.IDE = "openvscode" + + err := cmd.validate() + require.NoError(t, err) + assert.Equal(t, "openvscode", cmd.IDE, + "an explicit --ide value must not be overridden even when launch is skipped") +} + +func TestUpCmd_IDELaunchAutoDoesNotTouchIDE(t *testing.T) { + cmd := &UpCmd{GlobalFlags: &flags.GlobalFlags{}} + cmd.IDELaunch = opener.LaunchAuto + cmd.IDE = "" + + err := cmd.validate() + require.NoError(t, err) + assert.Equal(t, "", cmd.IDE, "auto launch must not force an IDE default in validate()") +} diff --git a/cmd/workspace/up/up_validate.go b/cmd/workspace/up/up_validate.go index 08bd57913..e851d3917 100644 --- a/cmd/workspace/up/up_validate.go +++ b/cmd/workspace/up/up_validate.go @@ -6,9 +6,11 @@ import ( "strconv" "strings" + "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/ide/opener" ) const ( @@ -21,6 +23,8 @@ const ( ) func (cmd *UpCmd) validate() error { + cmd.applySkipLaunchIDEDefault() + if err := devcontainer.ResolveSourceSpec(&cmd.CLIOptions); err != nil { return err } @@ -46,6 +50,15 @@ func (cmd *UpCmd) validate() error { return validateRemoteUserUID(cmd.UpdateRemoteUserUIDDefault) } +// installIDE gates the container-side IDE server download on IDE name alone, +// not IDELaunch, so skipping launch without also defaulting IDE to none +// still downloads an unwanted binary. +func (cmd *UpCmd) applySkipLaunchIDEDefault() { + if cmd.IDELaunch == opener.LaunchSkip && cmd.IDE == "" { + cmd.IDE = string(config.IDENone) + } +} + func (cmd *UpCmd) validateUserEnvProbe() error { if cmd.DefaultUserEnvProbe == "" { return nil diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go index cfdb513fb..1f5a995cf 100644 --- a/e2e/e2e_suite_test.go +++ b/e2e/e2e_suite_test.go @@ -24,6 +24,7 @@ import ( _ "github.com/devsy-org/devsy/e2e/tests/logs" _ "github.com/devsy-org/devsy/e2e/tests/machine" _ "github.com/devsy-org/devsy/e2e/tests/machineprovider" + _ "github.com/devsy-org/devsy/e2e/tests/mcp" _ "github.com/devsy-org/devsy/e2e/tests/outdated" _ "github.com/devsy-org/devsy/e2e/tests/provider" _ "github.com/devsy-org/devsy/e2e/tests/rename" diff --git a/e2e/framework/mcp.go b/e2e/framework/mcp.go new file mode 100644 index 000000000..13b1a845a --- /dev/null +++ b/e2e/framework/mcp.go @@ -0,0 +1,214 @@ +package framework + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os/exec" + "path/filepath" + "sync" +) + +const jsonRPCVersion = "2.0" + +type MCPClient struct { + cmd *exec.Cmd + stdin *bufio.Writer + stdout *bufio.Reader + mu sync.Mutex + nextID int64 + poisoned bool + closeFn func() error +} + +type jsonRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + ID int64 `json:"id,omitempty"` + Method string `json:"method"` + Params any `json:"params,omitempty"` +} + +type jsonRPCResponse struct { + JSONRPC string `json:"jsonrpc"` + ID *int64 `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *jsonRPCError `json:"error,omitempty"` +} + +type jsonRPCError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// StartMCPServer launches `devsy mcp serve` and completes the MCP initialize +// handshake. Callers must call Close when done. +func (f *Framework) StartMCPServer(ctx context.Context) (*MCPClient, error) { + // #nosec G204 -- fixed subcommand args against the compiled test binary, not user input + cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), "mcp", "serve") + stdinPipe, err := cmd.StdinPipe() + if err != nil { + return nil, fmt.Errorf("stdin pipe: %w", err) + } + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("stdout pipe: %w", err) + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start devsy mcp serve: %w", err) + } + + cleanup := func() { + _ = stdinPipe.Close() + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + + c := &MCPClient{ + cmd: cmd, + stdin: bufio.NewWriter(stdinPipe), + stdout: bufio.NewReader(stdoutPipe), + closeFn: func() error { + _ = stdinPipe.Close() + return cmd.Wait() + }, + } + + initID := c.nextRequestID() + if err := c.send(jsonRPCRequest{ + JSONRPC: jsonRPCVersion, + ID: initID, + Method: "initialize", + Params: map[string]any{ + "protocolVersion": "2024-11-05", + "capabilities": map[string]any{}, + "clientInfo": map[string]any{"name": "devsy-e2e", "version": "0.1"}, + }, + }); err != nil { + cleanup() + return nil, err + } + if _, err := c.readResponseFor(initID); err != nil { + cleanup() + return nil, fmt.Errorf("initialize handshake: %w", err) + } + if err := c.send(jsonRPCRequest{ + JSONRPC: jsonRPCVersion, + Method: "notifications/initialized", + }); err != nil { + cleanup() + return nil, err + } + + return c, nil +} + +func (c *MCPClient) CallTool( + ctx context.Context, name string, args map[string]any, +) (map[string]any, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.poisoned { + return nil, false, fmt.Errorf( + "mcp client unusable after a prior call's context was cancelled mid-read", + ) + } + + id := c.nextRequestID() + if err := c.send(jsonRPCRequest{ + JSONRPC: jsonRPCVersion, + ID: id, + Method: "tools/call", + Params: map[string]any{"name": name, "arguments": args}, + }); err != nil { + return nil, false, err + } + resp, err := c.readResponseForCtx(ctx, id) + if err != nil { + return nil, false, err + } + if resp.Error != nil { + return nil, false, fmt.Errorf("jsonrpc error %d: %s", resp.Error.Code, resp.Error.Message) + } + + var result struct { + IsError bool `json:"isError"` + StructuredContent map[string]any `json:"structuredContent"` + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return nil, false, fmt.Errorf("unmarshal tool result: %w", err) + } + return result.StructuredContent, result.IsError, nil +} + +func (c *MCPClient) Close() error { + return c.closeFn() +} + +// nextRequestID must only be called while holding c.mu, except during +// StartMCPServer's handshake before the client is returned to any caller. +func (c *MCPClient) nextRequestID() int64 { + c.nextID++ + return c.nextID +} + +func (c *MCPClient) send(req jsonRPCRequest) error { + data, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("marshal request: %w", err) + } + if _, err := c.stdin.Write(data); err != nil { + return fmt.Errorf("write request: %w", err) + } + if err := c.stdin.WriteByte('\n'); err != nil { + return fmt.Errorf("write newline: %w", err) + } + return c.stdin.Flush() +} + +// readResponseFor reads and discards notifications (no id) until it finds +// the response matching id — tool calls that stream log progress interleave +// notifications with the eventual response on the same stream. +func (c *MCPClient) readResponseFor(id int64) (*jsonRPCResponse, error) { + for { + line, err := c.stdout.ReadBytes('\n') + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + var resp jsonRPCResponse + if err := json.Unmarshal(line, &resp); err != nil { + return nil, fmt.Errorf("unmarshal response %q: %w", line, err) + } + if resp.ID == nil || *resp.ID != id { + continue + } + return &resp, nil + } +} + +// readResponseForCtx races readResponseFor against ctx so a hung server +// can't block the caller past its deadline. bufio.Reader isn't safe for +// concurrent use, so a cancellation that fires while the read is still in +// flight leaves that goroutine's read outstanding on c.stdout — the client +// is marked poisoned and every later call on it fails fast instead of +// risking a second concurrent read. +func (c *MCPClient) readResponseForCtx(ctx context.Context, id int64) (*jsonRPCResponse, error) { + type result struct { + resp *jsonRPCResponse + err error + } + done := make(chan result, 1) + go func() { + resp, err := c.readResponseFor(id) + done <- result{resp, err} + }() + + select { + case r := <-done: + return r.resp, r.err + case <-ctx.Done(): + c.poisoned = true + return nil, fmt.Errorf("waiting for response to request %d: %w", id, ctx.Err()) + } +} diff --git a/e2e/tests/ide/skip_launch_no_install.go b/e2e/tests/ide/skip_launch_no_install.go new file mode 100644 index 000000000..cf7e74eab --- /dev/null +++ b/e2e/tests/ide/skip_launch_no_install.go @@ -0,0 +1,39 @@ +//go:build !windows + +package ide + +import ( + "context" + "os" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/config" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("devsy up --ide-launch=skip", ginkgo.Label("ide"), ginkgo.Ordered, func() { + var initialDir string + + ginkgo.BeforeEach(func() { + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It("defaults the resolved IDE to none when --ide is omitted", + func(ctx context.Context) { + f, tempDir := setupBrowserIDE(ctx, initialDir) + + err := f.DevsyUpWithIDE(ctx, "--ide-launch=skip", tempDir) + framework.ExpectNoError(err) + + ws, err := f.FindWorkspace(ctx, tempDir) + framework.ExpectNoError(err) + gomega.Expect(ws).NotTo(gomega.BeNil()) + gomega.Expect(ws.IDE.Name).To(gomega.Equal(string(config.IDENone)), + "--ide-launch=skip without an explicit --ide must resolve IDE to "+ + "none, so installIDE never downloads an IDE server binary "+ + "container-side") + }, ginkgo.SpecTimeout(framework.TimeoutShort())) +}) diff --git a/e2e/tests/mcp/helper.go b/e2e/tests/mcp/helper.go new file mode 100644 index 000000000..a69c3005e --- /dev/null +++ b/e2e/tests/mcp/helper.go @@ -0,0 +1,37 @@ +package mcp + +import ( + "context" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" +) + +func setupWorkspace(testdataPath, initialDir string) (string, *framework.Framework, error) { + tempDir, err := framework.CopyToTempDir(testdataPath) + if err != nil { + return "", nil, err + } + + f, err := framework.SetupDockerProvider(initialDir+"/bin", "docker") + if err != nil { + return "", nil, err + } + + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) + ginkgo.DeferCleanup(f.DevsyWorkspaceDelete, tempDir) + + return tempDir, f, nil +} + +func setupWorkspaceAndUp( + ctx context.Context, + testdataPath, initialDir string, +) (string, *framework.Framework, error) { + tempDir, f, err := setupWorkspace(testdataPath, initialDir) + if err != nil { + return "", nil, err + } + + return tempDir, f, f.DevsyUp(ctx, tempDir) +} diff --git a/e2e/tests/mcp/mcp.go b/e2e/tests/mcp/mcp.go new file mode 100644 index 000000000..4ef8742bd --- /dev/null +++ b/e2e/tests/mcp/mcp.go @@ -0,0 +1,61 @@ +package mcp + +import ( + "context" + "os" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("devsy mcp serve", ginkgo.Label("mcp"), ginkgo.Ordered, func() { + var initialDir string + + ginkgo.BeforeEach(func() { + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It("lists a running workspace via workspace_list and execs a command via workspace_exec", + func(ctx context.Context) { + tempDir, f, err := setupWorkspaceAndUp(ctx, "tests/mcp/testdata/basic", initialDir) + framework.ExpectNoError(err) + + client, err := f.StartMCPServer(ctx) + framework.ExpectNoError(err) + defer func() { _ = client.Close() }() + + listResult, isErr, err := client.CallTool(ctx, "workspace_list", map[string]any{}) + framework.ExpectNoError(err) + gomega.Expect(isErr).To(gomega.BeFalse()) + workspaces, ok := listResult["workspaces"].([]any) + gomega.Expect(ok).To(gomega.BeTrue()) + gomega.Expect(workspaces).NotTo(gomega.BeEmpty()) + + execResult, isErr, err := client.CallTool(ctx, "workspace_exec", map[string]any{ + "name": tempDir, + "command": []string{"echo", "-n", "hello-from-mcp"}, + }) + framework.ExpectNoError(err) + gomega.Expect(isErr).To(gomega.BeFalse()) + gomega.Expect(execResult["stdout"]).To(gomega.Equal("hello-from-mcp")) + gomega.Expect(execResult["exit_code"]).To(gomega.BeNumerically("==", 0)) + }, ginkgo.SpecTimeout(framework.TimeoutShort())) + + ginkgo.It("returns an isError result for an unknown workspace name", + func(ctx context.Context) { + f := framework.NewDefaultFramework(initialDir + "/bin") + client, err := f.StartMCPServer(ctx) + framework.ExpectNoError(err) + defer func() { _ = client.Close() }() + + _, isErr, err := client.CallTool(ctx, "workspace_exec", map[string]any{ + "name": "definitely-not-a-real-workspace", + "command": []string{"echo", "hi"}, + }) + framework.ExpectNoError(err) + gomega.Expect(isErr).To(gomega.BeTrue()) + }, ginkgo.SpecTimeout(framework.TimeoutShort())) +}) diff --git a/e2e/tests/mcp/testdata/basic/.devcontainer/devcontainer.json b/e2e/tests/mcp/testdata/basic/.devcontainer/devcontainer.json new file mode 100644 index 000000000..b2446f418 --- /dev/null +++ b/e2e/tests/mcp/testdata/basic/.devcontainer/devcontainer.json @@ -0,0 +1,4 @@ +{ + "name": "MCP Test", + "image": "ghcr.io/devsy-org/test-images/base:ubuntu" +} diff --git a/pkg/flags/names/names.go b/pkg/flags/names/names.go index d3a79f02b..a8b1b01f8 100644 --- a/pkg/flags/names/names.go +++ b/pkg/flags/names/names.go @@ -320,6 +320,7 @@ const ( Hidden = "hidden" Interval = "interval" KeyFile = "key-file" + MaxConcurrentOps = "mcp-max-concurrent-ops" MaxDepth = "max-depth" Namespace = "namespace" Port = "port" diff --git a/pkg/provider/atomic.go b/pkg/provider/atomic.go index e729530c2..02fa8e74d 100644 --- a/pkg/provider/atomic.go +++ b/pkg/provider/atomic.go @@ -6,19 +6,10 @@ import ( "path/filepath" ) -// WriteFileAtomic writes data to path atomically by writing to a sibling -// temp file then renaming. POSIX rename(2) ensures concurrent readers see -// either the old or the new content, never a partially-written file — -// which is the guarantee callers of this helper rely on for config files -// like workspace.json. -// -// This helper does NOT guarantee crash durability: it syncs the temp file -// but not the parent directory, so a power loss between rename(2) -// returning and the directory entry being flushed could lose the rename. -// That tradeoff is acceptable for the config files this is used for -// (callers retry / re-resolve on the next run). If you need crash-safe -// persistence for a new caller, add a parent-directory fsync here and -// audit existing callers for the latency cost. +// WriteFileAtomic writes data to path atomically via a temp file and rename, +// so concurrent readers never see a partial write. On POSIX it also fsyncs +// the parent directory so the rename survives a crash; on Windows that sync +// is a no-op, so callers there should still tolerate re-resolving state. func WriteFileAtomic(path string, data []byte, perm os.FileMode) error { dir := filepath.Dir(path) tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") @@ -51,5 +42,8 @@ func WriteFileAtomic(path string, data []byte, perm os.FileMode) error { cleanup() return fmt.Errorf("rename temp file: %w", err) } + if err := syncDir(dir); err != nil { + return fmt.Errorf("sync parent dir: %w", err) + } return nil } diff --git a/pkg/provider/atomic_posix.go b/pkg/provider/atomic_posix.go new file mode 100644 index 000000000..00ade1457 --- /dev/null +++ b/pkg/provider/atomic_posix.go @@ -0,0 +1,17 @@ +//go:build !windows + +package provider + +import "os" + +// syncDir fsyncs a directory so a prior rename(2) into it survives a crash, +// not just becomes visible to concurrent readers. +func syncDir(dir string) error { + //nolint:gosec // dir is controlled, derived from WriteFileAtomic's path argument + f, err := os.OpenFile(dir, os.O_RDONLY, os.ModePerm) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + return f.Sync() +} diff --git a/pkg/provider/atomic_test.go b/pkg/provider/atomic_test.go index 87618c726..85505a5b6 100644 --- a/pkg/provider/atomic_test.go +++ b/pkg/provider/atomic_test.go @@ -66,3 +66,20 @@ func atomicReader(stop *atomic.Bool, path string, out chan<- error) { } } } + +func TestWriteFileAtomic_SucceedsAndDataIsDurable(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "durable.json") + + if err := WriteFileAtomic(path, []byte(`{"durable":true}`), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + data, err := os.ReadFile(path) //nolint:gosec // test reads a path under t.TempDir + if err != nil { + t.Fatalf("read back: %v", err) + } + if string(data) != `{"durable":true}` { + t.Fatalf("got %q, want the written content", data) + } +} diff --git a/pkg/provider/atomic_windows.go b/pkg/provider/atomic_windows.go new file mode 100644 index 000000000..cb63b17ec --- /dev/null +++ b/pkg/provider/atomic_windows.go @@ -0,0 +1,8 @@ +//go:build windows + +package provider + +// syncDir is a no-op on Windows; see atomic_posix.go for the rationale. +func syncDir(_ string) error { + return nil +} diff --git a/pkg/workspace/exec.go b/pkg/workspace/exec.go index ae8eccb2a..2c07ef6ab 100644 --- a/pkg/workspace/exec.go +++ b/pkg/workspace/exec.go @@ -31,9 +31,24 @@ const ( // applies. Long enough for typical build/test, short enough to surface hangs. const defaultExecTimeoutSeconds = 300 -// ResolveDockerCommand returns the docker binary to invoke. Precedence: -// override → provider config (agent.docker.path) → default. The override is -// honored even when workspace is nil. +type execLocker interface { + Lock(ctx context.Context) error + Unlock() +} + +const defaultExecLockTimeout = 15 * time.Second + +// acquireExecLock returns nil on success (caller must Unlock) or an error. +func acquireExecLock(ctx context.Context, client execLocker, timeout time.Duration) error { + lockCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err := client.Lock(lockCtx); err != nil { + return fmt.Errorf("workspace busy (locked by a concurrent operation): %w", err) + } + return nil +} + +// ResolveDockerCommand precedence: override -> provider config -> default. func ResolveDockerCommand( workspace *provider2.Workspace, override string, @@ -451,6 +466,7 @@ func ExecOneShot(ctx context.Context, opts ExecOneShotOptions) (*ExecOneShotResu if err != nil { return nil, err } + defer resolved.unlock() execCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -502,6 +518,7 @@ type resolvedExecTarget struct { target ContainerTarget workdir string envMap map[string]string + unlock func() } func resolveExecTarget(ctx context.Context, opts ExecOneShotOptions) (resolvedExecTarget, error) { @@ -519,9 +536,15 @@ func resolveExecTarget(ctx context.Context, opts ExecOneShotOptions) (resolvedEx return resolvedExecTarget{}, fmt.Errorf("resolve workspace: %w", err) } + if err := acquireExecLock(ctx, client, defaultExecLockTimeout); err != nil { + return resolvedExecTarget{}, err + } + unlock := client.Unlock + workspaceConfig := client.WorkspaceConfig() runtime, err := NewContainerRuntime(workspaceConfig, "") if err != nil { + unlock() return resolvedExecTarget{}, err } @@ -529,6 +552,7 @@ func resolveExecTarget(ctx context.Context, opts ExecOneShotOptions) (resolvedEx ctx, devcontainer.GetRunnerIDFromWorkspace(workspaceConfig), opts.IDLabels, ) if err != nil { + unlock() return resolvedExecTarget{}, err } @@ -546,6 +570,7 @@ func resolveExecTarget(ctx context.Context, opts ExecOneShotOptions) (resolvedEx target: target, workdir: workdir, envMap: envMap, + unlock: unlock, }, nil } diff --git a/pkg/workspace/exec_test.go b/pkg/workspace/exec_test.go index 5cdf7e63c..97a97d4f6 100644 --- a/pkg/workspace/exec_test.go +++ b/pkg/workspace/exec_test.go @@ -3,6 +3,7 @@ package workspace import ( "bytes" "context" + "fmt" "io" "testing" @@ -180,3 +181,53 @@ func TestExecOneShot_PartialOutputOnError(t *testing.T) { t.Fatalf("expected partial stdout %q to be preserved, got %q", "partial", stdout.String()) } } + +type fakeLockClient struct { + lockErr error + lockCalls int + unlockCalls int +} + +func (f *fakeLockClient) Lock(_ context.Context) error { + f.lockCalls++ + return f.lockErr +} + +func (f *fakeLockClient) Unlock() { + f.unlockCalls++ +} + +func TestExecOneShot_LockedWorkspaceReturnsBusyError(t *testing.T) { + lockClient := &fakeLockClient{lockErr: fmt.Errorf("timed out waiting to lock workspace")} + err := acquireExecLock(context.Background(), lockClient, defaultExecLockTimeout) + if err == nil { + t.Fatal("expected lock error, got nil") + } + if lockClient.lockCalls != 1 { + t.Fatalf("expected 1 lock call, got %d", lockClient.lockCalls) + } + if lockClient.unlockCalls != 0 { + t.Fatalf( + "unlock must not be called when lock itself failed, got %d calls", + lockClient.unlockCalls, + ) + } +} + +func TestExecOneShot_UnlocksAfterSuccessfulLock(t *testing.T) { + lockClient := &fakeLockClient{} + err := acquireExecLock(context.Background(), lockClient, defaultExecLockTimeout) + if err != nil { + t.Fatalf("unexpected lock error: %v", err) + } + if lockClient.lockCalls != 1 { + t.Fatalf("expected 1 lock call, got %d", lockClient.lockCalls) + } + + // acquireExecLock never calls Unlock itself; the caller does (see + // resolveExecTarget/ExecOneShot), simulated here. + lockClient.Unlock() + if lockClient.unlockCalls != 1 { + t.Fatalf("expected 1 unlock call, got %d", lockClient.unlockCalls) + } +}