From 481d1a3f427ee8ffe5a4d2a41fd30e317d09b2b5 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 13:00:36 -0700 Subject: [PATCH 01/34] feat: add the experimentation skill to the agent space in setup Setup now adds the experiment-orchestration skill to the agent space. The skill tells the agent how to move a goal through an experiment, a guarded rollout, and a report. The skill text comes from the setup guide. If the space has a skill with the same name and other instructions, setup does not change it. Pass --replace-skill to replace the instructions. The status and agent types of the skill stay the same. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/confirm_test.go | 12 + cmd/aws_devops_agent/plan_print_test.go | 11 + cmd/aws_devops_agent/select_test.go | 16 ++ cmd/aws_devops_agent/setup.go | 20 +- internal/awsdevops/awsdevops.go | 4 + internal/awsdevops/awsdevops_test.go | 69 +++++- internal/awsdevops/lookup.go | 22 ++ internal/awsdevops/setup.go | 26 +- internal/awsdevops/skill.go | 232 ++++++++++++++++++ internal/awsdevops/skill_test.go | 226 +++++++++++++++++ .../skills/experiment-orchestration/SKILL.md | 167 +++++++++++++ 11 files changed, 797 insertions(+), 8 deletions(-) create mode 100644 internal/awsdevops/skill.go create mode 100644 internal/awsdevops/skill_test.go create mode 100644 internal/awsdevops/skills/experiment-orchestration/SKILL.md diff --git a/cmd/aws_devops_agent/confirm_test.go b/cmd/aws_devops_agent/confirm_test.go index 8cd36ccf6..7bad8b3d2 100644 --- a/cmd/aws_devops_agent/confirm_test.go +++ b/cmd/aws_devops_agent/confirm_test.go @@ -43,6 +43,12 @@ func (a countingAgent) CreateAgentSpace(context.Context, *devopsagent.CreateAgen return &devopsagent.CreateAgentSpaceOutput{AgentSpace: &agenttypes.AgentSpace{AgentSpaceId: aws.String("space-new")}}, nil } +func (a countingAgent) CreateAsset(context.Context, *devopsagent.CreateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { + a.writes.add("CreateAsset") + + return &devopsagent.CreateAssetOutput{Asset: &agenttypes.Asset{AssetId: aws.String("ki-1")}}, nil +} + func (a countingAgent) DeregisterService(context.Context, *devopsagent.DeregisterServiceInput, ...func(*devopsagent.Options)) (*devopsagent.DeregisterServiceOutput, error) { a.writes.add("DeregisterService") @@ -77,6 +83,12 @@ func (a countingAgent) RegisterService(context.Context, *devopsagent.RegisterSer return &devopsagent.RegisterServiceOutput{ServiceId: aws.String("mcp-1")}, nil } +func (a countingAgent) UpdateAsset(context.Context, *devopsagent.UpdateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) { + a.writes.add("UpdateAsset") + + return &devopsagent.UpdateAssetOutput{}, nil +} + type countingIAM struct{ writes *writes } func (c countingIAM) AttachRolePolicy(context.Context, *iam.AttachRolePolicyInput, ...func(*iam.Options)) (*iam.AttachRolePolicyOutput, error) { diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index 47afff3d8..0ef842869 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -65,6 +65,17 @@ func TestSetupPlanNamesRolesAndOperatorApp(t *testing.T) { assert.Contains(t, out.String(), awsdevops.AgentSpaceRoleName) assert.Contains(t, out.String(), awsdevops.OperatorAppRoleName) assert.Contains(t, out.String(), "operator app") + assert.Contains(t, out.String(), "Skill: Create skill "+awsdevops.ExperimentSkillName) +} + +// A canceled plan does not create the skill. +func TestSetupCanceledCreatesNoSkill(t *testing.T) { + cmd, _ := setupTestCmd(t, "n\n") + clients, w := countingClients() + + require.Error(t, setupWith(cmd, clients, newSpaceOptions(), true, true)) + + assert.NotContains(t, w.calls, "CreateAsset") } func noTokenOptions() awsdevops.SetupOptions { diff --git a/cmd/aws_devops_agent/select_test.go b/cmd/aws_devops_agent/select_test.go index 54458388d..40b879468 100644 --- a/cmd/aws_devops_agent/select_test.go +++ b/cmd/aws_devops_agent/select_test.go @@ -32,6 +32,22 @@ func (s stubAgent) AssociateService(context.Context, *devopsagent.AssociateServi return nil, nil } +func (s stubAgent) CreateAsset(context.Context, *devopsagent.CreateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { + return &devopsagent.CreateAssetOutput{Asset: &agenttypes.Asset{AssetId: aws.String("ki-1")}}, nil +} + +func (s stubAgent) GetAssetFile(context.Context, *devopsagent.GetAssetFileInput, ...func(*devopsagent.Options)) (*devopsagent.GetAssetFileOutput, error) { + return nil, &agenttypes.ResourceNotFoundException{} +} + +func (s stubAgent) ListAssets(context.Context, *devopsagent.ListAssetsInput, ...func(*devopsagent.Options)) (*devopsagent.ListAssetsOutput, error) { + return &devopsagent.ListAssetsOutput{}, nil +} + +func (s stubAgent) UpdateAsset(context.Context, *devopsagent.UpdateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) { + return &devopsagent.UpdateAssetOutput{}, nil +} + func (s stubAgent) CreateAgentSpace(context.Context, *devopsagent.CreateAgentSpaceInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAgentSpaceOutput, error) { return nil, nil } diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 2dc40a2a2..c3187b5e1 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -27,6 +27,7 @@ const ( replaceMCPTokenFlag = "replace-mcp-token" mcpServiceIDFlag = "mcp-service-id" mcpEndpointFlag = "mcp-endpoint" + replaceSkillFlag = "replace-skill" yesFlag = "yes" ) @@ -34,12 +35,13 @@ func NewSetupCmd(analyticsTrackerFn analytics.TrackerFn) *cobra.Command { cmd := &cobra.Command{ Use: "setup", Short: "Provision the AWS DevOps Agent for LaunchDarkly", - Long: `Create the IAM roles, agent space, AWS account association, operator app and -LaunchDarkly MCP server connection the AWS DevOps Agent needs. + Long: `Create the IAM roles, agent space, AWS account association, operator app, +LaunchDarkly MCP server connection and experimentation skill the AWS DevOps +Agent needs. When the account already has agent spaces, setup asks which one to connect LaunchDarkly to. On a space that setup did not make, setup only registers the -MCP server. On a space that setup made, or with --auth-flow, setup runs the +MCP server and adds the skill. On a space that setup made, or with --auth-flow, setup runs the full setup. Pass --agent-space-id to pick a space without the list, or --new-agent-space to provision another space. A new space needs a name: pass --agent-space-name, or type one when setup asks. The run ends with the @@ -53,7 +55,12 @@ session token from 'ldcli login' stops working once it expires. If the agent space already has an MCP server, setup reuses it and does not change its token. If it has none, setup asks for a service token in a terminal and checks it before it changes AWS. Without a terminal, or with --yes, it stops and asks -for --access-token.`, +for --access-token. + +The experimentation skill tells the agent how to run an experiment from a +goal to a report. If the space has a skill with the same name and other +instructions, setup does not change it. To replace the instructions, pass +--replace-skill.`, Args: cobra.NoArgs, PreRun: trackRun(analyticsTrackerFn), RunE: runSetup, @@ -73,6 +80,7 @@ for --access-token.`, cmd.Flags().Bool(replaceMCPTokenFlag, false, "Give the agent space a new LaunchDarkly MCP server with the token passed with --access-token. Other agent spaces keep their server") cmd.Flags().String(mcpServiceIDFlag, "", "Registered MCP server to connect, instead of the server of the agent space") cmd.Flags().String(mcpEndpointFlag, "", "MCP endpoint to register. Setup reuses only a server at this endpoint. Defaults to "+awsdevops.MCPServerEndpoint) + cmd.Flags().Bool(replaceSkillFlag, false, "Replace the instructions of the "+awsdevops.ExperimentSkillName+" skill if they differ from the ldcli version") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -495,7 +503,8 @@ func confirmSetup(cmd *cobra.Command, plan awsdevops.SetupPlan, ask bool) error _, _ = fmt.Fprintf(out, " Agent space: %s\n", text.AgentSpace) _, _ = fmt.Fprintf(out, " Association: %s\n", text.Association) _, _ = fmt.Fprintf(out, " Operator app: %s\n", text.OperatorApp) - _, _ = fmt.Fprintf(out, " MCP server: %s\n\n", text.MCPServer) + _, _ = fmt.Fprintf(out, " MCP server: %s\n", text.MCPServer) + _, _ = fmt.Fprintf(out, " Skill: %s\n\n", text.Skill) if !ask { return nil } @@ -681,6 +690,7 @@ func setupOptions(cmd *cobra.Command) (awsdevops.SetupOptions, error) { ReplaceMCPToken: mustBool(cmd, replaceMCPTokenFlag), MCPServiceID: mustString(cmd, mcpServiceIDFlag), MCPEndpoint: mustString(cmd, mcpEndpointFlag), + ReplaceSkill: mustBool(cmd, replaceSkillFlag), } idcSet := opts.IdcInstanceARN != "" diff --git a/internal/awsdevops/awsdevops.go b/internal/awsdevops/awsdevops.go index 72155518a..2afca795d 100644 --- a/internal/awsdevops/awsdevops.go +++ b/internal/awsdevops/awsdevops.go @@ -51,16 +51,20 @@ var ( type AgentAPI interface { AssociateService(context.Context, *devopsagent.AssociateServiceInput, ...func(*devopsagent.Options)) (*devopsagent.AssociateServiceOutput, error) CreateAgentSpace(context.Context, *devopsagent.CreateAgentSpaceInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAgentSpaceOutput, error) + CreateAsset(context.Context, *devopsagent.CreateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) DeregisterService(context.Context, *devopsagent.DeregisterServiceInput, ...func(*devopsagent.Options)) (*devopsagent.DeregisterServiceOutput, error) DisassociateService(context.Context, *devopsagent.DisassociateServiceInput, ...func(*devopsagent.Options)) (*devopsagent.DisassociateServiceOutput, error) EnableOperatorApp(context.Context, *devopsagent.EnableOperatorAppInput, ...func(*devopsagent.Options)) (*devopsagent.EnableOperatorAppOutput, error) GetOperatorApp(context.Context, *devopsagent.GetOperatorAppInput, ...func(*devopsagent.Options)) (*devopsagent.GetOperatorAppOutput, error) ListTagsForResource(context.Context, *devopsagent.ListTagsForResourceInput, ...func(*devopsagent.Options)) (*devopsagent.ListTagsForResourceOutput, error) GetAgentSpace(context.Context, *devopsagent.GetAgentSpaceInput, ...func(*devopsagent.Options)) (*devopsagent.GetAgentSpaceOutput, error) + GetAssetFile(context.Context, *devopsagent.GetAssetFileInput, ...func(*devopsagent.Options)) (*devopsagent.GetAssetFileOutput, error) ListAgentSpaces(context.Context, *devopsagent.ListAgentSpacesInput, ...func(*devopsagent.Options)) (*devopsagent.ListAgentSpacesOutput, error) + ListAssets(context.Context, *devopsagent.ListAssetsInput, ...func(*devopsagent.Options)) (*devopsagent.ListAssetsOutput, error) ListAssociations(context.Context, *devopsagent.ListAssociationsInput, ...func(*devopsagent.Options)) (*devopsagent.ListAssociationsOutput, error) ListServices(context.Context, *devopsagent.ListServicesInput, ...func(*devopsagent.Options)) (*devopsagent.ListServicesOutput, error) RegisterService(context.Context, *devopsagent.RegisterServiceInput, ...func(*devopsagent.Options)) (*devopsagent.RegisterServiceOutput, error) + UpdateAsset(context.Context, *devopsagent.UpdateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) } // IAMAPI is the subset of the IAM API this package uses. diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index 6a0de2f5a..a14d15966 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -14,6 +14,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/devopsagent" + "github.com/aws/aws-sdk-go-v2/service/devopsagent/document" agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" "github.com/aws/aws-sdk-go-v2/service/iam" iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types" @@ -64,6 +65,65 @@ type fakeAgent struct { // paginate makes every list API return one item per page, so callers are // exercised against the NextToken AWS sends for a long list. paginate bool + // assets holds the skill assets of every space. + assets []agenttypes.Asset + // skillFiles holds the SKILL.md text by asset ID. + skillFiles map[string]string + listAssetsErr error + createAssetErr error + createAssets []*devopsagent.CreateAssetInput + updateAssets []*devopsagent.UpdateAssetInput +} + +// skillAsset is a skill asset named name. +func skillAsset(id, name string) agenttypes.Asset { + return agenttypes.Asset{ + AssetId: aws.String(id), + AssetType: aws.String("skill"), + Metadata: document.NewLazyDocument(map[string]any{"name": name, "agent_types": []string{"GENERIC"}}), + } +} + +func (f *fakeAgent) CreateAsset(_ context.Context, in *devopsagent.CreateAssetInput, _ ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { + f.calls = append(f.calls, "CreateAsset") + f.createAssets = append(f.createAssets, in) + if f.createAssetErr != nil { + return nil, f.createAssetErr + } + asset := agenttypes.Asset{AssetId: aws.String("ki-created"), AssetType: in.AssetType, Metadata: in.Metadata} + f.assets = append(f.assets, asset) + + return &devopsagent.CreateAssetOutput{Asset: &asset}, nil +} + +func (f *fakeAgent) GetAssetFile(_ context.Context, in *devopsagent.GetAssetFileInput, _ ...func(*devopsagent.Options)) (*devopsagent.GetAssetFileOutput, error) { + f.calls = append(f.calls, "GetAssetFile") + text, ok := f.skillFiles[aws.ToString(in.AssetId)] + if !ok || aws.ToString(in.Path) != "SKILL.md" { + return nil, &agenttypes.ResourceNotFoundException{} + } + + return &devopsagent.GetAssetFileOutput{File: &agenttypes.AssetFile{ + Path: in.Path, + Content: &agenttypes.AssetFileBodyMemberText{Value: text}, + }}, nil +} + +func (f *fakeAgent) ListAssets(_ context.Context, in *devopsagent.ListAssetsInput, _ ...func(*devopsagent.Options)) (*devopsagent.ListAssetsOutput, error) { + f.calls = append(f.calls, "ListAssets") + if f.listAssetsErr != nil { + return nil, f.listAssetsErr + } + items, next := page(f.assets, in.NextToken, f.paginate) + + return &devopsagent.ListAssetsOutput{Items: items, NextToken: next}, nil +} + +func (f *fakeAgent) UpdateAsset(_ context.Context, in *devopsagent.UpdateAssetInput, _ ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) { + f.calls = append(f.calls, "UpdateAsset") + f.updateAssets = append(f.updateAssets, in) + + return &devopsagent.UpdateAssetOutput{Asset: &agenttypes.Asset{AssetId: in.AssetId}}, nil } func page[T any](items []T, token *string, paginate bool) ([]T, *string) { @@ -365,6 +425,7 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { assert.Equal(t, "mcp-1", result.MCPServiceID) assert.Equal(t, "assoc-mcp-1", result.MCPAssociationID) assert.Equal(t, "https://space-1.aidevops.global.app.aws", result.OperatorAppURL) + assert.Equal(t, "ki-created", result.SkillAssetID) assert.Equal(t, map[string]string{awsdevops.ManagedTagKey: awsdevops.ManagedTagValue}, agent.createdTags) assert.Equal(t, []string{awsdevops.AgentSpaceRoleName, awsdevops.OperatorAppRoleName}, iamClient.createdRoles) assert.Equal(t, awsdevops.AgentSpacePolicyARN, iamClient.attachedPolicy[awsdevops.AgentSpaceRoleName]) @@ -373,8 +434,12 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { assert.Equal( t, // The plan does not look for the server of a new space. The check - // before the MCP step looks, because the space exists by then. - []string{"CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "RegisterService", "AssociateService"}, + // before the MCP step looks, because the space exists by then. The + // skill goes before the MCP step. + []string{ + "CreateAgentSpace", "AssociateService", "EnableOperatorApp", + "ListAssets", "CreateAsset", "ListServices", "RegisterService", "AssociateService", + }, agent.calls, ) } diff --git a/internal/awsdevops/lookup.go b/internal/awsdevops/lookup.go index 444d41e46..2e2c86a47 100644 --- a/internal/awsdevops/lookup.go +++ b/internal/awsdevops/lookup.go @@ -340,3 +340,25 @@ func serviceName(service agenttypes.RegisteredService) string { return aws.ToString(service.Name) } + +func listAssets(ctx context.Context, clients Clients, agentSpaceID, assetType string) ([]agenttypes.Asset, error) { + var ( + assets []agenttypes.Asset + nextToken *string + ) + for { + page, err := clients.Agent.ListAssets(ctx, &devopsagent.ListAssetsInput{ + AgentSpaceId: aws.String(agentSpaceID), + AssetType: aws.String(assetType), + NextToken: nextToken, + }) + if err != nil { + return nil, fmt.Errorf("unable to list the %s assets of agent space %s: %w", assetType, agentSpaceID, err) + } + assets = append(assets, page.Items...) + if page.NextToken == nil { + return assets, nil + } + nextToken = page.NextToken + } +} diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index eb68019cd..0d433bb71 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -115,6 +115,10 @@ type SetupOptions struct { // registration, so a self-hosted one is never confused with ours. MCPEndpoint string + // ReplaceSkill replaces the instructions of a skill with the name of the + // experimentation skill when they differ. + ReplaceSkill bool + // Logf, when set, reports progress as each step completes. Logf func(format string, args ...any) } @@ -132,6 +136,7 @@ type SetupResult struct { OperatorAppEnabled bool `json:"operatorAppEnabled"` MCPServiceID string `json:"mcpServiceId,omitempty"` MCPAssociationID string `json:"mcpAssociationId,omitempty"` + SkillAssetID string `json:"skillAssetId,omitempty"` } // ErrPlanChanged means AWS changed after the plan. @@ -396,6 +401,7 @@ type SetupPlan struct { // AuthFlow is the sign-in that a full setup sets. AuthFlow string MCP MCPStep + Skill SkillStep } // PlanText is the plan as lines for a person to read. @@ -405,11 +411,12 @@ type PlanText struct { Roles []string OperatorApp string MCPServer string + Skill string } // Text describes each step of the plan. func (p SetupPlan) Text() PlanText { - text := PlanText{AgentSpace: p.Space.describe(), MCPServer: p.MCP.describe()} + text := PlanText{AgentSpace: p.Space.describe(), MCPServer: p.MCP.describe(), Skill: p.Skill.describe()} if p.Space.Action == SpaceConnect { text.Association = "No change" text.Roles = []string{"No change"} @@ -492,6 +499,11 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio return plan, err } + plan.Skill, err = planSkill(ctx, clients, opts, plan.Space) + if err != nil { + return plan, err + } + if !provision { // Read the app, so the run does not print a URL that does not work. plan.App, err = appState(ctx, clients, plan.Space.ID) @@ -884,6 +896,11 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := checkSentToken(ctx, opts, plan.MCP); err != nil { return result, err } + if plan.Space.Action != SpaceCreate { + if err := checkSkill(ctx, clients, plan.Space.ID, plan.Skill); err != nil { + return result, err + } + } spaceRole, err := checkRole(ctx, clients.IAM, plan, AgentSpaceRoleName) if err != nil { return result, err @@ -969,6 +986,10 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup result.OperatorAppURL = OperatorAppURL(result.AgentSpaceID) } + if err := installSkill(ctx, clients, plan.Skill, &result, logf); err != nil { + return result, err + } + // The role and association steps can take minutes. A server that appeared // or went in that time makes AWS reject the step with an error that does // not say that the plan changed, so check the server again. A new space @@ -1054,6 +1075,9 @@ func ConnectPlan(ctx context.Context, clients Clients, opts SetupOptions, plan S if err := checkSentToken(ctx, opts, plan.MCP); err != nil { return result, err } + if err := installSkill(ctx, clients, plan.Skill, &result, logf); err != nil { + return result, err + } if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go new file mode 100644 index 000000000..6ed760744 --- /dev/null +++ b/internal/awsdevops/skill.go @@ -0,0 +1,232 @@ +package awsdevops + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/devopsagent" + "github.com/aws/aws-sdk-go-v2/service/devopsagent/document" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" +) + +// ExperimentSkill is the SKILL.md of the experimentation skill. Its front +// matter gives the name and the description that setup sends as metadata. +// +//go:embed skills/experiment-orchestration/SKILL.md +var ExperimentSkill string + +var ( + ExperimentSkillName = frontMatterField(ExperimentSkill, "name") + experimentSkillDescription = frontMatterField(ExperimentSkill, "description") +) + +const ( + skillAssetType = "skill" + skillFilePath = "SKILL.md" +) + +// frontMatterField reads one key from the front matter of a SKILL.md. +func frontMatterField(text, key string) string { + lines := strings.Split(text, "\n") + if len(lines) == 0 || strings.TrimSpace(lines[0]) != "---" { + return "" + } + for _, line := range lines[1:] { + if strings.TrimSpace(line) == "---" { + return "" + } + if value, ok := strings.CutPrefix(line, key+":"); ok { + return strings.Trim(strings.TrimSpace(value), `"`) + } + } + + return "" +} + +// SkillAction is the planned skill action. +type SkillAction int + +const ( + SkillCreate SkillAction = iota + SkillKeep + // SkillLeave means that a skill with the name has other instructions. Skills + // have no tags, so setup cannot tell an edit by the customer from an older + // ldcli version, and it does not replace them without --replace-skill. + SkillLeave + SkillReplace +) + +// SkillStep is the planned experimentation skill step. +type SkillStep struct { + Action SkillAction + AssetID string +} + +func (s SkillStep) describe() string { + label := fmt.Sprintf("%s (%s)", ExperimentSkillName, s.AssetID) + switch s.Action { + case SkillKeep: + return "Keep skill " + label + case SkillLeave: + return fmt.Sprintf("Leave skill %s unchanged because its instructions differ. "+ + "Pass --replace-skill to replace them", label) + case SkillReplace: + return "Replace the instructions of skill " + label + default: + return "Create skill " + ExperimentSkillName + } +} + +func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep) (SkillStep, error) { + if space.Action == SpaceCreate { + return SkillStep{Action: SkillCreate}, nil + } + id, err := findSkill(ctx, clients, space.ID, ExperimentSkillName) + if err != nil || id == "" { + return SkillStep{Action: SkillCreate}, err + } + same, err := skillMatches(ctx, clients, space.ID, id) + switch { + case err != nil: + return SkillStep{}, err + case same: + return SkillStep{Action: SkillKeep, AssetID: id}, nil + case opts.ReplaceSkill: + return SkillStep{Action: SkillReplace, AssetID: id}, nil + default: + return SkillStep{Action: SkillLeave, AssetID: id}, nil + } +} + +// findSkill returns the asset ID of the skill with this name in the space, or +// "" if there is none. AWS keeps skill names unique in a space. +func findSkill(ctx context.Context, clients Clients, spaceID, name string) (string, error) { + assets, err := listAssets(ctx, clients, spaceID, skillAssetType) + if err != nil { + return "", err + } + for _, asset := range assets { + if assetName(asset) == name { + return aws.ToString(asset.AssetId), nil + } + } + + return "", nil +} + +// assetName reads the name from the metadata of an asset. It decodes the JSON +// of the document, because UnmarshalSmithyDocument fails on a document from +// NewLazyDocument. +func assetName(asset agenttypes.Asset) string { + if asset.Metadata == nil { + return "" + } + encoded, err := asset.Metadata.MarshalSmithyDocument() + if err != nil { + return "" + } + var metadata struct { + Name string `json:"name"` + } + if err := json.Unmarshal(encoded, &metadata); err != nil { + return "" + } + + return metadata.Name +} + +// skillMatches reports whether the SKILL.md of the skill is ours. +func skillMatches(ctx context.Context, clients Clients, spaceID, assetID string) (bool, error) { + file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ + AgentSpaceId: aws.String(spaceID), + AssetId: aws.String(assetID), + Path: aws.String(skillFilePath), + }) + var missing *agenttypes.ResourceNotFoundException + switch { + case errors.As(err, &missing): + return false, nil + case err != nil: + return false, fmt.Errorf("unable to read skill %s (%s): %w", ExperimentSkillName, assetID, err) + } + text, ok := file.File.Content.(*agenttypes.AssetFileBodyMemberText) + + return ok && text.Value == ExperimentSkill, nil +} + +// checkSkill compares the space with the skill step. +func checkSkill(ctx context.Context, clients Clients, spaceID string, step SkillStep) error { + id, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) + switch { + case err != nil: + return err + case step.Action == SkillCreate && id != "": + return planChanged("skill %s appeared after the plan", ExperimentSkillName) + case step.Action != SkillCreate && id != step.AssetID: + return planChanged("skill %s disappeared after the plan", ExperimentSkillName) + } + + return nil +} + +func installSkill( + ctx context.Context, + clients Clients, + step SkillStep, + result *SetupResult, + logf func(string, ...any), +) error { + // The other steps can take minutes, so check the skill again. + if err := checkSkill(ctx, clients, result.AgentSpaceID, step); err != nil { + return err + } + content := &agenttypes.AssetContentMemberFile{Value: agenttypes.AssetFileContent{ + Path: aws.String(skillFilePath), + Body: &agenttypes.AssetFileBodyMemberText{Value: ExperimentSkill}, + }} + switch step.Action { + case SkillKeep: + result.SkillAssetID = step.AssetID + logf("Kept skill %s (%s)", ExperimentSkillName, step.AssetID) + case SkillLeave: + result.SkillAssetID = step.AssetID + logf("Skill %s (%s) has other instructions, so ldcli did not change it", ExperimentSkillName, step.AssetID) + logf("Pass --replace-skill to replace them") + case SkillReplace: + // AWS updates only the metadata keys in the request, so the status + // and the agent types that the customer set stay. + if _, err := clients.Agent.UpdateAsset(ctx, &devopsagent.UpdateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetId: aws.String(step.AssetID), + Content: content, + Metadata: document.NewLazyDocument(map[string]any{"description": experimentSkillDescription}), + }); err != nil { + return fmt.Errorf("unable to replace the instructions of skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) + } + result.SkillAssetID = step.AssetID + logf("Replaced the instructions of skill %s (%s)", ExperimentSkillName, step.AssetID) + default: + created, err := clients.Agent.CreateAsset(ctx, &devopsagent.CreateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetType: aws.String(skillAssetType), + Content: content, + Metadata: document.NewLazyDocument(map[string]any{ + "name": ExperimentSkillName, + "description": experimentSkillDescription, + "agent_types": []string{"GENERIC"}, + }), + }) + if err != nil { + return fmt.Errorf("unable to create skill %s: %w", ExperimentSkillName, err) + } + result.SkillAssetID = aws.ToString(created.Asset.AssetId) + logf("Created skill %s (%s)", ExperimentSkillName, result.SkillAssetID) + } + + return nil +} diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go new file mode 100644 index 000000000..b024b080e --- /dev/null +++ b/internal/awsdevops/skill_test.go @@ -0,0 +1,226 @@ +package awsdevops_test + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/devopsagent/document" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) + +func reuseOptions() awsdevops.SetupOptions { + return awsdevops.SetupOptions{AgentSpaceID: "space-2", AuthFlow: "iam", LDAccessToken: "api-token"} +} + +func metadataOf(t *testing.T, doc document.Interface) map[string]any { + t.Helper() + encoded, err := doc.MarshalSmithyDocument() + require.NoError(t, err) + var metadata map[string]any + require.NoError(t, json.Unmarshal(encoded, &metadata)) + + return metadata +} + +func TestExperimentSkillFrontMatter(t *testing.T) { + assert.Equal(t, "experiment-orchestration", awsdevops.ExperimentSkillName) + assert.Contains(t, awsdevops.ExperimentSkill, "# Automated Experimentation") +} + +func TestSetupCreatesSkillInNewSpace(t *testing.T) { + agent := &fakeAgent{} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), awsdevops.SetupOptions{ + AgentSpaceName: "launchdarkly", + AuthFlow: "iam", + LDAccessToken: "api-token", + }) + require.NoError(t, err) + + require.Len(t, agent.createAssets, 1) + in := agent.createAssets[0] + assert.Equal(t, "space-1", aws.ToString(in.AgentSpaceId)) + assert.Equal(t, "skill", aws.ToString(in.AssetType)) + metadata := metadataOf(t, in.Metadata) + assert.Equal(t, awsdevops.ExperimentSkillName, metadata["name"]) + assert.Contains(t, metadata["description"], "Orchestrates automated experimentation lifecycle") + assert.Equal(t, []any{"GENERIC"}, metadata["agent_types"]) + assert.NotContains(t, metadata, "status") + file, ok := in.Content.(*agenttypes.AssetContentMemberFile) + require.True(t, ok) + assert.Equal(t, "SKILL.md", aws.ToString(file.Value.Path)) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentSkill}, file.Value.Body) + assert.Equal(t, "ki-created", result.SkillAssetID) +} + +// AWS adds sample skills to a space. Only the name of our skill counts. +func TestPlanSetupFindsSkillOnLaterPage(t *testing.T) { + agent := &fakeAgent{ + paginate: true, + assets: []agenttypes.Asset{skillAsset("ki-sample", "sample-skip-scheduled-maintenance"), skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + } + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillStep{Action: awsdevops.SkillKeep, AssetID: "ki-ours"}, plan.Skill) + assert.Equal(t, "Keep skill experiment-orchestration (ki-ours)", plan.Text().Skill) +} + +func TestPlanSetupCreatesMissingSkill(t *testing.T) { + agent := &fakeAgent{assets: []agenttypes.Asset{skillAsset("ki-sample", "sample-skip-scheduled-maintenance")}} + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillCreate, plan.Skill.Action) + assert.Equal(t, "Create skill experiment-orchestration", plan.Text().Skill) +} + +func TestSetupKeepsSameSkill(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Empty(t, agent.createAssets) + assert.Empty(t, agent.updateAssets) + assert.Equal(t, "ki-ours", result.SkillAssetID) +} + +func TestSetupLeavesChangedSkill(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillLeave, plan.Skill.Action) + assert.Contains(t, plan.Text().Skill, "--replace-skill") + + _, err = awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Empty(t, agent.createAssets) + assert.Empty(t, agent.updateAssets) +} + +func TestSetupReplacesChangedSkillWithFlag(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": "# Older instructions"}, + } + opts := reuseOptions() + opts.ReplaceSkill = true + + _, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) + require.NoError(t, err) + + assert.Empty(t, agent.createAssets) + require.Len(t, agent.updateAssets, 1) + in := agent.updateAssets[0] + assert.Equal(t, "ki-ours", aws.ToString(in.AssetId)) + // Only the description, so the status and agent types that the + // customer set stay. + assert.Equal(t, []string{"description"}, keysOf(metadataOf(t, in.Metadata))) + file, ok := in.Content.(*agenttypes.AssetContentMemberFile) + require.True(t, ok) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentSkill}, file.Value.Body) +} + +// The flag does nothing to a skill that is already the same. +func TestPlanSetupKeepsSameSkillWithFlag(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + } + opts := reuseOptions() + opts.ReplaceSkill = true + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), opts, true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillKeep, plan.Skill.Action) +} + +func TestConnectCreatesSkill(t *testing.T) { + agent := &fakeAgent{} + + result, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + require.Len(t, agent.createAssets, 1) + assert.Equal(t, "space-2", aws.ToString(agent.createAssets[0].AgentSpaceId)) + assert.Equal(t, "ki-created", result.SkillAssetID) +} + +func TestRunPlanStopsWhenSkillAppears(t *testing.T) { + agent := &fakeAgent{} + clients := newTestClients(agent, newFakeIAM()) + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + agent.assets = []agenttypes.Asset{skillAsset("ki-other", "experiment-orchestration")} + + _, err = awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + + require.ErrorIs(t, err, awsdevops.ErrPlanChanged) + assert.Empty(t, agent.createAssets) + assert.NotContains(t, agent.calls, "EnableOperatorApp") +} + +func TestRunPlanStopsWhenSkillIsGone(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": "# Older instructions"}, + } + clients := newTestClients(agent, newFakeIAM()) + opts := reuseOptions() + opts.ReplaceSkill = true + plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) + require.NoError(t, err) + agent.assets = nil + + _, err = awsdevops.RunPlan(context.Background(), clients, opts, plan) + + require.ErrorIs(t, err, awsdevops.ErrPlanChanged) + assert.Empty(t, agent.updateAssets) +} + +func TestPlanSetupFailsWhenAssetsCannotBeListed(t *testing.T) { + agent := &fakeAgent{listAssetsErr: errors.New("AccessDeniedException: no aidevops:ListAssets")} + + _, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + + require.ErrorContains(t, err, "aidevops:ListAssets") + assert.ErrorContains(t, err, "space-2") +} + +func TestSetupReportsSkillCreateFailure(t *testing.T) { + agent := &fakeAgent{createAssetErr: errors.New("ServiceQuotaExceededException: 200 skills")} + + _, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + + require.ErrorContains(t, err, "unable to create skill experiment-orchestration") + assert.NotContains(t, agent.calls, "RegisterService") +} + +func keysOf(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + + return keys +} diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md new file mode 100644 index 000000000..ba54057c4 --- /dev/null +++ b/internal/awsdevops/skills/experiment-orchestration/SKILL.md @@ -0,0 +1,167 @@ +--- +name: "experiment-orchestration" +description: "Orchestrates automated experimentation lifecycle using LaunchDarkly Guarded Rollouts, an AI coding agent for implementation, and GitHub Actions for deployment." +--- + +# Automated Experimentation + +Use this skill when you have a goal you want to move through experimentation (e.g., "increase checkout conversion by 15%", "decrease page load time by 20%"). + +**Core principle: experiment first, then guarded rollout.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before ramping it up through a guarded rollout. Never start a guarded rollout blind — it exists only to scale a change the experiment has already shown to work. + +**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Only stop and ask the user something if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result). + +**The final report (Step 8) is mandatory, not optional.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. + +## Step 1: Goal Clarification + +Before doing anything, get answers to: + +1. **What metric measures success?** *(Required)* e.g. conversion rate, page load time, bounce rate. Reserve your error-rate metric as a safety guardrail — never use it as the primary success metric. +2. **What's the target improvement?** *(Required)* e.g. 15% increase, 200ms decrease. +3. **What's the current baseline?** *(Optional — infer from production metrics if not given)* +4. **What parts of the app are in scope?** *(Optional — infer from the codebase if not given)* +5. **Any constraints?** *(Optional)* e.g. no changes to the payment flow. + +Questions 1–2 are required before proceeding; infer 3–5 where possible and confirm your assumptions with the user before implementing. + +## Step 2: Hypothesis Generation + +Explore the target repository/codebase to find a plausible change: + +1. Search and read the relevant code paths. +2. Think through what UI/UX or logic change could plausibly move the chosen metric. +3. Check whether this hypothesis (or something close to it) has already been tried and failed — look at flag history or archived flags with similar naming. Avoid repeating a known failure. +4. Present the hypothesis to the user before proceeding, along with your reasoning and any inferred assumptions from Step 1. + +**Before finalizing a flag key, check for collisions:** look up any candidate flag key first. +- Already fully shipped (100% one variation, no split) → already decided, pick a different hypothesis. +- Actively running an experiment → mid-flight, don't compete with it, pick a different hypothesis. +- Doesn't exist → safe to create. + +## Step 3: Implementation + +1. Create a boolean feature flag, OFF by default in all environments. Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. +2. Hand off implementation to your coding agent/tool of choice, with clear instructions to gate the change behind the exact flag key from step 1. +3. This step is asynchronous — check status periodically rather than looping tightly on it. +4. Once implementation completes, move to Step 4 with the resulting branch/PR. If it fails, report the error and stop. + +## Step 4: Release Readiness + +Run your standard release/risk review on the PR before merging. + +- If it passes: merge the PR. +- If it fails: feed the review's specific feedback back into implementation and retry. Cap retries at a small fixed number (e.g. 3 attempts total) — if it still hasn't passed, stop and report the last failure to the user rather than retrying indefinitely. + +*(If your environment genuinely has no review capability available — e.g., a fully unattended automation context — you can skip straight to merge, but treat that as a deliberate, narrow exception you call out explicitly, not a default. Skipping review removes your only gate against shipping broken code.)* + +## Step 5: Deployment + +Deployment typically won't fire automatically on merge if your workflow is manually-triggered (`workflow_dispatch`-only) — you'll need to trigger it explicitly. + +1. Trigger the deploy workflow on the merge target branch. Treat "already an in-progress deployment for this ref" as expected de-duplication, not an error — don't re-trigger. +2. Poll for status, but let your polling tool's own internal long-poll do the waiting rather than looping tightly yourself. +3. Watch for a "stale" status specifically: if a deployment reports "running" for far longer than normal, cross-check the actual CI run history by commit SHA/timing before assuming it's still in progress — a background poll process may have died without updating the record. +4. **Trigger a deployment at most once per attempt.** If you're unsure whether a previous trigger succeeded, check status first — never re-trigger just because you're unsure. +5. On timeout: stop, check the CI run directly, report the situation, ask how to proceed. +6. On explicit failure: stop and report — do not proceed to the experiment. +7. On success: proceed immediately to Step 6. + +## Step 6: Experiment Phase (fixed 10%) + +Prove the change on a small, fixed slice of traffic. Do **not** start a guarded rollout here — that's Step 7, and only after this proves out. + +1. Turn the flag ON. +2. Configure a fixed 50/50 split across 10% of traffic (a flat allocation, not a staged ramp) on your chosen randomization unit (typically "user"). The remaining 90% of traffic is excluded from the experiment entirely. +3. Create an experiment with: + - Exactly one primary metric: the success metric from Step 1. + - Guardrail metric(s): always include your error-rate metric; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. + - Treatments: control (off) at 50%, treatment (on) at 50%, allocated to 10% of total traffic. +4. Start the experiment/data collection. +5. Move to Step 7 to monitor toward a decision. + +## Step 7: Monitoring & Outcome + +Check status periodically — don't tight-loop. In an interactive session, check once and report progress, then pick back up later. In an unattended/scheduled context, check once per invocation and persist your progress somewhere durable between runs. + +**Phase 1 — Prove the experiment at 10% (gate before any rollout):** + +Watch for statistical significance on the primary metric: + +- **Significant + positive lift** → experiment proven. Stop the experiment iteration and move to Phase 2. +- **Significant + negative lift** → declare a loser, archive the flag, skip Phase 2, go straight to the Step 8 report. +- **No significance after a reasonable ceiling (e.g. 30 minutes)** → report "inconclusive, need more traffic" and stop; don't proceed to Phase 2. + +Never declare a winner off a single data point or before your stats engine confirms significance. + +**Phase 2 — Guarded rollout ramp (only after Phase 1 proves the change):** + +Start a guarded rollout with: +- The winning ("on") variation as the test, the original as control. +- Same randomization unit as the experiment. +- **Exactly 3 monitored stages, capped well below 100%** — e.g. 20% → 30% → 40%, ~60 minutes monitoring each. Don't add a stage at or above 100%; Guarded-rollout implementations reject stages above 50% audience allocation, and the rollout auto-promotes to 100% itself once the final monitored stage completes cleanly — no explicit 100% stage needed. +- The same primary + guardrail metrics as the experiment, each configured to notify and auto-rollback on regression. + +Track stage progression. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, clean up the flag (deprecate/archive it), and go to the Step 8 report. + +Once the final stage completes cleanly and auto-promotes to 100%, declare a winner and go to the Step 8 report. + +**Retrying after a rollback:** a rollback isn't always caused by your monitored metrics genuinely regressing — it can also be triggered by an unrelated application error surfacing mid-ramp. Before blindly restarting after the user says they've fixed something: +1. Confirm the flag's current state (should be back to 100% control, nothing stuck mid-rollout). +2. Check the change history timing between "advanced to next stage" and "reverted." A rollback within seconds of advancing is inconsistent with a full metric-window regression and points to an external cause instead. +3. If the flag is cleanly reverted and the external cause is confirmed fixed, it's safe to restart the guarded rollout from scratch with the same parameters. +4. Don't silently retry without this check, and don't refuse to retry just because a prior attempt rolled back — a genuinely fixed external cause is a legitimate reason to retry. A metric-driven loser is not — don't retry that. + +**On any terminal outcome, immediately produce the Step 8 report in the same turn** — a one-line "it worked!" note is fine as a lead-in, but the structured report must follow, not wait for a follow-up request. + +## Step 8: Report + +Runs automatically the instant Step 7 reaches a terminal outcome (winner + auto-promoted to 100%; loser; inconclusive; or rollback/failure). Use this exact structure from Experiment Report to Next Steps: + +## Experiment Report: [Goal Description] + +**Date:** [YYYY-MM-DD] +**Goal:** [metric] [direction] by [target]% +**Status:** [achieved / in progress / stalled] + +### Hypothesis +[What we tried and why] + +### Implementation +- Flag: [flag_key] +- Files modified: [list] +- Branch: [branch name] + +### Release Readiness +- [reviewed, passed after N attempt(s) / skipped, per your environment's process] + +### Experiment Phase (10% fixed split) +- Status: [proven / loser / inconclusive] +- Duration: [time] +- Metric change: [before] → [after] ([+/-]%) +- Statistical significance: [value, confidence interval] + +### Guarded Rollout Phase (if reached) +- Status: [completed / rolled_back / not started] +- Duration: [time] +- Stages reached: [N of 3 monitored stages] +- If rolled back and retried: [root cause, outcome of retry] + +### Safety Metrics +- error-rate: [baseline] → [final] ([no regression / regression detected]) +- [other guardrails]: [baseline] → [final] ([status]) + +### Next Steps +[What to do next based on the outcome] + +## Safety Rules (the non-negotiables) + +- Always present the hypothesis before implementing. +- Always run a release/risk review before merging, unless your environment has a deliberate, explicitly-called-out exception. +- **Always prove a change via a fixed small-percentage experiment before starting any guarded rollout** — never ramp blind. +- Always include an error-rate (or equivalent "don't break prod") metric as a guardrail, separate from your success metric. +- Add a performance guardrail (e.g. p95 latency) for performance-focused changes. +- Every rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. +- **Cap guarded rollout stages well below 100%** (most platforms reject stages ≥50% audience allocation) and let the platform auto-promote to 100% after the final stage — don't try to add an explicit 100% stage. +- Distinguish a metric-driven rollback (don't retry) from an external-cause rollback (safe to retry once fixed) before restarting a rolled-back rollout. +- The final report is automatic and mandatory on every terminal outcome — never defer it to a follow-up ask. From 80289612f66531b6edca8003ec64825e6d58a889 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 13:18:13 -0700 Subject: [PATCH 02/34] fix: make the skill step optional and update copies from older versions Setup now does the skill step after the MCP server step. If the skill step fails, setup shows a warning and puts the error in skillError. The other steps still run. If setup cannot read the skills of the space, the plan skips the skill. Setup keeps the SHA-256 of each SKILL.md that it sent. If a space has one of these texts, setup updates it without --replace-skill. Other text is a customer edit. The plan keeps the version of the skill asset. If the version changes before the write, setup does not change the skill. The plan line and the help text say that every agent type in the space can load the skill. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/plan_print_test.go | 21 +++ cmd/aws_devops_agent/setup.go | 28 ++-- internal/awsdevops/awsdevops_test.go | 7 +- internal/awsdevops/export_test.go | 14 ++ internal/awsdevops/setup.go | 29 ++-- internal/awsdevops/skill.go | 185 ++++++++++++++++-------- internal/awsdevops/skill_test.go | 110 +++++++++++--- 7 files changed, 284 insertions(+), 110 deletions(-) diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index 0ef842869..4129d451f 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -469,3 +469,24 @@ func TestSetupSaysWhenTheSecondPlanReusesAServer(t *testing.T) { assert.Contains(t, stderrOf(cmd), "MCP server Prod LD (mcp-7) was registered after the first plan") assert.Contains(t, stderrOf(cmd), "--access-token and --replace-mcp-token") } + +type failingSkillAgent struct{ countingAgent } + +func (failingSkillAgent) CreateAsset(context.Context, *devopsagent.CreateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { + return nil, errors.New("AccessDeniedException: no aidevops:CreateAsset") +} + +// A failed skill step warns at the end, and setup still prints the app URL. +func TestSetupWarnsWhenSkillFails(t *testing.T) { + cmd, errOut := setupTestCmd(t, "") + cmd.SetIn(noReadStdin{t}) + clients, w := countingClients() + clients.Agent = failingSkillAgent{clients.Agent.(countingAgent)} + + require.NoError(t, setupWith(cmd, clients, newSpaceOptions(), true, false)) + + assert.Contains(t, w.calls, "RegisterService") + assert.Contains(t, errOut.String(), "Warning: unable to create skill "+awsdevops.ExperimentSkillName) + assert.Contains(t, stdoutOf(cmd), "The "+awsdevops.ExperimentSkillName+" skill is not set up") + assert.Contains(t, stdoutOf(cmd), "Open the DevOps Agent at:") +} diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index c3187b5e1..a3f2e7e52 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -41,11 +41,11 @@ Agent needs. When the account already has agent spaces, setup asks which one to connect LaunchDarkly to. On a space that setup did not make, setup only registers the -MCP server and adds the skill. On a space that setup made, or with --auth-flow, setup runs the -full setup. Pass --agent-space-id to pick a space without the list, or ---new-agent-space to provision another space. A new space needs a name: pass ---agent-space-name, or type one when setup asks. The run ends with the -operator app URL, where the agent is used. +MCP server and adds the skill. On a space that setup made, or with +--auth-flow, setup runs the full setup. Pass --agent-space-id to pick a space +without the list, or --new-agent-space to provision another space. A new +space needs a name: pass --agent-space-name, or type one when setup asks. The +run ends with the operator app URL, where the agent is used. Each agent space gets its own LaunchDarkly MCP server registration, so each space can use a different token. The MCP server is connected with @@ -57,10 +57,12 @@ If it has none, setup asks for a service token in a terminal and checks it before it changes AWS. Without a terminal, or with --yes, it stops and asks for --access-token. -The experimentation skill tells the agent how to run an experiment from a -goal to a report. If the space has a skill with the same name and other -instructions, setup does not change it. To replace the instructions, pass ---replace-skill.`, +The experiment-orchestration skill tells the agent how to run an experiment +from a goal to a report. Every agent type in the space can load it. If the +skill has the text of an older ldcli version, setup updates it. If someone +edited it, setup does not change it. To overwrite the edits, pass +--replace-skill. The skill is optional: if its step fails, setup warns and +does the other steps.`, Args: cobra.NoArgs, PreRun: trackRun(analyticsTrackerFn), RunE: runSetup, @@ -80,7 +82,7 @@ instructions, setup does not change it. To replace the instructions, pass cmd.Flags().Bool(replaceMCPTokenFlag, false, "Give the agent space a new LaunchDarkly MCP server with the token passed with --access-token. Other agent spaces keep their server") cmd.Flags().String(mcpServiceIDFlag, "", "Registered MCP server to connect, instead of the server of the agent space") cmd.Flags().String(mcpEndpointFlag, "", "MCP endpoint to register. Setup reuses only a server at this endpoint. Defaults to "+awsdevops.MCPServerEndpoint) - cmd.Flags().Bool(replaceSkillFlag, false, "Replace the instructions of the "+awsdevops.ExperimentSkillName+" skill if they differ from the ldcli version") + cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -181,6 +183,12 @@ func setupWith( awsdevops.AccessTokenURL(opts.LDBaseURI)) } + if result.SkillError != "" { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), + "\nThe %s skill is not set up: %s. Run setup again to add it\n", + awsdevops.ExperimentSkillName, result.SkillError) + } + // Only a connect run leaves the app off, on a space that setup did not // make. The full setup would change that space, so point to the console. if !result.OperatorAppEnabled { diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index a14d15966..04e576a6f 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -434,11 +434,10 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { assert.Equal( t, // The plan does not look for the server of a new space. The check - // before the MCP step looks, because the space exists by then. The - // skill goes before the MCP step. + // before the MCP step looks, because the space exists by then, + // then the skill last. []string{ - "CreateAgentSpace", "AssociateService", "EnableOperatorApp", - "ListAssets", "CreateAsset", "ListServices", "RegisterService", "AssociateService", + "CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "RegisterService", "AssociateService", "ListAssets", "CreateAsset", }, agent.calls, ) diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 38904942a..9ee486db2 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -21,3 +21,17 @@ func ConnectExisting(ctx context.Context, clients Clients, opts SetupOptions) (S return ConnectPlan(ctx, clients, opts, plan) } + +// ShipSkill adds text to the shipped skill versions until the test ends. +func ShipSkill(text string) func() { + saved := shippedSkillHashes + shippedSkillHashes = append(append([]string(nil), saved...), skillHash(text)) + + return func() { shippedSkillHashes = saved } +} + +// ShippedSkillHashes are the hashes of the shipped skill versions. +func ShippedSkillHashes() []string { return shippedSkillHashes } + +// SkillHash is the hash that setup compares. +func SkillHash(text string) string { return skillHash(text) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 0d433bb71..270429039 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -115,8 +115,9 @@ type SetupOptions struct { // registration, so a self-hosted one is never confused with ours. MCPEndpoint string - // ReplaceSkill replaces the instructions of a skill with the name of the - // experimentation skill when they differ. + // ReplaceSkill overwrites the SKILL.md of an existing + // experiment-orchestration skill that someone edited. Setup updates a + // SKILL.md from an older ldcli version without this option. ReplaceSkill bool // Logf, when set, reports progress as each step completes. @@ -137,6 +138,9 @@ type SetupResult struct { MCPServiceID string `json:"mcpServiceId,omitempty"` MCPAssociationID string `json:"mcpAssociationId,omitempty"` SkillAssetID string `json:"skillAssetId,omitempty"` + // SkillError is set if the skill step failed. The skill is optional, so + // the rest of the setup ran. + SkillError string `json:"skillError,omitempty"` } // ErrPlanChanged means AWS changed after the plan. @@ -499,10 +503,7 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio return plan, err } - plan.Skill, err = planSkill(ctx, clients, opts, plan.Space) - if err != nil { - return plan, err - } + plan.Skill = planSkill(ctx, clients, opts, plan.Space) if !provision { // Read the app, so the run does not print a URL that does not work. @@ -896,11 +897,6 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := checkSentToken(ctx, opts, plan.MCP); err != nil { return result, err } - if plan.Space.Action != SpaceCreate { - if err := checkSkill(ctx, clients, plan.Space.ID, plan.Skill); err != nil { - return result, err - } - } spaceRole, err := checkRole(ctx, clients.IAM, plan, AgentSpaceRoleName) if err != nil { return result, err @@ -986,10 +982,6 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup result.OperatorAppURL = OperatorAppURL(result.AgentSpaceID) } - if err := installSkill(ctx, clients, plan.Skill, &result, logf); err != nil { - return result, err - } - // The role and association steps can take minutes. A server that appeared // or went in that time makes AWS reject the step with an error that does // not say that the plan changed, so check the server again. A new space @@ -1000,6 +992,9 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } + // The skill goes last, so a failure of the optional step cannot stop the + // MCP server step. + installSkill(ctx, clients, plan.Skill, &result, logf) return result, nil } @@ -1075,12 +1070,10 @@ func ConnectPlan(ctx context.Context, clients Clients, opts SetupOptions, plan S if err := checkSentToken(ctx, opts, plan.MCP); err != nil { return result, err } - if err := installSkill(ctx, clients, plan.Skill, &result, logf); err != nil { - return result, err - } if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } + installSkill(ctx, clients, plan.Skill, &result, logf) return result, nil } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 6ed760744..785a1f754 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -2,10 +2,13 @@ package awsdevops import ( "context" + "crypto/sha256" _ "embed" + "encoding/hex" "encoding/json" "errors" "fmt" + "slices" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -25,6 +28,14 @@ var ( experimentSkillDescription = frontMatterField(ExperimentSkill, "description") ) +// shippedSkillHashes holds the SHA-256 of each SKILL.md that an ldcli release +// sent. A skill with one of these texts has no customer edits, so setup +// updates it without --replace-skill. When you change SKILL.md, add the hash +// of the new text. Never remove a hash. +var shippedSkillHashes = []string{ + "a5a063f2f3739ca73c10602e05f18d36f5d0e29c372be025e7e6c71d9846c4eb", +} + const ( skillAssetType = "skill" skillFilePath = "SKILL.md" @@ -48,23 +59,37 @@ func frontMatterField(text, key string) string { return "" } +func skillHash(text string) string { + sum := sha256.Sum256([]byte(text)) + + return hex.EncodeToString(sum[:]) +} + // SkillAction is the planned skill action. type SkillAction int const ( SkillCreate SkillAction = iota SkillKeep - // SkillLeave means that a skill with the name has other instructions. Skills - // have no tags, so setup cannot tell an edit by the customer from an older - // ldcli version, and it does not replace them without --replace-skill. + // SkillUpdate replaces a SKILL.md that an older ldcli sent. + SkillUpdate + // SkillLeave means that someone edited the SKILL.md, so setup does not + // replace it without --replace-skill. SkillLeave SkillReplace + // SkillSkip means that setup could not read the skills of the space. The + // skill is optional, so the run continues without it. + SkillSkip ) // SkillStep is the planned experimentation skill step. type SkillStep struct { Action SkillAction AssetID string + // Version is the asset version at plan time. A write stops if it changed. + Version int32 + // Reason says why the step is skipped. + Reason string } func (s SkillStep) describe() string { @@ -72,51 +97,63 @@ func (s SkillStep) describe() string { switch s.Action { case SkillKeep: return "Keep skill " + label + case SkillUpdate: + return fmt.Sprintf("Update skill %s to the instructions of this ldcli version", label) case SkillLeave: - return fmt.Sprintf("Leave skill %s unchanged because its instructions differ. "+ - "Pass --replace-skill to replace them", label) + return fmt.Sprintf("Leave skill %s unchanged because someone edited it. "+ + "Pass --replace-skill to overwrite it", label) case SkillReplace: - return "Replace the instructions of skill " + label + return fmt.Sprintf("Overwrite the edited instructions of skill %s", label) + case SkillSkip: + return fmt.Sprintf("Skip skill %s: %s", ExperimentSkillName, s.Reason) default: - return "Create skill " + ExperimentSkillName + return fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) } } -func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep) (SkillStep, error) { +func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep) SkillStep { if space.Action == SpaceCreate { - return SkillStep{Action: SkillCreate}, nil + return SkillStep{Action: SkillCreate} } - id, err := findSkill(ctx, clients, space.ID, ExperimentSkillName) - if err != nil || id == "" { - return SkillStep{Action: SkillCreate}, err + asset, err := findSkill(ctx, clients, space.ID, ExperimentSkillName) + if err != nil { + return SkillStep{Action: SkillSkip, Reason: errorCode(err)} + } + if asset == nil { + return SkillStep{Action: SkillCreate} } - same, err := skillMatches(ctx, clients, space.ID, id) + step := SkillStep{AssetID: aws.ToString(asset.AssetId), Version: aws.ToInt32(asset.Version)} + text, err := skillText(ctx, clients, space.ID, step.AssetID) switch { case err != nil: - return SkillStep{}, err - case same: - return SkillStep{Action: SkillKeep, AssetID: id}, nil + return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Reason: errorCode(err)} + case text == ExperimentSkill: + step.Action = SkillKeep + case slices.Contains(shippedSkillHashes, skillHash(text)): + step.Action = SkillUpdate case opts.ReplaceSkill: - return SkillStep{Action: SkillReplace, AssetID: id}, nil + step.Action = SkillReplace default: - return SkillStep{Action: SkillLeave, AssetID: id}, nil + step.Action = SkillLeave } + + return step } -// findSkill returns the asset ID of the skill with this name in the space, or -// "" if there is none. AWS keeps skill names unique in a space. -func findSkill(ctx context.Context, clients Clients, spaceID, name string) (string, error) { +// findSkill returns the skill with this name in the space, or nil if there is +// none. AWS keeps skill names unique in a space. +func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { assets, err := listAssets(ctx, clients, spaceID, skillAssetType) if err != nil { - return "", err + return nil, err } for _, asset := range assets { if assetName(asset) == name { - return aws.ToString(asset.AssetId), nil + return &asset, nil } } - return "", nil + return nil, nil } // assetName reads the name from the metadata of an asset. It decodes the JSON @@ -140,8 +177,8 @@ func assetName(asset agenttypes.Asset) string { return metadata.Name } -// skillMatches reports whether the SKILL.md of the skill is ours. -func skillMatches(ctx context.Context, clients Clients, spaceID, assetID string) (bool, error) { +// skillText returns the SKILL.md of the skill, or "" if it has none. +func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (string, error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), AssetId: aws.String(assetID), @@ -150,38 +187,79 @@ func skillMatches(ctx context.Context, clients Clients, spaceID, assetID string) var missing *agenttypes.ResourceNotFoundException switch { case errors.As(err, &missing): - return false, nil + return "", nil case err != nil: - return false, fmt.Errorf("unable to read skill %s (%s): %w", ExperimentSkillName, assetID, err) + return "", fmt.Errorf("unable to read skill %s (%s): %w", ExperimentSkillName, assetID, err) + case file.File == nil: + return "", nil + } + text, _ := file.File.Content.(*agenttypes.AssetFileBodyMemberText) + if text == nil { + return "", nil } - text, ok := file.File.Content.(*agenttypes.AssetFileBodyMemberText) - return ok && text.Value == ExperimentSkill, nil + return text.Value, nil } -// checkSkill compares the space with the skill step. +// checkSkill compares the space with a step that writes. A change to the +// version means that someone edited the skill after the plan. func checkSkill(ctx context.Context, clients Clients, spaceID string, step SkillStep) error { - id, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) + asset, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) switch { case err != nil: return err - case step.Action == SkillCreate && id != "": + case step.Action == SkillCreate && asset != nil: return planChanged("skill %s appeared after the plan", ExperimentSkillName) - case step.Action != SkillCreate && id != step.AssetID: + case step.Action == SkillCreate: + return nil + case asset == nil || aws.ToString(asset.AssetId) != step.AssetID: return planChanged("skill %s disappeared after the plan", ExperimentSkillName) + case aws.ToInt32(asset.Version) != step.Version: + return planChanged("skill %s changed after the plan", ExperimentSkillName) } return nil } +// installSkill does the skill step. The skill is optional, so a failure does +// not stop setup. It goes in result.SkillError, and the caller reports it. func installSkill( ctx context.Context, clients Clients, step SkillStep, result *SetupResult, logf func(string, ...any), +) { + result.SkillAssetID = step.AssetID + if err := doSkillStep(ctx, clients, step, result, logf); err != nil { + result.SkillError = err.Error() + logf("Warning: %v", err) + } +} + +func doSkillStep( + ctx context.Context, + clients Clients, + step SkillStep, + result *SetupResult, + logf func(string, ...any), ) error { - // The other steps can take minutes, so check the skill again. + switch step.Action { + case SkillKeep: + logf("Kept skill %s (%s)", ExperimentSkillName, step.AssetID) + + return nil + case SkillLeave: + logf("Skill %s (%s) has edits, so ldcli did not change it", ExperimentSkillName, step.AssetID) + logf("Pass --replace-skill to overwrite it") + + return nil + case SkillSkip: + return fmt.Errorf("skipped skill %s: %s", ExperimentSkillName, step.Reason) + } + + // The other steps can take minutes, so check the skill right before the + // write. if err := checkSkill(ctx, clients, result.AgentSpaceID, step); err != nil { return err } @@ -189,28 +267,7 @@ func installSkill( Path: aws.String(skillFilePath), Body: &agenttypes.AssetFileBodyMemberText{Value: ExperimentSkill}, }} - switch step.Action { - case SkillKeep: - result.SkillAssetID = step.AssetID - logf("Kept skill %s (%s)", ExperimentSkillName, step.AssetID) - case SkillLeave: - result.SkillAssetID = step.AssetID - logf("Skill %s (%s) has other instructions, so ldcli did not change it", ExperimentSkillName, step.AssetID) - logf("Pass --replace-skill to replace them") - case SkillReplace: - // AWS updates only the metadata keys in the request, so the status - // and the agent types that the customer set stay. - if _, err := clients.Agent.UpdateAsset(ctx, &devopsagent.UpdateAssetInput{ - AgentSpaceId: aws.String(result.AgentSpaceID), - AssetId: aws.String(step.AssetID), - Content: content, - Metadata: document.NewLazyDocument(map[string]any{"description": experimentSkillDescription}), - }); err != nil { - return fmt.Errorf("unable to replace the instructions of skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) - } - result.SkillAssetID = step.AssetID - logf("Replaced the instructions of skill %s (%s)", ExperimentSkillName, step.AssetID) - default: + if step.Action == SkillCreate { created, err := clients.Agent.CreateAsset(ctx, &devopsagent.CreateAssetInput{ AgentSpaceId: aws.String(result.AgentSpaceID), AssetType: aws.String(skillAssetType), @@ -226,7 +283,21 @@ func installSkill( } result.SkillAssetID = aws.ToString(created.Asset.AssetId) logf("Created skill %s (%s)", ExperimentSkillName, result.SkillAssetID) + + return nil + } + + // AWS updates only the metadata keys in the request, so the status and + // the agent types that the customer set stay. + if _, err := clients.Agent.UpdateAsset(ctx, &devopsagent.UpdateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetId: aws.String(step.AssetID), + Content: content, + Metadata: document.NewLazyDocument(map[string]any{"description": experimentSkillDescription}), + }); err != nil { + return fmt.Errorf("unable to update skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) } + logf("Updated skill %s (%s)", ExperimentSkillName, step.AssetID) return nil } diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index b024b080e..4fa207e28 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -34,6 +34,13 @@ func TestExperimentSkillFrontMatter(t *testing.T) { assert.Contains(t, awsdevops.ExperimentSkill, "# Automated Experimentation") } +// A change to SKILL.md must add its hash, so the next release updates the +// copies that this release sends. +func TestShippedSkillHashesHoldTheCurrentSkill(t *testing.T) { + assert.Contains(t, awsdevops.ShippedSkillHashes(), awsdevops.SkillHash(awsdevops.ExperimentSkill), + "add the hash of the new SKILL.md to shippedSkillHashes") +} + func TestSetupCreatesSkillInNewSpace(t *testing.T) { agent := &fakeAgent{} @@ -82,7 +89,7 @@ func TestPlanSetupCreatesMissingSkill(t *testing.T) { require.NoError(t, err) assert.Equal(t, awsdevops.SkillCreate, plan.Skill.Action) - assert.Equal(t, "Create skill experiment-orchestration", plan.Text().Skill) + assert.Equal(t, "Create skill experiment-orchestration for every agent type in the space", plan.Text().Skill) } func TestSetupKeepsSameSkill(t *testing.T) { @@ -97,9 +104,30 @@ func TestSetupKeepsSameSkill(t *testing.T) { assert.Empty(t, agent.createAssets) assert.Empty(t, agent.updateAssets) assert.Equal(t, "ki-ours", result.SkillAssetID) + assert.Empty(t, result.SkillError) } -func TestSetupLeavesChangedSkill(t *testing.T) { +// A copy from an older ldcli has no edits, so setup updates it without the flag. +func TestSetupUpdatesOlderShippedSkill(t *testing.T) { + t.Cleanup(awsdevops.ShipSkill("# Instructions of an older ldcli")) + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": "# Instructions of an older ldcli"}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) + assert.Contains(t, plan.Text().Skill, "Update skill experiment-orchestration (ki-ours)") + + _, err = awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + require.Len(t, agent.updateAssets, 1) + assert.Equal(t, "ki-ours", aws.ToString(agent.updateAssets[0].AssetId)) +} + +func TestSetupLeavesEditedSkill(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, @@ -111,16 +139,17 @@ func TestSetupLeavesChangedSkill(t *testing.T) { assert.Equal(t, awsdevops.SkillLeave, plan.Skill.Action) assert.Contains(t, plan.Text().Skill, "--replace-skill") - _, err = awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) assert.Empty(t, agent.createAssets) assert.Empty(t, agent.updateAssets) + assert.Empty(t, result.SkillError) } -func TestSetupReplacesChangedSkillWithFlag(t *testing.T) { +func TestSetupOverwritesEditedSkillWithFlag(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": "# Older instructions"}, + skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, } opts := reuseOptions() opts.ReplaceSkill = true @@ -166,24 +195,27 @@ func TestConnectCreatesSkill(t *testing.T) { assert.Equal(t, "ki-created", result.SkillAssetID) } -func TestRunPlanStopsWhenSkillAppears(t *testing.T) { +// The skill step goes after the MCP server, and a plan change in it only +// skips the skill. +func TestRunPlanSkipsSkillThatAppears(t *testing.T) { agent := &fakeAgent{} clients := newTestClients(agent, newFakeIAM()) plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) agent.assets = []agenttypes.Asset{skillAsset("ki-other", "experiment-orchestration")} - _, err = awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) - require.ErrorIs(t, err, awsdevops.ErrPlanChanged) + require.NoError(t, err) assert.Empty(t, agent.createAssets) - assert.NotContains(t, agent.calls, "EnableOperatorApp") + assert.Contains(t, agent.calls, "RegisterService") + assert.Contains(t, result.SkillError, "appeared after the plan") } -func TestRunPlanStopsWhenSkillIsGone(t *testing.T) { +func TestRunPlanSkipsSkillThatIsGone(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": "# Older instructions"}, + skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, } clients := newTestClients(agent, newFakeIAM()) opts := reuseOptions() @@ -192,28 +224,64 @@ func TestRunPlanStopsWhenSkillIsGone(t *testing.T) { require.NoError(t, err) agent.assets = nil - _, err = awsdevops.RunPlan(context.Background(), clients, opts, plan) + result, err := awsdevops.RunPlan(context.Background(), clients, opts, plan) - require.ErrorIs(t, err, awsdevops.ErrPlanChanged) + require.NoError(t, err) assert.Empty(t, agent.updateAssets) + assert.Contains(t, result.SkillError, "disappeared after the plan") } -func TestPlanSetupFailsWhenAssetsCannotBeListed(t *testing.T) { +// An edit after the plan changes the version. The overwrite stops, so it +// cannot remove an edit that the plan did not show. +func TestRunPlanSkipsSkillEditedAfterPlan(t *testing.T) { + asset := skillAsset("ki-ours", "experiment-orchestration") + asset.Version = aws.Int32(3) + agent := &fakeAgent{ + assets: []agenttypes.Asset{asset}, + skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, + } + clients := newTestClients(agent, newFakeIAM()) + opts := reuseOptions() + opts.ReplaceSkill = true + plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) + require.NoError(t, err) + require.Equal(t, int32(3), plan.Skill.Version) + agent.assets[0].Version = aws.Int32(4) + + result, err := awsdevops.RunPlan(context.Background(), clients, opts, plan) + + require.NoError(t, err) + assert.Empty(t, agent.updateAssets) + assert.Contains(t, result.SkillError, "changed after the plan") +} + +// Without aidevops:ListAssets, the plan skips the skill and setup does the +// other steps. +func TestSetupSkipsSkillWhenAssetsCannotBeListed(t *testing.T) { agent := &fakeAgent{listAssetsErr: errors.New("AccessDeniedException: no aidevops:ListAssets")} + clients := newTestClients(agent, newFakeIAM()) - _, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillSkip, plan.Skill.Action) + assert.Contains(t, plan.Text().Skill, "aidevops:ListAssets") - require.ErrorContains(t, err, "aidevops:ListAssets") - assert.ErrorContains(t, err, "space-2") + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Contains(t, agent.calls, "RegisterService") + assert.Contains(t, result.SkillError, "aidevops:ListAssets") } -func TestSetupReportsSkillCreateFailure(t *testing.T) { +func TestSetupWarnsOnSkillCreateFailure(t *testing.T) { agent := &fakeAgent{createAssetErr: errors.New("ServiceQuotaExceededException: 200 skills")} - _, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + result, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) - require.ErrorContains(t, err, "unable to create skill experiment-orchestration") - assert.NotContains(t, agent.calls, "RegisterService") + require.NoError(t, err) + assert.Contains(t, agent.calls, "RegisterService") + assert.Equal(t, "mcp-1", result.MCPServiceID) + assert.Contains(t, result.SkillError, "unable to create skill experiment-orchestration") + assert.Empty(t, result.SkillAssetID) } func keysOf(m map[string]any) []string { From 254e9a5e3d7c7314d9cfb810ad088edd8e935db1 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 13:29:21 -0700 Subject: [PATCH 03/34] fix: keep the AWS message in skill warnings and report the skill action The skill warning now keeps the full AWS error, so it names the denied action. If AWS denies access, the warning also names the permissions that the skill step needs. The warning shows one time, at the end of the run. An empty SKILL.md now gets an update without --replace-skill. A skill with no SKILL.md gets its own plan line. A SKILL.md that AWS sends as bytes is compared as text. The setup result now has skillAction, so a JSON run shows if setup kept, updated, or left the skill. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/plan_print_test.go | 8 +- cmd/aws_devops_agent/setup.go | 4 +- internal/awsdevops/awsdevops_test.go | 15 ++- internal/awsdevops/setup.go | 6 +- internal/awsdevops/skill.go | 116 ++++++++++++++++-------- internal/awsdevops/skill_test.go | 85 +++++++++++++++-- 6 files changed, 179 insertions(+), 55 deletions(-) diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index 4129d451f..c3df5644b 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -473,7 +473,7 @@ func TestSetupSaysWhenTheSecondPlanReusesAServer(t *testing.T) { type failingSkillAgent struct{ countingAgent } func (failingSkillAgent) CreateAsset(context.Context, *devopsagent.CreateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { - return nil, errors.New("AccessDeniedException: no aidevops:CreateAsset") + return nil, &agenttypes.AccessDeniedException{Message: aws.String("not authorized to perform: aidevops:CreateAsset")} } // A failed skill step warns at the end, and setup still prints the app URL. @@ -486,7 +486,9 @@ func TestSetupWarnsWhenSkillFails(t *testing.T) { require.NoError(t, setupWith(cmd, clients, newSpaceOptions(), true, false)) assert.Contains(t, w.calls, "RegisterService") - assert.Contains(t, errOut.String(), "Warning: unable to create skill "+awsdevops.ExperimentSkillName) - assert.Contains(t, stdoutOf(cmd), "The "+awsdevops.ExperimentSkillName+" skill is not set up") + assert.Contains(t, stdoutOf(cmd), "Warning: unable to create skill "+awsdevops.ExperimentSkillName) + assert.Contains(t, stdoutOf(cmd), "aidevops:CreateAsset") assert.Contains(t, stdoutOf(cmd), "Open the DevOps Agent at:") + // The warning shows one time. + assert.Equal(t, 1, strings.Count(errOut.String()+stdoutOf(cmd), "unable to create skill")) } diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index a3f2e7e52..8be8cdbed 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -184,9 +184,7 @@ func setupWith( } if result.SkillError != "" { - _, _ = fmt.Fprintf(cmd.OutOrStdout(), - "\nThe %s skill is not set up: %s. Run setup again to add it\n", - awsdevops.ExperimentSkillName, result.SkillError) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.SkillError) } // Only a connect run leaves the app off, on a space that setup did not diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index 04e576a6f..fd0258125 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -68,7 +68,9 @@ type fakeAgent struct { // assets holds the skill assets of every space. assets []agenttypes.Asset // skillFiles holds the SKILL.md text by asset ID. - skillFiles map[string]string + skillFiles map[string]string + // skillBytes holds a SKILL.md that AWS sends as bytes, by asset ID. + skillBytes map[string][]byte listAssetsErr error createAssetErr error createAssets []*devopsagent.CreateAssetInput @@ -98,8 +100,17 @@ func (f *fakeAgent) CreateAsset(_ context.Context, in *devopsagent.CreateAssetIn func (f *fakeAgent) GetAssetFile(_ context.Context, in *devopsagent.GetAssetFileInput, _ ...func(*devopsagent.Options)) (*devopsagent.GetAssetFileOutput, error) { f.calls = append(f.calls, "GetAssetFile") + if aws.ToString(in.Path) != "SKILL.md" { + return nil, &agenttypes.ResourceNotFoundException{} + } + if body, ok := f.skillBytes[aws.ToString(in.AssetId)]; ok { + return &devopsagent.GetAssetFileOutput{File: &agenttypes.AssetFile{ + Path: in.Path, + Content: &agenttypes.AssetFileBodyMemberBytes{Value: body}, + }}, nil + } text, ok := f.skillFiles[aws.ToString(in.AssetId)] - if !ok || aws.ToString(in.Path) != "SKILL.md" { + if !ok { return nil, &agenttypes.ResourceNotFoundException{} } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 270429039..9998107b5 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -138,8 +138,10 @@ type SetupResult struct { MCPServiceID string `json:"mcpServiceId,omitempty"` MCPAssociationID string `json:"mcpAssociationId,omitempty"` SkillAssetID string `json:"skillAssetId,omitempty"` - // SkillError is set if the skill step failed. The skill is optional, so - // the rest of the setup ran. + // SkillAction is what the skill step did, for example "kept" or "left". + SkillAction string `json:"skillAction,omitempty"` + // SkillError is set if the skill step failed or was skipped. The skill is + // optional, so the rest of the setup ran. SkillError string `json:"skillError,omitempty"` } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 785a1f754..715fab97f 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -71,10 +71,11 @@ type SkillAction int const ( SkillCreate SkillAction = iota SkillKeep - // SkillUpdate replaces a SKILL.md that an older ldcli sent. + // SkillUpdate replaces a SKILL.md that an older ldcli sent, or one that + // is empty. SkillUpdate - // SkillLeave means that someone edited the SKILL.md, so setup does not - // replace it without --replace-skill. + // SkillLeave means that someone edited the SKILL.md, or that the skill has + // none, so setup does not change it without --replace-skill. SkillLeave SkillReplace // SkillSkip means that setup could not read the skills of the space. The @@ -88,48 +89,67 @@ type SkillStep struct { AssetID string // Version is the asset version at plan time. A write stops if it changed. Version int32 - // Reason says why the step is skipped. - Reason string + // NoFile means that the skill has no SKILL.md, for example a skill that + // someone made from a PDF. + NoFile bool + // Err says why the step is skipped. + Err error } func (s SkillStep) describe() string { label := fmt.Sprintf("%s (%s)", ExperimentSkillName, s.AssetID) + why := "someone edited it" + if s.NoFile { + why = "it has no " + skillFilePath + } switch s.Action { case SkillKeep: return "Keep skill " + label case SkillUpdate: return fmt.Sprintf("Update skill %s to the instructions of this ldcli version", label) case SkillLeave: - return fmt.Sprintf("Leave skill %s unchanged because someone edited it. "+ - "Pass --replace-skill to overwrite it", label) + return fmt.Sprintf("Leave skill %s unchanged because %s. Pass --replace-skill to overwrite it", label, why) case SkillReplace: - return fmt.Sprintf("Overwrite the edited instructions of skill %s", label) + return fmt.Sprintf("Overwrite skill %s, which is not the ldcli version because %s", label, why) case SkillSkip: - return fmt.Sprintf("Skip skill %s: %s", ExperimentSkillName, s.Reason) + return skillFailure(fmt.Errorf("skip skill %s: %w", ExperimentSkillName, s.Err)) default: return fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) } } +// skillFailure is the text of a failed skill step. A missing permission is +// the most likely cause, so it names the permissions that the step needs. +func skillFailure(err error) string { + if errorCode(err) != "AccessDeniedException" { + return err.Error() + } + + return fmt.Sprintf("%v. The skill step needs aidevops:ListAssets, aidevops:GetAssetFile, "+ + "aidevops:CreateAsset, and aidevops:UpdateAsset", err) +} + func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep) SkillStep { if space.Action == SpaceCreate { return SkillStep{Action: SkillCreate} } asset, err := findSkill(ctx, clients, space.ID, ExperimentSkillName) if err != nil { - return SkillStep{Action: SkillSkip, Reason: errorCode(err)} + return SkillStep{Action: SkillSkip, Err: err} } if asset == nil { return SkillStep{Action: SkillCreate} } step := SkillStep{AssetID: aws.ToString(asset.AssetId), Version: aws.ToInt32(asset.Version)} - text, err := skillText(ctx, clients, space.ID, step.AssetID) + text, found, err := skillText(ctx, clients, space.ID, step.AssetID) + step.NoFile = !found switch { case err != nil: - return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Reason: errorCode(err)} + return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} case text == ExperimentSkill: step.Action = SkillKeep - case slices.Contains(shippedSkillHashes, skillHash(text)): + case found && (text == "" || slices.Contains(shippedSkillHashes, skillHash(text))): + // An empty file has nothing to lose. step.Action = SkillUpdate case opts.ReplaceSkill: step.Action = SkillReplace @@ -177,8 +197,9 @@ func assetName(asset agenttypes.Asset) string { return metadata.Name } -// skillText returns the SKILL.md of the skill, or "" if it has none. -func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (string, error) { +// skillText returns the SKILL.md of the skill. found is false if the skill +// has no SKILL.md. +func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (text string, found bool, err error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), AssetId: aws.String(assetID), @@ -187,18 +208,20 @@ func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (s var missing *agenttypes.ResourceNotFoundException switch { case errors.As(err, &missing): - return "", nil + return "", false, nil case err != nil: - return "", fmt.Errorf("unable to read skill %s (%s): %w", ExperimentSkillName, assetID, err) + return "", false, fmt.Errorf("unable to read skill %s (%s): %w", ExperimentSkillName, assetID, err) case file.File == nil: - return "", nil + return "", false, nil } - text, _ := file.File.Content.(*agenttypes.AssetFileBodyMemberText) - if text == nil { - return "", nil + switch body := file.File.Content.(type) { + case *agenttypes.AssetFileBodyMemberText: + return body.Value, true, nil + case *agenttypes.AssetFileBodyMemberBytes: + return string(body.Value), true, nil } - return text.Value, nil + return "", false, nil } // checkSkill compares the space with a step that writes. A change to the @@ -221,8 +244,20 @@ func checkSkill(ctx context.Context, clients Clients, spaceID string, step Skill return nil } +// The values of SetupResult.SkillAction. +const ( + SkillCreated = "created" + SkillKept = "kept" + SkillUpdated = "updated" + SkillLeft = "left" + SkillOverwritten = "overwritten" + SkillSkipped = "skipped" + SkillFailed = "failed" +) + // installSkill does the skill step. The skill is optional, so a failure does -// not stop setup. It goes in result.SkillError, and the caller reports it. +// not stop setup. The failure goes in result.SkillError, and the caller +// reports it. func installSkill( ctx context.Context, clients Clients, @@ -231,9 +266,14 @@ func installSkill( logf func(string, ...any), ) { result.SkillAssetID = step.AssetID - if err := doSkillStep(ctx, clients, step, result, logf); err != nil { - result.SkillError = err.Error() - logf("Warning: %v", err) + action, err := doSkillStep(ctx, clients, step, result, logf) + result.SkillAction = action + if err != nil { + result.SkillAction = SkillFailed + if step.Action == SkillSkip { + result.SkillAction = SkillSkipped + } + result.SkillError = skillFailure(err) } } @@ -243,25 +283,24 @@ func doSkillStep( step SkillStep, result *SetupResult, logf func(string, ...any), -) error { +) (string, error) { switch step.Action { case SkillKeep: logf("Kept skill %s (%s)", ExperimentSkillName, step.AssetID) - return nil + return SkillKept, nil case SkillLeave: - logf("Skill %s (%s) has edits, so ldcli did not change it", ExperimentSkillName, step.AssetID) - logf("Pass --replace-skill to overwrite it") + logf("Left skill %s (%s) unchanged. Pass --replace-skill to overwrite it", ExperimentSkillName, step.AssetID) - return nil + return SkillLeft, nil case SkillSkip: - return fmt.Errorf("skipped skill %s: %s", ExperimentSkillName, step.Reason) + return "", fmt.Errorf("setup skipped skill %s: %w", ExperimentSkillName, step.Err) } // The other steps can take minutes, so check the skill right before the // write. if err := checkSkill(ctx, clients, result.AgentSpaceID, step); err != nil { - return err + return "", err } content := &agenttypes.AssetContentMemberFile{Value: agenttypes.AssetFileContent{ Path: aws.String(skillFilePath), @@ -279,12 +318,12 @@ func doSkillStep( }), }) if err != nil { - return fmt.Errorf("unable to create skill %s: %w", ExperimentSkillName, err) + return "", fmt.Errorf("unable to create skill %s: %w", ExperimentSkillName, err) } result.SkillAssetID = aws.ToString(created.Asset.AssetId) logf("Created skill %s (%s)", ExperimentSkillName, result.SkillAssetID) - return nil + return SkillCreated, nil } // AWS updates only the metadata keys in the request, so the status and @@ -295,9 +334,12 @@ func doSkillStep( Content: content, Metadata: document.NewLazyDocument(map[string]any{"description": experimentSkillDescription}), }); err != nil { - return fmt.Errorf("unable to update skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) + return "", fmt.Errorf("unable to update skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) } logf("Updated skill %s (%s)", ExperimentSkillName, step.AssetID) + if step.Action == SkillReplace { + return SkillOverwritten, nil + } - return nil + return SkillUpdated, nil } diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index 4fa207e28..37437b186 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -3,7 +3,7 @@ package awsdevops_test import ( "context" "encoding/json" - "errors" + "strings" "testing" "github.com/aws/aws-sdk-go-v2/aws" @@ -104,9 +104,49 @@ func TestSetupKeepsSameSkill(t *testing.T) { assert.Empty(t, agent.createAssets) assert.Empty(t, agent.updateAssets) assert.Equal(t, "ki-ours", result.SkillAssetID) + assert.Equal(t, awsdevops.SkillKept, result.SkillAction) assert.Empty(t, result.SkillError) } +// AWS can send the file as bytes. The text is the same, so setup keeps it. +func TestPlanSetupKeepsSameSkillSentAsBytes(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillBytes: map[string][]byte{"ki-ours": []byte(awsdevops.ExperimentSkill)}, + } + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillKeep, plan.Skill.Action) +} + +// An empty SKILL.md has no edits to lose, so setup updates it without the flag. +func TestPlanSetupUpdatesEmptySkill(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": ""}, + } + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) +} + +// A skill without SKILL.md, for example one made from a PDF, is not ours. +func TestPlanSetupLeavesSkillWithoutFile(t *testing.T) { + agent := &fakeAgent{assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}} + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillLeave, plan.Skill.Action) + assert.Equal(t, + "Leave skill experiment-orchestration (ki-ours) unchanged because it has no SKILL.md. Pass --replace-skill to overwrite it", + plan.Text().Skill) +} + // A copy from an older ldcli has no edits, so setup updates it without the flag. func TestSetupUpdatesOlderShippedSkill(t *testing.T) { t.Cleanup(awsdevops.ShipSkill("# Instructions of an older ldcli")) @@ -121,10 +161,11 @@ func TestSetupUpdatesOlderShippedSkill(t *testing.T) { assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) assert.Contains(t, plan.Text().Skill, "Update skill experiment-orchestration (ki-ours)") - _, err = awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) require.Len(t, agent.updateAssets, 1) assert.Equal(t, "ki-ours", aws.ToString(agent.updateAssets[0].AssetId)) + assert.Equal(t, awsdevops.SkillUpdated, result.SkillAction) } func TestSetupLeavesEditedSkill(t *testing.T) { @@ -143,6 +184,8 @@ func TestSetupLeavesEditedSkill(t *testing.T) { require.NoError(t, err) assert.Empty(t, agent.createAssets) assert.Empty(t, agent.updateAssets) + // JSON output shows that the skill is not the ldcli version. + assert.Equal(t, awsdevops.SkillLeft, result.SkillAction) assert.Empty(t, result.SkillError) } @@ -154,10 +197,11 @@ func TestSetupOverwritesEditedSkillWithFlag(t *testing.T) { opts := reuseOptions() opts.ReplaceSkill = true - _, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) require.NoError(t, err) assert.Empty(t, agent.createAssets) + assert.Equal(t, awsdevops.SkillOverwritten, result.SkillAction) require.Len(t, agent.updateAssets, 1) in := agent.updateAssets[0] assert.Equal(t, "ki-ours", aws.ToString(in.AssetId)) @@ -193,6 +237,7 @@ func TestConnectCreatesSkill(t *testing.T) { require.Len(t, agent.createAssets, 1) assert.Equal(t, "space-2", aws.ToString(agent.createAssets[0].AgentSpaceId)) assert.Equal(t, "ki-created", result.SkillAssetID) + assert.Equal(t, awsdevops.SkillCreated, result.SkillAction) } // The skill step goes after the MCP server, and a plan change in it only @@ -253,37 +298,61 @@ func TestRunPlanSkipsSkillEditedAfterPlan(t *testing.T) { require.NoError(t, err) assert.Empty(t, agent.updateAssets) assert.Contains(t, result.SkillError, "changed after the plan") + assert.Equal(t, awsdevops.SkillFailed, result.SkillAction) +} + +func accessDenied(action string) error { + return &agenttypes.AccessDeniedException{ + Message: aws.String("User: arn:aws:sts::123456789012:assumed-role/Dev/me is not authorized to perform: " + action), + } } // Without aidevops:ListAssets, the plan skips the skill and setup does the -// other steps. +// other steps. The AWS error has a code, and the text keeps its message, +// which names the denied action. func TestSetupSkipsSkillWhenAssetsCannotBeListed(t *testing.T) { - agent := &fakeAgent{listAssetsErr: errors.New("AccessDeniedException: no aidevops:ListAssets")} + agent := &fakeAgent{listAssetsErr: accessDenied("aidevops:ListAssets")} clients := newTestClients(agent, newFakeIAM()) plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) assert.Equal(t, awsdevops.SkillSkip, plan.Skill.Action) - assert.Contains(t, plan.Text().Skill, "aidevops:ListAssets") + assert.Contains(t, plan.Text().Skill, "not authorized to perform: aidevops:ListAssets") + assert.Contains(t, plan.Text().Skill, "The skill step needs") result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) assert.Contains(t, agent.calls, "RegisterService") - assert.Contains(t, result.SkillError, "aidevops:ListAssets") + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) + assert.Contains(t, result.SkillError, "not authorized to perform: aidevops:ListAssets") + assert.Equal(t, 1, strings.Count(result.SkillError, awsdevops.ExperimentSkillName)) } func TestSetupWarnsOnSkillCreateFailure(t *testing.T) { - agent := &fakeAgent{createAssetErr: errors.New("ServiceQuotaExceededException: 200 skills")} + agent := &fakeAgent{createAssetErr: accessDenied("aidevops:CreateAsset")} result, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) require.NoError(t, err) assert.Contains(t, agent.calls, "RegisterService") assert.Equal(t, "mcp-1", result.MCPServiceID) + assert.Equal(t, awsdevops.SkillFailed, result.SkillAction) assert.Contains(t, result.SkillError, "unable to create skill experiment-orchestration") + assert.Contains(t, result.SkillError, "not authorized to perform: aidevops:CreateAsset") assert.Empty(t, result.SkillAssetID) } +// Other failures do not get the permission text. +func TestSkillFailureNamesPermissionsOnlyWhenDenied(t *testing.T) { + agent := &fakeAgent{createAssetErr: &agenttypes.ServiceQuotaExceededException{Message: aws.String("200 skills")}} + + result, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + + require.NoError(t, err) + assert.Contains(t, result.SkillError, "200 skills") + assert.NotContains(t, result.SkillError, "The skill step needs") +} + func keysOf(m map[string]any) []string { keys := make([]string, 0, len(m)) for key := range m { From 6dfcc5688e2eb8656625962edf3af9e2fa0e976d Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 13:41:19 -0700 Subject: [PATCH 04/34] fix: report a plan change as a skipped skill and warn on stderr in JSON runs If the skill changes between the plan and the write, skillAction is now skipped, because setup tried no write. The warning for a denied GetAssetFile now names the skill one time. The plan line for a skipped skill starts with "Skip skill". A JSON run now also writes the skill warning to stderr. If someone deletes the skill after the list, the plan now creates a new skill. Before, the plan said that the skill had no SKILL.md. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/plan_print_test.go | 14 +++++++++++ cmd/aws_devops_agent/setup.go | 5 ++++ internal/awsdevops/awsdevops_test.go | 13 ++++++++-- internal/awsdevops/skill.go | 22 ++++++++++++++--- internal/awsdevops/skill_test.go | 32 ++++++++++++++++++++++++- 5 files changed, 80 insertions(+), 6 deletions(-) diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index c3df5644b..b03b87140 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -492,3 +492,17 @@ func TestSetupWarnsWhenSkillFails(t *testing.T) { // The warning shows one time. assert.Equal(t, 1, strings.Count(errOut.String()+stdoutOf(cmd), "unable to create skill")) } + +// stdout holds only the JSON, so the warning goes to stderr. +func TestSetupJSONWarnsOnStderrWhenSkillFails(t *testing.T) { + cmd, errOut := setupTestCmd(t, "") + cmd.SetIn(noReadStdin{t}) + clients, _ := countingClients() + clients.Agent = failingSkillAgent{clients.Agent.(countingAgent)} + + require.NoError(t, setupWith(cmd, clients, newSpaceOptions(), false, false)) + + assert.True(t, json.Valid([]byte(stdoutOf(cmd))), "stdout is not JSON: %s", stdoutOf(cmd)) + assert.Contains(t, stdoutOf(cmd), `"skillAction": "failed"`) + assert.Contains(t, errOut.String(), "Warning: unable to create skill "+awsdevops.ExperimentSkillName) +} diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 8be8cdbed..82bc832e7 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -173,6 +173,11 @@ func setupWith( } if !plaintext { + // stdout holds only the JSON, so a CI log shows the warning on stderr. + if result.SkillError != "" { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.SkillError) + } + return printJSON(cmd, result) } diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index fd0258125..ebe4195e6 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -70,8 +70,11 @@ type fakeAgent struct { // skillFiles holds the SKILL.md text by asset ID. skillFiles map[string]string // skillBytes holds a SKILL.md that AWS sends as bytes, by asset ID. - skillBytes map[string][]byte - listAssetsErr error + skillBytes map[string][]byte + listAssetsErr error + getFileErr error + // onGetFile runs at the start of each GetAssetFile call. + onGetFile func() createAssetErr error createAssets []*devopsagent.CreateAssetInput updateAssets []*devopsagent.UpdateAssetInput @@ -100,6 +103,12 @@ func (f *fakeAgent) CreateAsset(_ context.Context, in *devopsagent.CreateAssetIn func (f *fakeAgent) GetAssetFile(_ context.Context, in *devopsagent.GetAssetFileInput, _ ...func(*devopsagent.Options)) (*devopsagent.GetAssetFileOutput, error) { f.calls = append(f.calls, "GetAssetFile") + if f.onGetFile != nil { + f.onGetFile() + } + if f.getFileErr != nil { + return nil, f.getFileErr + } if aws.ToString(in.Path) != "SKILL.md" { return nil, &agenttypes.ResourceNotFoundException{} } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 715fab97f..c797d4291 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -112,7 +112,7 @@ func (s SkillStep) describe() string { case SkillReplace: return fmt.Sprintf("Overwrite skill %s, which is not the ldcli version because %s", label, why) case SkillSkip: - return skillFailure(fmt.Errorf("skip skill %s: %w", ExperimentSkillName, s.Err)) + return skillFailure(fmt.Errorf("Skip skill %s: %w", ExperimentSkillName, s.Err)) default: return fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) } @@ -142,6 +142,19 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp } step := SkillStep{AssetID: aws.ToString(asset.AssetId), Version: aws.ToInt32(asset.Version)} text, found, err := skillText(ctx, clients, space.ID, step.AssetID) + if err == nil && !found { + // AWS also sends ResourceNotFoundException for an asset that someone + // deleted after the list. A deleted skill gets a new one. + asset, err = findSkill(ctx, clients, space.ID, ExperimentSkillName) + switch { + case err != nil: + return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} + case asset == nil: + return SkillStep{Action: SkillCreate} + case aws.ToString(asset.AssetId) != step.AssetID: + return SkillStep{Action: SkillSkip, Err: planChanged("skill %s changed during the plan", ExperimentSkillName)} + } + } step.NoFile = !found switch { case err != nil: @@ -210,7 +223,8 @@ func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (t case errors.As(err, &missing): return "", false, nil case err != nil: - return "", false, fmt.Errorf("unable to read skill %s (%s): %w", ExperimentSkillName, assetID, err) + // The caller names the skill. + return "", false, fmt.Errorf("unable to read %s of asset %s: %w", skillFilePath, assetID, err) case file.File == nil: return "", false, nil } @@ -270,7 +284,9 @@ func installSkill( result.SkillAction = action if err != nil { result.SkillAction = SkillFailed - if step.Action == SkillSkip { + // A plan change stops the step before the write, so setup tried + // nothing. + if step.Action == SkillSkip || errors.Is(err, ErrPlanChanged) { result.SkillAction = SkillSkipped } result.SkillError = skillFailure(err) diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index 37437b186..f79cbf33d 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -255,6 +255,7 @@ func TestRunPlanSkipsSkillThatAppears(t *testing.T) { assert.Empty(t, agent.createAssets) assert.Contains(t, agent.calls, "RegisterService") assert.Contains(t, result.SkillError, "appeared after the plan") + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) } func TestRunPlanSkipsSkillThatIsGone(t *testing.T) { @@ -298,7 +299,8 @@ func TestRunPlanSkipsSkillEditedAfterPlan(t *testing.T) { require.NoError(t, err) assert.Empty(t, agent.updateAssets) assert.Contains(t, result.SkillError, "changed after the plan") - assert.Equal(t, awsdevops.SkillFailed, result.SkillAction) + // The step stopped before the write. + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) } func accessDenied(action string) error { @@ -317,6 +319,7 @@ func TestSetupSkipsSkillWhenAssetsCannotBeListed(t *testing.T) { plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) assert.Equal(t, awsdevops.SkillSkip, plan.Skill.Action) + assert.True(t, strings.HasPrefix(plan.Text().Skill, "Skip skill experiment-orchestration: "), plan.Text().Skill) assert.Contains(t, plan.Text().Skill, "not authorized to perform: aidevops:ListAssets") assert.Contains(t, plan.Text().Skill, "The skill step needs") @@ -328,6 +331,33 @@ func TestSetupSkipsSkillWhenAssetsCannotBeListed(t *testing.T) { assert.Equal(t, 1, strings.Count(result.SkillError, awsdevops.ExperimentSkillName)) } +func TestSetupSkipsSkillWhenFileCannotBeRead(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + getFileErr: accessDenied("aidevops:GetAssetFile"), + } + + result, err := awsdevops.ConnectExisting(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) + assert.Contains(t, result.SkillError, "not authorized to perform: aidevops:GetAssetFile") + assert.Contains(t, result.SkillError, "ki-ours") + assert.Equal(t, 1, strings.Count(result.SkillError, awsdevops.ExperimentSkillName)) +} + +// AWS sends ResourceNotFoundException also for a skill that someone deleted +// after the list. That skill does not exist, so the plan creates one. +func TestPlanSetupCreatesSkillDeletedDuringPlan(t *testing.T) { + agent := &fakeAgent{assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}} + agent.onGetFile = func() { agent.assets = nil } + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillStep{Action: awsdevops.SkillCreate}, plan.Skill) +} + func TestSetupWarnsOnSkillCreateFailure(t *testing.T) { agent := &fakeAgent{createAssetErr: accessDenied("aidevops:CreateAsset")} From 44041f78c7c57cc10e0bcdf37aaa72a06273ad42 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 14:56:45 -0700 Subject: [PATCH 05/34] fix: let the experiment skill resume across runs and wait 2 weeks for a result A scheduled run of the agent has no memory of the run that started the experiment. The skill now keeps an experiment record, tags the flags that it owns, and resumes in-flight work before it starts a new goal. The experiment now runs at least 2 weeks before a decision, not 30 minutes. A result that is not a win stops the iteration with control as the winner and turns the flag off. Guarded rollout stages stay at or below 50%, as LaunchDarkly requires. If no coding agent is available, the skill waits for a person to confirm the deployment. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/skill.go | 2 +- .../skills/experiment-orchestration/SKILL.md | 76 ++++++++++++++----- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index c797d4291..1fa6c72fd 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -33,7 +33,7 @@ var ( // updates it without --replace-skill. When you change SKILL.md, add the hash // of the new text. Never remove a hash. var shippedSkillHashes = []string{ - "a5a063f2f3739ca73c10602e05f18d36f5d0e29c372be025e7e6c71d9846c4eb", + "ec5723b6afbaa0f791e71153fb83594b83912f77901030dcf92ba4d55471789c", } const ( diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md index ba54057c4..f7ef92631 100644 --- a/internal/awsdevops/skills/experiment-orchestration/SKILL.md +++ b/internal/awsdevops/skills/experiment-orchestration/SKILL.md @@ -9,10 +9,35 @@ Use this skill when you have a goal you want to move through experimentation (e. **Core principle: experiment first, then guarded rollout.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before ramping it up through a guarded rollout. Never start a guarded rollout blind — it exists only to scale a change the experiment has already shown to work. -**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Only stop and ask the user something if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result). +**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end, one step per invocation while you are resuming. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Only stop and ask the user something if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result), or if a person must implement and deploy the change (Step 3). + +**Resuming:** an experiment runs for weeks, so most invocations pick up work that an earlier one started. Always follow "Resuming in-flight work" below first. Start at Step 1 only for a goal that no open record already covers. **The final report (Step 8) is mandatory, not optional.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. +## Experiment Record + +Keep one record per flag in the durable store your environment provides, and update it whenever the step changes. A later invocation has no memory of this one, so the record must hold everything it needs: + +- Flag key and project and environment keys +- Goal: metric, direction, target, and baseline +- Hypothesis +- Current step (for example, "Step 3: waiting for deployment", "Step 7: Phase 1 running", "Step 8: report produced") +- Implementation details for the report: files, branch, and release readiness result ("implemented by a person" when no coding agent did it) +- Deployment confirmation: who confirmed that the change is deployed, and when +- Experiment key, start date, and stop date +- Guardrail metric baselines at the experiment start +- Guarded rollout start date and end date + +Every flag this skill creates carries the tag `experiment-orchestration`, and its description holds the goal and the hypothesis. This marks the flags this skill owns, and it keeps the goal recoverable if the store is lost. + +## Resuming in-flight work + +1. Read every experiment record that is not at "Step 8: report produced". If you have no store, list the flags that are tagged `experiment-orchestration` and are not archived, and rebuild the records from their descriptions and their experiments. Never act on a flag without this tag. +2. For each one, read the current state from LaunchDarkly (flag, experiment, guarded rollout) and compare it with the record. +3. Advance each one by at most one step, following the step in this skill that the record names. A terminal outcome also produces its Step 8 report in the same invocation. Never start a new goal or accept a new hypothesis while resuming; those need a person. +4. Update the record. If nothing is in flight, say so and stop. + ## Step 1: Goal Clarification Before doing anything, get answers to: @@ -41,10 +66,18 @@ Explore the target repository/codebase to find a plausible change: ## Step 3: Implementation -1. Create a boolean feature flag, OFF by default in all environments. Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. -2. Hand off implementation to your coding agent/tool of choice, with clear instructions to gate the change behind the exact flag key from step 1. -3. This step is asynchronous — check status periodically rather than looping tightly on it. -4. Once implementation completes, move to Step 4 with the resulting branch/PR. If it fails, report the error and stop. +1. Create a boolean feature flag, OFF by default in all environments, with the tag `experiment-orchestration` and a description that holds the goal and the hypothesis. Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. If the change is in a browser or mobile app, make the flag available to client-side SDKs (`clientSideAvailability.usingEnvironmentId`, or `usingMobileKey` for mobile); otherwise the app cannot read the flag and the experiment collects no data. +2. Create the experiment record (see "Experiment Record"). +3. Hand off implementation to your coding agent/tool of choice, with clear instructions to gate the change behind the exact flag key from step 1. +4. This step is asynchronous — check status periodically rather than looping tightly on it. +5. Once implementation completes, move to Step 4 with the resulting branch/PR. If it fails, report the error and stop. + +**If no coding agent is available:** report the flag key and the change to make, set the record to "Step 3: waiting for deployment", and stop. A person implements, reviews, and deploys the change. Continue at Step 6 (skipping Steps 4–5) only when one of these is true: + +- A person says in this invocation that the change is deployed. Write their confirmation to the record. +- The record already holds a person's deployment confirmation. + +Until then, a resumed invocation reports "waiting for deployment" for this flag and changes nothing. Flag evaluations from the app show that the code evaluates the flag, but not that the treatment works, so they never replace a person's confirmation. ## Step 4: Release Readiness @@ -72,12 +105,14 @@ Deployment typically won't fire automatically on merge if your workflow is manua Prove the change on a small, fixed slice of traffic. Do **not** start a guarded rollout here — that's Step 7, and only after this proves out. 1. Turn the flag ON. -2. Configure a fixed 50/50 split across 10% of traffic (a flat allocation, not a staged ramp) on your chosen randomization unit (typically "user"). The remaining 90% of traffic is excluded from the experiment entirely. -3. Create an experiment with: - - Exactly one primary metric: the success metric from Step 1. +2. Create an experiment on the flag's default rule (`"fallthrough"`) with: + - Audience allocation of 10% of the rule's traffic (a flat allocation, not a staged ramp) on your chosen randomization unit (typically "user"). The other 90% receive the control variation and are not analyzed. + - Treatments: control (off) at 50%, treatment (on) at 50%. Treatment allocations must sum to 100%; the 10% is the audience allocation, not the treatments. + - Exactly one primary metric: the success metric from Step 1. If it does not exist, create the metric and the event that feeds it first. - Guardrail metric(s): always include your error-rate metric; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. - - Treatments: control (off) at 50%, treatment (on) at 50%, allocated to 10% of total traffic. -4. Start the experiment/data collection. + - Sequential testing, if your methodology offers it, so that checking results on every invocation does not inflate false positives. +3. Start the experiment/data collection. +4. Record the experiment key, the start date, and the guardrail baselines, and set the record to "Step 7: Phase 1 running". 5. Move to Step 7 to monitor toward a decision. ## Step 7: Monitoring & Outcome @@ -86,23 +121,26 @@ Check status periodically — don't tight-loop. In an interactive session, check **Phase 1 — Prove the experiment at 10% (gate before any rollout):** -Watch for statistical significance on the primary metric: +Run the experiment for at least 2 full weeks, so that weekday and weekend behavior are both in the data. Check results once per invocation (daily is enough), and decide on the primary metric: -- **Significant + positive lift** → experiment proven. Stop the experiment iteration and move to Phase 2. -- **Significant + negative lift** → declare a loser, archive the flag, skip Phase 2, go straight to the Step 8 report. -- **No significance after a reasonable ceiling (e.g. 30 minutes)** → report "inconclusive, need more traffic" and stop; don't proceed to Phase 2. +- **Guardrail regression at any time** → stop early with control as the winner (see "Ending without a win" below). +- **Before 2 weeks** → keep running, even if the primary metric looks significant. Report progress only. +- **After 2 weeks, significant + positive lift** → experiment proven. Stop the iteration with the treatment as the winner and move to Phase 2. +- **After 2 weeks, significant + negative lift** → declare a loser and end without a win. +- **No significance after 4 weeks** → declare "inconclusive, need more traffic" and end without a win. -Never declare a winner off a single data point or before your stats engine confirms significance. +**Ending without a win:** stop the iteration with control as the winner and a reason that says why, because stopping an iteration requires a winning treatment. Stopping does not change targeting, so then turn the flag off so that everyone gets control. Record the stop date, skip Phase 2, and go to the Step 8 report. Don't archive the flag: the code still evaluates it. Recommend removing the flag from the code, and then archiving it, in the report's Next Steps. + Never declare a winner off a single data point or before your stats engine confirms significance. **Phase 2 — Guarded rollout ramp (only after Phase 1 proves the change):** Start a guarded rollout with: - The winning ("on") variation as the test, the original as control. - Same randomization unit as the experiment. -- **Exactly 3 monitored stages, capped well below 100%** — e.g. 20% → 30% → 40%, ~60 minutes monitoring each. Don't add a stage at or above 100%; Guarded-rollout implementations reject stages above 50% audience allocation, and the rollout auto-promotes to 100% itself once the final monitored stage completes cleanly — no explicit 100% stage needed. +- **Exactly 3 monitored stages, none above 50%** — e.g. 10% → 25% → 50%, ~24 hours monitoring each. LaunchDarkly rejects any step above 50%, because each step needs an equal split between the original and new variations, and the rollout auto-promotes to 100% itself once the final monitored stage completes cleanly — no explicit 100% stage needed. - The same primary + guardrail metrics as the experiment, each configured to notify and auto-rollback on regression. -Track stage progression. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, clean up the flag (deprecate/archive it), and go to the Step 8 report. +LaunchDarkly advances the stages and rolls back on its own, so each invocation only reads the rollout status and updates the record. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, confirm that everyone gets control, and go to the Step 8 report. Recommend flag cleanup in Next Steps rather than archiving the flag yourself. Once the final stage completes cleanly and auto-promotes to 100%, declare a winner and go to the Step 8 report. @@ -162,6 +200,8 @@ Runs automatically the instant Step 7 reaches a terminal outcome (winner + auto- - Always include an error-rate (or equivalent "don't break prod") metric as a guardrail, separate from your success metric. - Add a performance guardrail (e.g. p95 latency) for performance-focused changes. - Every rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. -- **Cap guarded rollout stages well below 100%** (most platforms reject stages ≥50% audience allocation) and let the platform auto-promote to 100% after the final stage — don't try to add an explicit 100% stage. +- **Run an experiment at least 2 full weeks** before deciding, unless a guardrail regresses. +- **Keep every guarded rollout stage at or below 50%** (LaunchDarkly rejects larger steps) and let the platform auto-promote to 100% after the final stage — don't try to add an explicit 100% stage. +- Update the experiment record on every step change, so that the next invocation can resume. - Distinguish a metric-driven rollback (don't retry) from an external-cause rollback (safe to retry once fixed) before restarting a rolled-back rollout. - The final report is automatic and mandatory on every terminal outcome — never defer it to a follow-up ask. From 9a09cc6bdd52bb20eb8098ad538932bd884d650c Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 16:15:34 -0700 Subject: [PATCH 06/34] fix: shape the experiment skill for a chat run with approved writes In AWS DevOps Agent, every LaunchDarkly write needs a person to approve it in chat, and no coding agent or CI/CD is connected. The skill now runs as two chats: one that starts an experiment and one that checks on experiments. A person implements and deploys the change, and the agent stops at each wait. - The flag serves false when it is on, so turning it on before the experiment starts does not give everyone the treatment. - The experiment uses 10% of traffic through 5% and 5% treatments and notInExperimentVariationId, with sequential testing. - The agent shows the draft experiment and waits for a confirmation before it starts it, as the create-experiment tool asks. - The record in the flag description holds the project and environment. - The release review, merge, and deploy steps are gone. The allowlist adds update-flag-settings, for the record, and create-metric. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/setup.go | 7 +- internal/awsdevops/skill.go | 2 +- internal/awsdevops/skill_test.go | 4 +- .../skills/experiment-orchestration/SKILL.md | 268 ++++++++---------- 4 files changed, 135 insertions(+), 146 deletions(-) diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 9998107b5..95aafcf51 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -48,7 +48,10 @@ const ( // The agent invokes read-only tools without asking for approval, so anything // that changes LaunchDarkly state has to be classified explicitly. The lists // cover the steps of the experiment-orchestration skill: find a flag, create -// it, run an experiment, read its results, and ramp or clean up the flag. +// it, run an experiment, read its results, and ramp or clean up the flag. The +// skill keeps its experiment record in the flag description and creates a +// missing primary metric, so the lists include update-flag-settings and +// create-metric. // They also cover incident work: turn a change off for one segment or // context, and ask for approval where an environment needs it. var ( @@ -76,6 +79,8 @@ var ( mcpMutativeTools = []string{ "toggle-flag", "create-flag", + "update-flag-settings", + "create-metric", "update-rollout", "archive-flag", "create-experiment", diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 1fa6c72fd..0028c62df 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -33,7 +33,7 @@ var ( // updates it without --replace-skill. When you change SKILL.md, add the hash // of the new text. Never remove a hash. var shippedSkillHashes = []string{ - "ec5723b6afbaa0f791e71153fb83594b83912f77901030dcf92ba4d55471789c", + "6972a3b18f2f8a52f9be2edc9157551f1bf5a7547026926b8eef9a348200d70e", } const ( diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index f79cbf33d..d44dd8440 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -31,7 +31,7 @@ func metadataOf(t *testing.T, doc document.Interface) map[string]any { func TestExperimentSkillFrontMatter(t *testing.T) { assert.Equal(t, "experiment-orchestration", awsdevops.ExperimentSkillName) - assert.Contains(t, awsdevops.ExperimentSkill, "# Automated Experimentation") + assert.Contains(t, awsdevops.ExperimentSkill, "# Experimentation") } // A change to SKILL.md must add its hash, so the next release updates the @@ -57,7 +57,7 @@ func TestSetupCreatesSkillInNewSpace(t *testing.T) { assert.Equal(t, "skill", aws.ToString(in.AssetType)) metadata := metadataOf(t, in.Metadata) assert.Equal(t, awsdevops.ExperimentSkillName, metadata["name"]) - assert.Contains(t, metadata["description"], "Orchestrates automated experimentation lifecycle") + assert.Contains(t, metadata["description"], "Start a LaunchDarkly experiment for a product goal") assert.Equal(t, []any{"GENERIC"}, metadata["agent_types"]) assert.NotContains(t, metadata, "status") file, ok := in.Content.(*agenttypes.AssetContentMemberFile) diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md index f7ef92631..16ee2a62d 100644 --- a/internal/awsdevops/skills/experiment-orchestration/SKILL.md +++ b/internal/awsdevops/skills/experiment-orchestration/SKILL.md @@ -1,207 +1,191 @@ --- name: "experiment-orchestration" -description: "Orchestrates automated experimentation lifecycle using LaunchDarkly Guarded Rollouts, an AI coding agent for implementation, and GitHub Actions for deployment." +description: "Start a LaunchDarkly experiment for a product goal, or check on running ones. Use when someone wants to move a goal (for example, increase checkout conversion by 15%) through a flag, an A/B experiment, and then a guarded rollout, or asks how their experiments are doing. Not for a standalone guarded rollout or flag change." --- -# Automated Experimentation +# Experimentation -Use this skill when you have a goal you want to move through experimentation (e.g., "increase checkout conversion by 15%", "decrease page load time by 20%"). +Use this skill in chat when a person wants to move a goal through experimentation (for example, "increase checkout conversion by 15%"), or asks how their experiments are doing. -**Core principle: experiment first, then guarded rollout.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before ramping it up through a guarded rollout. Never start a guarded rollout blind — it exists only to scale a change the experiment has already shown to work. +**Core principle: experiment first, then guarded rollout.** Always prove a change on a small, fixed slice of traffic with an A/B experiment before you ramp it up with a guarded rollout. A guarded rollout only scales a change that the experiment has already shown to work. -**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end, one step per invocation while you are resuming. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Only stop and ask the user something if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result), or if a person must implement and deploy the change (Step 3). +An experiment runs for weeks, so the work happens in separate chats: -**Resuming:** an experiment runs for weeks, so most invocations pick up work that an earlier one started. Always follow "Resuming in-flight work" below first. Start at Step 1 only for a goal that no open record already covers. +1. **Start an experiment** (Steps 1–4): goal, hypothesis, flag, a person ships the change, then the experiment starts. +2. **Check on experiments** (Steps 5–6): read where each experiment stands, decide when a decision is due, and report. -**The final report (Step 8) is mandatory, not optional.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. +Decide which one the person asked for. If they ask about experiments that already exist, go to "Check on experiments". Never start a goal that the person did not ask for. -## Experiment Record +## How you work -Keep one record per flag in the durable store your environment provides, and update it whenever the step changes. A later invocation has no memory of this one, so the record must hold everything it needs: +- **A person implements and deploys the code.** You cannot read or change the code, and you cannot deploy. You create and change things in LaunchDarkly only. +- **Every write needs the person's approval.** Before each write (create a flag, change a flag, start or stop an experiment, start a guarded rollout), say in one sentence what it changes. If the person rejects it, stop that step and do not try another way. +- **A blocked write is a setup problem.** If a write fails with "This action is not available in this agent space", the agent space does not allow LaunchDarkly writes: directed actions are off, or the tool is not allowed. Tell the person, and do not retry. +- **Change only your flags.** Change only flags that have the tag `experiment-orchestration`. Read other flags for collision checks and history, but never change them. +- **Stop at each wait.** When the next step needs a person (shipping code, or two weeks of data), say what happens next and end. Do not wait or poll in the chat. -- Flag key and project and environment keys -- Goal: metric, direction, target, and baseline -- Hypothesis -- Current step (for example, "Step 3: waiting for deployment", "Step 7: Phase 1 running", "Step 8: report produced") -- Implementation details for the report: files, branch, and release readiness result ("implemented by a person" when no coding agent did it) -- Deployment confirmation: who confirmed that the change is deployed, and when -- Experiment key, start date, and stop date -- Guardrail metric baselines at the experiment start -- Guarded rollout start date and end date - -Every flag this skill creates carries the tag `experiment-orchestration`, and its description holds the goal and the hypothesis. This marks the flags this skill owns, and it keeps the goal recoverable if the store is lost. +## Experiment record -## Resuming in-flight work +The next chat may be with another person, and you have no memory of this one. Keep the record in the description of the flag, and update it whenever a field changes: -1. Read every experiment record that is not at "Step 8: report produced". If you have no store, list the flags that are tagged `experiment-orchestration` and are not archived, and rebuild the records from their descriptions and their experiments. Never act on a flag without this tag. -2. For each one, read the current state from LaunchDarkly (flag, experiment, guarded rollout) and compare it with the record. -3. Advance each one by at most one step, following the step in this skill that the record names. A terminal outcome also produces its Step 8 report in the same invocation. Never start a new goal or accept a new hypothesis while resuming; those need a person. -4. Update the record. If nothing is in flight, say so and stop. +- Project and environment keys +- Goal: metric, direction, target, and baseline +- Hypothesis, and the change that the person implements +- Current step, for example "waiting for deployment", "experiment running", "guarded rollout running", or "report produced" +- Deployment: who confirmed that the change is deployed, and when -## Step 1: Goal Clarification +Read everything else (the experiment key, dates, results, and rollout stages) from LaunchDarkly. -Before doing anything, get answers to: +# Start an experiment -1. **What metric measures success?** *(Required)* e.g. conversion rate, page load time, bounce rate. Reserve your error-rate metric as a safety guardrail — never use it as the primary success metric. -2. **What's the target improvement?** *(Required)* e.g. 15% increase, 200ms decrease. -3. **What's the current baseline?** *(Optional — infer from production metrics if not given)* -4. **What parts of the app are in scope?** *(Optional — infer from the codebase if not given)* -5. **Any constraints?** *(Optional)* e.g. no changes to the payment flow. +## Step 1: Goal -Questions 1–2 are required before proceeding; infer 3–5 where possible and confirm your assumptions with the user before implementing. +Get answers to: -## Step 2: Hypothesis Generation +1. **What metric measures success?** *(Required)* For example, conversion rate, page load time, or bounce rate. Keep the error-rate metric as a safety guardrail; never use it as the primary metric. +2. **What is the target improvement?** *(Required)* For example, a 15% increase or a 200ms decrease. +3. **What is the baseline?** *(Optional: read it from the metric if the person does not know.)* +4. **What part of the product is in scope?** *(Required, because you cannot read the code.)* +5. **Which LaunchDarkly project and environment?** *(Required)* The environment is the one that the app's production traffic evaluates in. Confirm both; do not guess. +6. **Any constraints?** *(Optional)* For example, no changes to the payment flow. -Explore the target repository/codebase to find a plausible change: +## Step 2: Hypothesis -1. Search and read the relevant code paths. -2. Think through what UI/UX or logic change could plausibly move the chosen metric. -3. Check whether this hypothesis (or something close to it) has already been tried and failed — look at flag history or archived flags with similar naming. Avoid repeating a known failure. -4. Present the hypothesis to the user before proceeding, along with your reasoning and any inferred assumptions from Step 1. +You cannot read the code, so build the hypothesis with the person: -**Before finalizing a flag key, check for collisions:** look up any candidate flag key first. -- Already fully shipped (100% one variation, no split) → already decided, pick a different hypothesis. -- Actively running an experiment → mid-flight, don't compete with it, pick a different hypothesis. -- Doesn't exist → safe to create. +1. Ask what the product surface does today and what change could move the metric. +2. Use LaunchDarkly for context: existing flags and metrics, archived flags with similar keys, and the change history. Avoid a hypothesis that was already tried and failed. +3. Propose one hypothesis in this shape: "Changing **X** will improve **metric** for **audience**, because **reason**." Confirm it with the person before you continue. -## Step 3: Implementation +**Before you pick a flag key, check for collisions:** -1. Create a boolean feature flag, OFF by default in all environments, with the tag `experiment-orchestration` and a description that holds the goal and the hypothesis. Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. If the change is in a browser or mobile app, make the flag available to client-side SDKs (`clientSideAvailability.usingEnvironmentId`, or `usingMobileKey` for mobile); otherwise the app cannot read the flag and the experiment collects no data. -2. Create the experiment record (see "Experiment Record"). -3. Hand off implementation to your coding agent/tool of choice, with clear instructions to gate the change behind the exact flag key from step 1. -4. This step is asynchronous — check status periodically rather than looping tightly on it. -5. Once implementation completes, move to Step 4 with the resulting branch/PR. If it fails, report the error and stop. +- The key exists: pick a different key. If that flag tests the same idea, check its result first. + - Already shipped (100% one variation, no split): the idea is already decided, so pick a different hypothesis. + - An experiment is running on it: do not compete with it, so pick a different hypothesis. +- Does not exist: safe to create. -**If no coding agent is available:** report the flag key and the change to make, set the record to "Step 3: waiting for deployment", and stop. A person implements, reviews, and deploys the change. Continue at Step 6 (skipping Steps 4–5) only when one of these is true: +## Step 3: Flag and handoff -- A person says in this invocation that the change is deployed. Write their confirmation to the record. -- The record already holds a person's deployment confirmation. +1. Create a boolean flag, OFF in all environments, with the tag `experiment-orchestration` and `defaults` of `{"onVariation": 1, "offVariation": 1}`, so that it serves false whether it is on or off. A flag that is on and matches no rule serves its on variation, so without these defaults, turning it on would give every user the treatment before the experiment starts. Name it `exp--` (for example `exp-checkout-conversion-cta-color`), lowercase with hyphens, about 50 characters at most. If the change is in a browser or mobile app, make the flag available to client-side SDKs (`clientSideAvailability.usingEnvironmentId`, or `usingMobileKey` for mobile). Otherwise the app cannot read the flag, and the experiment collects no data. +2. Write the experiment record to the flag description, with the step "waiting for deployment". +3. Give the person the handoff: + - The flag key, and that the change must be behind it + - The change, in plain words + - What the code must do when the flag is off or cannot be evaluated: keep the current behavior + - The event that the primary metric needs, if the app does not send it yet +4. Stop. The person implements, reviews, and deploys the change, then comes back. -Until then, a resumed invocation reports "waiting for deployment" for this flag and changes nothing. Flag evaluations from the app show that the code evaluates the flag, but not that the treatment works, so they never replace a person's confirmation. +## Step 4: Start the experiment -## Step 4: Release Readiness +Start only after a person says that the change is deployed. -Run your standard release/risk review on the PR before merging. +**Design the experiment:** -- If it passes: merge the PR. -- If it fails: feed the review's specific feedback back into implementation and retry. Cap retries at a small fixed number (e.g. 3 attempts total) — if it still hasn't passed, stop and report the last failure to the user rather than retrying indefinitely. +1. Make sure that the metrics exist. You can create a metric only for an event that the app already sends. If the event is not sent yet, ask the person to add it and come back. +2. Create the experiment as a draft, on the flag's default rule (`"fallthrough"`), in the environment from the record: + - Only 10% of the rule's traffic takes part: treatments of control (false) at 5% and treatment (true) at 5%, with `notInExperimentVariationId` set to the false variation, so that the other 90% get control and are not analyzed. If the tool rejects allocations that do not sum to 100%, do not fall back to 50/50 of all traffic: tell the person, and ask them to set a 10% audience on the design page before they confirm. + - Your randomization unit (usually "user"). + - Exactly one primary metric: the success metric from Step 1. + - Guardrail metrics: always the error-rate metric, and a performance metric (for example p95 page load time) for a performance change. + - `analysisConfig.sequentialTestingEnabled` set to true, so that checking results at each check-in does not inflate false positives. +3. Show the person the design and its `designUrl`, check with them that 10% of traffic takes part, and wait for their confirmation. Do not start the experiment in the same reply. -*(If your environment genuinely has no review capability available — e.g., a fully unattended automation context — you can skip straight to merge, but treat that as a deliberate, narrow exception you call out explicitly, not a default. Skipping review removes your only gate against shipping broken code.)* +**After the person confirms the design:** -## Step 5: Deployment +4. Turn the flag on. Its defaults serve false, so no user gets the treatment yet. Starting an experiment needs the flag on. +5. Start the experiment. +6. Write the record once: the step "experiment running", and who confirmed the deployment, and when. Tell the person to check back in 2 weeks, or sooner for a guardrail check, and end. -Deployment typically won't fire automatically on merge if your workflow is manually-triggered (`workflow_dispatch`-only) — you'll need to trigger it explicitly. +# Check on experiments -1. Trigger the deploy workflow on the merge target branch. Treat "already an in-progress deployment for this ref" as expected de-duplication, not an error — don't re-trigger. -2. Poll for status, but let your polling tool's own internal long-poll do the waiting rather than looping tightly yourself. -3. Watch for a "stale" status specifically: if a deployment reports "running" for far longer than normal, cross-check the actual CI run history by commit SHA/timing before assuming it's still in progress — a background poll process may have died without updating the record. -4. **Trigger a deployment at most once per attempt.** If you're unsure whether a previous trigger succeeded, check status first — never re-trigger just because you're unsure. -5. On timeout: stop, check the CI run directly, report the situation, ask how to proceed. -6. On explicit failure: stop and report — do not proceed to the experiment. -7. On success: proceed immediately to Step 6. +## Step 5: Read and decide -## Step 6: Experiment Phase (fixed 10%) +For each project in the person's account (or the one they name), list the flags with the tag `experiment-orchestration` that are not archived. For each one, read the record and the current state in the environment from the record, then report where it stands. Act only where a decision is due. -Prove the change on a small, fixed slice of traffic. Do **not** start a guarded rollout here — that's Step 7, and only after this proves out. +If the person asks about an experiment on a flag without the tag, read it and report on it, but never change it. -1. Turn the flag ON. -2. Create an experiment on the flag's default rule (`"fallthrough"`) with: - - Audience allocation of 10% of the rule's traffic (a flat allocation, not a staged ramp) on your chosen randomization unit (typically "user"). The other 90% receive the control variation and are not analyzed. - - Treatments: control (off) at 50%, treatment (on) at 50%. Treatment allocations must sum to 100%; the 10% is the audience allocation, not the treatments. - - Exactly one primary metric: the success metric from Step 1. If it does not exist, create the metric and the event that feeds it first. - - Guardrail metric(s): always include your error-rate metric; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. - - Sequential testing, if your methodology offers it, so that checking results on every invocation does not inflate false positives. -3. Start the experiment/data collection. -4. Record the experiment key, the start date, and the guardrail baselines, and set the record to "Step 7: Phase 1 running". -5. Move to Step 7 to monitor toward a decision. +**Waiting for deployment:** report it. If the person says now that the change is deployed, go to Step 4. -## Step 7: Monitoring & Outcome +**Experiment running:** -Check status periodically — don't tight-loop. In an interactive session, check once and report progress, then pick back up later. In an unattended/scheduled context, check once per invocation and persist your progress somewhere durable between runs. +- **A guardrail regressed, at any time:** end without a win (below). +- **Less than 2 full weeks of data:** keep it running, even if the primary metric looks significant. Report progress only. +- **2 weeks or more, significant positive lift:** the experiment is proven. Stop the iteration with the treatment as the winner, then start the guarded rollout (below). +- **2 weeks or more, significant negative lift:** a loser. End without a win. +- **2 to 4 weeks, not significant:** keep it running. Report progress only. +- **No significance after 4 weeks:** inconclusive, need more traffic. End without a win. -**Phase 1 — Prove the experiment at 10% (gate before any rollout):** +Never declare a winner off a single data point, or before the stats engine confirms significance. -Run the experiment for at least 2 full weeks, so that weekday and weekend behavior are both in the data. Check results once per invocation (daily is enough), and decide on the primary metric: +**Ending without a win:** stop the iteration with control as the winner and a reason that says why, because stopping an iteration requires a winning treatment. Stopping does not change targeting, so then turn the flag off so that everyone gets control. Do not archive the flag, because the code still evaluates it. Go to Step 6. -- **Guardrail regression at any time** → stop early with control as the winner (see "Ending without a win" below). -- **Before 2 weeks** → keep running, even if the primary metric looks significant. Report progress only. -- **After 2 weeks, significant + positive lift** → experiment proven. Stop the iteration with the treatment as the winner and move to Phase 2. -- **After 2 weeks, significant + negative lift** → declare a loser and end without a win. -- **No significance after 4 weeks** → declare "inconclusive, need more traffic" and end without a win. +**Starting the guarded rollout:** -**Ending without a win:** stop the iteration with control as the winner and a reason that says why, because stopping an iteration requires a winning treatment. Stopping does not change targeting, so then turn the flag off so that everyone gets control. Record the stop date, skip Phase 2, and go to the Step 8 report. Don't archive the flag: the code still evaluates it. Recommend removing the flag from the code, and then archiving it, in the report's Next Steps. - Never declare a winner off a single data point or before your stats engine confirms significance. +- The winning ("on") variation as the test, the original as control, and the same randomization unit as the experiment. +- Exactly 3 monitored stages, none above 50% (for example 10% → 25% → 50%), with about 24 hours of monitoring each. LaunchDarkly rejects any step above 50%, and it promotes the rollout to 100% itself after the last stage, so do not add a 100% stage. +- The same primary and guardrail metrics as the experiment, each set to notify and roll back on regression. -**Phase 2 — Guarded rollout ramp (only after Phase 1 proves the change):** +Set the record to "guarded rollout running". LaunchDarkly advances the stages and rolls back on its own, so later check-ins only read its status. -Start a guarded rollout with: -- The winning ("on") variation as the test, the original as control. -- Same randomization unit as the experiment. -- **Exactly 3 monitored stages, none above 50%** — e.g. 10% → 25% → 50%, ~24 hours monitoring each. LaunchDarkly rejects any step above 50%, because each step needs an equal split between the original and new variations, and the rollout auto-promotes to 100% itself once the final monitored stage completes cleanly — no explicit 100% stage needed. -- The same primary + guardrail metrics as the experiment, each configured to notify and auto-rollback on regression. +**Guarded rollout running:** -LaunchDarkly advances the stages and rolls back on its own, so each invocation only reads the rollout status and updates the record. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, confirm that everyone gets control, and go to the Step 8 report. Recommend flag cleanup in Next Steps rather than archiving the flag yourself. +- **Still in a stage:** report the stage. +- **Rolled back or stopped:** the rollout failed. Confirm that everyone gets control, and go to Step 6. +- **Reached 100%:** a winner. Go to Step 6. -Once the final stage completes cleanly and auto-promotes to 100%, declare a winner and go to the Step 8 report. +**Retrying after a rollback:** a rollback can come from an unrelated application error during the ramp, not from the change. Retry only when the person asks, and only after these checks: -**Retrying after a rollback:** a rollback isn't always caused by your monitored metrics genuinely regressing — it can also be triggered by an unrelated application error surfacing mid-ramp. Before blindly restarting after the user says they've fixed something: -1. Confirm the flag's current state (should be back to 100% control, nothing stuck mid-rollout). -2. Check the change history timing between "advanced to next stage" and "reverted." A rollback within seconds of advancing is inconsistent with a full metric-window regression and points to an external cause instead. -3. If the flag is cleanly reverted and the external cause is confirmed fixed, it's safe to restart the guarded rollout from scratch with the same parameters. -4. Don't silently retry without this check, and don't refuse to retry just because a prior attempt rolled back — a genuinely fixed external cause is a legitimate reason to retry. A metric-driven loser is not — don't retry that. +1. The flag is back to 100% control, with nothing stuck mid-rollout. +2. The change history shows the timing between "advanced to next stage" and "reverted". A rollback within seconds of advancing does not fit a regression over a full metric window, so it points to an external cause. +3. The person confirms that the external cause is fixed. -**On any terminal outcome, immediately produce the Step 8 report in the same turn** — a one-line "it worked!" note is fine as a lead-in, but the structured report must follow, not wait for a follow-up request. +Then restart the guarded rollout from the start with the same settings. Never retry a rollback that the metrics caused. -## Step 8: Report +## Step 6: Report -Runs automatically the instant Step 7 reaches a terminal outcome (winner + auto-promoted to 100%; loser; inconclusive; or rollback/failure). Use this exact structure from Experiment Report to Next Steps: +When an experiment reaches an outcome (a winner at 100%, a loser, inconclusive, or a failed rollout), produce this report in the same reply, and set the record to "report produced": -## Experiment Report: [Goal Description] +## Experiment Report: [Goal] **Date:** [YYYY-MM-DD] **Goal:** [metric] [direction] by [target]% -**Status:** [achieved / in progress / stalled] +**Status:** [achieved / not achieved / inconclusive] ### Hypothesis [What we tried and why] -### Implementation -- Flag: [flag_key] -- Files modified: [list] -- Branch: [branch name] - -### Release Readiness -- [reviewed, passed after N attempt(s) / skipped, per your environment's process] +### Change +- Flag: [flag key] +- Change: [as described in the handoff] +- Deployed: [who confirmed, and when] -### Experiment Phase (10% fixed split) -- Status: [proven / loser / inconclusive] +### Experiment (10% of traffic) +- Status: [proven / loser / inconclusive / stopped for a guardrail] - Duration: [time] - Metric change: [before] → [after] ([+/-]%) - Statistical significance: [value, confidence interval] -### Guarded Rollout Phase (if reached) -- Status: [completed / rolled_back / not started] +### Guarded rollout (if reached) +- Status: [completed / rolled back / not started] - Duration: [time] -- Stages reached: [N of 3 monitored stages] -- If rolled back and retried: [root cause, outcome of retry] - -### Safety Metrics -- error-rate: [baseline] → [final] ([no regression / regression detected]) -- [other guardrails]: [baseline] → [final] ([status]) - -### Next Steps -[What to do next based on the outcome] - -## Safety Rules (the non-negotiables) - -- Always present the hypothesis before implementing. -- Always run a release/risk review before merging, unless your environment has a deliberate, explicitly-called-out exception. -- **Always prove a change via a fixed small-percentage experiment before starting any guarded rollout** — never ramp blind. -- Always include an error-rate (or equivalent "don't break prod") metric as a guardrail, separate from your success metric. -- Add a performance guardrail (e.g. p95 latency) for performance-focused changes. -- Every rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. -- **Run an experiment at least 2 full weeks** before deciding, unless a guardrail regresses. -- **Keep every guarded rollout stage at or below 50%** (LaunchDarkly rejects larger steps) and let the platform auto-promote to 100% after the final stage — don't try to add an explicit 100% stage. -- Update the experiment record on every step change, so that the next invocation can resume. -- Distinguish a metric-driven rollback (don't retry) from an external-cause rollback (safe to retry once fixed) before restarting a rolled-back rollout. -- The final report is automatic and mandatory on every terminal outcome — never defer it to a follow-up ask. +- Stages reached: [N of 3] +- If rolled back and retried: [cause, and the result of the retry] + +### Safety metrics +- Error rate: [baseline] → [final] ([no regression / regression]) +- [Other guardrails]: [baseline] → [final] ([status]) + +### Next steps +[What to do next. For an experiment that did not win, recommend removing the flag from the code, and then archiving it.] + +## Safety rules + +- Present the hypothesis, and get the person's agreement, before you create anything. +- Never treat a change as deployed without a person's confirmation. +- Always prove a change with a fixed 10% experiment before any guarded rollout. +- Run an experiment for at least 2 full weeks before you decide, unless a guardrail regresses. +- Always include an error-rate guardrail, separate from the success metric. Add a performance guardrail for a performance change. +- Set every rollout metric to both notify and roll back on regression. +- Keep every guarded rollout stage at or below 50%, and let LaunchDarkly promote to 100%. +- Tell a rollback that the metrics caused (do not retry) from one with an external cause (safe to retry once it is fixed). +- Change only flags with the tag `experiment-orchestration`, and keep the record in the flag description current. From 800dc4d656e27bbf2a74dcdfc4771060d7b17bde Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 16:29:11 -0700 Subject: [PATCH 07/34] fix: add missing MCP tools to an existing allowlist on request, and keep the win out of the 100% path Stopping an experiment iteration serves the chosen winner to everyone that the rule matches. The skill now stops a won experiment with control as the winner, records the win first, makes sure that the default rule serves control, and then ramps the treatment with the guarded rollout. A space that reuses an MCP server kept its old allowlist, so the tools that the skill now uses were missing. Setup now names the missing tools in the plan and warns. With --add-mcp-tools it adds them. It rewrites the allowlist only if every tool on it is an ldcli tool, because AWS replaces the whole configuration and does not send back each tool's classification, and an unclassified tool runs without approval. The run skips the rewrite if the allowlist changed after the plan. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/select_test.go | 4 + cmd/aws_devops_agent/setup.go | 15 ++ internal/awsdevops/allowlist_test.go | 162 ++++++++++++++++++ internal/awsdevops/awsdevops.go | 1 + internal/awsdevops/awsdevops_test.go | 15 +- internal/awsdevops/export_test.go | 8 +- internal/awsdevops/setup.go | 142 ++++++++++++++- internal/awsdevops/skill.go | 2 +- .../skills/experiment-orchestration/SKILL.md | 10 +- 9 files changed, 347 insertions(+), 12 deletions(-) create mode 100644 internal/awsdevops/allowlist_test.go diff --git a/cmd/aws_devops_agent/select_test.go b/cmd/aws_devops_agent/select_test.go index 40b879468..87484e9e3 100644 --- a/cmd/aws_devops_agent/select_test.go +++ b/cmd/aws_devops_agent/select_test.go @@ -109,6 +109,10 @@ func (s stubAgent) ListAssociations(context.Context, *devopsagent.ListAssociatio return &devopsagent.ListAssociationsOutput{Associations: s.associations}, nil } +func (s stubAgent) UpdateAssociation(context.Context, *devopsagent.UpdateAssociationInput, ...func(*devopsagent.Options)) (*devopsagent.UpdateAssociationOutput, error) { + return &devopsagent.UpdateAssociationOutput{}, nil +} + func (s stubAgent) ListServices(context.Context, *devopsagent.ListServicesInput, ...func(*devopsagent.Options)) (*devopsagent.ListServicesOutput, error) { return &devopsagent.ListServicesOutput{Services: s.services}, nil } diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 82bc832e7..4c7bb5454 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -28,6 +28,7 @@ const ( mcpServiceIDFlag = "mcp-service-id" mcpEndpointFlag = "mcp-endpoint" replaceSkillFlag = "replace-skill" + addMCPToolsFlag = "add-mcp-tools" yesFlag = "yes" ) @@ -57,6 +58,11 @@ If it has none, setup asks for a service token in a terminal and checks it before it changes AWS. Without a terminal, or with --yes, it stops and asks for --access-token. +A space that reuses a server keeps its tool allowlist. If the allowlist misses +tools that this ldcli version allows, setup warns. To add them, pass +--add-mcp-tools. AWS does not send back the classification of each tool, so +the rewrite gives every tool the ldcli classification. + The experiment-orchestration skill tells the agent how to run an experiment from a goal to a report. Every agent type in the space can load it. If the skill has the text of an older ldcli version, setup updates it. If someone @@ -83,6 +89,8 @@ does the other steps.`, cmd.Flags().String(mcpServiceIDFlag, "", "Registered MCP server to connect, instead of the server of the agent space") cmd.Flags().String(mcpEndpointFlag, "", "MCP endpoint to register. Setup reuses only a server at this endpoint. Defaults to "+awsdevops.MCPServerEndpoint) cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") + cmd.Flags().Bool(addMCPToolsFlag, false, "Add the ldcli tools that the MCP server allowlist of an existing agent space misses. "+ + "Every tool gets the ldcli classification") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -174,6 +182,9 @@ func setupWith( if !plaintext { // stdout holds only the JSON, so a CI log shows the warning on stderr. + if result.MCPToolsError != "" { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.MCPToolsError) + } if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.SkillError) } @@ -188,6 +199,9 @@ func setupWith( awsdevops.AccessTokenURL(opts.LDBaseURI)) } + if result.MCPToolsError != "" { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.MCPToolsError) + } if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.SkillError) } @@ -702,6 +716,7 @@ func setupOptions(cmd *cobra.Command) (awsdevops.SetupOptions, error) { MCPServiceID: mustString(cmd, mcpServiceIDFlag), MCPEndpoint: mustString(cmd, mcpEndpointFlag), ReplaceSkill: mustBool(cmd, replaceSkillFlag), + AddMCPTools: mustBool(cmd, addMCPToolsFlag), } idcSet := opts.IdcInstanceARN != "" diff --git a/internal/awsdevops/allowlist_test.go b/internal/awsdevops/allowlist_test.go new file mode 100644 index 000000000..1b34d5278 --- /dev/null +++ b/internal/awsdevops/allowlist_test.go @@ -0,0 +1,162 @@ +package awsdevops_test + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) + +// mcpAssociation is the association of server mcp-1 with space-2 that allows +// these tools. +func mcpAssociation(tools ...string) agenttypes.Association { + return agenttypes.Association{ + AgentSpaceId: aws.String("space-2"), + AssociationId: aws.String("assoc-mcp-1"), + ServiceId: aws.String("mcp-1"), + Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{ + Value: agenttypes.MCPServerConfiguration{Tools: tools}, + }, + Capabilities: map[string]agenttypes.CapabilityConfiguration{"kept": {}}, + } +} + +func reuseServerOptions() awsdevops.SetupOptions { + opts := reuseOptions() + opts.LDAccessToken = "" + opts.AddMCPTools = true + + return opts +} + +// An older ldcli allowed fewer tools. A re-run adds the new ones, so the skill +// can use them. +func TestSetupAddsMissingToolsToAnExistingAllowlist(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation("list-projects", "get-flag", "toggle-flag")}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseServerOptions(), true) + require.NoError(t, err) + require.Equal(t, awsdevops.MCPReuse, plan.MCP.Action) + assert.Contains(t, plan.Text().MCPServer, "and add get-project, get-environment, list-flags,") + assert.Contains(t, plan.Text().MCPServer, "update-flag-settings") + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseServerOptions(), plan) + require.NoError(t, err) + + require.Len(t, agent.updateAssociations, 1) + in := agent.updateAssociations[0] + assert.Equal(t, "assoc-mcp-1", aws.ToString(in.AssociationId)) + config, ok := in.Configuration.(*agenttypes.ServiceConfigurationMemberMcpserver) + require.True(t, ok) + assert.Contains(t, config.Value.Tools, "update-flag-settings") + assert.Contains(t, config.Value.ToolDetails, agenttypes.MCPToolDetail{ + Name: aws.String("update-flag-settings"), + ToolClassification: agenttypes.ToolClassificationMutative, + }) + // AWS replaces the whole association, so setup sends the capabilities back. + assert.Contains(t, in.Capabilities, "kept") + assert.Contains(t, result.MCPToolsAdded, "create-metric") + assert.NotContains(t, result.MCPToolsAdded, "get-flag") + assert.Empty(t, result.MCPToolsError) +} + +func TestSetupKeepsACompleteAllowlist(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation(awsdevops.AllowedTools()...)}, + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseServerOptions()) + require.NoError(t, err) + + assert.Empty(t, agent.updateAssociations) + assert.Empty(t, result.MCPToolsAdded) +} + +// AWS does not send the classification of each tool back. Rewriting an +// allowlist with a tool that ldcli does not know would make that tool +// read-only, so setup warns instead. +func TestSetupLeavesAnAllowlistWithForeignTools(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation("get-flag", "delete-everything")}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseServerOptions(), true) + require.NoError(t, err) + assert.Contains(t, plan.Text().MCPServer, "which setup cannot add") + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseServerOptions(), plan) + require.NoError(t, err) + + assert.Empty(t, agent.updateAssociations) + assert.Contains(t, result.MCPToolsError, "Add the missing tools in the AWS console") +} + +// Setup cannot tell an older ldcli allowlist from one that someone trimmed, +// so without --add-mcp-tools it only warns. +func TestSetupWarnsAboutMissingToolsWithoutTheFlag(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation("get-flag")}, + } + clients := newTestClients(agent, newFakeIAM()) + opts := reuseServerOptions() + opts.AddMCPTools = false + + plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) + require.NoError(t, err) + assert.Contains(t, plan.Text().MCPServer, "Pass --add-mcp-tools to add them") + + result, err := awsdevops.RunPlan(context.Background(), clients, opts, plan) + require.NoError(t, err) + + assert.Empty(t, agent.updateAssociations) + assert.Contains(t, result.MCPToolsError, "--add-mcp-tools") +} + +// The run makes only the change that the plan showed. +func TestSetupSkipsAnAllowlistThatChangedAfterThePlan(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation("get-flag", "delete-everything")}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseServerOptions(), true) + require.NoError(t, err) + agent.associations = []agenttypes.Association{mcpAssociation("get-flag")} + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseServerOptions(), plan) + require.NoError(t, err) + + assert.Empty(t, agent.updateAssociations) + assert.Contains(t, result.MCPToolsError, "changed after the plan") +} + +// The server works with the tools that it has, so a failed update does not +// fail setup. +func TestSetupWarnsWhenTheAllowlistUpdateFails(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation("get-flag")}, + updateAssociationErr: errors.New("AccessDeniedException: not allowed"), + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseServerOptions()) + require.NoError(t, err) + + assert.Contains(t, result.MCPToolsError, "unable to add") + assert.Empty(t, result.MCPToolsAdded) +} diff --git a/internal/awsdevops/awsdevops.go b/internal/awsdevops/awsdevops.go index 2afca795d..83061513f 100644 --- a/internal/awsdevops/awsdevops.go +++ b/internal/awsdevops/awsdevops.go @@ -65,6 +65,7 @@ type AgentAPI interface { ListServices(context.Context, *devopsagent.ListServicesInput, ...func(*devopsagent.Options)) (*devopsagent.ListServicesOutput, error) RegisterService(context.Context, *devopsagent.RegisterServiceInput, ...func(*devopsagent.Options)) (*devopsagent.RegisterServiceOutput, error) UpdateAsset(context.Context, *devopsagent.UpdateAssetInput, ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) + UpdateAssociation(context.Context, *devopsagent.UpdateAssociationInput, ...func(*devopsagent.Options)) (*devopsagent.UpdateAssociationOutput, error) } // IAMAPI is the subset of the IAM API this package uses. diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index ebe4195e6..b59a018ec 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -77,7 +77,10 @@ type fakeAgent struct { onGetFile func() createAssetErr error createAssets []*devopsagent.CreateAssetInput - updateAssets []*devopsagent.UpdateAssetInput + // updateAssociations holds the allowlist updates. + updateAssociations []*devopsagent.UpdateAssociationInput + updateAssociationErr error + updateAssets []*devopsagent.UpdateAssetInput } // skillAsset is a skill asset named name. @@ -302,6 +305,16 @@ func (f *fakeAgent) ListAssociations(_ context.Context, in *devopsagent.ListAsso return &devopsagent.ListAssociationsOutput{Associations: items, NextToken: next}, nil } +func (f *fakeAgent) UpdateAssociation(_ context.Context, in *devopsagent.UpdateAssociationInput, _ ...func(*devopsagent.Options)) (*devopsagent.UpdateAssociationOutput, error) { + f.calls = append(f.calls, "UpdateAssociation") + f.updateAssociations = append(f.updateAssociations, in) + if f.updateAssociationErr != nil { + return nil, f.updateAssociationErr + } + + return &devopsagent.UpdateAssociationOutput{}, nil +} + func (f *fakeAgent) RegisterService(_ context.Context, in *devopsagent.RegisterServiceInput, _ ...func(*devopsagent.Options)) (*devopsagent.RegisterServiceOutput, error) { f.calls = append(f.calls, "RegisterService") f.registerInputs = append(f.registerInputs, in) diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 9ee486db2..1526e3aac 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -1,6 +1,9 @@ package awsdevops -import "context" +import ( + "context" + "slices" +) // Setup plans a full setup and runs it, for tests. func Setup(ctx context.Context, clients Clients, opts SetupOptions) (SetupResult, error) { @@ -35,3 +38,6 @@ func ShippedSkillHashes() []string { return shippedSkillHashes } // SkillHash is the hash that setup compares. func SkillHash(text string) string { return skillHash(text) } + +// AllowedTools are the tools that setup allows on the MCP server. +func AllowedTools() []string { return slices.Concat(mcpReadOnlyTools, mcpMutativeTools) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 95aafcf51..c21ec95cf 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -124,6 +124,10 @@ type SetupOptions struct { // experiment-orchestration skill that someone edited. Setup updates a // SKILL.md from an older ldcli version without this option. ReplaceSkill bool + // AddMCPTools adds the ldcli tools that the allowlist of an existing MCP + // server association misses. AWS does not send back the classification of + // each tool, so the rewrite gives every tool the ldcli classification. + AddMCPTools bool // Logf, when set, reports progress as each step completes. Logf func(format string, args ...any) @@ -142,7 +146,13 @@ type SetupResult struct { OperatorAppEnabled bool `json:"operatorAppEnabled"` MCPServiceID string `json:"mcpServiceId,omitempty"` MCPAssociationID string `json:"mcpAssociationId,omitempty"` - SkillAssetID string `json:"skillAssetId,omitempty"` + // MCPToolsAdded lists the ldcli tools that setup added to the allowlist of + // an existing MCP server association. + MCPToolsAdded []string `json:"mcpToolsAdded,omitempty"` + // MCPToolsError is set if the allowlist of an existing association misses + // ldcli tools that setup could not add. The rest of the setup ran. + MCPToolsError string `json:"mcpToolsError,omitempty"` + SkillAssetID string `json:"skillAssetId,omitempty"` // SkillAction is what the skill step did, for example "kept" or "left". SkillAction string `json:"skillAction,omitempty"` // SkillError is set if the skill step failed or was skipped. The skill is @@ -376,6 +386,13 @@ type MCPStep struct { // space uses. A replace on a shared server removes it first, so that the // new server can have the name. An earlier run that failed leaves one. Unused string + // MissingTools are the ldcli tools that the allowlist of the existing + // association in the space misses. CanAddTools says whether setup can add + // them. + MissingTools []string + CanAddTools bool + // AddTools is set by --add-mcp-tools. + AddTools bool } // shared reports whether other spaces use the server. @@ -510,6 +527,17 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio return plan, err } + if plan.MCP.Action == MCPReuse && plan.Space.Action != SpaceCreate { + association, err := findMCPAssociation(ctx, clients, plan.Space.ID, plan.MCP.ServiceID) + if err != nil { + return plan, err + } + if association != nil { + plan.MCP.MissingTools, plan.MCP.CanAddTools = missingTools(*association) + plan.MCP.AddTools = opts.AddMCPTools + } + } + plan.Skill = planSkill(ctx, clients, opts, plan.Space) if !provision { @@ -803,6 +831,18 @@ func (m MCPStep) describe() string { strings.Join(m.otherSpaces(), ", "), ) } + switch { + case len(m.MissingTools) == 0: + case m.CanAddTools && m.AddTools: + line += ", and add " + strings.Join(m.MissingTools, ", ") + + " to its allowlist. The rewrite gives each tool the ldcli classification" + case m.CanAddTools: + line += ". Its allowlist misses " + strings.Join(m.MissingTools, ", ") + + ". Pass --add-mcp-tools to add them" + default: + line += ". Its allowlist misses " + strings.Join(m.MissingTools, ", ") + + ", which setup cannot add because the allowlist has tools that ldcli does not classify" + } return line case MCPRegister: @@ -1213,7 +1253,7 @@ func registerMCPServer( logf("Reused MCP server %s (%s) with its current token", step.name(), existing) logf("Pass --access-token and --replace-mcp-token to change the token") - return associateMCPServer(ctx, clients, result, logf) + return associateMCPServer(ctx, clients, opts, step, result, logf) } // A run after a failure finds an unconnected server by this name. name := MCPServerNameFor(result.AgentSpaceID) @@ -1302,7 +1342,7 @@ func registerMCPServer( } } - return associateMCPServer(ctx, clients, result, logf) + return associateMCPServer(ctx, clients, opts, step, result, logf) } // leaveSharedServer disconnects a shared server from this space only, after @@ -1453,16 +1493,19 @@ func disassociateEverywhere( func associateMCPServer( ctx context.Context, clients Clients, + opts SetupOptions, + step MCPStep, result *SetupResult, logf func(string, ...any), ) error { - existing, err := FindAssociation(ctx, clients, result.AgentSpaceID, result.MCPServiceID) + existing, err := findMCPAssociation(ctx, clients, result.AgentSpaceID, result.MCPServiceID) if err != nil { return err } - if existing != "" { - result.MCPAssociationID = existing + if existing != nil { + result.MCPAssociationID = aws.ToString(existing.AssociationId) logf("MCP server %s is already associated with the agent space", MCPServerName) + extendAllowlist(ctx, clients, opts, step, *existing, result, logf) return nil } @@ -1477,6 +1520,93 @@ func associateMCPServer( return nil } +func findMCPAssociation(ctx context.Context, clients Clients, agentSpaceID, serviceID string) (*agenttypes.Association, error) { + associations, err := listAssociations(ctx, clients, agentSpaceID) + if err != nil { + return nil, err + } + for _, association := range associations { + if aws.ToString(association.ServiceId) == serviceID { + return &association, nil + } + } + + return nil, nil +} + +// missingTools returns the ldcli tools that the allowlist of an association +// misses, and whether setup can add them. AWS replaces the whole configuration +// on an update and does not send the classification of each tool back, so +// setup can rewrite the allowlist only if it holds ldcli tools alone. An +// unclassified tool counts as read-only, so a rewrite would let a mutative +// tool that the customer added run without approval. +func missingTools(association agenttypes.Association) (missing []string, canAdd bool) { + if _, ok := association.Configuration.(*agenttypes.ServiceConfigurationMemberMcpserver); !ok { + // Not an allowlist that setup knows how to read, so leave it. + return nil, false + } + allowed := associationTools(association.Configuration) + ldcli := slices.Concat(mcpReadOnlyTools, mcpMutativeTools) + for _, tool := range ldcli { + if !slices.Contains(allowed, tool) { + missing = append(missing, tool) + } + } + canAdd = !slices.ContainsFunc(allowed, func(tool string) bool { return !slices.Contains(ldcli, tool) }) + + return missing, canAdd +} + +// extendAllowlist adds the ldcli tools that an existing association misses, +// for example after an ldcli upgrade that allows more tools. Setup cannot tell +// an older ldcli allowlist from one that someone trimmed, so it adds tools only +// with --add-mcp-tools. A failure does not stop setup, because the server works +// with the tools that it has. +func extendAllowlist( + ctx context.Context, + clients Clients, + opts SetupOptions, + step MCPStep, + association agenttypes.Association, + result *SetupResult, + logf func(string, ...any), +) { + missing, canAdd := missingTools(association) + tools := strings.Join(missing, ", ") + switch { + case len(missing) == 0: + return + case !canAdd: + result.MCPToolsError = fmt.Sprintf("the MCP server allowlist of the agent space misses %s, and it has tools "+ + "that ldcli does not classify, so setup did not change it. Add the missing tools in the AWS console", tools) + + return + case !opts.AddMCPTools: + result.MCPToolsError = fmt.Sprintf("the MCP server allowlist of the agent space misses %s, which the "+ + "experiment-orchestration skill uses. To add them, run setup again with --add-mcp-tools", tools) + + return + case !slices.Equal(missing, step.MissingTools) || canAdd != step.CanAddTools: + // The plan showed another change, so do not make this one. + result.MCPToolsError = "the MCP server allowlist changed after the plan, so setup did not change it. Run setup again" + + return + } + _, err := clients.Agent.UpdateAssociation(ctx, &devopsagent.UpdateAssociationInput{ + AgentSpaceId: association.AgentSpaceId, + AssociationId: association.AssociationId, + Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{Value: mcpServerConfiguration()}, + Capabilities: association.Capabilities, + }) + if err != nil { + result.MCPToolsError = fmt.Sprintf("unable to add %s to the MCP server allowlist: %v", strings.Join(missing, ", "), err) + + return + } + result.MCPToolsAdded = missing + logf("Added %s to the MCP server allowlist", strings.Join(missing, ", ")) +} + func associateMCPServerWith(ctx context.Context, clients Clients, agentSpaceID, serviceID string) (string, error) { association, err := clients.Agent.AssociateService(ctx, &devopsagent.AssociateServiceInput{ AgentSpaceId: aws.String(agentSpaceID), diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 0028c62df..dd223e1cf 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -33,7 +33,7 @@ var ( // updates it without --replace-skill. When you change SKILL.md, add the hash // of the new text. Never remove a hash. var shippedSkillHashes = []string{ - "6972a3b18f2f8a52f9be2edc9157551f1bf5a7547026926b8eef9a348200d70e", + "53b05092fd7f9ecca7027c60699c88162f1f4410430ea27afa0dc18205dd894c", } const ( diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md index 16ee2a62d..761ebddee 100644 --- a/internal/awsdevops/skills/experiment-orchestration/SKILL.md +++ b/internal/awsdevops/skills/experiment-orchestration/SKILL.md @@ -31,7 +31,7 @@ The next chat may be with another person, and you have no memory of this one. Ke - Project and environment keys - Goal: metric, direction, target, and baseline - Hypothesis, and the change that the person implements -- Current step, for example "waiting for deployment", "experiment running", "guarded rollout running", or "report produced" +- Current step, for example "waiting for deployment", "experiment running", "won, guarded rollout pending", "guarded rollout running", or "report produced" - Deployment: who confirmed that the change is deployed, and when Read everything else (the experiment key, dates, results, and rollout stages) from LaunchDarkly. @@ -110,23 +110,27 @@ If the person asks about an experiment on a flag without the tag, read it and re - **A guardrail regressed, at any time:** end without a win (below). - **Less than 2 full weeks of data:** keep it running, even if the primary metric looks significant. Report progress only. -- **2 weeks or more, significant positive lift:** the experiment is proven. Stop the iteration with the treatment as the winner, then start the guarded rollout (below). +- **2 weeks or more, significant positive lift:** the experiment is proven. First set the record to "won, guarded rollout pending", so that a later check-in can tell the win apart from a loss. Then stop the iteration with control as the winner, and a reason that says that the treatment won, with its lift. Stopping serves the winner to everyone that the default rule matches, so a treatment winner would reach 100% of users at once and skip the guarded rollout. Then start the guarded rollout (below), which ramps the treatment. - **2 weeks or more, significant negative lift:** a loser. End without a win. - **2 to 4 weeks, not significant:** keep it running. Report progress only. - **No significance after 4 weeks:** inconclusive, need more traffic. End without a win. Never declare a winner off a single data point, or before the stats engine confirms significance. -**Ending without a win:** stop the iteration with control as the winner and a reason that says why, because stopping an iteration requires a winning treatment. Stopping does not change targeting, so then turn the flag off so that everyone gets control. Do not archive the flag, because the code still evaluates it. Go to Step 6. +**Ending without a win:** stop the iteration with control as the winner and a reason that says why, because stopping an iteration requires a winning treatment. Stopping serves control to everyone that the default rule matches. Then turn the flag off, so that its state shows that it is not in use. Do not archive the flag, because the code still evaluates it. Go to Step 6. **Starting the guarded rollout:** +Before you start it, read the flag's default rule. It must serve control (false) to everyone. If it still has the experiment's split, set it to serve control first. + - The winning ("on") variation as the test, the original as control, and the same randomization unit as the experiment. - Exactly 3 monitored stages, none above 50% (for example 10% → 25% → 50%), with about 24 hours of monitoring each. LaunchDarkly rejects any step above 50%, and it promotes the rollout to 100% itself after the last stage, so do not add a 100% stage. - The same primary and guardrail metrics as the experiment, each set to notify and roll back on regression. Set the record to "guarded rollout running". LaunchDarkly advances the stages and rolls back on its own, so later check-ins only read its status. +**Won, guarded rollout pending:** the iteration is stopped and the treatment won. Start the guarded rollout (above). + **Guarded rollout running:** - **Still in a stage:** report the stage. From 3b9332040586af7801afc451649c0e11198d3125 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Tue, 27 Oct 2026 09:52:37 -0700 Subject: [PATCH 08/34] fix: address review feedback on the experiment skill - The report template is in a fenced markdown block, so its headings stay out of the skill's outline. - The report compares control with treatment, not before with after. - "Design waiting for confirmation" is a record state, so a later chat resumes the draft experiment instead of making another one. - The skill says why the 2-week minimum holds with sequential testing on: sequential testing lets a guardrail regression end the experiment early, and the minimum covers weekly cycles and novelty effects. - The report recommends a flag cleanup for a winner too. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/skill.go | 2 +- .../skills/experiment-orchestration/SKILL.md | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index dd223e1cf..1d4ca7ad1 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -33,7 +33,7 @@ var ( // updates it without --replace-skill. When you change SKILL.md, add the hash // of the new text. Never remove a hash. var shippedSkillHashes = []string{ - "53b05092fd7f9ecca7027c60699c88162f1f4410430ea27afa0dc18205dd894c", + "57844bc3c5aca292ef88b5f949cd791206074b5cfc850ea52ebdd40020ead695", } const ( diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md index 761ebddee..98390cf77 100644 --- a/internal/awsdevops/skills/experiment-orchestration/SKILL.md +++ b/internal/awsdevops/skills/experiment-orchestration/SKILL.md @@ -31,7 +31,7 @@ The next chat may be with another person, and you have no memory of this one. Ke - Project and environment keys - Goal: metric, direction, target, and baseline - Hypothesis, and the change that the person implements -- Current step, for example "waiting for deployment", "experiment running", "won, guarded rollout pending", "guarded rollout running", or "report produced" +- Current step, for example "waiting for deployment", "design waiting for confirmation", "experiment running", "won, guarded rollout pending", "guarded rollout running", or "report produced" - Deployment: who confirmed that the change is deployed, and when Read everything else (the experiment key, dates, results, and rollout stages) from LaunchDarkly. @@ -88,13 +88,14 @@ Start only after a person says that the change is deployed. - Exactly one primary metric: the success metric from Step 1. - Guardrail metrics: always the error-rate metric, and a performance metric (for example p95 page load time) for a performance change. - `analysisConfig.sequentialTestingEnabled` set to true, so that checking results at each check-in does not inflate false positives. -3. Show the person the design and its `designUrl`, check with them that 10% of traffic takes part, and wait for their confirmation. Do not start the experiment in the same reply. +3. Write the record: the step "design waiting for confirmation", the experiment key, and who confirmed the deployment, and when. A later chat then resumes the draft instead of making another one. +4. Show the person the design and its `designUrl`, check with them that 10% of traffic takes part, and wait for their confirmation. Do not start the experiment in the same reply. **After the person confirms the design:** -4. Turn the flag on. Its defaults serve false, so no user gets the treatment yet. Starting an experiment needs the flag on. -5. Start the experiment. -6. Write the record once: the step "experiment running", and who confirmed the deployment, and when. Tell the person to check back in 2 weeks, or sooner for a guardrail check, and end. +5. Turn the flag on. Its defaults serve false, so no user gets the treatment yet. Starting an experiment needs the flag on. +6. Start the experiment. +7. Set the record to "experiment running". Tell the person to check back in 2 weeks, or sooner for a guardrail check, and end. # Check on experiments @@ -106,6 +107,8 @@ If the person asks about an experiment on a flag without the tag, read it and re **Waiting for deployment:** report it. If the person says now that the change is deployed, go to Step 4. +**Design waiting for confirmation:** the draft experiment from the record exists but has not started. Show its `designUrl` again and ask the person to confirm it, then continue at item 5 of Step 4. Do not create another draft. + **Experiment running:** - **A guardrail regressed, at any time:** end without a win (below). @@ -117,6 +120,8 @@ If the person asks about an experiment on a flag without the tag, read it and re Never declare a winner off a single data point, or before the stats engine confirms significance. +Sequential testing keeps each check-in valid, which is what lets a guardrail regression end the experiment early. The 2-week minimum for a win is about the product, not the statistics: behavior differs between weekdays and weekends, and a new experience can get a short novelty bump. A win needs at least two full weekly cycles. + **Ending without a win:** stop the iteration with control as the winner and a reason that says why, because stopping an iteration requires a winning treatment. Stopping serves control to everyone that the default rule matches. Then turn the flag off, so that its state shows that it is not in use. Do not archive the flag, because the code still evaluates it. Go to Step 6. **Starting the guarded rollout:** @@ -149,6 +154,7 @@ Then restart the guarded rollout from the start with the same settings. Never re When an experiment reaches an outcome (a winner at 100%, a loser, inconclusive, or a failed rollout), produce this report in the same reply, and set the record to "report produced": +```markdown ## Experiment Report: [Goal] **Date:** [YYYY-MM-DD] @@ -166,7 +172,7 @@ When an experiment reaches an outcome (a winner at 100%, a loser, inconclusive, ### Experiment (10% of traffic) - Status: [proven / loser / inconclusive / stopped for a guardrail] - Duration: [time] -- Metric change: [before] → [after] ([+/-]%) +- Metric change: [control] → [treatment] ([+/-]%) - Statistical significance: [value, confidence interval] ### Guarded rollout (if reached) @@ -180,7 +186,8 @@ When an experiment reaches an outcome (a winner at 100%, a loser, inconclusive, - [Other guardrails]: [baseline] → [final] ([status]) ### Next steps -[What to do next. For an experiment that did not win, recommend removing the flag from the code, and then archiving it.] +[What to do next. For a winner at 100%, recommend a cleanup change that removes the flag from the code and keeps the winning path, and then archiving the flag. For an experiment that did not win, recommend removing the flag from the code and keeping the current behavior, and then archiving the flag.] +``` ## Safety rules From 087d39e68fd4ad4244d5ce2ab1f82e0aa2aadf86 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 15:23:06 -0700 Subject: [PATCH 09/34] feat: add the experiment-orchestrator custom agent to the agent space in setup Setup now creates a custom agent that runs the experiment-orchestration skill with the LaunchDarkly MCP tools that the space allows. The tool IDs are "_", the format that AWS uses for a custom agent made in the console. An update adds the ldcli tools and skill to the agent and keeps the ones that the customer added. If someone edited the prompt, setup leaves the agent unless --replace-agent is passed. The agent is optional: if its step fails, or the MCP server or the skill is missing, setup warns and does the other steps. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/plan_print_test.go | 1 + cmd/aws_devops_agent/setup.go | 21 +- internal/awsdevops/agent.go | 369 ++++++++++++++++++ internal/awsdevops/agent_test.go | 272 +++++++++++++ .../agents/experiment-orchestrator.md | 39 ++ internal/awsdevops/awsdevops_test.go | 64 ++- internal/awsdevops/export_test.go | 6 + internal/awsdevops/setup.go | 31 +- internal/awsdevops/skill.go | 18 +- 9 files changed, 803 insertions(+), 18 deletions(-) create mode 100644 internal/awsdevops/agent.go create mode 100644 internal/awsdevops/agent_test.go create mode 100644 internal/awsdevops/agents/experiment-orchestrator.md diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index b03b87140..f9d7091c7 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -66,6 +66,7 @@ func TestSetupPlanNamesRolesAndOperatorApp(t *testing.T) { assert.Contains(t, out.String(), awsdevops.OperatorAppRoleName) assert.Contains(t, out.String(), "operator app") assert.Contains(t, out.String(), "Skill: Create skill "+awsdevops.ExperimentSkillName) + assert.Contains(t, out.String(), "Custom agent: Create custom agent "+awsdevops.ExperimentAgentName) } // A canceled plan does not create the skill. diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 4c7bb5454..6c98d8fdc 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -29,6 +29,7 @@ const ( mcpEndpointFlag = "mcp-endpoint" replaceSkillFlag = "replace-skill" addMCPToolsFlag = "add-mcp-tools" + replaceAgentFlag = "replace-agent" yesFlag = "yes" ) @@ -68,7 +69,14 @@ from a goal to a report. Every agent type in the space can load it. If the skill has the text of an older ldcli version, setup updates it. If someone edited it, setup does not change it. To overwrite the edits, pass --replace-skill. The skill is optional: if its step fails, setup warns and -does the other steps.`, +does the other steps. + +The experiment-orchestrator custom agent runs the skill with the LaunchDarkly +MCP tools that the space allows. Setup updates an agent with the prompt of an +older ldcli version. If someone edited the prompt, setup does not change the +agent. To overwrite the edits, pass --replace-agent. The agent is optional and +needs the MCP server and the skill: if its step fails, setup warns and does +the other steps.`, Args: cobra.NoArgs, PreRun: trackRun(analyticsTrackerFn), RunE: runSetup, @@ -91,6 +99,7 @@ does the other steps.`, cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") cmd.Flags().Bool(addMCPToolsFlag, false, "Add the ldcli tools that the MCP server allowlist of an existing agent space misses. "+ "Every tool gets the ldcli classification") + cmd.Flags().Bool(replaceAgentFlag, false, "Overwrite the "+awsdevops.ExperimentAgentName+" custom agent if someone edited it") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -188,6 +197,9 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.SkillError) } + if result.AgentError != "" { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.AgentError) + } return printJSON(cmd, result) } @@ -205,6 +217,9 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.SkillError) } + if result.AgentError != "" { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.AgentError) + } // Only a connect run leaves the app off, on a space that setup did not // make. The full setup would change that space, so point to the console. @@ -529,7 +544,8 @@ func confirmSetup(cmd *cobra.Command, plan awsdevops.SetupPlan, ask bool) error _, _ = fmt.Fprintf(out, " Association: %s\n", text.Association) _, _ = fmt.Fprintf(out, " Operator app: %s\n", text.OperatorApp) _, _ = fmt.Fprintf(out, " MCP server: %s\n", text.MCPServer) - _, _ = fmt.Fprintf(out, " Skill: %s\n\n", text.Skill) + _, _ = fmt.Fprintf(out, " Skill: %s\n", text.Skill) + _, _ = fmt.Fprintf(out, " Custom agent: %s\n\n", text.Agent) if !ask { return nil } @@ -717,6 +733,7 @@ func setupOptions(cmd *cobra.Command) (awsdevops.SetupOptions, error) { MCPEndpoint: mustString(cmd, mcpEndpointFlag), ReplaceSkill: mustBool(cmd, replaceSkillFlag), AddMCPTools: mustBool(cmd, addMCPToolsFlag), + ReplaceAgent: mustBool(cmd, replaceAgentFlag), } idcSet := opts.IdcInstanceARN != "" diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go new file mode 100644 index 000000000..e91949468 --- /dev/null +++ b/internal/awsdevops/agent.go @@ -0,0 +1,369 @@ +package awsdevops + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/devopsagent" + "github.com/aws/aws-sdk-go-v2/service/devopsagent/document" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" +) + +// ExperimentAgentPrompt is the system prompt of the custom agent that runs the +// experimentation skill. +// +//go:embed agents/experiment-orchestrator.md +var ExperimentAgentPrompt string + +// ExperimentAgentName is the name of the custom agent. AWS keeps custom agent +// names unique in a space. +const ExperimentAgentName = "experiment-orchestrator" + +// shippedAgentHashes holds the SHA-256 of each system prompt that an ldcli +// release sent. An agent with one of these prompts has no customer edits, so +// setup updates it without --replace-agent. When you change the prompt, add +// the hash of the new text. Never remove a hash. +var shippedAgentHashes = []string{ + "997dd932b26120de3ccfa19a8f0bf53c02b23d58f33a0ac9e40c188623e41f18", +} + +const ( + agentAssetType = "custom_agent" + // agentFilePath is where the console keeps the system prompt of a custom + // agent. + agentFilePath = ExperimentAgentName + ".md" +) + +// agentTools are the IDs of the MCP tools that the agent can use. AWS names a +// tool of an MCP server "_". +func agentTools(serverName string) []string { + return toolIDs(serverName, slices.Concat(mcpReadOnlyTools, mcpMutativeTools)) +} + +func toolIDs(serverName string, tools []string) []string { + ids := make([]string, 0, len(tools)) + for _, tool := range tools { + ids = append(ids, serverName+"_"+tool) + } + + return ids +} + +// spaceAgentTools are the tool IDs of agentTools that the space allows. An +// agent can use only the tools that the association of the server allows, and +// setup keeps the allowlist of an existing association. With no association +// yet, setup makes one that allows every ldcli tool. +func spaceAgentTools(ctx context.Context, clients Clients, spaceID, serviceID, serverName string) ([]string, error) { + tools := slices.Concat(mcpReadOnlyTools, mcpMutativeTools) + if spaceID == "" || serviceID == "" { + return toolIDs(serverName, tools), nil + } + associations, err := listAssociations(ctx, clients, spaceID) + if err != nil { + return nil, err + } + for _, association := range associations { + if aws.ToString(association.ServiceId) != serviceID { + continue + } + config, ok := association.Configuration.(*agenttypes.ServiceConfigurationMemberMcpserver) + if !ok { + break + } + tools = slices.DeleteFunc(tools, func(tool string) bool { return !slices.Contains(config.Value.Tools, tool) }) + } + + return toolIDs(serverName, tools), nil +} + +// AgentAction is the planned custom agent action. +type AgentAction int + +const ( + AgentCreate AgentAction = iota + AgentKeep + // AgentUpdate gives an agent with the prompt of this or an older ldcli + // version the prompt, the tools, and the skill of this version. + AgentUpdate + // AgentLeave means that someone edited the prompt, so setup does not + // change the agent without --replace-agent. + AgentLeave + AgentReplace + // AgentSkip means that setup could not read the custom agents of the + // space. The agent is optional, so the run continues without it. + AgentSkip +) + +// AgentStep is the planned custom agent step. +type AgentStep struct { + Action AgentAction + AssetID string + // Version is the asset version at plan time. A write stops if it changed. + Version int32 + // ServerName is the name of the MCP server, which the tool IDs start with. + ServerName string + // Tools and Skills are what the existing agent has. An update adds the + // ldcli tools and skill to them, so the ones that the customer added stay. + Tools []string + Skills []string + // NoFile means that the agent has no prompt file of this name. + NoFile bool + // Err says why the step is skipped. + Err error +} + +func (s AgentStep) describe() string { + label := fmt.Sprintf("%s (%s)", ExperimentAgentName, s.AssetID) + why := "someone edited its prompt" + if s.NoFile { + why = "it has no " + agentFilePath + } + switch s.Action { + case AgentKeep: + return "Keep custom agent " + label + case AgentUpdate: + return fmt.Sprintf("Update custom agent %s to the prompt and tools of this ldcli version", label) + case AgentLeave: + return fmt.Sprintf("Leave custom agent %s unchanged because %s. Pass --replace-agent to overwrite it", label, why) + case AgentReplace: + return fmt.Sprintf("Overwrite custom agent %s, which is not the ldcli version because %s", label, why) + case AgentSkip: + return agentFailure(fmt.Errorf("Skip custom agent %s: %w", ExperimentAgentName, s.Err)) + default: + return fmt.Sprintf("Create custom agent %s with skill %s and the MCP tools of server %s", + ExperimentAgentName, ExperimentSkillName, s.ServerName) + } +} + +// agentFailure is the text of a failed agent step. A missing permission is the +// most likely cause, so it names the permissions that the step needs. +func agentFailure(err error) string { + if errorCode(err) != "AccessDeniedException" { + return err.Error() + } + + return fmt.Sprintf("%v. The custom agent step needs aidevops:ListAssets, aidevops:GetAssetFile, "+ + "aidevops:CreateAsset, and aidevops:UpdateAsset", err) +} + +func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, mcp MCPStep, skill SkillStep) AgentStep { + step := AgentStep{ServerName: mcp.name()} + if space.Action == SpaceCreate { + return step + } + asset, err := findAsset(ctx, clients, space.ID, agentAssetType, ExperimentAgentName) + if err != nil { + step.Action, step.Err = AgentSkip, err + return step + } + if asset == nil { + return step + } + step.AssetID, step.Version = aws.ToString(asset.AssetId), aws.ToInt32(asset.Version) + step.Tools, step.Skills = agentMetadata(*asset) + prompt, found, err := assetText(ctx, clients, space.ID, step.AssetID, agentFilePath) + step.NoFile = !found + if err != nil { + step.Action, step.Err = AgentSkip, err + return step + } + want, err := spaceAgentTools(ctx, clients, space.ID, mcp.ServiceID, step.ServerName) + switch { + case err != nil: + step.Action, step.Err = AgentSkip, err + case prompt == ExperimentAgentPrompt && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && + !slices.ContainsFunc(want, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): + step.Action = AgentKeep + case found && (prompt == "" || slices.Contains(shippedAgentHashes, skillHash(prompt))): + // An empty prompt has nothing to lose. + step.Action = AgentUpdate + case opts.ReplaceAgent: + step.Action = AgentReplace + default: + step.Action = AgentLeave + } + + return step +} + +// agentMetadata reads the tools and the skills of a custom agent. +func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { + if asset.Metadata == nil { + return nil, nil + } + encoded, err := asset.Metadata.MarshalSmithyDocument() + if err != nil { + return nil, nil + } + var metadata struct { + Tools []string `json:"tools"` + Skills []string `json:"skills"` + } + if err := json.Unmarshal(encoded, &metadata); err != nil { + return nil, nil + } + + return metadata.Tools, metadata.Skills +} + +// union is a followed by the items of b that a does not have. +func union(a, b []string) []string { + out := slices.Clone(a) + for _, item := range b { + if !slices.Contains(out, item) { + out = append(out, item) + } + } + + return out +} + +// checkAgent compares the space with a step that writes. A change to the +// version means that someone edited the agent after the plan. +func checkAgent(ctx context.Context, clients Clients, spaceID string, step AgentStep) error { + asset, err := findAsset(ctx, clients, spaceID, agentAssetType, ExperimentAgentName) + switch { + case err != nil: + return err + case step.Action == AgentCreate && asset != nil: + return planChanged("custom agent %s appeared after the plan", ExperimentAgentName) + case step.Action == AgentCreate: + return nil + case asset == nil || aws.ToString(asset.AssetId) != step.AssetID: + return planChanged("custom agent %s disappeared after the plan", ExperimentAgentName) + case aws.ToInt32(asset.Version) != step.Version: + return planChanged("custom agent %s changed after the plan", ExperimentAgentName) + } + + return nil +} + +// The values of SetupResult.AgentAction. +const ( + AgentCreated = "created" + AgentKept = "kept" + AgentUpdated = "updated" + AgentLeft = "left" + AgentOverwritten = "overwritten" + AgentSkipped = "skipped" + AgentFailed = "failed" +) + +// installAgent does the custom agent step after the MCP server and skill +// steps, because the agent needs both. The agent is optional, so a failure does +// not stop setup. The failure goes in result.AgentError, and the caller reports +// it. +func installAgent( + ctx context.Context, + clients Clients, + step AgentStep, + result *SetupResult, + logf func(string, ...any), +) { + result.AgentAssetID = step.AssetID + action, err := doAgentStep(ctx, clients, step, result, logf) + result.AgentAction = action + if err != nil { + result.AgentAction = AgentFailed + if step.Action == AgentSkip || errors.Is(err, ErrPlanChanged) || errors.Is(err, errAgentNeeds) { + result.AgentAction = AgentSkipped + } + result.AgentError = agentFailure(err) + } +} + +// errAgentNeeds means that an earlier step left the agent without its MCP +// server or its skill, so setup does not write the agent. +var errAgentNeeds = errors.New("the custom agent needs the MCP server and the skill") + +func doAgentStep( + ctx context.Context, + clients Clients, + step AgentStep, + result *SetupResult, + logf func(string, ...any), +) (string, error) { + switch step.Action { + case AgentKeep: + logf("Kept custom agent %s (%s)", ExperimentAgentName, step.AssetID) + + return AgentKept, nil + case AgentLeave: + logf("Left custom agent %s (%s) unchanged. Pass --replace-agent to overwrite it", ExperimentAgentName, step.AssetID) + + return AgentLeft, nil + case AgentSkip: + return "", fmt.Errorf("setup skipped custom agent %s: %w", ExperimentAgentName, step.Err) + } + switch { + case result.MCPServiceID == "": + return "", fmt.Errorf("setup skipped custom agent %s, because no MCP server is connected: %w", + ExperimentAgentName, errAgentNeeds) + case result.SkillAssetID == "" || result.SkillAction == SkillSkipped: + // A skipped skill step can keep the ID of a skill that someone + // deleted. + return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", + ExperimentAgentName, ExperimentSkillName, errAgentNeeds) + } + tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, step.ServerName) + if err != nil { + return "", err + } + if len(tools) == 0 { + return "", fmt.Errorf("setup skipped custom agent %s, because the space allows none of the ldcli MCP tools: %w", + ExperimentAgentName, errAgentNeeds) + } + + // The other steps can take minutes, so check the agent right before the + // write. + if err := checkAgent(ctx, clients, result.AgentSpaceID, step); err != nil { + return "", err + } + content := &agenttypes.AssetContentMemberFile{Value: agenttypes.AssetFileContent{ + Path: aws.String(agentFilePath), + Body: &agenttypes.AssetFileBodyMemberText{Value: ExperimentAgentPrompt}, + }} + skills := []string{result.SkillAssetID} + if step.Action == AgentUpdate { + tools, skills = union(step.Tools, tools), union(step.Skills, skills) + } + metadata := map[string]any{"tools": tools, "skills": skills} + if step.Action == AgentCreate { + metadata["name"] = ExperimentAgentName + created, err := clients.Agent.CreateAsset(ctx, &devopsagent.CreateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetType: aws.String(agentAssetType), + Content: content, + Metadata: document.NewLazyDocument(metadata), + }) + if err != nil { + return "", fmt.Errorf("unable to create custom agent %s: %w", ExperimentAgentName, err) + } + result.AgentAssetID = aws.ToString(created.Asset.AssetId) + logf("Created custom agent %s (%s)", ExperimentAgentName, result.AgentAssetID) + + return AgentCreated, nil + } + + // AWS updates only the metadata keys in the request, so the memory stores + // that the customer attached stay. + if _, err := clients.Agent.UpdateAsset(ctx, &devopsagent.UpdateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetId: aws.String(step.AssetID), + Content: content, + Metadata: document.NewLazyDocument(metadata), + }); err != nil { + return "", fmt.Errorf("unable to update custom agent %s (%s): %w", ExperimentAgentName, step.AssetID, err) + } + logf("Updated custom agent %s (%s)", ExperimentAgentName, step.AssetID) + if step.Action == AgentReplace { + return AgentOverwritten, nil + } + + return AgentUpdated, nil +} diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go new file mode 100644 index 000000000..0837815c0 --- /dev/null +++ b/internal/awsdevops/agent_test.go @@ -0,0 +1,272 @@ +package awsdevops_test + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) + +// A change to the prompt must add its hash, so the next release updates the +// agents that this release makes. +func TestShippedAgentHashesHoldTheCurrentPrompt(t *testing.T) { + assert.Contains(t, awsdevops.ShippedAgentHashes(), awsdevops.SkillHash(awsdevops.ExperimentAgentPrompt), + "add the hash of the new prompt to shippedAgentHashes") +} + +// AWS names a tool of an MCP server "_". +func TestAgentToolsStartWithTheServerName(t *testing.T) { + tools := awsdevops.AgentTools("LaunchDarkly") + + assert.Contains(t, tools, "LaunchDarkly_get-flag") + assert.Contains(t, tools, "LaunchDarkly_toggle-flag") + for _, tool := range tools { + assert.Regexp(t, `^LaunchDarkly_[a-z-]+$`, tool) + } +} + +func TestSetupCreatesAgentInNewSpace(t *testing.T) { + agent := &fakeAgent{} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), awsdevops.SetupOptions{ + AgentSpaceName: "launchdarkly", + AuthFlow: "iam", + LDAccessToken: "api-token", + }) + require.NoError(t, err) + + require.Len(t, agent.createAgents, 1) + in := agent.createAgents[0] + assert.Equal(t, "space-1", aws.ToString(in.AgentSpaceId)) + metadata := metadataOf(t, in.Metadata) + assert.Equal(t, awsdevops.ExperimentAgentName, metadata["name"]) + assert.Equal(t, []any{"ki-created"}, metadata["skills"]) + assert.Contains(t, metadata["tools"], awsdevops.MCPServerName+"_get-flag") + assert.NotContains(t, metadata, "memory_stores") + file, ok := in.Content.(*agenttypes.AssetContentMemberFile) + require.True(t, ok) + assert.Equal(t, "experiment-orchestrator.md", aws.ToString(file.Value.Path)) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentAgentPrompt}, file.Value.Body) + assert.Equal(t, "ki-agent-created", result.AgentAssetID) + assert.Equal(t, awsdevops.AgentCreated, result.AgentAction) + assert.Empty(t, result.AgentError) +} + +// The tool IDs use the name of the server in AWS, which the customer can +// change. +func TestPlanSetupNamesAgentToolsAfterTheExistingServer(t *testing.T) { + agent := &fakeAgent{services: []agenttypes.RegisteredService{ + mcpService("mcp-1", "Prod-LD", awsdevops.MCPServerEndpoint), + }} + opts := reuseOptions() + opts.LDAccessToken = "" + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), opts, true) + require.NoError(t, err) + + require.Equal(t, awsdevops.MCPReuse, plan.MCP.Action) + assert.Equal(t, "Prod-LD", plan.Agent.ServerName) + assert.Equal(t, "Create custom agent experiment-orchestrator with skill experiment-orchestration "+ + "and the MCP tools of server Prod-LD", plan.Text().Agent) +} + +func TestSetupKeepsCurrentAgent(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", awsdevops.AgentTools(awsdevops.MCPServerName), []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Empty(t, agent.createAgents) + assert.Empty(t, agent.updateAgents) + assert.Equal(t, "ki-agent", result.AgentAssetID) + assert.Equal(t, awsdevops.AgentKept, result.AgentAction) +} + +// A newer ldcli allows more tools, so an agent with the ldcli prompt and +// fewer tools gets the new list. +func TestSetupUpdatesAgentWithOldTools(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", []string{"LaunchDarkly_get-flag"}, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1) + metadata := metadataOf(t, agent.updateAgents[0].Metadata) + assert.ElementsMatch(t, awsdevops.AgentTools(awsdevops.MCPServerName), metadata["tools"]) + assert.Equal(t, []any{"ki-ours"}, metadata["skills"]) + // AWS keeps the keys that the request leaves out, such as memory stores. + assert.NotContains(t, metadata, "memory_stores") + assert.Equal(t, awsdevops.AgentUpdated, result.AgentAction) +} + +func TestSetupLeavesEditedAgent(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) + assert.Contains(t, plan.Text().Agent, "--replace-agent") + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Empty(t, agent.updateAgents) + assert.Equal(t, awsdevops.AgentLeft, result.AgentAction) + assert.Empty(t, result.AgentError) +} + +func TestSetupOverwritesEditedAgentWithFlag(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, + } + opts := reuseOptions() + opts.ReplaceAgent = true + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1) + assert.Equal(t, "ki-agent", aws.ToString(agent.updateAgents[0].AssetId)) + assert.Equal(t, awsdevops.AgentOverwritten, result.AgentAction) +} + +// Without the skill, the agent could not run, so setup does not make it. +func TestSetupSkipsAgentWhenSkillFails(t *testing.T) { + agent := &fakeAgent{createAssetErr: errors.New("AccessDeniedException: not allowed")} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Empty(t, agent.createAgents) + assert.Equal(t, awsdevops.SkillFailed, result.SkillAction) + assert.Equal(t, awsdevops.AgentSkipped, result.AgentAction) + assert.Contains(t, result.AgentError, "skill experiment-orchestration is not ready") +} + +// The agent is optional, so its failure does not fail setup. +func TestSetupWarnsWhenAgentCreateFails(t *testing.T) { + agent := &fakeAgent{createAgentErr: accessDenied("aidevops:CreateAsset")} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillCreated, result.SkillAction) + assert.Equal(t, awsdevops.AgentFailed, result.AgentAction) + assert.Contains(t, result.AgentError, "The custom agent step needs aidevops:ListAssets") +} + +func TestPlanSetupSkipsAgentWhenAssetsCannotBeListed(t *testing.T) { + agent := &fakeAgent{listAssetsErr: accessDenied("aidevops:ListAssets")} + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.AgentSkip, plan.Agent.Action) + assert.Contains(t, plan.Text().Agent, "The custom agent step needs aidevops:ListAssets") +} + +// A customer can give the agent more tools or skills. The ldcli ones are +// there, so setup keeps the agent. +func TestPlanSetupKeepsAgentWithAddedTools(t *testing.T) { + tools := append(awsdevops.AgentTools(awsdevops.MCPServerName), "Datadog_query-logs") + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", tools, []string{"ki-other", "ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.AgentKeep, plan.Agent.Action) +} + +// An update adds the ldcli tools and skill, and keeps the ones that the +// customer added. +func TestSetupUpdateKeepsAddedToolsAndSkills(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + agentAsset("ki-agent", "experiment-orchestrator", []string{"Datadog_query-logs"}, []string{"ki-other"}), + }, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + _, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1) + metadata := metadataOf(t, agent.updateAgents[0].Metadata) + assert.Contains(t, metadata["tools"], "Datadog_query-logs") + assert.Contains(t, metadata["tools"], awsdevops.MCPServerName+"_get-flag") + assert.Equal(t, []any{"ki-other", "ki-created"}, metadata["skills"]) +} + +// Setup keeps the allowlist of an existing association, and an agent can use +// only the tools that it allows. +func TestSetupGivesAgentOnlyTheToolsTheSpaceAllows(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{{ + AgentSpaceId: aws.String("space-2"), + AssociationId: aws.String("assoc-mcp-1"), + ServiceId: aws.String("mcp-1"), + Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{ + Value: agenttypes.MCPServerConfiguration{Tools: []string{"get-flag", "not-an-ldcli-tool"}}, + }, + }}, + } + opts := reuseOptions() + opts.LDAccessToken = "" + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) + require.NoError(t, err) + + require.Len(t, agent.createAgents, 1, result.AgentError) + assert.Equal(t, []any{awsdevops.MCPServerName + "_get-flag"}, metadataOf(t, agent.createAgents[0].Metadata)["tools"]) +} + +func TestPlanTextSkipsAgentWithoutMCPServer(t *testing.T) { + plan := awsdevops.SetupPlan{ + MCP: awsdevops.MCPStep{Action: awsdevops.MCPSkip}, + Agent: awsdevops.AgentStep{Action: awsdevops.AgentCreate}, + } + + assert.Equal(t, "Skip custom agent experiment-orchestrator, because no MCP server is connected", plan.Text().Agent) +} + +func TestPlanSetupLeavesAgentWithoutPromptFile(t *testing.T) { + agent := &fakeAgent{assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}} + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) + assert.Contains(t, plan.Text().Agent, "because it has no experiment-orchestrator.md") +} diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md new file mode 100644 index 000000000..3c8f58fd1 --- /dev/null +++ b/internal/awsdevops/agents/experiment-orchestrator.md @@ -0,0 +1,39 @@ +You are an experimentation agent. You move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. + +## Goal + +Take a goal (for example, "increase checkout conversion by 15%") from a hypothesis to the final report, and keep the experiments that are in flight moving. + +## Approach + +1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. This prompt only says how this agent runs it. +2. Always follow the skill's "Resuming in-flight work" section first. +3. Then pick the mode for this invocation: + - **A person gave a goal in this invocation** that no open experiment record covers: start at Step 1 of the skill. + - **Anything else**, such as a scheduled run or a run with no prompt: resume only. Do not start a goal. +4. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. + +## State + +This agent has no memory store. LaunchDarkly holds the experiment records: each flag that the skill creates has the tag `experiment-orchestration`, and its description holds the whole experiment record from the skill. Update the description whenever a record field changes. A later invocation reads only the description, so write these fields there as soon as you know them: + +- The goal, the hypothesis, and the current step +- Who confirmed that the change is deployed, and when +- The guardrail baselines at the experiment start +- The experiment key, and the start and stop dates of the experiment and of the guarded rollout + +## Implementation + +No coding agent and no CI/CD pipeline are connected to this agent. Always use the skill's path for "If no coding agent is available": create the flag, report the change to make, and wait for a person to confirm that it is deployed. + +## Constraints + +- Change only flags that have the tag `experiment-orchestration`. Read other flags for collision checks and history, but never change them. +- Without a person in this invocation, never start a goal, accept a hypothesis, treat a change as deployed, or retry after a rollback. +- If a tool is not available or a change needs approval, report it and stop that flag. Do not work around it. + +## Output + +- When a person gave a goal: a short status, then any question that the skill tells you to ask. +- When resuming: one line for each flag in flight, with the flag key, the step, and what changed in this invocation. If nothing is in flight, say so. +- On a terminal outcome: the Step 8 report, exactly as the skill defines it. diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index b59a018ec..061fc08fe 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -65,8 +65,10 @@ type fakeAgent struct { // paginate makes every list API return one item per page, so callers are // exercised against the NextToken AWS sends for a long list. paginate bool - // assets holds the skill assets of every space. + // assets holds the skill and custom agent assets of every space. assets []agenttypes.Asset + // agentFiles holds the prompt of each custom agent by asset ID. + agentFiles map[string]string // skillFiles holds the SKILL.md text by asset ID. skillFiles map[string]string // skillBytes holds a SKILL.md that AWS sends as bytes, by asset ID. @@ -81,6 +83,20 @@ type fakeAgent struct { updateAssociations []*devopsagent.UpdateAssociationInput updateAssociationErr error updateAssets []*devopsagent.UpdateAssetInput + // createAgents and updateAgents hold the custom agent writes, apart from + // the skill writes. + createAgents []*devopsagent.CreateAssetInput + updateAgents []*devopsagent.UpdateAssetInput + createAgentErr error +} + +// agentAsset is a custom agent asset named name with these tools and skills. +func agentAsset(id, name string, tools, skills []string) agenttypes.Asset { + return agenttypes.Asset{ + AssetId: aws.String(id), + AssetType: aws.String("custom_agent"), + Metadata: document.NewLazyDocument(map[string]any{"name": name, "tools": tools, "skills": skills}), + } } // skillAsset is a skill asset named name. @@ -94,11 +110,20 @@ func skillAsset(id, name string) agenttypes.Asset { func (f *fakeAgent) CreateAsset(_ context.Context, in *devopsagent.CreateAssetInput, _ ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { f.calls = append(f.calls, "CreateAsset") - f.createAssets = append(f.createAssets, in) - if f.createAssetErr != nil { - return nil, f.createAssetErr + id := "ki-created" + if aws.ToString(in.AssetType) == "custom_agent" { + f.createAgents = append(f.createAgents, in) + if f.createAgentErr != nil { + return nil, f.createAgentErr + } + id = "ki-agent-created" + } else { + f.createAssets = append(f.createAssets, in) + if f.createAssetErr != nil { + return nil, f.createAssetErr + } } - asset := agenttypes.Asset{AssetId: aws.String("ki-created"), AssetType: in.AssetType, Metadata: in.Metadata} + asset := agenttypes.Asset{AssetId: aws.String(id), AssetType: in.AssetType, Metadata: in.Metadata} f.assets = append(f.assets, asset) return &devopsagent.CreateAssetOutput{Asset: &asset}, nil @@ -112,6 +137,12 @@ func (f *fakeAgent) GetAssetFile(_ context.Context, in *devopsagent.GetAssetFile if f.getFileErr != nil { return nil, f.getFileErr } + if prompt, ok := f.agentFiles[aws.ToString(in.AssetId)]; ok && aws.ToString(in.Path) == "experiment-orchestrator.md" { + return &devopsagent.GetAssetFileOutput{File: &agenttypes.AssetFile{ + Path: in.Path, + Content: &agenttypes.AssetFileBodyMemberText{Value: prompt}, + }}, nil + } if aws.ToString(in.Path) != "SKILL.md" { return nil, &agenttypes.ResourceNotFoundException{} } @@ -137,13 +168,26 @@ func (f *fakeAgent) ListAssets(_ context.Context, in *devopsagent.ListAssetsInpu if f.listAssetsErr != nil { return nil, f.listAssetsErr } - items, next := page(f.assets, in.NextToken, f.paginate) + var assets []agenttypes.Asset + for _, asset := range f.assets { + if in.AssetType == nil || aws.ToString(asset.AssetType) == aws.ToString(in.AssetType) { + assets = append(assets, asset) + } + } + items, next := page(assets, in.NextToken, f.paginate) return &devopsagent.ListAssetsOutput{Items: items, NextToken: next}, nil } func (f *fakeAgent) UpdateAsset(_ context.Context, in *devopsagent.UpdateAssetInput, _ ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) { f.calls = append(f.calls, "UpdateAsset") + for _, asset := range f.assets { + if aws.ToString(asset.AssetId) == aws.ToString(in.AssetId) && aws.ToString(asset.AssetType) == "custom_agent" { + f.updateAgents = append(f.updateAgents, in) + + return &devopsagent.UpdateAssetOutput{Asset: &agenttypes.Asset{AssetId: in.AssetId}}, nil + } + } f.updateAssets = append(f.updateAssets, in) return &devopsagent.UpdateAssetOutput{Asset: &agenttypes.Asset{AssetId: in.AssetId}}, nil @@ -467,10 +511,12 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { assert.Equal( t, // The plan does not look for the server of a new space. The check - // before the MCP step looks, because the space exists by then, - // then the skill last. + // before the MCP step looks, because the space exists by then, then + // the skill and the custom agent last. []string{ - "CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "RegisterService", "AssociateService", "ListAssets", "CreateAsset", + "CreateAgentSpace", "AssociateService", "EnableOperatorApp", + "ListServices", "RegisterService", "AssociateService", + "ListAssets", "CreateAsset", "ListAssets", "CreateAsset", }, agent.calls, ) diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 1526e3aac..eede802ec 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -41,3 +41,9 @@ func SkillHash(text string) string { return skillHash(text) } // AllowedTools are the tools that setup allows on the MCP server. func AllowedTools() []string { return slices.Concat(mcpReadOnlyTools, mcpMutativeTools) } + +// ShippedAgentHashes are the hashes of the shipped custom agent prompts. +func ShippedAgentHashes() []string { return shippedAgentHashes } + +// AgentTools are the tool IDs that setup gives the custom agent. +func AgentTools(serverName string) []string { return agentTools(serverName) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index c21ec95cf..73ce4b9fc 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -128,6 +128,9 @@ type SetupOptions struct { // server association misses. AWS does not send back the classification of // each tool, so the rewrite gives every tool the ldcli classification. AddMCPTools bool + // ReplaceAgent overwrites the experiment-orchestrator custom agent if + // someone edited its prompt. + ReplaceAgent bool // Logf, when set, reports progress as each step completes. Logf func(format string, args ...any) @@ -157,7 +160,13 @@ type SetupResult struct { SkillAction string `json:"skillAction,omitempty"` // SkillError is set if the skill step failed or was skipped. The skill is // optional, so the rest of the setup ran. - SkillError string `json:"skillError,omitempty"` + SkillError string `json:"skillError,omitempty"` + AgentAssetID string `json:"agentAssetId,omitempty"` + // AgentAction is what the custom agent step did, for example "created". + AgentAction string `json:"agentAction,omitempty"` + // AgentError is set if the custom agent step failed or was skipped. The + // agent is optional, so the rest of the setup ran. + AgentError string `json:"agentError,omitempty"` } // ErrPlanChanged means AWS changed after the plan. @@ -430,6 +439,7 @@ type SetupPlan struct { AuthFlow string MCP MCPStep Skill SkillStep + Agent AgentStep } // PlanText is the plan as lines for a person to read. @@ -440,11 +450,21 @@ type PlanText struct { OperatorApp string MCPServer string Skill string + Agent string } // Text describes each step of the plan. func (p SetupPlan) Text() PlanText { - text := PlanText{AgentSpace: p.Space.describe(), MCPServer: p.MCP.describe(), Skill: p.Skill.describe()} + text := PlanText{ + AgentSpace: p.Space.describe(), + MCPServer: p.MCP.describe(), + Skill: p.Skill.describe(), + Agent: p.Agent.describe(), + } + if p.MCP.Action == MCPSkip && slices.Contains([]AgentAction{AgentCreate, AgentUpdate, AgentReplace}, p.Agent.Action) { + // The run skips the agent, which would have no tools. + text.Agent = fmt.Sprintf("Skip custom agent %s, because no MCP server is connected", ExperimentAgentName) + } if p.Space.Action == SpaceConnect { text.Association = "No change" text.Roles = []string{"No change"} @@ -539,6 +559,7 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } plan.Skill = planSkill(ctx, clients, opts, plan.Space) + plan.Agent = planAgent(ctx, clients, opts, plan.Space, plan.MCP, plan.Skill) if !provision { // Read the app, so the run does not print a URL that does not work. @@ -1039,9 +1060,10 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } - // The skill goes last, so a failure of the optional step cannot stop the - // MCP server step. + // The skill and the agent go last, so a failure of the optional steps + // cannot stop the MCP server step. installSkill(ctx, clients, plan.Skill, &result, logf) + installAgent(ctx, clients, plan.Agent, &result, logf) return result, nil } @@ -1121,6 +1143,7 @@ func ConnectPlan(ctx context.Context, clients Clients, opts SetupOptions, plan S return result, err } installSkill(ctx, clients, plan.Skill, &result, logf) + installAgent(ctx, clients, plan.Agent, &result, logf) return result, nil } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 1d4ca7ad1..f6dc9fc13 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -176,7 +176,13 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp // findSkill returns the skill with this name in the space, or nil if there is // none. AWS keeps skill names unique in a space. func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { - assets, err := listAssets(ctx, clients, spaceID, skillAssetType) + return findAsset(ctx, clients, spaceID, skillAssetType, name) +} + +// findAsset returns the asset of this type and name in the space, or nil if +// there is none. +func findAsset(ctx context.Context, clients Clients, spaceID, assetType, name string) (*agenttypes.Asset, error) { + assets, err := listAssets(ctx, clients, spaceID, assetType) if err != nil { return nil, err } @@ -213,10 +219,16 @@ func assetName(asset agenttypes.Asset) string { // skillText returns the SKILL.md of the skill. found is false if the skill // has no SKILL.md. func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (text string, found bool, err error) { + return assetText(ctx, clients, spaceID, assetID, skillFilePath) +} + +// assetText returns the file at path in the asset. found is false if the +// asset has no such file. +func assetText(ctx context.Context, clients Clients, spaceID, assetID, path string) (text string, found bool, err error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), AssetId: aws.String(assetID), - Path: aws.String(skillFilePath), + Path: aws.String(path), }) var missing *agenttypes.ResourceNotFoundException switch { @@ -224,7 +236,7 @@ func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (t return "", false, nil case err != nil: // The caller names the skill. - return "", false, fmt.Errorf("unable to read %s of asset %s: %w", skillFilePath, assetID, err) + return "", false, fmt.Errorf("unable to read %s of asset %s: %w", path, assetID, err) case file.File == nil: return "", false, nil } From 6d91b769d88e7b4515139ee88d114257c76fb4ab Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 15:31:35 -0700 Subject: [PATCH 10/34] feat: show the skill and the custom agent in status status now reports the experiment-orchestration skill and the experiment-orchestrator custom agent of each agent space: whether each is the ldcli version, an older one that setup updates, edited, without its file, or missing. For the agent, it also gives the number of tools and whether the skill is attached. An agent with the ldcli prompt but without the skill or an allowed tool is outdated, because setup updates it. Like the operator app, an asset that status cannot read does not stop it. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/status.go | 51 ++++++ cmd/aws_devops_agent/status_fixture_test.go | 12 +- cmd/aws_devops_agent/status_test.go | 53 +++++++ .../testdata/status-partial.json | 12 +- internal/awsdevops/agent_test.go | 108 +++++++++++++ internal/awsdevops/status.go | 149 ++++++++++++++++++ 6 files changed, 383 insertions(+), 2 deletions(-) diff --git a/cmd/aws_devops_agent/status.go b/cmd/aws_devops_agent/status.go index a11ceba44..66a26ae2c 100644 --- a/cmd/aws_devops_agent/status.go +++ b/cmd/aws_devops_agent/status.go @@ -107,11 +107,62 @@ func printStatus(out io.Writer, status awsdevops.Status) { _, _ = fmt.Fprintf(out, " %s: %s\n", associationLabel(status, association), associationState(association)) } + if space.Skill != nil { + _, _ = fmt.Fprintf(out, " Skill %s: %s\n", + assetLabel(awsdevops.ExperimentSkillName, *space.Skill), + assetLine(*space.Skill, "SKILL.md", "--replace-skill")) + } + if space.Agent != nil { + _, _ = fmt.Fprintf(out, " Custom agent %s: %s\n", + assetLabel(awsdevops.ExperimentAgentName, space.Agent.AssetStatus), agentLine(*space.Agent)) + } } printConsoleURL(out, status.Region) } +// assetLabel is the name of the skill or the agent, with its asset ID. +func assetLabel(name string, asset awsdevops.AssetStatus) string { + if asset.AssetID == "" { + return name + } + + return name + " (" + asset.AssetID + ")" +} + +// assetLine gives the state of the skill or the agent, and what setup does +// with it. file is the file that setup writes, and flag is the setup flag that +// overwrites an edited one. +func assetLine(asset awsdevops.AssetStatus, file, flag string) string { + switch asset.State { + case awsdevops.AssetCurrent: + return "this ldcli version" + case awsdevops.AssetOutdated: + return "an older ldcli version. Setup updates it" + case awsdevops.AssetEdited: + return "edited. Setup leaves it unless you pass " + flag + case awsdevops.AssetNoFile: + return "has no " + file + ". Setup leaves it unless you pass " + flag + case awsdevops.AssetMissing: + return "missing. " + setupHint + } + + return "unknown (" + asset.Error + ")" +} + +func agentLine(agent awsdevops.AgentStatus) string { + line := assetLine(agent.AssetStatus, awsdevops.ExperimentAgentName+".md", "--replace-agent") + if agent.State == awsdevops.AssetMissing || agent.AssetID == "" { + return line + } + line += fmt.Sprintf(". It has %d tools", len(agent.Tools)) + if agent.SkillAttached != nil && !*agent.SkillAttached { + line += " and not skill " + awsdevops.ExperimentSkillName + } + + return line +} + func printConsoleURL(out io.Writer, region string) { _, _ = fmt.Fprintf(out, "\nAWS DevOps Agent console: %s\n", awsdevops.ConsoleURL(region)) } diff --git a/cmd/aws_devops_agent/status_fixture_test.go b/cmd/aws_devops_agent/status_fixture_test.go index 3e727c7bb..04c21c274 100644 --- a/cmd/aws_devops_agent/status_fixture_test.go +++ b/cmd/aws_devops_agent/status_fixture_test.go @@ -1,6 +1,10 @@ package awsdevopsagent -import "github.com/launchdarkly/ldcli/internal/awsdevops" +import ( + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) func realisticStatus() awsdevops.Status { return awsdevops.Status{ @@ -40,6 +44,12 @@ func partialStatus() awsdevops.Status { AgentSpaceID: "as-prod01", Name: "Production", OperatorAppError: "AccessDeniedException", + Skill: &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetEdited}, + Agent: &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetOutdated}, + Tools: []string{"LaunchDarkly_get-flag"}, + SkillAttached: aws.Bool(false), + }, }}, MissingSetup: []string{"IAM role " + awsdevops.OperatorAppRoleName + " does not exist"}, } diff --git a/cmd/aws_devops_agent/status_test.go b/cmd/aws_devops_agent/status_test.go index d5ba6245e..19a18e936 100644 --- a/cmd/aws_devops_agent/status_test.go +++ b/cmd/aws_devops_agent/status_test.go @@ -1,6 +1,8 @@ package awsdevopsagent import ( + "github.com/aws/aws-sdk-go-v2/aws" + "bytes" "os" "strings" @@ -328,3 +330,54 @@ func TestStatusShowsAnOperatorAppThatIsOff(t *testing.T) { assert.Contains(t, out, "Operator app: off") assert.NotContains(t, out, "aidevops.global.app.aws") } + +func TestStatusShowsSkillAndAgent(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetCurrent} + status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetEdited}, + Tools: []string{"LaunchDarkly_get-flag", "LaunchDarkly_toggle-flag"}, + SkillAttached: aws.Bool(true), + } + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + + assert.Equal(t, " Skill experiment-orchestration (ki-skill): this ldcli version", lineWith(t, section, "Skill")) + assert.Equal(t, + " Custom agent experiment-orchestrator (ki-agent): edited. Setup leaves it unless you pass --replace-agent. "+ + "It has 2 tools", + lineWith(t, section, "Custom agent")) +} + +func TestStatusShowsMissingAgentAndAgentWithoutSkill(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{AssetStatus: awsdevops.AssetStatus{State: awsdevops.AssetMissing}} + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + assert.Equal(t, " Custom agent experiment-orchestrator: missing. "+setupHint, lineWith(t, section, "Custom agent")) + + status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetOutdated}, + SkillAttached: aws.Bool(false), + } + section = spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + assert.Equal(t, + " Custom agent experiment-orchestrator (ki-agent): an older ldcli version. Setup updates it. "+ + "It has 0 tools and not skill experiment-orchestration", + lineWith(t, section, "Custom agent")) +} + +func TestStatusShowsSkillWithoutFile(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetNoFile} + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + + assert.Equal(t, " Skill experiment-orchestration (ki-skill): has no SKILL.md. Setup leaves it unless you pass "+ + "--replace-skill", lineWith(t, section, "Skill")) +} + +func TestStatusShowsUnreadableSkill(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{Error: "AccessDeniedException"} + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + + assert.Equal(t, " Skill experiment-orchestration: unknown (AccessDeniedException)", lineWith(t, section, "Skill")) +} diff --git a/cmd/aws_devops_agent/testdata/status-partial.json b/cmd/aws_devops_agent/testdata/status-partial.json index d54be8ce6..e7772fd0a 100644 --- a/cmd/aws_devops_agent/testdata/status-partial.json +++ b/cmd/aws_devops_agent/testdata/status-partial.json @@ -18,7 +18,17 @@ "name": "Production", "operatorAppUrl": "", "operatorAppEnabled": false, - "operatorAppError": "AccessDeniedException" + "operatorAppError": "AccessDeniedException", + "skill": { + "assetId": "ki-skill", + "state": "edited" + }, + "agent": { + "assetId": "ki-agent", + "state": "outdated", + "tools": ["LaunchDarkly_get-flag"], + "skillAttached": false + } } ], "missingSetup": [ diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 0837815c0..9b1598940 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -270,3 +270,111 @@ func TestPlanSetupLeavesAgentWithoutPromptFile(t *testing.T) { assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) assert.Contains(t, plan.Text().Agent, "because it has no experiment-orchestrator.md") } + +func TestStatusReportsSkillAndAgent(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", []string{"LaunchDarkly_get-flag"}, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + require.Len(t, status.AgentSpaces, 1) + space := status.AgentSpaces[0] + assert.Equal(t, &awsdevops.AssetStatus{AssetID: "ki-ours", State: awsdevops.AssetCurrent}, space.Skill) + assert.Equal(t, &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetEdited}, + Tools: []string{"LaunchDarkly_get-flag"}, + SkillAttached: aws.Bool(true), + }, space.Agent) +} + +// Setup updates an agent with the ldcli prompt but without the skill or an +// allowed tool, so status calls it outdated. +func TestStatusReportsAgentWithoutToolsAsOutdated(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + associations: []agenttypes.Association{{ + AgentSpaceId: aws.String("space-1"), + ServiceId: aws.String("mcp-1"), + Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{ + Value: agenttypes.MCPServerConfiguration{Tools: []string{"get-flag", "toggle-flag"}}, + }, + }}, + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", []string{"Prod-LD_get-flag"}, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + clients := newTestClients(agent, newFakeIAM()) + + status, err := awsdevops.GetStatus(context.Background(), clients, "") + require.NoError(t, err) + assert.Equal(t, awsdevops.AssetOutdated, status.AgentSpaces[0].Agent.State) + + agent.assets[1] = agentAsset("ki-agent", "experiment-orchestrator", + []string{"Prod-LD_get-flag", "Prod-LD_toggle-flag"}, []string{"ki-ours"}) + status, err = awsdevops.GetStatus(context.Background(), clients, "") + require.NoError(t, err) + assert.Equal(t, awsdevops.AssetCurrent, status.AgentSpaces[0].Agent.State) +} + +func TestStatusReportsMissingSkillAndAgent(t *testing.T) { + agent := &fakeAgent{agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}} + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, awsdevops.AssetMissing, status.AgentSpaces[0].Skill.State) + assert.Equal(t, awsdevops.AssetMissing, status.AgentSpaces[0].Agent.State) + assert.Nil(t, status.AgentSpaces[0].Agent.SkillAttached) +} + +// A skill without SKILL.md is not ours, and setup leaves it. +func TestStatusReportsSkillWithoutFile(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, &awsdevops.AssetStatus{AssetID: "ki-ours", State: awsdevops.AssetNoFile}, status.AgentSpaces[0].Skill) +} + +// An older ldcli prompt is one that setup updates. +func TestStatusReportsOutdatedAgent(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": ""}, + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, awsdevops.AssetOutdated, status.AgentSpaces[0].Agent.State) +} + +// Like the operator app, assets that status cannot read do not stop it. +func TestStatusReportsUnreadableAssets(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + listAssetsErr: accessDenied("aidevops:ListAssets"), + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Skill.Error) + assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Agent.Error) +} diff --git a/internal/awsdevops/status.go b/internal/awsdevops/status.go index 76a8acae0..2b35b0f98 100644 --- a/internal/awsdevops/status.go +++ b/internal/awsdevops/status.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "slices" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/devopsagent" @@ -47,6 +49,44 @@ type AgentSpaceStatus struct { // and OperatorAppEnabled is false. OperatorAppError string `json:"operatorAppError,omitempty"` Associations []AssociationStatus `json:"associations,omitempty"` + // Skill is the experiment-orchestration skill, and Agent is the + // experiment-orchestrator custom agent. + Skill *AssetStatus `json:"skill,omitempty"` + Agent *AgentStatus `json:"agent,omitempty"` +} + +// The values of AssetStatus.State. +const ( + // AssetCurrent is the text of this ldcli version. + AssetCurrent = "current" + // AssetOutdated is the text of an older ldcli version, or an empty file. + // Setup updates it. + AssetOutdated = "outdated" + // AssetEdited is a text that no ldcli version sent. Setup leaves it + // without --replace-skill or --replace-agent. + AssetEdited = "edited" + // AssetNoFile is an asset without the file that setup writes, for example + // a skill that someone made from a PDF. Setup leaves it too. + AssetNoFile = "no-file" + AssetMissing = "missing" +) + +// AssetStatus is the state of an asset that setup makes. +type AssetStatus struct { + AssetID string `json:"assetId,omitempty"` + State string `json:"state,omitempty"` + // Error is set if status could not read the asset, for example without + // aidevops:ListAssets. State is then empty. + Error string `json:"error,omitempty"` +} + +// AgentStatus is the state of the custom agent. +type AgentStatus struct { + AssetStatus + Tools []string `json:"tools,omitempty"` + // SkillAttached says whether the agent loads the experiment-orchestration + // skill. It is nil if status could not read the agent or the skill. + SkillAttached *bool `json:"skillAttached,omitempty"` } type AssociationStatus struct { @@ -150,12 +190,121 @@ func GetStatus(ctx context.Context, clients Clients, agentSpaceID string) (Statu }) } + spaceStatus.Skill = skillStatus(ctx, clients, id) + spaceStatus.Agent = agentStatus(ctx, clients, id, *spaceStatus.Skill, associations) + status.AgentSpaces = append(status.AgentSpaces, spaceStatus) } return status, nil } +// assetState compares the text of an asset with the text of this ldcli +// version and the texts of older versions. +func assetState(text string, found bool, current string, shipped []string) string { + switch { + case !found: + return AssetNoFile + case text == current: + return AssetCurrent + case found && (text == "" || slices.Contains(shipped, skillHash(text))): + return AssetOutdated + } + + return AssetEdited +} + +// skillStatus reads the skill. Like the operator app, an unreadable skill does +// not stop status. +func skillStatus(ctx context.Context, clients Clients, spaceID string) *AssetStatus { + asset, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) + switch { + case err != nil: + return &AssetStatus{Error: errorCode(err)} + case asset == nil: + return &AssetStatus{State: AssetMissing} + } + status := &AssetStatus{AssetID: aws.ToString(asset.AssetId)} + text, found, err := skillText(ctx, clients, spaceID, status.AssetID) + switch { + case err != nil: + status.Error = errorCode(err) + case !found && !stillListed(ctx, clients, spaceID, skillAssetType, ExperimentSkillName, status.AssetID): + // AWS also sends ResourceNotFoundException for an asset that someone + // deleted after the list, and setup then makes a new one. + return &AssetStatus{State: AssetMissing} + default: + status.State = assetState(text, found, ExperimentSkill, shippedSkillHashes) + } + + return status +} + +// stillListed reports whether the asset is still in the space. +func stillListed(ctx context.Context, clients Clients, spaceID, assetType, name, assetID string) bool { + asset, err := findAsset(ctx, clients, spaceID, assetType, name) + + return err != nil || (asset != nil && aws.ToString(asset.AssetId) == assetID) +} + +// agentStatus reads the custom agent. An agent with the ldcli prompt is +// current only if it also has the skill and the ldcli tools that the space +// allows, because setup updates it otherwise. +func agentStatus( + ctx context.Context, + clients Clients, + spaceID string, + skill AssetStatus, + associations []agenttypes.Association, +) *AgentStatus { + asset, err := findAsset(ctx, clients, spaceID, agentAssetType, ExperimentAgentName) + switch { + case err != nil: + return &AgentStatus{AssetStatus: AssetStatus{Error: errorCode(err)}} + case asset == nil: + return &AgentStatus{AssetStatus: AssetStatus{State: AssetMissing}} + } + status := &AgentStatus{AssetStatus: AssetStatus{AssetID: aws.ToString(asset.AssetId)}} + var skills []string + status.Tools, skills = agentMetadata(*asset) + if skill.Error == "" { + attached := skill.AssetID != "" && slices.Contains(skills, skill.AssetID) + status.SkillAttached = &attached + } + prompt, found, err := assetText(ctx, clients, spaceID, status.AssetID, agentFilePath) + if err != nil { + status.Error = errorCode(err) + + return status + } + status.State = assetState(prompt, found, ExperimentAgentPrompt, shippedAgentHashes) + if status.State == AssetCurrent && (status.SkillAttached == nil || !*status.SkillAttached || + missingAgentTools(status.Tools, associations)) { + status.State = AssetOutdated + } + + return status +} + +// missingAgentTools reports whether the agent lacks an ldcli tool that an MCP +// server association of the space allows. status does not know which server is +// the LaunchDarkly one, so it matches the tool by the part after the server +// name. +func missingAgentTools(tools []string, associations []agenttypes.Association) bool { + for _, association := range associations { + for _, allowed := range associationTools(association.Configuration) { + if !slices.Contains(slices.Concat(mcpReadOnlyTools, mcpMutativeTools), allowed) { + continue + } + if !slices.ContainsFunc(tools, func(tool string) bool { return strings.HasSuffix(tool, "_"+allowed) }) { + return true + } + } + } + + return false +} + // associationAccount returns the AWS account of an association. Each AWS // account association has the service ID "aws", so only the configuration // tells the accounts apart. From 7263e56213c4577f95422f10443da026177e0dac Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 16:21:45 -0700 Subject: [PATCH 11/34] fix: shape the custom agent prompt for chat runs and read-only runs The skill now runs in chat, where a person approves each write. Outside chat, for example in a scheduled run, every write fails. In chat, the agent follows the skill. Outside chat, it only reads: it reports, for each experiment in flight, what is due and the message to send in chat, and it never writes. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 2 +- .../agents/experiment-orchestrator.md | 40 ++++++++----------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index e91949468..24b846974 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -29,7 +29,7 @@ const ExperimentAgentName = "experiment-orchestrator" // setup updates it without --replace-agent. When you change the prompt, add // the hash of the new text. Never remove a hash. var shippedAgentHashes = []string{ - "997dd932b26120de3ccfa19a8f0bf53c02b23d58f33a0ac9e40c188623e41f18", + "241729286c43fb2522fe98cbd2b8f6ba683ad65606fe60e8a41beb634607c497", } const ( diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md index 3c8f58fd1..977122e16 100644 --- a/internal/awsdevops/agents/experiment-orchestrator.md +++ b/internal/awsdevops/agents/experiment-orchestrator.md @@ -1,39 +1,31 @@ -You are an experimentation agent. You move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. +You are an experimentation agent. You help people move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. ## Goal -Take a goal (for example, "increase checkout conversion by 15%") from a hypothesis to the final report, and keep the experiments that are in flight moving. +Start experiments when a person asks, and tell people where their experiments stand and which decisions are due. ## Approach 1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. This prompt only says how this agent runs it. -2. Always follow the skill's "Resuming in-flight work" section first. -3. Then pick the mode for this invocation: - - **A person gave a goal in this invocation** that no open experiment record covers: start at Step 1 of the skill. - - **Anything else**, such as a scheduled run or a run with no prompt: resume only. Do not start a goal. -4. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. +2. Pick the mode for this invocation: + - **In chat:** a person can reply to you in this conversation. Follow the skill. Start an experiment, or check on experiments, as the person asks. + - **Not in chat:** anything else, such as a scheduled run or a run with a prompt from the agent page. When you are not sure, treat the run as not in chat. A write needs a person's approval in chat, so every write fails here. Follow "Not in chat" below instead of the skill's steps that write. +3. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. -## State +## Not in chat -This agent has no memory store. LaunchDarkly holds the experiment records: each flag that the skill creates has the tag `experiment-orchestration`, and its description holds the whole experiment record from the skill. Update the description whenever a record field changes. A later invocation reads only the description, so write these fields there as soon as you know them: - -- The goal, the hypothesis, and the current step -- Who confirmed that the change is deployed, and when -- The guardrail baselines at the experiment start -- The experiment key, and the start and stop dates of the experiment and of the guarded rollout - -## Implementation - -No coding agent and no CI/CD pipeline are connected to this agent. Always use the skill's path for "If no coding agent is available": create the flag, report the change to make, and wait for a person to confirm that it is deployed. +1. Find the experiments: list the projects (or use the ones that the prompt names), and in each one, list the flags with the tag `experiment-orchestration` that are not archived. A flag is in flight unless its record says "report produced". +2. For each flag in flight, read the record and the current state in the environment from the record, and apply the branches of the skill's Step 5. Where a branch says to act (stop the iteration, turn the flag off, start the guarded rollout, produce the report), report that action as due. Do not do it. +3. If a record has no project, environment, or step, report "record incomplete" for the flag. Never guess the environment. +4. If a write fails anyway, the reason is that no person can approve it here, not a setup problem. Report "needs chat approval". ## Constraints -- Change only flags that have the tag `experiment-orchestration`. Read other flags for collision checks and history, but never change them. -- Without a person in this invocation, never start a goal, accept a hypothesis, treat a change as deployed, or retry after a rollback. -- If a tool is not available or a change needs approval, report it and stop that flag. Do not work around it. +- Not in chat, never write: do not create, change, start, or stop anything, and do not update a record. +- Never start a goal that a person did not ask for. +- If a write is blocked or rejected in chat, report it and stop that flag. Do not work around it. ## Output -- When a person gave a goal: a short status, then any question that the skill tells you to ask. -- When resuming: one line for each flag in flight, with the flag key, the step, and what changed in this invocation. If nothing is in flight, say so. -- On a terminal outcome: the Step 8 report, exactly as the skill defines it. +- In chat: what the skill says to give at each step, and the report of the skill's Step 6 on an outcome. +- Not in chat: one line for each flag in flight, with the project key, the flag key, the environment, the step, and what is due. Name a guardrail regression first. For a flag with a decision due, give the message to send in chat, for example "Check on experiment `exp-checkout-conversion-cta-color` in project `checkout`". If nothing is in flight, say so. From 642f7e3d232aefc8a2a42353bfa5342c254eadf8 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Tue, 27 Oct 2026 10:39:34 -0700 Subject: [PATCH 12/34] fix: name the custom agent's tools after the server's name in AWS Since #968, setup registers a server for each agent space, named LaunchDarkly-, and a replace can rename a server. The agent's tool IDs used the fixed name LaunchDarkly, so they would not match. The run now reads the name of the registered server, and the plan shows the name that the server will have. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 40 ++++++++++++++++++++++++++-- internal/awsdevops/agent_test.go | 35 ++++++++++++++++++------ internal/awsdevops/awsdevops_test.go | 9 ++++++- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 24b846974..a2c4c4e8f 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -152,7 +152,7 @@ func agentFailure(err error) string { } func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, mcp MCPStep, skill SkillStep) AgentStep { - step := AgentStep{ServerName: mcp.name()} + step := AgentStep{ServerName: planServerName(mcp, space)} if space.Action == SpaceCreate { return step } @@ -191,6 +191,38 @@ func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space Sp return step } +// planServerName is the name that the MCP server of the space has after setup: +// a reused server keeps its name, and setup registers a new one as +// MCPServerNameFor. The run reads the real name, because a replace can rename +// the server. +func planServerName(mcp MCPStep, space SpaceStep) string { + switch { + case mcp.Action == MCPReuse && mcp.Name != "": + return mcp.Name + case space.ID == "": + // A new space gets its ID from AWS during the run. + return MCPServerNameFor("") + } + + return MCPServerNameFor(space.ID) +} + +// serverName is the name of the registered MCP server in AWS, which the tool +// IDs start with. +func serverName(ctx context.Context, clients Clients, serviceID string) (string, error) { + services, err := listServices(ctx, clients) + if err != nil { + return "", err + } + for _, service := range services { + if aws.ToString(service.ServiceId) == serviceID { + return serviceName(service), nil + } + } + + return "", fmt.Errorf("MCP server %s is not registered", serviceID) +} + // agentMetadata reads the tools and the skills of a custom agent. func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { if asset.Metadata == nil { @@ -310,7 +342,11 @@ func doAgentStep( return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", ExperimentAgentName, ExperimentSkillName, errAgentNeeds) } - tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, step.ServerName) + name, err := serverName(ctx, clients, result.MCPServiceID) + if err != nil { + return "", err + } + tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, name) if err != nil { return "", err } diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 9b1598940..94de05b68 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -47,7 +47,7 @@ func TestSetupCreatesAgentInNewSpace(t *testing.T) { metadata := metadataOf(t, in.Metadata) assert.Equal(t, awsdevops.ExperimentAgentName, metadata["name"]) assert.Equal(t, []any{"ki-created"}, metadata["skills"]) - assert.Contains(t, metadata["tools"], awsdevops.MCPServerName+"_get-flag") + assert.Contains(t, metadata["tools"], awsdevops.MCPServerNameFor("space-1")+"_get-flag") assert.NotContains(t, metadata, "memory_stores") file, ok := in.Content.(*agenttypes.AssetContentMemberFile) require.True(t, ok) @@ -61,9 +61,13 @@ func TestSetupCreatesAgentInNewSpace(t *testing.T) { // The tool IDs use the name of the server in AWS, which the customer can // change. func TestPlanSetupNamesAgentToolsAfterTheExistingServer(t *testing.T) { - agent := &fakeAgent{services: []agenttypes.RegisteredService{ - mcpService("mcp-1", "Prod-LD", awsdevops.MCPServerEndpoint), - }} + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{ + mcpService("mcp-1", "Prod-LD", awsdevops.MCPServerEndpoint), + }, + // Setup reuses only a server that the space already uses. + associations: []agenttypes.Association{mcpAssociation(awsdevops.AllowedTools()...)}, + } opts := reuseOptions() opts.LDAccessToken = "" @@ -80,7 +84,7 @@ func TestSetupKeepsCurrentAgent(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{ skillAsset("ki-ours", "experiment-orchestration"), - agentAsset("ki-agent", "experiment-orchestrator", awsdevops.AgentTools(awsdevops.MCPServerName), []string{"ki-ours"}), + agentAsset("ki-agent", "experiment-orchestrator", awsdevops.AgentTools(awsdevops.MCPServerNameFor("space-2")), []string{"ki-ours"}), }, skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, @@ -112,7 +116,9 @@ func TestSetupUpdatesAgentWithOldTools(t *testing.T) { require.Len(t, agent.updateAgents, 1) metadata := metadataOf(t, agent.updateAgents[0].Metadata) - assert.ElementsMatch(t, awsdevops.AgentTools(awsdevops.MCPServerName), metadata["tools"]) + // The update adds the tools of the server and keeps the ones the agent has. + assert.Subset(t, metadata["tools"], awsdevops.AgentTools(awsdevops.MCPServerNameFor("space-2"))) + assert.Contains(t, metadata["tools"], "LaunchDarkly_get-flag") assert.Equal(t, []any{"ki-ours"}, metadata["skills"]) // AWS keeps the keys that the request leaves out, such as memory stores. assert.NotContains(t, metadata, "memory_stores") @@ -192,7 +198,7 @@ func TestPlanSetupSkipsAgentWhenAssetsCannotBeListed(t *testing.T) { // A customer can give the agent more tools or skills. The ldcli ones are // there, so setup keeps the agent. func TestPlanSetupKeepsAgentWithAddedTools(t *testing.T) { - tools := append(awsdevops.AgentTools(awsdevops.MCPServerName), "Datadog_query-logs") + tools := append(awsdevops.AgentTools(awsdevops.MCPServerNameFor("space-2")), "Datadog_query-logs") agent := &fakeAgent{ assets: []agenttypes.Asset{ skillAsset("ki-ours", "experiment-orchestration"), @@ -224,7 +230,7 @@ func TestSetupUpdateKeepsAddedToolsAndSkills(t *testing.T) { require.Len(t, agent.updateAgents, 1) metadata := metadataOf(t, agent.updateAgents[0].Metadata) assert.Contains(t, metadata["tools"], "Datadog_query-logs") - assert.Contains(t, metadata["tools"], awsdevops.MCPServerName+"_get-flag") + assert.Contains(t, metadata["tools"], awsdevops.MCPServerNameFor("space-2")+"_get-flag") assert.Equal(t, []any{"ki-other", "ki-created"}, metadata["skills"]) } @@ -378,3 +384,16 @@ func TestStatusReportsUnreadableAssets(t *testing.T) { assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Skill.Error) assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Agent.Error) } + +// Setup registers a server for each space, so the agent's tools start with the +// name of that server. +func TestPlanSetupNamesAgentToolsAfterTheServerOfANewSpace(t *testing.T) { + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(&fakeAgent{}, newFakeIAM()), awsdevops.SetupOptions{ + AgentSpaceName: "launchdarkly", + AuthFlow: "iam", + LDAccessToken: "api-token", + }, true) + require.NoError(t, err) + + assert.Contains(t, plan.Text().Agent, "the MCP tools of server LaunchDarkly-") +} diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index 061fc08fe..5713c13e6 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -369,6 +369,13 @@ func (f *fakeAgent) RegisterService(_ context.Context, in *devopsagent.RegisterS return f.registerOutput, nil } + // AWS lists the new server, so a later step can read its name. + name := aws.ToString(in.Name) + if details, ok := in.ServiceDetails.(*agenttypes.ServiceDetailsMemberMcpserver); ok && details.Value.Name != nil { + name = aws.ToString(details.Value.Name) + } + f.services = append(f.services, mcpService("mcp-1", name, awsdevops.MCPServerEndpoint)) + return &devopsagent.RegisterServiceOutput{ServiceId: aws.String("mcp-1")}, nil } @@ -516,7 +523,7 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { []string{ "CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "RegisterService", "AssociateService", - "ListAssets", "CreateAsset", "ListAssets", "CreateAsset", + "ListAssets", "CreateAsset", "ListServices", "ListAssets", "CreateAsset", }, agent.calls, ) From 7b480c6845ae5cc494d4d7831a4404083d9afd7e Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 15:23:06 -0700 Subject: [PATCH 13/34] feat: add the experiment-orchestrator custom agent to the agent space in setup Setup now creates a custom agent that runs the experiment-orchestration skill with the LaunchDarkly MCP tools that the space allows. The tool IDs are "_", the format that AWS uses for a custom agent made in the console. An update adds the ldcli tools and skill to the agent and keeps the ones that the customer added. If someone edited the prompt, setup leaves the agent unless --replace-agent is passed. The agent is optional: if its step fails, or the MCP server or the skill is missing, setup warns and does the other steps. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/plan_print_test.go | 1 + cmd/aws_devops_agent/setup.go | 21 +- internal/awsdevops/agent.go | 369 ++++++++++++++++++ internal/awsdevops/agent_test.go | 261 +++++++++++++ .../agents/experiment-orchestrator.md | 39 ++ internal/awsdevops/awsdevops_test.go | 61 ++- internal/awsdevops/export_test.go | 10 + internal/awsdevops/setup.go | 36 +- internal/awsdevops/skill.go | 18 +- 9 files changed, 797 insertions(+), 19 deletions(-) create mode 100644 internal/awsdevops/agent.go create mode 100644 internal/awsdevops/agent_test.go create mode 100644 internal/awsdevops/agents/experiment-orchestrator.md diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index 1fbe0ae49..d723c7af0 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -66,6 +66,7 @@ func TestSetupPlanNamesRolesAndOperatorApp(t *testing.T) { assert.Contains(t, out.String(), awsdevops.OperatorAppRoleName) assert.Contains(t, out.String(), "operator app") assert.Contains(t, out.String(), "Skill: Create skill "+awsdevops.ExperimentSkillName) + assert.Contains(t, out.String(), "Custom agent: Create custom agent "+awsdevops.ExperimentAgentName) } // A canceled plan does not create the skill. diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 0720bb69e..681d0cc21 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -28,6 +28,7 @@ const ( mcpEndpointFlag = "mcp-endpoint" replaceSkillFlag = "replace-skill" addMCPToolsFlag = "add-mcp-tools" + replaceAgentFlag = "replace-agent" yesFlag = "yes" ) @@ -66,7 +67,14 @@ from a goal to a report. Every agent type in the space can load it. If the skill has the text of an older ldcli version, setup updates it. If someone edited it, setup does not change it. To overwrite the edits, pass --replace-skill. The skill is optional: if its step fails, setup warns and -does the other steps.`, +does the other steps. + +The experiment-orchestrator custom agent runs the skill with the LaunchDarkly +MCP tools that the space allows. Setup updates an agent with the prompt of an +older ldcli version. If someone edited the prompt, setup does not change the +agent. To overwrite the edits, pass --replace-agent. The agent is optional and +needs the MCP server and the skill: if its step fails, setup warns and does +the other steps.`, Args: cobra.NoArgs, PreRun: trackRun(analyticsTrackerFn), RunE: runSetup, @@ -89,6 +97,7 @@ does the other steps.`, cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") cmd.Flags().Bool(addMCPToolsFlag, false, "Add the ldcli tools that the MCP server allowlist of an existing agent space misses. "+ "Every tool gets the ldcli classification") + cmd.Flags().Bool(replaceAgentFlag, false, "Overwrite the "+awsdevops.ExperimentAgentName+" custom agent if someone edited it") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -178,6 +187,9 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.SkillError) } + if result.AgentError != "" { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.AgentError) + } return printJSON(cmd, result) } @@ -188,6 +200,9 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.SkillError) } + if result.AgentError != "" { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.AgentError) + } // Only a connect run leaves the app off, on a space that setup did not // make. The full setup would change that space, so point to the console. @@ -513,7 +528,8 @@ func confirmSetup(cmd *cobra.Command, plan awsdevops.SetupPlan, ask bool) error _, _ = fmt.Fprintf(out, " Association: %s\n", text.Association) _, _ = fmt.Fprintf(out, " Operator app: %s\n", text.OperatorApp) _, _ = fmt.Fprintf(out, " MCP server: %s\n", text.MCPServer) - _, _ = fmt.Fprintf(out, " Skill: %s\n\n", text.Skill) + _, _ = fmt.Fprintf(out, " Skill: %s\n", text.Skill) + _, _ = fmt.Fprintf(out, " Custom agent: %s\n\n", text.Agent) if !ask { return nil } @@ -642,6 +658,7 @@ func setupOptions(cmd *cobra.Command) (awsdevops.SetupOptions, error) { OpenURL: browser.OpenURL, ReplaceSkill: mustBool(cmd, replaceSkillFlag), AddMCPTools: mustBool(cmd, addMCPToolsFlag), + ReplaceAgent: mustBool(cmd, replaceAgentFlag), } idcSet := opts.IdcInstanceARN != "" diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go new file mode 100644 index 000000000..e91949468 --- /dev/null +++ b/internal/awsdevops/agent.go @@ -0,0 +1,369 @@ +package awsdevops + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/devopsagent" + "github.com/aws/aws-sdk-go-v2/service/devopsagent/document" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" +) + +// ExperimentAgentPrompt is the system prompt of the custom agent that runs the +// experimentation skill. +// +//go:embed agents/experiment-orchestrator.md +var ExperimentAgentPrompt string + +// ExperimentAgentName is the name of the custom agent. AWS keeps custom agent +// names unique in a space. +const ExperimentAgentName = "experiment-orchestrator" + +// shippedAgentHashes holds the SHA-256 of each system prompt that an ldcli +// release sent. An agent with one of these prompts has no customer edits, so +// setup updates it without --replace-agent. When you change the prompt, add +// the hash of the new text. Never remove a hash. +var shippedAgentHashes = []string{ + "997dd932b26120de3ccfa19a8f0bf53c02b23d58f33a0ac9e40c188623e41f18", +} + +const ( + agentAssetType = "custom_agent" + // agentFilePath is where the console keeps the system prompt of a custom + // agent. + agentFilePath = ExperimentAgentName + ".md" +) + +// agentTools are the IDs of the MCP tools that the agent can use. AWS names a +// tool of an MCP server "_". +func agentTools(serverName string) []string { + return toolIDs(serverName, slices.Concat(mcpReadOnlyTools, mcpMutativeTools)) +} + +func toolIDs(serverName string, tools []string) []string { + ids := make([]string, 0, len(tools)) + for _, tool := range tools { + ids = append(ids, serverName+"_"+tool) + } + + return ids +} + +// spaceAgentTools are the tool IDs of agentTools that the space allows. An +// agent can use only the tools that the association of the server allows, and +// setup keeps the allowlist of an existing association. With no association +// yet, setup makes one that allows every ldcli tool. +func spaceAgentTools(ctx context.Context, clients Clients, spaceID, serviceID, serverName string) ([]string, error) { + tools := slices.Concat(mcpReadOnlyTools, mcpMutativeTools) + if spaceID == "" || serviceID == "" { + return toolIDs(serverName, tools), nil + } + associations, err := listAssociations(ctx, clients, spaceID) + if err != nil { + return nil, err + } + for _, association := range associations { + if aws.ToString(association.ServiceId) != serviceID { + continue + } + config, ok := association.Configuration.(*agenttypes.ServiceConfigurationMemberMcpserver) + if !ok { + break + } + tools = slices.DeleteFunc(tools, func(tool string) bool { return !slices.Contains(config.Value.Tools, tool) }) + } + + return toolIDs(serverName, tools), nil +} + +// AgentAction is the planned custom agent action. +type AgentAction int + +const ( + AgentCreate AgentAction = iota + AgentKeep + // AgentUpdate gives an agent with the prompt of this or an older ldcli + // version the prompt, the tools, and the skill of this version. + AgentUpdate + // AgentLeave means that someone edited the prompt, so setup does not + // change the agent without --replace-agent. + AgentLeave + AgentReplace + // AgentSkip means that setup could not read the custom agents of the + // space. The agent is optional, so the run continues without it. + AgentSkip +) + +// AgentStep is the planned custom agent step. +type AgentStep struct { + Action AgentAction + AssetID string + // Version is the asset version at plan time. A write stops if it changed. + Version int32 + // ServerName is the name of the MCP server, which the tool IDs start with. + ServerName string + // Tools and Skills are what the existing agent has. An update adds the + // ldcli tools and skill to them, so the ones that the customer added stay. + Tools []string + Skills []string + // NoFile means that the agent has no prompt file of this name. + NoFile bool + // Err says why the step is skipped. + Err error +} + +func (s AgentStep) describe() string { + label := fmt.Sprintf("%s (%s)", ExperimentAgentName, s.AssetID) + why := "someone edited its prompt" + if s.NoFile { + why = "it has no " + agentFilePath + } + switch s.Action { + case AgentKeep: + return "Keep custom agent " + label + case AgentUpdate: + return fmt.Sprintf("Update custom agent %s to the prompt and tools of this ldcli version", label) + case AgentLeave: + return fmt.Sprintf("Leave custom agent %s unchanged because %s. Pass --replace-agent to overwrite it", label, why) + case AgentReplace: + return fmt.Sprintf("Overwrite custom agent %s, which is not the ldcli version because %s", label, why) + case AgentSkip: + return agentFailure(fmt.Errorf("Skip custom agent %s: %w", ExperimentAgentName, s.Err)) + default: + return fmt.Sprintf("Create custom agent %s with skill %s and the MCP tools of server %s", + ExperimentAgentName, ExperimentSkillName, s.ServerName) + } +} + +// agentFailure is the text of a failed agent step. A missing permission is the +// most likely cause, so it names the permissions that the step needs. +func agentFailure(err error) string { + if errorCode(err) != "AccessDeniedException" { + return err.Error() + } + + return fmt.Sprintf("%v. The custom agent step needs aidevops:ListAssets, aidevops:GetAssetFile, "+ + "aidevops:CreateAsset, and aidevops:UpdateAsset", err) +} + +func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, mcp MCPStep, skill SkillStep) AgentStep { + step := AgentStep{ServerName: mcp.name()} + if space.Action == SpaceCreate { + return step + } + asset, err := findAsset(ctx, clients, space.ID, agentAssetType, ExperimentAgentName) + if err != nil { + step.Action, step.Err = AgentSkip, err + return step + } + if asset == nil { + return step + } + step.AssetID, step.Version = aws.ToString(asset.AssetId), aws.ToInt32(asset.Version) + step.Tools, step.Skills = agentMetadata(*asset) + prompt, found, err := assetText(ctx, clients, space.ID, step.AssetID, agentFilePath) + step.NoFile = !found + if err != nil { + step.Action, step.Err = AgentSkip, err + return step + } + want, err := spaceAgentTools(ctx, clients, space.ID, mcp.ServiceID, step.ServerName) + switch { + case err != nil: + step.Action, step.Err = AgentSkip, err + case prompt == ExperimentAgentPrompt && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && + !slices.ContainsFunc(want, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): + step.Action = AgentKeep + case found && (prompt == "" || slices.Contains(shippedAgentHashes, skillHash(prompt))): + // An empty prompt has nothing to lose. + step.Action = AgentUpdate + case opts.ReplaceAgent: + step.Action = AgentReplace + default: + step.Action = AgentLeave + } + + return step +} + +// agentMetadata reads the tools and the skills of a custom agent. +func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { + if asset.Metadata == nil { + return nil, nil + } + encoded, err := asset.Metadata.MarshalSmithyDocument() + if err != nil { + return nil, nil + } + var metadata struct { + Tools []string `json:"tools"` + Skills []string `json:"skills"` + } + if err := json.Unmarshal(encoded, &metadata); err != nil { + return nil, nil + } + + return metadata.Tools, metadata.Skills +} + +// union is a followed by the items of b that a does not have. +func union(a, b []string) []string { + out := slices.Clone(a) + for _, item := range b { + if !slices.Contains(out, item) { + out = append(out, item) + } + } + + return out +} + +// checkAgent compares the space with a step that writes. A change to the +// version means that someone edited the agent after the plan. +func checkAgent(ctx context.Context, clients Clients, spaceID string, step AgentStep) error { + asset, err := findAsset(ctx, clients, spaceID, agentAssetType, ExperimentAgentName) + switch { + case err != nil: + return err + case step.Action == AgentCreate && asset != nil: + return planChanged("custom agent %s appeared after the plan", ExperimentAgentName) + case step.Action == AgentCreate: + return nil + case asset == nil || aws.ToString(asset.AssetId) != step.AssetID: + return planChanged("custom agent %s disappeared after the plan", ExperimentAgentName) + case aws.ToInt32(asset.Version) != step.Version: + return planChanged("custom agent %s changed after the plan", ExperimentAgentName) + } + + return nil +} + +// The values of SetupResult.AgentAction. +const ( + AgentCreated = "created" + AgentKept = "kept" + AgentUpdated = "updated" + AgentLeft = "left" + AgentOverwritten = "overwritten" + AgentSkipped = "skipped" + AgentFailed = "failed" +) + +// installAgent does the custom agent step after the MCP server and skill +// steps, because the agent needs both. The agent is optional, so a failure does +// not stop setup. The failure goes in result.AgentError, and the caller reports +// it. +func installAgent( + ctx context.Context, + clients Clients, + step AgentStep, + result *SetupResult, + logf func(string, ...any), +) { + result.AgentAssetID = step.AssetID + action, err := doAgentStep(ctx, clients, step, result, logf) + result.AgentAction = action + if err != nil { + result.AgentAction = AgentFailed + if step.Action == AgentSkip || errors.Is(err, ErrPlanChanged) || errors.Is(err, errAgentNeeds) { + result.AgentAction = AgentSkipped + } + result.AgentError = agentFailure(err) + } +} + +// errAgentNeeds means that an earlier step left the agent without its MCP +// server or its skill, so setup does not write the agent. +var errAgentNeeds = errors.New("the custom agent needs the MCP server and the skill") + +func doAgentStep( + ctx context.Context, + clients Clients, + step AgentStep, + result *SetupResult, + logf func(string, ...any), +) (string, error) { + switch step.Action { + case AgentKeep: + logf("Kept custom agent %s (%s)", ExperimentAgentName, step.AssetID) + + return AgentKept, nil + case AgentLeave: + logf("Left custom agent %s (%s) unchanged. Pass --replace-agent to overwrite it", ExperimentAgentName, step.AssetID) + + return AgentLeft, nil + case AgentSkip: + return "", fmt.Errorf("setup skipped custom agent %s: %w", ExperimentAgentName, step.Err) + } + switch { + case result.MCPServiceID == "": + return "", fmt.Errorf("setup skipped custom agent %s, because no MCP server is connected: %w", + ExperimentAgentName, errAgentNeeds) + case result.SkillAssetID == "" || result.SkillAction == SkillSkipped: + // A skipped skill step can keep the ID of a skill that someone + // deleted. + return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", + ExperimentAgentName, ExperimentSkillName, errAgentNeeds) + } + tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, step.ServerName) + if err != nil { + return "", err + } + if len(tools) == 0 { + return "", fmt.Errorf("setup skipped custom agent %s, because the space allows none of the ldcli MCP tools: %w", + ExperimentAgentName, errAgentNeeds) + } + + // The other steps can take minutes, so check the agent right before the + // write. + if err := checkAgent(ctx, clients, result.AgentSpaceID, step); err != nil { + return "", err + } + content := &agenttypes.AssetContentMemberFile{Value: agenttypes.AssetFileContent{ + Path: aws.String(agentFilePath), + Body: &agenttypes.AssetFileBodyMemberText{Value: ExperimentAgentPrompt}, + }} + skills := []string{result.SkillAssetID} + if step.Action == AgentUpdate { + tools, skills = union(step.Tools, tools), union(step.Skills, skills) + } + metadata := map[string]any{"tools": tools, "skills": skills} + if step.Action == AgentCreate { + metadata["name"] = ExperimentAgentName + created, err := clients.Agent.CreateAsset(ctx, &devopsagent.CreateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetType: aws.String(agentAssetType), + Content: content, + Metadata: document.NewLazyDocument(metadata), + }) + if err != nil { + return "", fmt.Errorf("unable to create custom agent %s: %w", ExperimentAgentName, err) + } + result.AgentAssetID = aws.ToString(created.Asset.AssetId) + logf("Created custom agent %s (%s)", ExperimentAgentName, result.AgentAssetID) + + return AgentCreated, nil + } + + // AWS updates only the metadata keys in the request, so the memory stores + // that the customer attached stay. + if _, err := clients.Agent.UpdateAsset(ctx, &devopsagent.UpdateAssetInput{ + AgentSpaceId: aws.String(result.AgentSpaceID), + AssetId: aws.String(step.AssetID), + Content: content, + Metadata: document.NewLazyDocument(metadata), + }); err != nil { + return "", fmt.Errorf("unable to update custom agent %s (%s): %w", ExperimentAgentName, step.AssetID, err) + } + logf("Updated custom agent %s (%s)", ExperimentAgentName, step.AssetID) + if step.Action == AgentReplace { + return AgentOverwritten, nil + } + + return AgentUpdated, nil +} diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go new file mode 100644 index 000000000..7e50eae01 --- /dev/null +++ b/internal/awsdevops/agent_test.go @@ -0,0 +1,261 @@ +package awsdevops_test + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) + +// A change to the prompt must add its hash, so the next release updates the +// agents that this release makes. +func TestShippedAgentHashesHoldTheCurrentPrompt(t *testing.T) { + assert.Contains(t, awsdevops.ShippedAgentHashes(), awsdevops.SkillHash(awsdevops.ExperimentAgentPrompt), + "add the hash of the new prompt to shippedAgentHashes") +} + +// AWS names a tool of an MCP server "_". +func TestAgentToolsStartWithTheServerName(t *testing.T) { + tools := awsdevops.AgentTools("LaunchDarkly") + + assert.Contains(t, tools, "LaunchDarkly_get-flag") + assert.Contains(t, tools, "LaunchDarkly_toggle-flag") + for _, tool := range tools { + assert.Regexp(t, `^LaunchDarkly_[a-z-]+$`, tool) + } +} + +func TestSetupCreatesAgentInNewSpace(t *testing.T) { + agent := &fakeAgent{} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), awsdevops.SetupOptions{ + AgentSpaceName: "launchdarkly", + AuthFlow: "iam", + }) + require.NoError(t, err) + + require.Len(t, agent.createAgents, 1) + in := agent.createAgents[0] + assert.Equal(t, "space-1", aws.ToString(in.AgentSpaceId)) + metadata := metadataOf(t, in.Metadata) + assert.Equal(t, awsdevops.ExperimentAgentName, metadata["name"]) + assert.Equal(t, []any{"ki-created"}, metadata["skills"]) + assert.Contains(t, metadata["tools"], awsdevops.MCPOAuthServerName+"_get-flag") + assert.NotContains(t, metadata, "memory_stores") + file, ok := in.Content.(*agenttypes.AssetContentMemberFile) + require.True(t, ok) + assert.Equal(t, "experiment-orchestrator.md", aws.ToString(file.Value.Path)) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentAgentPrompt}, file.Value.Body) + assert.Equal(t, "ki-agent-created", result.AgentAssetID) + assert.Equal(t, awsdevops.AgentCreated, result.AgentAction) + assert.Empty(t, result.AgentError) +} + +// The tool IDs use the name of the server in AWS, which the customer can +// change. +func TestPlanSetupNamesAgentToolsAfterTheExistingServer(t *testing.T) { + server := oauthMCPService("mcp-1") + server.Name = aws.String("Prod-LD") + server.AdditionalServiceDetails.(*agenttypes.AdditionalServiceDetailsMemberMcpserver).Value.Name = aws.String("Prod-LD") + agent := &fakeAgent{services: []agenttypes.RegisteredService{server}} + opts := reuseOptions() + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), opts, true) + require.NoError(t, err) + + require.Equal(t, awsdevops.MCPReuse, plan.MCP.Action) + assert.Equal(t, "Prod-LD", plan.Agent.ServerName) + assert.Equal(t, "Create custom agent experiment-orchestrator with skill experiment-orchestration "+ + "and the MCP tools of server Prod-LD", plan.Text().Agent) +} + +func TestSetupKeepsCurrentAgent(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", awsdevops.AgentTools(awsdevops.MCPOAuthServerName), []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Empty(t, agent.createAgents) + assert.Empty(t, agent.updateAgents) + assert.Equal(t, "ki-agent", result.AgentAssetID) + assert.Equal(t, awsdevops.AgentKept, result.AgentAction) +} + +// A newer ldcli allows more tools, so an agent with the ldcli prompt and +// fewer tools gets the new list. +func TestSetupUpdatesAgentWithOldTools(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", []string{awsdevops.MCPOAuthServerName + "_get-flag"}, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1) + metadata := metadataOf(t, agent.updateAgents[0].Metadata) + assert.ElementsMatch(t, awsdevops.AgentTools(awsdevops.MCPOAuthServerName), metadata["tools"]) + assert.Equal(t, []any{"ki-ours"}, metadata["skills"]) + // AWS keeps the keys that the request leaves out, such as memory stores. + assert.NotContains(t, metadata, "memory_stores") + assert.Equal(t, awsdevops.AgentUpdated, result.AgentAction) +} + +func TestSetupLeavesEditedAgent(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, + } + clients := newTestClients(agent, newFakeIAM()) + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) + assert.Contains(t, plan.Text().Agent, "--replace-agent") + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Empty(t, agent.updateAgents) + assert.Equal(t, awsdevops.AgentLeft, result.AgentAction) + assert.Empty(t, result.AgentError) +} + +func TestSetupOverwritesEditedAgentWithFlag(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, + } + opts := reuseOptions() + opts.ReplaceAgent = true + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1) + assert.Equal(t, "ki-agent", aws.ToString(agent.updateAgents[0].AssetId)) + assert.Equal(t, awsdevops.AgentOverwritten, result.AgentAction) +} + +// Without the skill, the agent could not run, so setup does not make it. +func TestSetupSkipsAgentWhenSkillFails(t *testing.T) { + agent := &fakeAgent{createAssetErr: errors.New("AccessDeniedException: not allowed")} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Empty(t, agent.createAgents) + assert.Equal(t, awsdevops.SkillFailed, result.SkillAction) + assert.Equal(t, awsdevops.AgentSkipped, result.AgentAction) + assert.Contains(t, result.AgentError, "skill experiment-orchestration is not ready") +} + +// The agent is optional, so its failure does not fail setup. +func TestSetupWarnsWhenAgentCreateFails(t *testing.T) { + agent := &fakeAgent{createAgentErr: accessDenied("aidevops:CreateAsset")} + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillCreated, result.SkillAction) + assert.Equal(t, awsdevops.AgentFailed, result.AgentAction) + assert.Contains(t, result.AgentError, "The custom agent step needs aidevops:ListAssets") +} + +func TestPlanSetupSkipsAgentWhenAssetsCannotBeListed(t *testing.T) { + agent := &fakeAgent{listAssetsErr: accessDenied("aidevops:ListAssets")} + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.AgentSkip, plan.Agent.Action) + assert.Contains(t, plan.Text().Agent, "The custom agent step needs aidevops:ListAssets") +} + +// A customer can give the agent more tools or skills. The ldcli ones are +// there, so setup keeps the agent. +func TestPlanSetupKeepsAgentWithAddedTools(t *testing.T) { + tools := append(awsdevops.AgentTools(awsdevops.MCPOAuthServerName), "Datadog_query-logs") + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", tools, []string{"ki-other", "ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.AgentKeep, plan.Agent.Action) +} + +// An update adds the ldcli tools and skill, and keeps the ones that the +// customer added. +func TestSetupUpdateKeepsAddedToolsAndSkills(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + agentAsset("ki-agent", "experiment-orchestrator", []string{"Datadog_query-logs"}, []string{"ki-other"}), + }, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + + _, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1) + metadata := metadataOf(t, agent.updateAgents[0].Metadata) + assert.Contains(t, metadata["tools"], "Datadog_query-logs") + assert.Contains(t, metadata["tools"], awsdevops.MCPOAuthServerName+"_get-flag") + assert.Equal(t, []any{"ki-other", "ki-created"}, metadata["skills"]) +} + +// Setup keeps the allowlist of an existing association, and an agent can use +// only the tools that it allows. +func TestSetupGivesAgentOnlyTheToolsTheSpaceAllows(t *testing.T) { + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{{ + AgentSpaceId: aws.String("space-2"), + AssociationId: aws.String("assoc-mcp-1"), + ServiceId: aws.String("mcp-1"), + Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{ + Value: agenttypes.MCPServerConfiguration{Tools: []string{"get-flag", "not-an-ldcli-tool"}}, + }, + }}, + } + opts := reuseOptions() + + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) + require.NoError(t, err) + + require.Len(t, agent.createAgents, 1, result.AgentError) + assert.Equal(t, []any{awsdevops.MCPServerName + "_get-flag"}, metadataOf(t, agent.createAgents[0].Metadata)["tools"]) +} + +func TestPlanSetupLeavesAgentWithoutPromptFile(t *testing.T) { + agent := &fakeAgent{assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}} + + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) + assert.Contains(t, plan.Text().Agent, "because it has no experiment-orchestrator.md") +} diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md new file mode 100644 index 000000000..3c8f58fd1 --- /dev/null +++ b/internal/awsdevops/agents/experiment-orchestrator.md @@ -0,0 +1,39 @@ +You are an experimentation agent. You move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. + +## Goal + +Take a goal (for example, "increase checkout conversion by 15%") from a hypothesis to the final report, and keep the experiments that are in flight moving. + +## Approach + +1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. This prompt only says how this agent runs it. +2. Always follow the skill's "Resuming in-flight work" section first. +3. Then pick the mode for this invocation: + - **A person gave a goal in this invocation** that no open experiment record covers: start at Step 1 of the skill. + - **Anything else**, such as a scheduled run or a run with no prompt: resume only. Do not start a goal. +4. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. + +## State + +This agent has no memory store. LaunchDarkly holds the experiment records: each flag that the skill creates has the tag `experiment-orchestration`, and its description holds the whole experiment record from the skill. Update the description whenever a record field changes. A later invocation reads only the description, so write these fields there as soon as you know them: + +- The goal, the hypothesis, and the current step +- Who confirmed that the change is deployed, and when +- The guardrail baselines at the experiment start +- The experiment key, and the start and stop dates of the experiment and of the guarded rollout + +## Implementation + +No coding agent and no CI/CD pipeline are connected to this agent. Always use the skill's path for "If no coding agent is available": create the flag, report the change to make, and wait for a person to confirm that it is deployed. + +## Constraints + +- Change only flags that have the tag `experiment-orchestration`. Read other flags for collision checks and history, but never change them. +- Without a person in this invocation, never start a goal, accept a hypothesis, treat a change as deployed, or retry after a rollback. +- If a tool is not available or a change needs approval, report it and stop that flag. Do not work around it. + +## Output + +- When a person gave a goal: a short status, then any question that the skill tells you to ask. +- When resuming: one line for each flag in flight, with the flag key, the step, and what changed in this invocation. If nothing is in flight, say so. +- On a terminal outcome: the Step 8 report, exactly as the skill defines it. diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index 80ef0e5c0..e0547f707 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -63,8 +63,10 @@ type fakeAgent struct { // paginate makes every list API return one item per page, so callers are // exercised against the NextToken AWS sends for a long list. paginate bool - // assets holds the skill assets of every space. + // assets holds the skill and custom agent assets of every space. assets []agenttypes.Asset + // agentFiles holds the prompt of each custom agent by asset ID. + agentFiles map[string]string // skillFiles holds the SKILL.md text by asset ID. skillFiles map[string]string // skillBytes holds a SKILL.md that AWS sends as bytes, by asset ID. @@ -79,6 +81,20 @@ type fakeAgent struct { updateAssociations []*devopsagent.UpdateAssociationInput updateAssociationErr error updateAssets []*devopsagent.UpdateAssetInput + // createAgents and updateAgents hold the custom agent writes, apart from + // the skill writes. + createAgents []*devopsagent.CreateAssetInput + updateAgents []*devopsagent.UpdateAssetInput + createAgentErr error +} + +// agentAsset is a custom agent asset named name with these tools and skills. +func agentAsset(id, name string, tools, skills []string) agenttypes.Asset { + return agenttypes.Asset{ + AssetId: aws.String(id), + AssetType: aws.String("custom_agent"), + Metadata: document.NewLazyDocument(map[string]any{"name": name, "tools": tools, "skills": skills}), + } } // skillAsset is a skill asset named name. @@ -92,11 +108,20 @@ func skillAsset(id, name string) agenttypes.Asset { func (f *fakeAgent) CreateAsset(_ context.Context, in *devopsagent.CreateAssetInput, _ ...func(*devopsagent.Options)) (*devopsagent.CreateAssetOutput, error) { f.calls = append(f.calls, "CreateAsset") - f.createAssets = append(f.createAssets, in) - if f.createAssetErr != nil { - return nil, f.createAssetErr + id := "ki-created" + if aws.ToString(in.AssetType) == "custom_agent" { + f.createAgents = append(f.createAgents, in) + if f.createAgentErr != nil { + return nil, f.createAgentErr + } + id = "ki-agent-created" + } else { + f.createAssets = append(f.createAssets, in) + if f.createAssetErr != nil { + return nil, f.createAssetErr + } } - asset := agenttypes.Asset{AssetId: aws.String("ki-created"), AssetType: in.AssetType, Metadata: in.Metadata} + asset := agenttypes.Asset{AssetId: aws.String(id), AssetType: in.AssetType, Metadata: in.Metadata} f.assets = append(f.assets, asset) return &devopsagent.CreateAssetOutput{Asset: &asset}, nil @@ -110,6 +135,12 @@ func (f *fakeAgent) GetAssetFile(_ context.Context, in *devopsagent.GetAssetFile if f.getFileErr != nil { return nil, f.getFileErr } + if prompt, ok := f.agentFiles[aws.ToString(in.AssetId)]; ok && aws.ToString(in.Path) == "experiment-orchestrator.md" { + return &devopsagent.GetAssetFileOutput{File: &agenttypes.AssetFile{ + Path: in.Path, + Content: &agenttypes.AssetFileBodyMemberText{Value: prompt}, + }}, nil + } if aws.ToString(in.Path) != "SKILL.md" { return nil, &agenttypes.ResourceNotFoundException{} } @@ -135,13 +166,26 @@ func (f *fakeAgent) ListAssets(_ context.Context, in *devopsagent.ListAssetsInpu if f.listAssetsErr != nil { return nil, f.listAssetsErr } - items, next := page(f.assets, in.NextToken, f.paginate) + var assets []agenttypes.Asset + for _, asset := range f.assets { + if in.AssetType == nil || aws.ToString(asset.AssetType) == aws.ToString(in.AssetType) { + assets = append(assets, asset) + } + } + items, next := page(assets, in.NextToken, f.paginate) return &devopsagent.ListAssetsOutput{Items: items, NextToken: next}, nil } func (f *fakeAgent) UpdateAsset(_ context.Context, in *devopsagent.UpdateAssetInput, _ ...func(*devopsagent.Options)) (*devopsagent.UpdateAssetOutput, error) { f.calls = append(f.calls, "UpdateAsset") + for _, asset := range f.assets { + if aws.ToString(asset.AssetId) == aws.ToString(in.AssetId) && aws.ToString(asset.AssetType) == "custom_agent" { + f.updateAgents = append(f.updateAgents, in) + + return &devopsagent.UpdateAssetOutput{Asset: &agenttypes.Asset{AssetId: in.AssetId}}, nil + } + } f.updateAssets = append(f.updateAssets, in) return &devopsagent.UpdateAssetOutput{Asset: &agenttypes.Asset{AssetId: in.AssetId}}, nil @@ -461,10 +505,11 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { // The plan and the first check look only for the server of the // account, because a new space has no server. The check before the // MCP step also looks for the server of the space, which exists by - // then. The skill is last. + // then. The skill and the custom agent are last. []string{ "ListServices", "ListServices", "CreateAgentSpace", "AssociateService", "EnableOperatorApp", - "ListServices", "ListServices", "RegisterService", "AssociateService", "ListAssets", "CreateAsset", + "ListServices", "ListServices", "RegisterService", "AssociateService", + "ListAssets", "CreateAsset", "ListAssets", "CreateAsset", }, agent.calls, ) diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 789550eed..082f94e38 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -2,6 +2,7 @@ package awsdevops import ( "context" + "slices" "time" ) @@ -52,3 +53,12 @@ func SetOAuthPollInterval(d time.Duration) func() { return func() { oauthPollInterval = old } } + +// AllowedTools are the tools that setup allows on the MCP server. +func AllowedTools() []string { return slices.Concat(mcpReadOnlyTools, mcpMutativeTools) } + +// ShippedAgentHashes are the hashes of the shipped custom agent prompts. +func ShippedAgentHashes() []string { return shippedAgentHashes } + +// AgentTools are the tool IDs that setup gives the custom agent. +func AgentTools(serverName string) []string { return agentTools(serverName) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index d19fbd5c3..f025f283c 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -146,6 +146,9 @@ type SetupOptions struct { // server association misses. AWS does not send back the classification of // each tool, so the rewrite gives every tool the ldcli classification. AddMCPTools bool + // ReplaceAgent overwrites the experiment-orchestrator custom agent if + // someone edited its prompt. + ReplaceAgent bool // Logf, when set, reports progress as each step completes. Logf func(format string, args ...any) @@ -175,7 +178,13 @@ type SetupResult struct { SkillAction string `json:"skillAction,omitempty"` // SkillError is set if the skill step failed or was skipped. The skill is // optional, so the rest of the setup ran. - SkillError string `json:"skillError,omitempty"` + SkillError string `json:"skillError,omitempty"` + AgentAssetID string `json:"agentAssetId,omitempty"` + // AgentAction is what the custom agent step did, for example "created". + AgentAction string `json:"agentAction,omitempty"` + // AgentError is set if the custom agent step failed or was skipped. The + // agent is optional, so the rest of the setup ran. + AgentError string `json:"agentError,omitempty"` } // ErrPlanChanged means AWS changed after the plan. @@ -445,6 +454,7 @@ type SetupPlan struct { AuthFlow string MCP MCPStep Skill SkillStep + Agent AgentStep } // PlanText is the plan as lines for a person to read. @@ -457,11 +467,18 @@ type PlanText struct { OperatorApp string MCPServer string Skill string + Agent string } // Text describes each step of the plan. func (p SetupPlan) Text() PlanText { - text := PlanText{Scope: "Full setup", AgentSpace: p.Space.describe(), MCPServer: p.MCP.describe(), Skill: p.Skill.describe()} + text := PlanText{ + Scope: "Full setup", + AgentSpace: p.Space.describe(), + MCPServer: p.MCP.describe(), + Skill: p.Skill.describe(), + Agent: p.Agent.describe(), + } if p.Space.Action == SpaceConnect { text.Scope = "Connect the MCP server only. To also update the IAM roles, " + "account association, and operator app of the space, run setup again with --full-setup" @@ -558,6 +575,7 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } plan.Skill = planSkill(ctx, clients, opts, plan.Space) + plan.Agent = planAgent(ctx, clients, opts, plan.Space, plan.MCP, plan.Skill) if !provision { // Read the app, so the run does not print a URL that does not work. @@ -793,9 +811,13 @@ func checkMCPServiceID(ctx context.Context, clients Clients, id, endpoint string return "", fmt.Errorf("ID %s names no registered MCP server", id) } -// name is the name of the existing server in AWS, or MCPServerName when AWS -// sent none. +// name is the name of the server in AWS. A new server gets +// MCPOAuthServerName. An existing server without a name in AWS gets +// MCPServerName. func (m MCPStep) name() string { + if m.Action == MCPRegister { + return MCPOAuthServerName + } if m.Name != "" { return m.Name } @@ -1005,9 +1027,10 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } - // The skill goes last, so a failure of the optional step cannot stop the - // MCP server step. + // The skill and the agent go last, so a failure of the optional steps + // cannot stop the MCP server step. installSkill(ctx, clients, plan.Skill, &result, logf) + installAgent(ctx, clients, plan.Agent, &result, logf) return result, nil } @@ -1067,6 +1090,7 @@ func ConnectPlan(ctx context.Context, clients Clients, opts SetupOptions, plan S return result, err } installSkill(ctx, clients, plan.Skill, &result, logf) + installAgent(ctx, clients, plan.Agent, &result, logf) return result, nil } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 959e1a067..79dda7ec2 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -177,7 +177,13 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp // findSkill returns the skill with this name in the space, or nil if there is // none. AWS keeps skill names unique in a space. func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { - assets, err := listAssets(ctx, clients, spaceID, skillAssetType) + return findAsset(ctx, clients, spaceID, skillAssetType, name) +} + +// findAsset returns the asset of this type and name in the space, or nil if +// there is none. +func findAsset(ctx context.Context, clients Clients, spaceID, assetType, name string) (*agenttypes.Asset, error) { + assets, err := listAssets(ctx, clients, spaceID, assetType) if err != nil { return nil, err } @@ -214,10 +220,16 @@ func assetName(asset agenttypes.Asset) string { // skillText returns the SKILL.md of the skill. found is false if the skill // has no SKILL.md. func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (text string, found bool, err error) { + return assetText(ctx, clients, spaceID, assetID, skillFilePath) +} + +// assetText returns the file at path in the asset. found is false if the +// asset has no such file. +func assetText(ctx context.Context, clients Clients, spaceID, assetID, path string) (text string, found bool, err error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), AssetId: aws.String(assetID), - Path: aws.String(skillFilePath), + Path: aws.String(path), }) var missing *agenttypes.ResourceNotFoundException switch { @@ -225,7 +237,7 @@ func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (t return "", false, nil case err != nil: // The caller names the skill. - return "", false, fmt.Errorf("unable to read %s of asset %s: %w", skillFilePath, assetID, err) + return "", false, fmt.Errorf("unable to read %s of asset %s: %w", path, assetID, err) case file.File == nil: return "", false, nil } From 32b9e64923155c228321796c64dbfb5d2229ed16 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 15:31:35 -0700 Subject: [PATCH 14/34] feat: show the skill and the custom agent in status status now reports the experiment-orchestration skill and the experiment-orchestrator custom agent of each agent space: whether each is the ldcli version, an older one that setup updates, edited, without its file, or missing. For the agent, it also gives the number of tools and whether the skill is attached. An agent with the ldcli prompt but without the skill or an allowed tool is outdated, because setup updates it. Like the operator app, an asset that status cannot read does not stop it. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/status.go | 51 ++++++ cmd/aws_devops_agent/status_fixture_test.go | 12 +- cmd/aws_devops_agent/status_test.go | 53 +++++++ .../testdata/status-partial.json | 12 +- internal/awsdevops/agent_test.go | 108 +++++++++++++ internal/awsdevops/status.go | 149 ++++++++++++++++++ 6 files changed, 383 insertions(+), 2 deletions(-) diff --git a/cmd/aws_devops_agent/status.go b/cmd/aws_devops_agent/status.go index a11ceba44..66a26ae2c 100644 --- a/cmd/aws_devops_agent/status.go +++ b/cmd/aws_devops_agent/status.go @@ -107,11 +107,62 @@ func printStatus(out io.Writer, status awsdevops.Status) { _, _ = fmt.Fprintf(out, " %s: %s\n", associationLabel(status, association), associationState(association)) } + if space.Skill != nil { + _, _ = fmt.Fprintf(out, " Skill %s: %s\n", + assetLabel(awsdevops.ExperimentSkillName, *space.Skill), + assetLine(*space.Skill, "SKILL.md", "--replace-skill")) + } + if space.Agent != nil { + _, _ = fmt.Fprintf(out, " Custom agent %s: %s\n", + assetLabel(awsdevops.ExperimentAgentName, space.Agent.AssetStatus), agentLine(*space.Agent)) + } } printConsoleURL(out, status.Region) } +// assetLabel is the name of the skill or the agent, with its asset ID. +func assetLabel(name string, asset awsdevops.AssetStatus) string { + if asset.AssetID == "" { + return name + } + + return name + " (" + asset.AssetID + ")" +} + +// assetLine gives the state of the skill or the agent, and what setup does +// with it. file is the file that setup writes, and flag is the setup flag that +// overwrites an edited one. +func assetLine(asset awsdevops.AssetStatus, file, flag string) string { + switch asset.State { + case awsdevops.AssetCurrent: + return "this ldcli version" + case awsdevops.AssetOutdated: + return "an older ldcli version. Setup updates it" + case awsdevops.AssetEdited: + return "edited. Setup leaves it unless you pass " + flag + case awsdevops.AssetNoFile: + return "has no " + file + ". Setup leaves it unless you pass " + flag + case awsdevops.AssetMissing: + return "missing. " + setupHint + } + + return "unknown (" + asset.Error + ")" +} + +func agentLine(agent awsdevops.AgentStatus) string { + line := assetLine(agent.AssetStatus, awsdevops.ExperimentAgentName+".md", "--replace-agent") + if agent.State == awsdevops.AssetMissing || agent.AssetID == "" { + return line + } + line += fmt.Sprintf(". It has %d tools", len(agent.Tools)) + if agent.SkillAttached != nil && !*agent.SkillAttached { + line += " and not skill " + awsdevops.ExperimentSkillName + } + + return line +} + func printConsoleURL(out io.Writer, region string) { _, _ = fmt.Fprintf(out, "\nAWS DevOps Agent console: %s\n", awsdevops.ConsoleURL(region)) } diff --git a/cmd/aws_devops_agent/status_fixture_test.go b/cmd/aws_devops_agent/status_fixture_test.go index 3e727c7bb..04c21c274 100644 --- a/cmd/aws_devops_agent/status_fixture_test.go +++ b/cmd/aws_devops_agent/status_fixture_test.go @@ -1,6 +1,10 @@ package awsdevopsagent -import "github.com/launchdarkly/ldcli/internal/awsdevops" +import ( + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) func realisticStatus() awsdevops.Status { return awsdevops.Status{ @@ -40,6 +44,12 @@ func partialStatus() awsdevops.Status { AgentSpaceID: "as-prod01", Name: "Production", OperatorAppError: "AccessDeniedException", + Skill: &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetEdited}, + Agent: &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetOutdated}, + Tools: []string{"LaunchDarkly_get-flag"}, + SkillAttached: aws.Bool(false), + }, }}, MissingSetup: []string{"IAM role " + awsdevops.OperatorAppRoleName + " does not exist"}, } diff --git a/cmd/aws_devops_agent/status_test.go b/cmd/aws_devops_agent/status_test.go index d5ba6245e..19a18e936 100644 --- a/cmd/aws_devops_agent/status_test.go +++ b/cmd/aws_devops_agent/status_test.go @@ -1,6 +1,8 @@ package awsdevopsagent import ( + "github.com/aws/aws-sdk-go-v2/aws" + "bytes" "os" "strings" @@ -328,3 +330,54 @@ func TestStatusShowsAnOperatorAppThatIsOff(t *testing.T) { assert.Contains(t, out, "Operator app: off") assert.NotContains(t, out, "aidevops.global.app.aws") } + +func TestStatusShowsSkillAndAgent(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetCurrent} + status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetEdited}, + Tools: []string{"LaunchDarkly_get-flag", "LaunchDarkly_toggle-flag"}, + SkillAttached: aws.Bool(true), + } + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + + assert.Equal(t, " Skill experiment-orchestration (ki-skill): this ldcli version", lineWith(t, section, "Skill")) + assert.Equal(t, + " Custom agent experiment-orchestrator (ki-agent): edited. Setup leaves it unless you pass --replace-agent. "+ + "It has 2 tools", + lineWith(t, section, "Custom agent")) +} + +func TestStatusShowsMissingAgentAndAgentWithoutSkill(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{AssetStatus: awsdevops.AssetStatus{State: awsdevops.AssetMissing}} + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + assert.Equal(t, " Custom agent experiment-orchestrator: missing. "+setupHint, lineWith(t, section, "Custom agent")) + + status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetOutdated}, + SkillAttached: aws.Bool(false), + } + section = spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + assert.Equal(t, + " Custom agent experiment-orchestrator (ki-agent): an older ldcli version. Setup updates it. "+ + "It has 0 tools and not skill experiment-orchestration", + lineWith(t, section, "Custom agent")) +} + +func TestStatusShowsSkillWithoutFile(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetNoFile} + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + + assert.Equal(t, " Skill experiment-orchestration (ki-skill): has no SKILL.md. Setup leaves it unless you pass "+ + "--replace-skill", lineWith(t, section, "Skill")) +} + +func TestStatusShowsUnreadableSkill(t *testing.T) { + status := realisticStatus() + status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{Error: "AccessDeniedException"} + section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) + + assert.Equal(t, " Skill experiment-orchestration: unknown (AccessDeniedException)", lineWith(t, section, "Skill")) +} diff --git a/cmd/aws_devops_agent/testdata/status-partial.json b/cmd/aws_devops_agent/testdata/status-partial.json index d54be8ce6..e7772fd0a 100644 --- a/cmd/aws_devops_agent/testdata/status-partial.json +++ b/cmd/aws_devops_agent/testdata/status-partial.json @@ -18,7 +18,17 @@ "name": "Production", "operatorAppUrl": "", "operatorAppEnabled": false, - "operatorAppError": "AccessDeniedException" + "operatorAppError": "AccessDeniedException", + "skill": { + "assetId": "ki-skill", + "state": "edited" + }, + "agent": { + "assetId": "ki-agent", + "state": "outdated", + "tools": ["LaunchDarkly_get-flag"], + "skillAttached": false + } } ], "missingSetup": [ diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 7e50eae01..7bea69ade 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -259,3 +259,111 @@ func TestPlanSetupLeavesAgentWithoutPromptFile(t *testing.T) { assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) assert.Contains(t, plan.Text().Agent, "because it has no experiment-orchestrator.md") } + +func TestStatusReportsSkillAndAgent(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", []string{"LaunchDarkly_get-flag"}, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + require.Len(t, status.AgentSpaces, 1) + space := status.AgentSpaces[0] + assert.Equal(t, &awsdevops.AssetStatus{AssetID: "ki-ours", State: awsdevops.AssetCurrent}, space.Skill) + assert.Equal(t, &awsdevops.AgentStatus{ + AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetEdited}, + Tools: []string{"LaunchDarkly_get-flag"}, + SkillAttached: aws.Bool(true), + }, space.Agent) +} + +// Setup updates an agent with the ldcli prompt but without the skill or an +// allowed tool, so status calls it outdated. +func TestStatusReportsAgentWithoutToolsAsOutdated(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + associations: []agenttypes.Association{{ + AgentSpaceId: aws.String("space-1"), + ServiceId: aws.String("mcp-1"), + Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{ + Value: agenttypes.MCPServerConfiguration{Tools: []string{"get-flag", "toggle-flag"}}, + }, + }}, + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", []string{"Prod-LD_get-flag"}, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + clients := newTestClients(agent, newFakeIAM()) + + status, err := awsdevops.GetStatus(context.Background(), clients, "") + require.NoError(t, err) + assert.Equal(t, awsdevops.AssetOutdated, status.AgentSpaces[0].Agent.State) + + agent.assets[1] = agentAsset("ki-agent", "experiment-orchestrator", + []string{"Prod-LD_get-flag", "Prod-LD_toggle-flag"}, []string{"ki-ours"}) + status, err = awsdevops.GetStatus(context.Background(), clients, "") + require.NoError(t, err) + assert.Equal(t, awsdevops.AssetCurrent, status.AgentSpaces[0].Agent.State) +} + +func TestStatusReportsMissingSkillAndAgent(t *testing.T) { + agent := &fakeAgent{agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}} + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, awsdevops.AssetMissing, status.AgentSpaces[0].Skill.State) + assert.Equal(t, awsdevops.AssetMissing, status.AgentSpaces[0].Agent.State) + assert.Nil(t, status.AgentSpaces[0].Agent.SkillAttached) +} + +// A skill without SKILL.md is not ours, and setup leaves it. +func TestStatusReportsSkillWithoutFile(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, &awsdevops.AssetStatus{AssetID: "ki-ours", State: awsdevops.AssetNoFile}, status.AgentSpaces[0].Skill) +} + +// An older ldcli prompt is one that setup updates. +func TestStatusReportsOutdatedAgent(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": ""}, + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, awsdevops.AssetOutdated, status.AgentSpaces[0].Agent.State) +} + +// Like the operator app, assets that status cannot read do not stop it. +func TestStatusReportsUnreadableAssets(t *testing.T) { + agent := &fakeAgent{ + agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, + listAssetsErr: accessDenied("aidevops:ListAssets"), + } + + status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") + require.NoError(t, err) + + assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Skill.Error) + assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Agent.Error) +} diff --git a/internal/awsdevops/status.go b/internal/awsdevops/status.go index 76a8acae0..2b35b0f98 100644 --- a/internal/awsdevops/status.go +++ b/internal/awsdevops/status.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "slices" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/devopsagent" @@ -47,6 +49,44 @@ type AgentSpaceStatus struct { // and OperatorAppEnabled is false. OperatorAppError string `json:"operatorAppError,omitempty"` Associations []AssociationStatus `json:"associations,omitempty"` + // Skill is the experiment-orchestration skill, and Agent is the + // experiment-orchestrator custom agent. + Skill *AssetStatus `json:"skill,omitempty"` + Agent *AgentStatus `json:"agent,omitempty"` +} + +// The values of AssetStatus.State. +const ( + // AssetCurrent is the text of this ldcli version. + AssetCurrent = "current" + // AssetOutdated is the text of an older ldcli version, or an empty file. + // Setup updates it. + AssetOutdated = "outdated" + // AssetEdited is a text that no ldcli version sent. Setup leaves it + // without --replace-skill or --replace-agent. + AssetEdited = "edited" + // AssetNoFile is an asset without the file that setup writes, for example + // a skill that someone made from a PDF. Setup leaves it too. + AssetNoFile = "no-file" + AssetMissing = "missing" +) + +// AssetStatus is the state of an asset that setup makes. +type AssetStatus struct { + AssetID string `json:"assetId,omitempty"` + State string `json:"state,omitempty"` + // Error is set if status could not read the asset, for example without + // aidevops:ListAssets. State is then empty. + Error string `json:"error,omitempty"` +} + +// AgentStatus is the state of the custom agent. +type AgentStatus struct { + AssetStatus + Tools []string `json:"tools,omitempty"` + // SkillAttached says whether the agent loads the experiment-orchestration + // skill. It is nil if status could not read the agent or the skill. + SkillAttached *bool `json:"skillAttached,omitempty"` } type AssociationStatus struct { @@ -150,12 +190,121 @@ func GetStatus(ctx context.Context, clients Clients, agentSpaceID string) (Statu }) } + spaceStatus.Skill = skillStatus(ctx, clients, id) + spaceStatus.Agent = agentStatus(ctx, clients, id, *spaceStatus.Skill, associations) + status.AgentSpaces = append(status.AgentSpaces, spaceStatus) } return status, nil } +// assetState compares the text of an asset with the text of this ldcli +// version and the texts of older versions. +func assetState(text string, found bool, current string, shipped []string) string { + switch { + case !found: + return AssetNoFile + case text == current: + return AssetCurrent + case found && (text == "" || slices.Contains(shipped, skillHash(text))): + return AssetOutdated + } + + return AssetEdited +} + +// skillStatus reads the skill. Like the operator app, an unreadable skill does +// not stop status. +func skillStatus(ctx context.Context, clients Clients, spaceID string) *AssetStatus { + asset, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) + switch { + case err != nil: + return &AssetStatus{Error: errorCode(err)} + case asset == nil: + return &AssetStatus{State: AssetMissing} + } + status := &AssetStatus{AssetID: aws.ToString(asset.AssetId)} + text, found, err := skillText(ctx, clients, spaceID, status.AssetID) + switch { + case err != nil: + status.Error = errorCode(err) + case !found && !stillListed(ctx, clients, spaceID, skillAssetType, ExperimentSkillName, status.AssetID): + // AWS also sends ResourceNotFoundException for an asset that someone + // deleted after the list, and setup then makes a new one. + return &AssetStatus{State: AssetMissing} + default: + status.State = assetState(text, found, ExperimentSkill, shippedSkillHashes) + } + + return status +} + +// stillListed reports whether the asset is still in the space. +func stillListed(ctx context.Context, clients Clients, spaceID, assetType, name, assetID string) bool { + asset, err := findAsset(ctx, clients, spaceID, assetType, name) + + return err != nil || (asset != nil && aws.ToString(asset.AssetId) == assetID) +} + +// agentStatus reads the custom agent. An agent with the ldcli prompt is +// current only if it also has the skill and the ldcli tools that the space +// allows, because setup updates it otherwise. +func agentStatus( + ctx context.Context, + clients Clients, + spaceID string, + skill AssetStatus, + associations []agenttypes.Association, +) *AgentStatus { + asset, err := findAsset(ctx, clients, spaceID, agentAssetType, ExperimentAgentName) + switch { + case err != nil: + return &AgentStatus{AssetStatus: AssetStatus{Error: errorCode(err)}} + case asset == nil: + return &AgentStatus{AssetStatus: AssetStatus{State: AssetMissing}} + } + status := &AgentStatus{AssetStatus: AssetStatus{AssetID: aws.ToString(asset.AssetId)}} + var skills []string + status.Tools, skills = agentMetadata(*asset) + if skill.Error == "" { + attached := skill.AssetID != "" && slices.Contains(skills, skill.AssetID) + status.SkillAttached = &attached + } + prompt, found, err := assetText(ctx, clients, spaceID, status.AssetID, agentFilePath) + if err != nil { + status.Error = errorCode(err) + + return status + } + status.State = assetState(prompt, found, ExperimentAgentPrompt, shippedAgentHashes) + if status.State == AssetCurrent && (status.SkillAttached == nil || !*status.SkillAttached || + missingAgentTools(status.Tools, associations)) { + status.State = AssetOutdated + } + + return status +} + +// missingAgentTools reports whether the agent lacks an ldcli tool that an MCP +// server association of the space allows. status does not know which server is +// the LaunchDarkly one, so it matches the tool by the part after the server +// name. +func missingAgentTools(tools []string, associations []agenttypes.Association) bool { + for _, association := range associations { + for _, allowed := range associationTools(association.Configuration) { + if !slices.Contains(slices.Concat(mcpReadOnlyTools, mcpMutativeTools), allowed) { + continue + } + if !slices.ContainsFunc(tools, func(tool string) bool { return strings.HasSuffix(tool, "_"+allowed) }) { + return true + } + } + } + + return false +} + // associationAccount returns the AWS account of an association. Each AWS // account association has the service ID "aws", so only the configuration // tells the accounts apart. From bcd71050b44362edc7365353935919c656303856 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Mon, 26 Oct 2026 16:21:45 -0700 Subject: [PATCH 15/34] fix: shape the custom agent prompt for chat runs and read-only runs The skill now runs in chat, where a person approves each write. Outside chat, for example in a scheduled run, every write fails. In chat, the agent follows the skill. Outside chat, it only reads: it reports, for each experiment in flight, what is due and the message to send in chat, and it never writes. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 2 +- .../agents/experiment-orchestrator.md | 40 ++++++++----------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index e91949468..24b846974 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -29,7 +29,7 @@ const ExperimentAgentName = "experiment-orchestrator" // setup updates it without --replace-agent. When you change the prompt, add // the hash of the new text. Never remove a hash. var shippedAgentHashes = []string{ - "997dd932b26120de3ccfa19a8f0bf53c02b23d58f33a0ac9e40c188623e41f18", + "241729286c43fb2522fe98cbd2b8f6ba683ad65606fe60e8a41beb634607c497", } const ( diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md index 3c8f58fd1..977122e16 100644 --- a/internal/awsdevops/agents/experiment-orchestrator.md +++ b/internal/awsdevops/agents/experiment-orchestrator.md @@ -1,39 +1,31 @@ -You are an experimentation agent. You move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. +You are an experimentation agent. You help people move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. ## Goal -Take a goal (for example, "increase checkout conversion by 15%") from a hypothesis to the final report, and keep the experiments that are in flight moving. +Start experiments when a person asks, and tell people where their experiments stand and which decisions are due. ## Approach 1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. This prompt only says how this agent runs it. -2. Always follow the skill's "Resuming in-flight work" section first. -3. Then pick the mode for this invocation: - - **A person gave a goal in this invocation** that no open experiment record covers: start at Step 1 of the skill. - - **Anything else**, such as a scheduled run or a run with no prompt: resume only. Do not start a goal. -4. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. +2. Pick the mode for this invocation: + - **In chat:** a person can reply to you in this conversation. Follow the skill. Start an experiment, or check on experiments, as the person asks. + - **Not in chat:** anything else, such as a scheduled run or a run with a prompt from the agent page. When you are not sure, treat the run as not in chat. A write needs a person's approval in chat, so every write fails here. Follow "Not in chat" below instead of the skill's steps that write. +3. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. -## State +## Not in chat -This agent has no memory store. LaunchDarkly holds the experiment records: each flag that the skill creates has the tag `experiment-orchestration`, and its description holds the whole experiment record from the skill. Update the description whenever a record field changes. A later invocation reads only the description, so write these fields there as soon as you know them: - -- The goal, the hypothesis, and the current step -- Who confirmed that the change is deployed, and when -- The guardrail baselines at the experiment start -- The experiment key, and the start and stop dates of the experiment and of the guarded rollout - -## Implementation - -No coding agent and no CI/CD pipeline are connected to this agent. Always use the skill's path for "If no coding agent is available": create the flag, report the change to make, and wait for a person to confirm that it is deployed. +1. Find the experiments: list the projects (or use the ones that the prompt names), and in each one, list the flags with the tag `experiment-orchestration` that are not archived. A flag is in flight unless its record says "report produced". +2. For each flag in flight, read the record and the current state in the environment from the record, and apply the branches of the skill's Step 5. Where a branch says to act (stop the iteration, turn the flag off, start the guarded rollout, produce the report), report that action as due. Do not do it. +3. If a record has no project, environment, or step, report "record incomplete" for the flag. Never guess the environment. +4. If a write fails anyway, the reason is that no person can approve it here, not a setup problem. Report "needs chat approval". ## Constraints -- Change only flags that have the tag `experiment-orchestration`. Read other flags for collision checks and history, but never change them. -- Without a person in this invocation, never start a goal, accept a hypothesis, treat a change as deployed, or retry after a rollback. -- If a tool is not available or a change needs approval, report it and stop that flag. Do not work around it. +- Not in chat, never write: do not create, change, start, or stop anything, and do not update a record. +- Never start a goal that a person did not ask for. +- If a write is blocked or rejected in chat, report it and stop that flag. Do not work around it. ## Output -- When a person gave a goal: a short status, then any question that the skill tells you to ask. -- When resuming: one line for each flag in flight, with the flag key, the step, and what changed in this invocation. If nothing is in flight, say so. -- On a terminal outcome: the Step 8 report, exactly as the skill defines it. +- In chat: what the skill says to give at each step, and the report of the skill's Step 6 on an outcome. +- Not in chat: one line for each flag in flight, with the project key, the flag key, the environment, the step, and what is due. Name a guardrail regression first. For a flag with a decision due, give the message to send in chat, for example "Check on experiment `exp-checkout-conversion-cta-color` in project `checkout`". If nothing is in flight, say so. From 9b175012e0cbceea98f8fb0d7067115bf0af1423 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Tue, 27 Oct 2026 10:39:34 -0700 Subject: [PATCH 16/34] fix: name the custom agent's tools after the server's name in AWS Since #968, setup registers a server for each agent space, named LaunchDarkly-, and a replace can rename a server. The agent's tool IDs used the fixed name LaunchDarkly, so they would not match. The run now reads the name of the registered server, and the plan shows the name that the server will have. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 23 ++++++++++++++++++++++- internal/awsdevops/agent_test.go | 12 ++++++++++++ internal/awsdevops/awsdevops_test.go | 12 ++++++++++-- internal/awsdevops/runplan_test.go | 3 ++- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 24b846974..24a87a4e5 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -191,6 +191,23 @@ func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space Sp return step } +// serverName is the name of the registered MCP server in AWS, which the tool +// IDs start with. The run reads it, because the name in the plan can be out of +// date. +func serverName(ctx context.Context, clients Clients, serviceID string) (string, error) { + services, err := listServices(ctx, clients) + if err != nil { + return "", err + } + for _, service := range services { + if aws.ToString(service.ServiceId) == serviceID { + return serviceName(service), nil + } + } + + return "", fmt.Errorf("MCP server %s is not registered", serviceID) +} + // agentMetadata reads the tools and the skills of a custom agent. func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { if asset.Metadata == nil { @@ -310,7 +327,11 @@ func doAgentStep( return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", ExperimentAgentName, ExperimentSkillName, errAgentNeeds) } - tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, step.ServerName) + name, err := serverName(ctx, clients, result.MCPServiceID) + if err != nil { + return "", err + } + tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, name) if err != nil { return "", err } diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 7bea69ade..a46c37374 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -367,3 +367,15 @@ func TestStatusReportsUnreadableAssets(t *testing.T) { assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Skill.Error) assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Agent.Error) } + +// Setup registers one OAuth server for the account, so the agent's tools start +// with the name of that server. +func TestPlanSetupNamesAgentToolsAfterANewServer(t *testing.T) { + plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(&fakeAgent{}, newFakeIAM()), awsdevops.SetupOptions{ + AgentSpaceName: "launchdarkly", + AuthFlow: "iam", + }, true) + require.NoError(t, err) + + assert.Contains(t, plan.Text().Agent, "the MCP tools of server "+awsdevops.MCPOAuthServerName) +} diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index e0547f707..2a242d116 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -362,6 +362,13 @@ func (f *fakeAgent) RegisterService(_ context.Context, in *devopsagent.RegisterS return f.registerOutput, nil } + // AWS lists the new server, so a later step can read its name. + name := aws.ToString(in.Name) + if details, ok := in.ServiceDetails.(*agenttypes.ServiceDetailsMemberMcpserver); ok && details.Value.Name != nil { + name = aws.ToString(details.Value.Name) + } + f.services = append(f.services, mcpService("mcp-1", name, awsdevops.MCPServerEndpoint)) + return &devopsagent.RegisterServiceOutput{ServiceId: aws.String("mcp-1")}, nil } @@ -505,11 +512,12 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { // The plan and the first check look only for the server of the // account, because a new space has no server. The check before the // MCP step also looks for the server of the space, which exists by - // then. The skill and the custom agent are last. + // then. The skill and the custom agent are last. The custom agent + // step reads the name of the server. []string{ "ListServices", "ListServices", "CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "ListServices", "RegisterService", "AssociateService", - "ListAssets", "CreateAsset", "ListAssets", "CreateAsset", + "ListAssets", "CreateAsset", "ListServices", "ListAssets", "CreateAsset", }, agent.calls, ) diff --git a/internal/awsdevops/runplan_test.go b/internal/awsdevops/runplan_test.go index 5d7f288be..5a294f1fd 100644 --- a/internal/awsdevops/runplan_test.go +++ b/internal/awsdevops/runplan_test.go @@ -963,7 +963,8 @@ func TestOAuthPollsUntilTheServerAppears(t *testing.T) { } polls := 0 agent.onListServices = func() { - if !opened { + // The custom agent step lists the servers again after the poll. + if !opened || polls == 3 { return } polls++ From f4b32130b0253b3020352ca688217116b029c5e4 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:16:06 -0700 Subject: [PATCH 17/34] fix: make the custom agent prompt a thin layer on the experiment skill The prompt now loads the experiment-orchestration skill from #970 and follows it as is. It adds only the LaunchDarkly MCP tools and one rule: never start a goal that a person did not ask for. The old prompt never shipped, so its hash is replaced. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 2 +- .../agents/experiment-orchestrator.md | 23 ++++--------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 24a87a4e5..75a650a64 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -29,7 +29,7 @@ const ExperimentAgentName = "experiment-orchestrator" // setup updates it without --replace-agent. When you change the prompt, add // the hash of the new text. Never remove a hash. var shippedAgentHashes = []string{ - "241729286c43fb2522fe98cbd2b8f6ba683ad65606fe60e8a41beb634607c497", + "cad2fb81776dee1029bf7bf8d7b466e869be23590f918433de698922be001c08", } const ( diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md index 977122e16..d9f491b7b 100644 --- a/internal/awsdevops/agents/experiment-orchestrator.md +++ b/internal/awsdevops/agents/experiment-orchestrator.md @@ -1,31 +1,18 @@ -You are an experimentation agent. You help people move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. +You are an experimentation agent. You move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. ## Goal -Start experiments when a person asks, and tell people where their experiments stand and which decisions are due. +Run the `experiment-orchestration` skill for the goal that a person gives you. ## Approach -1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. This prompt only says how this agent runs it. -2. Pick the mode for this invocation: - - **In chat:** a person can reply to you in this conversation. Follow the skill. Start an experiment, or check on experiments, as the person asks. - - **Not in chat:** anything else, such as a scheduled run or a run with a prompt from the agent page. When you are not sure, treat the run as not in chat. A write needs a person's approval in chat, so every write fails here. Follow "Not in chat" below instead of the skill's steps that write. -3. Use the LaunchDarkly MCP tools for every flag, experiment, metric, and guarded rollout action. - -## Not in chat - -1. Find the experiments: list the projects (or use the ones that the prompt names), and in each one, list the flags with the tag `experiment-orchestration` that are not archived. A flag is in flight unless its record says "report produced". -2. For each flag in flight, read the record and the current state in the environment from the record, and apply the branches of the skill's Step 5. Where a branch says to act (stop the iteration, turn the flag off, start the guarded rollout, produce the report), report that action as due. Do not do it. -3. If a record has no project, environment, or step, report "record incomplete" for the flag. Never guess the environment. -4. If a write fails anyway, the reason is that no person can approve it here, not a setup problem. Report "needs chat approval". +1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. +2. Use the LaunchDarkly MCP tools for every flag, metric, experiment, approval request, and guarded rollout action. ## Constraints -- Not in chat, never write: do not create, change, start, or stop anything, and do not update a record. - Never start a goal that a person did not ask for. -- If a write is blocked or rejected in chat, report it and stop that flag. Do not work around it. ## Output -- In chat: what the skill says to give at each step, and the report of the skill's Step 6 on an outcome. -- Not in chat: one line for each flag in flight, with the project key, the flag key, the environment, the step, and what is due. Name a guardrail regression first. For a flag with a decision due, give the message to send in chat, for example "Check on experiment `exp-checkout-conversion-cta-color` in project `checkout`". If nothing is in flight, say so. +What the skill says to give at each step, and the report of the skill's final step on every terminal outcome. From 5fe9961e97fa2ef682e53312e7742597f10000cd Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:29:13 -0700 Subject: [PATCH 18/34] fix: shorten the custom agent prompt to one paragraph The prompt now has one paragraph that loads the experiment-orchestration skill and follows it. The help text for the custom agent is shorter. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/setup.go | 7 ++----- internal/awsdevops/agent.go | 2 +- .../agents/experiment-orchestrator.md | 19 +------------------ 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 681d0cc21..a02098c51 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -70,11 +70,8 @@ edited it, setup does not change it. To overwrite the edits, pass does the other steps. The experiment-orchestrator custom agent runs the skill with the LaunchDarkly -MCP tools that the space allows. Setup updates an agent with the prompt of an -older ldcli version. If someone edited the prompt, setup does not change the -agent. To overwrite the edits, pass --replace-agent. The agent is optional and -needs the MCP server and the skill: if its step fails, setup warns and does -the other steps.`, +MCP tools that the space allows. The agent is optional and needs the MCP +server and the skill: if its step fails, setup warns and does the other steps.`, Args: cobra.NoArgs, PreRun: trackRun(analyticsTrackerFn), RunE: runSetup, diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 75a650a64..0c68448ed 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -29,7 +29,7 @@ const ExperimentAgentName = "experiment-orchestrator" // setup updates it without --replace-agent. When you change the prompt, add // the hash of the new text. Never remove a hash. var shippedAgentHashes = []string{ - "cad2fb81776dee1029bf7bf8d7b466e869be23590f918433de698922be001c08", + "d3d2124c29e290df9cadd38e9bae2619ac8622cabb180fa0ad4fc59d5358dee5", } const ( diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md index d9f491b7b..058e3e8ce 100644 --- a/internal/awsdevops/agents/experiment-orchestrator.md +++ b/internal/awsdevops/agents/experiment-orchestrator.md @@ -1,18 +1 @@ -You are an experimentation agent. You move a product goal through a LaunchDarkly experiment and, when the experiment wins, a guarded rollout. - -## Goal - -Run the `experiment-orchestration` skill for the goal that a person gives you. - -## Approach - -1. Load the `experiment-orchestration` skill and follow it. The skill is the source of truth for the procedure. -2. Use the LaunchDarkly MCP tools for every flag, metric, experiment, approval request, and guarded rollout action. - -## Constraints - -- Never start a goal that a person did not ask for. - -## Output - -What the skill says to give at each step, and the report of the skill's final step on every terminal outcome. +You are an experimentation agent that moves a product goal through a LaunchDarkly experiment and, if the experiment wins, a rollout. Load the experiment-orchestration skill and follow it as the source of truth for the procedure, using the LaunchDarkly MCP tools for every flag, metric, experiment, approval request, and rollout action. Never start a goal that a person did not ask for. At each step, give the output the skill specifies, and on every terminal outcome, deliver the report defined by the skill's final step. From a020cc5ca2b47b7cbdbc771a07fea29d72534bf3 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:31:40 -0700 Subject: [PATCH 19/34] fix: remove --replace-agent from aws-devops-agent setup Setup never overwrites an edited prompt of the custom agent. The plan and the log say that setup leaves the agent, and that a person can delete the agent in the AWS console and run setup again to get the ldcli version. Status no longer names a flag for an edited agent. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/setup.go | 3 --- cmd/aws_devops_agent/status.go | 16 +++++++++++--- cmd/aws_devops_agent/status_test.go | 2 +- internal/awsdevops/agent.go | 34 ++++++++++++----------------- internal/awsdevops/agent_test.go | 18 +-------------- internal/awsdevops/setup.go | 3 --- internal/awsdevops/status.go | 3 ++- 7 files changed, 31 insertions(+), 48 deletions(-) diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index a02098c51..2afd609a2 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -28,7 +28,6 @@ const ( mcpEndpointFlag = "mcp-endpoint" replaceSkillFlag = "replace-skill" addMCPToolsFlag = "add-mcp-tools" - replaceAgentFlag = "replace-agent" yesFlag = "yes" ) @@ -94,7 +93,6 @@ server and the skill: if its step fails, setup warns and does the other steps.`, cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") cmd.Flags().Bool(addMCPToolsFlag, false, "Add the ldcli tools that the MCP server allowlist of an existing agent space misses. "+ "Every tool gets the ldcli classification") - cmd.Flags().Bool(replaceAgentFlag, false, "Overwrite the "+awsdevops.ExperimentAgentName+" custom agent if someone edited it") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -655,7 +653,6 @@ func setupOptions(cmd *cobra.Command) (awsdevops.SetupOptions, error) { OpenURL: browser.OpenURL, ReplaceSkill: mustBool(cmd, replaceSkillFlag), AddMCPTools: mustBool(cmd, addMCPToolsFlag), - ReplaceAgent: mustBool(cmd, replaceAgentFlag), } idcSet := opts.IdcInstanceARN != "" diff --git a/cmd/aws_devops_agent/status.go b/cmd/aws_devops_agent/status.go index 66a26ae2c..a6452fbec 100644 --- a/cmd/aws_devops_agent/status.go +++ b/cmd/aws_devops_agent/status.go @@ -140,9 +140,9 @@ func assetLine(asset awsdevops.AssetStatus, file, flag string) string { case awsdevops.AssetOutdated: return "an older ldcli version. Setup updates it" case awsdevops.AssetEdited: - return "edited. Setup leaves it unless you pass " + flag + return "edited. Setup leaves it" + leaveHint(flag) case awsdevops.AssetNoFile: - return "has no " + file + ". Setup leaves it unless you pass " + flag + return "has no " + file + ". Setup leaves it" + leaveHint(flag) case awsdevops.AssetMissing: return "missing. " + setupHint } @@ -150,8 +150,18 @@ func assetLine(asset awsdevops.AssetStatus, file, flag string) string { return "unknown (" + asset.Error + ")" } +// leaveHint names the flag that overwrites an asset. The custom agent has no +// such flag. +func leaveHint(flag string) string { + if flag == "" { + return "" + } + + return " unless you pass " + flag +} + func agentLine(agent awsdevops.AgentStatus) string { - line := assetLine(agent.AssetStatus, awsdevops.ExperimentAgentName+".md", "--replace-agent") + line := assetLine(agent.AssetStatus, awsdevops.ExperimentAgentName+".md", "") if agent.State == awsdevops.AssetMissing || agent.AssetID == "" { return line } diff --git a/cmd/aws_devops_agent/status_test.go b/cmd/aws_devops_agent/status_test.go index 19a18e936..98f549cb9 100644 --- a/cmd/aws_devops_agent/status_test.go +++ b/cmd/aws_devops_agent/status_test.go @@ -343,7 +343,7 @@ func TestStatusShowsSkillAndAgent(t *testing.T) { assert.Equal(t, " Skill experiment-orchestration (ki-skill): this ldcli version", lineWith(t, section, "Skill")) assert.Equal(t, - " Custom agent experiment-orchestrator (ki-agent): edited. Setup leaves it unless you pass --replace-agent. "+ + " Custom agent experiment-orchestrator (ki-agent): edited. Setup leaves it. "+ "It has 2 tools", lineWith(t, section, "Custom agent")) } diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 0c68448ed..1897419e7 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -26,12 +26,16 @@ const ExperimentAgentName = "experiment-orchestrator" // shippedAgentHashes holds the SHA-256 of each system prompt that an ldcli // release sent. An agent with one of these prompts has no customer edits, so -// setup updates it without --replace-agent. When you change the prompt, add +// setup updates it. When you change the prompt, add // the hash of the new text. Never remove a hash. var shippedAgentHashes = []string{ "d3d2124c29e290df9cadd38e9bae2619ac8622cabb180fa0ad4fc59d5358dee5", } +// agentResetHint says how to get the ldcli prompt back. Setup never +// overwrites an edited prompt. +const agentResetHint = "To use the ldcli version, delete the agent in the AWS console and run setup again" + const ( agentAssetType = "custom_agent" // agentFilePath is where the console keeps the system prompt of a custom @@ -91,9 +95,8 @@ const ( // version the prompt, the tools, and the skill of this version. AgentUpdate // AgentLeave means that someone edited the prompt, so setup does not - // change the agent without --replace-agent. + // change the agent. AgentLeave - AgentReplace // AgentSkip means that setup could not read the custom agents of the // space. The agent is optional, so the run continues without it. AgentSkip @@ -129,9 +132,7 @@ func (s AgentStep) describe() string { case AgentUpdate: return fmt.Sprintf("Update custom agent %s to the prompt and tools of this ldcli version", label) case AgentLeave: - return fmt.Sprintf("Leave custom agent %s unchanged because %s. Pass --replace-agent to overwrite it", label, why) - case AgentReplace: - return fmt.Sprintf("Overwrite custom agent %s, which is not the ldcli version because %s", label, why) + return fmt.Sprintf("Leave custom agent %s unchanged because %s. %s", label, why, agentResetHint) case AgentSkip: return agentFailure(fmt.Errorf("Skip custom agent %s: %w", ExperimentAgentName, s.Err)) default: @@ -182,8 +183,6 @@ func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space Sp case found && (prompt == "" || slices.Contains(shippedAgentHashes, skillHash(prompt))): // An empty prompt has nothing to lose. step.Action = AgentUpdate - case opts.ReplaceAgent: - step.Action = AgentReplace default: step.Action = AgentLeave } @@ -262,13 +261,12 @@ func checkAgent(ctx context.Context, clients Clients, spaceID string, step Agent // The values of SetupResult.AgentAction. const ( - AgentCreated = "created" - AgentKept = "kept" - AgentUpdated = "updated" - AgentLeft = "left" - AgentOverwritten = "overwritten" - AgentSkipped = "skipped" - AgentFailed = "failed" + AgentCreated = "created" + AgentKept = "kept" + AgentUpdated = "updated" + AgentLeft = "left" + AgentSkipped = "skipped" + AgentFailed = "failed" ) // installAgent does the custom agent step after the MCP server and skill @@ -311,7 +309,7 @@ func doAgentStep( return AgentKept, nil case AgentLeave: - logf("Left custom agent %s (%s) unchanged. Pass --replace-agent to overwrite it", ExperimentAgentName, step.AssetID) + logf("Left custom agent %s (%s) unchanged. %s", ExperimentAgentName, step.AssetID, agentResetHint) return AgentLeft, nil case AgentSkip: @@ -382,9 +380,5 @@ func doAgentStep( return "", fmt.Errorf("unable to update custom agent %s (%s): %w", ExperimentAgentName, step.AssetID, err) } logf("Updated custom agent %s (%s)", ExperimentAgentName, step.AssetID) - if step.Action == AgentReplace { - return AgentOverwritten, nil - } - return AgentUpdated, nil } diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index a46c37374..d66fa4e87 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -128,7 +128,7 @@ func TestSetupLeavesEditedAgent(t *testing.T) { plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) assert.Equal(t, awsdevops.AgentLeave, plan.Agent.Action) - assert.Contains(t, plan.Text().Agent, "--replace-agent") + assert.Contains(t, plan.Text().Agent, "delete the agent in the AWS console") result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) @@ -137,22 +137,6 @@ func TestSetupLeavesEditedAgent(t *testing.T) { assert.Empty(t, result.AgentError) } -func TestSetupOverwritesEditedAgentWithFlag(t *testing.T) { - agent := &fakeAgent{ - assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, - agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, - } - opts := reuseOptions() - opts.ReplaceAgent = true - - result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), opts) - require.NoError(t, err) - - require.Len(t, agent.updateAgents, 1) - assert.Equal(t, "ki-agent", aws.ToString(agent.updateAgents[0].AssetId)) - assert.Equal(t, awsdevops.AgentOverwritten, result.AgentAction) -} - // Without the skill, the agent could not run, so setup does not make it. func TestSetupSkipsAgentWhenSkillFails(t *testing.T) { agent := &fakeAgent{createAssetErr: errors.New("AccessDeniedException: not allowed")} diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index f025f283c..978ad4584 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -146,9 +146,6 @@ type SetupOptions struct { // server association misses. AWS does not send back the classification of // each tool, so the rewrite gives every tool the ldcli classification. AddMCPTools bool - // ReplaceAgent overwrites the experiment-orchestrator custom agent if - // someone edited its prompt. - ReplaceAgent bool // Logf, when set, reports progress as each step completes. Logf func(format string, args ...any) diff --git a/internal/awsdevops/status.go b/internal/awsdevops/status.go index 2b35b0f98..154ae72b5 100644 --- a/internal/awsdevops/status.go +++ b/internal/awsdevops/status.go @@ -63,7 +63,8 @@ const ( // Setup updates it. AssetOutdated = "outdated" // AssetEdited is a text that no ldcli version sent. Setup leaves it - // without --replace-skill or --replace-agent. + // without --replace-skill, and it never changes an + // edited custom agent. AssetEdited = "edited" // AssetNoFile is an asset without the file that setup writes, for example // a skill that someone made from a PDF. Setup leaves it too. From 1c987288c0fa56026fd6f3bcbd3a2477769f8b49 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:37:19 -0700 Subject: [PATCH 20/34] fix: leave the skill and the custom agent out of status Status does not show the experiment-orchestration skill or the experiment-orchestrator custom agent. Status is the same as on main. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/status.go | 61 ------- cmd/aws_devops_agent/status_fixture_test.go | 12 +- cmd/aws_devops_agent/status_test.go | 53 ------- .../testdata/status-partial.json | 12 +- internal/awsdevops/agent_test.go | 103 ------------ internal/awsdevops/status.go | 150 ------------------ 6 files changed, 2 insertions(+), 389 deletions(-) diff --git a/cmd/aws_devops_agent/status.go b/cmd/aws_devops_agent/status.go index a6452fbec..a11ceba44 100644 --- a/cmd/aws_devops_agent/status.go +++ b/cmd/aws_devops_agent/status.go @@ -107,72 +107,11 @@ func printStatus(out io.Writer, status awsdevops.Status) { _, _ = fmt.Fprintf(out, " %s: %s\n", associationLabel(status, association), associationState(association)) } - if space.Skill != nil { - _, _ = fmt.Fprintf(out, " Skill %s: %s\n", - assetLabel(awsdevops.ExperimentSkillName, *space.Skill), - assetLine(*space.Skill, "SKILL.md", "--replace-skill")) - } - if space.Agent != nil { - _, _ = fmt.Fprintf(out, " Custom agent %s: %s\n", - assetLabel(awsdevops.ExperimentAgentName, space.Agent.AssetStatus), agentLine(*space.Agent)) - } } printConsoleURL(out, status.Region) } -// assetLabel is the name of the skill or the agent, with its asset ID. -func assetLabel(name string, asset awsdevops.AssetStatus) string { - if asset.AssetID == "" { - return name - } - - return name + " (" + asset.AssetID + ")" -} - -// assetLine gives the state of the skill or the agent, and what setup does -// with it. file is the file that setup writes, and flag is the setup flag that -// overwrites an edited one. -func assetLine(asset awsdevops.AssetStatus, file, flag string) string { - switch asset.State { - case awsdevops.AssetCurrent: - return "this ldcli version" - case awsdevops.AssetOutdated: - return "an older ldcli version. Setup updates it" - case awsdevops.AssetEdited: - return "edited. Setup leaves it" + leaveHint(flag) - case awsdevops.AssetNoFile: - return "has no " + file + ". Setup leaves it" + leaveHint(flag) - case awsdevops.AssetMissing: - return "missing. " + setupHint - } - - return "unknown (" + asset.Error + ")" -} - -// leaveHint names the flag that overwrites an asset. The custom agent has no -// such flag. -func leaveHint(flag string) string { - if flag == "" { - return "" - } - - return " unless you pass " + flag -} - -func agentLine(agent awsdevops.AgentStatus) string { - line := assetLine(agent.AssetStatus, awsdevops.ExperimentAgentName+".md", "") - if agent.State == awsdevops.AssetMissing || agent.AssetID == "" { - return line - } - line += fmt.Sprintf(". It has %d tools", len(agent.Tools)) - if agent.SkillAttached != nil && !*agent.SkillAttached { - line += " and not skill " + awsdevops.ExperimentSkillName - } - - return line -} - func printConsoleURL(out io.Writer, region string) { _, _ = fmt.Fprintf(out, "\nAWS DevOps Agent console: %s\n", awsdevops.ConsoleURL(region)) } diff --git a/cmd/aws_devops_agent/status_fixture_test.go b/cmd/aws_devops_agent/status_fixture_test.go index 04c21c274..3e727c7bb 100644 --- a/cmd/aws_devops_agent/status_fixture_test.go +++ b/cmd/aws_devops_agent/status_fixture_test.go @@ -1,10 +1,6 @@ package awsdevopsagent -import ( - "github.com/aws/aws-sdk-go-v2/aws" - - "github.com/launchdarkly/ldcli/internal/awsdevops" -) +import "github.com/launchdarkly/ldcli/internal/awsdevops" func realisticStatus() awsdevops.Status { return awsdevops.Status{ @@ -44,12 +40,6 @@ func partialStatus() awsdevops.Status { AgentSpaceID: "as-prod01", Name: "Production", OperatorAppError: "AccessDeniedException", - Skill: &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetEdited}, - Agent: &awsdevops.AgentStatus{ - AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetOutdated}, - Tools: []string{"LaunchDarkly_get-flag"}, - SkillAttached: aws.Bool(false), - }, }}, MissingSetup: []string{"IAM role " + awsdevops.OperatorAppRoleName + " does not exist"}, } diff --git a/cmd/aws_devops_agent/status_test.go b/cmd/aws_devops_agent/status_test.go index 98f549cb9..d5ba6245e 100644 --- a/cmd/aws_devops_agent/status_test.go +++ b/cmd/aws_devops_agent/status_test.go @@ -1,8 +1,6 @@ package awsdevopsagent import ( - "github.com/aws/aws-sdk-go-v2/aws" - "bytes" "os" "strings" @@ -330,54 +328,3 @@ func TestStatusShowsAnOperatorAppThatIsOff(t *testing.T) { assert.Contains(t, out, "Operator app: off") assert.NotContains(t, out, "aidevops.global.app.aws") } - -func TestStatusShowsSkillAndAgent(t *testing.T) { - status := realisticStatus() - status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetCurrent} - status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{ - AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetEdited}, - Tools: []string{"LaunchDarkly_get-flag", "LaunchDarkly_toggle-flag"}, - SkillAttached: aws.Bool(true), - } - section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) - - assert.Equal(t, " Skill experiment-orchestration (ki-skill): this ldcli version", lineWith(t, section, "Skill")) - assert.Equal(t, - " Custom agent experiment-orchestrator (ki-agent): edited. Setup leaves it. "+ - "It has 2 tools", - lineWith(t, section, "Custom agent")) -} - -func TestStatusShowsMissingAgentAndAgentWithoutSkill(t *testing.T) { - status := realisticStatus() - status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{AssetStatus: awsdevops.AssetStatus{State: awsdevops.AssetMissing}} - section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) - assert.Equal(t, " Custom agent experiment-orchestrator: missing. "+setupHint, lineWith(t, section, "Custom agent")) - - status.AgentSpaces[0].Agent = &awsdevops.AgentStatus{ - AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetOutdated}, - SkillAttached: aws.Bool(false), - } - section = spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) - assert.Equal(t, - " Custom agent experiment-orchestrator (ki-agent): an older ldcli version. Setup updates it. "+ - "It has 0 tools and not skill experiment-orchestration", - lineWith(t, section, "Custom agent")) -} - -func TestStatusShowsSkillWithoutFile(t *testing.T) { - status := realisticStatus() - status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{AssetID: "ki-skill", State: awsdevops.AssetNoFile} - section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) - - assert.Equal(t, " Skill experiment-orchestration (ki-skill): has no SKILL.md. Setup leaves it unless you pass "+ - "--replace-skill", lineWith(t, section, "Skill")) -} - -func TestStatusShowsUnreadableSkill(t *testing.T) { - status := realisticStatus() - status.AgentSpaces[0].Skill = &awsdevops.AssetStatus{Error: "AccessDeniedException"} - section := spaceSection(t, renderStatus(status), status.AgentSpaces[0].Name) - - assert.Equal(t, " Skill experiment-orchestration: unknown (AccessDeniedException)", lineWith(t, section, "Skill")) -} diff --git a/cmd/aws_devops_agent/testdata/status-partial.json b/cmd/aws_devops_agent/testdata/status-partial.json index e7772fd0a..d54be8ce6 100644 --- a/cmd/aws_devops_agent/testdata/status-partial.json +++ b/cmd/aws_devops_agent/testdata/status-partial.json @@ -18,17 +18,7 @@ "name": "Production", "operatorAppUrl": "", "operatorAppEnabled": false, - "operatorAppError": "AccessDeniedException", - "skill": { - "assetId": "ki-skill", - "state": "edited" - }, - "agent": { - "assetId": "ki-agent", - "state": "outdated", - "tools": ["LaunchDarkly_get-flag"], - "skillAttached": false - } + "operatorAppError": "AccessDeniedException" } ], "missingSetup": [ diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index d66fa4e87..2e1614870 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -244,114 +244,11 @@ func TestPlanSetupLeavesAgentWithoutPromptFile(t *testing.T) { assert.Contains(t, plan.Text().Agent, "because it has no experiment-orchestrator.md") } -func TestStatusReportsSkillAndAgent(t *testing.T) { - agent := &fakeAgent{ - agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, - assets: []agenttypes.Asset{ - skillAsset("ki-ours", "experiment-orchestration"), - agentAsset("ki-agent", "experiment-orchestrator", []string{"LaunchDarkly_get-flag"}, []string{"ki-ours"}), - }, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, - agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, - } - - status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") - require.NoError(t, err) - - require.Len(t, status.AgentSpaces, 1) - space := status.AgentSpaces[0] - assert.Equal(t, &awsdevops.AssetStatus{AssetID: "ki-ours", State: awsdevops.AssetCurrent}, space.Skill) - assert.Equal(t, &awsdevops.AgentStatus{ - AssetStatus: awsdevops.AssetStatus{AssetID: "ki-agent", State: awsdevops.AssetEdited}, - Tools: []string{"LaunchDarkly_get-flag"}, - SkillAttached: aws.Bool(true), - }, space.Agent) -} - // Setup updates an agent with the ldcli prompt but without the skill or an // allowed tool, so status calls it outdated. -func TestStatusReportsAgentWithoutToolsAsOutdated(t *testing.T) { - agent := &fakeAgent{ - agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, - associations: []agenttypes.Association{{ - AgentSpaceId: aws.String("space-1"), - ServiceId: aws.String("mcp-1"), - Configuration: &agenttypes.ServiceConfigurationMemberMcpserver{ - Value: agenttypes.MCPServerConfiguration{Tools: []string{"get-flag", "toggle-flag"}}, - }, - }}, - assets: []agenttypes.Asset{ - skillAsset("ki-ours", "experiment-orchestration"), - agentAsset("ki-agent", "experiment-orchestrator", []string{"Prod-LD_get-flag"}, []string{"ki-ours"}), - }, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, - agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, - } - clients := newTestClients(agent, newFakeIAM()) - - status, err := awsdevops.GetStatus(context.Background(), clients, "") - require.NoError(t, err) - assert.Equal(t, awsdevops.AssetOutdated, status.AgentSpaces[0].Agent.State) - - agent.assets[1] = agentAsset("ki-agent", "experiment-orchestrator", - []string{"Prod-LD_get-flag", "Prod-LD_toggle-flag"}, []string{"ki-ours"}) - status, err = awsdevops.GetStatus(context.Background(), clients, "") - require.NoError(t, err) - assert.Equal(t, awsdevops.AssetCurrent, status.AgentSpaces[0].Agent.State) -} - -func TestStatusReportsMissingSkillAndAgent(t *testing.T) { - agent := &fakeAgent{agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}} - - status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") - require.NoError(t, err) - - assert.Equal(t, awsdevops.AssetMissing, status.AgentSpaces[0].Skill.State) - assert.Equal(t, awsdevops.AssetMissing, status.AgentSpaces[0].Agent.State) - assert.Nil(t, status.AgentSpaces[0].Agent.SkillAttached) -} - // A skill without SKILL.md is not ours, and setup leaves it. -func TestStatusReportsSkillWithoutFile(t *testing.T) { - agent := &fakeAgent{ - agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, - assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - } - - status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") - require.NoError(t, err) - - assert.Equal(t, &awsdevops.AssetStatus{AssetID: "ki-ours", State: awsdevops.AssetNoFile}, status.AgentSpaces[0].Skill) -} - // An older ldcli prompt is one that setup updates. -func TestStatusReportsOutdatedAgent(t *testing.T) { - agent := &fakeAgent{ - agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, - assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, - agentFiles: map[string]string{"ki-agent": ""}, - } - - status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") - require.NoError(t, err) - - assert.Equal(t, awsdevops.AssetOutdated, status.AgentSpaces[0].Agent.State) -} - // Like the operator app, assets that status cannot read do not stop it. -func TestStatusReportsUnreadableAssets(t *testing.T) { - agent := &fakeAgent{ - agentSpaces: []agenttypes.AgentSpace{namedSpace("space-1", "launchdarkly")}, - listAssetsErr: accessDenied("aidevops:ListAssets"), - } - - status, err := awsdevops.GetStatus(context.Background(), newTestClients(agent, newFakeIAM()), "") - require.NoError(t, err) - - assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Skill.Error) - assert.Equal(t, "AccessDeniedException", status.AgentSpaces[0].Agent.Error) -} - // Setup registers one OAuth server for the account, so the agent's tools start // with the name of that server. func TestPlanSetupNamesAgentToolsAfterANewServer(t *testing.T) { diff --git a/internal/awsdevops/status.go b/internal/awsdevops/status.go index 154ae72b5..76a8acae0 100644 --- a/internal/awsdevops/status.go +++ b/internal/awsdevops/status.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "slices" - "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/devopsagent" @@ -49,45 +47,6 @@ type AgentSpaceStatus struct { // and OperatorAppEnabled is false. OperatorAppError string `json:"operatorAppError,omitempty"` Associations []AssociationStatus `json:"associations,omitempty"` - // Skill is the experiment-orchestration skill, and Agent is the - // experiment-orchestrator custom agent. - Skill *AssetStatus `json:"skill,omitempty"` - Agent *AgentStatus `json:"agent,omitempty"` -} - -// The values of AssetStatus.State. -const ( - // AssetCurrent is the text of this ldcli version. - AssetCurrent = "current" - // AssetOutdated is the text of an older ldcli version, or an empty file. - // Setup updates it. - AssetOutdated = "outdated" - // AssetEdited is a text that no ldcli version sent. Setup leaves it - // without --replace-skill, and it never changes an - // edited custom agent. - AssetEdited = "edited" - // AssetNoFile is an asset without the file that setup writes, for example - // a skill that someone made from a PDF. Setup leaves it too. - AssetNoFile = "no-file" - AssetMissing = "missing" -) - -// AssetStatus is the state of an asset that setup makes. -type AssetStatus struct { - AssetID string `json:"assetId,omitempty"` - State string `json:"state,omitempty"` - // Error is set if status could not read the asset, for example without - // aidevops:ListAssets. State is then empty. - Error string `json:"error,omitempty"` -} - -// AgentStatus is the state of the custom agent. -type AgentStatus struct { - AssetStatus - Tools []string `json:"tools,omitempty"` - // SkillAttached says whether the agent loads the experiment-orchestration - // skill. It is nil if status could not read the agent or the skill. - SkillAttached *bool `json:"skillAttached,omitempty"` } type AssociationStatus struct { @@ -191,121 +150,12 @@ func GetStatus(ctx context.Context, clients Clients, agentSpaceID string) (Statu }) } - spaceStatus.Skill = skillStatus(ctx, clients, id) - spaceStatus.Agent = agentStatus(ctx, clients, id, *spaceStatus.Skill, associations) - status.AgentSpaces = append(status.AgentSpaces, spaceStatus) } return status, nil } -// assetState compares the text of an asset with the text of this ldcli -// version and the texts of older versions. -func assetState(text string, found bool, current string, shipped []string) string { - switch { - case !found: - return AssetNoFile - case text == current: - return AssetCurrent - case found && (text == "" || slices.Contains(shipped, skillHash(text))): - return AssetOutdated - } - - return AssetEdited -} - -// skillStatus reads the skill. Like the operator app, an unreadable skill does -// not stop status. -func skillStatus(ctx context.Context, clients Clients, spaceID string) *AssetStatus { - asset, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) - switch { - case err != nil: - return &AssetStatus{Error: errorCode(err)} - case asset == nil: - return &AssetStatus{State: AssetMissing} - } - status := &AssetStatus{AssetID: aws.ToString(asset.AssetId)} - text, found, err := skillText(ctx, clients, spaceID, status.AssetID) - switch { - case err != nil: - status.Error = errorCode(err) - case !found && !stillListed(ctx, clients, spaceID, skillAssetType, ExperimentSkillName, status.AssetID): - // AWS also sends ResourceNotFoundException for an asset that someone - // deleted after the list, and setup then makes a new one. - return &AssetStatus{State: AssetMissing} - default: - status.State = assetState(text, found, ExperimentSkill, shippedSkillHashes) - } - - return status -} - -// stillListed reports whether the asset is still in the space. -func stillListed(ctx context.Context, clients Clients, spaceID, assetType, name, assetID string) bool { - asset, err := findAsset(ctx, clients, spaceID, assetType, name) - - return err != nil || (asset != nil && aws.ToString(asset.AssetId) == assetID) -} - -// agentStatus reads the custom agent. An agent with the ldcli prompt is -// current only if it also has the skill and the ldcli tools that the space -// allows, because setup updates it otherwise. -func agentStatus( - ctx context.Context, - clients Clients, - spaceID string, - skill AssetStatus, - associations []agenttypes.Association, -) *AgentStatus { - asset, err := findAsset(ctx, clients, spaceID, agentAssetType, ExperimentAgentName) - switch { - case err != nil: - return &AgentStatus{AssetStatus: AssetStatus{Error: errorCode(err)}} - case asset == nil: - return &AgentStatus{AssetStatus: AssetStatus{State: AssetMissing}} - } - status := &AgentStatus{AssetStatus: AssetStatus{AssetID: aws.ToString(asset.AssetId)}} - var skills []string - status.Tools, skills = agentMetadata(*asset) - if skill.Error == "" { - attached := skill.AssetID != "" && slices.Contains(skills, skill.AssetID) - status.SkillAttached = &attached - } - prompt, found, err := assetText(ctx, clients, spaceID, status.AssetID, agentFilePath) - if err != nil { - status.Error = errorCode(err) - - return status - } - status.State = assetState(prompt, found, ExperimentAgentPrompt, shippedAgentHashes) - if status.State == AssetCurrent && (status.SkillAttached == nil || !*status.SkillAttached || - missingAgentTools(status.Tools, associations)) { - status.State = AssetOutdated - } - - return status -} - -// missingAgentTools reports whether the agent lacks an ldcli tool that an MCP -// server association of the space allows. status does not know which server is -// the LaunchDarkly one, so it matches the tool by the part after the server -// name. -func missingAgentTools(tools []string, associations []agenttypes.Association) bool { - for _, association := range associations { - for _, allowed := range associationTools(association.Configuration) { - if !slices.Contains(slices.Concat(mcpReadOnlyTools, mcpMutativeTools), allowed) { - continue - } - if !slices.ContainsFunc(tools, func(tool string) bool { return strings.HasSuffix(tool, "_"+allowed) }) { - return true - } - } - } - - return false -} - // associationAccount returns the AWS account of an association. Each AWS // account association has the service ID "aws", so only the configuration // tells the accounts apart. From 549cb8120b4e8842a9adcd3f31fdd9d883899388 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:47:19 -0700 Subject: [PATCH 21/34] chore: remove comments that repeat the code The comments that stay give a reason that the code does not show. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 70 ++++++---------------------- internal/awsdevops/agent_test.go | 21 --------- internal/awsdevops/awsdevops_test.go | 19 +++----- internal/awsdevops/export_test.go | 3 -- internal/awsdevops/setup.go | 9 ++-- internal/awsdevops/skill.go | 4 -- 6 files changed, 23 insertions(+), 103 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 1897419e7..a69e29b82 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -14,37 +14,25 @@ import ( agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" ) -// ExperimentAgentPrompt is the system prompt of the custom agent that runs the -// experimentation skill. -// //go:embed agents/experiment-orchestrator.md var ExperimentAgentPrompt string -// ExperimentAgentName is the name of the custom agent. AWS keeps custom agent -// names unique in a space. const ExperimentAgentName = "experiment-orchestrator" -// shippedAgentHashes holds the SHA-256 of each system prompt that an ldcli -// release sent. An agent with one of these prompts has no customer edits, so -// setup updates it. When you change the prompt, add -// the hash of the new text. Never remove a hash. +// shippedAgentHashes are the prompts that ldcli releases sent. When you change +// the prompt, add its hash. Never remove a hash. var shippedAgentHashes = []string{ "d3d2124c29e290df9cadd38e9bae2619ac8622cabb180fa0ad4fc59d5358dee5", } -// agentResetHint says how to get the ldcli prompt back. Setup never -// overwrites an edited prompt. const agentResetHint = "To use the ldcli version, delete the agent in the AWS console and run setup again" const ( agentAssetType = "custom_agent" - // agentFilePath is where the console keeps the system prompt of a custom - // agent. - agentFilePath = ExperimentAgentName + ".md" + agentFilePath = ExperimentAgentName + ".md" ) -// agentTools are the IDs of the MCP tools that the agent can use. AWS names a -// tool of an MCP server "_". +// AWS names a tool of an MCP server "_". func agentTools(serverName string) []string { return toolIDs(serverName, slices.Concat(mcpReadOnlyTools, mcpMutativeTools)) } @@ -58,10 +46,8 @@ func toolIDs(serverName string, tools []string) []string { return ids } -// spaceAgentTools are the tool IDs of agentTools that the space allows. An -// agent can use only the tools that the association of the server allows, and -// setup keeps the allowlist of an existing association. With no association -// yet, setup makes one that allows every ldcli tool. +// spaceAgentTools are the ldcli tools that the association of the server +// allows, because an agent can use no others. func spaceAgentTools(ctx context.Context, clients Clients, spaceID, serviceID, serverName string) ([]string, error) { tools := slices.Concat(mcpReadOnlyTools, mcpMutativeTools) if spaceID == "" || serviceID == "" { @@ -85,39 +71,27 @@ func spaceAgentTools(ctx context.Context, clients Clients, spaceID, serviceID, s return toolIDs(serverName, tools), nil } -// AgentAction is the planned custom agent action. type AgentAction int const ( AgentCreate AgentAction = iota AgentKeep - // AgentUpdate gives an agent with the prompt of this or an older ldcli - // version the prompt, the tools, and the skill of this version. AgentUpdate - // AgentLeave means that someone edited the prompt, so setup does not - // change the agent. AgentLeave - // AgentSkip means that setup could not read the custom agents of the - // space. The agent is optional, so the run continues without it. AgentSkip ) -// AgentStep is the planned custom agent step. type AgentStep struct { Action AgentAction AssetID string // Version is the asset version at plan time. A write stops if it changed. - Version int32 - // ServerName is the name of the MCP server, which the tool IDs start with. + Version int32 ServerName string - // Tools and Skills are what the existing agent has. An update adds the - // ldcli tools and skill to them, so the ones that the customer added stay. + // Tools and Skills are what the existing agent has. Tools []string Skills []string - // NoFile means that the agent has no prompt file of this name. NoFile bool - // Err says why the step is skipped. - Err error + Err error } func (s AgentStep) describe() string { @@ -141,8 +115,6 @@ func (s AgentStep) describe() string { } } -// agentFailure is the text of a failed agent step. A missing permission is the -// most likely cause, so it names the permissions that the step needs. func agentFailure(err error) string { if errorCode(err) != "AccessDeniedException" { return err.Error() @@ -190,9 +162,8 @@ func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space Sp return step } -// serverName is the name of the registered MCP server in AWS, which the tool -// IDs start with. The run reads it, because the name in the plan can be out of -// date. +// serverName reads the name of the server at run time, because the name in the +// plan can be out of date. func serverName(ctx context.Context, clients Clients, serviceID string) (string, error) { services, err := listServices(ctx, clients) if err != nil { @@ -207,7 +178,6 @@ func serverName(ctx context.Context, clients Clients, serviceID string) (string, return "", fmt.Errorf("MCP server %s is not registered", serviceID) } -// agentMetadata reads the tools and the skills of a custom agent. func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { if asset.Metadata == nil { return nil, nil @@ -227,7 +197,6 @@ func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { return metadata.Tools, metadata.Skills } -// union is a followed by the items of b that a does not have. func union(a, b []string) []string { out := slices.Clone(a) for _, item := range b { @@ -239,8 +208,6 @@ func union(a, b []string) []string { return out } -// checkAgent compares the space with a step that writes. A change to the -// version means that someone edited the agent after the plan. func checkAgent(ctx context.Context, clients Clients, spaceID string, step AgentStep) error { asset, err := findAsset(ctx, clients, spaceID, agentAssetType, ExperimentAgentName) switch { @@ -269,10 +236,8 @@ const ( AgentFailed = "failed" ) -// installAgent does the custom agent step after the MCP server and skill -// steps, because the agent needs both. The agent is optional, so a failure does -// not stop setup. The failure goes in result.AgentError, and the caller reports -// it. +// installAgent runs after the MCP server and skill steps, because the agent +// needs both. A failure does not stop setup. func installAgent( ctx context.Context, clients Clients, @@ -292,8 +257,6 @@ func installAgent( } } -// errAgentNeeds means that an earlier step left the agent without its MCP -// server or its skill, so setup does not write the agent. var errAgentNeeds = errors.New("the custom agent needs the MCP server and the skill") func doAgentStep( @@ -320,8 +283,6 @@ func doAgentStep( return "", fmt.Errorf("setup skipped custom agent %s, because no MCP server is connected: %w", ExperimentAgentName, errAgentNeeds) case result.SkillAssetID == "" || result.SkillAction == SkillSkipped: - // A skipped skill step can keep the ID of a skill that someone - // deleted. return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", ExperimentAgentName, ExperimentSkillName, errAgentNeeds) } @@ -338,8 +299,6 @@ func doAgentStep( ExperimentAgentName, errAgentNeeds) } - // The other steps can take minutes, so check the agent right before the - // write. if err := checkAgent(ctx, clients, result.AgentSpaceID, step); err != nil { return "", err } @@ -369,8 +328,7 @@ func doAgentStep( return AgentCreated, nil } - // AWS updates only the metadata keys in the request, so the memory stores - // that the customer attached stay. + // AWS keeps the metadata keys that the request leaves out. if _, err := clients.Agent.UpdateAsset(ctx, &devopsagent.UpdateAssetInput{ AgentSpaceId: aws.String(result.AgentSpaceID), AssetId: aws.String(step.AssetID), diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 2e1614870..12769cc88 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -20,7 +20,6 @@ func TestShippedAgentHashesHoldTheCurrentPrompt(t *testing.T) { "add the hash of the new prompt to shippedAgentHashes") } -// AWS names a tool of an MCP server "_". func TestAgentToolsStartWithTheServerName(t *testing.T) { tools := awsdevops.AgentTools("LaunchDarkly") @@ -57,8 +56,6 @@ func TestSetupCreatesAgentInNewSpace(t *testing.T) { assert.Empty(t, result.AgentError) } -// The tool IDs use the name of the server in AWS, which the customer can -// change. func TestPlanSetupNamesAgentToolsAfterTheExistingServer(t *testing.T) { server := oauthMCPService("mcp-1") server.Name = aws.String("Prod-LD") @@ -94,8 +91,6 @@ func TestSetupKeepsCurrentAgent(t *testing.T) { assert.Equal(t, awsdevops.AgentKept, result.AgentAction) } -// A newer ldcli allows more tools, so an agent with the ldcli prompt and -// fewer tools gets the new list. func TestSetupUpdatesAgentWithOldTools(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{ @@ -113,7 +108,6 @@ func TestSetupUpdatesAgentWithOldTools(t *testing.T) { metadata := metadataOf(t, agent.updateAgents[0].Metadata) assert.ElementsMatch(t, awsdevops.AgentTools(awsdevops.MCPOAuthServerName), metadata["tools"]) assert.Equal(t, []any{"ki-ours"}, metadata["skills"]) - // AWS keeps the keys that the request leaves out, such as memory stores. assert.NotContains(t, metadata, "memory_stores") assert.Equal(t, awsdevops.AgentUpdated, result.AgentAction) } @@ -137,7 +131,6 @@ func TestSetupLeavesEditedAgent(t *testing.T) { assert.Empty(t, result.AgentError) } -// Without the skill, the agent could not run, so setup does not make it. func TestSetupSkipsAgentWhenSkillFails(t *testing.T) { agent := &fakeAgent{createAssetErr: errors.New("AccessDeniedException: not allowed")} @@ -150,7 +143,6 @@ func TestSetupSkipsAgentWhenSkillFails(t *testing.T) { assert.Contains(t, result.AgentError, "skill experiment-orchestration is not ready") } -// The agent is optional, so its failure does not fail setup. func TestSetupWarnsWhenAgentCreateFails(t *testing.T) { agent := &fakeAgent{createAgentErr: accessDenied("aidevops:CreateAsset")} @@ -172,8 +164,6 @@ func TestPlanSetupSkipsAgentWhenAssetsCannotBeListed(t *testing.T) { assert.Contains(t, plan.Text().Agent, "The custom agent step needs aidevops:ListAssets") } -// A customer can give the agent more tools or skills. The ldcli ones are -// there, so setup keeps the agent. func TestPlanSetupKeepsAgentWithAddedTools(t *testing.T) { tools := append(awsdevops.AgentTools(awsdevops.MCPOAuthServerName), "Datadog_query-logs") agent := &fakeAgent{ @@ -191,8 +181,6 @@ func TestPlanSetupKeepsAgentWithAddedTools(t *testing.T) { assert.Equal(t, awsdevops.AgentKeep, plan.Agent.Action) } -// An update adds the ldcli tools and skill, and keeps the ones that the -// customer added. func TestSetupUpdateKeepsAddedToolsAndSkills(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{ @@ -211,8 +199,6 @@ func TestSetupUpdateKeepsAddedToolsAndSkills(t *testing.T) { assert.Equal(t, []any{"ki-other", "ki-created"}, metadata["skills"]) } -// Setup keeps the allowlist of an existing association, and an agent can use -// only the tools that it allows. func TestSetupGivesAgentOnlyTheToolsTheSpaceAllows(t *testing.T) { agent := &fakeAgent{ services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, @@ -244,13 +230,6 @@ func TestPlanSetupLeavesAgentWithoutPromptFile(t *testing.T) { assert.Contains(t, plan.Text().Agent, "because it has no experiment-orchestrator.md") } -// Setup updates an agent with the ldcli prompt but without the skill or an -// allowed tool, so status calls it outdated. -// A skill without SKILL.md is not ours, and setup leaves it. -// An older ldcli prompt is one that setup updates. -// Like the operator app, assets that status cannot read do not stop it. -// Setup registers one OAuth server for the account, so the agent's tools start -// with the name of that server. func TestPlanSetupNamesAgentToolsAfterANewServer(t *testing.T) { plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(&fakeAgent{}, newFakeIAM()), awsdevops.SetupOptions{ AgentSpaceName: "launchdarkly", diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index 2a242d116..7d2155e89 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -62,10 +62,8 @@ type fakeAgent struct { failAssociateFor map[string]bool // paginate makes every list API return one item per page, so callers are // exercised against the NextToken AWS sends for a long list. - paginate bool - // assets holds the skill and custom agent assets of every space. - assets []agenttypes.Asset - // agentFiles holds the prompt of each custom agent by asset ID. + paginate bool + assets []agenttypes.Asset agentFiles map[string]string // skillFiles holds the SKILL.md text by asset ID. skillFiles map[string]string @@ -81,14 +79,11 @@ type fakeAgent struct { updateAssociations []*devopsagent.UpdateAssociationInput updateAssociationErr error updateAssets []*devopsagent.UpdateAssetInput - // createAgents and updateAgents hold the custom agent writes, apart from - // the skill writes. - createAgents []*devopsagent.CreateAssetInput - updateAgents []*devopsagent.UpdateAssetInput - createAgentErr error + createAgents []*devopsagent.CreateAssetInput + updateAgents []*devopsagent.UpdateAssetInput + createAgentErr error } -// agentAsset is a custom agent asset named name with these tools and skills. func agentAsset(id, name string, tools, skills []string) agenttypes.Asset { return agenttypes.Asset{ AssetId: aws.String(id), @@ -362,7 +357,6 @@ func (f *fakeAgent) RegisterService(_ context.Context, in *devopsagent.RegisterS return f.registerOutput, nil } - // AWS lists the new server, so a later step can read its name. name := aws.ToString(in.Name) if details, ok := in.ServiceDetails.(*agenttypes.ServiceDetailsMemberMcpserver); ok && details.Value.Name != nil { name = aws.ToString(details.Value.Name) @@ -512,8 +506,7 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { // The plan and the first check look only for the server of the // account, because a new space has no server. The check before the // MCP step also looks for the server of the space, which exists by - // then. The skill and the custom agent are last. The custom agent - // step reads the name of the server. + // then. The skill and the custom agent are last. []string{ "ListServices", "ListServices", "CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "ListServices", "RegisterService", "AssociateService", diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 082f94e38..b6548b212 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -54,11 +54,8 @@ func SetOAuthPollInterval(d time.Duration) func() { return func() { oauthPollInterval = old } } -// AllowedTools are the tools that setup allows on the MCP server. func AllowedTools() []string { return slices.Concat(mcpReadOnlyTools, mcpMutativeTools) } -// ShippedAgentHashes are the hashes of the shipped custom agent prompts. func ShippedAgentHashes() []string { return shippedAgentHashes } -// AgentTools are the tool IDs that setup gives the custom agent. func AgentTools(serverName string) []string { return agentTools(serverName) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 978ad4584..04cb7e184 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -179,8 +179,7 @@ type SetupResult struct { AgentAssetID string `json:"agentAssetId,omitempty"` // AgentAction is what the custom agent step did, for example "created". AgentAction string `json:"agentAction,omitempty"` - // AgentError is set if the custom agent step failed or was skipped. The - // agent is optional, so the rest of the setup ran. + // AgentError is set if the custom agent step failed or was skipped. AgentError string `json:"agentError,omitempty"` } @@ -809,8 +808,7 @@ func checkMCPServiceID(ctx context.Context, clients Clients, id, endpoint string } // name is the name of the server in AWS. A new server gets -// MCPOAuthServerName. An existing server without a name in AWS gets -// MCPServerName. +// MCPOAuthServerName. func (m MCPStep) name() string { if m.Action == MCPRegister { return MCPOAuthServerName @@ -1024,8 +1022,7 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } - // The skill and the agent go last, so a failure of the optional steps - // cannot stop the MCP server step. + // The optional skill and agent steps go last. installSkill(ctx, clients, plan.Skill, &result, logf) installAgent(ctx, clients, plan.Agent, &result, logf) diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 79dda7ec2..f4131a961 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -180,8 +180,6 @@ func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*age return findAsset(ctx, clients, spaceID, skillAssetType, name) } -// findAsset returns the asset of this type and name in the space, or nil if -// there is none. func findAsset(ctx context.Context, clients Clients, spaceID, assetType, name string) (*agenttypes.Asset, error) { assets, err := listAssets(ctx, clients, spaceID, assetType) if err != nil { @@ -223,8 +221,6 @@ func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (t return assetText(ctx, clients, spaceID, assetID, skillFilePath) } -// assetText returns the file at path in the asset. found is false if the -// asset has no such file. func assetText(ctx context.Context, clients Clients, spaceID, assetID, path string) (text string, found bool, err error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), From 25884c113f659778432eed7fda0f6e5ce39bea5c Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:50:10 -0700 Subject: [PATCH 22/34] chore: remove dead code from the custom agent step Setup always connects the MCP server before the agent step, so the check for a missing server is gone. planAgent no longer takes the options, and the unused AllowedTools test helper is gone. The message for a missing permission now names aidevops:ListAssociations. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 16 ++++++---------- internal/awsdevops/export_test.go | 3 --- internal/awsdevops/setup.go | 2 +- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index a69e29b82..d5a25ef25 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -49,10 +49,10 @@ func toolIDs(serverName string, tools []string) []string { // spaceAgentTools are the ldcli tools that the association of the server // allows, because an agent can use no others. func spaceAgentTools(ctx context.Context, clients Clients, spaceID, serviceID, serverName string) ([]string, error) { - tools := slices.Concat(mcpReadOnlyTools, mcpMutativeTools) if spaceID == "" || serviceID == "" { - return toolIDs(serverName, tools), nil + return agentTools(serverName), nil } + tools := slices.Concat(mcpReadOnlyTools, mcpMutativeTools) associations, err := listAssociations(ctx, clients, spaceID) if err != nil { return nil, err @@ -121,10 +121,10 @@ func agentFailure(err error) string { } return fmt.Sprintf("%v. The custom agent step needs aidevops:ListAssets, aidevops:GetAssetFile, "+ - "aidevops:CreateAsset, and aidevops:UpdateAsset", err) + "aidevops:CreateAsset, aidevops:UpdateAsset, and aidevops:ListAssociations", err) } -func planAgent(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, mcp MCPStep, skill SkillStep) AgentStep { +func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPStep, skill SkillStep) AgentStep { step := AgentStep{ServerName: mcp.name()} if space.Action == SpaceCreate { return step @@ -257,7 +257,7 @@ func installAgent( } } -var errAgentNeeds = errors.New("the custom agent needs the MCP server and the skill") +var errAgentNeeds = errors.New("the custom agent needs the skill") func doAgentStep( ctx context.Context, @@ -278,11 +278,7 @@ func doAgentStep( case AgentSkip: return "", fmt.Errorf("setup skipped custom agent %s: %w", ExperimentAgentName, step.Err) } - switch { - case result.MCPServiceID == "": - return "", fmt.Errorf("setup skipped custom agent %s, because no MCP server is connected: %w", - ExperimentAgentName, errAgentNeeds) - case result.SkillAssetID == "" || result.SkillAction == SkillSkipped: + if result.SkillAssetID == "" || result.SkillAction == SkillSkipped { return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", ExperimentAgentName, ExperimentSkillName, errAgentNeeds) } diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index b6548b212..c7d4d1cc0 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -2,7 +2,6 @@ package awsdevops import ( "context" - "slices" "time" ) @@ -54,8 +53,6 @@ func SetOAuthPollInterval(d time.Duration) func() { return func() { oauthPollInterval = old } } -func AllowedTools() []string { return slices.Concat(mcpReadOnlyTools, mcpMutativeTools) } - func ShippedAgentHashes() []string { return shippedAgentHashes } func AgentTools(serverName string) []string { return agentTools(serverName) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 04cb7e184..d2c5bc079 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -571,7 +571,7 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } plan.Skill = planSkill(ctx, clients, opts, plan.Space) - plan.Agent = planAgent(ctx, clients, opts, plan.Space, plan.MCP, plan.Skill) + plan.Agent = planAgent(ctx, clients, plan.Space, plan.MCP, plan.Skill) if !provision { // Read the app, so the run does not print a URL that does not work. From fbe2e521091dbad38d7448e49f0603d8334db0b4 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 14:54:41 -0700 Subject: [PATCH 23/34] chore: pass the MCP server name to the custom agent step The MCP step knows the name of the server, so it keeps the name in the result. The agent step no longer lists the servers again, and the OAuth poll test is the same as on main. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 22 +--------------------- internal/awsdevops/agent_test.go | 8 +++++++- internal/awsdevops/awsdevops_test.go | 2 +- internal/awsdevops/runplan_test.go | 3 +-- internal/awsdevops/setup.go | 4 ++++ 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index d5a25ef25..ddffd7f2a 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -162,22 +162,6 @@ func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPSte return step } -// serverName reads the name of the server at run time, because the name in the -// plan can be out of date. -func serverName(ctx context.Context, clients Clients, serviceID string) (string, error) { - services, err := listServices(ctx, clients) - if err != nil { - return "", err - } - for _, service := range services { - if aws.ToString(service.ServiceId) == serviceID { - return serviceName(service), nil - } - } - - return "", fmt.Errorf("MCP server %s is not registered", serviceID) -} - func agentMetadata(asset agenttypes.Asset) (tools, skills []string) { if asset.Metadata == nil { return nil, nil @@ -282,11 +266,7 @@ func doAgentStep( return "", fmt.Errorf("setup skipped custom agent %s, because skill %s is not ready: %w", ExperimentAgentName, ExperimentSkillName, errAgentNeeds) } - name, err := serverName(ctx, clients, result.MCPServiceID) - if err != nil { - return "", err - } - tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, name) + tools, err := spaceAgentTools(ctx, clients, result.AgentSpaceID, result.MCPServiceID, result.mcpServerName) if err != nil { return "", err } diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 12769cc88..2be6e6ca3 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -61,15 +61,21 @@ func TestPlanSetupNamesAgentToolsAfterTheExistingServer(t *testing.T) { server.Name = aws.String("Prod-LD") server.AdditionalServiceDetails.(*agenttypes.AdditionalServiceDetailsMemberMcpserver).Value.Name = aws.String("Prod-LD") agent := &fakeAgent{services: []agenttypes.RegisteredService{server}} + clients := newTestClients(agent, newFakeIAM()) opts := reuseOptions() - plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), opts, true) + plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) require.NoError(t, err) require.Equal(t, awsdevops.MCPReuse, plan.MCP.Action) assert.Equal(t, "Prod-LD", plan.Agent.ServerName) assert.Equal(t, "Create custom agent experiment-orchestrator with skill experiment-orchestration "+ "and the MCP tools of server Prod-LD", plan.Text().Agent) + + result, err := awsdevops.RunPlan(context.Background(), clients, opts, plan) + require.NoError(t, err) + require.Len(t, agent.createAgents, 1, result.AgentError) + assert.Contains(t, metadataOf(t, agent.createAgents[0].Metadata)["tools"], "Prod-LD_get-flag") } func TestSetupKeepsCurrentAgent(t *testing.T) { diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index 7d2155e89..da72fcf8e 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -510,7 +510,7 @@ func TestSetupCreatesRolesAgentSpaceAndAssociations(t *testing.T) { []string{ "ListServices", "ListServices", "CreateAgentSpace", "AssociateService", "EnableOperatorApp", "ListServices", "ListServices", "RegisterService", "AssociateService", - "ListAssets", "CreateAsset", "ListServices", "ListAssets", "CreateAsset", + "ListAssets", "CreateAsset", "ListAssets", "CreateAsset", }, agent.calls, ) diff --git a/internal/awsdevops/runplan_test.go b/internal/awsdevops/runplan_test.go index 5a294f1fd..5d7f288be 100644 --- a/internal/awsdevops/runplan_test.go +++ b/internal/awsdevops/runplan_test.go @@ -963,8 +963,7 @@ func TestOAuthPollsUntilTheServerAppears(t *testing.T) { } polls := 0 agent.onListServices = func() { - // The custom agent step lists the servers again after the poll. - if !opened || polls == 3 { + if !opened { return } polls++ diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index d2c5bc079..f28acb31d 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -181,6 +181,9 @@ type SetupResult struct { AgentAction string `json:"agentAction,omitempty"` // AgentError is set if the custom agent step failed or was skipped. AgentError string `json:"agentError,omitempty"` + // mcpServerName is the name of the server in AWS, which the tool IDs of + // the custom agent start with. + mcpServerName string } // ErrPlanChanged means AWS changed after the plan. @@ -1316,6 +1319,7 @@ func associateMCPServer( result *SetupResult, logf func(string, ...any), ) error { + result.mcpServerName = name existing, err := findMCPAssociation(ctx, clients, result.AgentSpaceID, result.MCPServiceID) if err != nil { return err From 0ef40ca2bff62277bb57919db94d3d4dfa1f1913 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 15:02:18 -0700 Subject: [PATCH 24/34] fix: give the custom agent the tools that --add-mcp-tools adds The plan compared the agent with the allowlist before the MCP step. With --add-mcp-tools, the MCP step adds tools first, so the plan kept the agent without them. The plan now expects those tools, so setup updates the agent in the same run. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/agent.go | 4 ++++ internal/awsdevops/agent_test.go | 26 ++++++++++++++++++++++++++ internal/awsdevops/awsdevops_test.go | 5 +++++ 3 files changed, 35 insertions(+) diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index ddffd7f2a..dc5e4c9e9 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -146,6 +146,10 @@ func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPSte return step } want, err := spaceAgentTools(ctx, clients, space.ID, mcp.ServiceID, step.ServerName) + if mcp.AddTools && mcp.CanAddTools { + // The MCP step adds these tools to the allowlist before the agent step. + want = union(want, toolIDs(step.ServerName, mcp.MissingTools)) + } switch { case err != nil: step.Action, step.Err = AgentSkip, err diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 2be6e6ca3..f2527dfc9 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -245,3 +245,29 @@ func TestPlanSetupNamesAgentToolsAfterANewServer(t *testing.T) { assert.Contains(t, plan.Text().Agent, "the MCP tools of server "+awsdevops.MCPOAuthServerName) } + +func TestSetupGivesAgentTheToolsThatAddMCPToolsAdds(t *testing.T) { + old := awsdevops.AgentTools(awsdevops.MCPServerName)[:1] + agent := &fakeAgent{ + services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, + associations: []agenttypes.Association{mcpAssociation("list-projects")}, + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", old, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + } + clients := newTestClients(agent, newFakeIAM()) + opts := reuseOptions() + opts.AddMCPTools = true + + plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) + require.NoError(t, err) + require.Equal(t, awsdevops.AgentUpdate, plan.Agent.Action) + + result, err := awsdevops.RunPlan(context.Background(), clients, opts, plan) + require.NoError(t, err) + require.Len(t, agent.updateAgents, 1, result.AgentError) + assert.ElementsMatch(t, awsdevops.AgentTools(awsdevops.MCPServerName), metadataOf(t, agent.updateAgents[0].Metadata)["tools"]) +} diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index da72fcf8e..c1860f580 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -343,6 +343,11 @@ func (f *fakeAgent) UpdateAssociation(_ context.Context, in *devopsagent.UpdateA if f.updateAssociationErr != nil { return nil, f.updateAssociationErr } + for i, association := range f.associations { + if aws.ToString(association.AssociationId) == aws.ToString(in.AssociationId) { + f.associations[i].Configuration = in.Configuration + } + } return &devopsagent.UpdateAssociationOutput{}, nil } From 66eb76726ebc5e848bed4567a52de4406ec042ab Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Wed, 28 Oct 2026 15:17:56 -0700 Subject: [PATCH 25/34] feat: sync the AWS DevOps Agent files from ai-tooling at a fixed commit The experiment-orchestration skill and the custom agent prompt now come from launchdarkly/ai-tooling. scripts/sync-aws-devops-agent.sh downloads them at the commit in AI_TOOLING_COMMIT. go generate and make aws-devops-agent-sync run it. ldcli embeds the files, so they stay committed. CI fails if they differ from that commit. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/go.yml | 2 ++ Makefile | 9 ++++++++- internal/awsdevops/skill.go | 2 ++ scripts/check-aws-devops-agent-changed.sh | 9 +++++++++ scripts/sync-aws-devops-agent.sh | 14 ++++++++++++++ 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100755 scripts/check-aws-devops-agent-changed.sh create mode 100755 scripts/sync-aws-devops-agent.sh diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7599052ac..1d1fa2f91 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -25,6 +25,8 @@ jobs: go-version-file: go.mod - name: build run: go build . + - name: check AWS DevOps Agent files + run: make aws-devops-agent-check - uses: actions/setup-python@v7 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/Makefile b/Makefile index 59ba190ae..193ae2411 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,11 @@ -.PHONY: build generate log test vendor +.PHONY: aws-devops-agent-check aws-devops-agent-sync build generate log test vendor + +aws-devops-agent-check: + make aws-devops-agent-sync + ./scripts/check-aws-devops-agent-changed.sh + +aws-devops-agent-sync: + ./scripts/sync-aws-devops-agent.sh build: go build -o ldcli diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index f4131a961..ed524710e 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -17,6 +17,8 @@ import ( agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" ) +//go:generate ../../scripts/sync-aws-devops-agent.sh + // ExperimentSkill is the SKILL.md of the experimentation skill. Its front // matter gives the name and the description that setup sends as metadata. // diff --git a/scripts/check-aws-devops-agent-changed.sh b/scripts/check-aws-devops-agent-changed.sh new file mode 100755 index 000000000..49c8dcc98 --- /dev/null +++ b/scripts/check-aws-devops-agent-changed.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +DIFF=$(git diff internal/awsdevops/skills internal/awsdevops/agents) +if [ "$DIFF" ]; then + echo "The AWS DevOps Agent files differ from launchdarkly/ai-tooling. Run 'make aws-devops-agent-sync' and commit the files." + exit 1 +else + echo "The AWS DevOps Agent files match launchdarkly/ai-tooling." +fi diff --git a/scripts/sync-aws-devops-agent.sh b/scripts/sync-aws-devops-agent.sh new file mode 100755 index 000000000..64ab6cec8 --- /dev/null +++ b/scripts/sync-aws-devops-agent.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Downloads the AWS DevOps Agent skill and custom agent prompt from +# launchdarkly/ai-tooling at a fixed commit. ldcli embeds the files, so they +# are committed here. To update them, change AI_TOOLING_COMMIT, run this +# script, and add the new hashes in internal/awsdevops. +set -euo pipefail + +AI_TOOLING_COMMIT=bbdf962eb696d649d24ff128d4db0ff894c6943c +BASE_URL="https://raw.githubusercontent.com/launchdarkly/ai-tooling/${AI_TOOLING_COMMIT}/skills/aws-devops-agent" +ROOT=$(git rev-parse --show-toplevel) +DEST="${ROOT}/internal/awsdevops" + +curl -fsSL -o "${DEST}/skills/experiment-orchestration/SKILL.md" "${BASE_URL}/experiment-orchestration/SKILL.md" +curl -fsSL -o "${DEST}/agents/experiment-orchestrator.md" "${BASE_URL}/agents/experiment-orchestrator.md" From 1c7eeb4de871fa3967908913edd96bc44a5211a7 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 10:11:30 -0700 Subject: [PATCH 26/34] chore: sync the AWS DevOps Agent files from the ai-tooling merge commit The sync now pins 887d4cb, the merge commit of launchdarkly/ai-tooling#230. SKILL.md changed, so its hash is in shippedSkillHashes. The agent prompt did not change. The skill test now compares the description with the front matter of the embedded SKILL.md. Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/export_test.go | 3 + internal/awsdevops/skill.go | 1 + internal/awsdevops/skill_test.go | 2 +- .../skills/experiment-orchestration/SKILL.md | 155 ++++++++++++------ scripts/sync-aws-devops-agent.sh | 2 +- 5 files changed, 111 insertions(+), 52 deletions(-) diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index c7d4d1cc0..6ff076d1a 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -36,6 +36,9 @@ func ShipSkill(text string) func() { // ShippedSkillHashes are the hashes of the shipped skill versions. func ShippedSkillHashes() []string { return shippedSkillHashes } +// ExperimentSkillDescription is the description in the front matter of SKILL.md. +func ExperimentSkillDescription() string { return experimentSkillDescription } + // SkillHash is the hash that setup compares. func SkillHash(text string) string { return skillHash(text) } diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index ed524710e..0c719196d 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -37,6 +37,7 @@ var ( var shippedSkillHashes = []string{ "57844bc3c5aca292ef88b5f949cd791206074b5cfc850ea52ebdd40020ead695", "c4c4e12aeff900402dc17ff370bc1e78eadcb34abac5b256319c7cd793bd73f3", + "cbda81714171e1f725db7a7e10973c38ae0b71550e14636d8d463919349bd9e8", } const ( diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index 0c12ed03e..b41f704ba 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -56,7 +56,7 @@ func TestSetupCreatesSkillInNewSpace(t *testing.T) { assert.Equal(t, "skill", aws.ToString(in.AssetType)) metadata := metadataOf(t, in.Metadata) assert.Equal(t, awsdevops.ExperimentSkillName, metadata["name"]) - assert.Contains(t, metadata["description"], "Orchestrates automated experimentation lifecycle") + assert.Equal(t, awsdevops.ExperimentSkillDescription(), metadata["description"]) assert.Equal(t, []any{"GENERIC"}, metadata["agent_types"]) assert.NotContains(t, metadata, "status") file, ok := in.Content.(*agenttypes.AssetContentMemberFile) diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md index bf816eb50..eed2d61a9 100644 --- a/internal/awsdevops/skills/experiment-orchestration/SKILL.md +++ b/internal/awsdevops/skills/experiment-orchestration/SKILL.md @@ -1,19 +1,35 @@ --- name: "experiment-orchestration" -description: "Orchestrates automated experimentation lifecycle using LaunchDarkly Guarded Rollouts, an AI coding agent for implementation, and GitHub Actions for deployment." +description: "Takes a product goal through a LaunchDarkly experiment and, if it wins, a guarded rollout (or a regular rollout on plans without guarded rollouts). Uses GitHub for the code change and deployment when it is connected to the agent space; otherwise hands those steps to a person." +license: Apache-2.0 +compatibility: Requires the remotely hosted LaunchDarkly MCP server in an AWS DevOps Agent space. GitHub is optional. +metadata: + author: launchdarkly + version: "0.1.0" --- # Automated Experimentation Use this skill when you have a goal you want to move through experimentation (e.g., "increase checkout conversion by 15%", "decrease page load time by 20%"). -**Core principle: experiment first, then guarded rollout.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before ramping it up through a guarded rollout. Never start a guarded rollout blind — it exists only to scale a change the experiment has already shown to work. +**Core principle: experiment first, then a guarded rollout, or a regular rollout if the plan has no guarded rollouts.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before rolling it out. Never roll out blind — the rollout exists only to scale a change the experiment has already shown to work. -**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Only stop and ask the user something if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result). +**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Stop and ask the user only for a planned handoff (merging the PR; any code, review, or deploy step you hand over without GitHub) or if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result). -**Approvals:** if the LaunchDarkly environment requires approval for a change (for example, to turn a flag on or off, start or stop an experiment, or start a guarded rollout), create an approval request instead of the direct change. Tell the person who must review it. Continue only after `list-approval-requests` shows it approved, then apply it with `apply-approval-request` and go to the next step. If a reviewer rejects it, stop and report. +**Approvals:** if the LaunchDarkly environment requires approval for a change (for example, to turn a flag on or off, start or stop an experiment, or start a guarded rollout), create an approval request instead of the direct change. The first time you change an entity (a flag, an experiment) in an environment, call the change tool directly; don't guess beforehand whether the environment requires approval. If the response has `requiresApproval: true`, tell the user the change was not made because it needs LaunchDarkly approval, ask who should review it, and submit the returned instructions with `create-approval-request` (or the approval tool that response names). From then on, send every change to that entity in that environment straight to an approval request with the same instructions you would give the direct call, instead of calling the change tool first. Tell the person who must review it. Continue only after `list-approval-requests` shows it approved, then apply it with `apply-approval-request` and go to the next step. If it still shows pending, stop and tell the user it is waiting for review; never apply a pending request, and don't keep re-checking in the same turn. If a reviewer rejects it, stop and report. -**The final report (Step 8) is mandatory, not optional.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. +**Blocked tools:** an agent space can block MCP tools that it marks destructive. Likely candidates include, but aren't limited to, `toggle-flag`, `start-experiment-iteration`, `update-rollout`, `update-targeting-rules`, `start-guarded-rollout`, and `stop-experiment-iteration`. A blocked call fails without a LaunchDarkly error, or returns nothing, so you can't tell why it failed. If a write fails without a LaunchDarkly error that explains it, or returns nothing, first read the state back (`get-flag` or `get-experiment`): an empty response may still have made the change. If it didn't, don't retry: tell the user the tool may be blocked as destructive in this agent space, and hand them the step as below. + +**When a step can't succeed through you:** a blocked tool, or any other write you can't complete (including `create-experiment`, `create-metric`, and the approval tools). Hand the step to the user with exact instructions, not a summary. Give the link, what to click, and every value the call would have sent, using the real names and keys from this run: variation names, percentages, metric keys, thresholds, randomization unit, stage weights and durations, the winning treatment. If the environment requires approval, tell them the UI will ask them to submit the change for review and who should review it. Then wait until they confirm it's done, read the state back to check it, and continue from the next step. For example: +- Turn the flag on or off, or change its default rule or off variation: open `https://app.launchdarkly.com/projects//flags//targeting?env=&selected-env=`, set the toggle, the default rule, or the off variation to the exact variation name or percentages, and save. +- Start a guarded rollout: on the same targeting page, set the default rule to a guarded rollout and give the variation to roll out, the randomization unit, each metric key with its regression threshold and roll back on regression, and each stage's percentage and monitoring window from Step 7. +- Start or stop the experiment: open the `designUrl` that `create-experiment` returned, or `https://app.launchdarkly.com/projects//experiments//design?env=&selected-env=`. When the user stops it, tell them to choose Control as the winner, for the reason Step 7 gives. + +**Plan limits:** if LaunchDarkly returns an error saying the plan doesn't include a feature this skill needs, and this skill has no fallback for it (it has one only for guarded rollouts), tell the user which feature is missing and that it needs a plan that includes it, then stop. Treat a failure as a plan limit only when the error says so; a silent failure is a possibly blocked tool, above. If the account is on another LaunchDarkly instance (for example, federal), use that instance's host instead of `app.launchdarkly.com`. + +**GitHub:** Steps 2–5 use GitHub when it is connected to the agent space. Before Step 2, check whether you have GitHub access to the target repository. If you don't, read no code: ask the user where the change belongs, and in Step 3 give them the flag key and the change to make, then wait until they confirm the change is reviewed, merged, and deployed, and skip to Step 6. Triggering workflows and proposing fixes need the GitHub connection's Read and write permissions; with Read-only, hand those steps to the user. + +**Always produce the Step 8 report on every terminal outcome.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. ## Step 1: Goal Clarification @@ -24,8 +40,16 @@ Before doing anything, get answers to: 3. **What's the current baseline?** *(Optional — infer from production metrics if not given)* 4. **What parts of the app are in scope?** *(Optional — infer from the codebase if not given)* 5. **Any constraints?** *(Optional)* e.g. no changes to the payment flow. +6. **How much regression should roll a change back?** *(Optional — propose one for the success metric and each guardrail metric if not given)* Used as each metric's `regressionThreshold` in Step 7. -Questions 1–2 are required before proceeding; infer 3–5 where possible and confirm your assumptions with the user before implementing. +Questions 1–2 are required before proceeding: until the user has answered both, make no changes (no flag, metric, or experiment). If a question gets no answer, ask again or stop; never assume an answer. Infer 3–6 where possible and confirm your assumptions with the user before implementing. + +**Find the metrics.** The experiment needs the success metric. LaunchDarkly requires at least one metric to start a guarded rollout, and this skill requires an error-rate metric as a guardrail in both the experiment and the rollout. Find them before Step 2: + +1. Choose the randomization unit (typically "user"). Every metric must list it in its randomization units, or the guarded rollout rejects the metric. +2. Call `list-metrics` with a `query` for the success metric. For errors, look first for LaunchDarkly's built-in error-rate metric (key ending in `telemetry-default-error-rate`, event `$ld:telemetry:error`), then `query: "error"`. Check that each metric's randomization units include the unit from item 1. +3. For the success metric, call `list-metric-events` with `query` set to its event key, in the environment the experiment will use. It returns at most 50 keys, so always pass the query. Don't run this check for the error-rate metric: no recent error events may just mean no errors. +4. Confirm the metric keys with the user. If a metric doesn't exist, plan to create it in Step 3: Implementation. If the success metric exists but its event isn't arriving, plan to add its `track()` call in Step 3: Implementation instead of creating a second metric. ## Step 2: Hypothesis Generation @@ -34,7 +58,7 @@ Explore the target repository/codebase to find a plausible change: 1. Search and read the relevant code paths. 2. Think through what UI/UX or logic change could plausibly move the chosen metric. 3. Check whether this hypothesis (or something close to it) has already been tried and failed — look at flag history or archived flags with similar naming. Avoid repeating a known failure. -4. Present the hypothesis to the user before proceeding, along with your reasoning and any inferred assumptions from Step 1. +4. Present the hypothesis to the user before proceeding, along with your reasoning and any inferred assumptions from Step 1. If the user wants to test more than one version of the change (for example, three button labels), list each version; Step 3 then creates a multivariate flag. Each version splits the same 10% of traffic, so more than three versions makes significance slower to reach; say so to the user. **Before finalizing a flag key, check for collisions:** look up any candidate flag key first. - Already fully shipped (100% one variation, no split) → already decided, pick a different hypothesis. @@ -43,72 +67,100 @@ Explore the target repository/codebase to find a plausible change: ## Step 3: Implementation -1. Create a boolean feature flag, OFF by default in all environments, with `defaults` of `{"onVariation": 1, "offVariation": 1}` so that it serves false even when it is on, and `clientSideAvailability` of `{"usingEnvironmentId": true, "usingMobileKey": true}` so that every SDK can read it. Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. If the project has its own flag naming conventions (for example, a required prefix or case), follow them instead. -2. Hand off implementation to your coding agent/tool of choice, with clear instructions to gate the change behind the exact flag key from step 1. -3. This step is asynchronous — check status periodically rather than looping tightly on it. -4. Once implementation completes, move to Step 4 with the resulting branch/PR. If it fails, report the error and stop. +1. Create the feature flag, OFF by default in all environments, with `clientSideAvailability` of `{"usingEnvironmentId": true, "usingMobileKey": true}` so that every SDK can read it. Pick its kind from the hypothesis: + - **One version of the change:** a boolean flag with `defaults` of `{"onVariation": 1, "offVariation": 1}`, so that it serves `false` (the control) even when it is on. + - **Several versions of the change:** a multivariate flag with `kind: "multivariate"` and `variations` listing the control (current behavior) first, then one variation per version, each with a `name` and the `value` the code reads. Set `defaults` to `{"onVariation": 0, "offVariation": 0}`, so that it serves the control even when it is on. + + If `create-flag` errors on `defaults`, create the flag without `defaults`, then call `update-off-variation` with `allEnvs: true`, `variationIndex` and `fallthroughVariationIndex` both set to the control's index (`1` for a boolean flag, `0` for a multivariate flag), and `updateFlagDefaults: true`. Do this right after creating the flag, before any code that reads it is deployed: without it, the flag can serve a treatment to everyone, off or on. Environments that need approval come back blocked with instructions for `create-approval-request`; the others still apply. + + Then read the flag back with `get-flag` in the environment the experiment will use: its off variation and default rule must both be the control. If they aren't, hand the change to the user (see "When a step can't succeed through you") and don't continue until they are. + + Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. If the project has its own flag naming conventions (for example, a required prefix or case), follow them instead. +2. If Step 1: Goal Clarification found no success or error-rate metric, create it now with `create-metric`, with `randomizationUnits` set to the unit from Step 1: Goal Clarification. For an error-rate metric, use `kind: "custom"`, `measureType: "occurrence"` (the share of contexts that hit an error), and `successCriteria: "LowerThanBaseline"`. Include the `track()` call for each new or silent metric's event key in the code change below. +3. If a coding agent is available, hand off the implementation to it, with instructions to gate the change behind the exact flag key from item 1 above. Without a coding agent or GitHub write access, give the user the flag key and the change instead, and wait for their PR (with GitHub connected) or their confirmation that it is deployed (without GitHub). +4. This step is asynchronous — check status periodically rather than looping tightly on it. +5. Once implementation completes, move to Step 4: Release Readiness with the resulting branch/PR. If it fails, report the error and stop. ## Step 4: Release Readiness -Run your standard release/risk review on the PR before merging. +Run an AWS DevOps Agent release readiness code review on the PR before it merges. It runs automatically on connected repositories with **Auto trigger change review** enabled; otherwise request one. -- If it passes: merge the PR. -- If it fails: feed the review's specific feedback back into implementation and retry. Cap retries at a small fixed number (e.g. 3 attempts total) — if it still hasn't passed, stop and report the last failure to the user rather than retrying indefinitely. +- If it passes: ask the user to merge the PR, and wait until they confirm it is merged. +- If it fails: feed the review's specific feedback back into implementation (or to the user, if they made the change) and retry. Cap retries at a small fixed number (e.g. 3 attempts total) — if it still hasn't passed, stop and report the last failure to the user rather than retrying indefinitely. -*(If your environment genuinely has no review capability available — e.g., a fully unattended automation context — you can skip straight to merge, but treat that as a deliberate, narrow exception you call out explicitly, not a default. Skipping review removes your only gate against shipping broken code.)* +Never skip the review. Without GitHub, ask the user to have the PR reviewed and merged. ## Step 5: Deployment -Deployment typically won't fire automatically on merge if your workflow is manually-triggered (`workflow_dispatch`-only) — you'll need to trigger it explicitly. +The flagged code must be running in the environment the experiment uses before Step 6 turns the flag on; otherwise the experiment measures nothing. Confirm the merged change is deployed there. If the deploy workflow runs only when triggered (for example, `workflow_dispatch`), trigger it as below. -1. Trigger the deploy workflow on the merge target branch. Treat "already an in-progress deployment for this ref" as expected de-duplication, not an error — don't re-trigger. -2. Poll for status, but let your polling tool's own internal long-poll do the waiting rather than looping tightly yourself. -3. Watch for a "stale" status specifically: if a deployment reports "running" for far longer than normal, cross-check the actual CI run history by commit SHA/timing before assuming it's still in progress — a background poll process may have died without updating the record. +1. Trigger the deploy workflow on the merge target branch (this needs Read and write GitHub access; otherwise ask the user to deploy and confirm when it's done). Treat "already an in-progress deployment for this ref" as expected de-duplication, not an error — don't re-trigger. +2. Check the workflow run's status periodically rather than looping tightly. +3. If a run reports "running" for far longer than normal, check the workflow run history for that commit before assuming it's still in progress. 4. **Trigger a deployment at most once per attempt.** If you're unsure whether a previous trigger succeeded, check status first — never re-trigger just because you're unsure. 5. On timeout: stop, check the CI run directly, report the situation, ask how to proceed. 6. On explicit failure: stop and report — do not proceed to the experiment. -7. On success: proceed immediately to Step 6. +7. On success: proceed immediately to Step 6: Experiment Phase. ## Step 6: Experiment Phase (fixed 10%) -Prove the change on a small, fixed slice of traffic. Do **not** start a guarded rollout here — that's Step 7, and only after this proves out. +Prove the change on a small, fixed slice of traffic. The experiment and the rollout never run at the same time: a guarded rollout can't start while an experiment is running on the flag, so Step 7 stops the experiment iteration before Phase 2 starts any rollout. -1. Turn the flag ON. -2. Configure a fixed 50/50 split across 10% of traffic (a flat allocation, not a staged ramp) on your chosen randomization unit (typically "user"). The remaining 90% of traffic is excluded from the experiment entirely. +The control is the variation that keeps current behavior: `false` on a boolean flag, the first variation on a multivariate flag. Every other variation is a treatment: `true` on a boolean flag, one per version on a multivariate flag. + +1. Turn the flag ON; starting an experiment fails while the flag is off. It still serves the control to everyone, because Step 3: Implementation set its defaults to the control. +2. Read the flag to get its current version (required by `create-experiment`) and the ID of every variation. 3. Create an experiment with: - - Exactly one primary metric: the success metric from Step 1. - - Guardrail metric(s): always include your error-rate metric; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. - - Treatments: control (off) at 50%, treatment (on) at 50%, allocated to 10% of total traffic. -4. Start the experiment/data collection. + - `flags`: `{"": {"ruleId": "fallthrough", "flagConfigVersion": , "notInExperimentVariationId": ""}}`. The experiment runs on the default rule, the only rule the Step 7 rollouts act on, and every context outside the experiment keeps getting the control. + - `hypothesis`: the hypothesis from Step 2: Hypothesis Generation. + - `treatments`: one per variation. The control is `{"name": "Control", "baseline": true, "allocationPercent": , "parameters": [{"flagKey": "", "variationId": ""}]}`; each treatment is the same with its variation's name, `"baseline": false`, and its variation ID. Unless the user chose another split, split 10% of traffic evenly: each `` is 10 divided by the number of variations, rounded down to two decimals (2 variations: 5 each; 3: 3.33 each; 4: 2.5 each). Each `allocationPercent` is that treatment's share of all traffic on the rule, so the shares add up to 10%. Never use 50 and 50, or any shares that add up to more than 10: 50 and 50 puts 100% of traffic in the experiment. + - `canReshuffleTraffic: false`, so contexts keep their variation if the traffic changes. + - `randomizationUnit`: the randomization unit from Step 1: Goal Clarification. + - `metrics`: the success metric from Step 1: Goal Clarification and the guardrail metrics, each as `{"key": ""}`. Always include your error-rate metric as a guardrail; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. + - `primarySingleMetricKey`: the success metric's key. It is the only primary metric. + - `methodology: "frequentist"` with `analysisConfig` `{"sequentialTestingEnabled": true, "significanceThreshold": "5", "testDirection": "two-sided"}` (the last two are LaunchDarkly's defaults, sent so the config is complete). With two or more treatments, also set `"multipleComparisonCorrectionMethod": "benjamini-hochberg"` and `"multipleComparisonCorrectionScope": "variations"`; LaunchDarkly rejects a method without a scope, and without the correction, comparing several treatments to the control declares more false winners. Step 7 checks results many times; sequential testing keeps significance valid under repeated checks, where a fixed-horizon test would declare false winners. Don't use the Bayesian methodology: it reports a probability of being best, not the significance Step 7 decides on. +4. Start the experiment iteration. Starting it writes the split onto the default rule: 10% of contexts in the experiment, split across the variations, and the other 90% on the control, not analyzed. 5. Move to Step 7 to monitor toward a decision. ## Step 7: Monitoring & Outcome -Check status periodically — don't tight-loop. In an interactive session, check once and report progress, then pick back up later. In an unattended/scheduled context, check once per invocation and persist your progress somewhere durable between runs. +Check status periodically — don't tight-loop. In an interactive session, check once and report progress, then pick back up later. Every MCP write needs a person to approve it, so an unattended run stops at its first write: run checks that may lead to a write in a session where someone can approve, and report progress so the next session can pick it up. **Phase 1 — Prove the experiment at 10% (gate before any rollout):** -Watch for statistical significance on the primary metric: +Watch for statistical significance on the primary metric. For each metric, call `get-experiment-metric-results`: it returns per treatment `analyzedUnitCount` (contexts in the experiment), `conversionCount` (contexts that sent the metric's event), and `isSignificant`. Decide on `isSignificant`, not on the `isBest` that `get-experiment-results` reports; `isBest` marks the leading treatment even when nothing is significant. + +**Check the instrumentation after the first day.** If the success metric, or any guardrail other than the error-rate metric, has a `conversionCount` of 0 in every treatment, stop and report an instrumentation problem instead of waiting for significance. Zero error events is fine: it means no errors. -- **Significant + positive lift** → experiment proven. Stop the experiment iteration with control as the winner, then move to Phase 2. Stopping serves the winner to everyone, so a treatment winner would skip the guarded rollout. -- **Significant + negative lift** → declare a loser, stop the iteration with control as the winner, turn the flag off (do not archive it while the code still evaluates it), skip Phase 2, go straight to the Step 8 report. -- **No significance after a reasonable ceiling (e.g. 4 weeks)** → report "inconclusive, need more traffic", stop the iteration with control as the winner, and stop; don't proceed to Phase 2. +Judge each treatment against the control on its own. A treatment **qualifies** if it has a significant positive lift on the primary metric and no guardrail regression. A treatment **fails** if it has a significant negative lift on the primary metric or a guardrail regression. -Never declare a winner off a single data point or before your stats engine confirms significance. +- **Proven:** at least one treatment qualifies, even if another fails. The winner is the qualifying treatment with the largest lift. Stop the experiment iteration with control as the winner, then move to Phase 2. Stopping serves the winner to the experiment's targeting rule (everyone, when the experiment is on the default rule), so a treatment winner would skip the guarded rollout. If the plan has no guarded rollouts, still stop with control as the winner and use the regular rollout in Phase 2. +- **Loser:** no treatment qualifies, and either every treatment fails or any treatment has a guardrail regression. A regressing treatment must not keep serving traffic while the others collect data. Declare a loser, stop the iteration with control as the winner, turn the flag off (do not archive it while the code still evaluates it), skip Phase 2, go straight to the Step 8 report. +- **Anything else** (no significance, or mixed results such as one treatment worse and another flat): keep collecting data. At a reasonable ceiling (e.g. 4 weeks), report "inconclusive, need more traffic", naming any treatment that was significantly worse, stop the iteration with control as the winner, and stop; don't proceed to Phase 2. + +To stop the iteration, pass `stop-experiment-iteration` the `_id` of the control treatment (`baseline: true`) from `get-experiment` as `winningTreatmentId`, in every outcome; it takes a treatment ID, not a variation ID. Stopping with the real winner would serve it to everyone and skip the rollout. + +Never declare a winner off a single data point or before your stats engine confirms significance. Also require an `analyzedUnitCount` of at least 100 in each variation before acting on a significant result; with fewer, keep collecting data. **Phase 2 — Guarded rollout ramp (only after Phase 1 proves the change):** Start a guarded rollout with: -- The winning ("on") variation as the test, the original as control. +- `testVariationId`: the winning treatment's variation ID; `controlVariationId`: the control variation ID (both from Step 6). - Same randomization unit as the experiment. -- **Exactly 3 monitored stages, capped well below 100%** — e.g. 20% → 30% → 40%, ~60 minutes monitoring each. Don't add a stage at or above 100%; Guarded-rollout implementations reject stages above 50% audience allocation, and the rollout auto-promotes to 100% itself once the final monitored stage completes cleanly — no explicit 100% stage needed. -- The same primary + guardrail metrics as the experiment, each configured to notify and auto-rollback on regression. +- **Exactly 3 monitored stages, each below 50%** — e.g. 20% → 30% → 40%, ~60 minutes monitoring each. `rolloutWeight` is in thousandths of a percent (20% is `20000`) and `monitoringWindowMilliseconds` is in milliseconds (60 minutes is `3600000`). Don't add a 100% stage. +- `updateDefaultRuleOnCompletion: true`, so the rollout moves the default rule to the winning variation once the final stage completes cleanly. +- Every metric the experiment used, primary and guardrails, each with `onRegression` set to notify and roll back, and a `regressionThreshold` (required) from Step 1. Track stage progression. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, turn the flag off (do not archive it while the code still evaluates it), and go to the Step 8 report. -Once the final stage completes cleanly and auto-promotes to 100%, declare a winner and go to the Step 8 report. +Once the final stage completes cleanly and the default rule serves the winning variation, declare a winner and go to the Step 8 report. + +**If the plan has no guarded rollouts:** if `start-guarded-rollout` returns an error saying guarded rollouts aren't available on the plan, use a regular rollout instead. A silent failure is not a plan error: read the flag back and, if no rollout started, hand the guarded rollout to the user (see "When a step can't succeed through you") instead of rolling out to everyone. +1. Read the flag's variations to find the indexes of the winning treatment's variation and the control's. Call `update-rollout` with `rolloutType: "variation"` and the winner's index on the default rule. This serves the winner to everyone at once, with no automatic rollback. +2. Check the guardrail metrics once per monitoring window, for about as long as the guarded rollout would have run (e.g. 3 hours). Reverting is a write that needs a person's approval, so run this watch in a session where someone can approve. +3. If a guardrail regresses, call `update-rollout` with the control's index, declare failed, and go to the Step 8 report. Otherwise, declare a winner and go to the Step 8 report. -**Retrying after a rollback:** a rollback isn't always caused by your monitored metrics genuinely regressing — it can also be triggered by an unrelated application error surfacing mid-ramp. Before blindly restarting after the user says they've fixed something: +**Retrying after a guarded rollout rollback:** a rollback isn't always caused by your monitored metrics genuinely regressing — it can also be triggered by an unrelated application error surfacing mid-ramp. Before blindly restarting after the user says they've fixed something: 1. Confirm the flag's current state (should be back to 100% control, nothing stuck mid-rollout). 2. Check the change history timing between "advanced to next stage" and "reverted." A rollback within seconds of advancing is inconsistent with a full metric-window regression and points to an external cause instead. 3. If the flag is cleanly reverted and the external cause is confirmed fixed, it's safe to restart the guarded rollout from scratch with the same parameters. @@ -118,7 +170,7 @@ Once the final stage completes cleanly and auto-promotes to 100%, declare a winn ## Step 8: Report -Runs automatically the instant Step 7 reaches a terminal outcome (winner + auto-promoted to 100%; loser; inconclusive; or rollback/failure). Use this exact structure from Experiment Report to Next Steps: +Runs automatically the instant Step 7 reaches a terminal outcome (winner, with the default rule serving it; loser; inconclusive; or rollback/failure). Use this exact structure from Experiment Report to Next Steps: ```markdown ## Experiment Report: [Goal Description] @@ -131,41 +183,44 @@ Runs automatically the instant Step 7 reaches a terminal outcome (winner + auto- [What we tried and why] ### Implementation -- Flag: [flag_key] +- Flag: [flag_key] ([boolean / multivariate]) +- Variations: [control, then each treatment] - Files modified: [list] - Branch: [branch name] ### Release Readiness -- [reviewed, passed after N attempt(s) / skipped, per your environment's process] +- [release readiness review passed after N attempt(s) / reviewed by the user] ### Experiment Phase (10% fixed split) - Status: [proven / loser / inconclusive] +- Winner: [variation name, or none] - Duration: [time] - Metric change: [before] → [after] ([+/-]%) - Statistical significance: [value, confidence interval] -### Guarded Rollout Phase (if reached) +### Rollout Phase (if reached) +- Type: [guarded rollout / regular rollout (no guarded rollouts on this plan)] - Status: [completed / rolled_back / not started] - Duration: [time] -- Stages reached: [N of 3 monitored stages] +- Stages reached: [N of 3 monitored stages, guarded rollout only] - If rolled back and retried: [root cause, outcome of retry] ### Safety Metrics -- error-rate: [baseline] → [final] ([no regression / regression detected]) +- [error-rate metric key]: [baseline] → [final] ([no regression / regression detected]) - [other guardrails]: [baseline] → [final] ([status]) ### Next Steps [What to do next based on the outcome] ``` -## Safety Rules (the non-negotiables) +## Safety Rules - Always present the hypothesis before implementing. -- Always run a release/risk review before merging, unless your environment has a deliberate, explicitly-called-out exception. -- **Always prove a change via a fixed small-percentage experiment before starting any guarded rollout** — never ramp blind. +- Always have the PR reviewed before merging: the release readiness review with GitHub, or the user's review without it. Never skip it. +- **Always prove a change via a fixed small-percentage experiment before starting any rollout** — never roll out blind. - Always include an error-rate (or equivalent "don't break prod") metric as a guardrail, separate from your success metric. - Add a performance guardrail (e.g. p95 latency) for performance-focused changes. -- Every rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. -- **Cap guarded rollout stages well below 100%** (most platforms reject stages ≥50% audience allocation) and let the platform auto-promote to 100% after the final stage — don't try to add an explicit 100% stage. +- Every guarded rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. A regular rollout has no automatic rollback, so watch its guardrail metrics after rolling out. +- **Keep guarded rollout stages below 50%** and let the rollout move to 100% after the final stage — don't add an explicit 100% stage. - Distinguish a metric-driven rollback (don't retry) from an external-cause rollback (safe to retry once fixed) before restarting a rolled-back rollout. -- The final report is automatic and mandatory on every terminal outcome — never defer it to a follow-up ask. +- Produce the Step 8 report on every terminal outcome — never defer it to a follow-up ask. diff --git a/scripts/sync-aws-devops-agent.sh b/scripts/sync-aws-devops-agent.sh index 64ab6cec8..39ac28669 100755 --- a/scripts/sync-aws-devops-agent.sh +++ b/scripts/sync-aws-devops-agent.sh @@ -5,7 +5,7 @@ # script, and add the new hashes in internal/awsdevops. set -euo pipefail -AI_TOOLING_COMMIT=bbdf962eb696d649d24ff128d4db0ff894c6943c +AI_TOOLING_COMMIT=887d4cb5022ccda7e57dcef9b23aa9bd9d8cf75d BASE_URL="https://raw.githubusercontent.com/launchdarkly/ai-tooling/${AI_TOOLING_COMMIT}/skills/aws-devops-agent" ROOT=$(git rev-parse --show-toplevel) DEST="${ROOT}/internal/awsdevops" From 07ed010120e2d2f3f2223d21dd27a8dd6efc4130 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 10:16:40 -0700 Subject: [PATCH 27/34] docs: say that setting up the skill and the custom agent is optional "The agent is optional" can mean that a space does not need the agent. The help text now says that the setup step is optional. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/setup.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 2afd609a2..76cc07396 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -65,12 +65,13 @@ The experiment-orchestration skill tells the agent how to run an experiment from a goal to a report. Every agent type in the space can load it. If the skill has the text of an older ldcli version, setup updates it. If someone edited it, setup does not change it. To overwrite the edits, pass ---replace-skill. The skill is optional: if its step fails, setup warns and -does the other steps. +--replace-skill. Setting up the skill is optional: if its step fails, setup +warns and does the other steps. The experiment-orchestrator custom agent runs the skill with the LaunchDarkly -MCP tools that the space allows. The agent is optional and needs the MCP -server and the skill: if its step fails, setup warns and does the other steps.`, +MCP tools that the space allows. It needs the MCP server and the skill. +Setting up the custom agent is optional: if its step fails, setup warns and +does the other steps.`, Args: cobra.NoArgs, PreRun: trackRun(analyticsTrackerFn), RunE: runSetup, From 9969b07257d34e2b7454274f8f38d414f10feb74 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 10:25:14 -0700 Subject: [PATCH 28/34] feat: allow the MCP tools of the whole experiment flow, and add them to existing spaces The allowlist now has the tools that the new SKILL.md names and that were missing: list-metric-events and update-off-variation. It also has tools for the rest of that work that the skill does not name: update-experiment, save-and-start-experiment-iteration, create-automated-rollout-config, get-flag-health, get-flag-unique-context-exposures, and search-contexts. Setup now adds the missing ldcli tools to the allowlist of an existing space without --add-mcp-tools, so spaces from older ldcli versions get the new tools. The flag still parses, has no effect, and is deprecated. A new test fails when SKILL.md names a tool that the allowlist does not have. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/setup.go | 13 ++++---- internal/awsdevops/agent.go | 2 +- internal/awsdevops/agent_test.go | 3 +- internal/awsdevops/allowlist_test.go | 45 +++++----------------------- internal/awsdevops/awsdevops_test.go | 4 +-- internal/awsdevops/setup.go | 38 +++++++++++------------ internal/awsdevops/skill_test.go | 14 +++++++++ 7 files changed, 49 insertions(+), 70 deletions(-) diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 76cc07396..4357488b5 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -56,10 +56,10 @@ which can ask you to sign in. The agent then acts as the LaunchDarkly member who approved access, in every agent space. If the agent space already has an MCP server, setup keeps it. -A space that reuses a server keeps its tool allowlist. If the allowlist misses -tools that this ldcli version allows, setup warns. To add them, pass ---add-mcp-tools. AWS does not send back the classification of each tool, so -the rewrite gives every tool the ldcli classification. +A space that reuses a server keeps the tools in its allowlist. If the +allowlist misses tools that this ldcli version allows, setup adds them. AWS +does not send back the classification of each tool, so the rewrite gives +every tool the ldcli classification. The experiment-orchestration skill tells the agent how to run an experiment from a goal to a report. Every agent type in the space can load it. If the @@ -92,8 +92,8 @@ does the other steps.`, cmd.Flags().String(mcpServiceIDFlag, "", "Registered MCP server to connect, instead of the server of the AWS account") cmd.Flags().String(mcpEndpointFlag, "", "MCP endpoint to register. Setup reuses only a server at this endpoint. Defaults to "+awsdevops.MCPServerEndpoint) cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") - cmd.Flags().Bool(addMCPToolsFlag, false, "Add the ldcli tools that the MCP server allowlist of an existing agent space misses. "+ - "Every tool gets the ldcli classification") + cmd.Flags().Bool(addMCPToolsFlag, false, "Has no effect. Setup adds the missing tools without it") + _ = cmd.Flags().MarkDeprecated(addMCPToolsFlag, "setup now adds the missing MCP tools without it") cmd.Flags().Bool(yesFlag, false, "Ask no questions and accept every step") @@ -653,7 +653,6 @@ func setupOptions(cmd *cobra.Command) (awsdevops.SetupOptions, error) { MCPEndpoint: mustString(cmd, mcpEndpointFlag), OpenURL: browser.OpenURL, ReplaceSkill: mustBool(cmd, replaceSkillFlag), - AddMCPTools: mustBool(cmd, addMCPToolsFlag), } idcSet := opts.IdcInstanceARN != "" diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index dc5e4c9e9..9981b4de4 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -146,7 +146,7 @@ func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPSte return step } want, err := spaceAgentTools(ctx, clients, space.ID, mcp.ServiceID, step.ServerName) - if mcp.AddTools && mcp.CanAddTools { + if mcp.CanAddTools { // The MCP step adds these tools to the allowlist before the agent step. want = union(want, toolIDs(step.ServerName, mcp.MissingTools)) } diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index f2527dfc9..503a28411 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -246,7 +246,7 @@ func TestPlanSetupNamesAgentToolsAfterANewServer(t *testing.T) { assert.Contains(t, plan.Text().Agent, "the MCP tools of server "+awsdevops.MCPOAuthServerName) } -func TestSetupGivesAgentTheToolsThatAddMCPToolsAdds(t *testing.T) { +func TestSetupGivesAgentTheToolsThatSetupAddsToTheAllowlist(t *testing.T) { old := awsdevops.AgentTools(awsdevops.MCPServerName)[:1] agent := &fakeAgent{ services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, @@ -260,7 +260,6 @@ func TestSetupGivesAgentTheToolsThatAddMCPToolsAdds(t *testing.T) { } clients := newTestClients(agent, newFakeIAM()) opts := reuseOptions() - opts.AddMCPTools = true plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) require.NoError(t, err) diff --git a/internal/awsdevops/allowlist_test.go b/internal/awsdevops/allowlist_test.go index caff35661..faaa9328a 100644 --- a/internal/awsdevops/allowlist_test.go +++ b/internal/awsdevops/allowlist_test.go @@ -27,13 +27,6 @@ func mcpAssociation(tools ...string) agenttypes.Association { } } -func reuseServerOptions() awsdevops.SetupOptions { - opts := reuseOptions() - opts.AddMCPTools = true - - return opts -} - // An older ldcli allowed fewer tools. A re-run adds the new ones, so the skill // can use them. func TestSetupAddsMissingToolsToAnExistingAllowlist(t *testing.T) { @@ -43,13 +36,13 @@ func TestSetupAddsMissingToolsToAnExistingAllowlist(t *testing.T) { } clients := newTestClients(agent, newFakeIAM()) - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseServerOptions(), true) + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) require.Equal(t, awsdevops.MCPReuse, plan.MCP.Action) assert.Contains(t, plan.Text().MCPServer, "and add get-project, get-environment, list-flags,") assert.Contains(t, plan.Text().MCPServer, "update-flag-settings") - result, err := awsdevops.RunPlan(context.Background(), clients, reuseServerOptions(), plan) + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) require.Len(t, agent.updateAssociations, 1) @@ -75,7 +68,7 @@ func TestSetupKeepsACompleteAllowlist(t *testing.T) { associations: []agenttypes.Association{mcpAssociation(awsdevops.MCPTools()...)}, } - result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseServerOptions()) + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) require.NoError(t, err) assert.Empty(t, agent.updateAssociations) @@ -92,39 +85,17 @@ func TestSetupLeavesAnAllowlistWithForeignTools(t *testing.T) { } clients := newTestClients(agent, newFakeIAM()) - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseServerOptions(), true) + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) assert.Contains(t, plan.Text().MCPServer, "which setup cannot add") - result, err := awsdevops.RunPlan(context.Background(), clients, reuseServerOptions(), plan) + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) assert.Empty(t, agent.updateAssociations) assert.Contains(t, result.MCPToolsError, "Add the missing tools in the AWS console") } -// Setup cannot tell an older ldcli allowlist from one that someone trimmed, -// so without --add-mcp-tools it only warns. -func TestSetupWarnsAboutMissingToolsWithoutTheFlag(t *testing.T) { - agent := &fakeAgent{ - services: []agenttypes.RegisteredService{launchDarklyMCPService("mcp-1")}, - associations: []agenttypes.Association{mcpAssociation("get-flag")}, - } - clients := newTestClients(agent, newFakeIAM()) - opts := reuseServerOptions() - opts.AddMCPTools = false - - plan, err := awsdevops.PlanSetup(context.Background(), clients, opts, true) - require.NoError(t, err) - assert.Contains(t, plan.Text().MCPServer, "Pass --add-mcp-tools to add them") - - result, err := awsdevops.RunPlan(context.Background(), clients, opts, plan) - require.NoError(t, err) - - assert.Empty(t, agent.updateAssociations) - assert.Contains(t, result.MCPToolsError, "--add-mcp-tools") -} - // The run makes only the change that the plan showed. func TestSetupSkipsAnAllowlistThatChangedAfterThePlan(t *testing.T) { agent := &fakeAgent{ @@ -133,11 +104,11 @@ func TestSetupSkipsAnAllowlistThatChangedAfterThePlan(t *testing.T) { } clients := newTestClients(agent, newFakeIAM()) - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseServerOptions(), true) + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) require.NoError(t, err) agent.associations = []agenttypes.Association{mcpAssociation("get-flag")} - result, err := awsdevops.RunPlan(context.Background(), clients, reuseServerOptions(), plan) + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) require.NoError(t, err) assert.Empty(t, agent.updateAssociations) @@ -153,7 +124,7 @@ func TestSetupWarnsWhenTheAllowlistUpdateFails(t *testing.T) { updateAssociationErr: errors.New("AccessDeniedException: not allowed"), } - result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseServerOptions()) + result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) require.NoError(t, err) assert.Contains(t, result.MCPToolsError, "unable to add") diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index c1860f580..d319220f4 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -909,14 +909,14 @@ func TestMCPAssociationClassifiesTheSkillTools(t *testing.T) { assert.Contains(t, classes, tool) } for _, tool := range []string{ - "toggle-flag", "create-flag", "update-rollout", "archive-flag", "create-experiment", + "toggle-flag", "create-flag", "update-off-variation", "update-rollout", "archive-flag", "create-experiment", "start-experiment-iteration", "stop-experiment-iteration", "start-guarded-rollout", "stop-guarded-rollout", "update-targeting-rules", "update-individual-targets", "create-approval-request", "apply-approval-request", } { assert.Equal(t, agenttypes.ToolClassificationMutative, classes[tool], tool) } - for _, tool := range []string{"get-flag", "list-experiments", "get-experiment-results", "query-change-history"} { + for _, tool := range []string{"get-flag", "list-experiments", "get-experiment-results", "query-change-history", "list-metric-events"} { assert.Equal(t, agenttypes.ToolClassificationReadOnly, classes[tool], tool) } } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index f28acb31d..61db9d5d6 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -67,7 +67,12 @@ var oauthPollInterval = 5 * time.Second // it, run an experiment, read its results, and ramp or clean up the flag. The // skill keeps its experiment record in the flag description and creates a // missing primary metric, so the lists include update-flag-settings and -// create-metric. +// create-metric. It checks that the metric's event arrives with +// list-metric-events, and sets the off variation with update-off-variation +// when create-flag does not take defaults. The lists also have the tools for +// the rest of that work that the skill does not name: change an experiment +// between iterations, check exposures and contexts before a rollout, and +// record an automated rollout for the change. // They also cover incident work: turn a change off for one segment or // context, and ask for approval where an environment needs it, then apply // the request after a reviewer approves it. @@ -83,10 +88,14 @@ var ( "query-change-history", "list-metrics", "get-metric", + "list-metric-events", "list-experiments", "get-experiment", "get-experiment-results", "get-experiment-metric-results", + "get-flag-health", + "get-flag-unique-context-exposures", + "search-contexts", "list-segments", "get-segment", "list-approval-requests", @@ -97,14 +106,18 @@ var ( "toggle-flag", "create-flag", "update-flag-settings", + "update-off-variation", "create-metric", "update-rollout", "archive-flag", "create-experiment", + "update-experiment", "start-experiment-iteration", + "save-and-start-experiment-iteration", "stop-experiment-iteration", "start-guarded-rollout", "stop-guarded-rollout", + "create-automated-rollout-config", "update-targeting-rules", "update-individual-targets", "create-approval-request", @@ -142,10 +155,6 @@ type SetupOptions struct { // experiment-orchestration skill that someone edited. Setup updates a // SKILL.md from an older ldcli version without this option. ReplaceSkill bool - // AddMCPTools adds the ldcli tools that the allowlist of an existing MCP - // server association misses. AWS does not send back the classification of - // each tool, so the rewrite gives every tool the ldcli classification. - AddMCPTools bool // Logf, when set, reports progress as each step completes. Logf func(format string, args ...any) @@ -414,8 +423,6 @@ type MCPStep struct { // them. MissingTools []string CanAddTools bool - // AddTools is set by --add-mcp-tools. - AddTools bool } // shared reports whether other spaces use the server. @@ -569,7 +576,6 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } if association != nil { plan.MCP.MissingTools, plan.MCP.CanAddTools = missingTools(*association) - plan.MCP.AddTools = opts.AddMCPTools } } @@ -836,12 +842,9 @@ func (m MCPStep) describe() string { } switch { case len(m.MissingTools) == 0: - case m.CanAddTools && m.AddTools: + case m.CanAddTools: line += ", and add " + strings.Join(m.MissingTools, ", ") + " to its allowlist. The rewrite gives each tool the ldcli classification" - case m.CanAddTools: - line += ". Its allowlist misses " + strings.Join(m.MissingTools, ", ") + - ". Pass --add-mcp-tools to add them" default: line += ". Its allowlist misses " + strings.Join(m.MissingTools, ", ") + ", which setup cannot add because the allowlist has tools that ldcli does not classify" @@ -1380,10 +1383,8 @@ func missingTools(association agenttypes.Association) (missing []string, canAdd } // extendAllowlist adds the ldcli tools that an existing association misses, -// for example after an ldcli upgrade that allows more tools. Setup cannot tell -// an older ldcli allowlist from one that someone trimmed, so it adds tools only -// with --add-mcp-tools. A failure does not stop setup, because the server works -// with the tools that it has. +// for example after an ldcli upgrade that allows more tools. A failure does not +// stop setup, because the server works with the tools that it has. func extendAllowlist( ctx context.Context, clients Clients, @@ -1402,11 +1403,6 @@ func extendAllowlist( result.MCPToolsError = fmt.Sprintf("the MCP server allowlist of the agent space misses %s, and it has tools "+ "that ldcli does not classify, so setup did not change it. Add the missing tools in the AWS console", tools) - return - case !opts.AddMCPTools: - result.MCPToolsError = fmt.Sprintf("the MCP server allowlist of the agent space misses %s, which the "+ - "experiment-orchestration skill uses. To add them, run setup again with --add-mcp-tools", tools) - return case !slices.Equal(missing, step.MissingTools) || canAdd != step.CanAddTools: // The plan showed another change, so do not make this one. diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index b41f704ba..b28e7bf7c 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -3,6 +3,7 @@ package awsdevops_test import ( "context" "encoding/json" + "regexp" "strings" "testing" @@ -41,6 +42,19 @@ func TestShippedSkillHashesHoldTheCurrentSkill(t *testing.T) { "add the hash of the new SKILL.md to shippedSkillHashes") } +// skillToolPattern matches an MCP tool name in backticks, such as `get-flag`. +var skillToolPattern = regexp.MustCompile("`((?:list|get|create|update|start|stop|toggle|archive|apply|check|query|match|delete|copy|save|search|find|manage|run|clone|setup|preview|link|invite|restrict|unrestrict)-[a-z-]+)`") + +// The agent can call only the allowlisted tools, so every tool that SKILL.md +// names must be in the allowlist. +func TestMCPAllowlistHoldsTheSkillTools(t *testing.T) { + matches := skillToolPattern.FindAllStringSubmatch(awsdevops.ExperimentSkill, -1) + require.NotEmpty(t, matches) + for _, match := range matches { + assert.Contains(t, awsdevops.MCPTools(), match[1], "add the tool that SKILL.md names to the MCP allowlist") + } +} + func TestSetupCreatesSkillInNewSpace(t *testing.T) { agent := &fakeAgent{} From 6a02bed772c17703f6835ef819124b53c6bd4caf Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 11:18:43 -0700 Subject: [PATCH 29/34] feat: download the AWS DevOps Agent skill and prompt from ai-tooling main Setup no longer embeds SKILL.md and the custom agent prompt. The plan downloads both from the main branch of launchdarkly/ai-tooling, and the run writes the texts that the plan compared. If the download fails, setup skips the skill and custom agent steps and warns. A space text counts as ours when it is empty, matches a frozen hash of a released ldcli, or equals an earlier version of the file on ai-tooling main. Setup reads that history only when nothing else matches, and skips the step if it cannot read it. The plan and the result warn when the skill names MCP tools that this ldcli version does not allow. The sync and check scripts, their Makefile targets, and the CI step are gone. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/go.yml | 2 - Makefile | 9 +- cmd/aws_devops_agent/confirm_test.go | 22 +- cmd/aws_devops_agent/plan_print_test.go | 15 + cmd/aws_devops_agent/setup.go | 27 +- internal/awsdevops/agent.go | 56 +++- internal/awsdevops/agent_test.go | 27 +- .../agents/experiment-orchestrator.md | 1 - internal/awsdevops/awsdevops.go | 19 +- internal/awsdevops/awsdevops_test.go | 9 +- internal/awsdevops/export_test.go | 19 +- internal/awsdevops/setup.go | 14 +- internal/awsdevops/skill.go | 125 ++++++-- internal/awsdevops/skill_test.go | 44 +-- .../skills/experiment-orchestration/SKILL.md | 226 --------------- internal/awsdevops/source.go | 144 +++++++++ internal/awsdevops/source_test.go | 273 ++++++++++++++++++ scripts/check-aws-devops-agent-changed.sh | 9 - scripts/sync-aws-devops-agent.sh | 14 - 19 files changed, 667 insertions(+), 388 deletions(-) delete mode 100644 internal/awsdevops/agents/experiment-orchestrator.md delete mode 100644 internal/awsdevops/skills/experiment-orchestration/SKILL.md create mode 100644 internal/awsdevops/source.go create mode 100644 internal/awsdevops/source_test.go delete mode 100755 scripts/check-aws-devops-agent-changed.sh delete mode 100755 scripts/sync-aws-devops-agent.sh diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1d1fa2f91..7599052ac 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -25,8 +25,6 @@ jobs: go-version-file: go.mod - name: build run: go build . - - name: check AWS DevOps Agent files - run: make aws-devops-agent-check - uses: actions/setup-python@v7 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/Makefile b/Makefile index 193ae2411..59ba190ae 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,4 @@ -.PHONY: aws-devops-agent-check aws-devops-agent-sync build generate log test vendor - -aws-devops-agent-check: - make aws-devops-agent-sync - ./scripts/check-aws-devops-agent-changed.sh - -aws-devops-agent-sync: - ./scripts/sync-aws-devops-agent.sh +.PHONY: build generate log test vendor build: go build -o ldcli diff --git a/cmd/aws_devops_agent/confirm_test.go b/cmd/aws_devops_agent/confirm_test.go index 14abe2b43..8443fd989 100644 --- a/cmd/aws_devops_agent/confirm_test.go +++ b/cmd/aws_devops_agent/confirm_test.go @@ -129,13 +129,27 @@ func countingClients(spaces ...agenttypes.AgentSpace) (awsdevops.Clients, *write w := &writes{} return awsdevops.Clients{ - Agent: countingAgent{stubAgent: stubAgent{spaces: spaces}, writes: w}, - IAM: countingIAM{writes: w}, - STS: accountSTS{}, - Region: "eu-west-2", + Agent: countingAgent{stubAgent: stubAgent{spaces: spaces}, writes: w}, + IAM: countingIAM{writes: w}, + STS: accountSTS{}, + Sources: stubSources{skill: "---\nname: experiment-orchestration\ndescription: Run experiments.\n---\n"}, + Region: "eu-west-2", }, w } +// stubSources serves the files of ai-tooling main from memory. +type stubSources struct{ skill string } + +func (s stubSources) File(_ context.Context, _, path string) (string, error) { + if strings.HasSuffix(path, "SKILL.md") { + return s.skill, nil + } + + return "# Experiment orchestrator\n", nil +} + +func (stubSources) Commits(context.Context, string) ([]string, error) { return nil, nil } + // noReadStdin fails the test on any read. type noReadStdin struct{ t *testing.T } diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index d723c7af0..cb165e481 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -406,3 +406,18 @@ func TestSetupJSONWarnsOnStderrWhenSkillFails(t *testing.T) { assert.Contains(t, stdoutOf(cmd), `"skillAction": "failed"`) assert.Contains(t, errOut.String(), "Warning: unable to create skill "+awsdevops.ExperimentSkillName) } + +// A skill that names tools outside the allowlist goes in, and setup warns. +func TestSetupWarnsWhenSkillNamesUnknownTools(t *testing.T) { + cmd, errOut := setupTestCmd(t, "") + cmd.SetIn(noReadStdin{t}) + clients, _ := countingClients() + clients.Sources = stubSources{skill: "---\nname: experiment-orchestration\n---\nCall `delete-everything`.\n"} + + require.NoError(t, setupWith(cmd, clients, newSpaceOptions(), false, false)) + + assert.Contains(t, stdoutOf(cmd), `"skillUnknownTools": [`) + assert.Contains(t, errOut.String(), "The skill names MCP tools that this ldcli version does not allow: delete-everything") + assert.Contains(t, errOut.String(), "Warning: skill "+awsdevops.ExperimentSkillName+ + " names MCP tools that this ldcli version does not allow, so the agent cannot call them: delete-everything") +} diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 4357488b5..024b997f9 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -31,6 +31,13 @@ const ( yesFlag = "yes" ) +// unknownToolsWarning says that the skill names tools that the agent cannot +// call. +func unknownToolsWarning(tools []string) string { + return fmt.Sprintf("skill %s names MCP tools that this ldcli version does not allow, so the agent cannot call them: %s", + awsdevops.ExperimentSkillName, strings.Join(tools, ", ")) +} + func NewSetupCmd(analyticsTrackerFn analytics.TrackerFn) *cobra.Command { cmd := &cobra.Command{ Use: "setup", @@ -61,12 +68,18 @@ allowlist misses tools that this ldcli version allows, setup adds them. AWS does not send back the classification of each tool, so the rewrite gives every tool the ldcli classification. +Setup downloads the skill and the custom agent prompt from the main branch of +launchdarkly/ai-tooling on GitHub, so those steps need access to GitHub. If +the download fails, setup skips both steps and warns. + The experiment-orchestration skill tells the agent how to run an experiment from a goal to a report. Every agent type in the space can load it. If the -skill has the text of an older ldcli version, setup updates it. If someone -edited it, setup does not change it. To overwrite the edits, pass ---replace-skill. Setting up the skill is optional: if its step fails, setup -warns and does the other steps. +skill has the text of an older ldcli version or of an earlier version on +ai-tooling, setup updates it. If someone edited it, setup does not change it. +To overwrite the edits, pass --replace-skill. Setting up the skill is +optional: if its step fails, setup warns and does the other steps. If the +skill names MCP tools that this ldcli version does not allow, setup warns, +because the agent cannot call them. The experiment-orchestrator custom agent runs the skill with the LaunchDarkly MCP tools that the space allows. It needs the MCP server and the skill. @@ -183,6 +196,9 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.SkillError) } + if len(result.SkillUnknownTools) > 0 { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", unknownToolsWarning(result.SkillUnknownTools)) + } if result.AgentError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.AgentError) } @@ -196,6 +212,9 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.SkillError) } + if len(result.SkillUnknownTools) > 0 { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", unknownToolsWarning(result.SkillUnknownTools)) + } if result.AgentError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.AgentError) } diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 9981b4de4..6f10f9943 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -2,7 +2,6 @@ package awsdevops import ( "context" - _ "embed" "encoding/json" "errors" "fmt" @@ -14,13 +13,11 @@ import ( agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" ) -//go:embed agents/experiment-orchestrator.md -var ExperimentAgentPrompt string - const ExperimentAgentName = "experiment-orchestrator" -// shippedAgentHashes are the prompts that ldcli releases sent. When you change -// the prompt, add its hash. Never remove a hash. +// shippedAgentHashes are the prompts that released ldcli versions embedded. +// The list is frozen: setup now downloads the prompt from ai-tooling and +// compares its history there, so nobody adds to it. Never remove a hash. var shippedAgentHashes = []string{ "d3d2124c29e290df9cadd38e9bae2619ac8622cabb180fa0ad4fc59d5358dee5", } @@ -91,6 +88,9 @@ type AgentStep struct { Tools []string Skills []string NoFile bool + // Prompt is the prompt on ai-tooling main at plan time, which the step + // writes. + Prompt string Err error } @@ -104,7 +104,8 @@ func (s AgentStep) describe() string { case AgentKeep: return "Keep custom agent " + label case AgentUpdate: - return fmt.Sprintf("Update custom agent %s to the prompt and tools of this ldcli version", label) + return fmt.Sprintf("Update custom agent %s to the prompt on the main branch of %s and the tools of this ldcli version", + label, aiToolingRepo) case AgentLeave: return fmt.Sprintf("Leave custom agent %s unchanged because %s. %s", label, why, agentResetHint) case AgentSkip: @@ -124,8 +125,22 @@ func agentFailure(err error) string { "aidevops:CreateAsset, aidevops:UpdateAsset, and aidevops:ListAssociations", err) } -func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPStep, skill SkillStep) AgentStep { - step := AgentStep{ServerName: mcp.name()} +// planAgent compares the custom agent of the space with prompt, the prompt on +// ai-tooling main. fetchErr is set if the download failed. +func planAgent( + ctx context.Context, + clients Clients, + space SpaceStep, + mcp MCPStep, + skill SkillStep, + prompt string, + fetchErr error, +) AgentStep { + step := AgentStep{ServerName: mcp.name(), Prompt: prompt} + if fetchErr != nil { + step.Action, step.Err = AgentSkip, fetchErr + return step + } if space.Action == SpaceCreate { return step } @@ -139,7 +154,7 @@ func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPSte } step.AssetID, step.Version = aws.ToString(asset.AssetId), aws.ToInt32(asset.Version) step.Tools, step.Skills = agentMetadata(*asset) - prompt, found, err := assetText(ctx, clients, space.ID, step.AssetID, agentFilePath) + text, found, err := assetText(ctx, clients, space.ID, step.AssetID, agentFilePath) step.NoFile = !found if err != nil { step.Action, step.Err = AgentSkip, err @@ -150,14 +165,27 @@ func planAgent(ctx context.Context, clients Clients, space SpaceStep, mcp MCPSte // The MCP step adds these tools to the allowlist before the agent step. want = union(want, toolIDs(step.ServerName, mcp.MissingTools)) } + current := found && text == prompt switch { case err != nil: step.Action, step.Err = AgentSkip, err - case prompt == ExperimentAgentPrompt && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && + + return step + case current && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && !slices.ContainsFunc(want, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): step.Action = AgentKeep - case found && (prompt == "" || slices.Contains(shippedAgentHashes, skillHash(prompt))): - // An empty prompt has nothing to lose. + + return step + case current: + step.Action = AgentUpdate + + return step + } + ours, err := isOurs(ctx, clients, shippedAgentHashes, agentSourcePath, text, found) + switch { + case err != nil: + step.Action, step.Err = AgentSkip, err + case ours: step.Action = AgentUpdate default: step.Action = AgentLeave @@ -284,7 +312,7 @@ func doAgentStep( } content := &agenttypes.AssetContentMemberFile{Value: agenttypes.AssetFileContent{ Path: aws.String(agentFilePath), - Body: &agenttypes.AssetFileBodyMemberText{Value: ExperimentAgentPrompt}, + Body: &agenttypes.AssetFileBodyMemberText{Value: step.Prompt}, }} skills := []string{result.SkillAssetID} if step.Action == AgentUpdate { diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 503a28411..84b82029f 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -13,13 +13,6 @@ import ( "github.com/launchdarkly/ldcli/internal/awsdevops" ) -// A change to the prompt must add its hash, so the next release updates the -// agents that this release makes. -func TestShippedAgentHashesHoldTheCurrentPrompt(t *testing.T) { - assert.Contains(t, awsdevops.ShippedAgentHashes(), awsdevops.SkillHash(awsdevops.ExperimentAgentPrompt), - "add the hash of the new prompt to shippedAgentHashes") -} - func TestAgentToolsStartWithTheServerName(t *testing.T) { tools := awsdevops.AgentTools("LaunchDarkly") @@ -50,7 +43,7 @@ func TestSetupCreatesAgentInNewSpace(t *testing.T) { file, ok := in.Content.(*agenttypes.AssetContentMemberFile) require.True(t, ok) assert.Equal(t, "experiment-orchestrator.md", aws.ToString(file.Value.Path)) - assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentAgentPrompt}, file.Value.Body) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: testAgentPrompt}, file.Value.Body) assert.Equal(t, "ki-agent-created", result.AgentAssetID) assert.Equal(t, awsdevops.AgentCreated, result.AgentAction) assert.Empty(t, result.AgentError) @@ -84,8 +77,8 @@ func TestSetupKeepsCurrentAgent(t *testing.T) { skillAsset("ki-ours", "experiment-orchestration"), agentAsset("ki-agent", "experiment-orchestrator", awsdevops.AgentTools(awsdevops.MCPOAuthServerName), []string{"ki-ours"}), }, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, - agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + skillFiles: map[string]string{"ki-ours": testSkill}, + agentFiles: map[string]string{"ki-agent": testAgentPrompt}, } result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) @@ -103,8 +96,8 @@ func TestSetupUpdatesAgentWithOldTools(t *testing.T) { skillAsset("ki-ours", "experiment-orchestration"), agentAsset("ki-agent", "experiment-orchestrator", []string{awsdevops.MCPOAuthServerName + "_get-flag"}, []string{"ki-ours"}), }, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, - agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + skillFiles: map[string]string{"ki-ours": testSkill}, + agentFiles: map[string]string{"ki-agent": testAgentPrompt}, } result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) @@ -177,8 +170,8 @@ func TestPlanSetupKeepsAgentWithAddedTools(t *testing.T) { skillAsset("ki-ours", "experiment-orchestration"), agentAsset("ki-agent", "experiment-orchestrator", tools, []string{"ki-other", "ki-ours"}), }, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, - agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + skillFiles: map[string]string{"ki-ours": testSkill}, + agentFiles: map[string]string{"ki-agent": testAgentPrompt}, } plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) @@ -192,7 +185,7 @@ func TestSetupUpdateKeepsAddedToolsAndSkills(t *testing.T) { assets: []agenttypes.Asset{ agentAsset("ki-agent", "experiment-orchestrator", []string{"Datadog_query-logs"}, []string{"ki-other"}), }, - agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + agentFiles: map[string]string{"ki-agent": testAgentPrompt}, } _, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) @@ -255,8 +248,8 @@ func TestSetupGivesAgentTheToolsThatSetupAddsToTheAllowlist(t *testing.T) { skillAsset("ki-ours", "experiment-orchestration"), agentAsset("ki-agent", "experiment-orchestrator", old, []string{"ki-ours"}), }, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, - agentFiles: map[string]string{"ki-agent": awsdevops.ExperimentAgentPrompt}, + skillFiles: map[string]string{"ki-ours": testSkill}, + agentFiles: map[string]string{"ki-agent": testAgentPrompt}, } clients := newTestClients(agent, newFakeIAM()) opts := reuseOptions() diff --git a/internal/awsdevops/agents/experiment-orchestrator.md b/internal/awsdevops/agents/experiment-orchestrator.md deleted file mode 100644 index 058e3e8ce..000000000 --- a/internal/awsdevops/agents/experiment-orchestrator.md +++ /dev/null @@ -1 +0,0 @@ -You are an experimentation agent that moves a product goal through a LaunchDarkly experiment and, if the experiment wins, a rollout. Load the experiment-orchestration skill and follow it as the source of truth for the procedure, using the LaunchDarkly MCP tools for every flag, metric, experiment, approval request, and rollout action. Never start a goal that a person did not ask for. At each step, give the output the skill specifies, and on every terminal outcome, deliver the report defined by the skill's final step. diff --git a/internal/awsdevops/awsdevops.go b/internal/awsdevops/awsdevops.go index 848504ac5..632b46742 100644 --- a/internal/awsdevops/awsdevops.go +++ b/internal/awsdevops/awsdevops.go @@ -85,10 +85,12 @@ type STSAPI interface { // Clients bundles the AWS clients and the resolved session details the // provisioning steps run against. type Clients struct { - Agent AgentAPI - IAM IAMAPI - STS STSAPI - Region string + Agent AgentAPI + IAM IAMAPI + STS STSAPI + // Sources reads the skill and the custom agent prompt from ai-tooling. + Sources Sources + Region string } // NewClients builds AWS clients from the ambient credential chain: environment @@ -130,10 +132,11 @@ func NewClients(ctx context.Context, region, profile string) (Clients, error) { } return Clients{ - Agent: devopsagent.NewFromConfig(cfg), - IAM: iam.NewFromConfig(cfg), - STS: sts.NewFromConfig(cfg), - Region: cfg.Region, + Agent: devopsagent.NewFromConfig(cfg), + IAM: iam.NewFromConfig(cfg), + STS: sts.NewFromConfig(cfg), + Sources: NewGitHubSources(), + Region: cfg.Region, }, nil } diff --git a/internal/awsdevops/awsdevops_test.go b/internal/awsdevops/awsdevops_test.go index d319220f4..b58cc2330 100644 --- a/internal/awsdevops/awsdevops_test.go +++ b/internal/awsdevops/awsdevops_test.go @@ -478,10 +478,11 @@ func (f fakeSTS) GetCallerIdentity(_ context.Context, _ *sts.GetCallerIdentityIn func newTestClients(agent *fakeAgent, iamClient *fakeIAM) awsdevops.Clients { return awsdevops.Clients{ - Agent: agent, - IAM: iamClient, - STS: fakeSTS{account: testAccountID}, - Region: "us-east-1", + Agent: agent, + IAM: iamClient, + STS: fakeSTS{account: testAccountID}, + Sources: newFakeSources(), + Region: "us-east-1", } } diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 6ff076d1a..d8b648ad9 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -33,14 +33,15 @@ func ShipSkill(text string) func() { return func() { shippedSkillHashes = saved } } -// ShippedSkillHashes are the hashes of the shipped skill versions. -func ShippedSkillHashes() []string { return shippedSkillHashes } - -// ExperimentSkillDescription is the description in the front matter of SKILL.md. -func ExperimentSkillDescription() string { return experimentSkillDescription } - -// SkillHash is the hash that setup compares. -func SkillHash(text string) string { return skillHash(text) } +// UnknownSkillTools returns the tools that a SKILL.md names and that the +// allowlist misses, for tests. +func UnknownSkillTools(text string) []string { return unknownSkillTools(text) } + +// SkillSourcePath and AgentSourcePath are the paths of the files in ai-tooling. +const ( + SkillSourcePath = skillSourcePath + AgentSourcePath = agentSourcePath +) // MCPTools lists every tool in the allowlist, for tests. func MCPTools() []string { @@ -56,6 +57,4 @@ func SetOAuthPollInterval(d time.Duration) func() { return func() { oauthPollInterval = old } } -func ShippedAgentHashes() []string { return shippedAgentHashes } - func AgentTools(serverName string) []string { return agentTools(serverName) } diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 61db9d5d6..00a2fffa3 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -184,8 +184,11 @@ type SetupResult struct { SkillAction string `json:"skillAction,omitempty"` // SkillError is set if the skill step failed or was skipped. The skill is // optional, so the rest of the setup ran. - SkillError string `json:"skillError,omitempty"` - AgentAssetID string `json:"agentAssetId,omitempty"` + SkillError string `json:"skillError,omitempty"` + // SkillUnknownTools lists the MCP tools that the skill names and that this + // ldcli version does not allow, so the agent cannot call them. + SkillUnknownTools []string `json:"skillUnknownTools,omitempty"` + AgentAssetID string `json:"agentAssetId,omitempty"` // AgentAction is what the custom agent step did, for example "created". AgentAction string `json:"agentAction,omitempty"` // AgentError is set if the custom agent step failed or was skipped. @@ -579,8 +582,11 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } } - plan.Skill = planSkill(ctx, clients, opts, plan.Space) - plan.Agent = planAgent(ctx, clients, plan.Space, plan.MCP, plan.Skill) + // The skill and the prompt come from ai-tooling main. Both steps compare + // and write these texts, so a later change there does not reach the run. + skill, prompt, fetchErr := fetchSources(ctx, clients.Sources) + plan.Skill = planSkill(ctx, clients, opts, plan.Space, skill, fetchErr) + plan.Agent = planAgent(ctx, clients, plan.Space, plan.MCP, plan.Skill, prompt, fetchErr) if !provision { // Read the app, so the run does not print a URL that does not work. diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 0c719196d..7afdac714 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -3,11 +3,11 @@ package awsdevops import ( "context" "crypto/sha256" - _ "embed" "encoding/hex" "encoding/json" "errors" "fmt" + "regexp" "slices" "strings" @@ -17,23 +17,15 @@ import ( agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" ) -//go:generate ../../scripts/sync-aws-devops-agent.sh +// ExperimentSkillName is the name of the experimentation skill. Setup finds +// the skill by this name, so it does not come from the downloaded SKILL.md. +const ExperimentSkillName = "experiment-orchestration" -// ExperimentSkill is the SKILL.md of the experimentation skill. Its front -// matter gives the name and the description that setup sends as metadata. -// -//go:embed skills/experiment-orchestration/SKILL.md -var ExperimentSkill string - -var ( - ExperimentSkillName = frontMatterField(ExperimentSkill, "name") - experimentSkillDescription = frontMatterField(ExperimentSkill, "description") -) - -// shippedSkillHashes holds the SHA-256 of each SKILL.md that an ldcli release -// sent. A skill with one of these texts has no customer edits, so setup -// updates it without --replace-skill. When you change SKILL.md, add the hash -// of the new text. Never remove a hash. +// shippedSkillHashes holds the SHA-256 of each SKILL.md that a released ldcli +// embedded. A skill with one of these texts has no customer edits, so setup +// updates it without --replace-skill. The list is frozen: setup now downloads +// SKILL.md from ai-tooling and compares its history there, so nobody adds to +// it. Never remove a hash. var shippedSkillHashes = []string{ "57844bc3c5aca292ef88b5f949cd791206074b5cfc850ea52ebdd40020ead695", "c4c4e12aeff900402dc17ff370bc1e78eadcb34abac5b256319c7cd793bd73f3", @@ -63,6 +55,24 @@ func frontMatterField(text, key string) string { return "" } +// skillToolPattern matches an MCP tool name in backticks, such as `get-flag`. +var skillToolPattern = regexp.MustCompile("`((?:list|get|create|update|start|stop|toggle|archive|apply|check|query|match|delete|copy|save|search|find|manage|run|clone|setup|preview|link|invite|restrict|unrestrict)-[a-z-]+)`") + +// unknownSkillTools returns the MCP tools that a SKILL.md names and that this +// ldcli version does not allow. The agent cannot call them. +func unknownSkillTools(text string) []string { + var unknown []string + for _, match := range skillToolPattern.FindAllStringSubmatch(text, -1) { + tool := match[1] + if slices.Contains(mcpReadOnlyTools, tool) || slices.Contains(mcpMutativeTools, tool) || slices.Contains(unknown, tool) { + continue + } + unknown = append(unknown, tool) + } + + return unknown +} + func skillHash(text string) string { sum := sha256.Sum256([]byte(text)) @@ -96,6 +106,12 @@ type SkillStep struct { // NoFile means that the skill has no SKILL.md, for example a skill that // someone made from a PDF. NoFile bool + // Text is the SKILL.md on ai-tooling main at plan time, which the step + // writes. + Text string + // UnknownTools are the MCP tools that Text names and that this ldcli + // version does not allow. + UnknownTools []string // Err says why the step is skipped. Err error } @@ -106,20 +122,30 @@ func (s SkillStep) describe() string { if s.NoFile { why = "it has no " + skillFilePath } + var line string switch s.Action { case SkillKeep: - return "Keep skill " + label + line = "Keep skill " + label case SkillUpdate: - return fmt.Sprintf("Update skill %s to the instructions of this ldcli version", label) + line = fmt.Sprintf("Update skill %s to the instructions on the main branch of %s", label, aiToolingRepo) case SkillLeave: return fmt.Sprintf("Leave skill %s unchanged because %s. Pass --replace-skill to overwrite it", label, why) case SkillReplace: - return fmt.Sprintf("Overwrite skill %s, which is not the ldcli version because %s", label, why) + line = fmt.Sprintf("Overwrite skill %s, which is not the ldcli version because %s", label, why) case SkillSkip: return skillFailure(fmt.Errorf("Skip skill %s: %w", ExperimentSkillName, s.Err)) default: - return fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) + line = fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) } + if len(s.UnknownTools) > 0 { + line += ". " + unknownToolsText(s.UnknownTools) + } + + return line +} + +func unknownToolsText(tools []string) string { + return "The skill names MCP tools that this ldcli version does not allow: " + strings.Join(tools, ", ") } // skillFailure is the text of a failed skill step. A missing permission is @@ -133,7 +159,21 @@ func skillFailure(err error) string { "aidevops:CreateAsset, and aidevops:UpdateAsset", err) } -func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep) SkillStep { +// planSkill compares the skill of the space with want, the SKILL.md on +// ai-tooling main. fetchErr is set if the download failed. +func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, want string, fetchErr error) SkillStep { + if fetchErr != nil { + return SkillStep{Action: SkillSkip, Err: fetchErr} + } + step := planSkillAction(ctx, clients, opts, space, want) + if step.Action != SkillSkip && step.Action != SkillLeave { + step.Text, step.UnknownTools = want, unknownSkillTools(want) + } + + return step +} + +func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, want string) SkillStep { if space.Action == SpaceCreate { return SkillStep{Action: SkillCreate} } @@ -163,10 +203,16 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp switch { case err != nil: return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} - case text == ExperimentSkill: + case found && text == want: step.Action = SkillKeep - case found && (text == "" || slices.Contains(shippedSkillHashes, skillHash(text))): - // An empty file has nothing to lose. + + return step + } + ours, err := isOurs(ctx, clients, shippedSkillHashes, skillSourcePath, text, found) + switch { + case err != nil: + return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} + case ours: step.Action = SkillUpdate case opts.ReplaceSkill: step.Action = SkillReplace @@ -177,6 +223,24 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp return step } +// isOurs reports whether text, the file of an asset, came from ldcli or +// ai-tooling, so setup can replace it without losing an edit. shipped holds +// the hashes of the texts that released ldcli versions embedded. The history +// of path on ai-tooling main is read only when nothing else matches. +func isOurs(ctx context.Context, clients Clients, shipped []string, path, text string, found bool) (bool, error) { + switch { + case !found: + return false, nil + case text == "": + // An empty file has nothing to lose. + return true, nil + case slices.Contains(shipped, skillHash(text)): + return true, nil + } + + return pastVersion(ctx, clients.Sources, path, text) +} + // findSkill returns the skill with this name in the space, or nil if there is // none. AWS keeps skill names unique in a space. func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { @@ -294,6 +358,10 @@ func installSkill( result.SkillAssetID = step.AssetID action, err := doSkillStep(ctx, clients, step, result, logf) result.SkillAction = action + if err == nil { + // The skill in the space now names these tools. + result.SkillUnknownTools = step.UnknownTools + } if err != nil { result.SkillAction = SkillFailed // A plan change stops the step before the write, so setup tried @@ -332,8 +400,9 @@ func doSkillStep( } content := &agenttypes.AssetContentMemberFile{Value: agenttypes.AssetFileContent{ Path: aws.String(skillFilePath), - Body: &agenttypes.AssetFileBodyMemberText{Value: ExperimentSkill}, + Body: &agenttypes.AssetFileBodyMemberText{Value: step.Text}, }} + description := frontMatterField(step.Text, "description") if step.Action == SkillCreate { created, err := clients.Agent.CreateAsset(ctx, &devopsagent.CreateAssetInput{ AgentSpaceId: aws.String(result.AgentSpaceID), @@ -341,7 +410,7 @@ func doSkillStep( Content: content, Metadata: document.NewLazyDocument(map[string]any{ "name": ExperimentSkillName, - "description": experimentSkillDescription, + "description": description, "agent_types": []string{"GENERIC"}, }), }) @@ -360,7 +429,7 @@ func doSkillStep( AgentSpaceId: aws.String(result.AgentSpaceID), AssetId: aws.String(step.AssetID), Content: content, - Metadata: document.NewLazyDocument(map[string]any{"description": experimentSkillDescription}), + Metadata: document.NewLazyDocument(map[string]any{"description": description}), }); err != nil { return "", fmt.Errorf("unable to update skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) } diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index b28e7bf7c..0bc386ddd 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -3,7 +3,6 @@ package awsdevops_test import ( "context" "encoding/json" - "regexp" "strings" "testing" @@ -30,31 +29,6 @@ func metadataOf(t *testing.T, doc document.Interface) map[string]any { return metadata } -func TestExperimentSkillFrontMatter(t *testing.T) { - assert.Equal(t, "experiment-orchestration", awsdevops.ExperimentSkillName) - assert.Contains(t, awsdevops.ExperimentSkill, "# Automated Experimentation") -} - -// A change to SKILL.md must add its hash, so the next release updates the -// copies that this release sends. -func TestShippedSkillHashesHoldTheCurrentSkill(t *testing.T) { - assert.Contains(t, awsdevops.ShippedSkillHashes(), awsdevops.SkillHash(awsdevops.ExperimentSkill), - "add the hash of the new SKILL.md to shippedSkillHashes") -} - -// skillToolPattern matches an MCP tool name in backticks, such as `get-flag`. -var skillToolPattern = regexp.MustCompile("`((?:list|get|create|update|start|stop|toggle|archive|apply|check|query|match|delete|copy|save|search|find|manage|run|clone|setup|preview|link|invite|restrict|unrestrict)-[a-z-]+)`") - -// The agent can call only the allowlisted tools, so every tool that SKILL.md -// names must be in the allowlist. -func TestMCPAllowlistHoldsTheSkillTools(t *testing.T) { - matches := skillToolPattern.FindAllStringSubmatch(awsdevops.ExperimentSkill, -1) - require.NotEmpty(t, matches) - for _, match := range matches { - assert.Contains(t, awsdevops.MCPTools(), match[1], "add the tool that SKILL.md names to the MCP allowlist") - } -} - func TestSetupCreatesSkillInNewSpace(t *testing.T) { agent := &fakeAgent{} @@ -70,13 +44,13 @@ func TestSetupCreatesSkillInNewSpace(t *testing.T) { assert.Equal(t, "skill", aws.ToString(in.AssetType)) metadata := metadataOf(t, in.Metadata) assert.Equal(t, awsdevops.ExperimentSkillName, metadata["name"]) - assert.Equal(t, awsdevops.ExperimentSkillDescription(), metadata["description"]) + assert.Equal(t, testDescription, metadata["description"]) assert.Equal(t, []any{"GENERIC"}, metadata["agent_types"]) assert.NotContains(t, metadata, "status") file, ok := in.Content.(*agenttypes.AssetContentMemberFile) require.True(t, ok) assert.Equal(t, "SKILL.md", aws.ToString(file.Value.Path)) - assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentSkill}, file.Value.Body) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: testSkill}, file.Value.Body) assert.Equal(t, "ki-created", result.SkillAssetID) } @@ -85,13 +59,13 @@ func TestPlanSetupFindsSkillOnLaterPage(t *testing.T) { agent := &fakeAgent{ paginate: true, assets: []agenttypes.Asset{skillAsset("ki-sample", "sample-skip-scheduled-maintenance"), skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + skillFiles: map[string]string{"ki-ours": testSkill}, } plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) require.NoError(t, err) - assert.Equal(t, awsdevops.SkillStep{Action: awsdevops.SkillKeep, AssetID: "ki-ours"}, plan.Skill) + assert.Equal(t, awsdevops.SkillStep{Action: awsdevops.SkillKeep, AssetID: "ki-ours", Text: testSkill}, plan.Skill) assert.Equal(t, "Keep skill experiment-orchestration (ki-ours)", plan.Text().Skill) } @@ -108,7 +82,7 @@ func TestPlanSetupCreatesMissingSkill(t *testing.T) { func TestSetupKeepsSameSkill(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + skillFiles: map[string]string{"ki-ours": testSkill}, } result, err := awsdevops.Setup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions()) @@ -125,7 +99,7 @@ func TestSetupKeepsSameSkill(t *testing.T) { func TestPlanSetupKeepsSameSkillSentAsBytes(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillBytes: map[string][]byte{"ki-ours": []byte(awsdevops.ExperimentSkill)}, + skillBytes: map[string][]byte{"ki-ours": []byte(testSkill)}, } plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) @@ -223,14 +197,14 @@ func TestSetupOverwritesEditedSkillWithFlag(t *testing.T) { assert.Equal(t, []string{"description"}, keysOf(metadataOf(t, in.Metadata))) file, ok := in.Content.(*agenttypes.AssetContentMemberFile) require.True(t, ok) - assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: awsdevops.ExperimentSkill}, file.Value.Body) + assert.Equal(t, &agenttypes.AssetFileBodyMemberText{Value: testSkill}, file.Value.Body) } // The flag does nothing to a skill that is already the same. func TestPlanSetupKeepsSameSkillWithFlag(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": awsdevops.ExperimentSkill}, + skillFiles: map[string]string{"ki-ours": testSkill}, } opts := reuseOptions() opts.ReplaceSkill = true @@ -368,7 +342,7 @@ func TestPlanSetupCreatesSkillDeletedDuringPlan(t *testing.T) { plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) require.NoError(t, err) - assert.Equal(t, awsdevops.SkillStep{Action: awsdevops.SkillCreate}, plan.Skill) + assert.Equal(t, awsdevops.SkillStep{Action: awsdevops.SkillCreate, Text: testSkill}, plan.Skill) } func TestSetupWarnsOnSkillCreateFailure(t *testing.T) { diff --git a/internal/awsdevops/skills/experiment-orchestration/SKILL.md b/internal/awsdevops/skills/experiment-orchestration/SKILL.md deleted file mode 100644 index eed2d61a9..000000000 --- a/internal/awsdevops/skills/experiment-orchestration/SKILL.md +++ /dev/null @@ -1,226 +0,0 @@ ---- -name: "experiment-orchestration" -description: "Takes a product goal through a LaunchDarkly experiment and, if it wins, a guarded rollout (or a regular rollout on plans without guarded rollouts). Uses GitHub for the code change and deployment when it is connected to the agent space; otherwise hands those steps to a person." -license: Apache-2.0 -compatibility: Requires the remotely hosted LaunchDarkly MCP server in an AWS DevOps Agent space. GitHub is optional. -metadata: - author: launchdarkly - version: "0.1.0" ---- - -# Automated Experimentation - -Use this skill when you have a goal you want to move through experimentation (e.g., "increase checkout conversion by 15%", "decrease page load time by 20%"). - -**Core principle: experiment first, then a guarded rollout, or a regular rollout if the plan has no guarded rollouts.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before rolling it out. Never roll out blind — the rollout exists only to scale a change the experiment has already shown to work. - -**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Stop and ask the user only for a planned handoff (merging the PR; any code, review, or deploy step you hand over without GitHub) or if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result). - -**Approvals:** if the LaunchDarkly environment requires approval for a change (for example, to turn a flag on or off, start or stop an experiment, or start a guarded rollout), create an approval request instead of the direct change. The first time you change an entity (a flag, an experiment) in an environment, call the change tool directly; don't guess beforehand whether the environment requires approval. If the response has `requiresApproval: true`, tell the user the change was not made because it needs LaunchDarkly approval, ask who should review it, and submit the returned instructions with `create-approval-request` (or the approval tool that response names). From then on, send every change to that entity in that environment straight to an approval request with the same instructions you would give the direct call, instead of calling the change tool first. Tell the person who must review it. Continue only after `list-approval-requests` shows it approved, then apply it with `apply-approval-request` and go to the next step. If it still shows pending, stop and tell the user it is waiting for review; never apply a pending request, and don't keep re-checking in the same turn. If a reviewer rejects it, stop and report. - -**Blocked tools:** an agent space can block MCP tools that it marks destructive. Likely candidates include, but aren't limited to, `toggle-flag`, `start-experiment-iteration`, `update-rollout`, `update-targeting-rules`, `start-guarded-rollout`, and `stop-experiment-iteration`. A blocked call fails without a LaunchDarkly error, or returns nothing, so you can't tell why it failed. If a write fails without a LaunchDarkly error that explains it, or returns nothing, first read the state back (`get-flag` or `get-experiment`): an empty response may still have made the change. If it didn't, don't retry: tell the user the tool may be blocked as destructive in this agent space, and hand them the step as below. - -**When a step can't succeed through you:** a blocked tool, or any other write you can't complete (including `create-experiment`, `create-metric`, and the approval tools). Hand the step to the user with exact instructions, not a summary. Give the link, what to click, and every value the call would have sent, using the real names and keys from this run: variation names, percentages, metric keys, thresholds, randomization unit, stage weights and durations, the winning treatment. If the environment requires approval, tell them the UI will ask them to submit the change for review and who should review it. Then wait until they confirm it's done, read the state back to check it, and continue from the next step. For example: -- Turn the flag on or off, or change its default rule or off variation: open `https://app.launchdarkly.com/projects//flags//targeting?env=&selected-env=`, set the toggle, the default rule, or the off variation to the exact variation name or percentages, and save. -- Start a guarded rollout: on the same targeting page, set the default rule to a guarded rollout and give the variation to roll out, the randomization unit, each metric key with its regression threshold and roll back on regression, and each stage's percentage and monitoring window from Step 7. -- Start or stop the experiment: open the `designUrl` that `create-experiment` returned, or `https://app.launchdarkly.com/projects//experiments//design?env=&selected-env=`. When the user stops it, tell them to choose Control as the winner, for the reason Step 7 gives. - -**Plan limits:** if LaunchDarkly returns an error saying the plan doesn't include a feature this skill needs, and this skill has no fallback for it (it has one only for guarded rollouts), tell the user which feature is missing and that it needs a plan that includes it, then stop. Treat a failure as a plan limit only when the error says so; a silent failure is a possibly blocked tool, above. If the account is on another LaunchDarkly instance (for example, federal), use that instance's host instead of `app.launchdarkly.com`. - -**GitHub:** Steps 2–5 use GitHub when it is connected to the agent space. Before Step 2, check whether you have GitHub access to the target repository. If you don't, read no code: ask the user where the change belongs, and in Step 3 give them the flag key and the change to make, then wait until they confirm the change is reviewed, merged, and deployed, and skip to Step 6. Triggering workflows and proposing fixes need the GitHub connection's Read and write permissions; with Read-only, hand those steps to the user. - -**Always produce the Step 8 report on every terminal outcome.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. - -## Step 1: Goal Clarification - -Before doing anything, get answers to: - -1. **What metric measures success?** *(Required)* e.g. conversion rate, page load time, bounce rate. Reserve your error-rate metric as a safety guardrail — never use it as the primary success metric. -2. **What's the target improvement?** *(Required)* e.g. 15% increase, 200ms decrease. -3. **What's the current baseline?** *(Optional — infer from production metrics if not given)* -4. **What parts of the app are in scope?** *(Optional — infer from the codebase if not given)* -5. **Any constraints?** *(Optional)* e.g. no changes to the payment flow. -6. **How much regression should roll a change back?** *(Optional — propose one for the success metric and each guardrail metric if not given)* Used as each metric's `regressionThreshold` in Step 7. - -Questions 1–2 are required before proceeding: until the user has answered both, make no changes (no flag, metric, or experiment). If a question gets no answer, ask again or stop; never assume an answer. Infer 3–6 where possible and confirm your assumptions with the user before implementing. - -**Find the metrics.** The experiment needs the success metric. LaunchDarkly requires at least one metric to start a guarded rollout, and this skill requires an error-rate metric as a guardrail in both the experiment and the rollout. Find them before Step 2: - -1. Choose the randomization unit (typically "user"). Every metric must list it in its randomization units, or the guarded rollout rejects the metric. -2. Call `list-metrics` with a `query` for the success metric. For errors, look first for LaunchDarkly's built-in error-rate metric (key ending in `telemetry-default-error-rate`, event `$ld:telemetry:error`), then `query: "error"`. Check that each metric's randomization units include the unit from item 1. -3. For the success metric, call `list-metric-events` with `query` set to its event key, in the environment the experiment will use. It returns at most 50 keys, so always pass the query. Don't run this check for the error-rate metric: no recent error events may just mean no errors. -4. Confirm the metric keys with the user. If a metric doesn't exist, plan to create it in Step 3: Implementation. If the success metric exists but its event isn't arriving, plan to add its `track()` call in Step 3: Implementation instead of creating a second metric. - -## Step 2: Hypothesis Generation - -Explore the target repository/codebase to find a plausible change: - -1. Search and read the relevant code paths. -2. Think through what UI/UX or logic change could plausibly move the chosen metric. -3. Check whether this hypothesis (or something close to it) has already been tried and failed — look at flag history or archived flags with similar naming. Avoid repeating a known failure. -4. Present the hypothesis to the user before proceeding, along with your reasoning and any inferred assumptions from Step 1. If the user wants to test more than one version of the change (for example, three button labels), list each version; Step 3 then creates a multivariate flag. Each version splits the same 10% of traffic, so more than three versions makes significance slower to reach; say so to the user. - -**Before finalizing a flag key, check for collisions:** look up any candidate flag key first. -- Already fully shipped (100% one variation, no split) → already decided, pick a different hypothesis. -- Actively running an experiment → mid-flight, don't compete with it, pick a different hypothesis. -- Doesn't exist → safe to create. - -## Step 3: Implementation - -1. Create the feature flag, OFF by default in all environments, with `clientSideAvailability` of `{"usingEnvironmentId": true, "usingMobileKey": true}` so that every SDK can read it. Pick its kind from the hypothesis: - - **One version of the change:** a boolean flag with `defaults` of `{"onVariation": 1, "offVariation": 1}`, so that it serves `false` (the control) even when it is on. - - **Several versions of the change:** a multivariate flag with `kind: "multivariate"` and `variations` listing the control (current behavior) first, then one variation per version, each with a `name` and the `value` the code reads. Set `defaults` to `{"onVariation": 0, "offVariation": 0}`, so that it serves the control even when it is on. - - If `create-flag` errors on `defaults`, create the flag without `defaults`, then call `update-off-variation` with `allEnvs: true`, `variationIndex` and `fallthroughVariationIndex` both set to the control's index (`1` for a boolean flag, `0` for a multivariate flag), and `updateFlagDefaults: true`. Do this right after creating the flag, before any code that reads it is deployed: without it, the flag can serve a treatment to everyone, off or on. Environments that need approval come back blocked with instructions for `create-approval-request`; the others still apply. - - Then read the flag back with `get-flag` in the environment the experiment will use: its off variation and default rule must both be the control. If they aren't, hand the change to the user (see "When a step can't succeed through you") and don't continue until they are. - - Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. If the project has its own flag naming conventions (for example, a required prefix or case), follow them instead. -2. If Step 1: Goal Clarification found no success or error-rate metric, create it now with `create-metric`, with `randomizationUnits` set to the unit from Step 1: Goal Clarification. For an error-rate metric, use `kind: "custom"`, `measureType: "occurrence"` (the share of contexts that hit an error), and `successCriteria: "LowerThanBaseline"`. Include the `track()` call for each new or silent metric's event key in the code change below. -3. If a coding agent is available, hand off the implementation to it, with instructions to gate the change behind the exact flag key from item 1 above. Without a coding agent or GitHub write access, give the user the flag key and the change instead, and wait for their PR (with GitHub connected) or their confirmation that it is deployed (without GitHub). -4. This step is asynchronous — check status periodically rather than looping tightly on it. -5. Once implementation completes, move to Step 4: Release Readiness with the resulting branch/PR. If it fails, report the error and stop. - -## Step 4: Release Readiness - -Run an AWS DevOps Agent release readiness code review on the PR before it merges. It runs automatically on connected repositories with **Auto trigger change review** enabled; otherwise request one. - -- If it passes: ask the user to merge the PR, and wait until they confirm it is merged. -- If it fails: feed the review's specific feedback back into implementation (or to the user, if they made the change) and retry. Cap retries at a small fixed number (e.g. 3 attempts total) — if it still hasn't passed, stop and report the last failure to the user rather than retrying indefinitely. - -Never skip the review. Without GitHub, ask the user to have the PR reviewed and merged. - -## Step 5: Deployment - -The flagged code must be running in the environment the experiment uses before Step 6 turns the flag on; otherwise the experiment measures nothing. Confirm the merged change is deployed there. If the deploy workflow runs only when triggered (for example, `workflow_dispatch`), trigger it as below. - -1. Trigger the deploy workflow on the merge target branch (this needs Read and write GitHub access; otherwise ask the user to deploy and confirm when it's done). Treat "already an in-progress deployment for this ref" as expected de-duplication, not an error — don't re-trigger. -2. Check the workflow run's status periodically rather than looping tightly. -3. If a run reports "running" for far longer than normal, check the workflow run history for that commit before assuming it's still in progress. -4. **Trigger a deployment at most once per attempt.** If you're unsure whether a previous trigger succeeded, check status first — never re-trigger just because you're unsure. -5. On timeout: stop, check the CI run directly, report the situation, ask how to proceed. -6. On explicit failure: stop and report — do not proceed to the experiment. -7. On success: proceed immediately to Step 6: Experiment Phase. - -## Step 6: Experiment Phase (fixed 10%) - -Prove the change on a small, fixed slice of traffic. The experiment and the rollout never run at the same time: a guarded rollout can't start while an experiment is running on the flag, so Step 7 stops the experiment iteration before Phase 2 starts any rollout. - -The control is the variation that keeps current behavior: `false` on a boolean flag, the first variation on a multivariate flag. Every other variation is a treatment: `true` on a boolean flag, one per version on a multivariate flag. - -1. Turn the flag ON; starting an experiment fails while the flag is off. It still serves the control to everyone, because Step 3: Implementation set its defaults to the control. -2. Read the flag to get its current version (required by `create-experiment`) and the ID of every variation. -3. Create an experiment with: - - `flags`: `{"": {"ruleId": "fallthrough", "flagConfigVersion": , "notInExperimentVariationId": ""}}`. The experiment runs on the default rule, the only rule the Step 7 rollouts act on, and every context outside the experiment keeps getting the control. - - `hypothesis`: the hypothesis from Step 2: Hypothesis Generation. - - `treatments`: one per variation. The control is `{"name": "Control", "baseline": true, "allocationPercent": , "parameters": [{"flagKey": "", "variationId": ""}]}`; each treatment is the same with its variation's name, `"baseline": false`, and its variation ID. Unless the user chose another split, split 10% of traffic evenly: each `` is 10 divided by the number of variations, rounded down to two decimals (2 variations: 5 each; 3: 3.33 each; 4: 2.5 each). Each `allocationPercent` is that treatment's share of all traffic on the rule, so the shares add up to 10%. Never use 50 and 50, or any shares that add up to more than 10: 50 and 50 puts 100% of traffic in the experiment. - - `canReshuffleTraffic: false`, so contexts keep their variation if the traffic changes. - - `randomizationUnit`: the randomization unit from Step 1: Goal Clarification. - - `metrics`: the success metric from Step 1: Goal Clarification and the guardrail metrics, each as `{"key": ""}`. Always include your error-rate metric as a guardrail; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. - - `primarySingleMetricKey`: the success metric's key. It is the only primary metric. - - `methodology: "frequentist"` with `analysisConfig` `{"sequentialTestingEnabled": true, "significanceThreshold": "5", "testDirection": "two-sided"}` (the last two are LaunchDarkly's defaults, sent so the config is complete). With two or more treatments, also set `"multipleComparisonCorrectionMethod": "benjamini-hochberg"` and `"multipleComparisonCorrectionScope": "variations"`; LaunchDarkly rejects a method without a scope, and without the correction, comparing several treatments to the control declares more false winners. Step 7 checks results many times; sequential testing keeps significance valid under repeated checks, where a fixed-horizon test would declare false winners. Don't use the Bayesian methodology: it reports a probability of being best, not the significance Step 7 decides on. -4. Start the experiment iteration. Starting it writes the split onto the default rule: 10% of contexts in the experiment, split across the variations, and the other 90% on the control, not analyzed. -5. Move to Step 7 to monitor toward a decision. - -## Step 7: Monitoring & Outcome - -Check status periodically — don't tight-loop. In an interactive session, check once and report progress, then pick back up later. Every MCP write needs a person to approve it, so an unattended run stops at its first write: run checks that may lead to a write in a session where someone can approve, and report progress so the next session can pick it up. - -**Phase 1 — Prove the experiment at 10% (gate before any rollout):** - -Watch for statistical significance on the primary metric. For each metric, call `get-experiment-metric-results`: it returns per treatment `analyzedUnitCount` (contexts in the experiment), `conversionCount` (contexts that sent the metric's event), and `isSignificant`. Decide on `isSignificant`, not on the `isBest` that `get-experiment-results` reports; `isBest` marks the leading treatment even when nothing is significant. - -**Check the instrumentation after the first day.** If the success metric, or any guardrail other than the error-rate metric, has a `conversionCount` of 0 in every treatment, stop and report an instrumentation problem instead of waiting for significance. Zero error events is fine: it means no errors. - -Judge each treatment against the control on its own. A treatment **qualifies** if it has a significant positive lift on the primary metric and no guardrail regression. A treatment **fails** if it has a significant negative lift on the primary metric or a guardrail regression. - -- **Proven:** at least one treatment qualifies, even if another fails. The winner is the qualifying treatment with the largest lift. Stop the experiment iteration with control as the winner, then move to Phase 2. Stopping serves the winner to the experiment's targeting rule (everyone, when the experiment is on the default rule), so a treatment winner would skip the guarded rollout. If the plan has no guarded rollouts, still stop with control as the winner and use the regular rollout in Phase 2. -- **Loser:** no treatment qualifies, and either every treatment fails or any treatment has a guardrail regression. A regressing treatment must not keep serving traffic while the others collect data. Declare a loser, stop the iteration with control as the winner, turn the flag off (do not archive it while the code still evaluates it), skip Phase 2, go straight to the Step 8 report. -- **Anything else** (no significance, or mixed results such as one treatment worse and another flat): keep collecting data. At a reasonable ceiling (e.g. 4 weeks), report "inconclusive, need more traffic", naming any treatment that was significantly worse, stop the iteration with control as the winner, and stop; don't proceed to Phase 2. - -To stop the iteration, pass `stop-experiment-iteration` the `_id` of the control treatment (`baseline: true`) from `get-experiment` as `winningTreatmentId`, in every outcome; it takes a treatment ID, not a variation ID. Stopping with the real winner would serve it to everyone and skip the rollout. - -Never declare a winner off a single data point or before your stats engine confirms significance. Also require an `analyzedUnitCount` of at least 100 in each variation before acting on a significant result; with fewer, keep collecting data. - -**Phase 2 — Guarded rollout ramp (only after Phase 1 proves the change):** - -Start a guarded rollout with: -- `testVariationId`: the winning treatment's variation ID; `controlVariationId`: the control variation ID (both from Step 6). -- Same randomization unit as the experiment. -- **Exactly 3 monitored stages, each below 50%** — e.g. 20% → 30% → 40%, ~60 minutes monitoring each. `rolloutWeight` is in thousandths of a percent (20% is `20000`) and `monitoringWindowMilliseconds` is in milliseconds (60 minutes is `3600000`). Don't add a 100% stage. -- `updateDefaultRuleOnCompletion: true`, so the rollout moves the default rule to the winning variation once the final stage completes cleanly. -- Every metric the experiment used, primary and guardrails, each with `onRegression` set to notify and roll back, and a `regressionThreshold` (required) from Step 1. - -Track stage progression. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, turn the flag off (do not archive it while the code still evaluates it), and go to the Step 8 report. - -Once the final stage completes cleanly and the default rule serves the winning variation, declare a winner and go to the Step 8 report. - -**If the plan has no guarded rollouts:** if `start-guarded-rollout` returns an error saying guarded rollouts aren't available on the plan, use a regular rollout instead. A silent failure is not a plan error: read the flag back and, if no rollout started, hand the guarded rollout to the user (see "When a step can't succeed through you") instead of rolling out to everyone. -1. Read the flag's variations to find the indexes of the winning treatment's variation and the control's. Call `update-rollout` with `rolloutType: "variation"` and the winner's index on the default rule. This serves the winner to everyone at once, with no automatic rollback. -2. Check the guardrail metrics once per monitoring window, for about as long as the guarded rollout would have run (e.g. 3 hours). Reverting is a write that needs a person's approval, so run this watch in a session where someone can approve. -3. If a guardrail regresses, call `update-rollout` with the control's index, declare failed, and go to the Step 8 report. Otherwise, declare a winner and go to the Step 8 report. - -**Retrying after a guarded rollout rollback:** a rollback isn't always caused by your monitored metrics genuinely regressing — it can also be triggered by an unrelated application error surfacing mid-ramp. Before blindly restarting after the user says they've fixed something: -1. Confirm the flag's current state (should be back to 100% control, nothing stuck mid-rollout). -2. Check the change history timing between "advanced to next stage" and "reverted." A rollback within seconds of advancing is inconsistent with a full metric-window regression and points to an external cause instead. -3. If the flag is cleanly reverted and the external cause is confirmed fixed, it's safe to restart the guarded rollout from scratch with the same parameters. -4. Don't silently retry without this check, and don't refuse to retry just because a prior attempt rolled back — a genuinely fixed external cause is a legitimate reason to retry. A metric-driven loser is not — don't retry that. - -**On any terminal outcome, immediately produce the Step 8 report in the same turn** — a one-line "it worked!" note is fine as a lead-in, but the structured report must follow, not wait for a follow-up request. - -## Step 8: Report - -Runs automatically the instant Step 7 reaches a terminal outcome (winner, with the default rule serving it; loser; inconclusive; or rollback/failure). Use this exact structure from Experiment Report to Next Steps: - -```markdown -## Experiment Report: [Goal Description] - -**Date:** [YYYY-MM-DD] -**Goal:** [metric] [direction] by [target]% -**Status:** [achieved / in progress / stalled] - -### Hypothesis -[What we tried and why] - -### Implementation -- Flag: [flag_key] ([boolean / multivariate]) -- Variations: [control, then each treatment] -- Files modified: [list] -- Branch: [branch name] - -### Release Readiness -- [release readiness review passed after N attempt(s) / reviewed by the user] - -### Experiment Phase (10% fixed split) -- Status: [proven / loser / inconclusive] -- Winner: [variation name, or none] -- Duration: [time] -- Metric change: [before] → [after] ([+/-]%) -- Statistical significance: [value, confidence interval] - -### Rollout Phase (if reached) -- Type: [guarded rollout / regular rollout (no guarded rollouts on this plan)] -- Status: [completed / rolled_back / not started] -- Duration: [time] -- Stages reached: [N of 3 monitored stages, guarded rollout only] -- If rolled back and retried: [root cause, outcome of retry] - -### Safety Metrics -- [error-rate metric key]: [baseline] → [final] ([no regression / regression detected]) -- [other guardrails]: [baseline] → [final] ([status]) - -### Next Steps -[What to do next based on the outcome] -``` - -## Safety Rules - -- Always present the hypothesis before implementing. -- Always have the PR reviewed before merging: the release readiness review with GitHub, or the user's review without it. Never skip it. -- **Always prove a change via a fixed small-percentage experiment before starting any rollout** — never roll out blind. -- Always include an error-rate (or equivalent "don't break prod") metric as a guardrail, separate from your success metric. -- Add a performance guardrail (e.g. p95 latency) for performance-focused changes. -- Every guarded rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. A regular rollout has no automatic rollback, so watch its guardrail metrics after rolling out. -- **Keep guarded rollout stages below 50%** and let the rollout move to 100% after the final stage — don't add an explicit 100% stage. -- Distinguish a metric-driven rollback (don't retry) from an external-cause rollback (safe to retry once fixed) before restarting a rolled-back rollout. -- Produce the Step 8 report on every terminal outcome — never defer it to a follow-up ask. diff --git a/internal/awsdevops/source.go b/internal/awsdevops/source.go new file mode 100644 index 000000000..84a4422ef --- /dev/null +++ b/internal/awsdevops/source.go @@ -0,0 +1,144 @@ +package awsdevops + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// Sources reads the files of launchdarkly/ai-tooling, where the skill and the +// custom agent prompt live. +type Sources interface { + // File returns a file of ai-tooling at ref, a branch or a commit SHA. + File(ctx context.Context, ref, path string) (string, error) + // Commits returns the SHAs of the commits on main that changed path. + Commits(ctx context.Context, path string) ([]string, error) +} + +const ( + aiToolingRepo = "launchdarkly/ai-tooling" + aiToolingBranch = "main" + skillSourcePath = "skills/aws-devops-agent/experiment-orchestration/SKILL.md" + agentSourcePath = "skills/aws-devops-agent/agents/experiment-orchestrator.md" +) + +// ErrSourceNotFound means that ai-tooling has no such file at the ref, for +// example at a commit that deleted it. +var ErrSourceNotFound = errors.New("file not found") + +const ( + sourceTimeout = 30 * time.Second + // maxSourceBytes caps a download. The files are a few KB. + maxSourceBytes = 1 << 20 +) + +// GitHubSources reads ai-tooling from GitHub without authentication. +type GitHubSources struct { + Client *http.Client + RawURL string + APIURL string +} + +// NewGitHubSources reads from github.com. +func NewGitHubSources() GitHubSources { + return GitHubSources{ + Client: &http.Client{Timeout: sourceTimeout}, + RawURL: "https://raw.githubusercontent.com", + APIURL: "https://api.github.com", + } +} + +func (s GitHubSources) File(ctx context.Context, ref, path string) (string, error) { + body, err := s.get(ctx, fmt.Sprintf("%s/%s/%s/%s", s.RawURL, aiToolingRepo, url.PathEscape(ref), path)) + if err != nil { + return "", fmt.Errorf("unable to download %s from %s at %s: %w", path, aiToolingRepo, ref, err) + } + + return string(body), nil +} + +func (s GitHubSources) Commits(ctx context.Context, path string) ([]string, error) { + query := url.Values{"sha": {aiToolingBranch}, "path": {path}, "per_page": {"100"}} + body, err := s.get(ctx, fmt.Sprintf("%s/repos/%s/commits?%s", s.APIURL, aiToolingRepo, query.Encode())) + if err != nil { + return nil, fmt.Errorf("unable to list the commits of %s in %s: %w", path, aiToolingRepo, err) + } + var commits []struct { + SHA string `json:"sha"` + } + if err := json.Unmarshal(body, &commits); err != nil { + return nil, fmt.Errorf("unable to read the commits of %s in %s: %w", path, aiToolingRepo, err) + } + shas := make([]string, 0, len(commits)) + for _, commit := range commits { + shas = append(shas, commit.SHA) + } + + return shas, nil +} + +func (s GitHubSources) get(ctx context.Context, address string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) + if err != nil { + return nil, err + } + client := s.Client + if client == nil { + client = &http.Client{Timeout: sourceTimeout} + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + switch { + case resp.StatusCode == http.StatusNotFound: + return nil, ErrSourceNotFound + case resp.StatusCode != http.StatusOK: + return nil, fmt.Errorf("GitHub sent %s", resp.Status) + } + + return io.ReadAll(io.LimitReader(resp.Body, maxSourceBytes)) +} + +// fetchSources downloads the skill and the custom agent prompt from main. +func fetchSources(ctx context.Context, sources Sources) (skill, prompt string, err error) { + if sources == nil { + return "", "", fmt.Errorf("no source for the files of %s", aiToolingRepo) + } + if skill, err = sources.File(ctx, aiToolingBranch, skillSourcePath); err != nil { + return "", "", err + } + if prompt, err = sources.File(ctx, aiToolingBranch, agentSourcePath); err != nil { + return "", "", err + } + + return skill, prompt, nil +} + +// pastVersion reports whether text is an earlier version of path on main. +// A commit that deleted the file has no version, so it is passed over. +func pastVersion(ctx context.Context, sources Sources, path, text string) (bool, error) { + shas, err := sources.Commits(ctx, path) + if err != nil { + return false, err + } + for _, sha := range shas { + old, err := sources.File(ctx, sha, path) + switch { + case errors.Is(err, ErrSourceNotFound): + continue + case err != nil: + return false, err + case old == text: + return true, nil + } + } + + return false, nil +} diff --git a/internal/awsdevops/source_test.go b/internal/awsdevops/source_test.go new file mode 100644 index 000000000..6a2560acd --- /dev/null +++ b/internal/awsdevops/source_test.go @@ -0,0 +1,273 @@ +package awsdevops_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/awsdevops" +) + +// testSkill and testAgentPrompt stand in for the files on ai-tooling main. +const ( + testSkill = "---\n" + + "name: experiment-orchestration\n" + + "description: Run an experiment from a goal to a report.\n" + + "---\n\n" + + "# Automated Experimentation\n\n" + + "Read the flag with `get-flag`, then call `create-experiment`.\n" + testAgentPrompt = "# Experiment orchestrator\n\nRun the experiment-orchestration skill.\n" + testDescription = "Run an experiment from a goal to a report." +) + +// fakeSources serves ai-tooling from memory. files maps "ref path" to text. +type fakeSources struct { + files map[string]string + commits map[string][]string + fileErr error + commitsErr error + calls []string +} + +func newFakeSources() *fakeSources { + return &fakeSources{files: map[string]string{ + "main " + awsdevops.SkillSourcePath: testSkill, + "main " + awsdevops.AgentSourcePath: testAgentPrompt, + }} +} + +func (f *fakeSources) File(_ context.Context, ref, path string) (string, error) { + f.calls = append(f.calls, "File "+ref+" "+path) + if f.fileErr != nil { + return "", f.fileErr + } + text, ok := f.files[ref+" "+path] + if !ok { + return "", awsdevops.ErrSourceNotFound + } + + return text, nil +} + +func (f *fakeSources) Commits(_ context.Context, path string) ([]string, error) { + f.calls = append(f.calls, "Commits "+path) + if f.commitsErr != nil { + return nil, f.commitsErr + } + + return f.commits[path], nil +} + +// addPast puts text in the history of path at sha. +func (f *fakeSources) addPast(path, sha, text string) { + if f.commits == nil { + f.commits = map[string][]string{} + } + f.commits[path] = append(f.commits[path], sha) + f.files[sha+" "+path] = text +} + +func sourcesOf(clients awsdevops.Clients) *fakeSources { + return clients.Sources.(*fakeSources) +} + +func TestUnknownSkillToolsListsToolsThatTheAllowlistMisses(t *testing.T) { + text := "Call `get-flag`, then `delete-everything`, `get-flag`, and `run-magic`. " + + "Plain words like get-flag-x or `not a tool` do not count." + + assert.Equal(t, []string{"delete-everything", "run-magic"}, awsdevops.UnknownSkillTools(text)) +} + +func TestUnknownSkillToolsIsEmptyWhenTheAllowlistHoldsEveryTool(t *testing.T) { + assert.Empty(t, awsdevops.UnknownSkillTools(testSkill)) +} + +// Without GitHub, the plan skips the skill and the custom agent, and setup +// does the other steps. +func TestSetupSkipsSkillAndAgentWhenDownloadFails(t *testing.T) { + agent := &fakeAgent{} + clients := newTestClients(agent, newFakeIAM()) + sourcesOf(clients).fileErr = errors.New("unable to download SKILL.md: dial tcp: no route to host") + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillSkip, plan.Skill.Action) + assert.Equal(t, awsdevops.AgentSkip, plan.Agent.Action) + assert.Equal(t, "Skip skill experiment-orchestration: unable to download SKILL.md: dial tcp: no route to host", + plan.Text().Skill) + assert.Contains(t, plan.Text().Agent, "Skip custom agent experiment-orchestrator: unable to download SKILL.md") + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Contains(t, agent.calls, "RegisterService") + assert.Empty(t, agent.createAssets) + assert.Empty(t, agent.createAgents) + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) + assert.Contains(t, result.SkillError, "no route to host") + assert.Equal(t, awsdevops.AgentSkipped, result.AgentAction) + assert.Contains(t, result.AgentError, "no route to host") +} + +// A new space has nothing to compare, but the steps still need the files. +func TestSetupSkipsSkillAndAgentInNewSpaceWhenDownloadFails(t *testing.T) { + agent := &fakeAgent{} + clients := newTestClients(agent, newFakeIAM()) + sourcesOf(clients).fileErr = errors.New("GitHub sent 503 Service Unavailable") + + result, err := awsdevops.Setup(context.Background(), clients, awsdevops.SetupOptions{ + AgentSpaceName: "launchdarkly", + AuthFlow: "iam", + }) + require.NoError(t, err) + + assert.Equal(t, "space-1", result.AgentSpaceID) + assert.Empty(t, agent.createAssets) + assert.Empty(t, agent.createAgents) + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) + assert.Equal(t, awsdevops.AgentSkipped, result.AgentAction) + assert.Contains(t, result.SkillError, "503") +} + +// A skill with an earlier version from ai-tooling main has no edits, so setup +// updates it without --replace-skill. +func TestSetupUpdatesSkillWithPastMainVersion(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": "# Older instructions"}, + } + clients := newTestClients(agent, newFakeIAM()) + sources := sourcesOf(clients) + sources.addPast(awsdevops.SkillSourcePath, "sha-new", testSkill) + sources.addPast(awsdevops.SkillSourcePath, "sha-deleted", "") + delete(sources.files, "sha-deleted "+awsdevops.SkillSourcePath) + sources.addPast(awsdevops.SkillSourcePath, "sha-old", "# Older instructions") + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + require.Len(t, agent.updateAssets, 1) + assert.Equal(t, awsdevops.SkillUpdated, result.SkillAction) +} + +func TestSetupUpdatesAgentWithPastMainVersion(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, + agentFiles: map[string]string{"ki-agent": "# Older prompt"}, + } + clients := newTestClients(agent, newFakeIAM()) + sourcesOf(clients).addPast(awsdevops.AgentSourcePath, "sha-old", "# Older prompt") + + result, err := awsdevops.Setup(context.Background(), clients, reuseOptions()) + require.NoError(t, err) + + require.Len(t, agent.updateAgents, 1, result.AgentError) + assert.Equal(t, awsdevops.AgentUpdated, result.AgentAction) +} + +// The history is read only for a text that matches nothing else. +func TestPlanSetupReadsNoHistoryForTheCurrentSkill(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, + skillFiles: map[string]string{"ki-ours": testSkill}, + } + clients := newTestClients(agent, newFakeIAM()) + sourcesOf(clients).commitsErr = errors.New("history must not be read") + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + + assert.Equal(t, awsdevops.SkillKeep, plan.Skill.Action) + assert.NotContains(t, sourcesOf(clients).calls, "Commits "+awsdevops.SkillSourcePath) +} + +// Without the history, setup cannot tell an old version from an edit, so it +// skips the step instead of calling it an edit. +func TestSetupSkipsSkillAndAgentWhenHistoryCannotBeRead(t *testing.T) { + agent := &fakeAgent{ + assets: []agenttypes.Asset{ + skillAsset("ki-ours", "experiment-orchestration"), + agentAsset("ki-agent", "experiment-orchestrator", nil, []string{"ki-ours"}), + }, + skillFiles: map[string]string{"ki-ours": "# Edited or older"}, + agentFiles: map[string]string{"ki-agent": "# Edited or older"}, + } + clients := newTestClients(agent, newFakeIAM()) + sourcesOf(clients).commitsErr = errors.New("unable to list the commits: GitHub sent 403 rate limit exceeded") + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillSkip, plan.Skill.Action) + assert.Contains(t, plan.Text().Skill, "Skip skill experiment-orchestration: ") + assert.Contains(t, plan.Text().Skill, "rate limit exceeded") + assert.Equal(t, awsdevops.AgentSkip, plan.Agent.Action) + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Empty(t, agent.updateAssets) + assert.Empty(t, agent.updateAgents) + assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) + assert.Contains(t, result.SkillError, "rate limit exceeded") + assert.Equal(t, awsdevops.AgentSkipped, result.AgentAction) +} + +// A skill that names a tool outside the allowlist still goes in, with a +// warning in the plan and the result. +func TestSetupWarnsWhenSkillNamesUnknownTools(t *testing.T) { + agent := &fakeAgent{} + clients := newTestClients(agent, newFakeIAM()) + skill := testSkill + "Then call `launch-rocket` and `delete-everything`.\n" + sourcesOf(clients).files["main "+awsdevops.SkillSourcePath] = skill + + plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillCreate, plan.Skill.Action) + assert.Equal(t, "Create skill experiment-orchestration for every agent type in the space. "+ + "The skill names MCP tools that this ldcli version does not allow: delete-everything", plan.Text().Skill) + + result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) + require.NoError(t, err) + assert.Equal(t, awsdevops.SkillCreated, result.SkillAction) + assert.Equal(t, []string{"delete-everything"}, result.SkillUnknownTools) + assert.Empty(t, result.SkillError) +} + +func TestGitHubSourcesReadFilesAndCommits(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/launchdarkly/ai-tooling/main/"+awsdevops.SkillSourcePath: + _, _ = w.Write([]byte(testSkill)) + case r.URL.Path == "/repos/launchdarkly/ai-tooling/commits" && + r.URL.Query().Get("sha") == "main" && r.URL.Query().Get("path") == awsdevops.SkillSourcePath: + _, _ = w.Write([]byte(`[{"sha":"abc"},{"sha":"def"}]`)) + case r.URL.Path == "/launchdarkly/ai-tooling/main/"+awsdevops.AgentSourcePath: + w.WriteHeader(http.StatusTooManyRequests) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + sources := awsdevops.GitHubSources{Client: server.Client(), RawURL: server.URL, APIURL: server.URL} + + text, err := sources.File(context.Background(), "main", awsdevops.SkillSourcePath) + require.NoError(t, err) + assert.Equal(t, testSkill, text) + + shas, err := sources.Commits(context.Background(), awsdevops.SkillSourcePath) + require.NoError(t, err) + assert.Equal(t, []string{"abc", "def"}, shas) + + _, err = sources.File(context.Background(), "main", awsdevops.AgentSourcePath) + assert.ErrorContains(t, err, "GitHub sent 429 Too Many Requests") + + _, err = sources.File(context.Background(), "abc", awsdevops.AgentSourcePath) + assert.ErrorIs(t, err, awsdevops.ErrSourceNotFound) +} diff --git a/scripts/check-aws-devops-agent-changed.sh b/scripts/check-aws-devops-agent-changed.sh deleted file mode 100755 index 49c8dcc98..000000000 --- a/scripts/check-aws-devops-agent-changed.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -DIFF=$(git diff internal/awsdevops/skills internal/awsdevops/agents) -if [ "$DIFF" ]; then - echo "The AWS DevOps Agent files differ from launchdarkly/ai-tooling. Run 'make aws-devops-agent-sync' and commit the files." - exit 1 -else - echo "The AWS DevOps Agent files match launchdarkly/ai-tooling." -fi diff --git a/scripts/sync-aws-devops-agent.sh b/scripts/sync-aws-devops-agent.sh deleted file mode 100755 index 39ac28669..000000000 --- a/scripts/sync-aws-devops-agent.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# Downloads the AWS DevOps Agent skill and custom agent prompt from -# launchdarkly/ai-tooling at a fixed commit. ldcli embeds the files, so they -# are committed here. To update them, change AI_TOOLING_COMMIT, run this -# script, and add the new hashes in internal/awsdevops. -set -euo pipefail - -AI_TOOLING_COMMIT=887d4cb5022ccda7e57dcef9b23aa9bd9d8cf75d -BASE_URL="https://raw.githubusercontent.com/launchdarkly/ai-tooling/${AI_TOOLING_COMMIT}/skills/aws-devops-agent" -ROOT=$(git rev-parse --show-toplevel) -DEST="${ROOT}/internal/awsdevops" - -curl -fsSL -o "${DEST}/skills/experiment-orchestration/SKILL.md" "${BASE_URL}/experiment-orchestration/SKILL.md" -curl -fsSL -o "${DEST}/agents/experiment-orchestrator.md" "${BASE_URL}/agents/experiment-orchestrator.md" From c6aaed28deaf8c2c6a053870ca3aa243fcddea23 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 11:23:07 -0700 Subject: [PATCH 30/34] chore: drop the unknown-tool warning and the ai-tooling history check A future GitHub Action will compare the skill on ai-tooling main with the allowlist and open a pull request when they differ, so setup no longer warns about tools that the skill names. Setup also no longer reads the history of the files on ai-tooling main. A space whose text is the current main text or a text that a released ldcli embedded gets the new text. Other text counts as an edit. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/confirm_test.go | 2 - cmd/aws_devops_agent/plan_print_test.go | 15 --- cmd/aws_devops_agent/setup.go | 13 -- internal/awsdevops/agent.go | 19 +-- internal/awsdevops/export_test.go | 4 - internal/awsdevops/setup.go | 7 +- internal/awsdevops/skill.go | 71 ++--------- internal/awsdevops/source.go | 57 +-------- internal/awsdevops/source_test.go | 162 +----------------------- 9 files changed, 21 insertions(+), 329 deletions(-) diff --git a/cmd/aws_devops_agent/confirm_test.go b/cmd/aws_devops_agent/confirm_test.go index 8443fd989..f63c9573c 100644 --- a/cmd/aws_devops_agent/confirm_test.go +++ b/cmd/aws_devops_agent/confirm_test.go @@ -148,8 +148,6 @@ func (s stubSources) File(_ context.Context, _, path string) (string, error) { return "# Experiment orchestrator\n", nil } -func (stubSources) Commits(context.Context, string) ([]string, error) { return nil, nil } - // noReadStdin fails the test on any read. type noReadStdin struct{ t *testing.T } diff --git a/cmd/aws_devops_agent/plan_print_test.go b/cmd/aws_devops_agent/plan_print_test.go index cb165e481..d723c7af0 100644 --- a/cmd/aws_devops_agent/plan_print_test.go +++ b/cmd/aws_devops_agent/plan_print_test.go @@ -406,18 +406,3 @@ func TestSetupJSONWarnsOnStderrWhenSkillFails(t *testing.T) { assert.Contains(t, stdoutOf(cmd), `"skillAction": "failed"`) assert.Contains(t, errOut.String(), "Warning: unable to create skill "+awsdevops.ExperimentSkillName) } - -// A skill that names tools outside the allowlist goes in, and setup warns. -func TestSetupWarnsWhenSkillNamesUnknownTools(t *testing.T) { - cmd, errOut := setupTestCmd(t, "") - cmd.SetIn(noReadStdin{t}) - clients, _ := countingClients() - clients.Sources = stubSources{skill: "---\nname: experiment-orchestration\n---\nCall `delete-everything`.\n"} - - require.NoError(t, setupWith(cmd, clients, newSpaceOptions(), false, false)) - - assert.Contains(t, stdoutOf(cmd), `"skillUnknownTools": [`) - assert.Contains(t, errOut.String(), "The skill names MCP tools that this ldcli version does not allow: delete-everything") - assert.Contains(t, errOut.String(), "Warning: skill "+awsdevops.ExperimentSkillName+ - " names MCP tools that this ldcli version does not allow, so the agent cannot call them: delete-everything") -} diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 024b997f9..a6b25c58b 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -31,13 +31,6 @@ const ( yesFlag = "yes" ) -// unknownToolsWarning says that the skill names tools that the agent cannot -// call. -func unknownToolsWarning(tools []string) string { - return fmt.Sprintf("skill %s names MCP tools that this ldcli version does not allow, so the agent cannot call them: %s", - awsdevops.ExperimentSkillName, strings.Join(tools, ", ")) -} - func NewSetupCmd(analyticsTrackerFn analytics.TrackerFn) *cobra.Command { cmd := &cobra.Command{ Use: "setup", @@ -196,9 +189,6 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.SkillError) } - if len(result.SkillUnknownTools) > 0 { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", unknownToolsWarning(result.SkillUnknownTools)) - } if result.AgentError != "" { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", result.AgentError) } @@ -212,9 +202,6 @@ func setupWith( if result.SkillError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.SkillError) } - if len(result.SkillUnknownTools) > 0 { - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", unknownToolsWarning(result.SkillUnknownTools)) - } if result.AgentError != "" { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nWarning: %s\n", result.AgentError) } diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 6f10f9943..d34ed5621 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -16,8 +16,8 @@ import ( const ExperimentAgentName = "experiment-orchestrator" // shippedAgentHashes are the prompts that released ldcli versions embedded. -// The list is frozen: setup now downloads the prompt from ai-tooling and -// compares its history there, so nobody adds to it. Never remove a hash. +// The list is frozen, because setup now downloads the prompt from ai-tooling. +// Never remove a hash. var shippedAgentHashes = []string{ "d3d2124c29e290df9cadd38e9bae2619ac8622cabb180fa0ad4fc59d5358dee5", } @@ -169,23 +169,10 @@ func planAgent( switch { case err != nil: step.Action, step.Err = AgentSkip, err - - return step case current && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && !slices.ContainsFunc(want, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): step.Action = AgentKeep - - return step - case current: - step.Action = AgentUpdate - - return step - } - ours, err := isOurs(ctx, clients, shippedAgentHashes, agentSourcePath, text, found) - switch { - case err != nil: - step.Action, step.Err = AgentSkip, err - case ours: + case current || isOurs(shippedAgentHashes, text, found): step.Action = AgentUpdate default: step.Action = AgentLeave diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index d8b648ad9..796cfdd69 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -33,10 +33,6 @@ func ShipSkill(text string) func() { return func() { shippedSkillHashes = saved } } -// UnknownSkillTools returns the tools that a SKILL.md names and that the -// allowlist misses, for tests. -func UnknownSkillTools(text string) []string { return unknownSkillTools(text) } - // SkillSourcePath and AgentSourcePath are the paths of the files in ai-tooling. const ( SkillSourcePath = skillSourcePath diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 00a2fffa3..37189e490 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -184,11 +184,8 @@ type SetupResult struct { SkillAction string `json:"skillAction,omitempty"` // SkillError is set if the skill step failed or was skipped. The skill is // optional, so the rest of the setup ran. - SkillError string `json:"skillError,omitempty"` - // SkillUnknownTools lists the MCP tools that the skill names and that this - // ldcli version does not allow, so the agent cannot call them. - SkillUnknownTools []string `json:"skillUnknownTools,omitempty"` - AgentAssetID string `json:"agentAssetId,omitempty"` + SkillError string `json:"skillError,omitempty"` + AgentAssetID string `json:"agentAssetId,omitempty"` // AgentAction is what the custom agent step did, for example "created". AgentAction string `json:"agentAction,omitempty"` // AgentError is set if the custom agent step failed or was skipped. diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 7afdac714..7b5976432 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "regexp" "slices" "strings" @@ -23,9 +22,8 @@ const ExperimentSkillName = "experiment-orchestration" // shippedSkillHashes holds the SHA-256 of each SKILL.md that a released ldcli // embedded. A skill with one of these texts has no customer edits, so setup -// updates it without --replace-skill. The list is frozen: setup now downloads -// SKILL.md from ai-tooling and compares its history there, so nobody adds to -// it. Never remove a hash. +// updates it without --replace-skill. The list is frozen, because setup now +// downloads SKILL.md from ai-tooling. Never remove a hash. var shippedSkillHashes = []string{ "57844bc3c5aca292ef88b5f949cd791206074b5cfc850ea52ebdd40020ead695", "c4c4e12aeff900402dc17ff370bc1e78eadcb34abac5b256319c7cd793bd73f3", @@ -55,24 +53,6 @@ func frontMatterField(text, key string) string { return "" } -// skillToolPattern matches an MCP tool name in backticks, such as `get-flag`. -var skillToolPattern = regexp.MustCompile("`((?:list|get|create|update|start|stop|toggle|archive|apply|check|query|match|delete|copy|save|search|find|manage|run|clone|setup|preview|link|invite|restrict|unrestrict)-[a-z-]+)`") - -// unknownSkillTools returns the MCP tools that a SKILL.md names and that this -// ldcli version does not allow. The agent cannot call them. -func unknownSkillTools(text string) []string { - var unknown []string - for _, match := range skillToolPattern.FindAllStringSubmatch(text, -1) { - tool := match[1] - if slices.Contains(mcpReadOnlyTools, tool) || slices.Contains(mcpMutativeTools, tool) || slices.Contains(unknown, tool) { - continue - } - unknown = append(unknown, tool) - } - - return unknown -} - func skillHash(text string) string { sum := sha256.Sum256([]byte(text)) @@ -109,9 +89,6 @@ type SkillStep struct { // Text is the SKILL.md on ai-tooling main at plan time, which the step // writes. Text string - // UnknownTools are the MCP tools that Text names and that this ldcli - // version does not allow. - UnknownTools []string // Err says why the step is skipped. Err error } @@ -137,17 +114,10 @@ func (s SkillStep) describe() string { default: line = fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) } - if len(s.UnknownTools) > 0 { - line += ". " + unknownToolsText(s.UnknownTools) - } return line } -func unknownToolsText(tools []string) string { - return "The skill names MCP tools that this ldcli version does not allow: " + strings.Join(tools, ", ") -} - // skillFailure is the text of a failed skill step. A missing permission is // the most likely cause, so it names the permissions that the step needs. func skillFailure(err error) string { @@ -167,7 +137,7 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp } step := planSkillAction(ctx, clients, opts, space, want) if step.Action != SkillSkip && step.Action != SkillLeave { - step.Text, step.UnknownTools = want, unknownSkillTools(want) + step.Text = want } return step @@ -205,14 +175,7 @@ func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, sp return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} case found && text == want: step.Action = SkillKeep - - return step - } - ours, err := isOurs(ctx, clients, shippedSkillHashes, skillSourcePath, text, found) - switch { - case err != nil: - return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} - case ours: + case isOurs(shippedSkillHashes, text, found): step.Action = SkillUpdate case opts.ReplaceSkill: step.Action = SkillReplace @@ -223,22 +186,12 @@ func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, sp return step } -// isOurs reports whether text, the file of an asset, came from ldcli or -// ai-tooling, so setup can replace it without losing an edit. shipped holds -// the hashes of the texts that released ldcli versions embedded. The history -// of path on ai-tooling main is read only when nothing else matches. -func isOurs(ctx context.Context, clients Clients, shipped []string, path, text string, found bool) (bool, error) { - switch { - case !found: - return false, nil - case text == "": - // An empty file has nothing to lose. - return true, nil - case slices.Contains(shipped, skillHash(text)): - return true, nil - } - - return pastVersion(ctx, clients.Sources, path, text) +// isOurs reports whether text, the file of an asset, came from a released +// ldcli, so setup can replace it without losing an edit. shipped holds the +// hashes of the texts that those versions embedded. An empty file has nothing +// to lose. +func isOurs(shipped []string, text string, found bool) bool { + return found && (text == "" || slices.Contains(shipped, skillHash(text))) } // findSkill returns the skill with this name in the space, or nil if there is @@ -358,10 +311,6 @@ func installSkill( result.SkillAssetID = step.AssetID action, err := doSkillStep(ctx, clients, step, result, logf) result.SkillAction = action - if err == nil { - // The skill in the space now names these tools. - result.SkillUnknownTools = step.UnknownTools - } if err != nil { result.SkillAction = SkillFailed // A plan change stops the step before the write, so setup tried diff --git a/internal/awsdevops/source.go b/internal/awsdevops/source.go index 84a4422ef..28b64db7e 100644 --- a/internal/awsdevops/source.go +++ b/internal/awsdevops/source.go @@ -2,8 +2,6 @@ package awsdevops import ( "context" - "encoding/json" - "errors" "fmt" "io" "net/http" @@ -16,8 +14,6 @@ import ( type Sources interface { // File returns a file of ai-tooling at ref, a branch or a commit SHA. File(ctx context.Context, ref, path string) (string, error) - // Commits returns the SHAs of the commits on main that changed path. - Commits(ctx context.Context, path string) ([]string, error) } const ( @@ -27,10 +23,6 @@ const ( agentSourcePath = "skills/aws-devops-agent/agents/experiment-orchestrator.md" ) -// ErrSourceNotFound means that ai-tooling has no such file at the ref, for -// example at a commit that deleted it. -var ErrSourceNotFound = errors.New("file not found") - const ( sourceTimeout = 30 * time.Second // maxSourceBytes caps a download. The files are a few KB. @@ -41,7 +33,6 @@ const ( type GitHubSources struct { Client *http.Client RawURL string - APIURL string } // NewGitHubSources reads from github.com. @@ -49,7 +40,6 @@ func NewGitHubSources() GitHubSources { return GitHubSources{ Client: &http.Client{Timeout: sourceTimeout}, RawURL: "https://raw.githubusercontent.com", - APIURL: "https://api.github.com", } } @@ -62,26 +52,6 @@ func (s GitHubSources) File(ctx context.Context, ref, path string) (string, erro return string(body), nil } -func (s GitHubSources) Commits(ctx context.Context, path string) ([]string, error) { - query := url.Values{"sha": {aiToolingBranch}, "path": {path}, "per_page": {"100"}} - body, err := s.get(ctx, fmt.Sprintf("%s/repos/%s/commits?%s", s.APIURL, aiToolingRepo, query.Encode())) - if err != nil { - return nil, fmt.Errorf("unable to list the commits of %s in %s: %w", path, aiToolingRepo, err) - } - var commits []struct { - SHA string `json:"sha"` - } - if err := json.Unmarshal(body, &commits); err != nil { - return nil, fmt.Errorf("unable to read the commits of %s in %s: %w", path, aiToolingRepo, err) - } - shas := make([]string, 0, len(commits)) - for _, commit := range commits { - shas = append(shas, commit.SHA) - } - - return shas, nil -} - func (s GitHubSources) get(ctx context.Context, address string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) if err != nil { @@ -96,10 +66,7 @@ func (s GitHubSources) get(ctx context.Context, address string) ([]byte, error) return nil, err } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusNotFound: - return nil, ErrSourceNotFound - case resp.StatusCode != http.StatusOK: + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("GitHub sent %s", resp.Status) } @@ -120,25 +87,3 @@ func fetchSources(ctx context.Context, sources Sources) (skill, prompt string, e return skill, prompt, nil } - -// pastVersion reports whether text is an earlier version of path on main. -// A commit that deleted the file has no version, so it is passed over. -func pastVersion(ctx context.Context, sources Sources, path, text string) (bool, error) { - shas, err := sources.Commits(ctx, path) - if err != nil { - return false, err - } - for _, sha := range shas { - old, err := sources.File(ctx, sha, path) - switch { - case errors.Is(err, ErrSourceNotFound): - continue - case err != nil: - return false, err - case old == text: - return true, nil - } - } - - return false, nil -} diff --git a/internal/awsdevops/source_test.go b/internal/awsdevops/source_test.go index 6a2560acd..e30923a3a 100644 --- a/internal/awsdevops/source_test.go +++ b/internal/awsdevops/source_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "testing" - agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -28,11 +27,8 @@ const ( // fakeSources serves ai-tooling from memory. files maps "ref path" to text. type fakeSources struct { - files map[string]string - commits map[string][]string - fileErr error - commitsErr error - calls []string + files map[string]string + fileErr error } func newFakeSources() *fakeSources { @@ -43,53 +39,21 @@ func newFakeSources() *fakeSources { } func (f *fakeSources) File(_ context.Context, ref, path string) (string, error) { - f.calls = append(f.calls, "File "+ref+" "+path) if f.fileErr != nil { return "", f.fileErr } text, ok := f.files[ref+" "+path] if !ok { - return "", awsdevops.ErrSourceNotFound + return "", errors.New("GitHub sent 404 Not Found") } return text, nil } -func (f *fakeSources) Commits(_ context.Context, path string) ([]string, error) { - f.calls = append(f.calls, "Commits "+path) - if f.commitsErr != nil { - return nil, f.commitsErr - } - - return f.commits[path], nil -} - -// addPast puts text in the history of path at sha. -func (f *fakeSources) addPast(path, sha, text string) { - if f.commits == nil { - f.commits = map[string][]string{} - } - f.commits[path] = append(f.commits[path], sha) - f.files[sha+" "+path] = text -} - func sourcesOf(clients awsdevops.Clients) *fakeSources { return clients.Sources.(*fakeSources) } -func TestUnknownSkillToolsListsToolsThatTheAllowlistMisses(t *testing.T) { - text := "Call `get-flag`, then `delete-everything`, `get-flag`, and `run-magic`. " + - "Plain words like get-flag-x or `not a tool` do not count." - - assert.Equal(t, []string{"delete-everything", "run-magic"}, awsdevops.UnknownSkillTools(text)) -} - -func TestUnknownSkillToolsIsEmptyWhenTheAllowlistHoldsEveryTool(t *testing.T) { - assert.Empty(t, awsdevops.UnknownSkillTools(testSkill)) -} - -// Without GitHub, the plan skips the skill and the custom agent, and setup -// does the other steps. func TestSetupSkipsSkillAndAgentWhenDownloadFails(t *testing.T) { agent := &fakeAgent{} clients := newTestClients(agent, newFakeIAM()) @@ -134,120 +98,11 @@ func TestSetupSkipsSkillAndAgentInNewSpaceWhenDownloadFails(t *testing.T) { assert.Contains(t, result.SkillError, "503") } -// A skill with an earlier version from ai-tooling main has no edits, so setup -// updates it without --replace-skill. -func TestSetupUpdatesSkillWithPastMainVersion(t *testing.T) { - agent := &fakeAgent{ - assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": "# Older instructions"}, - } - clients := newTestClients(agent, newFakeIAM()) - sources := sourcesOf(clients) - sources.addPast(awsdevops.SkillSourcePath, "sha-new", testSkill) - sources.addPast(awsdevops.SkillSourcePath, "sha-deleted", "") - delete(sources.files, "sha-deleted "+awsdevops.SkillSourcePath) - sources.addPast(awsdevops.SkillSourcePath, "sha-old", "# Older instructions") - - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) - require.NoError(t, err) - assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) - - result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) - require.NoError(t, err) - require.Len(t, agent.updateAssets, 1) - assert.Equal(t, awsdevops.SkillUpdated, result.SkillAction) -} - -func TestSetupUpdatesAgentWithPastMainVersion(t *testing.T) { - agent := &fakeAgent{ - assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, - agentFiles: map[string]string{"ki-agent": "# Older prompt"}, - } - clients := newTestClients(agent, newFakeIAM()) - sourcesOf(clients).addPast(awsdevops.AgentSourcePath, "sha-old", "# Older prompt") - - result, err := awsdevops.Setup(context.Background(), clients, reuseOptions()) - require.NoError(t, err) - - require.Len(t, agent.updateAgents, 1, result.AgentError) - assert.Equal(t, awsdevops.AgentUpdated, result.AgentAction) -} - -// The history is read only for a text that matches nothing else. -func TestPlanSetupReadsNoHistoryForTheCurrentSkill(t *testing.T) { - agent := &fakeAgent{ - assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": testSkill}, - } - clients := newTestClients(agent, newFakeIAM()) - sourcesOf(clients).commitsErr = errors.New("history must not be read") - - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) - require.NoError(t, err) - - assert.Equal(t, awsdevops.SkillKeep, plan.Skill.Action) - assert.NotContains(t, sourcesOf(clients).calls, "Commits "+awsdevops.SkillSourcePath) -} - -// Without the history, setup cannot tell an old version from an edit, so it -// skips the step instead of calling it an edit. -func TestSetupSkipsSkillAndAgentWhenHistoryCannotBeRead(t *testing.T) { - agent := &fakeAgent{ - assets: []agenttypes.Asset{ - skillAsset("ki-ours", "experiment-orchestration"), - agentAsset("ki-agent", "experiment-orchestrator", nil, []string{"ki-ours"}), - }, - skillFiles: map[string]string{"ki-ours": "# Edited or older"}, - agentFiles: map[string]string{"ki-agent": "# Edited or older"}, - } - clients := newTestClients(agent, newFakeIAM()) - sourcesOf(clients).commitsErr = errors.New("unable to list the commits: GitHub sent 403 rate limit exceeded") - - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) - require.NoError(t, err) - assert.Equal(t, awsdevops.SkillSkip, plan.Skill.Action) - assert.Contains(t, plan.Text().Skill, "Skip skill experiment-orchestration: ") - assert.Contains(t, plan.Text().Skill, "rate limit exceeded") - assert.Equal(t, awsdevops.AgentSkip, plan.Agent.Action) - - result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) - require.NoError(t, err) - assert.Empty(t, agent.updateAssets) - assert.Empty(t, agent.updateAgents) - assert.Equal(t, awsdevops.SkillSkipped, result.SkillAction) - assert.Contains(t, result.SkillError, "rate limit exceeded") - assert.Equal(t, awsdevops.AgentSkipped, result.AgentAction) -} - -// A skill that names a tool outside the allowlist still goes in, with a -// warning in the plan and the result. -func TestSetupWarnsWhenSkillNamesUnknownTools(t *testing.T) { - agent := &fakeAgent{} - clients := newTestClients(agent, newFakeIAM()) - skill := testSkill + "Then call `launch-rocket` and `delete-everything`.\n" - sourcesOf(clients).files["main "+awsdevops.SkillSourcePath] = skill - - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) - require.NoError(t, err) - assert.Equal(t, awsdevops.SkillCreate, plan.Skill.Action) - assert.Equal(t, "Create skill experiment-orchestration for every agent type in the space. "+ - "The skill names MCP tools that this ldcli version does not allow: delete-everything", plan.Text().Skill) - - result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) - require.NoError(t, err) - assert.Equal(t, awsdevops.SkillCreated, result.SkillAction) - assert.Equal(t, []string{"delete-everything"}, result.SkillUnknownTools) - assert.Empty(t, result.SkillError) -} - -func TestGitHubSourcesReadFilesAndCommits(t *testing.T) { +func TestGitHubSourcesReadFiles(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/launchdarkly/ai-tooling/main/"+awsdevops.SkillSourcePath: _, _ = w.Write([]byte(testSkill)) - case r.URL.Path == "/repos/launchdarkly/ai-tooling/commits" && - r.URL.Query().Get("sha") == "main" && r.URL.Query().Get("path") == awsdevops.SkillSourcePath: - _, _ = w.Write([]byte(`[{"sha":"abc"},{"sha":"def"}]`)) case r.URL.Path == "/launchdarkly/ai-tooling/main/"+awsdevops.AgentSourcePath: w.WriteHeader(http.StatusTooManyRequests) default: @@ -255,19 +110,12 @@ func TestGitHubSourcesReadFilesAndCommits(t *testing.T) { } })) defer server.Close() - sources := awsdevops.GitHubSources{Client: server.Client(), RawURL: server.URL, APIURL: server.URL} + sources := awsdevops.GitHubSources{Client: server.Client(), RawURL: server.URL} text, err := sources.File(context.Background(), "main", awsdevops.SkillSourcePath) require.NoError(t, err) assert.Equal(t, testSkill, text) - shas, err := sources.Commits(context.Background(), awsdevops.SkillSourcePath) - require.NoError(t, err) - assert.Equal(t, []string{"abc", "def"}, shas) - _, err = sources.File(context.Background(), "main", awsdevops.AgentSourcePath) assert.ErrorContains(t, err, "GitHub sent 429 Too Many Requests") - - _, err = sources.File(context.Background(), "abc", awsdevops.AgentSourcePath) - assert.ErrorIs(t, err, awsdevops.ErrSourceNotFound) } From a4489f34ee83fdbab0fd2b5bc61f3542d4214b78 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 11:28:08 -0700 Subject: [PATCH 31/34] fix: skip an existing skill or agent that differs from ai-tooling main Setup no longer keeps the hashes of the texts that older ldcli versions embedded. If the skill in the space is not the latest SKILL.md on ai-tooling main, the plan says so and setup skips it. --replace-skill updates it to the latest version. A custom agent whose prompt differs is skipped the same way, with the hint to delete it and run setup again. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/setup.go | 12 +++--- internal/awsdevops/agent.go | 18 +++------ internal/awsdevops/export_test.go | 8 ---- internal/awsdevops/setup.go | 5 +-- internal/awsdevops/skill.go | 61 ++++++------------------------- internal/awsdevops/skill_test.go | 29 ++------------- 6 files changed, 27 insertions(+), 106 deletions(-) diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index a6b25c58b..63921693d 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -67,12 +67,10 @@ the download fails, setup skips both steps and warns. The experiment-orchestration skill tells the agent how to run an experiment from a goal to a report. Every agent type in the space can load it. If the -skill has the text of an older ldcli version or of an earlier version on -ai-tooling, setup updates it. If someone edited it, setup does not change it. -To overwrite the edits, pass --replace-skill. Setting up the skill is -optional: if its step fails, setup warns and does the other steps. If the -skill names MCP tools that this ldcli version does not allow, setup warns, -because the agent cannot call them. +space already has the skill and it differs from the latest version, setup +skips it. To update it to the latest version, pass --replace-skill. Setting +up the skill is optional: if its step fails, setup warns and does the other +steps. The experiment-orchestrator custom agent runs the skill with the LaunchDarkly MCP tools that the space allows. It needs the MCP server and the skill. @@ -97,7 +95,7 @@ does the other steps.`, cmd.Flags().String(idpClientSecretFlag, "", "OIDC client secret, required when --auth-flow=idp") cmd.Flags().String(mcpServiceIDFlag, "", "Registered MCP server to connect, instead of the server of the AWS account") cmd.Flags().String(mcpEndpointFlag, "", "MCP endpoint to register. Setup reuses only a server at this endpoint. Defaults to "+awsdevops.MCPServerEndpoint) - cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if someone edited it") + cmd.Flags().Bool(replaceSkillFlag, false, "Update the "+awsdevops.ExperimentSkillName+" skill to the latest version if it differs") cmd.Flags().Bool(addMCPToolsFlag, false, "Has no effect. Setup adds the missing tools without it") _ = cmd.Flags().MarkDeprecated(addMCPToolsFlag, "setup now adds the missing MCP tools without it") diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index d34ed5621..98034f079 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -15,14 +15,7 @@ import ( const ExperimentAgentName = "experiment-orchestrator" -// shippedAgentHashes are the prompts that released ldcli versions embedded. -// The list is frozen, because setup now downloads the prompt from ai-tooling. -// Never remove a hash. -var shippedAgentHashes = []string{ - "d3d2124c29e290df9cadd38e9bae2619ac8622cabb180fa0ad4fc59d5358dee5", -} - -const agentResetHint = "To use the ldcli version, delete the agent in the AWS console and run setup again" +const agentResetHint = "To use the latest version, delete the agent in the AWS console and run setup again" const ( agentAssetType = "custom_agent" @@ -96,7 +89,7 @@ type AgentStep struct { func (s AgentStep) describe() string { label := fmt.Sprintf("%s (%s)", ExperimentAgentName, s.AssetID) - why := "someone edited its prompt" + why := "its prompt differs from the latest version on the main branch of " + aiToolingRepo if s.NoFile { why = "it has no " + agentFilePath } @@ -104,10 +97,9 @@ func (s AgentStep) describe() string { case AgentKeep: return "Keep custom agent " + label case AgentUpdate: - return fmt.Sprintf("Update custom agent %s to the prompt on the main branch of %s and the tools of this ldcli version", - label, aiToolingRepo) + return fmt.Sprintf("Add the tools of this ldcli version and the skill to custom agent %s", label) case AgentLeave: - return fmt.Sprintf("Leave custom agent %s unchanged because %s. %s", label, why, agentResetHint) + return fmt.Sprintf("Skip custom agent %s because %s. %s", label, why, agentResetHint) case AgentSkip: return agentFailure(fmt.Errorf("Skip custom agent %s: %w", ExperimentAgentName, s.Err)) default: @@ -172,7 +164,7 @@ func planAgent( case current && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && !slices.ContainsFunc(want, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): step.Action = AgentKeep - case current || isOurs(shippedAgentHashes, text, found): + case current: step.Action = AgentUpdate default: step.Action = AgentLeave diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 796cfdd69..730ea3547 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -25,14 +25,6 @@ func ConnectExisting(ctx context.Context, clients Clients, opts SetupOptions) (S return ConnectPlan(ctx, clients, opts, plan) } -// ShipSkill adds text to the shipped skill versions until the test ends. -func ShipSkill(text string) func() { - saved := shippedSkillHashes - shippedSkillHashes = append(append([]string(nil), saved...), skillHash(text)) - - return func() { shippedSkillHashes = saved } -} - // SkillSourcePath and AgentSourcePath are the paths of the files in ai-tooling. const ( SkillSourcePath = skillSourcePath diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 37189e490..380a82db9 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -151,9 +151,8 @@ type SetupOptions struct { // OpenURL, when set, opens the OAuth approval page in a browser. OpenURL func(url string) error - // ReplaceSkill overwrites the SKILL.md of an existing - // experiment-orchestration skill that someone edited. Setup updates a - // SKILL.md from an older ldcli version without this option. + // ReplaceSkill writes the latest SKILL.md over an existing + // experiment-orchestration skill that differs from it. ReplaceSkill bool // Logf, when set, reports progress as each step completes. diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 7b5976432..16180c661 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -2,12 +2,9 @@ package awsdevops import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" - "slices" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -20,16 +17,6 @@ import ( // the skill by this name, so it does not come from the downloaded SKILL.md. const ExperimentSkillName = "experiment-orchestration" -// shippedSkillHashes holds the SHA-256 of each SKILL.md that a released ldcli -// embedded. A skill with one of these texts has no customer edits, so setup -// updates it without --replace-skill. The list is frozen, because setup now -// downloads SKILL.md from ai-tooling. Never remove a hash. -var shippedSkillHashes = []string{ - "57844bc3c5aca292ef88b5f949cd791206074b5cfc850ea52ebdd40020ead695", - "c4c4e12aeff900402dc17ff370bc1e78eadcb34abac5b256319c7cd793bd73f3", - "cbda81714171e1f725db7a7e10973c38ae0b71550e14636d8d463919349bd9e8", -} - const ( skillAssetType = "skill" skillFilePath = "SKILL.md" @@ -53,24 +40,17 @@ func frontMatterField(text, key string) string { return "" } -func skillHash(text string) string { - sum := sha256.Sum256([]byte(text)) - - return hex.EncodeToString(sum[:]) -} - // SkillAction is the planned skill action. type SkillAction int const ( SkillCreate SkillAction = iota SkillKeep - // SkillUpdate replaces a SKILL.md that an older ldcli sent, or one that - // is empty. - SkillUpdate - // SkillLeave means that someone edited the SKILL.md, or that the skill has - // none, so setup does not change it without --replace-skill. + // SkillLeave means that the SKILL.md of the space is not the latest one on + // ai-tooling main, or that the skill has none, so setup skips it without + // --replace-skill. SkillLeave + // SkillReplace writes the latest SKILL.md over the one in the space. SkillReplace // SkillSkip means that setup could not read the skills of the space. The // skill is optional, so the run continues without it. @@ -95,27 +75,22 @@ type SkillStep struct { func (s SkillStep) describe() string { label := fmt.Sprintf("%s (%s)", ExperimentSkillName, s.AssetID) - why := "someone edited it" + why := "it differs from the latest version on the main branch of " + aiToolingRepo if s.NoFile { why = "it has no " + skillFilePath } - var line string switch s.Action { case SkillKeep: - line = "Keep skill " + label - case SkillUpdate: - line = fmt.Sprintf("Update skill %s to the instructions on the main branch of %s", label, aiToolingRepo) + return "Keep skill " + label case SkillLeave: - return fmt.Sprintf("Leave skill %s unchanged because %s. Pass --replace-skill to overwrite it", label, why) + return fmt.Sprintf("Skip skill %s because %s. Pass --replace-skill to update it to the latest version", label, why) case SkillReplace: - line = fmt.Sprintf("Overwrite skill %s, which is not the ldcli version because %s", label, why) + return fmt.Sprintf("Update skill %s to the latest version on the main branch of %s", label, aiToolingRepo) case SkillSkip: return skillFailure(fmt.Errorf("Skip skill %s: %w", ExperimentSkillName, s.Err)) default: - line = fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) + return fmt.Sprintf("Create skill %s for every agent type in the space", ExperimentSkillName) } - - return line } // skillFailure is the text of a failed skill step. A missing permission is @@ -175,8 +150,6 @@ func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, sp return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} case found && text == want: step.Action = SkillKeep - case isOurs(shippedSkillHashes, text, found): - step.Action = SkillUpdate case opts.ReplaceSkill: step.Action = SkillReplace default: @@ -186,14 +159,6 @@ func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, sp return step } -// isOurs reports whether text, the file of an asset, came from a released -// ldcli, so setup can replace it without losing an edit. shipped holds the -// hashes of the texts that those versions embedded. An empty file has nothing -// to lose. -func isOurs(shipped []string, text string, found bool) bool { - return found && (text == "" || slices.Contains(shipped, skillHash(text))) -} - // findSkill returns the skill with this name in the space, or nil if there is // none. AWS keeps skill names unique in a space. func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { @@ -291,7 +256,6 @@ func checkSkill(ctx context.Context, clients Clients, spaceID string, step Skill const ( SkillCreated = "created" SkillKept = "kept" - SkillUpdated = "updated" SkillLeft = "left" SkillOverwritten = "overwritten" SkillSkipped = "skipped" @@ -335,7 +299,7 @@ func doSkillStep( return SkillKept, nil case SkillLeave: - logf("Left skill %s (%s) unchanged. Pass --replace-skill to overwrite it", ExperimentSkillName, step.AssetID) + logf("Skipped skill %s (%s). Pass --replace-skill to update it to the latest version", ExperimentSkillName, step.AssetID) return SkillLeft, nil case SkillSkip: @@ -383,9 +347,6 @@ func doSkillStep( return "", fmt.Errorf("unable to update skill %s (%s): %w", ExperimentSkillName, step.AssetID, err) } logf("Updated skill %s (%s)", ExperimentSkillName, step.AssetID) - if step.Action == SkillReplace { - return SkillOverwritten, nil - } - return SkillUpdated, nil + return SkillOverwritten, nil } diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index 0bc386ddd..ab91bf9cc 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -108,8 +108,8 @@ func TestPlanSetupKeepsSameSkillSentAsBytes(t *testing.T) { assert.Equal(t, awsdevops.SkillKeep, plan.Skill.Action) } -// An empty SKILL.md has no edits to lose, so setup updates it without the flag. -func TestPlanSetupUpdatesEmptySkill(t *testing.T) { +// An empty SKILL.md is not the latest one, so setup skips it without the flag. +func TestPlanSetupLeavesEmptySkill(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, skillFiles: map[string]string{"ki-ours": ""}, @@ -118,7 +118,7 @@ func TestPlanSetupUpdatesEmptySkill(t *testing.T) { plan, err := awsdevops.PlanSetup(context.Background(), newTestClients(agent, newFakeIAM()), reuseOptions(), true) require.NoError(t, err) - assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) + assert.Equal(t, awsdevops.SkillLeave, plan.Skill.Action) } // A skill without SKILL.md, for example one made from a PDF, is not ours. @@ -130,31 +130,10 @@ func TestPlanSetupLeavesSkillWithoutFile(t *testing.T) { assert.Equal(t, awsdevops.SkillLeave, plan.Skill.Action) assert.Equal(t, - "Leave skill experiment-orchestration (ki-ours) unchanged because it has no SKILL.md. Pass --replace-skill to overwrite it", + "Skip skill experiment-orchestration (ki-ours) because it has no SKILL.md. Pass --replace-skill to update it to the latest version", plan.Text().Skill) } -// A copy from an older ldcli has no edits, so setup updates it without the flag. -func TestSetupUpdatesOlderShippedSkill(t *testing.T) { - t.Cleanup(awsdevops.ShipSkill("# Instructions of an older ldcli")) - agent := &fakeAgent{ - assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, - skillFiles: map[string]string{"ki-ours": "# Instructions of an older ldcli"}, - } - clients := newTestClients(agent, newFakeIAM()) - - plan, err := awsdevops.PlanSetup(context.Background(), clients, reuseOptions(), true) - require.NoError(t, err) - assert.Equal(t, awsdevops.SkillUpdate, plan.Skill.Action) - assert.Contains(t, plan.Text().Skill, "Update skill experiment-orchestration (ki-ours)") - - result, err := awsdevops.RunPlan(context.Background(), clients, reuseOptions(), plan) - require.NoError(t, err) - require.Len(t, agent.updateAssets, 1) - assert.Equal(t, "ki-ours", aws.ToString(agent.updateAssets[0].AssetId)) - assert.Equal(t, awsdevops.SkillUpdated, result.SkillAction) -} - func TestSetupLeavesEditedSkill(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, From d57f70c069b05da1c6a951c7e1c02accfc346a08 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 11:32:32 -0700 Subject: [PATCH 32/34] chore: trim comments and dead code in the skill and custom agent steps Sources.File drops the ref, which is always main, and GitHubSources drops the nil-client fallback. planSkill is one function again, and skillText is inlined. Comments that repeat the code, or that describe the embedded files and the hash lists, are gone. The allowlist comment no longer says that the skill keeps its record in the flag description. Tests named for edits are named for a different text. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/confirm_test.go | 3 +- internal/awsdevops/agent.go | 16 +++--- internal/awsdevops/agent_test.go | 2 +- internal/awsdevops/awsdevops.go | 7 ++- internal/awsdevops/export_test.go | 1 - internal/awsdevops/setup.go | 22 ++------- internal/awsdevops/skill.go | 74 +++++++--------------------- internal/awsdevops/skill_test.go | 4 +- internal/awsdevops/source.go | 49 +++++++----------- internal/awsdevops/source_test.go | 14 +++--- 10 files changed, 60 insertions(+), 132 deletions(-) diff --git a/cmd/aws_devops_agent/confirm_test.go b/cmd/aws_devops_agent/confirm_test.go index f63c9573c..b37f956a9 100644 --- a/cmd/aws_devops_agent/confirm_test.go +++ b/cmd/aws_devops_agent/confirm_test.go @@ -137,10 +137,9 @@ func countingClients(spaces ...agenttypes.AgentSpace) (awsdevops.Clients, *write }, w } -// stubSources serves the files of ai-tooling main from memory. type stubSources struct{ skill string } -func (s stubSources) File(_ context.Context, _, path string) (string, error) { +func (s stubSources) File(_ context.Context, path string) (string, error) { if strings.HasSuffix(path, "SKILL.md") { return s.skill, nil } diff --git a/internal/awsdevops/agent.go b/internal/awsdevops/agent.go index 98034f079..1171fef71 100644 --- a/internal/awsdevops/agent.go +++ b/internal/awsdevops/agent.go @@ -81,8 +81,7 @@ type AgentStep struct { Tools []string Skills []string NoFile bool - // Prompt is the prompt on ai-tooling main at plan time, which the step - // writes. + // Prompt is the prompt downloaded at plan time. Prompt string Err error } @@ -117,8 +116,6 @@ func agentFailure(err error) string { "aidevops:CreateAsset, aidevops:UpdateAsset, and aidevops:ListAssociations", err) } -// planAgent compares the custom agent of the space with prompt, the prompt on -// ai-tooling main. fetchErr is set if the download failed. func planAgent( ctx context.Context, clients Clients, @@ -152,17 +149,17 @@ func planAgent( step.Action, step.Err = AgentSkip, err return step } - want, err := spaceAgentTools(ctx, clients, space.ID, mcp.ServiceID, step.ServerName) + wantTools, err := spaceAgentTools(ctx, clients, space.ID, mcp.ServiceID, step.ServerName) if mcp.CanAddTools { - // The MCP step adds these tools to the allowlist before the agent step. - want = union(want, toolIDs(step.ServerName, mcp.MissingTools)) + // The MCP step adds these before the agent step runs. + wantTools = union(wantTools, toolIDs(step.ServerName, mcp.MissingTools)) } current := found && text == prompt switch { case err != nil: step.Action, step.Err = AgentSkip, err case current && skill.AssetID != "" && slices.Contains(step.Skills, skill.AssetID) && - !slices.ContainsFunc(want, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): + !slices.ContainsFunc(wantTools, func(tool string) bool { return !slices.Contains(step.Tools, tool) }): step.Action = AgentKeep case current: step.Action = AgentUpdate @@ -267,7 +264,7 @@ func doAgentStep( return AgentKept, nil case AgentLeave: - logf("Left custom agent %s (%s) unchanged. %s", ExperimentAgentName, step.AssetID, agentResetHint) + logf("Skipped custom agent %s (%s). %s", ExperimentAgentName, step.AssetID, agentResetHint) return AgentLeft, nil case AgentSkip: @@ -325,5 +322,6 @@ func doAgentStep( return "", fmt.Errorf("unable to update custom agent %s (%s): %w", ExperimentAgentName, step.AssetID, err) } logf("Updated custom agent %s (%s)", ExperimentAgentName, step.AssetID) + return AgentUpdated, nil } diff --git a/internal/awsdevops/agent_test.go b/internal/awsdevops/agent_test.go index 84b82029f..cd5a3757e 100644 --- a/internal/awsdevops/agent_test.go +++ b/internal/awsdevops/agent_test.go @@ -111,7 +111,7 @@ func TestSetupUpdatesAgentWithOldTools(t *testing.T) { assert.Equal(t, awsdevops.AgentUpdated, result.AgentAction) } -func TestSetupLeavesEditedAgent(t *testing.T) { +func TestSetupSkipsDifferentAgent(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{agentAsset("ki-agent", "experiment-orchestrator", nil, nil)}, agentFiles: map[string]string{"ki-agent": "# Edited by the customer"}, diff --git a/internal/awsdevops/awsdevops.go b/internal/awsdevops/awsdevops.go index 632b46742..ac93d91b4 100644 --- a/internal/awsdevops/awsdevops.go +++ b/internal/awsdevops/awsdevops.go @@ -85,10 +85,9 @@ type STSAPI interface { // Clients bundles the AWS clients and the resolved session details the // provisioning steps run against. type Clients struct { - Agent AgentAPI - IAM IAMAPI - STS STSAPI - // Sources reads the skill and the custom agent prompt from ai-tooling. + Agent AgentAPI + IAM IAMAPI + STS STSAPI Sources Sources Region string } diff --git a/internal/awsdevops/export_test.go b/internal/awsdevops/export_test.go index 730ea3547..a21de5b5f 100644 --- a/internal/awsdevops/export_test.go +++ b/internal/awsdevops/export_test.go @@ -25,7 +25,6 @@ func ConnectExisting(ctx context.Context, clients Clients, opts SetupOptions) (S return ConnectPlan(ctx, clients, opts, plan) } -// SkillSourcePath and AgentSourcePath are the paths of the files in ai-tooling. const ( SkillSourcePath = skillSourcePath AgentSourcePath = agentSourcePath diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index 380a82db9..c6e28484c 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -61,21 +61,10 @@ const ( // registers after a person approves access. Tests make it shorter. var oauthPollInterval = 5 * time.Second -// The agent invokes read-only tools without asking for approval, so anything -// that changes LaunchDarkly state has to be classified explicitly. The lists -// cover the steps of the experiment-orchestration skill: find a flag, create -// it, run an experiment, read its results, and ramp or clean up the flag. The -// skill keeps its experiment record in the flag description and creates a -// missing primary metric, so the lists include update-flag-settings and -// create-metric. It checks that the metric's event arrives with -// list-metric-events, and sets the off variation with update-off-variation -// when create-flag does not take defaults. The lists also have the tools for -// the rest of that work that the skill does not name: change an experiment -// between iterations, check exposures and contexts before a rollout, and -// record an automated rollout for the change. -// They also cover incident work: turn a change off for one segment or -// context, and ask for approval where an environment needs it, then apply -// the request after a reviewer approves it. +// The agent calls read-only tools without approval, so every tool that +// changes LaunchDarkly state must be in mcpMutativeTools. The lists cover the +// experiment flow (flags, metrics, experiments, rollouts, approvals) and +// incident work (targeting changes for a segment or context). var ( mcpReadOnlyTools = []string{ "list-projects", @@ -578,8 +567,7 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } } - // The skill and the prompt come from ai-tooling main. Both steps compare - // and write these texts, so a later change there does not reach the run. + // The run writes these texts, so a later change on main does not reach it. skill, prompt, fetchErr := fetchSources(ctx, clients.Sources) plan.Skill = planSkill(ctx, clients, opts, plan.Space, skill, fetchErr) plan.Agent = planAgent(ctx, clients, plan.Space, plan.MCP, plan.Skill, prompt, fetchErr) diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 16180c661..35b6b5657 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -13,8 +13,6 @@ import ( agenttypes "github.com/aws/aws-sdk-go-v2/service/devopsagent/types" ) -// ExperimentSkillName is the name of the experimentation skill. Setup finds -// the skill by this name, so it does not come from the downloaded SKILL.md. const ExperimentSkillName = "experiment-orchestration" const ( @@ -22,7 +20,6 @@ const ( skillFilePath = "SKILL.md" ) -// frontMatterField reads one key from the front matter of a SKILL.md. func frontMatterField(text, key string) string { lines := strings.Split(text, "\n") if len(lines) == 0 || strings.TrimSpace(lines[0]) != "---" { @@ -40,37 +37,28 @@ func frontMatterField(text, key string) string { return "" } -// SkillAction is the planned skill action. type SkillAction int const ( SkillCreate SkillAction = iota SkillKeep - // SkillLeave means that the SKILL.md of the space is not the latest one on - // ai-tooling main, or that the skill has none, so setup skips it without - // --replace-skill. + // SkillLeave: the space has a SKILL.md that differs from main, or none. SkillLeave - // SkillReplace writes the latest SKILL.md over the one in the space. SkillReplace - // SkillSkip means that setup could not read the skills of the space. The - // skill is optional, so the run continues without it. + // SkillSkip: setup could not download or read the skill. SkillSkip ) -// SkillStep is the planned experimentation skill step. type SkillStep struct { Action SkillAction AssetID string // Version is the asset version at plan time. A write stops if it changed. Version int32 - // NoFile means that the skill has no SKILL.md, for example a skill that - // someone made from a PDF. + // NoFile: the skill has no SKILL.md, for example one made from a PDF. NoFile bool - // Text is the SKILL.md on ai-tooling main at plan time, which the step - // writes. + // Text is the SKILL.md downloaded at plan time. Text string - // Err says why the step is skipped. - Err error + Err error } func (s SkillStep) describe() string { @@ -93,8 +81,7 @@ func (s SkillStep) describe() string { } } -// skillFailure is the text of a failed skill step. A missing permission is -// the most likely cause, so it names the permissions that the step needs. +// skillFailure names the permissions the step needs, the most likely cause. func skillFailure(err error) string { if errorCode(err) != "AccessDeniedException" { return err.Error() @@ -104,33 +91,22 @@ func skillFailure(err error) string { "aidevops:CreateAsset, and aidevops:UpdateAsset", err) } -// planSkill compares the skill of the space with want, the SKILL.md on -// ai-tooling main. fetchErr is set if the download failed. func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, want string, fetchErr error) SkillStep { - if fetchErr != nil { + switch { + case fetchErr != nil: return SkillStep{Action: SkillSkip, Err: fetchErr} - } - step := planSkillAction(ctx, clients, opts, space, want) - if step.Action != SkillSkip && step.Action != SkillLeave { - step.Text = want - } - - return step -} - -func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, space SpaceStep, want string) SkillStep { - if space.Action == SpaceCreate { - return SkillStep{Action: SkillCreate} + case space.Action == SpaceCreate: + return SkillStep{Action: SkillCreate, Text: want} } asset, err := findSkill(ctx, clients, space.ID, ExperimentSkillName) if err != nil { return SkillStep{Action: SkillSkip, Err: err} } if asset == nil { - return SkillStep{Action: SkillCreate} + return SkillStep{Action: SkillCreate, Text: want} } - step := SkillStep{AssetID: aws.ToString(asset.AssetId), Version: aws.ToInt32(asset.Version)} - text, found, err := skillText(ctx, clients, space.ID, step.AssetID) + step := SkillStep{AssetID: aws.ToString(asset.AssetId), Version: aws.ToInt32(asset.Version), Text: want} + text, found, err := assetText(ctx, clients, space.ID, step.AssetID, skillFilePath) if err == nil && !found { // AWS also sends ResourceNotFoundException for an asset that someone // deleted after the list. A deleted skill gets a new one. @@ -139,7 +115,7 @@ func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, sp case err != nil: return SkillStep{Action: SkillSkip, AssetID: step.AssetID, Err: err} case asset == nil: - return SkillStep{Action: SkillCreate} + return SkillStep{Action: SkillCreate, Text: want} case aws.ToString(asset.AssetId) != step.AssetID: return SkillStep{Action: SkillSkip, Err: planChanged("skill %s changed during the plan", ExperimentSkillName)} } @@ -159,8 +135,7 @@ func planSkillAction(ctx context.Context, clients Clients, opts SetupOptions, sp return step } -// findSkill returns the skill with this name in the space, or nil if there is -// none. AWS keeps skill names unique in a space. +// findSkill returns nil if the space has no skill with this name. func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { return findAsset(ctx, clients, spaceID, skillAssetType, name) } @@ -200,12 +175,6 @@ func assetName(asset agenttypes.Asset) string { return metadata.Name } -// skillText returns the SKILL.md of the skill. found is false if the skill -// has no SKILL.md. -func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (text string, found bool, err error) { - return assetText(ctx, clients, spaceID, assetID, skillFilePath) -} - func assetText(ctx context.Context, clients Clients, spaceID, assetID, path string) (text string, found bool, err error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), @@ -217,7 +186,6 @@ func assetText(ctx context.Context, clients Clients, spaceID, assetID, path stri case errors.As(err, &missing): return "", false, nil case err != nil: - // The caller names the skill. return "", false, fmt.Errorf("unable to read %s of asset %s: %w", path, assetID, err) case file.File == nil: return "", false, nil @@ -232,8 +200,7 @@ func assetText(ctx context.Context, clients Clients, spaceID, assetID, path stri return "", false, nil } -// checkSkill compares the space with a step that writes. A change to the -// version means that someone edited the skill after the plan. +// checkSkill stops a write if the skill changed after the plan. func checkSkill(ctx context.Context, clients Clients, spaceID string, step SkillStep) error { asset, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) switch { @@ -262,9 +229,8 @@ const ( SkillFailed = "failed" ) -// installSkill does the skill step. The skill is optional, so a failure does -// not stop setup. The failure goes in result.SkillError, and the caller -// reports it. +// installSkill reports a failure in result.SkillError instead of stopping +// setup, because the skill is optional. func installSkill( ctx context.Context, clients Clients, @@ -277,8 +243,6 @@ func installSkill( result.SkillAction = action if err != nil { result.SkillAction = SkillFailed - // A plan change stops the step before the write, so setup tried - // nothing. if step.Action == SkillSkip || errors.Is(err, ErrPlanChanged) { result.SkillAction = SkillSkipped } @@ -306,8 +270,6 @@ func doSkillStep( return "", fmt.Errorf("setup skipped skill %s: %w", ExperimentSkillName, step.Err) } - // The other steps can take minutes, so check the skill right before the - // write. if err := checkSkill(ctx, clients, result.AgentSpaceID, step); err != nil { return "", err } diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index ab91bf9cc..2d8f340e0 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -134,7 +134,7 @@ func TestPlanSetupLeavesSkillWithoutFile(t *testing.T) { plan.Text().Skill) } -func TestSetupLeavesEditedSkill(t *testing.T) { +func TestSetupSkipsDifferentSkill(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, @@ -155,7 +155,7 @@ func TestSetupLeavesEditedSkill(t *testing.T) { assert.Empty(t, result.SkillError) } -func TestSetupOverwritesEditedSkillWithFlag(t *testing.T) { +func TestSetupUpdatesDifferentSkillWithFlag(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, diff --git a/internal/awsdevops/source.go b/internal/awsdevops/source.go index 28b64db7e..ede2e71e6 100644 --- a/internal/awsdevops/source.go +++ b/internal/awsdevops/source.go @@ -5,83 +5,68 @@ import ( "fmt" "io" "net/http" - "net/url" "time" ) -// Sources reads the files of launchdarkly/ai-tooling, where the skill and the -// custom agent prompt live. +// Sources reads files from the main branch of launchdarkly/ai-tooling. type Sources interface { - // File returns a file of ai-tooling at ref, a branch or a commit SHA. - File(ctx context.Context, ref, path string) (string, error) + File(ctx context.Context, path string) (string, error) } const ( aiToolingRepo = "launchdarkly/ai-tooling" - aiToolingBranch = "main" skillSourcePath = "skills/aws-devops-agent/experiment-orchestration/SKILL.md" agentSourcePath = "skills/aws-devops-agent/agents/experiment-orchestrator.md" + maxSourceBytes = 1 << 20 ) -const ( - sourceTimeout = 30 * time.Second - // maxSourceBytes caps a download. The files are a few KB. - maxSourceBytes = 1 << 20 -) - -// GitHubSources reads ai-tooling from GitHub without authentication. type GitHubSources struct { Client *http.Client RawURL string } -// NewGitHubSources reads from github.com. func NewGitHubSources() GitHubSources { return GitHubSources{ - Client: &http.Client{Timeout: sourceTimeout}, + Client: &http.Client{Timeout: 30 * time.Second}, RawURL: "https://raw.githubusercontent.com", } } -func (s GitHubSources) File(ctx context.Context, ref, path string) (string, error) { - body, err := s.get(ctx, fmt.Sprintf("%s/%s/%s/%s", s.RawURL, aiToolingRepo, url.PathEscape(ref), path)) +func (s GitHubSources) File(ctx context.Context, path string) (string, error) { + text, err := s.get(ctx, fmt.Sprintf("%s/%s/main/%s", s.RawURL, aiToolingRepo, path)) if err != nil { - return "", fmt.Errorf("unable to download %s from %s at %s: %w", path, aiToolingRepo, ref, err) + return "", fmt.Errorf("unable to download %s from %s: %w", path, aiToolingRepo, err) } - return string(body), nil + return text, nil } -func (s GitHubSources) get(ctx context.Context, address string) ([]byte, error) { +func (s GitHubSources) get(ctx context.Context, address string) (string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) if err != nil { - return nil, err - } - client := s.Client - if client == nil { - client = &http.Client{Timeout: sourceTimeout} + return "", err } - resp, err := client.Do(req) + resp, err := s.Client.Do(req) if err != nil { - return nil, err + return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GitHub sent %s", resp.Status) + return "", fmt.Errorf("GitHub sent %s", resp.Status) } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxSourceBytes)) - return io.ReadAll(io.LimitReader(resp.Body, maxSourceBytes)) + return string(body), err } -// fetchSources downloads the skill and the custom agent prompt from main. func fetchSources(ctx context.Context, sources Sources) (skill, prompt string, err error) { if sources == nil { return "", "", fmt.Errorf("no source for the files of %s", aiToolingRepo) } - if skill, err = sources.File(ctx, aiToolingBranch, skillSourcePath); err != nil { + if skill, err = sources.File(ctx, skillSourcePath); err != nil { return "", "", err } - if prompt, err = sources.File(ctx, aiToolingBranch, agentSourcePath); err != nil { + if prompt, err = sources.File(ctx, agentSourcePath); err != nil { return "", "", err } diff --git a/internal/awsdevops/source_test.go b/internal/awsdevops/source_test.go index e30923a3a..2b1b899a4 100644 --- a/internal/awsdevops/source_test.go +++ b/internal/awsdevops/source_test.go @@ -13,7 +13,6 @@ import ( "github.com/launchdarkly/ldcli/internal/awsdevops" ) -// testSkill and testAgentPrompt stand in for the files on ai-tooling main. const ( testSkill = "---\n" + "name: experiment-orchestration\n" + @@ -25,7 +24,6 @@ const ( testDescription = "Run an experiment from a goal to a report." ) -// fakeSources serves ai-tooling from memory. files maps "ref path" to text. type fakeSources struct { files map[string]string fileErr error @@ -33,16 +31,16 @@ type fakeSources struct { func newFakeSources() *fakeSources { return &fakeSources{files: map[string]string{ - "main " + awsdevops.SkillSourcePath: testSkill, - "main " + awsdevops.AgentSourcePath: testAgentPrompt, + awsdevops.SkillSourcePath: testSkill, + awsdevops.AgentSourcePath: testAgentPrompt, }} } -func (f *fakeSources) File(_ context.Context, ref, path string) (string, error) { +func (f *fakeSources) File(_ context.Context, path string) (string, error) { if f.fileErr != nil { return "", f.fileErr } - text, ok := f.files[ref+" "+path] + text, ok := f.files[path] if !ok { return "", errors.New("GitHub sent 404 Not Found") } @@ -112,10 +110,10 @@ func TestGitHubSourcesReadFiles(t *testing.T) { defer server.Close() sources := awsdevops.GitHubSources{Client: server.Client(), RawURL: server.URL} - text, err := sources.File(context.Background(), "main", awsdevops.SkillSourcePath) + text, err := sources.File(context.Background(), awsdevops.SkillSourcePath) require.NoError(t, err) assert.Equal(t, testSkill, text) - _, err = sources.File(context.Background(), "main", awsdevops.AgentSourcePath) + _, err = sources.File(context.Background(), awsdevops.AgentSourcePath) assert.ErrorContains(t, err, "GitHub sent 429 Too Many Requests") } From e8db6e7b9fc04534779b85a5b189126284e6223a Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 11:38:08 -0700 Subject: [PATCH 33/34] docs: clarify why setup downloads the skill and prompt once Co-Authored-By: Claude Opus 5.5 --- internal/awsdevops/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index c6e28484c..eeca3c162 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -567,7 +567,7 @@ func PlanSetup(ctx context.Context, clients Clients, opts SetupOptions, provisio } } - // The run writes these texts, so a later change on main does not reach it. + // Download once, so the run writes the same texts that the plan compared. skill, prompt, fetchErr := fetchSources(ctx, clients.Sources) plan.Skill = planSkill(ctx, clients, opts, plan.Space, skill, fetchErr) plan.Agent = planAgent(ctx, clients, plan.Space, plan.MCP, plan.Skill, prompt, fetchErr) From 03c96a9362891160b620249fc18bbf665172f453 Mon Sep 17 00:00:00 2001 From: Christina Chu Date: Thu, 29 Oct 2026 11:40:27 -0700 Subject: [PATCH 34/34] chore: restore comments that the earlier cleanup only rephrased Comments whose code did not change get their wording from main back, and skillText and two test names return. Comments now change only where the behavior did. Co-Authored-By: Claude Opus 5.5 --- cmd/aws_devops_agent/setup.go | 10 ++++---- internal/awsdevops/setup.go | 19 ++++++++------ internal/awsdevops/skill.go | 43 ++++++++++++++++++++++++-------- internal/awsdevops/skill_test.go | 4 +-- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/cmd/aws_devops_agent/setup.go b/cmd/aws_devops_agent/setup.go index 63921693d..26e9e3e29 100644 --- a/cmd/aws_devops_agent/setup.go +++ b/cmd/aws_devops_agent/setup.go @@ -56,10 +56,10 @@ which can ask you to sign in. The agent then acts as the LaunchDarkly member who approved access, in every agent space. If the agent space already has an MCP server, setup keeps it. -A space that reuses a server keeps the tools in its allowlist. If the -allowlist misses tools that this ldcli version allows, setup adds them. AWS -does not send back the classification of each tool, so the rewrite gives -every tool the ldcli classification. +A space that reuses a server keeps its tool allowlist. If the allowlist misses +tools that this ldcli version allows, setup adds them. AWS does not send back +the classification of each tool, so the rewrite gives every tool the ldcli +classification. Setup downloads the skill and the custom agent prompt from the main branch of launchdarkly/ai-tooling on GitHub, so those steps need access to GitHub. If @@ -95,7 +95,7 @@ does the other steps.`, cmd.Flags().String(idpClientSecretFlag, "", "OIDC client secret, required when --auth-flow=idp") cmd.Flags().String(mcpServiceIDFlag, "", "Registered MCP server to connect, instead of the server of the AWS account") cmd.Flags().String(mcpEndpointFlag, "", "MCP endpoint to register. Setup reuses only a server at this endpoint. Defaults to "+awsdevops.MCPServerEndpoint) - cmd.Flags().Bool(replaceSkillFlag, false, "Update the "+awsdevops.ExperimentSkillName+" skill to the latest version if it differs") + cmd.Flags().Bool(replaceSkillFlag, false, "Overwrite the "+awsdevops.ExperimentSkillName+" skill if it differs from the latest version") cmd.Flags().Bool(addMCPToolsFlag, false, "Has no effect. Setup adds the missing tools without it") _ = cmd.Flags().MarkDeprecated(addMCPToolsFlag, "setup now adds the missing MCP tools without it") diff --git a/internal/awsdevops/setup.go b/internal/awsdevops/setup.go index eeca3c162..58870314c 100644 --- a/internal/awsdevops/setup.go +++ b/internal/awsdevops/setup.go @@ -61,10 +61,14 @@ const ( // registers after a person approves access. Tests make it shorter. var oauthPollInterval = 5 * time.Second -// The agent calls read-only tools without approval, so every tool that -// changes LaunchDarkly state must be in mcpMutativeTools. The lists cover the -// experiment flow (flags, metrics, experiments, rollouts, approvals) and -// incident work (targeting changes for a segment or context). +// The agent invokes read-only tools without asking for approval, so anything +// that changes LaunchDarkly state has to be classified explicitly. The lists +// cover the steps of the experiment-orchestration skill: find a flag, create +// it, run an experiment, read its results, and ramp or clean up the flag. The +// skill creates a missing primary metric, so the lists include create-metric. +// They also cover incident work: turn a change off for one segment or +// context, and ask for approval where an environment needs it, then apply +// the request after a reviewer approves it. var ( mcpReadOnlyTools = []string{ "list-projects", @@ -140,8 +144,8 @@ type SetupOptions struct { // OpenURL, when set, opens the OAuth approval page in a browser. OpenURL func(url string) error - // ReplaceSkill writes the latest SKILL.md over an existing - // experiment-orchestration skill that differs from it. + // ReplaceSkill overwrites the SKILL.md of an existing + // experiment-orchestration skill that differs from the latest one. ReplaceSkill bool // Logf, when set, reports progress as each step completes. @@ -1018,7 +1022,8 @@ func RunPlan(ctx context.Context, clients Clients, opts SetupOptions, plan Setup if err := registerMCPServer(ctx, clients, opts, plan.MCP, &result, logf); err != nil { return result, err } - // The optional skill and agent steps go last. + // The skill and the agent go last, so a failure of the optional steps + // cannot stop the MCP server step. installSkill(ctx, clients, plan.Skill, &result, logf) installAgent(ctx, clients, plan.Agent, &result, logf) diff --git a/internal/awsdevops/skill.go b/internal/awsdevops/skill.go index 35b6b5657..2c3d12a02 100644 --- a/internal/awsdevops/skill.go +++ b/internal/awsdevops/skill.go @@ -20,6 +20,7 @@ const ( skillFilePath = "SKILL.md" ) +// frontMatterField reads one key from the front matter of a SKILL.md. func frontMatterField(text, key string) string { lines := strings.Split(text, "\n") if len(lines) == 0 || strings.TrimSpace(lines[0]) != "---" { @@ -37,28 +38,35 @@ func frontMatterField(text, key string) string { return "" } +// SkillAction is the planned skill action. type SkillAction int const ( SkillCreate SkillAction = iota SkillKeep - // SkillLeave: the space has a SKILL.md that differs from main, or none. + // SkillLeave means that the SKILL.md differs from the latest one, or that + // the skill has none, so setup does not change it without --replace-skill. SkillLeave SkillReplace - // SkillSkip: setup could not download or read the skill. + // SkillSkip means that setup could not download the skill or read the + // skills of the space. The skill is optional, so the run continues without + // it. SkillSkip ) +// SkillStep is the planned experimentation skill step. type SkillStep struct { Action SkillAction AssetID string // Version is the asset version at plan time. A write stops if it changed. Version int32 - // NoFile: the skill has no SKILL.md, for example one made from a PDF. + // NoFile means that the skill has no SKILL.md, for example a skill that + // someone made from a PDF. NoFile bool // Text is the SKILL.md downloaded at plan time. Text string - Err error + // Err says why the step is skipped. + Err error } func (s SkillStep) describe() string { @@ -81,7 +89,8 @@ func (s SkillStep) describe() string { } } -// skillFailure names the permissions the step needs, the most likely cause. +// skillFailure is the text of a failed skill step. A missing permission is +// the most likely cause, so it names the permissions that the step needs. func skillFailure(err error) string { if errorCode(err) != "AccessDeniedException" { return err.Error() @@ -106,7 +115,7 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp return SkillStep{Action: SkillCreate, Text: want} } step := SkillStep{AssetID: aws.ToString(asset.AssetId), Version: aws.ToInt32(asset.Version), Text: want} - text, found, err := assetText(ctx, clients, space.ID, step.AssetID, skillFilePath) + text, found, err := skillText(ctx, clients, space.ID, step.AssetID) if err == nil && !found { // AWS also sends ResourceNotFoundException for an asset that someone // deleted after the list. A deleted skill gets a new one. @@ -135,7 +144,8 @@ func planSkill(ctx context.Context, clients Clients, opts SetupOptions, space Sp return step } -// findSkill returns nil if the space has no skill with this name. +// findSkill returns the skill with this name in the space, or nil if there is +// none. AWS keeps skill names unique in a space. func findSkill(ctx context.Context, clients Clients, spaceID, name string) (*agenttypes.Asset, error) { return findAsset(ctx, clients, spaceID, skillAssetType, name) } @@ -175,6 +185,12 @@ func assetName(asset agenttypes.Asset) string { return metadata.Name } +// skillText returns the SKILL.md of the skill. found is false if the skill +// has no SKILL.md. +func skillText(ctx context.Context, clients Clients, spaceID, assetID string) (text string, found bool, err error) { + return assetText(ctx, clients, spaceID, assetID, skillFilePath) +} + func assetText(ctx context.Context, clients Clients, spaceID, assetID, path string) (text string, found bool, err error) { file, err := clients.Agent.GetAssetFile(ctx, &devopsagent.GetAssetFileInput{ AgentSpaceId: aws.String(spaceID), @@ -186,6 +202,7 @@ func assetText(ctx context.Context, clients Clients, spaceID, assetID, path stri case errors.As(err, &missing): return "", false, nil case err != nil: + // The caller names the skill. return "", false, fmt.Errorf("unable to read %s of asset %s: %w", path, assetID, err) case file.File == nil: return "", false, nil @@ -200,7 +217,8 @@ func assetText(ctx context.Context, clients Clients, spaceID, assetID, path stri return "", false, nil } -// checkSkill stops a write if the skill changed after the plan. +// checkSkill compares the space with a step that writes. A change to the +// version means that someone edited the skill after the plan. func checkSkill(ctx context.Context, clients Clients, spaceID string, step SkillStep) error { asset, err := findSkill(ctx, clients, spaceID, ExperimentSkillName) switch { @@ -229,8 +247,9 @@ const ( SkillFailed = "failed" ) -// installSkill reports a failure in result.SkillError instead of stopping -// setup, because the skill is optional. +// installSkill does the skill step. The skill is optional, so a failure does +// not stop setup. The failure goes in result.SkillError, and the caller +// reports it. func installSkill( ctx context.Context, clients Clients, @@ -243,6 +262,8 @@ func installSkill( result.SkillAction = action if err != nil { result.SkillAction = SkillFailed + // A plan change stops the step before the write, so setup tried + // nothing. if step.Action == SkillSkip || errors.Is(err, ErrPlanChanged) { result.SkillAction = SkillSkipped } @@ -270,6 +291,8 @@ func doSkillStep( return "", fmt.Errorf("setup skipped skill %s: %w", ExperimentSkillName, step.Err) } + // The other steps can take minutes, so check the skill right before the + // write. if err := checkSkill(ctx, clients, result.AgentSpaceID, step); err != nil { return "", err } diff --git a/internal/awsdevops/skill_test.go b/internal/awsdevops/skill_test.go index 2d8f340e0..ab91bf9cc 100644 --- a/internal/awsdevops/skill_test.go +++ b/internal/awsdevops/skill_test.go @@ -134,7 +134,7 @@ func TestPlanSetupLeavesSkillWithoutFile(t *testing.T) { plan.Text().Skill) } -func TestSetupSkipsDifferentSkill(t *testing.T) { +func TestSetupLeavesEditedSkill(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, skillFiles: map[string]string{"ki-ours": "# Edited by the customer"}, @@ -155,7 +155,7 @@ func TestSetupSkipsDifferentSkill(t *testing.T) { assert.Empty(t, result.SkillError) } -func TestSetupUpdatesDifferentSkillWithFlag(t *testing.T) { +func TestSetupOverwritesEditedSkillWithFlag(t *testing.T) { agent := &fakeAgent{ assets: []agenttypes.Asset{skillAsset("ki-ours", "experiment-orchestration")}, skillFiles: map[string]string{"ki-ours": "# Edited by the customer"},