diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6630ef2f9..b9af336cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -240,6 +240,7 @@ jobs: filters: | skill: - 'skills/basecamp/SKILL.md' + - 'skills/basecamp-connect/SKILL.md' - 'skill-evals/**' - name: Set up Ruby diff --git a/.surface b/.surface index c3d974102..95c60dd98 100644 --- a/.surface +++ b/.surface @@ -670,6 +670,7 @@ CMD basecamp config unset CMD basecamp config untrust CMD basecamp connect CMD basecamp connect setup +CMD basecamp connect show CMD basecamp docs CMD basecamp docs archive CMD basecamp docs doc @@ -5399,6 +5400,27 @@ FLAG basecamp connect setup --trust type=string FLAG basecamp connect setup --verbose type=count FLAG basecamp connect setup --watch-completions type=stringArray FLAG basecamp connect setup --worktrees type=bool +FLAG basecamp connect show --account type=string +FLAG basecamp connect show --agent type=bool +FLAG basecamp connect show --cache-dir type=string +FLAG basecamp connect show --count type=bool +FLAG basecamp connect show --help type=bool +FLAG basecamp connect show --hints type=bool +FLAG basecamp connect show --ids-only type=bool +FLAG basecamp connect show --in type=string +FLAG basecamp connect show --jq type=string +FLAG basecamp connect show --json type=bool +FLAG basecamp connect show --markdown type=bool +FLAG basecamp connect show --md type=bool +FLAG basecamp connect show --no-hints type=bool +FLAG basecamp connect show --no-stats type=bool +FLAG basecamp connect show --profile type=string +FLAG basecamp connect show --project type=string +FLAG basecamp connect show --quiet type=bool +FLAG basecamp connect show --stats type=bool +FLAG basecamp connect show --styled type=bool +FLAG basecamp connect show --todolist type=string +FLAG basecamp connect show --verbose type=count FLAG basecamp docs --account type=string FLAG basecamp docs --agent type=bool FLAG basecamp docs --cache-dir type=string @@ -18577,6 +18599,7 @@ SUB basecamp config unset SUB basecamp config untrust SUB basecamp connect SUB basecamp connect setup +SUB basecamp connect show SUB basecamp docs SUB basecamp docs archive SUB basecamp docs doc diff --git a/AGENTS.md b/AGENTS.md index 9b83a7b68..893ccb66c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,16 +75,18 @@ Run it early and often — after finishing a feature, after fixing a bug, before pushing. If you're about to `git push` and haven't run `bin/ci` in this session, stop and run it first. -**Skill drift**: `make check-skill-drift` runs over both `skills/basecamp/SKILL.md` -and `skills/basecamp-doctor/SKILL.md`, checking that the commands and flags each one +**Skill drift**: `make check-skill-drift` runs over `skills/basecamp/SKILL.md`, +`skills/basecamp-doctor/SKILL.md` and `skills/basecamp-connect/SKILL.md`, checking that the commands and flags each one *references* still exist in the `.surface` snapshot. It catches stale references, not -missing coverage, so adding a command breaks neither skill. +missing coverage, so adding a command breaks none of them. Removal is only partly caught. `resolve_cmd` walks up to the nearest existing ancestor, so dropping a nested subcommand leaves the reference resolving against its parent and the check still passes — `basecamp setup ` resolves as `basecamp setup`. Removing a top-level command is caught; removing a subcommand is not. Don't lean on CI for this. Update the skill the change actually affects; -basecamp-doctor deliberately covers only doctor, setup and auth remediation. +basecamp-doctor deliberately covers only doctor, setup and auth remediation; +basecamp-connect covers `basecamp auth agent connect` and `basecamp connect`, and its +evals run against its own SKILL.md (`make -C skill-evals eval-connect`). ```bash bin/ci # The single command — run this diff --git a/Makefile b/Makefile index fad4bb1d5..e70b3ffcf 100644 --- a/Makefile +++ b/Makefile @@ -457,6 +457,7 @@ check-surface-compat: build check-skill-drift: @scripts/check-skill-drift.sh @scripts/check-skill-drift.sh skills/basecamp-doctor/SKILL.md + @scripts/check-skill-drift.sh skills/basecamp-connect/SKILL.md # Verify group commands show help bare (no RunE on parents with subcommands) .PHONY: check-bare-groups @@ -555,6 +556,7 @@ tools: .PHONY: skill-eval skill-eval: $(MAKE) -C skill-evals eval + $(MAKE) -C skill-evals eval-connect # Sync skills to basecamp/skills distribution repo # Usage: make sync-skills TAG=v1.2.3 diff --git a/e2e/smoke/smoke_lifecycle.bats b/e2e/smoke/smoke_lifecycle.bats index 4fee36ec9..df00a6567 100644 --- a/e2e/smoke/smoke_lifecycle.bats +++ b/e2e/smoke/smoke_lifecycle.bats @@ -20,6 +20,10 @@ load smoke_helper mark_out_of_scope "Needs a connected agent profile and writes local connector policy — covered by Go tests in internal/commands and internal/connector/setup" } +@test "connect show is out of scope" { + mark_out_of_scope "Reads the connector policy a connected profile's setup wrote — covered by Go tests in internal/commands" +} + @test "auth refresh is out of scope" { mark_out_of_scope "Requires OAuth credentials" } diff --git a/internal/commands/commands.go b/internal/commands/commands.go index c76a9a23f..8c2c75b7a 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -146,7 +146,7 @@ func CommandCategories() []CommandCategory { {Name: "bonfire", Category: "additional", Description: "Multi-chat orchestration", Actions: []string{"split", "layout"}, Experimental: true, DevOnly: true}, {Name: "api", Category: "additional", Description: "Raw API access"}, {Name: "mcp", Category: "additional", Description: "Serve Basecamp to MCP clients over stdio"}, - {Name: "connect", Category: "additional", Description: "Run a local agent connector for a Basecamp agent", Actions: []string{"setup"}}, + {Name: "connect", Category: "additional", Description: "Set up a local agent connector for a Basecamp agent", Actions: []string{"setup", "show"}}, {Name: "help", Category: "additional", Description: "Show help"}, {Name: "version", Category: "additional", Description: "Show version"}, }, diff --git a/internal/commands/connect.go b/internal/commands/connect.go index bf33857ad..3be7c24e4 100644 --- a/internal/commands/connect.go +++ b/internal/commands/connect.go @@ -6,9 +6,11 @@ import ( "fmt" "net/http" "os" + "runtime" "strconv" "strings" "time" + "unicode" "github.com/spf13/cobra" @@ -36,12 +38,184 @@ the work to a local coding agent that replies in Basecamp as the agent. Connect the agent to a profile first (basecamp auth agent connect -P ), then run setup on that profile: it records who may drive the agent, maps projects to the directories their work runs in, and checks the connector is -ready.`, +ready. Show prints what setup recorded.`, } cmd.AddCommand(newConnectSetupCmd()) + cmd.AddCommand(newConnectShowCmd()) return cmd } +// newConnectShowCmd shows a profile's connect.json, read the way the +// connector reads it: through setup.Load's safety checks, so a symlink, a +// file someone else could have written, or anything oversized is refused +// before a byte of it is shown. +func newConnectShowCmd() *cobra.Command { + return &cobra.Command{ + Use: "show", + Short: "Show a profile's connector setup without changing it", + Long: `Show the connect.json a profile's setup wrote: the agent, the operator and +trust mode, each routed project and the worker settings. + +The file is read through the same checks setup and the connector apply: +it is refused, and nothing of it shown, when it is a symlink, is not a +regular file, could have been changed by another user, or does not parse. +Nothing is changed and nothing is fetched: show reads no credential and +makes no request. To check readiness, run setup again. + +A profile that does not exist is refused before this runs, naming the +profiles there are, as it is for every command; a profile that exists and +has never been set up is reported as not found. + +Examples: + basecamp connect show -P agent + basecamp connect show -P agent --json`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + app := appctx.FromContext(cmd.Context()) + if app == nil { + return fmt.Errorf("app not initialized") + } + return runConnectShow(app) + }, + } +} + +func runConnectShow(app *appctx.App) error { + name := app.Config.ActiveProfile + if name == "" { + return output.ErrUsageHint("Show needs the agent's profile", "Pass -P/--profile .") + } + if !isValidProfileName(name) { + return output.ErrUsage(fmt.Sprintf("Invalid profile name %q: use only letters, numbers, hyphens, and underscores", name)) + } + path, err := setup.Path(config.GlobalConfigDir(), name) + if err != nil { + return output.ErrUsage(err.Error()) + } + f, err := setup.Load(path) + switch { + case errors.Is(err, os.ErrNotExist): + return output.ErrNotFoundHint("connect.json for profile", name, + "The profile has not been set up. Set it up: basecamp connect setup -P "+shellQuote(name)+" --operator-profile '' --route '='") + case err != nil && runtime.GOOS == "windows": + return output.ErrUsageHint("connect.json cannot be used: "+setup.ErrorText(err), + "The connector's setup is not supported on Windows: this CLI cannot verify who can change connect.json there.") + case err != nil: + return output.ErrUsageHint("connect.json cannot be used: "+setup.ErrorText(err), + "Nothing of it was shown. Fix or remove "+richtext.SanitizeSingleLine(path)+", then run setup again.") + } + if f.Profile != name { + // A policy copied from another profile's directory is not this + // profile's, however valid it is on its own: setup refuses it too. + return output.ErrUsageHint(fmt.Sprintf("%s names profile %q, not %q", richtext.SanitizeSingleLine(path), f.Profile, name), + "Nothing of it was shown. Remove "+richtext.SanitizeSingleLine(path)+" and run setup again for this profile.") + } + + // The generic object renderer drops nested maps, which is where the + // agent, the trust and the routes live, so a person's formats get them + // flattened to one line each; JSON keeps the file's own shape. + markdown := app.Output.EffectiveFormat() == output.FormatMarkdown + return app.OK(connectShowResult{Path: path, File: f}, + output.WithDisplayData(connectShowDisplay(path, f, markdown)), + output.WithSummary(fmt.Sprintf("Connector setup for profile %q: trust %s, %d routed project(s)", name, f.Trust.Mode, len(f.Projects)))) +} + +// connectShowDisplay is show's data for a person: every setting connect.json +// records as a flat field, one per route. Paths are shown exactly: quoted, +// or as a code span in Markdown, so nothing in one renders as formatting or +// reaches a terminal as a control byte. +func connectShowDisplay(path string, f setup.File, markdown bool) map[string]any { + exact := func(s string) string { + // strconv.Quote escapes controls and backslashes; "<" is escaped too, + // or the renderer would take a path holding "" for HTML and + // rewrite it. + return strings.ReplaceAll(strconv.Quote(s), "<", `\x3c`) + } + if markdown { + exact = func(s string) string { return markdownCode(escapeControls(s)) } + } + agent := fmt.Sprintf("person %d (%s)", f.Agent.PersonID, f.Agent.Kind) + if f.Agent.IdentityID != 0 { + agent += fmt.Sprintf(", identity %d", f.Agent.IdentityID) + } + trust := string(f.Trust.Mode) + if len(f.Trust.AllowlistIDs) > 0 { + ids := make([]string, len(f.Trust.AllowlistIDs)) + for i, id := range f.Trust.AllowlistIDs { + ids[i] = strconv.FormatInt(id, 10) + } + trust += ": people " + strings.Join(ids, ", ") + } + worktrees := "off" + if f.Worktrees { + worktrees = "on" + } + d := map[string]any{ + "file": exact(path), + "account": f.AccountID, + "agent": agent, + "operator": fmt.Sprintf("person %d", f.Trust.OperatorID), + "trust": trust, + "workers": fmt.Sprintf("%s, concurrency %d, deadline %s, worktrees %s", f.Driver, f.Concurrency, time.Duration(f.Deadline), worktrees), + "projects": strconv.Itoa(len(f.Projects)) + " routed", + } + for id, r := range f.Projects { + route := exact(r.Path) + if r.Class != "" { + route += ", class " + r.Class + } + if r.WatchCompletions { + route += ", watches completions" + } + d[fmt.Sprintf("route_%d", id)] = route + } + return d +} + +// escapeControls writes every control or non-printable rune in s, the +// backslash and "<" (which the renderer would read as HTML) as a Go escape +// (\x1b, \u009b, \\, \x3c), so a path can reach a +// terminal or a pager without a single control byte and still reads exactly: +// an escape in the output never stands for text the path already held. +func escapeControls(s string) string { + var b strings.Builder + for _, r := range s { + if r == '<' { + b.WriteString(`\x3c`) + continue + } + if r == '\\' || unicode.IsControl(r) || !unicode.IsPrint(r) { + q := strconv.QuoteRune(r) + b.WriteString(q[1 : len(q)-1]) + continue + } + b.WriteRune(r) + } + return b.String() +} + +// markdownCode is s as a Markdown code span, fenced with one more backtick +// than the longest run inside it so no backtick in s can close it. +func markdownCode(s string) string { + longest, run := 0, 0 + for _, r := range s { + if r == '`' { + run++ + longest = max(longest, run) + } else { + run = 0 + } + } + fence := strings.Repeat("`", longest+1) + return fence + " " + s + " " + fence +} + +// connectShowResult is show's data: where the file is, and what it holds. +type connectShowResult struct { + Path string `json:"path"` + setup.File +} + // connectSetupFlags are setup's flags, as typed. type connectSetupFlags struct { expectIdentity string diff --git a/internal/commands/connect_setup_test.go b/internal/commands/connect_setup_test.go index 1fd306324..18ce4799d 100644 --- a/internal/commands/connect_setup_test.go +++ b/internal/commands/connect_setup_test.go @@ -1234,3 +1234,219 @@ func TestConnectSetupClassifiesCancellationUnderTheLock(t *testing.T) { }) } } + +func runConnectShowCmd(t *testing.T, app *appctx.App, args ...string) (string, error) { + t.Helper() + cmd := NewConnectCmd() + cmd.SetArgs(append([]string{"show"}, args...)) + cmd.SetContext(appctx.WithApp(context.Background(), app)) + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + err := cmd.Execute() + return out.String(), err +} + +// Show prints what setup recorded, and changes nothing. +func TestConnectShowPrintsWhatSetupRecorded(t *testing.T) { + s := startConnectSetupServer(t) + firstSetup(t, s) + path := connectSetupPath(t, "agent") + before, err := os.ReadFile(path) + require.NoError(t, err) + + app := newConnectSetupApp(t, s, "agent") + var buf bytes.Buffer + app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &buf}) + out, err := runConnectShowCmd(t, app) + require.NoError(t, err, out) + + var envelope struct { + OK bool `json:"ok"` + Data struct { + Path string `json:"path"` + Profile string `json:"profile"` + Agent setup.Agent `json:"agent"` + Trust admission.Trust `json:"trust"` + Projects map[int64]admission.Route `json:"projects"` + Deadline string `json:"deadline"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &envelope), buf.String()) + assert.True(t, envelope.OK) + assert.Equal(t, path, envelope.Data.Path) + assert.Equal(t, "agent", envelope.Data.Profile) + assert.Equal(t, setup.Agent{PersonID: setupAgentPerson, Kind: setup.KindAgent}, envelope.Data.Agent) + assert.Equal(t, setupOperatorPerson, envelope.Data.Trust.OperatorID) + assert.Contains(t, envelope.Data.Projects, setupProject) + assert.Equal(t, "45m0s", envelope.Data.Deadline) + assert.NotContains(t, buf.String(), fakeConnectSecret) + + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, before, after, "show changes nothing") +} + +// A profile that exists and has never been set up is not found. A profile +// that does not exist at all never reaches this command: the root refuses an +// unknown -P for every command, which is what tells a typo from a profile +// with no connector policy. +func TestConnectShowOnAProfileNeverSetUpIsNotFound(t *testing.T) { + s := startConnectSetupServer(t) + app := connectSetupApp(t, s, "agent") + out, err := runConnectShowCmd(t, app) + var apiErr *output.Error + require.ErrorAs(t, err, &apiErr, out) + assert.Equal(t, output.CodeNotFound, apiErr.Code) + assert.Contains(t, apiErr.Hint, "basecamp connect setup -P agent") +} + +// A connect.json that is a symlink, or that someone else could have written, +// is refused before any of it is shown: show is not a way to print an +// arbitrary local file. +func TestConnectShowRefusesAnUnsafeConnectJSON(t *testing.T) { + for _, tc := range []struct { + name string + plant func(t *testing.T, path string) + }{ + {"a symlink to another file", func(t *testing.T, path string) { + // A well-formed policy, so only the refusal to follow the link + // stops it being shown. + data, err := os.ReadFile(path) + require.NoError(t, err) + target := filepath.Join(t.TempDir(), "elsewhere.json") + require.NoError(t, os.WriteFile(target, data, 0o600)) + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Symlink(target, path)) + }}, + {"a file others can write", func(t *testing.T, path string) { + require.NoError(t, os.Chmod(path, 0o666)) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := startConnectSetupServer(t) + firstSetup(t, s) + tc.plant(t, connectSetupPath(t, "agent")) + + app := newConnectSetupApp(t, s, "agent") + var buf bytes.Buffer + app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &buf}) + out, err := runConnectShowCmd(t, app) + var apiErr *output.Error + require.ErrorAs(t, err, &apiErr, out) + assert.Equal(t, output.CodeUsage, apiErr.Code) + assert.NotContains(t, out+buf.String()+apiErr.Message+apiErr.Hint, `"projects"`, "nothing of the file is shown") + }) + } +} + +// A person reading show in a terminal or as Markdown sees what matters: the +// agent, the operator, the trust and every route, which the generic object +// renderer would drop, through the same output pipeline as every command. +func TestConnectShowTellsAPersonEverySetting(t *testing.T) { + s := startConnectSetupServer(t) + firstSetup(t, s) + f, err := setup.Load(connectSetupPath(t, "agent")) + require.NoError(t, err) + var route string + for _, r := range f.Projects { + route = r.Path + } + + for _, format := range []output.Format{output.FormatStyled, output.FormatMarkdown} { + app := newConnectSetupApp(t, s, "agent") + var buf bytes.Buffer + app.Output = output.New(output.Options{Format: format, Writer: &buf}) + out, err := runConnectShowCmd(t, app) + require.NoError(t, err, out) + shown := buf.String() + for _, want := range []string{ + fmt.Sprintf("person %d (agent)", setupAgentPerson), + fmt.Sprintf("person %d", setupOperatorPerson), + "operator", + fmt.Sprintf("Route %d", setupProject), + route, + "deadline 45m0s", + } { + assert.Contains(t, shown, want, "format %v", format) + } + } +} + +// A valid policy copied in from another profile is not this profile's: show +// refuses it rather than label it as this profile's setup. +func TestConnectShowRefusesAnotherProfilesPolicy(t *testing.T) { + s := startConnectSetupServer(t) + firstSetup(t, s) + data, err := os.ReadFile(connectSetupPath(t, "agent")) + require.NoError(t, err) + other := connectSetupPath(t, "other") + require.NoError(t, os.MkdirAll(filepath.Dir(other), 0o700)) + require.NoError(t, os.WriteFile(other, data, 0o600)) + + app := newConnectSetupApp(t, s, "other") + var buf bytes.Buffer + app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &buf}) + out, err := runConnectShowCmd(t, app) + var apiErr *output.Error + require.ErrorAs(t, err, &apiErr, out) + assert.Equal(t, output.CodeUsage, apiErr.Code) + assert.Contains(t, apiErr.Message, `names profile "agent", not "other"`) + assert.NotContains(t, out+buf.String(), `"projects"`) +} + +// A route path is a clean absolute path, which may still hold control +// characters: human output shows them escaped, never raw. +func TestConnectShowEscapesControlsInARoutePath(t *testing.T) { + path := "/home/me/Work/Q3 \x1b[31mred\u009b $launch" + f := setup.New("agent") + f.AccountID = "999" + f.Agent = setup.Agent{PersonID: 4001, Kind: setup.KindAgent} + f.Trust.OperatorID = 1001 + f.Projects[222] = admission.Route{Path: path} + for _, markdown := range []bool{false, true} { + route := connectShowDisplay("/x/connect.json", f, markdown)["route_222"].(string) + assert.NotContains(t, route, "\x1b", "markdown %v", markdown) + assert.NotContains(t, route, "\u009b", "markdown %v", markdown) + assert.Contains(t, route, `Q3 \x1b[31mred\u009b $launch`, "markdown %v", markdown) + } +} + +// The file's own path is shown as literally as the routes: a configuration +// directory holding Markdown syntax or a backslash does not render as a link +// or read as an escape. +func TestConnectShowShowsTheFilePathLiterally(t *testing.T) { + f := setup.New("agent") + file := connectShowDisplay("/home/[me](x)/`cfg`/a\\x1b/connect.json", f, true)["file"] + assert.Equal(t, "`` /home/[me](x)/`cfg`/a\\\\x1b/connect.json ``", file) +} + +// A route path holding what looks like HTML or Markdown survives the real +// renderers: nothing in it is converted, dropped or rendered. +func TestConnectShowKeepsAPathThatLooksLikeMarkup(t *testing.T) { + s := startConnectSetupServer(t) + firstSetup(t, s) + dir := filepath.Join(t.TempDir(), "Q3 bold [x](y) `tick`") + require.NoError(t, os.Mkdir(dir, 0o700)) + out, err := runConnectSetupCmd(t, newConnectSetupApp(t, s, "agent"), "--route", fmt.Sprintf("%d=%s", setupProject, dir)) + require.NoError(t, err, out) + + for format, want := range map[output.Format]string{ + output.FormatStyled: `Q3 \x3cb>bold\x3ci> [x](y) ` + "`tick`", + output.FormatMarkdown: `Q3 \x3cb>bold\x3ci> [x](y) ` + "`tick`", + } { + app := newConnectSetupApp(t, s, "agent") + var buf bytes.Buffer + app.Output = output.New(output.Options{Format: format, Writer: &buf}) + out, err := runConnectShowCmd(t, app) + require.NoError(t, err, out) + assert.Contains(t, buf.String(), want, "format %v", format) + } +} + +func TestMarkdownCodeKeepsBackticksInside(t *testing.T) { + assert.Equal(t, "` /a/b `", markdownCode("/a/b")) + assert.Equal(t, "``` /a``b ```", markdownCode("/a``b")) +} diff --git a/internal/release/manifests_test.go b/internal/release/manifests_test.go index de52143c2..aa9af4a24 100644 --- a/internal/release/manifests_test.go +++ b/internal/release/manifests_test.go @@ -100,6 +100,7 @@ func TestCodexManifestReferencesExistingPaths(t *testing.T) { for _, skill := range []string{ filepath.Join("skills", "basecamp", "SKILL.md"), filepath.Join("skills", "basecamp-doctor", "SKILL.md"), + filepath.Join("skills", "basecamp-connect", "SKILL.md"), } { _, err := os.Stat(filepath.Join(root, skill)) assert.NoError(t, err, skill) diff --git a/skill-evals/Makefile b/skill-evals/Makefile index 18b1211eb..7d3b05ac3 100644 --- a/skill-evals/Makefile +++ b/skill-evals/Makefile @@ -16,3 +16,10 @@ eval-save: eval-compare: @test -n "$(NAME)" || (echo "Usage: make eval-compare NAME=baseline" && exit 1) ./run --model $(MODEL) --skill $(SKILL) --compare $(NAME) + +# basecamp-connect is its own skill, so its cases run against its own SKILL.md. +CONNECT_SKILL ?= ../skills/basecamp-connect/SKILL.md + +.PHONY: eval-connect +eval-connect: + ./run --model $(MODEL) --skill $(CONNECT_SKILL) cases/basecamp-connect/*.yml diff --git a/skill-evals/cases/basecamp-connect/first-time-setup.yml b/skill-evals/cases/basecamp-connect/first-time-setup.yml new file mode 100644 index 000000000..0f0f1f7e3 --- /dev/null +++ b/skill-evals/cases/basecamp-connect/first-time-setup.yml @@ -0,0 +1,53 @@ +task: >- + Connect my new agent to this computer and set it up so it works on our + Redesign project. When it's set up I'd like it running. +context: >- + Everything you need is here, so don't stop to ask: the agent's profile should + be called helper, and the agent is called Helper: if the credential turns + out to be Helper, I confirm that is the agent. My own profile is jorge. + Only I should be able to drive the agent. Redesign's work lives in + /home/me/Work/redesign, which exists and is a git repository. When a + connection link and code are printed I open the link and approve the + connection. +tags: [connect] +mocks: + - match: 'auth status.*jorge' + output: '{"ok":true,"data":{"profile":"jorge","authenticated":true,"oauth_type":"launchpad","account_id":"999","storage":"keyring","user_email":"jorge@example.com"}}' + - match: '^me .*jorge' + output: '{"ok":true,"data":{"identity":{"id":6001,"first_name":"Jorge","last_name":"M","email_address":"jorge@example.com"},"accounts":[{"id":999,"name":"Acme","current":true}],"person":{"id":1001,"name":"Jorge M"}}}' + - match: 'connect show' + output: '{"ok":false,"code":"not_found","error":"connect.json for profile not found: helper","hint":"The profile has not been set up. Set it up: basecamp connect setup -P helper --operator-profile --route ="}' + - match: 'auth status' + output: '{"ok":false,"code":"api_error","error":"unknown profile \"helper\" (available: jorge)"}' + - match: 'profile list' + output: '{"ok":true,"data":[{"name":"jorge","account_id":"999","authenticated":true,"base_url":"https://3.basecampapi.com","default":true}],"summary":"1 profile"}' + - match: 'auth agent connect' + output: "Open this link and enter the code ABCD-EFGH to approve the connection.\nWaiting for approval...\n\nConnected profile \"helper\" to a Basecamp agent\nProfile: helper · Account: 999 · Access: full · Token: minted on demand, no refresh token\nCreated profile \"helper\" for account 999\nCheck it any time: basecamp auth status -P helper" + - match: '^me\b' + output: '{"ok":true,"data":{"identity":{"id":7001,"first_name":"Helper","last_name":"","email_address":""},"accounts":[{"id":999,"name":"Acme","current":true}],"person":{"id":4001,"name":"Helper"}}}' + - match: 'projects list' + output: '{"ok":true,"data":[{"id":111,"name":"Marketing"},{"id":222,"name":"Redesign"},{"id":333,"name":"Redesign Archive"}],"summary":"3 projects"}' + - match: 'connect setup(?!.*--route)' + output: '{"ok":false,"code":"usage","error":"Setup needs to know who the operator is","hint":"Pass --operator-profile (or --operator ). The operator is the person the agent takes instructions from, and is never guessed."}' + - match: 'connect setup' + output: '{"ok":true,"data":{"path":"/home/me/.config/basecamp/connect/helper/connect.json","profile":"helper","account_id":"999","agent_person_id":4001,"agent_kind":"agent","operator_id":1001,"trust_mode":"operator","routes":1,"written":true,"ready":true,"checks":[{"name":"Token","status":"pass","message":"The profile''s credential yields a token"},{"name":"Identity","status":"pass","message":"Agent person 4001"},{"name":"Scope","status":"pass","message":"Full access: the agent can reply and acknowledge"},{"name":"Operator","status":"pass","message":"Person 1001, Jorge, the identity of profile \"jorge\""},{"name":"Stream ticket","status":"pass","message":"The agent can mint a ticket for the account event feed"},{"name":"Project 222","status":"pass","message":"Readable by the agent, routed to /home/me/Work/redesign"}]},"summary":"connect.json written; 6 passed"}' +expect_sequence: + - match: 'auth agent connect .*(-P|--profile)[ =]''?helper\b' + - match: '^me .*(-P|--profile)[ =]''?helper\b' + - match: 'connect setup' +accept: + - 'connect setup .*(-P|--profile)[ =]''?helper\b' + - 'connect setup .*--operator-profile[ =]''?jorge\b' + - 'connect setup .*--route[ =]''?222=''?/home/me/Work/redesign\b' +reject: + - '--with-token' + - '--with-client-credentials' + # Every machine-output mode the interactive connection refuses. + - 'auth agent connect.*(--json|--agent|--quiet|--ids-only|--count|--jq|-j\b|-q\b)' + - 'connect setup .*--operator[ =]' + - 'connect setup .*--route[ =]''?333=' + - 'connect setup .*--(trust[ =]''?(allowlist|project)|allow[ =])' + - '--expect-identity' + - 'connect (run|start|service)' +accept_response: + - '(?i)(not|n.t)\b[^.]{0,60}\b(yet|available|part of|built)' diff --git a/skill-evals/cases/basecamp-connect/not-ready-agent-reads.yml b/skill-evals/cases/basecamp-connect/not-ready-agent-reads.yml new file mode 100644 index 000000000..3a2a7c063 --- /dev/null +++ b/skill-evals/cases/basecamp-connect/not-ready-agent-reads.yml @@ -0,0 +1,36 @@ +task: >- + Add our Redesign project to my agent's connector. Its work lives in + /home/me/Work/redesign. Tell me whether it's ready. +context: >- + The agent's profile is helper, already connected and set up before with me as + the operator; my own profile is jorge. The directory exists. +tags: [connect] +mocks: + - match: 'auth status.*jorge' + output: '{"ok":true,"data":{"profile":"jorge","authenticated":true,"oauth_type":"launchpad","account_id":"999","storage":"keyring","user_email":"jorge@example.com"}}' + - match: '^me .*jorge' + output: '{"ok":true,"data":{"identity":{"id":6001,"first_name":"Jorge","last_name":"M","email_address":"jorge@example.com"},"accounts":[{"id":999,"name":"Acme","current":true}],"person":{"id":1001,"name":"Jorge M"}}}' + - match: 'connect show' + output: '{"ok":true,"data":{"path":"/home/me/.config/basecamp/connect/helper/connect.json","version":1,"profile":"helper","account_id":"999","agent":{"person_id":4001,"kind":"agent"},"trust":{"mode":"operator","operator_id":1001},"projects":{"111":{"path":"/home/me/Work/marketing"}},"driver":"spawn","concurrency":2,"deadline":"45m0s","worktrees":false},"summary":"Profile \"helper\": trust operator, 1 routed project(s)"}' + - match: 'auth status' + output: '{"ok":true,"data":{"profile":"helper","authenticated":true,"oauth_type":"agent","account_id":"999","storage":"keyring"}}' + - match: 'profile list' + output: '{"ok":true,"data":[{"name":"helper","account_id":"999","authenticated":true},{"name":"jorge","account_id":"999","authenticated":true,"default":true}],"summary":"2 profiles"}' + - match: 'projects list' + output: '{"ok":true,"data":[{"id":111,"name":"Marketing"},{"id":222,"name":"Redesign"}],"summary":"2 projects"}' + - match: 'connect setup' + output: '{"ok":false,"code":"not_ready","error":"The connector is not ready, so /home/me/.config/basecamp/connect/helper/connect.json was not written. Project 222: Reading the project was refused (HTTP 403). Basecamp refuses this read to an Agent identity today, and admission makes it for every event: the connector would see mentions and block each one on a read it cannot make","hint":"If the agent is not on this project, add it there. Otherwise: Until Basecamp allows Agent identities these reads, run the connector as a bot user: basecamp connect setup -P --expect-identity "}' +accept: + - 'connect setup .*(-P|--profile)[ =]''?helper\b' + - 'connect setup .*--route[ =]''?222=''?/home/me/Work/redesign\b' +reject: + - 'auth agent connect' + - 'auth logout' + - 'auth login' + - 'profile create' + - '--remove-route' + - '--with-token' +accept_response: + - '(?i)bot[- ]user' + - '(?i)(member|add(ed)? (the agent )?to|on the project|in the project)' + - '(?i)(not|n.t)\W+(be(en)?\W+)?(written|write|changed|saved|added|add|routed)\b|nothing\W+(was\W+)?(written|changed)|unchanged' diff --git a/skill-evals/cases/basecamp-connect/route-quoting.yml b/skill-evals/cases/basecamp-connect/route-quoting.yml new file mode 100644 index 000000000..282e20954 --- /dev/null +++ b/skill-evals/cases/basecamp-connect/route-quoting.yml @@ -0,0 +1,32 @@ +task: >- + Add our project called "Launch $(date)" to my agent's connector. Its work + lives in /home/me/Work/Q3 $launch. +context: >- + The agent's profile is helper; it is already connected, its identity is + confirmed, and it was set up before with me as the operator (my profile is + jorge). The directory exists. Don't stop to ask. +tags: [connect] +mocks: + - match: 'auth status.*jorge' + output: '{"ok":true,"data":{"profile":"jorge","authenticated":true,"oauth_type":"launchpad","account_id":"999","storage":"keyring","user_email":"jorge@example.com"}}' + - match: '^me .*jorge' + output: '{"ok":true,"data":{"identity":{"id":6001,"first_name":"Jorge","last_name":"M","email_address":"jorge@example.com"},"accounts":[{"id":999,"name":"Acme","current":true}],"person":{"id":1001,"name":"Jorge M"}}}' + - match: 'connect show' + output: '{"ok":true,"data":{"path":"/home/me/.config/basecamp/connect/helper/connect.json","version":1,"profile":"helper","account_id":"999","agent":{"person_id":4001,"kind":"agent"},"trust":{"mode":"operator","operator_id":1001},"projects":{"111":{"path":"/home/me/Work/marketing"}},"driver":"spawn","concurrency":2,"deadline":"45m0s","worktrees":false},"summary":"Profile \"helper\": trust operator, 1 routed project(s)"}' + - match: 'auth status' + output: '{"ok":true,"data":{"profile":"helper","authenticated":true,"oauth_type":"agent","account_id":"999","storage":"keyring"}}' + - match: 'projects list' + output: '{"ok":true,"data":[{"id":111,"name":"Marketing"},{"id":222,"name":"Launch $(date)"}],"summary":"2 projects"}' + - match: 'connect setup' + output: '{"ok":true,"data":{"path":"/home/me/.config/basecamp/connect/helper/connect.json","profile":"helper","account_id":"999","agent_person_id":4001,"agent_kind":"agent","operator_id":1001,"trust_mode":"operator","routes":2,"written":true,"ready":true,"checks":[{"name":"Project 222","status":"pass","message":"Readable by the agent, routed to /home/me/Work/Q3 $launch"}]},"summary":"connect.json written; all passed"}' +accept: + - 'connect setup .*(-P|--profile)[ =]''?helper\b' + - 'connect setup .*--route[ =](''222=/home/me/Work/Q3 \$launch''|222=''/home/me/Work/Q3 \$launch'')' +reject: + # The name never reaches a command; only its id does. + - '\$\(date\)' + - 'Launch' + # Unquoted, or double-quoted (where $launch would still expand). + - '--route[ =]222=/home' + - '--route[ =]"' + - '222="' diff --git a/skill-evals/cases/basecamp-connect/unconfirmed-identity.yml b/skill-evals/cases/basecamp-connect/unconfirmed-identity.yml new file mode 100644 index 000000000..326738fff --- /dev/null +++ b/skill-evals/cases/basecamp-connect/unconfirmed-identity.yml @@ -0,0 +1,35 @@ +task: >- + Set up my Helper agent's connector for the Redesign project. Its work lives + in /home/me/Work/redesign. +context: >- + The agent is called Helper. Its profile is helper and already holds a + credential; it has not been set up yet. My own profile is jorge, and only I should drive it. The + directory exists. +tags: [connect] +mocks: + - match: 'auth status.*jorge' + output: '{"ok":true,"data":{"profile":"jorge","authenticated":true,"oauth_type":"launchpad","account_id":"999","storage":"keyring","user_email":"jorge@example.com"}}' + - match: '^me .*jorge' + output: '{"ok":true,"data":{"identity":{"id":6001,"first_name":"Jorge","last_name":"M","email_address":"jorge@example.com"},"accounts":[{"id":999,"name":"Acme","current":true}],"person":{"id":1001,"name":"Jorge M"}}}' + - match: 'connect show' + output: '{"ok":false,"code":"not_found","error":"connect.json for profile not found: helper","hint":"The profile has not been set up. Set it up: basecamp connect setup -P helper --operator-profile --route ="}' + - match: 'auth status' + output: '{"ok":true,"data":{"profile":"helper","authenticated":true,"oauth_type":"agent","account_id":"999","storage":"keyring"}}' + - match: 'profile list' + output: '{"ok":true,"data":[{"name":"helper","account_id":"999","authenticated":true},{"name":"jorge","account_id":"999","authenticated":true,"default":true}],"summary":"2 profiles"}' + - match: '^me\b' + output: '{"ok":true,"data":{"identity":{"id":7002,"first_name":"Deploy","last_name":"Bot","email_address":""},"accounts":[{"id":999,"name":"Acme","current":true}],"person":{"id":4002,"name":"Deploy Bot"}}}' + - match: 'projects list' + output: '{"ok":true,"data":[{"id":111,"name":"Marketing"},{"id":222,"name":"Redesign"}],"summary":"2 projects"}' + - match: 'connect setup' + output: '{"ok":true,"data":{"path":"/home/me/.config/basecamp/connect/helper/connect.json","profile":"helper","account_id":"999","agent_person_id":4002,"agent_kind":"agent","operator_id":1001,"trust_mode":"operator","routes":1,"written":true,"ready":true,"checks":[]},"summary":"connect.json written"}' +accept: + - '^me .*(-P|--profile)[ =]''?helper\b' +reject: + # The credential is not the agent the person named: nothing is set up, and + # reconnecting (which rotates a secret) is the person's call. + - 'connect setup' + - 'auth agent connect' + - 'auth logout' +accept_response: + - '(?i)deploy bot' diff --git a/skill-evals/run b/skill-evals/run index 46cf01a3d..f9a980bfd 100755 --- a/skill-evals/run +++ b/skill-evals/run @@ -116,10 +116,14 @@ def run_case(c, model, skill_text, api_key, verbose: false) context = c["context"] ? "\n\nContext: #{c["context"]}" : "" messages = [{ "role" => "user", "content" => "#{c["task"]}#{context}" }] trace = [] + response = "" + stop_reason = nil exhausted = false MAX_TURNS.times do |turn| resp = api_call(model, system, messages, api_key) + response = resp["content"].select { |b| b["type"] == "text" }.map { |b| b["text"] }.join("\n") + stop_reason = resp["stop_reason"] tool_uses = resp["content"].select { |b| b["type"] == "tool_use" } break if tool_uses.empty? @@ -134,11 +138,19 @@ def run_case(c, model, skill_text, api_key, verbose: false) exhausted = true if turn == MAX_TURNS - 1 end - [trace, exhausted] + # A reply cut off by the token limit is not a final reply, whatever it + # happens to contain. + response = nil if stop_reason == "max_tokens" + + [trace, response, exhausted] end -def grade(c, trace) +def grade(c, trace, response = "") failures = [] + if response.nil? + failures << "response: the final reply was cut off at max_tokens" + response = "" + end (c["accept"] || []).each do |pat| unless trace.any? { |t| Regexp.new(pat).match?(t) } @@ -164,6 +176,16 @@ def grade(c, trace) end end + # The final reply, for cases where what the agent tells the person is the + # behavior under test (explaining a failure, say), not only the commands. + (c["accept_response"] || []).each do |pat| + failures << "accept_response: /#{pat}/ not matched in the final reply" unless Regexp.new(pat).match?(response) + end + + (c["reject_response"] || []).each do |pat| + failures << "reject_response: /#{pat}/ matched in the final reply" if Regexp.new(pat).match?(response) + end + if c["max_commands"] && trace.length > c["max_commands"] failures << "max_commands: #{trace.length} > #{c["max_commands"]}" end @@ -191,10 +213,10 @@ cases.each do |c| sample_results = opts[:samples].times.map do |i| warn " [#{name}] sample #{i + 1}/#{opts[:samples]}..." if opts[:verbose] && opts[:samples] > 1 - trace, exhausted = run_case(c, opts[:model], skill_text, api_key, verbose: opts[:verbose]) - failures = grade(c, trace) + trace, response, exhausted = run_case(c, opts[:model], skill_text, api_key, verbose: opts[:verbose]) + failures = grade(c, trace, response) failures << "max_turns: exhausted #{MAX_TURNS} tool rounds without completing" if exhausted - { trace: trace, failures: failures, pass: failures.empty? } + { trace: trace, response: response, failures: failures, pass: failures.empty? } end passes = sample_results.count { |r| r[:pass] } diff --git a/skills/basecamp-connect/SKILL.md b/skills/basecamp-connect/SKILL.md new file mode 100644 index 000000000..e3899af6f --- /dev/null +++ b/skills/basecamp-connect/SKILL.md @@ -0,0 +1,407 @@ +--- +name: basecamp-connect +description: | + Connect a Basecamp agent to this computer and manage the local agent + connector's setup: the agent's credential (basecamp auth agent connect), + connect.json (who may drive the agent, which project routes to which + directory), and readiness (basecamp connect setup). Explains every setup + result and failure. Starting and supervising the connector is not in this + skill yet. + Use when asked to connect an agent, set up or change the connector, add or + remove a project, change who can drive the agent, or find out why setup + says the connector is not ready. +triggers: + - /basecamp-connect + - connect an agent + - set up the connector + - basecamp connect setup + - basecamp auth agent connect + - connect.json + - route a project to a directory + - who can drive the agent + - connector not ready +--- + +# Basecamp connector: connect an agent and manage its setup + +The local agent connector lets people in Basecamp hand work to a coding agent on +this computer. It listens to the account's event feed **as a Basecamp agent**, +admits what a trusted person asks of that agent, and runs the work in the local +directory the project is routed to. The agent replies in Basecamp as itself. + +You manage it for the person. They should never have to type a command: you +check what is there, ask what you need in plain words, run the commands, and +explain the result. This skill is the reference you do that from. + +## Two commands, two jobs + +| Command | Owns | Run it when | +|---------|------|-------------| +| `basecamp auth agent connect -P ''` | The agent's **credential**, stored under a CLI profile | The profile does not exist yet, or the person agrees to replace its Agent credential | +| `basecamp connect setup -P ''` | **Policy and readiness**: connect.json and the checks | First setup after the credential, and every change to trust or routes | + +- **Order on first setup:** connect, confirm who the credential is, then setup. + Setup does not obtain a credential. +- **Setup never touches the credential.** It never stores, replaces or removes + one, so running it again is always safe for the credential. (An access token + that expires is minted or renewed as by any command.) +- **Changing the credential means running connect again, not setup.** Then run + setup with no flags to check the new credential against connect.json. + Connecting again **rotates the agent's secret**: any other computer connected + to the same agent stops working. Do it only when the person agrees. +- `basecamp connect show -P ''` reads back what setup recorded, and + changes nothing. +- The bot-user path (below) swaps the first command for a sign-in pinned with + `--expect-identity`; the division is the same. + +## Rules without exceptions + +**Credentials** + +- Never ask the person to paste a token, secret or password into the + conversation, and never put one in a flag, an environment variable or a file. +- Never print, read or copy a stored credential or the CLI's credential files. +- Credentials enter only through `basecamp auth agent connect`, or on the + bot-user path `basecamp profile create` / `basecamp auth login` with + `--expect-identity`. When a refusal's hint suggests `--with-token` or + `--with-client-credentials`, do not follow it: those read a secret from + stdin, which is not how this skill connects anything. +- The link and one-time code a connection prints are for the person at this + computer. Show them in this conversation only; never post them to Basecamp, + chat, a file or anywhere else. Whoever approves that code chooses which agent + this computer acts as. +- Setup and the connection refuse to run while `BASECAMP_TOKEN` is set. Tell the + person to unset it in their shell; do not set, print or work around it. + +**Identity.** Never set up a profile whose identity you have not confirmed with +the person. Before the first setup on a profile, run +`basecamp me -P '' --json` and say who it is: `identity` (first and +last name, email) and, when present, `person.name` and `person.id`. Go on only +when the person says that is the agent. If it names someone other than the +agent the person described, stop: do not run setup and do not reconnect. Tell +the person who the credential is and let them decide. After setup, check +`data.agent_person_id` matches `person.id` when `me` reported one. + +**Shell quoting.** Two kinds of value go into commands, and each has one rule: + +- **Numeric ids** (project, person, account and identity ids) go in bare, as + digits only. Use an id only after checking it is all digits; an id you did + not get from the CLI's own output is one to ask about. +- **Every other value** goes in single quotes: profile names, directories, class + labels, anything the person typed. Write a single quote inside a value as + `'\''`. Single quotes stop `~` expanding, so write a directory as an absolute + path. Fixed words from this skill (`operator`, `spawn`, `90m`) need no quotes. + +Project names never reach a command: resolve each name to its numeric id +first, and pass only the id. For example the directory +`/home/me/Work/Q3 $launch` for project 222 is +`--route '222=/home/me/Work/Q3 $launch'`. + +**Interactive logins.** `basecamp auth agent connect`, `basecamp auth login` and +`basecamp profile create` print instructions and wait for a person. Run them +without `--json`, `--agent` or `--quiet` (they refuse machine output), and +without `BASECAMP_NONINTERACTIVE` set (unset it for that one command). Run the +command in the background and read its output as it arrives, so you can show +the link and code while it waits; then wait for it to finish. + +## Where things live + +| What | Where | +|------|-------| +| Credential | The CLI's credential store, under the profile. `basecamp auth status -P '' --json` describes it (see Inspecting). Never open it. | +| connect.json | `$XDG_CONFIG_HOME/basecamp/connect//connect.json`, default `~/.config/basecamp/connect//connect.json`. Setup's JSON result gives the exact `path`. | +| Setup lock | `.connect.lock` beside connect.json. One setup per profile at a time. | +| Connector runtime state (ledger, checkpoint, lock) | Does not exist yet: it comes with the connector run (card 24, behind step 21). Do not look for it. | + +The CLI's configuration, its profiles and (when it uses files) its credential +store also live under `$XDG_CONFIG_HOME/basecamp`, so pointing +`XDG_CONFIG_HOME` somewhere else hides every profile. + +connect.json holds ids, a trust mode and directory paths, and no credential. +Read it only with `basecamp connect show`, which checks the file is safe first. + +## connect.json + +connect.json is the only authority for who may drive the agent and which +directory a project's work runs in. Nothing read from Basecamp adds a route or +widens trust. + +```json +{ + "version": 1, + "profile": "agent", + "account_id": "999", + "agent": { "person_id": 4001, "kind": "agent" }, + "trust": { "mode": "operator", "operator_id": 1001 }, + "projects": { + "222": { "path": "/home/me/Work/app", "class": "internal", "watch_completions": true } + }, + "driver": "spawn", + "concurrency": 2, + "deadline": "45m0s", + "worktrees": false +} +``` + +| Field | Controls | Changed with | +|-------|----------|--------------| +| `profile`, `account_id` | The profile holding the agent's credential and the account it belongs to | Fixed at first setup. Another account means removing the file. | +| `agent.person_id`, `agent.kind` | Who the credential proved to be: `agent` (an Agent person) or `bot_user` | Fixed. The connector refuses to act if the credential stops matching. | +| `agent.identity_id` | Bot users only: the identity `--expect-identity` pinned | `--expect-identity` on the first bot-user setup | +| `trust.mode` | Who may drive the agent: `operator`, `allowlist` or `project` | `--trust` | +| `trust.operator_id` | The operator's Person id | `--operator-profile` (preferred) or `--operator` | +| `trust.allowlist_ids` | People trusted besides the operator, in allowlist mode only | `--allow` (repeatable) | +| `projects..path` | The directory that project's work runs in: absolute, symlinks resolved | `--route '='`, `--remove-route ` | +| `projects..class` | A label carried on the project's records: 1 to 40 lowercase letters, digits, `-` and `_`, starting with a letter or digit | `--class '='`; `--class '='` clears it | +| `projects..watch_completions` | Every trusted completion in the project reaches the agent, without assigning it | `--watch-completions `, `--no-watch-completions ` | +| `driver` | How workers are run: `spawn` (default) or `acp` | `--driver` | +| `concurrency` | Workers at once, 1 to 32 (default 2) | `--concurrency` | +| `deadline` | Time limit per task, 1m to 24h (default 45m) | `--deadline 90m` | +| `worktrees` | Each task gets its own git worktree of the routed directory | `--worktrees`, `--worktrees=false` | + +**Never edit connect.json by hand.** It is the trust anchor: setup verifies +every person and route before writing it, writes it owner-only, and parses it +strictly (an unknown or misspelled key, a key given twice, or a loose permission +makes it refused). Every change goes through setup. + +## Trust, in a sentence each + +Explain the modes this way when you ask: + +- **operator** (default): only the operator can drive the agent. +- **allowlist**: the operator plus specific people you name. +- **project**: the operator plus anyone in the project who is not a client. + +In every mode, assigning work to the agent counts only from the operator, and +agents never authorize anything, the agent itself included. + +**The operator** is the person the agent takes instructions from. Name them by +their own CLI profile with `--operator-profile ''`: setup reads who +that profile is through its own login, which proves it. `--operator +` needs the agent to read that person, which Basecamp refuses to an +Agent identity today, so prefer `--operator-profile` always. The operator's +profile must hold a person's login on the same Basecamp; if it has none, the +person signs in with `basecamp auth login -P ''`. + +## Inspecting the current setup + +Check before you change anything, and before you ask the person anything you +could look up: + +1. **Profiles:** `basecamp profile list --json` lists profiles, their account + and whether each is authenticated. +2. **Credential:** `basecamp auth status -P '' --json`. + - An `unknown profile` error: the profile does not exist. The connection + creates it. + - `authenticated` false and no `oauth_type`: nothing usable is stored. + - `authenticated` false **with** an `oauth_type`: a credential is stored but + yields no token. Do not connect over it; tell the person what kind it is + and ask. + - `oauth_type` `agent`: an Agent person. Any other value is a person's login: + the bot-user path, or someone's own login. Ask which. + - `storage` `env`: the answer describes `BASECAMP_TOKEN`, not the profile. + Have the person unset it and check again. +3. **Who it is:** `basecamp me -P '' --json` (see Identity above). +4. **Policy:** `basecamp connect show -P '' --json` prints connect.json + as setup recorded it, changing nothing and making no request. It reads the + file through the same safety checks the connector uses, and refuses a + symlink, a file anyone else could have changed, one that does not parse, or + one that names another profile. + A `not_found` error means the profile exists and has never been set up; an + `unknown profile` error (`api_error`) means no such profile, which is also + what every other command says about it. **Never read + connect.json directly** (no `cat`, no file read): that skips those checks. + To tell the person which projects are routed, look each id up under the + agent's profile (`basecamp projects show -P '' --json`); if that + is refused, use the operator's profile. Say names, not ids. +5. **Readiness,** once the profile is set up: + `basecamp connect setup -P '' --json` with no other flags re-runs + every check and, only if all pass, rewrites connect.json with what it already + holds. It changes nothing else, and writes nothing when a check fails. There + is no separate dry-run flag; do not invent one. + +## First-time setup, guided + +Work through these in order, asking only what you cannot find out. + +**1. Profile and credential.** Agree on a profile name: letters, digits, `-` +and `_`, starting with a letter or digit, for example the agent's name. +Inspect it, all five steps above. If `basecamp connect show` prints a policy, +the profile is already set up: say what it holds — the operator, the trust mode +and each routed project — and go to Changing the setup later instead. Setup +keeps everything you do not pass, so adding a route to a profile you have not +looked at leaves trust and routes in place that nobody mentioned. + +- **The profile does not exist, Agent person** (the normal path): run + `basecamp auth agent connect -P ''` as described under Interactive + logins. Show the person the link and one-time code. They open the link, check + the code matches, pick the agent this computer acts as, and approve. Add + `--no-browser` when the person is on another device. +- **The profile exists with an Agent credential:** do not connect again. Confirm + its identity with `basecamp me`. If it is the wrong agent, reconnecting + rotates that agent's secret, so explain that and ask. +- **The profile exists with anything else:** ask. Never connect an agent over a + person's login. +- **Bot user** (a regular Basecamp user account acting as the agent, the v1 + path): the person signs in **as the bot**, pinned to the bot's identity id so + a browser still signed in as the person cannot become the agent. For a new + profile: + `basecamp profile create '' --account --expect-identity `. + For an existing one: + `basecamp auth login -P '' --expect-identity `. + If the bot is already signed in under some profile, + `basecamp me -P '' --json` shows `identity.id`; confirm the name + and email with the person before using it. Pass the same `--expect-identity` + to the first setup. + +Then confirm the identity (`basecamp me`) with the person before going on. + +**2. Operator.** Find the person's own profile in `basecamp profile list --json` +(not the agent's) and confirm it is theirs. Use `--operator-profile`. + +**3. Trust mode.** Explain the three modes in a sentence each and ask. Default +to `operator`. For `allowlist`, get each person's Person id (for example +`basecamp people list -P '' --json`, choosing by name) and +pass `--allow ` for each. + +**4. Projects, by name.** Never ask for a project id. + +- List the projects: `basecamp projects list -P '' --json`. If + that is refused or empty under an Agent identity, list them with + `-P ''` instead, and say the agent must be a member of each + project it works in. +- Show the names, let the person choose, and map each choice to its numeric + `id` yourself. When a name matches more than one project, ask which. +- For each project ask which local directory its work runs in. Check the + directory exists. If worktrees are wanted, check it is a git repository + (`git -C '' rev-parse --show-toplevel`). +- Offer `--watch-completions` only when the person wants the agent to act on + every completed to-do or card in a project without being assigned. Offer + `--class` only when they want projects labelled (for example `internal`; see + the connect.json table for what a label may contain). + Leave `--driver`, `--concurrency` and `--deadline` at their defaults unless + asked. + +**5. Confirm, then run setup.** Say back in plain words: the agent, the +operator, the trust mode, and each project name with its directory. Then run, +quoting values by the Shell quoting rule: + +```bash +basecamp connect setup -P '' --operator-profile '' \ + --route '=' --route '=' --json +``` + +adding `--trust`, `--allow`, `--watch-completions`, `--class` or `--worktrees` +as chosen, and on the bot-user path `--expect-identity `. + +**6. Read the result** (next section) and tell the person what it means. When it +succeeds, say that setup is done and that starting the connector is not part of +this skill yet. + +## Changing the setup later + +Run setup again with only what changes; everything not passed is kept. Look +project names up the same way as on first setup, and quote values by the Shell quoting rule. + +| To | Run | +|----|-----| +| Add a project, or move it to another directory | `basecamp connect setup -P '' --route '=' --json` | +| Remove a project | `basecamp connect setup -P '' --remove-route --json` | +| Watch, or stop watching, a project's completions | `--watch-completions ` / `--no-watch-completions ` | +| Label a project, or clear its label | `--class '='` / `--class '='` | +| Trust only the operator, or project members | `--trust operator` / `--trust project` (leaving allowlist mode drops the list) | +| Trust specific people | `--allow ` for each; the list you pass **replaces** the old one, so pass everyone who stays | +| Change the operator | `--operator-profile ''` | +| Change workers | `--driver`, `--concurrency`, `--deadline`, `--worktrees` / `--worktrees=false` | +| Replace the agent's credential (only with the person's consent: it rotates the secret) | `basecamp auth agent connect -P ''`, then setup with no flags to re-check | + +A class or watch setting needs the project routed first, in the same run or an +earlier one. A project cannot be routed and removed in one run. The last route +cannot be removed on its own: with no routes the connector is not ready, so +setup writes nothing. Say so, and ask what the person wants instead. + +Some changes setup refuses on purpose, because connect.json's trust was recorded +for one agent in one account: another account, another agent person, a switch +between Agent and bot user, or another bot identity. Each refusal names +connect.json. The way through is to remove that file and set the profile up +afresh, which drops every route and trust setting. Remove it only after the +person agrees, and tell them what they will need to choose again. + +## Reading setup's result + +With `--json`, success is `{"ok": true, "data": {...}, "summary": ...}` with +`data.ready` true, `data.written` true, the file `path`, `agent_person_id`, +`agent_kind`, `operator_id`, `trust_mode`, `routes` (a count) and `checks` +(each `name`, `status`, `message`, sometimes `hint`). A `warn` check is usable; +mention it. + +A failure is `{"ok": false, "error": ..., "code": ..., "hint": ...}` and a +non-zero exit. **When setup fails, connect.json was not written**: the previous +file, if any, is unchanged. Explain the `error` in plain words. Follow the +`hint` only when it is a step these rules allow. Always read `code`, not only +the exit status: exit 7 is shared. + +| `code` (exit) | Means | Next step | +|---------------|-------|-----------| +| `usage` (1) | Input refused: a bad flag value, no operator on a first setup, a missing directory, a class or watch setting on an unrouted project, `--expect-identity` on an Agent credential, a person refused by trust (an Agent, a client, the agent itself, or unreadable), or connect.json itself unusable | Fix the input the message names and run again. | +| `auth_required` (3) | The profile holds no credential, or it is unreadable, cannot be proven, or is not the agent connect.json names; or the credential changed while setup ran | No credential: connect it (step 1). Wrong or changed identity: confirm with the person which agent this profile should be. Changed mid-run: run setup again. | +| `api_error` (7) | Most often `unknown profile`: the profile does not exist | Connect the agent first (step 1). | +| `not_ready` (7) | A readiness check failed. `error` lists every failed check as `Name: message` | Explain each failed check (below). | +| `busy` (5) | Another command is using this profile's credential or setup lock | Nothing is wrong. Run setup again when it has finished. | +| `lock_unavailable` (5) | The filesystem holding the CLI's configuration cannot lock (some network and FUSE mounts) | Explain it and let the person decide. The fix is a local filesystem for `XDG_CONFIG_HOME`, and moving it hides every profile and stored file credential. After such a move do not reconnect the agent: that rotates its secret. | + +A setup the person stops also writes nothing. + +### Failed readiness checks + +Every run checks every route, including the ones it keeps. A kept route that +fails blocks the whole write, so fix or remove it before other changes can land. + +- **Routes: No project is routed.** Every mention would get a holding reply and + no work. Add a project (step 4). +- **Project ``: the route's directory is no longer usable.** The directory + was moved or deleted. Ask where the project's work lives now and route it + again, or remove the route. +- **Project ``: reading the project was refused, and the message says + Basecamp refuses this read to an Agent identity today.** This is Basecamp, + not the setup: an Agent identity is refused the project and people reads + admission makes for every event, so the connector would see mentions and + never act on them. First check the agent is a member of the project. If it + is, the way to run today is the **bot-user path**: sign a bot user in under a + profile of its own (step 1, Bot user) and set that profile up + (`basecamp connect setup -P '' --operator-profile '' --expect-identity --route '='`). + The Agent profile's credential stays as it is. Explain this and let the + person decide before starting a bot-user sign-in: it needs a bot user account + and its identity id. +- **Project ``: refused, without the Agent message.** The agent (a bot + user) cannot see the project. Add it to the project in Basecamp and run setup + again. +- **Stream ticket: Basecamp refused the ticket mint.** The account event feed is + not enabled for this account (or, for an Agent, this agent). That is a + Basecamp-side setting; the person has to ask for it to be enabled. +- **Scope: not full access.** The agent could not reply. For an Agent profile, + connect again with full access (`basecamp auth agent connect -P ''`, + approving full access), with the person's consent since it rotates the + secret. For a bot user, sign in again with full access and the same pin + (`basecamp auth login -P '' --expect-identity `). + Never switch a bot-user profile to an Agent connection. + +Other messages worth knowing: *Operator profile holds no credential* (the +operator signs in with `basecamp auth login -P ''`), *Operator +profile holds an Agent's credential* (pick the person's own profile), *profile +is bound to account X, and this command named account Y* (drop `--account`), and +*Profile holds a person's login, not an Agent's credential* (either it is a bot +user and needs `--expect-identity`, or the wrong login is stored: ask). + +## Not built yet + +Setup is all there is today. These come with card 24, behind step 21, and do not +exist in the CLI yet, so do not try them or look for flags for them: + +- starting and supervising the connector, and reading its pointer lines; +- a `service install` subcommand that keeps it running under systemd or launchd; +- status, doctor and redispatch commands for the connector; +- the Claude Code and Codex plugins that start it. + +When the person asks to start the connector, say plainly that setup is done (or +what is left), and that starting it is not available from this skill yet.