Skip to content
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ pushing. If you're about to `git push` and haven't run `bin/ci` in this
session, stop and run it first.

**Skill drift**: `make check-skill-drift` runs over `skills/basecamp/SKILL.md`,
`skills/basecamp-doctor/SKILL.md` and `skills/basecamp-connect/SKILL.md`, checking that the commands and flags each one
`skills/basecamp-doctor/SKILL.md`, `skills/basecamp-connect/SKILL.md` and its
`skills/basecamp-connect/subagent.md`, checking that the commands and flags each one
*references* still exist in the `.surface` snapshot. It catches stale references, not
missing coverage, so adding a command breaks none of them.

Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,7 @@ check-skill-drift:
@scripts/check-skill-drift.sh
@scripts/check-skill-drift.sh skills/basecamp-doctor/SKILL.md
@scripts/check-skill-drift.sh skills/basecamp-connect/SKILL.md
@scripts/check-skill-drift.sh skills/basecamp-connect/subagent.md
Comment thread
jeremy marked this conversation as resolved.
Comment thread
jeremy marked this conversation as resolved.

# Verify the skill drift check itself: what it accepts, and what it refuses
.PHONY: test-skill-drift
Expand Down
95 changes: 95 additions & 0 deletions internal/connector/skill_layout_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package connector

import (
"os"
"path/filepath"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// perRequestBudget bounds where the orchestrator's per-request steps may end
// in the skill's body. After a compaction Claude Code re-attaches a skill cut
// to its first 5,000 tokens (code.claude.com/docs/en/skills, "Skill content
// lifecycle"); this skill's Markdown runs near 2.9 characters a token, so
// 14,000 characters stays under the cut with room to spare. A session cut
// short of the handoff can acknowledge a request and never dispatch it.
const perRequestBudget = 14000

func readConnectSkill(t *testing.T, name string) string {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "skills", "basecamp-connect", name))
require.NoError(t, err)
return strings.ReplaceAll(string(raw), "\r\n", "\n")
}

// skillBody drops the frontmatter, which Claude Code doesn't render into the
// conversation.
func skillBody(t *testing.T, skill string) string {
t.Helper()
parts := strings.SplitN(skill, "---\n", 3)
require.Len(t, parts, 3, "the skill opens with frontmatter")
return parts[2]
}

func TestTheConnectSkillKeepsEachRequestsStepsInsideTheCompactionCut(t *testing.T) {
body := skillBody(t, readConnectSkill(t, "SKILL.md"))
dispatch := strings.Index(body, "### c. Dispatch one background subagent")
require.GreaterOrEqual(t, dispatch, 0, "the handoff step is where this test looks for it")
end := strings.Index(body[dispatch:], "\n## ")
require.GreaterOrEqual(t, end, 0, "a section follows the handoff")
end += dispatch

previous := -1
for _, step := range []string{
"### 1. Start it, and watch its output",
"### 2. The request line",
"## For each request: acknowledge, choose the repo, hand off",
"### a. Acknowledge, within seconds",
"### b. Choose the repo",
"### c. Dispatch one background subagent",
} {
at := strings.Index(body, step)
assert.Greater(t, at, previous, "%q comes after the step before it", step)
previous = at
}
assert.Contains(t, body[:dispatch], "~/.config/basecamp-connect/last.json",
"choosing the repo names the file of confirmed mappings it reads")
assert.Contains(t, body[:dispatch], "read `${CLAUDE_SKILL_DIR}/SKILL.md` in full (if that reads literally, the\nbase directory this skill loaded from)",
"a connector change the owner asks for mid-run reads the setup steps past the cut first, from the base directory where the placeholder isn't expanded")
assert.Contains(t, body[:dispatch], "goes out as the agent,\nwith `--profile '<profile>'`",
"the orchestrator's own replies say to post as the agent")
// The refusal and the holding reply are posted before any subagent starts,
// so the orchestrator needs the reply commands, which live in subagent.md.
assert.Contains(t, body[:dispatch], "post as step 5 of `subagent.md` does",
"the orchestrator's own replies point at the reply commands")
assert.Contains(t, body[:dispatch], "after a change it restarts",
"a setup change the owner asks for mid-run says the connector restarts to take it")
assert.LessOrEqual(t, end, perRequestBudget,
"the per-request steps end at character %d of the skill's body; keep them within %d, moving setup and reference material after them",
end, perRequestBudget)
}

func TestTheSubagentReadsItsInstructionsFromTheirOwnFile(t *testing.T) {
skill := readConnectSkill(t, "SKILL.md")
subagent := readConnectSkill(t, "subagent.md")

assert.NotContains(t, skill, "## The subagent's instructions", "the subagent's instructions live in subagent.md")
dispatch := strings.Index(skill, "### c. Dispatch one background subagent")
require.GreaterOrEqual(t, dispatch, 0)
handoff := skill[dispatch:]
handoff = handoff[:strings.Index(handoff, "\n## ")]
assert.Contains(t, handoff, "`${CLAUDE_SKILL_DIR}/subagent.md`", "the handoff passes the file's path, which Claude Code expands")
assert.Contains(t, handoff, "read it in full before anything else", "the handoff tells the subagent to read it first")

// An acknowledgement such as "I'm on it" breaks a single-quoted command
// unless its quote is escaped, so both files say how.
acknowledge := skill[strings.Index(skill, "### a. Acknowledge, within seconds"):dispatch]
assert.Contains(t, acknowledge, "`'\\''`", "the acknowledgement step says how to quote a value")
assert.Contains(t, subagent, "`'\\''`", "the subagent's instructions say how to quote a value")
Comment thread
jeremy marked this conversation as resolved.
for _, step := range []string{"**Shell quoting.**", "**1. Acknowledge", "**5. Reply at `reply_to`", "**6. Report back", "**A participant asks, an operator authorizes.**"} {
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
assert.Contains(t, subagent, step)
}
}
1 change: 1 addition & 0 deletions internal/release/manifests_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ func TestCodexManifestReferencesExistingPaths(t *testing.T) {
filepath.Join("skills", "basecamp", "SKILL.md"),
filepath.Join("skills", "basecamp-doctor", "SKILL.md"),
filepath.Join("skills", "basecamp-connect", "SKILL.md"),
filepath.Join("skills", "basecamp-connect", "subagent.md"),
} {
_, err := os.Stat(filepath.Join(root, skill))
assert.NoError(t, err, skill)
Expand Down
6 changes: 4 additions & 2 deletions scripts/check-skill-drift.sh
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,10 @@ flag_exists() {
}

# Strip YAML frontmatter — trigger keywords (e.g. "basecamp project") are
# natural-language match phrases, not CLI command references.
skill_body=$(awk '/^---$/{n++; next} n>=2' "$SKILL")
# natural-language match phrases, not CLI command references. Only a file
# that opens with "---" has any: a supporting file such as
# basecamp-connect/subagent.md has none, and is read whole.
skill_body=$(awk 'NR==1 && /^---$/ {fm=1; next} fm && /^---$/ {fm=0; next} !fm' "$SKILL")
Comment thread
jeremy marked this conversation as resolved.

# --- Phase 1: Command references ---
# Extract "basecamp <subcommand>..." patterns and verify each resolves.
Expand Down
13 changes: 13 additions & 0 deletions scripts/test-skill-drift.sh
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,19 @@ else
not_ok "a command that fails is not also reported once per flag — ${count} DRIFT lines: ${out}"
fi

# --- A supporting file with no frontmatter is read whole ---
# The check used to drop everything before a second "---", which skipped a
# file without frontmatter entirely, or everything up to a later thematic
# break in it.

printf '%s\n' 'Run `basecamp profile set agent`.' '' '---' '' 'Then `basecamp profile list`.' > "${work}/no_frontmatter.md"
assert_fails no_frontmatter 'cannot verify' \
'a command in a file without frontmatter is checked'

printf '%s\n' 'Run `basecamp profile list`.' '' '---' '' 'Then `basecamp profile set agent`.' > "${work}/after_break.md"
assert_fails after_break 'cannot verify' \
'a command after a thematic break in a file without frontmatter is checked'

echo
if [ "$failures" -gt 0 ]; then
echo "${failures} failure(s)"
Expand Down
Loading
Loading