conformance_test.go

package main

import (
	"io/fs"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"gopkg.in/yaml.v3"
)

// transportOwnershipPatterns are the delivery/reconciliation ownership
// concepts a manifest-producing plugin must never carry. A Codefly plugin
// renders deterministic Kubernetes output from normalized inputs; how that
// output is transported, reviewed, or reconciled belongs to a separate
// promotion driver, not here. Each entry is matched case-insensitively.
//
// Patterns target the tokens these concepts actually carry in Go source and
// Kubernetes manifests — import paths, API groups, resource kinds, and
// process invocations — not the plain-English word, which real integrations
// never emit. "flux" appears only as "fluxcd" (the module path and the
// *.toolkit.fluxcd.io API groups); Git integration arrives as the go-git
// library or an exec of the git binary, never as the prose "git clone".
var transportOwnershipPatterns = []*regexp.Regexp{
	regexp.MustCompile(`(?i)gitops`),
	regexp.MustCompile(`(?i)repourl`),
	regexp.MustCompile(`(?i)targetrevision`),
	regexp.MustCompile(`(?i)appproject`),
	regexp.MustCompile(`(?i)argocd`),
	regexp.MustCompile(`(?i)argoproj`),
	regexp.MustCompile(`(?i)fluxcd`),
	regexp.MustCompile(`(?i)kubeconfig`),
	regexp.MustCompile(`(?i)reconcil`),
	regexp.MustCompile(`(?i)go-git`),
	regexp.MustCompile(`(?i)exec\.Command(?:Context)?\([^"]*"git"`),
	regexp.MustCompile(`(?i)\bgit\s+(clone|push|commit|checkout|fetch|remote|tag|init|pull)\b`),
	regexp.MustCompile(`(?i)\bpull\s+request\b`),
	regexp.MustCompile(`(?i)kind:\s*Application\b`),
	regexp.MustCompile(`(?i)kind:\s*AppProject\b`),
	regexp.MustCompile(`(?i)kind:\s*GitRepository\b`),
}

// scanTransportOwnership reports every line in content that matches a
// forbidden pattern, formatted as "<name>:<line>: <trimmed source>".
func scanTransportOwnership(name, content string) []string {
	var hits []string
	for i, line := range strings.Split(content, "\n") {
		for _, re := range transportOwnershipPatterns {
			if re.MatchString(line) {
				hits = append(hits, name+":"+strconv.Itoa(i+1)+": "+strings.TrimSpace(line))
				break
			}
		}
	}
	return hits
}

// isIgnoredSourceDir reports whether a directory is excluded from a module's
// compiled runtime source. This mirrors the Go toolchain's own rule:
// directories named "testdata" and those beginning with "." or "_" are never
// built.
func isIgnoredSourceDir(name string) bool {
	return name == "testdata" || strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_")
}

// runtimeGoFiles returns every compiled runtime Go file under root: all
// non-test .go files, recursively, excluding toolchain-ignored directories.
// Test files are excluded because boundary guards (this file included) name
// the forbidden concepts as data.
func runtimeGoFiles(root string) ([]string, error) {
	var files []string
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			if path != root && isIgnoredSourceDir(d.Name()) {
				return fs.SkipDir
			}
			return nil
		}
		if strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") {
			files = append(files, path)
		}
		return nil
	})
	return files, err
}

// TestRuntimeSourceIsTransportNeutral enforces the manifest-bundle boundary
// on this plugin's runtime source: it must not integrate with Git
// repositories, GitHub/pull requests, Argo CD/Flux reconcilers, kubeconfig,
// or own reconciliation objects (repoURL, targetRevision, Application,
// AppProject). The scan is recursive so a future subpackage cannot escape it.
func TestRuntimeSourceIsTransportNeutral(t *testing.T) {
	files, err := runtimeGoFiles(".")
	if err != nil {
		t.Fatalf("enumerate runtime source: %v", err)
	}
	for _, name := range files {
		src, err := os.ReadFile(name)
		if err != nil {
			t.Fatalf("read %s: %v", name, err)
		}
		if hits := scanTransportOwnership(name, string(src)); hits != nil {
			t.Errorf("runtime source names a transport/reconciler concept:\n  %s", strings.Join(hits, "\n  "))
		}
	}
}

// TestOwnedManifestsAreTransportNeutral enforces the same boundary on the
// plugin-owned deployment templates: they carry only workload/resources,
// never reconciliation control-plane objects (Argo Application/AppProject,
// Flux GitRepository) or repository source bindings (repoURL, targetRevision).
func TestOwnedManifestsAreTransportNeutral(t *testing.T) {
	err := fs.WalkDir(deploymentFS, ".", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		content, err := fs.ReadFile(deploymentFS, path)
		if err != nil {
			return err
		}
		if hits := scanTransportOwnership(path, string(content)); hits != nil {
			t.Errorf("owned manifest names a transport/reconciler concept:\n  %s", strings.Join(hits, "\n  "))
		}
		return nil
	})
	if err != nil {
		t.Fatalf("walk deployment templates: %v", err)
	}
}

// TestScanTransportOwnershipDetectsRealVectors pins the detector to the forms
// transport ownership actually takes in Go source and manifests — library
// imports, API groups, resource kinds, and process invocations — and to a set
// of benign lines that must not trip it.
func TestScanTransportOwnershipDetectsRealVectors(t *testing.T) {
	forbidden := []string{
		`_ "github.com/fluxcd/pkg/apis/meta"`,
		`apiVersion: source.toolkit.fluxcd.io/v1`,
		`kind: GitRepository`,
		`_ "github.com/argoproj/argo-cd/v2/pkg/apis"`,
		`import gogit "github.com/go-git/go-git/v5"`,
		`r, _ := exec.Command("git", "clone", url).Output()`,
		`out, _ := exec.CommandContext(ctx, "git", "push").CombinedOutput()`,
		`kind: Application`,
		`spec: { repoURL: https://example.com, targetRevision: main }`,
		`// reconcile the cluster state`,
	}
	for _, line := range forbidden {
		if hits := scanTransportOwnership("x.go", line); hits == nil {
			t.Errorf("expected forbidden line to be flagged: %q", line)
		}
	}

	benign := []string{
		`import _ "github.com/codefly-dev/core/agents/services"`,
		`s.Wool.Debug("git status is clean")`,
		`kind: Deployment`,
		`kind: Service`,
		`newTag: {{.Image.Tag}}`,
		`// influx of requests handled by the digit parser`,
	}
	for _, line := range benign {
		if hits := scanTransportOwnership("x.go", line); hits != nil {
			t.Errorf("benign line falsely flagged: %q -> %v", line, hits)
		}
	}
}

// TestRuntimeGoFilesRecursesAndFilters proves the enumerator descends into
// subpackages (so no future package escapes the boundary check) while
// excluding test files and toolchain-ignored directories.
func TestRuntimeGoFilesRecursesAndFilters(t *testing.T) {
	root := t.TempDir()
	write := func(rel string) {
		p := filepath.Join(root, rel)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte("package p\n"), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	write("top.go")
	write("sub/pkg/deep.go")
	write("sub/pkg/deep_test.go")
	write("testdata/fixture.go")
	write(".hidden/skip.go")

	files, err := runtimeGoFiles(root)
	if err != nil {
		t.Fatalf("runtimeGoFiles: %v", err)
	}
	got := map[string]bool{}
	for _, f := range files {
		rel, _ := filepath.Rel(root, f)
		got[filepath.ToSlash(rel)] = true
	}

	want := []string{"top.go", "sub/pkg/deep.go"}
	for _, w := range want {
		if !got[w] {
			t.Errorf("expected %s to be scanned, got %v", w, files)
		}
	}
	excluded := []string{"sub/pkg/deep_test.go", "testdata/fixture.go", ".hidden/skip.go"}
	for _, e := range excluded {
		if got[e] {
			t.Errorf("%s should have been excluded from the scan", e)
		}
	}
}

// releaseConfig is the subset of .goreleaser.yaml this repo asserts on: the
// per-target build environments.
type releaseConfig struct {
	Builds []struct {
		ID  string   `yaml:"id"`
		Env []string `yaml:"env"`
	} `yaml:"builds"`
}

// binaryNeedsCgo reports whether the agent binary can still be built with CGO
// disabled. Core's source inspection links real tree-sitter grammars, whose
// bindings carry `//go:build cgo`; with CGO off they contribute no files and
// the build fails at load time rather than degrading. Loading is enough to
// find that out, so this costs a fraction of a second.
func binaryNeedsCgo(t *testing.T) bool {
	t.Helper()
	cmd := exec.Command("go", "build", "-o", os.DevNull, ".")
	cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
	out, err := cmd.CombinedOutput()
	if err != nil && !strings.Contains(string(out), "build constraints exclude all Go files") {
		t.Fatalf("cgo-free build failed for an unexpected reason: %v\n%s", err, out)
	}
	return err != nil
}

// TestReleaseCanCompileTheCgoTheBinaryNeeds is the guard the release path had
// no way to fail on: `ci.yml` builds with the runner's default (CGO on) and
// the release workflow only runs on a tag, so a dependency bump that pulls in
// tree-sitter turns every release red and nothing says so until a tag is cut.
//
// Two halves, and both are load-bearing. `CGO_ENABLED=0` in the build config
// fails the compile outright. `CGO_ENABLED=1` on a bare runner fails too, just
// later and differently — an ubuntu runner has no macOS compiler — which is
// why the release has to go through the cross toolchain image.
func TestReleaseCanCompileTheCgoTheBinaryNeeds(t *testing.T) {
	if !binaryNeedsCgo(t) {
		t.Skip("the binary builds without cgo, so the release toolchain is unconstrained")
	}

	raw, err := os.ReadFile(".goreleaser.yaml")
	if err != nil {
		t.Fatalf("read .goreleaser.yaml: %v", err)
	}
	var config releaseConfig
	if err := yaml.Unmarshal(raw, &config); err != nil {
		t.Fatalf("parse .goreleaser.yaml: %v", err)
	}
	if len(config.Builds) == 0 {
		t.Fatal(".goreleaser.yaml declares no builds")
	}
	for _, build := range config.Builds {
		var enabled bool
		for _, env := range build.Env {
			if env == "CGO_ENABLED=1" {
				enabled = true
			}
			if env == "CGO_ENABLED=0" {
				t.Errorf("build %q disables cgo; the binary links tree-sitter and will not compile", build.ID)
			}
		}
		if !enabled {
			t.Errorf("build %q does not set CGO_ENABLED=1; goreleaser-cross defaults are not guaranteed", build.ID)
		}
	}

	workflow, err := os.ReadFile(filepath.Join(".github", "workflows", "releaser.yml"))
	if err != nil {
		t.Fatalf("read releaser.yml: %v", err)
	}
	if !strings.Contains(string(workflow), "goreleaser-cross") {
		t.Error("the release workflow does not run goreleaser through the cross toolchain image, " +
			"so CGO_ENABLED=1 has no macOS or cross-linux compiler to use")
	}
}

// agentContextCap extracts the line budget AGENTS.md states for itself. The
// budget is parsed rather than hardcoded so the assertion is that the file
// obeys its own stated rule: a file that tells the next agent to stay under a
// number it has itself exceeded is an instruction that reads as false.
var agentContextCap = regexp.MustCompile(`hard cap (\d+)`)

// skillFrontmatter is the Agent Skills contract: name and description are the
// only required fields, and the name is what the directory must be called.
type skillFrontmatter struct {
	Name        string `yaml:"name"`
	Description string `yaml:"description"`
}

func TestAgentContextFilesHonourTheirOwnContract(t *testing.T) {
	raw, err := os.ReadFile("AGENTS.md")
	if err != nil {
		t.Fatalf("read AGENTS.md: %v", err)
	}
	lines := strings.Count(string(raw), "\n")

	match := agentContextCap.FindSubmatch(raw)
	if match == nil {
		t.Fatal("AGENTS.md does not state a hard cap; the length budget is unenforceable")
	}
	stated, err := strconv.Atoi(string(match[1]))
	if err != nil {
		t.Fatalf("parse hard cap %q: %v", match[1], err)
	}
	if lines > stated {
		t.Errorf("AGENTS.md is %d lines but states a hard cap of %d; push depth into a skill", lines, stated)
	}

	// A CLAUDE.md is optional, but when present it must stay a pointer rather
	// than a second copy that drifts from this file.
	if pointer, err := os.ReadFile("CLAUDE.md"); err == nil {
		if strings.TrimSpace(string(pointer)) != "@AGENTS.md" {
			t.Errorf("CLAUDE.md must be the single line @AGENTS.md, got %q", string(pointer))
		}
	}
}

func TestSkillsDeclareTheRequiredFrontmatter(t *testing.T) {
	entries, err := os.ReadDir(filepath.Join(".claude", "skills"))
	if err != nil {
		if os.IsNotExist(err) {
			t.Skip("no skills directory")
		}
		t.Fatalf("read skills dir: %v", err)
	}

	valid := regexp.MustCompile(`^[a-z0-9-]{1,64}$`)
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}
		path := filepath.Join(".claude", "skills", entry.Name(), "SKILL.md")
		raw, err := os.ReadFile(path)
		if err != nil {
			t.Errorf("%s: %v", path, err)
			continue
		}
		// Frontmatter is the block between the leading --- fence and the next.
		body := string(raw)
		if !strings.HasPrefix(body, "---\n") {
			t.Errorf("%s: no YAML frontmatter", path)
			continue
		}
		end := strings.Index(body[4:], "\n---\n")
		if end < 0 {
			t.Errorf("%s: frontmatter is not terminated", path)
			continue
		}
		var front skillFrontmatter
		if err := yaml.Unmarshal([]byte(body[4:4+end]), &front); err != nil {
			t.Errorf("%s: parse frontmatter: %v", path, err)
			continue
		}
		if front.Name != entry.Name() {
			t.Errorf("%s: frontmatter name %q does not match directory %q", path, front.Name, entry.Name())
		}
		if !valid.MatchString(front.Name) {
			t.Errorf("%s: name %q must be lowercase letters, digits and hyphens, at most 64 chars", path, front.Name)
		}
		// The description is all an agent sees before loading the skill, so an
		// empty one makes the skill undiscoverable and an over-long one is
		// rejected by the standard.
		if front.Description == "" {
			t.Errorf("%s: description is required", path)
		}
		if len(front.Description) > 1024 {
			t.Errorf("%s: description is %d chars, over the 1024 limit", path, len(front.Description))
		}
	}
}

// TestNixBackendIsAssertedInCI closes the one way ci.yml's `nix` job can pass
// while asserting nothing: `go test -run` that matches no test prints a warning
// and exits 0. A renamed or deleted test would turn the job green by absence —
// the exact shape the job exists to close — and nothing else would say so.
func TestNixBackendIsAssertedInCI(t *testing.T) {
	const backendTest = "TestNixBackendToolchain"

	workflow, err := os.ReadFile(filepath.Join(".github", "workflows", "ci.yml"))
	if err != nil {
		t.Fatalf("read ci.yml: %v", err)
	}
	if !strings.Contains(string(workflow), backendTest) {
		t.Fatalf("ci.yml runs no %s; the nix backend is covered nowhere", backendTest)
	}
	if !strings.Contains(string(workflow), "requirenix") {
		t.Error("ci.yml does not build the requirenix tag, so the nix assertion never compiles in")
	}

	source, err := os.ReadFile("nix_backend_test.go")
	if err != nil {
		t.Fatalf("read nix_backend_test.go: %v", err)
	}
	if !strings.Contains(string(source), "func "+backendTest+"(") {
		t.Errorf("ci.yml runs %s but no such test is declared; -run would match nothing", backendTest)
	}
	if !strings.Contains(string(source), `testmatrix.Only("nix")`) {
		t.Error("the nix backend test does not pass Only(\"nix\"), " +
			"so a runner that lost nix skips instead of failing")
	}
}

// TestEveryStartResponseNamesWhatItRan protects the trap in
// runtime-started-artifact/v1: core CONSUMES the report at each
// `StartResponse`, so a Start path that answers STARTED without going through
// `started()` stamps no artifact, and a native run then refuses a service that
// is up (cli#993). The failure is silent here — the agent builds, the suite
// passes, the service serves — and only shows up as a refused delivery, so the
// invariant is asserted on the source rather than left to review.
//
// `started()` is the one legal caller. A new path that needs its own response
// calls it too; a path that must NOT report is a failure, and failures go
// through StartError, which discards the report by contract.
func TestEveryStartResponseNamesWhatItRan(t *testing.T) {
	files, err := runtimeGoFiles(".")
	if err != nil {
		t.Fatalf("enumerate runtime source: %v", err)
	}
	sites := 0
	reported := false
	for _, name := range files {
		src, err := os.ReadFile(name)
		if err != nil {
			t.Fatalf("read %s: %v", name, err)
		}
		content := string(src)
		sites += strings.Count(content, "Runtime.StartResponse()")

		// The one legal call site, and the report that must precede it in the
		// same body.
		body := startedBody(content)
		if body == "" {
			continue
		}
		if !strings.Contains(body, "Runtime.StartResponse()") {
			t.Errorf("%s: started() does not answer STARTED", name)
		}
		if index := strings.Index(body, "ReportStartedArtifact("); index >= 0 && index < strings.Index(body, "Runtime.StartResponse()") {
			reported = true
		} else {
			t.Errorf("%s: started() answers STARTED without naming what it ran first; StartResponse consumes the report, so the order is the behaviour", name)
		}
	}
	if sites != 1 {
		t.Errorf("found %d StartResponse call sites, want exactly 1 (started()); a second one is a start that names no build", sites)
	}
	if !reported {
		t.Error("no started() reporting the artifact was found in the runtime source")
	}
}

// startedBody is the source of `func (s *Runtime) started()`, from its opening
// brace to the first line that closes it at column zero. Crude, and enough: it
// is one short function whose body is what the assertion above is about.
func startedBody(content string) string {
	const signature = "func (s *Runtime) started() "
	start := strings.Index(content, signature)
	if start < 0 {
		return ""
	}
	rest := content[start:]
	if end := strings.Index(rest, "\n}\n"); end >= 0 {
		return rest[:end]
	}
	return rest
}