Merge pull request #60 from codefly-dev/codex/formula-test-scope

Bind Python test requests to exact pytest scope

Author: AntoineToussaint

agent.codefly.yaml

@@ -1,4 +1,4 @@
 publisher: codefly.dev
 kind: codefly:service
 name: python
-version: 0.0.44
+version: 0.0.45

go.mod

@@ -5,7 +5,7 @@ go 1.25.12
 toolchain go1.26.4
 
 require (
-	github.com/codefly-dev/core v0.2.118
+	github.com/codefly-dev/core v0.2.119
 	github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827
 	gopkg.in/yaml.v3 v3.0.1
 )

go.sum

@@ -37,8 +37,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ
 github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
 github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
-github.com/codefly-dev/core v0.2.118 h1:oy7TUKPLxfpcOC1C3NnZ5ktEctw4ZidCoBjHfLgCu+U=
-github.com/codefly-dev/core v0.2.118/go.mod h1:e3xNBpBgMOzraxGaFbbiYtMYyhLAEoP7Cy2yaMrvV5U=
+github.com/codefly-dev/core v0.2.119 h1:krAElUdbrafbqNghApTVReeKrzrkzPpQ0TQ8oQBq6CY=
+github.com/codefly-dev/core v0.2.119/go.mod h1:e3xNBpBgMOzraxGaFbbiYtMYyhLAEoP7Cy2yaMrvV5U=
 github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ=
 github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg=
 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=

pkg/runtime/runtime.go

@@ -129,23 +129,13 @@ func (s *Runtime) Test(ctx context.Context, req *runtimev0.TestRequest) (*runtim
 	// plugin is the ONLY place the generic provisioning map becomes uv flags.
 	if cmd, output, env, prov, ok := resolveTestFormula(req, s.Base.Service, s.Service.SourceLocation); ok {
 		fspec := pythonhelpers.SpecFromFormula(cmd, output, env, prov, nil)
-		selectors, selectorErr := pythonTestRequestSelectors(req, fspec.Command, fspec.Cwd)
-		if selectorErr != nil {
+		if selectorErr := applyPythonFormulaTestScope(req, &fspec); selectorErr != nil {
 			return nil, fmt.Errorf("python runtime selection: %w", selectorErr)
 		}
-		// A typed selector is authoritative execution scope. Derived pytest
-		// commands may carry broad project discovery operands (for example,
-		// `pytest --pyargs astropy docs {posargs}`). Appending one node ID to
-		// those operands still collects unrelated modules, allowing their
-		// failures to masquerade as the selected case. Keep the real runner
-		// prefix and replace broad pytest discovery before adding the exact
-		// plugin-rendered selector.
-		if req.GetSelection() != nil {
-			fspec.Command = pythonhelpers.CommandForExactSelection(fspec.Command)
-		}
-		fspec.Selectors = selectors
 		s.Wool.Info("running test formula via uv",
-			wool.Field("command", fspec.Command), wool.Field("output", fspec.Output))
+			wool.Field("command", fspec.Command),
+			wool.Field("selectors", fspec.Selectors),
+			wool.Field("output", fspec.Output))
 		fStart := time.Now()
 		fRun, fErr := pythonhelpers.RunFormulaStructured(ctx, s.Service.SourceLocation, fspec)
 		evidence := pythonhelpers.RuntimeEvidenceForFormula(s.Service.SourceLocation, cmd, output, env, prov, formulaDerivedFromProject(req, s.Base.Service))
@@ -368,18 +358,83 @@ func boundedTestLogMessage(message string) string {
 	) + message[tailStart:]
 }
 
-// pythonTestRequestSelectors translates authoritative typed scope inside the
-// Python plugin. Legacy target/filter requests remain supported for interactive
-// broad runs, but never participate in the typed acknowledgement contract.
-func pythonTestRequestSelectors(req *runtimev0.TestRequest, command []string, cwd string) ([]string, error) {
+// applyPythonFormulaTestScope translates the language-neutral runtime request
+// into runner-private formula arguments. This is the boundary that owns the
+// distinction between pytest collection targets and pytest name expressions:
+// callers provide target/filter data and never construct runner flags.
+//
+// ARCHITECTURE: A target or exact pytest node identity is authoritative scope.
+// Derived formulas may contain broad discovery operands such as
+// `pytest --pyargs astropy docs`; retaining those operands would execute
+// unrelated tests in addition to the requested target. Exact scope therefore
+// replaces broad discovery, while ordinary filters preserve the derived scope
+// and become pytest's name-expression argument. Typed TestSelection remains
+// the acknowledged, fail-closed acceptance contract.
+func applyPythonFormulaTestScope(req *runtimev0.TestRequest, spec *pythonhelpers.TestFormulaSpec) error {
 	if req.GetSelection() != nil {
-		return pythonhelpers.RenderTestSelection(req.GetSelection(), command, cwd)
+		selectors, err := pythonhelpers.RenderTestSelection(req.GetSelection(), spec.Command, spec.Cwd)
+		if err != nil {
+			return err
+		}
+		spec.Command = pythonhelpers.CommandForExactSelection(spec.Command)
+		spec.Selectors = selectors
+		return nil
+	}
+
+	// JUnit XML selects Codefly's pytest adapter. The Python plugin, not Mind,
+	// owns pytest's private distinction between positional collection targets
+	// and -k name expressions.
+	if spec.Output == pythonhelpers.OutputJUnitXML {
+		var exactSelectors, nameFilters []string
+		for _, filter := range req.GetFilters() {
+			if isExactPytestSelector(filter) {
+				exactSelectors = append(exactSelectors, filter)
+			} else if strings.TrimSpace(filter) != "" {
+				nameFilters = append(nameFilters, filter)
+			}
+		}
+		if target := strings.TrimSpace(req.GetTarget()); target != "" {
+			exactSelectors = append(exactSelectors, target)
+		}
+		if len(exactSelectors) > 0 {
+			spec.Command = pythonhelpers.CommandForExactSelection(spec.Command)
+		}
+		if len(nameFilters) > 0 {
+			spec.ExtraArgs = append(spec.ExtraArgs, "-k", strings.Join(nameFilters, " or "))
+			// A filter-only request is a complete test execution even though it
+			// has no positional selector. Say so explicitly: formula Core's auto
+			// mode intentionally treats a selector-free invocation as an
+			// environment materialization probe.
+			spec.ExecutionMode = pythonhelpers.FormulaExecutionComplete
+		}
+		spec.Selectors = exactSelectors
+		return nil
+	}
+
+	// Non-pytest formula runners own their positional selector grammar. Core's
+	// formula executor performs the runner-specific normalization (for example,
+	// unittest display identities to Django dotted labels).
+	spec.Selectors = append(spec.Selectors, req.GetFilters()...)
+	if target := strings.TrimSpace(req.GetTarget()); target != "" {
+		spec.Selectors = append(spec.Selectors, target)
+	}
+	return nil
+}
+
+// isExactPytestSelector recognizes collection identities, not arbitrary name
+// expressions. This knowledge belongs in the Python runtime plugin. SWE-bench
+// node IDs and file targets are positional pytest operands; simple names and
+// boolean expressions remain filters.
+func isExactPytestSelector(value string) bool {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return false
 	}
-	selectors := append([]string(nil), req.GetFilters()...)
-	if target := req.GetTarget(); target != "" {
-		selectors = append(selectors, target)
+	if strings.Contains(value, "::") {
+		return true
 	}
-	return selectors, nil
+	normalized := strings.ReplaceAll(value, `\`, "/")
+	return strings.HasSuffix(normalized, ".py") || strings.Contains(normalized, ".py/")
 }
 
 func pythonDefaultTestScope(req *runtimev0.TestRequest) (string, []string, error) {

pkg/runtime/runtime_test.go

@@ -145,6 +145,126 @@ commands = pytest --pyargs broadpkg {posargs}
 	assertRuntimeTestLeftSourceClean(t, root)
 }
 
+// TestRuntimeFormulaTargetReplacesBroadPytestDiscovery proves the public
+// target field is execution scope even when the project-derived formula names
+// broad pytest roots. The runtime owns the argv translation; callers provide
+// only target data. An unrelated collection crash makes accidental widening
+// observable through real uv + pytest.
+func TestRuntimeFormulaTargetReplacesBroadPytestDiscovery(t *testing.T) {
+	if _, err := exec.LookPath("uv"); err != nil {
+		t.Fatalf("uv is required for the production Python runtime: %v", err)
+	}
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "broadpkg"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for path, content := range map[string]string{
+		"tox.ini": `[testenv]
+commands = pytest --pyargs broadpkg {posargs}
+`,
+		"broadpkg/__init__.py":    "",
+		"broadpkg/test_broken.py": "raise RuntimeError('unselected collection must not run')\n",
+		"test_selected.py":        "def test_selected_pass():\n    assert 2 + 2 == 4\n",
+	} {
+		if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(path)), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	svc := pythonservice.New(&resources.Agent{Kind: "codefly:service", Name: "python"})
+	svc.SourceLocation = root
+	resp, err := pythonruntime.New(svc).Test(context.Background(), &runtimev0.TestRequest{
+		Target: "test_selected.py::test_selected_pass",
+	})
+	if err != nil {
+		t.Fatalf("Test: %v", err)
+	}
+	if resp.GetResult().GetState() != runtimev0.TestRunResult_PASSED || resp.GetCounts().GetTotal() != 1 || resp.GetCounts().GetPassed() != 1 {
+		t.Fatalf("targeted result = %s counts=%+v, want only one passing case\n%s", resp.GetResult().GetState(), resp.GetCounts(), resp.GetOutput())
+	}
+	assertRuntimeTestLeftSourceClean(t, root)
+}
+
+// TestRuntimeFormulaExactFilterReplacesBroadPytestDiscovery proves exact node
+// identities carried in the repeated filters field remain exact targets. This
+// is the shape used by result-driven graders that own a set of test identities.
+func TestRuntimeFormulaExactFilterReplacesBroadPytestDiscovery(t *testing.T) {
+	if _, err := exec.LookPath("uv"); err != nil {
+		t.Fatalf("uv is required for the production Python runtime: %v", err)
+	}
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "broadpkg"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for path, content := range map[string]string{
+		"tox.ini": `[testenv]
+commands = pytest --pyargs broadpkg {posargs}
+`,
+		"broadpkg/__init__.py":    "",
+		"broadpkg/test_broken.py": "raise RuntimeError('unselected collection must not run')\n",
+		"test_selected.py":        "def test_selected_pass():\n    assert 2 + 2 == 4\n",
+	} {
+		if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(path)), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	svc := pythonservice.New(&resources.Agent{Kind: "codefly:service", Name: "python"})
+	svc.SourceLocation = root
+	resp, err := pythonruntime.New(svc).Test(context.Background(), &runtimev0.TestRequest{
+		Filters: []string{"test_selected.py::test_selected_pass"},
+	})
+	if err != nil {
+		t.Fatalf("Test: %v", err)
+	}
+	if resp.GetResult().GetState() != runtimev0.TestRunResult_PASSED || resp.GetCounts().GetTotal() != 1 || resp.GetCounts().GetPassed() != 1 {
+		t.Fatalf("filtered result = %s counts=%+v, want only one passing case\n%s", resp.GetResult().GetState(), resp.GetCounts(), resp.GetOutput())
+	}
+	assertRuntimeTestLeftSourceClean(t, root)
+}
+
+// TestRuntimeFormulaNameFilterUsesPytestExpression proves ordinary name
+// filters retain the project-derived discovery roots and become runner-owned
+// pytest expressions rather than invalid positional paths.
+func TestRuntimeFormulaNameFilterUsesPytestExpression(t *testing.T) {
+	if _, err := exec.LookPath("uv"); err != nil {
+		t.Fatalf("uv is required for the production Python runtime: %v", err)
+	}
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "broadpkg"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for path, content := range map[string]string{
+		"tox.ini": `[testenv]
+commands = pytest --pyargs broadpkg {posargs}
+`,
+		"broadpkg/__init__.py": "",
+		"broadpkg/test_cases.py": `def test_selected_pass():
+    assert 2 + 2 == 4
+
+def test_unselected_fail():
+    assert False
+`,
+	} {
+		if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(path)), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	svc := pythonservice.New(&resources.Agent{Kind: "codefly:service", Name: "python"})
+	svc.SourceLocation = root
+	resp, err := pythonruntime.New(svc).Test(context.Background(), &runtimev0.TestRequest{
+		Filters: []string{"test_selected_pass"},
+	})
+	if err != nil {
+		t.Fatalf("Test: %v", err)
+	}
+	if resp.GetResult().GetState() != runtimev0.TestRunResult_PASSED || resp.GetCounts().GetTotal() != 1 || resp.GetCounts().GetPassed() != 1 {
+		t.Fatalf("name-filtered result = %s counts=%+v, want one selected passing case\n%s", resp.GetResult().GetState(), resp.GetCounts(), resp.GetOutput())
+	}
+	assertRuntimeTestLeftSourceClean(t, root)
+}
+
 // TestRuntimeFormulaNormalizesUnittestDisplaySelection proves the production
 // agent boundary accepts the canonical case identity returned by unittest
 // (`method (module.Class)`) and routes it back through a formula that consumes