Refactor onboarding and testdrive orchestration

internal/cmd/ci_provider.go

@@ -22,6 +22,29 @@ import (
 // selectCIProvider resolves the CI provider; tests replace it.
 var selectCIProvider = onboard.SelectCIProvider
 
+// ciOptions keeps the shared CI flags and their resolution order together.
+// Each command owns an instance so flag values never leak between commands.
+type ciOptions struct {
+	provider string
+	runtimes []string
+	facts    []string
+}
+
+func (options *ciOptions) addFlags(command *cobra.Command) {
+	command.Flags().StringVar(&options.provider, "ci-provider", "", ciProviderFlagUsage())
+	ciStatementFlags(command, &options.runtimes, &options.facts)
+}
+
+func (options ciOptions) resolve(command *cobra.Command, structured bool) error {
+	if err := resolveCIProvider(command, options.provider); err != nil {
+		return ciProviderFailure(command, structured, err)
+	}
+	if err := resolveCIStatements(options.runtimes, options.facts); err != nil {
+		return ciStatementFailure(command, structured, err)
+	}
+	return nil
+}
+
 // ciProviderFlagUsage describes --ci-provider on onboard and testdrive.
 func ciProviderFlagUsage() string {
 	return "CI provider whose configuration to discover, set up and validate (" + strings.Join(onboard.CIProviderNames(), ", ") + "); required when several contain tests"

internal/cmd/onboard.go

@@ -22,10 +22,10 @@ var onboardCmd = newOnboardCommand(nil)
 
 func newOnboardCommand(run func(io.Writer) error) *cobra.Command {
 	var apply, yes, instructions, allowDirty, agent bool
-	var version, facts, site, format, resume, ciProvider string
+	var version, facts, site, format, resume string
 	var timeout time.Duration
 	var reviewed, substitute bool
-	var ciRuntimes, ciFacts []string
+	var ci ciOptions
 	command := &cobra.Command{
 		Use:   "onboard",
 		Short: "Start here: onboard this repository to Test Optimization",
@@ -52,12 +52,9 @@ func newOnboardCommand(run func(io.Writer) error) *cobra.Command {
 			if apply && instructions {
 				return fail("invalid_options", fmt.Errorf("--apply and --instructions are mutually exclusive"))
 			}
-			if err := resolveCIProvider(cmd, ciProvider); err != nil {
-				// Without --instructions, onboarding answers non-terminals in the agent protocol.
-				return ciProviderFailure(cmd, structured || (!instructions && !apply && !isTerminalWriter(cmd.OutOrStdout())), err)
-			}
-			if err := resolveCIStatements(ciRuntimes, ciFacts); err != nil {
-				return ciStatementFailure(cmd, structured || (!instructions && !apply && !isTerminalWriter(cmd.OutOrStdout())), err)
+			// Without --instructions, onboarding answers non-terminals in the agent protocol.
+			if err := ci.resolve(cmd, structured || (!instructions && !apply && !isTerminalWriter(cmd.OutOrStdout()))); err != nil {
+				return err
 			}
 			// start returns the onboarding handoff: the next command, or for a
 			// language whose CI setup the agent applies, the setup to apply first.
@@ -147,8 +144,7 @@ func newOnboardCommand(run func(io.Writer) error) *cobra.Command {
 	command.Flags().StringVar(&facts, "facts", "", "JSON build/service facts bound to input_fingerprint; must be outside the checkout")
 	command.Flags().StringVar(&site, "site", "", "Datadog site for a new action (default DD_SITE, otherwise datadoghq.com)")
 	command.Flags().StringVar(&format, "format", "agent", "Result rendering: agent or json for direct host consumption")
-	command.Flags().StringVar(&ciProvider, "ci-provider", "", ciProviderFlagUsage())
-	ciStatementFlags(command, &ciRuntimes, &ciFacts)
+	ci.addFlags(command)
 
 	command.Flags().BoolVar(&allowDirty, "allow-dirty", false, "Explicitly extend uncommitted target-file edits after showing their exact diff")
 	agentHelp(command)

internal/cmd/testdrive.go

@@ -35,10 +35,10 @@ func newTestdriveCommand() *cobra.Command {
 			return nil
 		},
 	}
-	var version, facts, format, resume, detail, site, ciProvider string
+	var version, facts, format, resume, detail, site string
 	var checkOnly, agent, report bool
 	var all bool
-	var ciRuntimes, ciFacts []string
+	var ci ciOptions
 	var allowReviewedBuilds, acceptSubstituteRuntime bool
 	var timeout time.Duration
 	command.Flags().DurationVar(&timeout, "timeout", 10*time.Minute, "Maximum duration of each subprocess, including each full test-suite run")
@@ -55,8 +55,7 @@ func newTestdriveCommand() *cobra.Command {
 	command.Flags().StringVar(&resume, "resume", "", "Resume the same repository/framework/scope; tests always execute fresh")
 	command.Flags().StringVar(&facts, "facts", "", "JSON build/service facts bound to input_fingerprint (requires --all)")
 	command.Flags().StringVar(&site, "site", "", "Datadog site retained in the resolved validation plan")
-	command.Flags().StringVar(&ciProvider, "ci-provider", "", ciProviderFlagUsage())
-	ciStatementFlags(command, &ciRuntimes, &ciFacts)
+	ci.addFlags(command)
 
 	command.RunE = func(cmd *cobra.Command, _ []string) error {
 		structured := agent || cmd.Flags().Changed("format")
@@ -83,11 +82,8 @@ func newTestdriveCommand() *cobra.Command {
 			return fail("invalid_options", fmt.Errorf("--report cannot be combined with --all or --check-only"))
 		}
 		// Reports are bound to the provider they validated, so --report resolves it too.
-		if err := resolveCIProvider(cmd, ciProvider); err != nil {
-			return ciProviderFailure(cmd, structured, err)
-		}
-		if err := resolveCIStatements(ciRuntimes, ciFacts); err != nil {
-			return ciStatementFailure(cmd, structured, err)
+		if err := ci.resolve(cmd, structured); err != nil {
+			return err
 		}
 		if report {
 			cmd.SilenceErrors = true

internal/onboard/README.md

@@ -0,0 +1,61 @@
+# Onboarding and testdrive code map
+
+`ddtest onboard` and `ddtest testdrive` share CI discovery and validation.
+Start at the boundary that owns the behavior you need to change:
+
+| Responsibility | Entry points |
+| --- | --- |
+| CLI flags, consent and output mode | `internal/cmd/onboard.go`, `internal/cmd/testdrive.go` |
+| Shared CI flags and error rendering | `internal/cmd/ci_provider.go` (`ciOptions`) |
+| Onboarding detection and setup handoffs | `internal/onboard/agent.go` |
+| Shared agent protocol and rendering | `internal/onboard/agent_result.go` |
+| Persistent executable lookup and follow-up commands | `internal/onboard/commands.go` |
+| Execution plans and setup-facts schema | `internal/onboard/execution_plan.go` |
+| CI provider selection and follow-up arguments | `internal/onboard/ci_selection.go` |
+| Provider contract and registrations | `internal/onboard/ci_provider.go`, `ci_providers.go` |
+| Workflow model, loading and shared test/setup classification | `internal/onboard/ci_workflows.go` |
+| Provider configuration syntax and edits | `internal/onboard/ci_provider_<provider>*.go` |
+| Shared setup model, support checks and setup wording | `internal/onboard/ci_setup.go` |
+| User statements about CI runtimes and server configuration | `internal/onboard/ci_statements.go` |
+| Language-specific discovery and setup | `internal/onboard/language.go`, `languages.go`, `<language>*.go` |
+| Resolved test commands and validation scope | `internal/onboard/discovery.go`, `scope.go`, `scripts.go` |
+| Literal command recognition without executing repository code | `internal/testcommand` |
+| Setup plans and bounded source edits | `internal/onboard/execution_plan.go`, `apply*.go` |
+| Validation preparation and execution | `internal/testdrive/testdrive.go` |
+| Shared paired-execution engine and framework adapters | `internal/testdrive/paired_validation.go` |
+| Shared preflight evidence and CI configuration checks | `internal/testdrive/preflight.go` |
+| Native result model, outcome comparison and telemetry matching | `internal/testdrive/native_results.go` |
+| Coordinated setup and validation | `internal/testdrive/workflow.go` |
+| Attempt state, failure policy and retry handoffs | `internal/testdrive/workflow_attempt.go` |
+| Shared CI questions and pinned tracer selection | `internal/testdrive/workflow_ci.go` |
+| Checkpoint-bound facts and setup failure recovery | `internal/testdrive/workflow_setup.go` |
+| Verified report completion and blocker handoffs | `internal/testdrive/workflow_completion.go` |
+| Restart and resume commands | `internal/testdrive/workflow_commands.go` |
+| Workflow tracer selection and metadata preflight | `internal/testdrive/workflow_tracer.go` |
+| Saved workflow reports and blocker classification | `internal/testdrive/workflow_report.go` |
+| Input fingerprints and accepted setup facts | `internal/testdrive/workflow_inputs.go` |
+| Evidence, cleanup and checkpoints | `internal/testdrive/workflow_checks.go`, `workflow_checkpoint.go`, `workflow_receipt.go` |
+
+Unqualified filenames in a row are in the same directory as that row's first file.
+
+Providers parse their own syntax into `ciWorkflow`, `runtimeJob` and
+`runtimeStep`. Shared discovery, setup and runtime checks consume those models;
+language decisions stay behind `discoveryLanguage`. Providers normally use
+`findTestWorkflows`' shared classification. GitHub manual instructions preserve
+their conservative literal-command inspection and use the same GitHub file
+loader as full discovery. Instructions decode only the fields they inspect;
+full discovery decodes the complete workflow model and expands composite actions.
+
+`internal/environment` detects the running CI environment and supplies runtime
+tags and runner configuration. Repository CI inspection belongs here, in
+`internal/onboard`, and is shared with testdrive.
+
+`workflowAttempt` owns a single invocation's result, checkpoint and recovery
+state. The coordinator sequences discovery, setup, execution and verification;
+the attempt handles failure and completion handoffs. Standalone validation
+shares the attempt's CI decisions and failure policy while keeping its own
+execution and verdict rules.
+
+After changing these paths, run `make test` and `make lint`. The end-to-end
+Codex catalog is described in
+[`skills/ddtest-onboarding-eval/SKILL.md`](../../skills/ddtest-onboarding-eval/SKILL.md).