diff --git a/cmd/astro/bundle_fixtures_test.go b/cmd/astro/bundle_fixtures_test.go new file mode 100644 index 000000000..5dd00f2fe --- /dev/null +++ b/cmd/astro/bundle_fixtures_test.go @@ -0,0 +1,220 @@ +package astro + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/config" + "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" + astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" +) + +const ( + dbtTestDeploymentID = "clxdbtdeployment000000000" + dbtTestDeployID = "clxdbtdeploy0000000000000" + dbtTestVersion = "2026-10-07T00:00:00.0000000Z" +) + +// fakeBlobStore answers the bundle upload the way Azure does: every request +// succeeds, and the commit reports the version the upload created. +func fakeBlobStore(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("x-ms-version-id", dbtTestVersion) + w.WriteHeader(http.StatusCreated) + })) + t.Cleanup(srv.Close) + return srv.URL + "/container/bundle.tar.gz?sig=x" +} + +// dbtProject writes a dbt project named jaffle_shop in a fresh directory outside +// any Astro project, and makes a second fresh directory the working path, so +// the bundle's tarball is written beside it rather than inside it. +func dbtProject(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte("name: jaffle_shop\n"), 0o600)) + prev := config.WorkingPath + config.WorkingPath = t.TempDir() + t.Cleanup(func() { config.WorkingPath = prev }) + return dir +} + +func dbtTestDeployment() astrov1.Deployment { + return astrov1.Deployment{ + Id: dbtTestDeploymentID, + Name: "analytics", + WorkspaceId: "clxdbtworkspace0000000000", + IsDagDeployEnabled: true, + Status: astrov1.DeploymentStatusHEALTHY, + } +} + +// dbtBundleMock is the API a bundle deploy or delete talks to. +func dbtBundleMock(t *testing.T, uploadURL string) *astrov1_mocks.ClientWithResponsesInterface { + t.Helper() + m := new(astrov1_mocks.ClientWithResponsesInterface) + t.Cleanup(func() { m.AssertExpectations(t) }) + d := dbtTestDeployment() + m.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID).Return(&astrov1.GetDeploymentResponse{ + HTTPResponse: &http.Response{StatusCode: http.StatusOK}, + JSON200: &d, + }, nil).Maybe() + deploy := astrov1.Deploy{Id: dbtTestDeployID} + if uploadURL != "" { + deploy.BundleUploadUrl = &uploadURL + } + m.On("CreateDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, mock.Anything).Return(&astrov1.CreateDeployResponse{ + HTTPResponse: &http.Response{StatusCode: http.StatusOK}, + JSON200: &deploy, + }, nil).Once() + m.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, dbtTestDeployID, mock.Anything).Return(&astrov1.FinalizeDeployResponse{ + HTTPResponse: &http.Response{StatusCode: http.StatusOK}, + JSON200: &deploy, + }, nil).Once() + return m +} + +// stubBundleWait stands in for the wait a --wait does, writing one line of +// progress where the real one writes its own and answering err. +func stubBundleWait(t *testing.T, err error) *[]string { + t.Helper() + var waited []string + prev := waitForBundle + waitForBundle = func(progress io.Writer, deploymentID string, _ time.Duration, _ astrov1.APIClient) error { + waited = append(waited, deploymentID) + fmt.Fprintln(progress, "Waiting for the deployment to become healthy…") + return err + } + t.Cleanup(func() { waitForBundle = prev }) + return &waited +} + +// A wait that fails and a result that then cannot be written are both +// reported: neither error hides the other. +func TestPublishThenWaitReportsBothFailures(t *testing.T) { + timedOut := errors.New("timed out") + stubBundleWait(t, timedOut) + writeFailed := errors.New("broken pipe") + + var stdout, stderr bytes.Buffer + root := &cobra.Command{Use: "astro"} + var format cliout.Format + cmd := &cobra.Command{Use: "x", RunE: func(cmd *cobra.Command, _ []string) error { + wait := bundleWait(cmd, &deployLogin{current: true}, dbtTestDeploymentID, time.Second) + return publishThenWaitWith(cmd, cliout.FormatJSON, true, wait, func(waitErr error) error { + assert.Same(t, timedOut, waitErr) + return writeFailed + }) + }} + cliout.AddOutputFlag(cmd, &format) + root.AddCommand(cmd) + root.SetOut(&stdout) + root.SetErr(&stderr) + + err := cliout.Execute(context.Background(), root, []string{"x", "-o", "json"}, &stdout, nil) + + require.ErrorIs(t, err, writeFailed) + require.ErrorIs(t, err, timedOut) + assert.Empty(t, stdout.String(), "the write to stdout failed, so no error object is written there after it") + assert.Equal(t, "Waiting for the deployment to become healthy…\nError: broken pipe\ntimed out\n", stderr.String(), "after the wait's progress, the write's failure first, then the wait's") + assert.Equal(t, cliout.ExitFailure, cliout.ExitCode(context.Background(), err)) +} + +// remote deploy's closing lines, as v2 printed them when the push succeeded. +const remoteDeployTail = "Successfully pushed client image to registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + + "\n--------------------------------\n" + + "The client image has been pushed to your private registry.\n" + + "Your next step would be to update the agent component to use the new client image.\n" + + "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.\n" + + "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + + "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components\n" + +// stubClientDeploy stands in for the build and the push, printing one of the +// progress lines the real one prints on the way. +func stubClientDeploy(t *testing.T, res *astrodeploy.ClientDeploy, err error) { + t.Helper() + prev := deployClientImage + deployClientImage = func(astrodeploy.InputClientDeploy, astrov1.APIClient) (astrodeploy.ClientDeploy, error) { + fmt.Println("Pushing client image to configured remote registry") + return *res, err + } + t.Cleanup(func() { deployClientImage = prev }) +} + +func execRemoteRun(t *testing.T, args ...string) tokenRun { + t.Helper() + // remote deploy runs in a project. + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + prev := config.WorkingPath + config.WorkingPath = dir + t.Cleanup(func() { config.WorkingPath = prev }) + return execAstroCmd(t, nil, "", newRemoteRootCmd, append([]string{"remote"}, args...)...) +} + +func TestRemoteDeployOutput(t *testing.T) { + pushed := astrodeploy.ClientDeploy{ + Image: "registry.example.com/agents/client:deploy-2026-10-07T14-53", + Registry: "registry.example.com/agents/client", + Tag: "deploy-2026-10-07T14-53", + Platforms: []string{"linux/amd64", "linux/arm64"}, + RuntimeCheck: &astrodeploy.ClientRuntimeCheck{ + DeploymentID: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2", + }, + } + t.Run("text", func(t *testing.T) { + stubClientDeploy(t, &pushed, nil) + r := execRemoteRun(t, "deploy") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + assert.Equal(t, "Pushing client image to configured remote registry\n"+remoteDeployTail, r.stdout) + }) + t.Run("json", func(t *testing.T) { + stubClientDeploy(t, &pushed, nil) + r := execRemoteRun(t, "deploy", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got remoteDeployJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, "registry.example.com/agents/client:deploy-2026-10-07T14-53", got.Image) + assert.Equal(t, "deploy-2026-10-07T14-53", got.Tag) + assert.Equal(t, []string{"linux/amd64", "linux/arm64"}, got.Platforms) + require.NotNil(t, got.RuntimeCheck) + assert.Equal(t, remoteRuntimeCheckJSON{Deployment: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2"}, *got.RuntimeCheck) + }) + t.Run("json, a prebuilt image", func(t *testing.T) { + stubClientDeploy(t, &astrodeploy.ClientDeploy{Image: "r/c:deploy-x", Registry: "r/c", Tag: "deploy-x", SourceImage: "local:1"}, nil) + r := execRemoteRun(t, "deploy", "--image-name", "local:1", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got remoteDeployJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, "local:1", got.SourceImage) + assert.JSONEq(t, "[]", string(fields["platforms"])) + assert.NotContains(t, fields, "runtime_check") + }) + t.Run("json, the push failing", func(t *testing.T) { + stubClientDeploy(t, &astrodeploy.ClientDeploy{}, errors.New("failed to push client image: denied")) + r := execRemoteRun(t, "deploy", "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitFailure, r.code) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Contains(t, got.Error, "denied") + }) +} diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index dc6e4ba97..4ac03ea31 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -1,348 +1,97 @@ package astro import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "time" + "strings" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" - - "github.com/astronomer/astro-cli/cmd/cliout" - "github.com/astronomer/astro-cli/cmd/utils" - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - "github.com/astronomer/astro-cli/internal/platform/astro/deployment" -) - -var ( - mountPath string - dbtProjectPath string - - DeployBundle = astrodeploy.DeployBundle - DeleteBundle = astrodeploy.DeleteBundle ) +// What the astro dbt group's commands say now. deploy and delete moved to +// astro deploy bundle, which deploys a dbt project, or any other non-DAG +// bundle, from its own directory. cleanup removed the Cosmos Boost sidecars +// earlier deploys wrote into the project; v2 writes none there, and leaves +// out the old ones from what it ships. const ( - dbtDefaultMountPathPrefix = "/usr/local/airflow/dbt/" - dbtProjectYmlFilename = "dbt_project.yml" - dbtBundleType = "dbt" - dbtWaitTime = 300 * time.Second + errDbtRemoved = "astro dbt was removed in Astro CLI v2: use astro deploy bundle, which deploys a dbt project " + + "(or any other non-DAG bundle) from its directory, and astro deploy bundle --delete, which removes one" + errDbtDeployRemoved = "astro dbt deploy was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment . " + + "PATH replaces --project-path, --deployment replaces the argument and --deployment-name, and --workspace replaces --workspace-id; " + + "--mount-path, --description, --wait and --wait-time are unchanged" + errDbtDeleteRemoved = "astro dbt delete was removed in Astro CLI v2: use astro deploy bundle --delete [PATH] --deployment . " + + "PATH replaces --project-path, --deployment replaces the argument and --deployment-name, and --workspace replaces --workspace-id; " + + "--mount-path, --description, --wait and --wait-time are unchanged" + errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + + "Astro CLI v2 no longer writes Cosmos Boost files into the project, and leaves out the ones earlier versions wrote " + + "from every bundle it deploys and every image it generates. " + + "Delete any old .astro/dbt_metadata.json and .astro/manifest.slim.json files by hand: " + + "a local Airflow running the Cosmos Boost plugin, or the build of a declared Dockerfile, may still read them" ) -func newDbtCmd(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "dbt", - Short: "Manage dbt projects deployed to your Deployments", - } - // The results leave by cmd.OutOrStdout(), which must be the stdout the - // tree was built with: under json strayStdoutToStderr points os.Stdout - // at stderr, and an unbound command would follow it there. - cmd.SetOut(out) - cmd.AddCommand( - newDbtDeployCmd(), - newDbtDeleteCmd(), - newDbtCleanupRemovedCmd(), - ) - applyPreferredFlagsIn(cmd) - return cmd -} - -// errDbtCleanupRemoved is what astro dbt cleanup says now. It removed the -// Cosmos Boost sidecars earlier deploys wrote into the project; v2 writes -// none there, and leaves out the old ones from what it ships. -const errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + - "Astro CLI v2 no longer writes Cosmos Boost files into the project, and leaves out the ones earlier versions wrote " + - "from every bundle it deploys and every image it generates. " + - "Delete any old .astro/dbt_metadata.json and .astro/manifest.slim.json files by hand: " + - "a local Airflow running the Cosmos Boost plugin, or the build of a declared Dockerfile, may still read them" - -// newDbtCleanupRemovedCmd builds the astro dbt cleanup tombstone -// (cliout.RemovedCommand), so a script that still runs it is told why rather -// than shown the dbt group's help. -func newDbtCleanupRemovedCmd() *cobra.Command { - return removedCmdStub("cleanup", nil, "Removed in v2 — Cosmos Boost no longer writes into the project", func([]string) string { - return errDbtCleanupRemoved - }) +// newDbtRemovedCmd builds the astro dbt tombstone (cliout.RemovedCommand): +// one hidden stub that takes any subcommand, so astro dbt deploy and its +// 1.x flags reach the guidance for what was typed rather than dying on a +// flag. +func newDbtRemovedCmd() *cobra.Command { + return removedCmdStub("dbt", nil, "Removed in v2 — use astro deploy bundle", dbtRemovedGuidance) } -//nolint:dupl // the duplication is acceptable here -func newDbtDeployCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "deploy [DEPLOYMENT_ID]", - Short: "Deploy your dbt project to a Deployment on Astro", - Long: "Deploy your dbt project to a Deployment on Astro. This command bundles your dbt project files and uploads it to your Deployment.", - Args: cobra.MaximumNArgs(1), - RunE: deployDbt, - Example: ` # Deploy the dbt project in the current directory, picking the Deployment from a list - astro dbt deploy - - # Deploy a dbt project elsewhere to a given Deployment - astro dbt deploy --project-path `, +// dbtRemovedGuidance names the replacement for the subcommand typed, the +// first argument that is not a flag. +func dbtRemovedGuidance(args []string) string { + if msg, ok := dbtSubcommandGuidance[dbtTypedSubcommand(args)]; ok { + return msg } - - cmd.Flags().StringVarP(&mountPath, "mount-path", "m", "", fmt.Sprintf("Path to mount dbt project in Airflow, for reference by Dags. Default %s{dbt project name}", dbtDefaultMountPathPrefix)) - cmd.Flags().StringVarP(&dbtProjectPath, "project-path", "p", "", "Path to the dbt project to deploy. Default current directory") - cmd.Flags().StringVar(&workspaceID, "workspace-id", "", "Workspace for your Deployment") - cmd.Flags().StringVarP(&deploymentName, "deployment-name", "n", "", "Name of the Deployment to deploy to") - addWorkspaceFlag(cmd.Flags(), "", "Workspace for your Deployment") - addDeploymentFlag(cmd.Flags(), "Deployment to deploy to: a Deployment id, or a Deployment name") - cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Description to store on the deploy") - cmd.Flags().BoolVarP(&waitForDeploy, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") - cmd.Flags().DurationVarP(&waitTime, "wait-time", "t", dbtWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") - cliout.AddOutputFlag(cmd, &dbtOutput) - - return cmd + return errDbtRemoved } -func deployDbt(cmd *cobra.Command, args []string) error { - format := dbtOutput - cmd.SilenceUsage = true - // The notes below, and the ones the platform code prints on the way, are - // the output text mode has always shown; under json they go to stderr. - defer strayStdoutToStderr(format)() - - // if the dbt project path is not provided, use the current directory - if dbtProjectPath == "" { - dbtProjectPath = config.WorkingPath - } - - // check that the dbt project path is not within an Astro project, by the - // walk every deploy decides by (astro deploy --non-dags checks the same) - if !utils.Locate(dbtProjectPath).None() { - return fmt.Errorf("dbt project is within an Astro project. Use 'astro deploy' to deploy your Astro project") - } - - // check that there is a valid dbt project at the dbt project path - err := validateDbtProjectExists(dbtProjectPath) - if err != nil { - return err - } - - // extract the dbt project's name - dbtProjectName, err := extractDbtProjectName(dbtProjectPath) - if err != nil { - return fmt.Errorf("dbt project name not found in %s: %w", dbtProjectPath, err) - } - - // if the workspace ID is not provided, try to find a valid workspace - if workspaceID == "" { - var err error - workspaceID, err = coalesceWorkspace() - if err != nil { - return fmt.Errorf("failed to find a valid workspace: %w", err) - } - } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return errors.New("cannot use --wait-time with --wait=false") - } - - // get the deployment id to deploy the dbt project to - deploymentID, read, err := resolveBundleDeployment(args, workspaceID, deploymentName, noCreateUnderJSON(format, "to deploy to")) - if err != nil { - return err - } - fmt.Println("Initiating dbt deploy for deployment ID: " + deploymentID) - - // if the mount path is not provided, derive it from the dbt project name - if mountPath == "" { - mountPath = dbtDefaultMountPathPrefix + dbtProjectName - fmt.Printf("Generated mount path from dbt project name: %s\n", mountPath) - } - - // deploy the dbt project as a bundle - deployBundleInput := &astrodeploy.DeployBundleInput{ - BundlePath: dbtProjectPath, - MountPath: mountPath, - DeploymentID: deploymentID, - Deployment: read, - BundleType: dbtBundleType, - Description: deployDescription, - AstroV1Client: astroV1Client, - } - res, err := DeployBundle(deployBundleInput) - if err != nil { - return err - } - projectPath, err := filepath.Abs(dbtProjectPath) - if err != nil { - projectPath = dbtProjectPath - } - r := cliout.Renderer{Format: format, Out: cmd.OutOrStdout()} - return publishThenWait(cmd, format, waitForDeploy, res.DeploymentID, waitTime, func(waitErr error) error { - return r.Emit(newDbtDeployJSON(&res, dbtProjectName, projectPath, waitForDeploy, waitErr), renderBundleUploaded(res.BundleVersion)) - }) +// dbtSubcommandGuidance is what each 1.x astro dbt subcommand says now. +var dbtSubcommandGuidance = map[string]string{ + "deploy": errDbtDeployRemoved, + "delete": errDbtDeleteRemoved, + "cleanup": errDbtCleanupRemoved, } -//nolint:dupl // the duplication is acceptable here -func newDbtDeleteCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "delete [DEPLOYMENT_ID]", - Short: "Delete a dbt project from a Deployment on Astro", - Long: "Delete a dbt project bundle from a Deployment. This removes the uploaded dbt project files but does not affect Dags that were generated from the project.", - Args: cobra.MaximumNArgs(1), - RunE: deleteDbt, - Example: ` # Delete the dbt project in the current directory, picking the Deployment from a list - astro dbt delete - - # Delete it from a given Deployment, by its mount path - astro dbt delete --mount-path `, - } - - cmd.Flags().StringVarP(&mountPath, "mount-path", "m", "", fmt.Sprintf("Mount path of the dbt project to be deleted from the Deployment. Default %s{dbt project name}", dbtDefaultMountPathPrefix)) - cmd.Flags().StringVarP(&dbtProjectPath, "project-path", "p", "", "Path to the dbt project to delete from the Deployment. Default current directory") - cmd.Flags().StringVar(&workspaceID, "workspace-id", "", "Workspace for your Deployment") - cmd.Flags().StringVarP(&deploymentName, "deployment-name", "n", "", "Name of the Deployment to delete the dbt project from") - addWorkspaceFlag(cmd.Flags(), "", "Workspace for your Deployment") - addDeploymentFlag(cmd.Flags(), "Deployment to delete the dbt project from: a Deployment id, or a Deployment name") - cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Description to store on the deploy") - cmd.Flags().BoolVarP(&waitForDeploy, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") - cmd.Flags().DurationVarP(&waitTime, "wait-time", "t", dbtWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") - cliout.AddOutputFlag(cmd, &dbtOutput) - return cmd +// dbtValueFlags are the flags that took a value on 1.x's astro dbt commands +// (v1_flags.tsv), and the --output and --verbosity v2 reads anywhere, so a +// flag's value typed before the subcommand is not read as the subcommand. +var dbtValueFlags = map[string]bool{ + "deployment-name": true, "n": true, + "description": true, + "mount-path": true, "m": true, + "project-path": true, "p": true, + "wait-time": true, "t": true, + "workspace-id": true, + "output": true, "o": true, + "verbosity": true, } -func deleteDbt(cmd *cobra.Command, args []string) error { - format := dbtOutput - cmd.SilenceUsage = true - defer strayStdoutToStderr(format)() - - // if the workspace ID is not provided, try to find a valid workspace - if workspaceID == "" { - var err error - workspaceID, err = coalesceWorkspace() - if err != nil { - return fmt.Errorf("failed to find a valid workspace: %w", err) +// dbtTypedSubcommand is the first argument that is neither a flag nor a +// value one took: astro dbt -n prod delete names delete. It stops at "--". +func dbtTypedSubcommand(args []string) string { + for i := 0; i < len(args); i++ { + a := args[i] + switch { + case a == "--": + return "" + case strings.HasPrefix(a, "--"): + if name := a[2:]; !strings.Contains(name, "=") && dbtValueFlags[name] { + i++ + } + case strings.HasPrefix(a, "-") && len(a) > 1: + // A group of shorthands: the first that takes a value takes the + // rest of the group (-nprod), or the next word when it is the + // group's last letter (-n prod, -wt 5m). + for j := 1; j < len(a); j++ { + if dbtValueFlags[a[j:j+1]] { + if j == len(a)-1 { + i++ + } + break + } + } + default: + return a } } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return errors.New("cannot use --wait-time with --wait=false") - } - - // get the deployment id to delete the dbt project - deploymentID, read, err := resolveBundleDeployment(args, workspaceID, deploymentName, noCreateUnderJSON(format, "to delete from")) - if err != nil { - return err - } - fmt.Println("Initiating dbt delete deploy for deployment ID: " + deploymentID) - - // if the mount path is not provided, derive it from the dbt project name - if mountPath == "" { - // if the dbt project path is not provided, use the current directory - if dbtProjectPath == "" { - dbtProjectPath = config.WorkingPath - } - - // check that there is a valid dbt project at the dbt project path - err := validateDbtProjectExists(dbtProjectPath) - if err != nil { - return err - } - - // extract the dbt project's name - dbtProjectName, err := extractDbtProjectName(dbtProjectPath) - if err != nil { - return fmt.Errorf("dbt project name not found in %s: %w", dbtProjectPath, err) - } - - mountPath = dbtDefaultMountPathPrefix + dbtProjectName - } - - deleteBundleInput := &astrodeploy.DeleteBundleInput{ - MountPath: mountPath, - DeploymentID: deploymentID, - Deployment: read, - BundleType: dbtBundleType, - Description: deployDescription, - AstroV1Client: astroV1Client, - } - res, err := DeleteBundle(deleteBundleInput) - if err != nil { - return err - } - r := cliout.Renderer{Format: format, Out: cmd.OutOrStdout()} - return publishThenWait(cmd, format, waitForDeploy, res.DeploymentID, waitTime, func(waitErr error) error { - return r.Emit(newDbtDeleteJSON(&res, waitForDeploy, waitErr), renderDbtDeleted(res.MountPath)) - }) -} - -func validateDbtProjectExists(dbtProjectPath string) error { - dbtProjectYamlPath := filepath.Join(dbtProjectPath, dbtProjectYmlFilename) - - _, err := os.Stat(dbtProjectYamlPath) - if os.IsNotExist(err) { - return fmt.Errorf("dbt project file not found at %s. Please run this command in the root of your dbt project, or use --project-path to specify the dbt project path", dbtProjectYamlPath) - } - return err -} - -func extractDbtProjectName(dbtProjectPath string) (string, error) { - dbtProjectYamlPath := filepath.Join(dbtProjectPath, dbtProjectYmlFilename) - - var dbtProject map[string]interface{} - file, err := os.Open(dbtProjectYamlPath) - if err != nil { - return "", fmt.Errorf("could not open %s: %w", dbtProjectYamlPath, err) - } - defer file.Close() - decoder := yaml.NewDecoder(file) - err = decoder.Decode(&dbtProject) - if err != nil { - return "", fmt.Errorf("could not decode %s: %w", dbtProjectYamlPath, err) - } - - dbtProjectName, ok := dbtProject["name"].(string) - if !ok || dbtProjectName == "" { - return "", errors.New("invalid dbt project name") - } - - return dbtProjectName, nil -} - -// resolveBundleDeployment is the Deployment a bundle command acts on: the -// argument, --deployment, the one --deployment-name names, or the one picked -// from the Workspace's. It returns the id, and the Deployment itself when it -// read it (picked, or found by name), so the command need not read it again; -// read is nil for one named by id. -// -// noCreateFor decides what a Workspace with no Deployment does. Empty, the -// run walks through creating one, as text mode always has. Set (under -// --output json, where nothing may be asked), it is what the command would -// have done with the Deployment ("to deploy to"), and the run fails saying -// so, rather than asking a question answered by a --name this command does -// not have. -func resolveBundleDeployment(args []string, workspaceID, deploymentName, noCreateFor string) (id string, read *astrov1.Deployment, err error) { - // if provided, use the deployment ID from the command argument - if len(args) > 0 { - return args[0], nil, nil - } - // or the one given as --deployment - if deploymentArg != "" { - return deploymentArg, nil, nil - } - - // otherwise, prompt the user to select a deployment - selectedDeployment, err := deployment.GetDeployment(workspaceID, "", deploymentName, noCreateFor != "", nil, astroV1Client) - if err != nil { - return "", nil, err - } - if selectedDeployment.Id == "" { - return "", nil, deployment.ErrNothingTo(workspaceID, "", deploymentName, noCreateFor) - } - return selectedDeployment.Id, &selectedDeployment, nil -} - -// noCreateUnderJSON is resolveBundleDeployment's noCreateFor for a command -// in format: what under json, and nothing (the create walk-through) in text. -func noCreateUnderJSON(format cliout.Format, what string) string { - if format == cliout.FormatJSON { - return what - } return "" } diff --git a/cmd/astro/dbt_output_test.go b/cmd/astro/dbt_output_test.go deleted file mode 100644 index c43b1b673..000000000 --- a/cmd/astro/dbt_output_test.go +++ /dev/null @@ -1,471 +0,0 @@ -package astro - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - "time" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/astronomer/astro-cli/cmd/cliout" - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" -) - -const ( - dbtTestDeploymentID = "clxdbtdeployment000000000" - dbtTestDeployID = "clxdbtdeploy0000000000000" - dbtTestVersion = "2026-10-07T00:00:00.0000000Z" -) - -// fakeBlobStore answers the bundle upload the way Azure does: every request -// succeeds, and the commit reports the version the upload created. -func fakeBlobStore(t *testing.T) string { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) - w.Header().Set("x-ms-version-id", dbtTestVersion) - w.WriteHeader(http.StatusCreated) - })) - t.Cleanup(srv.Close) - return srv.URL + "/container/bundle.tar.gz?sig=x" -} - -// dbtProject writes a dbt project named jaffle_shop in a fresh directory outside -// any Astro project, and makes a second fresh directory the working path, so -// the bundle's tarball is written beside it rather than inside it. -func dbtProject(t *testing.T) string { - t.Helper() - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte("name: jaffle_shop\n"), 0o600)) - prev := config.WorkingPath - config.WorkingPath = t.TempDir() - t.Cleanup(func() { config.WorkingPath = prev }) - return dir -} - -func dbtTestDeployment() astrov1.Deployment { - return astrov1.Deployment{ - Id: dbtTestDeploymentID, - Name: "analytics", - WorkspaceId: "clxdbtworkspace0000000000", - IsDagDeployEnabled: true, - Status: astrov1.DeploymentStatusHEALTHY, - } -} - -// dbtBundleMock is the API a bundle deploy or delete talks to. -func dbtBundleMock(t *testing.T, uploadURL string) *astrov1_mocks.ClientWithResponsesInterface { - t.Helper() - m := new(astrov1_mocks.ClientWithResponsesInterface) - t.Cleanup(func() { m.AssertExpectations(t) }) - d := dbtTestDeployment() - m.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID).Return(&astrov1.GetDeploymentResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &d, - }, nil).Maybe() - deploy := astrov1.Deploy{Id: dbtTestDeployID} - if uploadURL != "" { - deploy.BundleUploadUrl = &uploadURL - } - m.On("CreateDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, mock.Anything).Return(&astrov1.CreateDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &deploy, - }, nil).Once() - m.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, dbtTestDeployID, mock.Anything).Return(&astrov1.FinalizeDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &deploy, - }, nil).Once() - return m -} - -func execDbtRun(t *testing.T, client astrov1.APIClient, args ...string) tokenRun { - t.Helper() - return execAstroCmd(t, client, "", newDbtCmd, append([]string{"dbt"}, args...)...) -} - -// What v2 printed on stdout in text mode, recorded before --output json -// existed on these commands. Text output may not change, so these are pinned -// byte for byte. -func TestDbtTextOutputUnchanged(t *testing.T) { - t.Run("deploy", func(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--project-path", project) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Initiating dbt deploy for deployment ID: "+dbtTestDeploymentID+"\n"+ - "Generated mount path from dbt project name: /usr/local/airflow/dbt/jaffle_shop\n"+ - "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) - }) - t.Run("deploy --mount-path", func(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--project-path", project, "--mount-path", "/usr/local/airflow/dbt/custom") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Initiating dbt deploy for deployment ID: "+dbtTestDeploymentID+"\n"+ - "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) - }) - t.Run("delete", func(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, ""), "delete", dbtTestDeploymentID, "--project-path", project) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Initiating dbt delete deploy for deployment ID: "+dbtTestDeploymentID+"\n"+ - "Successfully requested bundle delete for mount path /usr/local/airflow/dbt/jaffle_shop from Astro.\n", r.stdout) - }) - // astro deploy --non-dags uploads a bundle the same way, and prints the - // same line. - t.Run("astro deploy --non-dags", func(t *testing.T) { - bundle := dbtProject(t) - r := execAstroCmd(t, dbtBundleMock(t, fakeBlobStore(t)), "", func(io.Writer) *cobra.Command { return NewDeployCmd() }, - "deploy", dbtTestDeploymentID, "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", bundle) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) - }) -} - -// stubBundleWait stands in for the wait a --wait does, writing one line of -// progress where the real one writes its own and answering err. -func stubBundleWait(t *testing.T, err error) *[]string { - t.Helper() - var waited []string - prev := waitForBundle - waitForBundle = func(progress io.Writer, deploymentID string, _ time.Duration, _ astrov1.APIClient) error { - waited = append(waited, deploymentID) - fmt.Fprintln(progress, "Waiting for the deployment to become healthy…") - return err - } - t.Cleanup(func() { waitForBundle = prev }) - return &waited -} - -func TestDbtDeployJSON(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--project-path", project, "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - - var got dbtDeployJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeploymentID, got.Deployment) - assert.Equal(t, "analytics", got.DeploymentName) - assert.Equal(t, "clxdbtworkspace0000000000", got.Workspace) - assert.Equal(t, dbtTestDeployID, got.DeployID) - assert.Equal(t, "jaffle_shop", got.Project) - assert.Equal(t, project, got.ProjectPath) - assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", got.MountPath) - assert.Equal(t, dbtTestVersion, got.BundleVersion) - assert.False(t, got.Waited) - assert.Nil(t, got.Git, "the project is in no git checkout") -} - -// A Deployment found by name is read once, by the lookup, and the deploy -// uses what the lookup read rather than reading it again. -func TestDbtDeployReadsANamedDeploymentOnce(t *testing.T) { - project := dbtProject(t) - m := dbtBundleMock(t, fakeBlobStore(t)) - m.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(listDeploymentsResp(dbtTestDeployment()), nil).Once() - - r := execDbtRun(t, m, "deploy", "--deployment-name", "analytics", "--project-path", project, "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got dbtDeployJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeploymentID, got.Deployment) - m.AssertNumberOfCalls(t, "GetDeploymentWithResponse", 1) -} - -// A --wait's progress is on stderr in both formats. In text the result comes -// first, as it always has, so stdout is what it is without --wait; under json -// the object comes after the wait it reports, and says it waited. -func TestDbtWait(t *testing.T) { - cases := []struct { - name string - upload bool - }{ - {"deploy", true}, - {"delete", false}, - } - for _, tc := range cases { - args := func(project string, more ...string) []string { - return append([]string{tc.name, dbtTestDeploymentID, "--project-path", project}, more...) - } - client := func(t *testing.T) *astrov1_mocks.ClientWithResponsesInterface { - if tc.upload { - return dbtBundleMock(t, fakeBlobStore(t)) - } - return dbtBundleMock(t, "") - } - t.Run(tc.name+" text", func(t *testing.T) { - project := dbtProject(t) - without := execDbtRun(t, client(t), args(project)...) - require.NoError(t, without.err) - - waited := stubBundleWait(t, nil) - r := execDbtRun(t, client(t), args(project, "--wait")...) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, []string{dbtTestDeploymentID}, *waited) - assert.Equal(t, without.stdout, r.stdout, "stdout is the result, the same with --wait as without") - assert.Contains(t, r.stderr, "Waiting for the deployment to become healthy…") - }) - t.Run(tc.name+" text, the wait failing", func(t *testing.T) { - project := dbtProject(t) - stubBundleWait(t, errors.New("timed out")) - r := execDbtRun(t, client(t), args(project, "--wait")...) - require.Error(t, r.err) - assert.Contains(t, r.stdout, "Successfully", "the result is printed before the wait begins") - }) - t.Run(tc.name+" json", func(t *testing.T) { - project := dbtProject(t) - waited := stubBundleWait(t, nil) - r := execDbtRun(t, client(t), args(project, "--wait", "-o", "json")...) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, []string{dbtTestDeploymentID}, *waited) - var got struct { - Waited bool `json:"waited"` - } - fields := decodeOne(t, r.stdout, &got) - assert.True(t, got.Waited) - assert.NotContains(t, fields, "wait_error", "a wait that succeeded has no error") - assert.Contains(t, r.stderr, "Waiting for the deployment to become healthy…") - }) - t.Run(tc.name+" json, the wait failing", func(t *testing.T) { - project := dbtProject(t) - stubBundleWait(t, errors.New("timed out")) - r := execDbtRun(t, client(t), args(project, "--wait", "-o", "json")...) - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - // The upload happened, so the result is still published, saying - // it waited and why the wait failed, and the run fails. - var got struct { - DeployID string `json:"deploy_id"` - Waited bool `json:"waited"` - WaitError string `json:"wait_error"` - } - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeployID, got.DeployID) - assert.True(t, got.Waited, "the wait was attempted") - assert.Equal(t, "timed out", got.WaitError) - assert.Contains(t, r.stderr, "Error: timed out") - }) - } -} - -func TestDbtDeleteJSON(t *testing.T) { - // Named by id, the Deployment is never read, so its Workspace is not - // known and not published. - t.Run("by id", func(t *testing.T) { - r := execDbtRun(t, dbtBundleMock(t, ""), "delete", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/dbt/custom", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - - var got dbtDeleteJSON - fields := decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtDeleteJSON{ - Action: "deleted", - Deployment: dbtTestDeploymentID, - DeployID: dbtTestDeployID, - MountPath: "/usr/local/airflow/dbt/custom", - }, got) - assert.NotContains(t, fields, "workspace") - assert.NotContains(t, fields, "wait_error") - }) - // Named by name, it was read from the Workspace, and its Workspace is - // published. - t.Run("by name", func(t *testing.T) { - m := dbtBundleMock(t, "") - m.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(listDeploymentsResp(dbtTestDeployment()), nil).Once() - r := execDbtRun(t, m, "delete", "--deployment-name", "analytics", "--mount-path", "/usr/local/airflow/dbt/custom", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - - var got dbtDeleteJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeploymentID, got.Deployment) - assert.Equal(t, "clxdbtworkspace0000000000", got.Workspace) - }) -} - -// A wait that fails and a result that then cannot be written are both -// reported: neither error hides the other. -func TestPublishThenWaitReportsBothFailures(t *testing.T) { - timedOut := errors.New("timed out") - stubBundleWait(t, timedOut) - writeFailed := errors.New("broken pipe") - - var stdout, stderr bytes.Buffer - root := &cobra.Command{Use: "astro"} - var format cliout.Format - cmd := &cobra.Command{Use: "x", RunE: func(cmd *cobra.Command, _ []string) error { - return publishThenWait(cmd, cliout.FormatJSON, true, dbtTestDeploymentID, time.Second, func(waitErr error) error { - assert.Same(t, timedOut, waitErr) - return writeFailed - }) - }} - cliout.AddOutputFlag(cmd, &format) - root.AddCommand(cmd) - root.SetOut(&stdout) - root.SetErr(&stderr) - - err := cliout.Execute(context.Background(), root, []string{"x", "-o", "json"}, &stdout, nil) - - require.ErrorIs(t, err, writeFailed) - require.ErrorIs(t, err, timedOut) - assert.Empty(t, stdout.String(), "the write to stdout failed, so no error object is written there after it") - assert.Equal(t, "Waiting for the deployment to become healthy…\nError: broken pipe\ntimed out\n", stderr.String(), "after the wait's progress, the write's failure first, then the wait's") - assert.Equal(t, cliout.ExitFailure, cliout.ExitCode(context.Background(), err)) -} - -// astro dbt cleanup is a tombstone: it touches nothing, and says why in text -// and as the one error object under json, whatever paths and flags it gets. -func TestDbtCleanupRemoved(t *testing.T) { - dir := t.TempDir() - artifact := filepath.Join(dir, ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(artifact), 0o755)) - require.NoError(t, os.WriteFile(artifact, []byte(`{"generated_by": {"application": "astro"}}`), 0o600)) - - r := execDbtRun(t, nil, "cleanup", dir) - require.Error(t, r.err) - assert.Equal(t, cliout.ExitUsage, r.code) - assert.Equal(t, errDbtCleanupRemoved, r.err.Error()) - assert.Contains(t, r.err.Error(), "no longer writes Cosmos Boost files") - assert.NotContains(t, r.err.Error(), "`", "command names are plain text") - assert.Empty(t, r.stdout) - assert.FileExists(t, artifact, "the tombstone removes nothing") - - r = execDbtRun(t, nil, "cleanup", dir, "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitUsage, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, errDbtCleanupRemoved, got.Error) - assert.FileExists(t, artifact) -} - -// Under --output json the deployment picker is refused as input_required, -// naming --deployment, before anything is uploaded or deleted. A Workspace -// with no Deployment fails instead of walking through a create, whose name -// question would name a --name these commands do not have. -func TestDbtJSONNeverAsks(t *testing.T) { - two := []astrov1.Deployment{dbtTestDeployment(), dbtTestDeployment()} - two[1].Id, two[1].Name = "clxdbtother00000000000000", "other" - for _, sub := range []string{"deploy", "delete"} { - t.Run(sub+" with several Deployments", func(t *testing.T) { - project := dbtProject(t) - r := execAstroCmd(t, coreMock(t, two...), "1\n", newDbtCmd, "dbt", sub, "--project-path", project, "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, string(cliout.KindInputRequired), got.Kind) - assert.Contains(t, got.Error, "pass --deployment") - }) - t.Run(sub+" with no Deployment", func(t *testing.T) { - project := dbtProject(t) - r := execAstroCmd(t, coreMock(t), "", newDbtCmd, "dbt", sub, "--project-path", project, "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "no Deployments found in workspace") - }) - // Named, it is a Deployment that was not found, as in a Workspace - // that has others. - t.Run(sub+" naming a Deployment, with none", func(t *testing.T) { - project := dbtProject(t) - r := execAstroCmd(t, coreMock(t), "", newDbtCmd, "dbt", sub, "--project-path", project, "--deployment-name", "analytics", "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "the Deployment specified was not found") - }) - } -} - -// remote deploy's closing lines, as v2 printed them when the push succeeded. -const remoteDeployTail = "Successfully pushed client image to registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + - "\n--------------------------------\n" + - "The client image has been pushed to your private registry.\n" + - "Your next step would be to update the agent component to use the new client image.\n" + - "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.\n" + - "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + - "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components\n" - -// stubClientDeploy stands in for the build and the push, printing one of the -// progress lines the real one prints on the way. -func stubClientDeploy(t *testing.T, res *astrodeploy.ClientDeploy, err error) { - t.Helper() - prev := deployClientImage - deployClientImage = func(astrodeploy.InputClientDeploy, astrov1.APIClient) (astrodeploy.ClientDeploy, error) { - fmt.Println("Pushing client image to configured remote registry") - return *res, err - } - t.Cleanup(func() { deployClientImage = prev }) -} - -func execRemoteRun(t *testing.T, args ...string) tokenRun { - t.Helper() - // remote deploy runs in a project. - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) - prev := config.WorkingPath - config.WorkingPath = dir - t.Cleanup(func() { config.WorkingPath = prev }) - return execAstroCmd(t, nil, "", newRemoteRootCmd, append([]string{"remote"}, args...)...) -} - -func TestRemoteDeployOutput(t *testing.T) { - pushed := astrodeploy.ClientDeploy{ - Image: "registry.example.com/agents/client:deploy-2026-10-07T14-53", - Registry: "registry.example.com/agents/client", - Tag: "deploy-2026-10-07T14-53", - Platforms: []string{"linux/amd64", "linux/arm64"}, - RuntimeCheck: &astrodeploy.ClientRuntimeCheck{ - DeploymentID: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2", - }, - } - t.Run("text", func(t *testing.T) { - stubClientDeploy(t, &pushed, nil) - r := execRemoteRun(t, "deploy") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Pushing client image to configured remote registry\n"+remoteDeployTail, r.stdout) - }) - t.Run("json", func(t *testing.T) { - stubClientDeploy(t, &pushed, nil) - r := execRemoteRun(t, "deploy", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got remoteDeployJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, "registry.example.com/agents/client:deploy-2026-10-07T14-53", got.Image) - assert.Equal(t, "deploy-2026-10-07T14-53", got.Tag) - assert.Equal(t, []string{"linux/amd64", "linux/arm64"}, got.Platforms) - require.NotNil(t, got.RuntimeCheck) - assert.Equal(t, remoteRuntimeCheckJSON{Deployment: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2"}, *got.RuntimeCheck) - }) - t.Run("json, a prebuilt image", func(t *testing.T) { - stubClientDeploy(t, &astrodeploy.ClientDeploy{Image: "r/c:deploy-x", Registry: "r/c", Tag: "deploy-x", SourceImage: "local:1"}, nil) - r := execRemoteRun(t, "deploy", "--image-name", "local:1", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got remoteDeployJSON - fields := decodeOne(t, r.stdout, &got) - assert.Equal(t, "local:1", got.SourceImage) - assert.JSONEq(t, "[]", string(fields["platforms"])) - assert.NotContains(t, fields, "runtime_check") - }) - t.Run("json, the push failing", func(t *testing.T) { - stubClientDeploy(t, &astrodeploy.ClientDeploy{}, errors.New("failed to push client image: denied")) - r := execRemoteRun(t, "deploy", "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "denied") - }) -} diff --git a/cmd/astro/dbt_render.go b/cmd/astro/dbt_render.go deleted file mode 100644 index a0b2991b8..000000000 --- a/cmd/astro/dbt_render.go +++ /dev/null @@ -1,230 +0,0 @@ -package astro - -// Rendering for `astro dbt deploy|delete`, `astro deploy --non-dags` -// and `astro remote deploy`. The platform packages return what happened; this -// file decides how it looks, in text and in json. - -import ( - "bufio" - "errors" - "fmt" - "io" - "time" - - "github.com/spf13/cobra" - - "github.com/astronomer/astro-cli/cmd/cliout" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - "github.com/astronomer/astro-cli/pkg/ansi" -) - -// dbtOutput is --output for the `dbt` family. -var dbtOutput cliout.Format - -// waitForBundle waits for a Deployment to take a bundle deploy or delete. A -// var so a test need not sit through the poll's sleeps. -var waitForBundle = astrodeploy.WaitForBundle - -// dbtDeployJSON is what `astro dbt deploy --output json` publishes: what was -// deployed where. Its deployment, workspace and git keys are the ones -// `astro deploy`'s result uses for the same facts. -type dbtDeployJSON struct { - Deployment string `json:"deployment"` - DeploymentName string `json:"deployment_name"` - Workspace string `json:"workspace"` - // DeployID is the deploy record the upload created on the Deployment. - DeployID string `json:"deploy_id"` - // Project is the dbt project's name, from its dbt_project.yml, and - // ProjectPath the directory bundled. - Project string `json:"project"` - ProjectPath string `json:"project_path"` - MountPath string `json:"mount_path"` - // BundleVersion is the version of the uploaded bundle. - BundleVersion string `json:"bundle_version"` - // Waited is true when the run waited for the Deployment to become - // healthy (--wait), and WaitError says why that wait failed; it is - // omitted when the wait succeeded or none was asked for. - Waited bool `json:"waited"` - WaitError string `json:"wait_error,omitempty"` - Git *deployGitJSON `json:"git,omitempty"` -} - -// dbtDeleteJSON is what `astro dbt delete --output json` publishes: the -// bundle it removed from the Deployment. Astro removes the files as the -// Deployment takes the deploy, which a wait (waited, and wait_error when it -// failed) watches for. workspace is the Deployment's, omitted when the -// Deployment was named by id and so never read. -type dbtDeleteJSON struct { - Action string `json:"action"` - Deployment string `json:"deployment"` - Workspace string `json:"workspace,omitempty"` - DeployID string `json:"deploy_id"` - MountPath string `json:"mount_path"` - Waited bool `json:"waited"` - WaitError string `json:"wait_error,omitempty"` -} - -// remoteDeployJSON is what `astro remote deploy --output json` publishes: the -// client image it pushed, which is what the agents' Helm values then name. -type remoteDeployJSON struct { - Image string `json:"image"` - Registry string `json:"registry"` - Tag string `json:"tag"` - // SourceImage is the local image pushed (--image-name), omitted when the - // run built one. - SourceImage string `json:"source_image,omitempty"` - // Platforms are the --platform the image was built for, [] for the host's - // or when nothing was built. - Platforms []string `json:"platforms"` - // RuntimeCheck is the check against --deployment, omitted without one. - RuntimeCheck *remoteRuntimeCheckJSON `json:"runtime_check,omitempty"` -} - -type remoteRuntimeCheckJSON struct { - Deployment string `json:"deployment"` - ClientRuntimeVersion string `json:"client_runtime_version"` - DeploymentRuntimeVersion string `json:"deployment_runtime_version"` -} - -func newDbtDeployJSON(res *astrodeploy.BundleDeploy, project, projectPath string, waited bool, waitErr error) dbtDeployJSON { - return dbtDeployJSON{ - Deployment: res.DeploymentID, - DeploymentName: res.DeploymentName, - Workspace: res.WorkspaceID, - DeployID: res.DeployID, - Project: project, - ProjectPath: projectPath, - MountPath: res.MountPath, - BundleVersion: res.BundleVersion, - Waited: waited, - WaitError: errText(waitErr), - Git: bundleGitJSON(res.Git), - } -} - -// bundleGitJSON is the commit a bundle deploy recorded, as a deploy result -// publishes it; nil when it recorded none. -func bundleGitJSON(g *astrodeploy.BundleGit) *deployGitJSON { - if g == nil { - return nil - } - return &deployGitJSON{CommitSHA: g.CommitSHA, Branch: g.Branch, CommitURL: g.CommitURL} -} - -// renderBundleUploaded is the text a finished bundle upload has always -// printed. -func renderBundleUploaded(version string) func(io.Writer) error { - return cliout.Text(func(b *bufio.Writer) { - fmt.Fprintf(b, "Successfully uploaded bundle with version %s to Astro.\n", version) - }) -} - -// publishThenWait publishes a bundle deploy's or delete's result and, with -// --wait, waits for the Deployment to take it. publish is handed the wait's -// failure, nil when it succeeded or there was no wait. -// -// The order depends on the format. In text the result comes first, as it -// always has, so a person sees the upload succeed before the wait begins; -// the wait's progress goes to stderr, and its failure is the run's error. -// Under json the one object is the last thing the run writes, after the wait -// it reports on. A wait that fails there still publishes the result, saying -// why, because the upload happened and a script needs its deploy id and -// version, and then exits 1 (failedAfterResult). -func publishThenWait(cmd *cobra.Command, format cliout.Format, wait bool, deploymentID string, waitTime time.Duration, publish func(waitErr error) error) error { - return publishThenWaitWith(cmd, format, wait, func() error { - return waitForBundle(cmd.ErrOrStderr(), deploymentID, waitTime, astroV1Client) - }, publish) -} - -// publishThenWaitWith is publishThenWait with the wait its caller's, for a -// Deployment reached under a login other than the current context's. -func publishThenWaitWith(cmd *cobra.Command, format cliout.Format, wait bool, waitFor func() error, publish func(waitErr error) error) error { - waitDone := func() error { - if !wait { - return nil - } - return waitFor() - } - if format == cliout.FormatJSON { - waitErr := waitDone() - if err := publish(waitErr); err != nil { - // The write failed, so stdout is what broke: the root must not - // try to write the error object there too. failedAfterResult - // marks it shown and puts the words on stderr, the write's - // failure first and the wait's after it, so neither hides the - // other. - return failedAfterResult(cmd, format, errors.Join(err, waitErr)) - } - return failedAfterResult(cmd, format, waitErr) - } - if err := publish(nil); err != nil { - return err - } - return waitDone() -} - -// errText is err's message, or "" for nil. -func errText(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func newDbtDeleteJSON(res *astrodeploy.BundleDelete, waited bool, waitErr error) dbtDeleteJSON { - return dbtDeleteJSON{ - Action: "deleted", - Deployment: res.DeploymentID, - Workspace: res.WorkspaceID, - DeployID: res.DeployID, - MountPath: res.MountPath, - Waited: waited, - WaitError: errText(waitErr), - } -} - -func renderDbtDeleted(mountPath string) func(io.Writer) error { - return cliout.Text(func(b *bufio.Writer) { - fmt.Fprintf(b, "Successfully requested bundle delete for mount path %s from Astro.\n", mountPath) - }) -} - -func newRemoteDeployJSON(res *astrodeploy.ClientDeploy) remoteDeployJSON { - obj := remoteDeployJSON{ - Image: res.Image, - Registry: res.Registry, - Tag: res.Tag, - SourceImage: res.SourceImage, - Platforms: nonNil(res.Platforms), - } - if c := res.RuntimeCheck; c != nil { - obj.RuntimeCheck = &remoteRuntimeCheckJSON{ - Deployment: c.DeploymentID, - ClientRuntimeVersion: c.ClientRuntimeVersion, - DeploymentRuntimeVersion: c.DeploymentRuntimeVersion, - } - } - return obj -} - -// renderRemoteDeploy is the text a finished client image push has always -// printed: where it went, and what to do with it next. -func renderRemoteDeploy(image string) func(io.Writer) error { - return cliout.Text(func(b *bufio.Writer) { - fmt.Fprintf(b, "Successfully pushed client image to %s\n", ansi.Bold(image)) - fmt.Fprintf(b, "\n--------------------------------\n") - fmt.Fprintln(b, "The client image has been pushed to your private registry.") - fmt.Fprintln(b, "Your next step would be to update the agent component to use the new client image.") - fmt.Fprintln(b, "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.") - fmt.Fprintf(b, "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: %s\n", image) - fmt.Fprintln(b, "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components") - }) -} - -// nonNil is s, or [] for nil, so a list publishes as [] and never null. -func nonNil(s []string) []string { - if s == nil { - return []string{} - } - return s -} diff --git a/cmd/astro/dbt_test.go b/cmd/astro/dbt_test.go deleted file mode 100644 index 871f1d633..000000000 --- a/cmd/astro/dbt_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package astro - -import ( - "net/http" - "os" - "path/filepath" - "testing" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - testUtil "github.com/astronomer/astro-cli/pkg/testing" -) - -type DbtSuite struct { - suite.Suite - mockV1Client *astrov1_mocks.ClientWithResponsesInterface - origV1Client astrov1.APIClient - origDeployBundle func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) - origDeleteBundle func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) - origWorkingPath string - origWd string - tmpWorkingDir string -} - -func (s *DbtSuite) SetupTest() { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - // Run each test from an isolated temp directory that is not within an Astro - // project. dbt deploy/delete default the project path to config.WorkingPath - // and reject paths nested under an Astro project (detected by walking up the - // real filesystem for a .astro/config.yaml). Relying on the package's own - // directory makes the suite fail whenever an ancestor of the repo happens to - // be an Astro project, so we point WorkingPath/cwd at a clean temp dir. - tmpDir, err := os.MkdirTemp("", "dbt-test") - s.Require().NoError(err) - s.tmpWorkingDir = tmpDir - s.origWd, err = os.Getwd() - s.Require().NoError(err) - s.Require().NoError(os.Chdir(tmpDir)) - s.origWorkingPath = config.WorkingPath - config.WorkingPath = tmpDir - - // the package depends on global variables so we need to manage overriding those - s.origV1Client = astroV1Client - s.origDeployBundle = DeployBundle - s.origDeleteBundle = DeleteBundle - s.mockV1Client = new(astrov1_mocks.ClientWithResponsesInterface) - astroV1Client = s.mockV1Client -} - -func (s *DbtSuite) TearDownTest() { - s.mockV1Client.AssertExpectations(s.T()) - astroV1Client = s.origV1Client - DeployBundle = s.origDeployBundle - DeleteBundle = s.origDeleteBundle - - config.WorkingPath = s.origWorkingPath - if s.origWd != "" { - _ = os.Chdir(s.origWd) - } - if s.tmpWorkingDir != "" { - _ = os.RemoveAll(s.tmpWorkingDir) - } -} - -func TestDbt(t *testing.T) { - suite.Run(t, new(DbtSuite)) -} - -func (s *DbtSuite) TestDbtDeploy_PickDeployment() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - return astrodeploy.BundleDeploy{}, nil - } - - s.mockListTestDeployments() - s.mockGetTestDeployment() - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeployCmd()) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDeploy_ProvidedDeploymentId() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDeploy_CustomProjectPath() { - projectPath, err := os.MkdirTemp("", "") - assert.NoError(s.T(), err) - defer os.RemoveAll(projectPath) - - s.createDbtProjectFile(filepath.Join(projectPath, "dbt_project.yml")) - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - if deployInput.BundlePath != projectPath { - return astrodeploy.BundleDeploy{}, assert.AnError - } - return astrodeploy.BundleDeploy{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err = testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", projectPath) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDeploy_CustomMountPath() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - if deployInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDeploy{}, assert.AnError - } - return astrodeploy.BundleDeploy{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--mount-path", dbtDefaultMountPathPrefix+"test_dbt_project") - assert.NoError(s.T(), err) -} - -// A dbt project in or below a 1.x project, by the walk every deploy decides -// by (a Dockerfile beside .astro, with or without .astro/config.yaml), is -// refused, as astro deploy --non-dags refuses the same bundle path. -func (s *DbtSuite) TestDbtDeploy_WithinAstroProject() { - projectDir := s.T().TempDir() - s.Require().NoError(os.WriteFile(filepath.Join(projectDir, "Dockerfile"), []byte("FROM x\n"), 0o600)) - s.Require().NoError(os.MkdirAll(filepath.Join(projectDir, ".astro"), 0o755)) - s.Require().NoError(os.WriteFile(filepath.Join(projectDir, ".astro", "config.yaml"), []byte("project:\n name: demo\n"), 0o600)) - dbtDir := filepath.Join(projectDir, "include", "dbt") - s.Require().NoError(os.MkdirAll(dbtDir, 0o755)) - - for _, path := range []string{projectDir, dbtDir} { - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", path) - s.Require().Error(err, path) - s.Contains(err.Error(), "dbt project is within an Astro project", path) - - resetDeployFlagVars() - nonDags := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", path) - s.Require().Error(nonDags, path) - s.Contains(nonDags.Error(), "within an Astro project", "dbt deploy and --non-dags agree: %s", path) - } -} - -// A project with a pyproject.toml carries no .astro/config.yaml, so the 1.x walk answers false at -// every level of it. Before this, a dbt project nested inside one bundled and -// deployed instead of being refused. -func (s *DbtSuite) TestDbtDeploy_WithinManifestProject() { - projectDir := s.T().TempDir() - manifest := "[project]\nname = \"demo\"\n\n[tool.astro]\n" - assert.NoError(s.T(), os.WriteFile(filepath.Join(projectDir, "pyproject.toml"), []byte(manifest), 0o600)) - nested := filepath.Join(projectDir, "analytics") - assert.NoError(s.T(), os.MkdirAll(nested, 0o755)) - - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", nested) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "dbt project is within an Astro project") -} - -func (s *DbtSuite) TestDbtDelete_PickDeployment() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - return astrodeploy.BundleDelete{}, nil - } - - s.mockListTestDeployments() - s.mockGetTestDeployment() - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd()) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_ProvidedDeploymentId() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - return astrodeploy.BundleDelete{}, nil - } - - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_CustomProjectPath() { - projectPath, err := os.MkdirTemp("", "") - assert.NoError(s.T(), err) - defer os.RemoveAll(projectPath) - - s.createDbtProjectFile(filepath.Join(projectPath, "dbt_project.yml")) - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - if deleteInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDelete{}, assert.AnError - } - return astrodeploy.BundleDelete{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err = testExecCmd(newDbtDeleteCmd(), "test-deployment-id", "--project-path", projectPath) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_NoMountPathOrProjectPath() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - if deleteInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDelete{}, assert.AnError - } - return astrodeploy.BundleDelete{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_NoMountPathOrProjectPath_MissingProject() { - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id") - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "dbt project file not found") -} - -func (s *DbtSuite) TestDbtDelete_CustomMountPath() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - if deleteInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDelete{}, assert.AnError - } - return astrodeploy.BundleDelete{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id", "--mount-path", dbtDefaultMountPathPrefix+"test_dbt_project") - assert.NoError(s.T(), err) -} - -func testExecCmd(cmd *cobra.Command, args ...string) error { - if args == nil { - args = []string{} - } - // deploy and dbt set the package's workspaceID from --workspace-id or the - // one they resolve, and coalesceWorkspace reads it before the context. - // Left set, it would choose the Workspace of whatever runs next. - defer func() { workspaceID = "" }() - testUtil.SetupOSArgsForGinkgo() - cmd.SetArgs(args) - _, err := cmd.ExecuteC() - return err -} - -func (s *DbtSuite) createDbtProjectFile(path string) { - file, err := os.Create(path) - assert.NoError(s.T(), err) - defer file.Close() - _, err = file.WriteString("name: test_dbt_project") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) mockListTestDeployments() { - s.mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&astrov1.ListDeploymentsResponse{ - HTTPResponse: &http.Response{ - StatusCode: http.StatusOK, - }, - JSON200: &astrov1.DeploymentsPaginated{ - Deployments: []astrov1.Deployment{ - { - Id: "test-deployment-id", - }, - }, - }, - }, nil) -} - -func (s *DbtSuite) mockGetTestDeployment() { - s.mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&astrov1.GetDeploymentResponse{ - HTTPResponse: &http.Response{ - StatusCode: http.StatusOK, - }, - JSON200: &astrov1.Deployment{ - Id: "test-deployment-id", - }, - }, nil) -} diff --git a/cmd/astro/deploy.go b/cmd/astro/deploy.go index 0177127b5..7bed0a6c6 100644 --- a/cmd/astro/deploy.go +++ b/cmd/astro/deploy.go @@ -2,7 +2,6 @@ package astro import ( "bufio" - "cmp" "context" goerrors "errors" "fmt" @@ -46,10 +45,6 @@ var ( imageName string deployDescription string noDagsBaseDir bool - nonDags bool - nonDagsMountPath string - nonDagsBundleType string - nonDagsBundlePath string manifestDeployment string manifestWorkspace string deployOutput string @@ -63,7 +58,10 @@ var ( astro deploy --dags # Deploy an image built on this machine, from any directory - astro deploy --image-name ` + astro deploy --image-name + + # Deploy a dbt project, or another non-DAG bundle, from its own directory + astro deploy bundle --deployment ` buildSecrets = []string{} ) @@ -72,14 +70,13 @@ const ( deployWaitTime = 300 * time.Second imageNameFlag = "image-name" - nonDagsFlag = "non-dags" ) func NewDeployCmd() *cobra.Command { cmd := &cobra.Command{ Use: "deploy [DEPLOYMENT_ID]", Short: "Ship this project's code to a Deployment", - Long: "Deploy your project to a Deployment on Astro. Run it in a project with a pyproject.toml ([tool.astro]): it builds the project into a Docker image and pushes that image, with the project's DAGs, to Astronomer. A project in the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml) is converted with astro init, or deployed with Astro CLI 1.x. With --image-name it deploys an image built on this machine, from any directory. In Deployments with Remote Execution enabled, this only updates the Orchestration Plane components (the API Server and Scheduler). For all other components, use astro remote deploy instead. It does not include any metadata associated with your local Airflow environment.", + Long: "Deploy your project to a Deployment on Astro. Run it in a project with a pyproject.toml ([tool.astro]): it builds the project into a Docker image and pushes that image, with the project's DAGs, to Astronomer. A project in the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml) is converted with astro init, or deployed with Astro CLI 1.x. With --image-name it deploys an image built on this machine, from any directory. In Deployments with Remote Execution enabled, this only updates the Orchestration Plane components (the API Server and Scheduler). For all other components, use astro remote deploy instead. It does not include any metadata associated with your local Airflow environment. To deploy a dbt project or another non-DAG bundle, use astro deploy bundle.", Args: cobra.MaximumNArgs(1), RunE: deploy, Example: deployExample, @@ -111,22 +108,17 @@ func NewDeployCmd() *cobra.Command { cmd.Flags().StringVar(&deployOutput, "output", string(cliout.FormatText), "Output format: text or json") cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Add a description for more context on this deploy") utils.AddBuildSecretFlag(cmd.Flags(), &buildSecrets) - cmd.Flags().BoolVar(&nonDags, nonDagsFlag, false, "Deploy a non-Dag bundle from a separate directory, instead of your Astro project. Requires --non-dags-mount-path") - cmd.Flags().StringVar(&nonDagsMountPath, "non-dags-mount-path", "", "Path to mount the non-Dag bundle in Airflow, for reference by Dags. Used with --non-dags") - cmd.Flags().StringVar(&nonDagsBundleType, "non-dags-bundle-type", "none", "Free-form label identifying the kind of non-Dag bundle (e.g. dbt). Any value is accepted. Defaults to \"none\". Used with --non-dags") - cmd.Flags().StringVar(&nonDagsBundlePath, "non-dags-local-path", "", "Path to the non-Dag bundle to deploy. Default current directory. Used with --non-dags") annotateDeployFlag(cmd, "image", deployGroupImage) annotateDeployFlag(cmd, imageNameFlag, deployGroupImage) annotateDeployFlag(cmd, "build-secret", deployGroupImage) annotateDeployFlag(cmd, "dags", deployGroupDAG) annotateDeployFlag(cmd, "no-dags-base-dir", deployGroupDAG) - annotateDeployFlag(cmd, nonDagsFlag, deployGroupNonDAG) - annotateDeployFlag(cmd, "non-dags-mount-path", deployGroupNonDAG) - annotateDeployFlag(cmd, "non-dags-bundle-type", deployGroupNonDAG) - annotateDeployFlag(cmd, "non-dags-local-path", deployGroupNonDAG) orderDeployFlagGroups(cmd) utils.MarkPreferredFlag(cmd.Flags(), "workspace", "workspace-id") + // astro deploy bundle is the subcommand, so the word bundle is never a + // Deployment argument: a link of that name is reached with --deployment. + cmd.AddCommand(newDeployBundleCmd()) applyPreferredFlagsIn(cmd) return cmd } @@ -134,10 +126,7 @@ func NewDeployCmd() *cobra.Command { // deploy ships a project with a pyproject.toml ([tool.astro]), the one kind // of project v2 deploys (deployManifest), run from the project's root. // -// --non-dags deploys a directory that must not be inside a project at all, so -// where it runs from says nothing about the project. It is checked first. -// -// Everywhere else the project decides, by utils.Locate's walk. In or below a +// The project decides, by utils.Locate's walk. In or below a // project in the Astro CLI 1.x layout every mode is refused, naming the two // ways forward: convert it with astro init, or deploy it with Astro CLI 1.x, // which keeps deploying that layout. --image-name is refused there too: a @@ -150,9 +139,6 @@ func NewDeployCmd() *cobra.Command { // Deployment that takes DAG deploys keeps the DAGs it had, and the deploy // says so. func deploy(cmd *cobra.Command, args []string) error { - if nonDags { - return deployNonDagsBundle(cmd, args) - } // One walk routes the deploy and words its refusal. where := utils.Locate(config.WorkingPath) if where.AtRoot(config.WorkingPath) { @@ -195,193 +181,6 @@ func refuseFlagCombinations(cmd *cobra.Command, inProject bool) error { return nil } -// nonDagsDeployJSON is what `astro deploy --non-dags --output json` -// publishes: the bundle deployed and where. Its keys are the ones -// `astro dbt deploy`'s result uses for the same facts. -type nonDagsDeployJSON struct { - Deployment string `json:"deployment"` - DeploymentName string `json:"deployment_name"` - Workspace string `json:"workspace"` - DeployID string `json:"deploy_id"` - // BundleType is --non-dags-bundle-type, BundlePath the directory bundled. - BundleType string `json:"bundle_type"` - BundlePath string `json:"bundle_path"` - MountPath string `json:"mount_path"` - BundleVersion string `json:"bundle_version"` - // Waited is true when the run waited for the Deployment to become - // healthy (--wait), and WaitError says why that wait failed. - Waited bool `json:"waited"` - WaitError string `json:"wait_error,omitempty"` - Git *deployGitJSON `json:"git,omitempty"` -} - -func newNonDagsDeployJSON(res *astrodeploy.BundleDeploy, bundleType, bundlePath string, waited bool, waitErr error) nonDagsDeployJSON { - return nonDagsDeployJSON{ - Deployment: res.DeploymentID, - DeploymentName: res.DeploymentName, - Workspace: res.WorkspaceID, - DeployID: res.DeployID, - BundleType: bundleType, - BundlePath: bundlePath, - MountPath: res.MountPath, - BundleVersion: res.BundleVersion, - Waited: waited, - WaitError: errText(waitErr), - Git: bundleGitJSON(res.Git), - } -} - -func deployNonDagsBundle(cmd *cobra.Command, args []string) error { - // Read first, so a bad --output is the usage error reported. - format, err := cliout.ParseFormat(deployOutput) - if err != nil { - return err - } - // The result goes where stdout is now. What the platform code prints on - // the way (the git note, the archive's) is the output text mode has always - // shown; under json it goes to stderr, as astro dbt deploy's does. - out := cmd.OutOrStdout() - defer strayStdoutToStderr(format)() - - for _, f := range []string{"dags", "image", imageNameFlag, "build-secret", "no-dags-base-dir"} { - if cmd.Flags().Changed(f) { - return cliout.Usage(fmt.Errorf("cannot use --%s with --non-dags; --non-dags performs a non-Dag bundle deploy", f)) - } - } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return cliout.Usage(errors.New("cannot use --wait-time with --wait=false")) - } - - if nonDagsMountPath == "" { - return cliout.Usage(errors.New("--non-dags-mount-path is required with --non-dags")) - } - - if nonDagsBundlePath == "" { - nonDagsBundlePath = config.WorkingPath - } - - info, err := os.Stat(nonDagsBundlePath) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("bundle path %s does not exist", nonDagsBundlePath) - } - return fmt.Errorf("failed to access bundle path %s: %w", nonDagsBundlePath, err) - } - if !info.IsDir() { - return fmt.Errorf("bundle path %s is not a directory", nonDagsBundlePath) - } - - // The walk every deploy decides by: a pyproject.toml project or a 1.x - // one at or above the bundle path. - if !utils.Locate(nonDagsBundlePath).None() { - return errors.New("bundle path is within an Astro project. Non-Dag bundles must be a separate directory") - } - - targetID, target, login, err := nonDagsTarget(cmd.Context(), args, format) - if err != nil { - return err - } - - cmd.SilenceUsage = true - - deployBundleInput := &astrodeploy.DeployBundleInput{ - BundlePath: nonDagsBundlePath, - MountPath: nonDagsMountPath, - DeploymentID: targetID, - Deployment: target, - BundleType: nonDagsBundleType, - Description: deployDescription, - AstroV1Client: login.client, - } - if !login.current { - deployBundleInput.Login = &login.context - } - res, err := DeployBundle(deployBundleInput) - if err != nil { - return err - } - waitDone := func() error { - if login.current { - return waitForBundle(cmd.ErrOrStderr(), res.DeploymentID, waitTime, login.client) - } - return astrodeploy.WaitForBundleIn(cmd.ErrOrStderr(), &login.context, res.DeploymentID, waitTime, login.client) - } - return publishThenWaitWith(cmd, format, waitForDeploy, waitDone, func(waitErr error) error { - if format == cliout.FormatJSON { - return cliout.Renderer{Format: format, Out: out}.Emit(newNonDagsDeployJSON(&res, nonDagsBundleType, nonDagsBundlePath, waitForDeploy, waitErr), nil) - } - return renderBundleUploaded(res.BundleVersion)(out) - }) -} - -// nonDagsTarget is the Deployment a non-Dag bundle deploys to, and the login -// it deploys under, the way astro deploy from the same directory picks them. -// -// The login: in a pyproject.toml project here whose manifest loads, the -// project's (its Astro host's), for everything; anywhere else, or when the -// manifest does not load, the current context's. -// -// The target: the argument or --deployment, which must agree, a link name of -// the project here or a Deployment id. With neither, the workspace's -// Deployments are offered (--workspace, else --workspace-id, else the -// login's): only to a run that can be asked, and only on the current -// context's host, as astro deploy offers them. A run that cannot be asked, -// under --output json or without a terminal, is refused as astro deploy's is, -// with kind input_required. -func nonDagsTarget(ctx context.Context, args []string, format cliout.Format) (string, *astrov1.Deployment, deployLogin, error) { - linkName := "" - if len(args) > 0 { - linkName = args[0] - } - ws := cmp.Or(manifestWorkspace, workspaceID) - // Agreement first, before anything is read. - if _, _, err := manifestdeploy.ResolveNamed(nil, linkName, manifestDeployment, ws, ""); err != nil { - current, _ := loginForDeploy(ctx, "") //nolint:errcheck // with no domain it is the current context, and never fails - return "", nil, current, cliout.Usage(err) - } - - m := manifestHere() - domain := "" - if m != nil { - domain = m.Astro.LoginDomain() - } - login, err := loginForDeploy(ctx, domain) - if err != nil { - return "", nil, login, err - } - - if name := cmp.Or(linkName, manifestDeployment); name != "" { - if m == nil || !hasLink(m, name) { - return name, nil, login, nil - } - named, _, err := manifestdeploy.ResolveNamed(m, name, "", ws, "") - if err != nil { - return "", nil, login, cliout.Usage(err) - } - return named.DeploymentID, nil, login, nil - } - - if format != cliout.FormatText || !stdinIsTerminal() { - return "", nil, login, input.Required(errNonDagsNoTarget) - } - if !login.current { - return "", nil, login, errPickerOtherHost(login.context.Domain) - } - if ws == "" { - ws, err = coalesceWorkspace() - if err != nil { - return "", nil, login, errors.Wrap(err, "failed to find a valid workspace") - } - } - id, read, err := resolveBundleDeployment(nil, ws, "", noCreateUnderJSON(format, "to deploy to")) - return id, read, login, err -} - -// errNonDagsNoTarget is a --non-dags deploy that names no Deployment and -// cannot be asked for one, in astro deploy's words for the same refusal. -var errNonDagsNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id as the argument or with --deployment") - // errPickerOtherHost is the workspace picker refused on a login other than // the current context's: it lists only the current context's Deployments. func errPickerOtherHost(domain string) error { diff --git a/cmd/astro/deploy_bundle.go b/cmd/astro/deploy_bundle.go new file mode 100644 index 000000000..2c8a7ceb3 --- /dev/null +++ b/cmd/astro/deploy_bundle.go @@ -0,0 +1,525 @@ +package astro + +import ( + "bufio" + "cmp" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/cmd/utils" + "github.com/astronomer/astro-cli/config" + manifestdeploy "github.com/astronomer/astro-cli/internal/deploy" + astrov1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/internal/platform/astro/deployment" + "github.com/astronomer/astro-cli/pkg/input" + "github.com/astronomer/astro-cli/pkg/util" +) + +// The transport astro deploy bundle calls, vars so a test can stand in for +// the upload and the delete. +var ( + DeployBundle = astrodeploy.DeployBundle + DeleteBundle = astrodeploy.DeleteBundle +) + +// waitForBundle waits for a Deployment to take a bundle deploy or delete. A +// var so a test need not sit through the poll's sleeps. +var waitForBundle = astrodeploy.WaitForBundle + +// deployBundleOptions are astro deploy bundle's flags. +type deployBundleOptions struct { + mountPath string + bundleType string + deployment string + workspace string + description string + wait bool + waitTime time.Duration + delete bool + output cliout.Format +} + +func newDeployBundleCmd() *cobra.Command { + o := &deployBundleOptions{} + types := strings.Join(astrodeploy.BundleTypeNames(), ", ") + cmd := &cobra.Command{ + Use: manifestdeploy.BundleCommand + " [PATH]", + Short: "Deploy a non-DAG bundle (for example a dbt project) to a Deployment", + Long: "Deploy a non-DAG bundle (for example a dbt project) to a Deployment on Astro. " + + "The bundle is a directory of its own, outside any Astro project: PATH, or the current directory. " + + "It is uploaded as it is and mounted at --mount-path in Airflow, for your DAGs to read; it carries no DAGs. " + + "With --delete it removes the bundle mounted at that path instead.\n\n" + + "A bundle's type decides its defaults. With no --type, the type is the first one PATH looks like (" + types + "), else none. " + + "A dbt project (a directory holding dbt_project.yml) mounts at " + astrodeploy.DbtDefaultMountPathPrefix + " unless --mount-path says otherwise. " + + "Any other --type is recorded as a label, with no defaults, so it needs --mount-path. " + + "With --delete and no bundle at PATH, a mount path below a type's default directory says the type.\n\n" + + "--deployment takes a link name of the project in the current directory, a Deployment id, or the name of a Deployment in the workspace. " + + "This deploys the bundle's files; astro deploy deploys the project and its DAGs. " + + "To list or change the bundles a Deployment has registered, use astro deployment bundle. " + + "A project link called bundle is not reachable as astro deploy bundle: deploy the project to it with astro deploy --deployment bundle.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDeployBundle(cmd, args, o) + }, + Example: ` # Deploy the dbt project in the current directory, picking the Deployment from a list + astro deploy bundle + + # Deploy a dbt project elsewhere to a given Deployment + astro deploy bundle --deployment + + # Deploy it to a Deployment named analytics in a given Workspace + astro deploy bundle --deployment analytics --workspace + + # Deploy another kind of bundle, which needs a mount path + astro deploy bundle --type --mount-path /usr/local/airflow/ \ + --deployment + + # Remove the dbt project in the current directory from a Deployment + astro deploy bundle --delete --deployment `, + } + f := cmd.Flags() + f.StringVarP(&o.mountPath, "mount-path", "m", "", "Path the bundle is mounted at in Airflow, for reference by Dags. Default: the type's, "+astrodeploy.DbtDefaultMountPathPrefix+" for a dbt project") + f.StringVar(&o.bundleType, "type", "", "Type of the bundle: "+types+", or any other label, which has no defaults. Default: detected from PATH, else none") + f.StringVar(&o.deployment, "deployment", "", "Deployment to deploy to: a link name from the project here, a Deployment id, or a Deployment name") + f.StringVar(&o.workspace, "workspace", "", "Workspace a Deployment name is looked up in, and whose Deployments are offered when no --deployment is given") + f.StringVar(&o.description, "description", "", "Description to store on the deploy") + f.BoolVarP(&o.wait, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") + f.DurationVarP(&o.waitTime, "wait-time", "t", deployWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") + f.BoolVar(&o.delete, "delete", false, "Remove the bundle mounted at --mount-path (by default the one PATH's type would mount at) instead of deploying") + cliout.AddOutputFlag(cmd, &o.output) + return cmd +} + +func runDeployBundle(cmd *cobra.Command, args []string, o *deployBundleOptions) error { + format := o.output + // The result goes where stdout is now. What the platform code prints on + // the way (the git note, the archive's) is output text mode has always + // shown; under json it goes to stderr. + out := cmd.OutOrStdout() + defer strayStdoutToStderr(format)() + + if cmd.Flags().Changed("wait-time") && !o.wait { + return cliout.Usage(errors.New("cannot use --wait-time with --wait=false")) + } + path := config.WorkingPath + if len(args) > 0 { + path = args[0] + } + if o.delete { + return deleteBundle(cmd, out, path, o) + } + return deployBundle(cmd, out, path, o) +} + +// deployBundle uploads the bundle at path. +func deployBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundleOptions) error { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("bundle path %s does not exist", path) + } + return fmt.Errorf("failed to access bundle path %s: %w", path, err) + } + if !info.IsDir() { + return fmt.Errorf("bundle path %s is not a directory", path) + } + // The walk every deploy decides by: a pyproject.toml project or a 1.x one + // at or above the bundle path. + if !utils.Locate(path).None() { + return fmt.Errorf("bundle path %s is within an Astro project: a non-DAG bundle is a directory of its own. To deploy the project, run astro deploy", path) + } + b, err := resolveBundle(cmd, path, o) + if err != nil { + return err + } + targetID, target, login, err := bundleTarget(cmd.Context(), o.deployment, o.workspace, o.output) + if err != nil { + return err + } + cmd.SilenceUsage = true + + in := &astrodeploy.DeployBundleInput{ + BundlePath: path, + MountPath: b.mountPath, + DeploymentID: targetID, + Deployment: target, + BundleType: b.bundleType, + Description: o.description, + AstroV1Client: login.client, + } + if !login.current { + in.Login = &login.context + } + res, err := DeployBundle(in) + if err != nil { + return err + } + bundlePath, err := filepath.Abs(path) + if err != nil { + bundlePath = path + } + return publishThenWaitWith(cmd, o.output, o.wait, bundleWait(cmd, &login, res.DeploymentID, o.waitTime), func(waitErr error) error { + r := cliout.Renderer{Format: o.output, Out: out} + return r.Emit(newBundleDeployJSON(&res, &b, bundlePath, o.wait, waitErr), renderBundleUploaded(res.BundleVersion)) + }) +} + +// deleteBundle removes the bundle mounted at --mount-path, or at the mount +// path the bundle at path would deploy to. +func deleteBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundleOptions) error { + b, err := resolveBundle(cmd, path, o) + if err != nil { + return err + } + targetID, target, login, err := bundleTarget(cmd.Context(), o.deployment, o.workspace, o.output) + if err != nil { + return err + } + cmd.SilenceUsage = true + + in := &astrodeploy.DeleteBundleInput{ + MountPath: b.mountPath, + DeploymentID: targetID, + Deployment: target, + BundleType: b.bundleType, + Description: o.description, + AstroV1Client: login.client, + } + if !login.current { + in.Login = &login.context + } + res, err := DeleteBundle(in) + if err != nil { + return err + } + return publishThenWaitWith(cmd, o.output, o.wait, bundleWait(cmd, &login, res.DeploymentID, o.waitTime), func(waitErr error) error { + r := cliout.Renderer{Format: o.output, Out: out} + return r.Emit(newBundleDeleteJSON(&res, &b, o.wait, waitErr), renderBundleDeleted(res.MountPath)) + }) +} + +// resolvedBundle is what a bundle deploy or delete acts on: the type it is +// recorded as, the mount path, and the bundle's name, when its type names +// it. +type resolvedBundle struct { + bundleType string + mountPath string + name string +} + +// resolveBundle settles the bundle at path, for a deploy and a delete alike. +// +// The type, for a deploy and a delete alike, is --type when given; else the +// first registered type that detects path; else, given a mount path, the +// type whose default directory that mount path is directly below; else +// none. So a deploy and a delete of the same bundle record the same type. +// +// The mount path is --mount-path when given, and then path is not checked: +// --type dbt with a mount path deploys any directory, as 1.x's +// --non-dags-bundle-type dbt did. Otherwise the type checks path and derives +// the mount path from it, which the run says on stderr; a type with no +// default needs --mount-path. +func resolveBundle(cmd *cobra.Command, path string, o *deployBundleOptions) (resolvedBundle, error) { + bt := astrodeploy.ResolveBundleType(o.bundleType, path) + if o.bundleType == "" && bt.Name == astrodeploy.BundleTypeNone && o.mountPath != "" { + if inferred, ok := astrodeploy.BundleTypeForMountPath(o.mountPath); ok { + bt = inferred + } + } + b := resolvedBundle{bundleType: bt.Name, mountPath: o.mountPath} + // The name is the bundle's when the mount path is the one this type + // derives from path: a mount path of its own names some other bundle, + // or none, and PATH says nothing of it. + named := func() string { + if !bt.HasDefaultMountPath() || (bt.Detect != nil && !bt.Detect(path)) { + return "" + } + derived, err := bt.DefaultMountPath(path) + if err != nil || derived != b.mountPath { + return "" + } + name, err := bt.BundleName(path) + if err != nil { + return "" + } + return name + } + if b.mountPath == "" { + if !bt.HasDefaultMountPath() { + return b, cliout.Usage(fmt.Errorf("--mount-path is required for a bundle of type %s: only a %s bundle has a default", bt.Name, strings.Join(astrodeploy.BundleTypeNames(), " or "))) + } + if bt.Validate != nil { + if err := bt.Validate(path); err != nil { + return b, err + } + } + mountPath, err := bt.DefaultMountPath(path) + if err != nil { + return b, err + } + b.mountPath = mountPath + fmt.Fprintf(cmd.ErrOrStderr(), "mount path %s: the default for a %s bundle\n", mountPath, bt.Name) + } + b.name = named() + return b, nil +} + +// bundleWait waits for the Deployment under the login the bundle went to. +func bundleWait(cmd *cobra.Command, login *deployLogin, deploymentID string, waitTime time.Duration) func() error { + return func() error { + if login.current { + return waitForBundle(cmd.ErrOrStderr(), deploymentID, waitTime, login.client) + } + return astrodeploy.WaitForBundleIn(cmd.ErrOrStderr(), &login.context, deploymentID, waitTime, login.client) + } +} + +// bundleTarget is the Deployment a bundle deploys to or is deleted from, and +// the login it does so under, the way astro deploy from the same directory +// picks them. +// +// The login: in a pyproject.toml project here whose manifest loads, the +// project's (its Astro host's), for everything; anywhere else, or when the +// manifest does not load, the current context's. +// +// The target: name (--deployment), a link name of the project here, else a +// Deployment id (a CUID), else the name of a Deployment in the workspace +// (deploymentNamed). Without one, the workspace's Deployments are offered +// (workspace, else the login's): only to a run that can be asked, and only on +// the current context's host, as astro deploy offers them. A run that cannot +// be asked, under --output json or without a terminal, is refused as astro +// deploy's is, with kind input_required. read is the Deployment when it was +// picked, so the deploy need not read it again. +func bundleTarget(ctx context.Context, name, workspace string, format cliout.Format) (id string, read *astrov1.Deployment, login deployLogin, err error) { + m := manifestHere() + domain := "" + if m != nil { + domain = m.Astro.LoginDomain() + } + login, err = loginForDeploy(ctx, domain) + if err != nil { + return "", nil, login, err + } + + if name != "" { + if m != nil && hasLink(m, name) { + named, _, err := manifestdeploy.ResolveNamed(m, name, "", workspace, "") + if err != nil { + return "", nil, login, cliout.Usage(err) + } + return named.DeploymentID, nil, login, nil + } + if util.IsCUID(name) { + return name, nil, login, nil + } + projectWorkspace := "" + if m != nil { + projectWorkspace = m.Astro.Workspace + } + dep, err := deploymentNamed(name, cmp.Or(workspace, projectWorkspace, login.context.Workspace), &login) + if err != nil { + return "", nil, login, err + } + return dep.Id, dep, login, nil + } + + if format != cliout.FormatText || !stdinIsTerminal() { + return "", nil, login, input.Required(errBundleNoTarget) + } + if !login.current { + return "", nil, login, errPickerOtherHost(login.context.Domain) + } + ws := workspace + if ws == "" { + if ws, err = coalesceWorkspace(); err != nil { + return "", nil, login, fmt.Errorf("failed to find a valid workspace: %w", err) + } + } + picked, err := deployment.GetDeployment(ws, "", "", false, nil, login.client) + if err != nil { + return "", nil, login, err + } + if picked.Id == "" { + return "", nil, login, deployment.ErrNothingTo(ws, "", "", "") + } + return picked.Id, &picked, login, nil +} + +// deploymentNamed is the one Deployment called name in workspace ws, listed +// under login. None, or more than one, is an error that says so, naming the +// ids of the ones that share the name. +func deploymentNamed(name, ws string, login *deployLogin) (*astrov1.Deployment, error) { + if ws == "" { + return nil, cliout.Usage(fmt.Errorf("--deployment %s is not a link of the project here or a Deployment id, and there is no Workspace to look it up by name in: pass --workspace, or the Deployment id", name)) + } + found, err := deployment.DeploymentsNamed(name, ws, login.context.Organization, login.client) + if err != nil { + return nil, err + } + switch len(found) { + case 0: + // Named wrong, as a link name that is not one would be: a usage + // mistake, not a failure of the deploy. + return nil, cliout.Usage(fmt.Errorf("no Deployment named %s in workspace %s: pass a Deployment id, a link name of the project here, or the name of a Deployment in that workspace", name, ws)) + case 1: + return &found[0], nil + } + return nil, cliout.Usage(fmt.Errorf("%d Deployments are named %s in workspace %s (%s): pass the id of the one you mean with --deployment", len(found), name, ws, strings.Join(deployment.DeploymentIDs(found), ", "))) +} + +// errBundleNoTarget is a bundle deploy or delete that names no Deployment and +// cannot be asked for one, in astro deploy's words for the same refusal. +var errBundleNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass --deployment with a Deployment id, a Deployment name, or a link name") + +// bundleDeployJSON is what astro deploy bundle --output json publishes: the +// bundle deployed and where. Its deployment, workspace and git keys are the +// ones astro deploy's result uses for the same facts. +type bundleDeployJSON struct { + Deployment string `json:"deployment"` + DeploymentName string `json:"deployment_name"` + Workspace string `json:"workspace"` + // DeployID is the deploy record the upload created on the Deployment. + DeployID string `json:"deploy_id"` + // BundleType is the type the bundle deployed as (--type, or detected), + // BundleName the bundle's name when its type names it (a dbt project's + // name), and BundlePath the directory bundled, as an absolute path. + BundleType string `json:"bundle_type"` + BundleName string `json:"bundle_name,omitempty"` + BundlePath string `json:"bundle_path"` + MountPath string `json:"mount_path"` + // BundleVersion is the version of the uploaded bundle. + BundleVersion string `json:"bundle_version"` + // Waited is true when the run waited for the Deployment to become + // healthy (--wait), and WaitError says why that wait failed; it is + // omitted when the wait succeeded or none was asked for. + Waited bool `json:"waited"` + WaitError string `json:"wait_error,omitempty"` + Git *deployGitJSON `json:"git,omitempty"` +} + +func newBundleDeployJSON(res *astrodeploy.BundleDeploy, b *resolvedBundle, bundlePath string, waited bool, waitErr error) bundleDeployJSON { + return bundleDeployJSON{ + Deployment: res.DeploymentID, + DeploymentName: res.DeploymentName, + Workspace: res.WorkspaceID, + DeployID: res.DeployID, + BundleType: b.bundleType, + BundleName: b.name, + BundlePath: bundlePath, + MountPath: res.MountPath, + BundleVersion: res.BundleVersion, + Waited: waited, + WaitError: errText(waitErr), + Git: bundleGitJSON(res.Git), + } +} + +// bundleDeleteJSON is what astro deploy bundle --delete --output json +// publishes: the bundle it removed from the Deployment. Astro removes the +// files as the Deployment takes the deploy, which a wait (waited, and +// wait_error when it failed) watches for. workspace is the Deployment's, +// omitted when the Deployment was named and so never read. +type bundleDeleteJSON struct { + Action string `json:"action"` + Deployment string `json:"deployment"` + Workspace string `json:"workspace,omitempty"` + DeployID string `json:"deploy_id"` + BundleType string `json:"bundle_type"` + // BundleName is the name of the bundle at PATH when its type names it, + // omitted when the delete was given a mount path and no bundle. + BundleName string `json:"bundle_name,omitempty"` + MountPath string `json:"mount_path"` + Waited bool `json:"waited"` + WaitError string `json:"wait_error,omitempty"` +} + +func newBundleDeleteJSON(res *astrodeploy.BundleDelete, b *resolvedBundle, waited bool, waitErr error) bundleDeleteJSON { + return bundleDeleteJSON{ + Action: "deleted", + Deployment: res.DeploymentID, + Workspace: res.WorkspaceID, + DeployID: res.DeployID, + BundleType: b.bundleType, + BundleName: b.name, + MountPath: res.MountPath, + Waited: waited, + WaitError: errText(waitErr), + } +} + +// bundleGitJSON is the commit a bundle deploy recorded, as a deploy result +// publishes it; nil when it recorded none. +func bundleGitJSON(g *astrodeploy.BundleGit) *deployGitJSON { + if g == nil { + return nil + } + return &deployGitJSON{CommitSHA: g.CommitSHA, Branch: g.Branch, CommitURL: g.CommitURL} +} + +// renderBundleUploaded is the text a finished bundle upload has always +// printed. +func renderBundleUploaded(version string) func(io.Writer) error { + return cliout.Text(func(b *bufio.Writer) { + fmt.Fprintf(b, "Successfully uploaded bundle with version %s to Astro.\n", version) + }) +} + +func renderBundleDeleted(mountPath string) func(io.Writer) error { + return cliout.Text(func(b *bufio.Writer) { + fmt.Fprintf(b, "Successfully requested bundle delete for mount path %s from Astro.\n", mountPath) + }) +} + +// publishThenWait publishes a bundle deploy's or delete's result and, with +// --wait, waits for the Deployment to take it (waitFor). publish is handed +// the wait's failure, nil when it succeeded or there was no wait. +// +// The order depends on the format. In text the result comes first, as it +// always has, so a person sees the upload succeed before the wait begins; +// the wait's progress goes to stderr, and its failure is the run's error. +// Under json the one object is the last thing the run writes, after the wait +// it reports on. A wait that fails there still publishes the result, saying +// why, because the upload happened and a script needs its deploy id and +// version, and then exits 1 (failedAfterResult). +func publishThenWaitWith(cmd *cobra.Command, format cliout.Format, wait bool, waitFor func() error, publish func(waitErr error) error) error { + waitDone := func() error { + if !wait { + return nil + } + return waitFor() + } + if format == cliout.FormatJSON { + waitErr := waitDone() + if err := publish(waitErr); err != nil { + // The write failed, so stdout is what broke: the root must not + // try to write the error object there too. failedAfterResult + // marks it shown and puts the words on stderr, the write's + // failure first and the wait's after it, so neither hides the + // other. + return failedAfterResult(cmd, format, errors.Join(err, waitErr)) + } + return failedAfterResult(cmd, format, waitErr) + } + if err := publish(nil); err != nil { + return err + } + return waitDone() +} + +// errText is err's message, or "" for nil. +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/cmd/astro/deploy_bundle_test.go b/cmd/astro/deploy_bundle_test.go new file mode 100644 index 000000000..24c09741c --- /dev/null +++ b/cmd/astro/deploy_bundle_test.go @@ -0,0 +1,626 @@ +package astro + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/config" + "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/pkg/input" +) + +// execBundleRun runs astro deploy bundle with args under a root, as a user +// runs it. +func execBundleRun(t *testing.T, client astrov1.APIClient, answers string, args ...string) tokenRun { + t.Helper() + return execAstroCmd(t, client, answers, func(io.Writer) *cobra.Command { return NewDeployCmd() }, append([]string{"deploy", "bundle"}, args...)...) +} + +// fakeDeployBundle stands in for the upload, recording what it was asked. +func fakeDeployBundle(t *testing.T) **astrodeploy.DeployBundleInput { + t.Helper() + var captured *astrodeploy.DeployBundleInput + prev := DeployBundle + t.Cleanup(func() { DeployBundle = prev }) + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = in + return astrodeploy.BundleDeploy{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath, BundleVersion: "v1"}, nil + } + return &captured +} + +// fakeDeleteBundle stands in for the delete, recording what it was asked. +func fakeDeleteBundle(t *testing.T) **astrodeploy.DeleteBundleInput { + t.Helper() + var captured *astrodeploy.DeleteBundleInput + prev := DeleteBundle + t.Cleanup(func() { DeleteBundle = prev }) + DeleteBundle = func(in *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { + captured = in + return astrodeploy.BundleDelete{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath}, nil + } + return &captured +} + +// noBundleTransport fails the test if the run reaches the upload or the +// delete. +func noBundleTransport(t *testing.T) { + t.Helper() + prevDeploy, prevDelete := DeployBundle, DeleteBundle + t.Cleanup(func() { DeployBundle, DeleteBundle = prevDeploy, prevDelete }) + DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + t.Error("nothing is deployed") + return astrodeploy.BundleDeploy{}, nil + } + DeleteBundle = func(*astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { + t.Error("nothing is deleted") + return astrodeploy.BundleDelete{}, nil + } +} + +// bundleTestID is a Deployment id, which --deployment takes as it is. +const bundleTestID = "clxbundle0000000000000000" + +// plainBundle is a directory outside any project that no bundle type +// detects, with the working path a fresh directory beside it. +func plainBundle(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seeds.csv"), []byte("a,b\n"), 0o600)) + prev := config.WorkingPath + config.WorkingPath = t.TempDir() + t.Cleanup(func() { config.WorkingPath = prev }) + return dir +} + +// astro deploy bundle is the subcommand, and any other word after deploy is +// still the Deployment the project deploys to. +func TestDeployBundleRouting(t *testing.T) { + root := deployUnderRoot() + cmd, rest, err := root.Find([]string{"deploy", "bundle", "some/path"}) + require.NoError(t, err) + assert.Equal(t, "astro deploy bundle", cmd.CommandPath()) + assert.Equal(t, []string{"some/path"}, rest) + + cmd, rest, err = root.Find([]string{"deploy", "clx-dep"}) + require.NoError(t, err) + assert.Equal(t, "astro deploy", cmd.CommandPath()) + assert.Equal(t, []string{"clx-dep"}, rest) +} + +// The type decides the defaults: a dbt project is detected and mounts below +// /usr/local/airflow/dbt by its name; --type dbt asks for one; any other type +// is a label that needs --mount-path. +func TestDeployBundleTypes(t *testing.T) { + t.Run("dbt, detected", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeployBundle(t) + r := execBundleRun(t, nil, "", project, "--deployment", bundleTestID, "--description", "nightly") + require.NoError(t, r.err, r.stderr) + in := *captured + require.NotNil(t, in) + assert.Equal(t, project, in.BundlePath) + assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", in.MountPath) + assert.Equal(t, "dbt", in.BundleType) + assert.Equal(t, bundleTestID, in.DeploymentID) + assert.Equal(t, "nightly", in.Description) + assert.Nil(t, in.Deployment, "named by id, so not read") + assert.Nil(t, in.Login, "the current context") + assert.Contains(t, r.stderr, "mount path /usr/local/airflow/dbt/jaffle_shop: the default for a dbt bundle") + assert.Equal(t, "Successfully uploaded bundle with version v1 to Astro.\n", r.stdout) + }) + t.Run("dbt, the working directory and a mount path given", func(t *testing.T) { + project := dbtProject(t) + config.WorkingPath = project + captured := fakeDeployBundle(t) + r := execBundleRun(t, nil, "", "--deployment", bundleTestID, "-m", "/usr/local/airflow/dbt/custom") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, project, (*captured).BundlePath) + assert.Equal(t, "/usr/local/airflow/dbt/custom", (*captured).MountPath) + assert.Equal(t, "dbt", (*captured).BundleType) + assert.NotContains(t, r.stderr, "the default for") + }) + // With no mount path, --type dbt derives one from the project, so it + // needs one; given a mount path, it labels any directory dbt, as 1.x's + // --non-dags-bundle-type dbt did. + t.Run("--type dbt, without a dbt project", func(t *testing.T) { + bundle := plainBundle(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", bundle, "--type", "dbt", "--deployment", bundleTestID) + require.Error(t, r.err) + assert.Contains(t, r.err.Error(), "no dbt_project.yml in "+bundle) + + captured := fakeDeployBundle(t) + r = execBundleRun(t, nil, "", bundle, "--type", "dbt", "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/dbt/seeds") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "dbt", (*captured).BundleType) + assert.Equal(t, "/usr/local/airflow/dbt/seeds", (*captured).MountPath) + }) + t.Run("--type of another kind needs a mount path", func(t *testing.T) { + bundle := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", bundleTestID) + require.Error(t, r.err) + assert.True(t, cliout.IsUsage(r.err), "%v", r.err) + assert.Equal(t, "--mount-path is required for a bundle of type seeds: only a dbt bundle has a default", r.err.Error()) + }) + t.Run("--type of another kind, with a mount path", func(t *testing.T) { + bundle := dbtProject(t) + captured := fakeDeployBundle(t) + r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/seeds") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "seeds", (*captured).BundleType, "a free-form label") + assert.Equal(t, "/usr/local/airflow/seeds", (*captured).MountPath) + }) + t.Run("nothing detected", func(t *testing.T) { + bundle := plainBundle(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", bundle, "--deployment", bundleTestID) + require.Error(t, r.err) + assert.Equal(t, "--mount-path is required for a bundle of type none: only a dbt bundle has a default", r.err.Error()) + + captured := fakeDeployBundle(t) + r = execBundleRun(t, nil, "", bundle, "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/x") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "none", (*captured).BundleType) + }) +} + +// PATH is a directory of its own: one that does not exist, a file, and one in +// or below a project are refused before anything is read. +func TestDeployBundleRefusesItsPath(t *testing.T) { + oneX := make1xProject(t) + manifestDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(manifestDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + nested := filepath.Join(manifestDir, "include", "dbt") + require.NoError(t, os.MkdirAll(nested, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nested, "dbt_project.yml"), []byte("name: x\n"), 0o600)) + file := filepath.Join(t.TempDir(), "a-file") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o600)) + + for path, want := range map[string]string{ + oneX: "is within an Astro project", + manifestDir: "is within an Astro project", + nested: "is within an Astro project", + filepath.Join(manifestDir, "..", "absent"): "does not exist", + file: "is not a directory", + } { + noBundleTransport(t) + r := execBundleRun(t, nil, "", path, "--deployment", bundleTestID, "--mount-path", "/x") + require.Error(t, r.err, path) + assert.Contains(t, r.err.Error(), want, path) + } + + noBundleTransport(t) + r := execBundleRun(t, nil, "", t.TempDir(), "--deployment", bundleTestID, "--mount-path", "/x", "--wait-time", "1m") + require.Error(t, r.err) + assert.True(t, cliout.IsUsage(r.err)) + assert.Contains(t, r.err.Error(), "cannot use --wait-time with --wait=false") +} + +// --delete removes the bundle at --mount-path, or at the mount path PATH's +// type gives it. +func TestDeployBundleDelete(t *testing.T) { + t.Run("the dbt project's default", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeleteBundle(t) + r := execBundleRun(t, nil, "", "--delete", project, "--deployment", bundleTestID) + require.NoError(t, r.err, r.stderr) + in := *captured + require.NotNil(t, in) + assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", in.MountPath) + assert.Equal(t, "dbt", in.BundleType) + assert.Equal(t, bundleTestID, in.DeploymentID) + assert.Nil(t, in.Login) + assert.Equal(t, "Successfully requested bundle delete for mount path /usr/local/airflow/dbt/jaffle_shop from Astro.\n", r.stdout) + }) + t.Run("a mount path needs no bundle here", func(t *testing.T) { + plainBundle(t) + captured := fakeDeleteBundle(t) + r := execBundleRun(t, nil, "", "--delete", "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/x", "--type", "seeds") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "/usr/local/airflow/x", (*captured).MountPath) + assert.Equal(t, "seeds", (*captured).BundleType) + }) + // From a directory that holds no bundle, a mount path below the dbt + // default directory is a dbt bundle's. + t.Run("a dbt mount path from elsewhere", func(t *testing.T) { + plainBundle(t) + r := execBundleRun(t, dbtBundleMock(t, ""), "", "--delete", "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/dbt/jaffle_shop", "-o", "json") + require.NoError(t, r.err, r.stderr) + var got bundleDeleteJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, "dbt", got.BundleType) + assert.NotContains(t, fields, "bundle_name", "no bundle here to name") + }) + t.Run("no mount path and nothing to derive one from", func(t *testing.T) { + bundle := plainBundle(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", "--delete", bundle, "--deployment", bundleTestID) + require.Error(t, r.err) + assert.True(t, cliout.IsUsage(r.err)) + assert.Contains(t, r.err.Error(), "--mount-path is required for a bundle of type none") + + r = execBundleRun(t, nil, "", "--delete", bundle, "--type", "dbt", "--deployment", bundleTestID) + require.Error(t, r.err) + assert.Contains(t, r.err.Error(), "no dbt_project.yml in") + }) +} + +// The target: --deployment, a link of the project here or an id. Naming +// none, a run that cannot be asked is refused as input_required before +// anything is uploaded or deleted. +func TestDeployBundleTarget(t *testing.T) { + for _, del := range []bool{false, true} { + name := "deploy" + if del { + name = "delete" + } + with := func(args ...string) []string { + if del { + return append([]string{"--delete"}, args...) + } + return args + } + t.Run(name+": a link of the project here", func(t *testing.T) { + bundle := dbtProject(t) + require.NoError(t, os.WriteFile(filepath.Join(config.WorkingPath, "pyproject.toml"), []byte("[project]\nname = \"demo\"\ndependencies = [\"apache-airflow==3.1.*\"]\n\n[tool.astro]\n\n[tool.astro.deployments.prod]\ntarget = \"astro\"\nworkspace = \"clw-ws\"\ndeployment = \"clx-prod\"\n"), 0o600)) + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + r := execBundleRun(t, nil, "", with(bundle, "--deployment", "prod")...) + require.NoError(t, r.err, r.stderr) + if del { + assert.Equal(t, "clx-prod", (*deleted).DeploymentID) + } else { + assert.Equal(t, "clx-prod", (*deployed).DeploymentID) + } + }) + t.Run(name+": none named, under json", func(t *testing.T) { + bundle := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", with(bundle, "-o", "json")...) + require.Error(t, r.err) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, string(cliout.KindInputRequired), got.Kind) + assert.Contains(t, got.Error, "pass --deployment with a Deployment id, a Deployment name, or a link name") + }) + t.Run(name+": none named, without a terminal", func(t *testing.T) { + bundle := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", with(bundle)...) + require.Error(t, r.err) + assert.True(t, input.IsRequired(r.err), "%v", r.err) + }) + t.Run(name+": none named, picked", func(t *testing.T) { + bundle := dbtProject(t) + interactiveDeploy(t) + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "1\n", with(bundle, "--workspace", "clxdbtworkspace0000000000")...) + require.NoError(t, r.err, r.stderr) + var read *astrov1.Deployment + if del { + read = (*deleted).Deployment + } else { + read = (*deployed).Deployment + } + require.NotNil(t, read, "the picked Deployment is handed on, not read again") + assert.Equal(t, dbtTestDeploymentID, read.Id) + }) + } +} + +// A deploy and a delete of the same bundle record the same type: --type, +// else what PATH holds, else what the mount path's shape says. +func TestDeployBundleDeployAndDeleteAgreeOnTheType(t *testing.T) { + for _, tc := range []struct { + name, mountPath, want string + args []string + dbtHere bool + }{ + {name: "a dbt mount path, no bundle here", mountPath: "/usr/local/airflow/dbt/jaffle_shop", want: "dbt"}, + {name: "a nested mount path", mountPath: "/usr/local/airflow/dbt/a/b", want: "none"}, + {name: "a mount path climbing out", mountPath: "/usr/local/airflow/dbt/../x", want: "none"}, + {name: "a dbt project here, another mount path", mountPath: "/usr/local/airflow/x", want: "dbt", dbtHere: true}, + {name: "--type wins", mountPath: "/usr/local/airflow/dbt/jaffle_shop", want: "seeds", args: []string{"--type", "seeds"}}, + } { + t.Run(tc.name, func(t *testing.T) { + bundle := plainBundle(t) + if tc.dbtHere { + bundle = dbtProject(t) + } + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + args := append([]string{bundle, "--deployment", bundleTestID, "--mount-path", tc.mountPath}, tc.args...) + r := execBundleRun(t, nil, "", args...) + require.NoError(t, r.err, r.stderr) + r = execBundleRun(t, nil, "", append([]string{"--delete"}, args...)...) + require.NoError(t, r.err, r.stderr) + assert.Equal(t, tc.want, (*deployed).BundleType) + assert.Equal(t, tc.want, (*deleted).BundleType) + }) + } +} + +// bundle_name names the bundle the mount path was derived from: a mount path +// of its own is some other bundle's, or none. +func TestDeployBundleNameOnlyForTheDerivedMountPath(t *testing.T) { + for mountPath, named := range map[string]bool{ + "": true, + "/usr/local/airflow/dbt/jaffle_shop": true, + "/usr/local/airflow/dbt/custom": false, + } { + project := dbtProject(t) + args := []string{project, "--deployment", dbtTestDeploymentID, "-o", "json"} + if mountPath != "" { + args = append(args, "--mount-path", mountPath) + } + r := execBundleRun(t, dbtBundleMock(t, fakeBlobStore(t)), "", args...) + require.NoError(t, r.err, r.stderr) + var got bundleDeployJSON + fields := decodeOne(t, r.stdout, &got) + if named { + assert.Equal(t, "jaffle_shop", got.BundleName, mountPath) + } else { + assert.NotContains(t, fields, "bundle_name", mountPath) + } + } +} + +// --deployment that is neither a link nor an id is a Deployment's name in +// the workspace: one match is used, as read; none, or several, are errors +// that say so. +func TestDeployBundleDeploymentByName(t *testing.T) { + other := dbtTestDeployment() + other.Id = "clxdbtother00000000000000" + t.Run("one", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeployBundle(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, dbtTestDeploymentID, (*captured).DeploymentID) + require.NotNil(t, (*captured).Deployment, "the listed Deployment is handed on") + }) + t.Run("one, deleted", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeleteBundle(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", "--delete", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, dbtTestDeploymentID, (*captured).DeploymentID) + }) + // None, or several, is a Deployment named wrong: a usage error, under + // json the usage error object. + t.Run("none", func(t *testing.T) { + project := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", project, "--deployment", "reporting", "--workspace", "clxdbtworkspace0000000000", "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitUsage, r.code) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, string(cliout.KindUsage), got.Kind) + assert.Contains(t, got.Error, "no Deployment named reporting in workspace clxdbtworkspace0000000000") + }) + t.Run("two", func(t *testing.T) { + project := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment(), other), "", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000", "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitUsage, r.code) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, string(cliout.KindUsage), got.Kind) + assert.Contains(t, got.Error, "2 Deployments are named analytics in workspace clxdbtworkspace0000000000 ("+dbtTestDeploymentID+", clxdbtother00000000000000)") + }) +} + +// From a project on another host, a bundle goes under that host's login, by +// link and by id alike, and the picker, which lists only the current +// context's Deployments, is refused. +func TestDeployBundleUsesTheManifestDomainsLogin(t *testing.T) { + for _, del := range []bool{false, true} { + for _, target := range []string{"test", bundleTestID} { + t.Run(fmt.Sprintf("%s, delete %t", target, del), func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + args := []string{"bundle", t.TempDir(), "--deployment", target, "--mount-path", "/x"} + if del { + args = append(args, "--delete") + } + _, err := execDeployCapture(args...) + require.NoError(t, err) + var login *config.Context + if del { + login = (*deleted).Login + } else { + login = (*deployed).Login + } + require.NotNil(t, login, "under the project's host") + assert.Equal(t, "astronomer.io", login.Domain) + assert.Equal(t, "prod-org", login.Organization) + }) + } + } + + t.Run("the picker on another host", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + interactiveDeploy(t) + noBundleTransport(t) + _, err := execDeployCapture("bundle", t.TempDir(), "--mount-path", "/x") + require.Error(t, err) + assert.Contains(t, err.Error(), "the deployment picker lists only the current context's Deployments") + assert.Contains(t, err.Error(), "astro context switch astronomer.io") + }) + + t.Run("outside a project", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + require.NoError(t, os.Remove(filepath.Join(config.WorkingPath, "pyproject.toml"))) + captured := fakeDeployBundle(t) + _, err := execDeployCapture("bundle", t.TempDir(), "--deployment", bundleTestID, "--mount-path", "/x") + require.NoError(t, err) + assert.Nil(t, (*captured).Login, "the current context") + }) +} + +func TestDeployBundleJSON(t *testing.T) { + t.Run("deploy", func(t *testing.T) { + project := dbtProject(t) + r := execBundleRun(t, dbtBundleMock(t, fakeBlobStore(t)), "", project, "--deployment", dbtTestDeploymentID, "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got bundleDeployJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, bundleDeployJSON{ + Deployment: dbtTestDeploymentID, + DeploymentName: "analytics", + Workspace: "clxdbtworkspace0000000000", + DeployID: dbtTestDeployID, + BundleType: "dbt", + BundleName: "jaffle_shop", + BundlePath: project, + MountPath: "/usr/local/airflow/dbt/jaffle_shop", + BundleVersion: dbtTestVersion, + }, got) + assert.NotContains(t, fields, "git", "the project is in no git checkout") + assert.NotContains(t, fields, "wait_error") + }) + // Named by id, the Deployment is never read, so its Workspace is not + // known and not published. + t.Run("delete", func(t *testing.T) { + plainBundle(t) + r := execBundleRun(t, dbtBundleMock(t, ""), "", "--delete", "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/seeds", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got bundleDeleteJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, bundleDeleteJSON{ + Action: "deleted", + Deployment: dbtTestDeploymentID, + DeployID: dbtTestDeployID, + BundleType: "none", + MountPath: "/usr/local/airflow/seeds", + }, got) + assert.NotContains(t, fields, "workspace") + }) +} + +// A --wait's progress is on stderr in both formats. In text the result comes +// first, so stdout is what it is without --wait; under json the object comes +// after the wait it reports, and says it waited, and why the wait failed. +func TestDeployBundleWait(t *testing.T) { + for _, del := range []bool{false, true} { + name := "deploy" + if del { + name = "delete" + } + run := func(t *testing.T, more ...string) tokenRun { + project := dbtProject(t) + args := []string{project, "--deployment", dbtTestDeploymentID} + var client astrov1.APIClient + if del { + client = dbtBundleMock(t, "") + args = append(args, "--delete") + } else { + client = dbtBundleMock(t, fakeBlobStore(t)) + } + return execBundleRun(t, client, "", append(args, more...)...) + } + t.Run(name+" text", func(t *testing.T) { + without := run(t) + require.NoError(t, without.err) + waited := stubBundleWait(t, nil) + r := run(t, "--wait") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + assert.Equal(t, []string{dbtTestDeploymentID}, *waited) + assert.Equal(t, without.stdout, r.stdout) + assert.Contains(t, r.stderr, "Waiting for the deployment to become healthy…") + }) + t.Run(name+" json, the wait failing", func(t *testing.T) { + stubBundleWait(t, errors.New("timed out")) + r := run(t, "--wait", "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitFailure, r.code) + var got struct { + DeployID string `json:"deploy_id"` + Waited bool `json:"waited"` + WaitError string `json:"wait_error"` + } + decodeOne(t, r.stdout, &got) + assert.Equal(t, dbtTestDeployID, got.DeployID) + assert.True(t, got.Waited) + assert.Equal(t, "timed out", got.WaitError) + }) + t.Run(name+" json", func(t *testing.T) { + waited := stubBundleWait(t, nil) + r := run(t, "--wait", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + assert.Equal(t, []string{dbtTestDeploymentID}, *waited) + var got struct { + Waited bool `json:"waited"` + } + fields := decodeOne(t, r.stdout, &got) + assert.True(t, got.Waited) + assert.NotContains(t, fields, "wait_error") + }) + } +} + +// The guidance is for the subcommand typed, wherever the flags stand, a +// flag's value never taken for it. +func TestDbtRemovedReadsTheSubcommand(t *testing.T) { + for _, tc := range []struct { + args []string + want string + }{ + {[]string{"--workspace-id", "ws1", "deploy"}, errDbtDeployRemoved}, + {[]string{"-n", "prod", "delete"}, errDbtDeleteRemoved}, + {[]string{"--project-path=.", "cleanup"}, errDbtCleanupRemoved}, + {[]string{"-w", "deploy"}, errDbtDeployRemoved}, + {[]string{"-wt", "5m", "deploy"}, errDbtDeployRemoved}, + {[]string{"-nprod", "delete"}, errDbtDeleteRemoved}, + {[]string{"--workspace-id", "deploy"}, errDbtRemoved}, + {[]string{"--", "deploy"}, errDbtRemoved}, + } { + assert.Equal(t, tc.want, dbtRemovedGuidance(tc.args), "%v", tc.args) + } +} + +// astro dbt is a tombstone: every subcommand, and the group bare, says what +// replaced it, in text and as the one error object under json, and runs +// nothing, whatever 1.x flags it is passed. +func TestDbtRemoved(t *testing.T) { + cases := map[string]string{ + "": errDbtRemoved, + "deploy": errDbtDeployRemoved, + "delete": errDbtDeleteRemoved, + "cleanup": errDbtCleanupRemoved, + } + for sub, want := range cases { + args := []string{"dbt"} + if sub != "" { + args = append(args, sub, "clx-dep", "--project-path", t.TempDir(), "-n", "analytics") + } + noBundleTransport(t) + r := execAstroCmd(t, nil, "", func(io.Writer) *cobra.Command { return newDbtRemovedCmd() }, args...) + require.Error(t, r.err, sub) + assert.Equal(t, cliout.ExitUsage, r.code, sub) + assert.Equal(t, want, r.err.Error(), sub) + assert.NotContains(t, want, "`", "command names are plain text") + assert.Empty(t, r.stdout, sub) + + r = execAstroCmd(t, nil, "", func(io.Writer) *cobra.Command { return newDbtRemovedCmd() }, append(args, "-o", "json")...) + require.Error(t, r.err, sub) + assert.Equal(t, cliout.ExitUsage, r.code, sub) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, want, got.Error, sub) + } +} diff --git a/cmd/astro/deploy_help.go b/cmd/astro/deploy_help.go index 8ebbdfaec..a7d60f37e 100644 --- a/cmd/astro/deploy_help.go +++ b/cmd/astro/deploy_help.go @@ -1,8 +1,6 @@ package astro import ( - "strings" - "github.com/spf13/cobra" ) @@ -10,9 +8,8 @@ import ( // apply to which kind of deploy. Each value is the section's title: the root's // help (cmd/help.go) lists a flag annotated "Image" under "Image Flags:". const ( - deployGroupImage = "Image" - deployGroupDAG = "Dag" - deployGroupNonDAG = "Non-Dag Bundle" + deployGroupImage = "Image" + deployGroupDAG = "Dag" ) // The annotation keys cmd/help.go reads a flag's section and the sections' @@ -29,12 +26,10 @@ func annotateDeployFlag(cmd *cobra.Command, name, group string) { } // orderDeployFlagGroups lists deploy's sections in the order a deploy narrows: -// what image ships, which DAGs, and what rides beside. +// what image ships, then which DAGs. func orderDeployFlagGroups(cmd *cobra.Command) { if cmd.Annotations == nil { cmd.Annotations = map[string]string{} } - cmd.Annotations[flagGroupOrderAnnotation] = strings.Join([]string{ - deployGroupImage, deployGroupDAG, deployGroupNonDAG, - }, ",") + cmd.Annotations[flagGroupOrderAnnotation] = deployGroupImage + "," + deployGroupDAG } diff --git a/cmd/astro/deploy_manifest_test.go b/cmd/astro/deploy_manifest_test.go index b8b67dd9f..070bf263b 100644 --- a/cmd/astro/deploy_manifest_test.go +++ b/cmd/astro/deploy_manifest_test.go @@ -54,7 +54,6 @@ func resetDeployFlagVars() { workspaceID = "" deploymentName = "" deployDescription = "" - nonDags = false deployOutput = string(cliout.FormatText) } @@ -1167,97 +1166,6 @@ func TestDeployManifestUsesTheManifestDomainsLogin(t *testing.T) { } } -// --non-dags from a project deploys under the project's host as the project's -// deploy does, a link and a bare Deployment id alike; outside a project a -// bare id goes under the current context. -func TestDeployNonDagsUsesTheManifestDomainsLogin(t *testing.T) { - for _, args := range [][]string{{"test"}, {"--deployment", "clx-bare"}} { - t.Run(strings.Join(args, " "), func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - var captured *astrodeploy.DeployBundleInput - prev := DeployBundle - t.Cleanup(func() { DeployBundle = prev }) - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - - _, err := execDeployCapture(append(args, "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", t.TempDir())...) - require.NoError(t, err) - require.NotNil(t, captured.Login, "deployed under the project's host") - assert.Equal(t, "astronomer.io", captured.Login.Domain) - assert.Equal(t, "prod-org", captured.Login.Organization) - }) - } - - t.Run("outside a project", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - require.NoError(t, os.Remove(filepath.Join(config.WorkingPath, "pyproject.toml"))) - var captured *astrodeploy.DeployBundleInput - prev := DeployBundle - t.Cleanup(func() { DeployBundle = prev }) - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - _, err := execDeployCapture("--deployment", "clx-bare", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", t.TempDir()) - require.NoError(t, err) - assert.Nil(t, captured.Login, "the current context") - }) -} - -// --non-dags naming no Deployment: a run that cannot be asked is refused as -// astro deploy's is, input_required, before any picker; one that can is -// offered the picker only on the current context's host, so from a project -// on another host it is refused with the same words astro deploy uses. -func TestDeployNonDagsWithNoTarget(t *testing.T) { - noBundle := func(t *testing.T) { - prev := DeployBundle - t.Cleanup(func() { DeployBundle = prev }) - DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - t.Fatal("no target, so nothing is deployed") - return astrodeploy.BundleDeploy{}, nil - } - } - nonDags := []string{"--non-dags", "--non-dags-mount-path", "/x"} - - t.Run("under --output json", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - noBundle(t) - out, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir(), "--output", "json")...) - require.Error(t, err) - var obj struct { - Error string `json:"error"` - Kind string `json:"kind"` - } - require.NoError(t, json.Unmarshal([]byte(out), &obj), out) - assert.Equal(t, "input_required", obj.Kind) - assert.Contains(t, obj.Error, "pass the Deployment id as the argument or with --deployment") - }) - - t.Run("without a terminal", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - noBundle(t) - _, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir())...) - require.Error(t, err) - assert.True(t, input.IsRequired(err), "%v", err) - }) - - t.Run("the picker on another host", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - interactiveDeploy(t) - noBundle(t) - _, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir())...) - require.Error(t, err) - assert.Contains(t, err.Error(), "the deployment picker lists only the current context's Deployments") - assert.Contains(t, err.Error(), "astro context switch astronomer.io") - }) -} - // ASTRO_API_TOKEN outranks the stored login, and goes out with its scheme like // a stored token, since the deploy's CI/CD check splits it off. func TestDeployManifestSendsTheAPITokenToTheManifestDomain(t *testing.T) { diff --git a/cmd/astro/deploy_test.go b/cmd/astro/deploy_test.go index 526a95133..eaf10cd41 100644 --- a/cmd/astro/deploy_test.go +++ b/cmd/astro/deploy_test.go @@ -1,7 +1,6 @@ package astro import ( - "encoding/json" "errors" "fmt" "io" @@ -9,17 +8,14 @@ import ( "path/filepath" "testing" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "github.com/astronomer/astro-cli/cmd/cliout" "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/config" manifestdeploy "github.com/astronomer/astro-cli/internal/deploy" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/internal/project" testUtil "github.com/astronomer/astro-cli/pkg/testing" ) @@ -239,7 +235,7 @@ func TestDeployRefusesFlagsThatCannotAllApply(t *testing.T) { // passes --force, and a pyproject.toml project's deploy always took both. func TestDeployDropsThe1xOnlyFlags(t *testing.T) { cmd := NewDeployCmd() - for _, name := range []string{"save", "pytest", "parse", "test", "env", "dags-path", "dag-bundle-name", "deployment-name"} { + for _, name := range []string{"save", "pytest", "parse", "test", "env", "dags-path", "dag-bundle-name", "deployment-name", "non-dags", "non-dags-mount-path", "non-dags-local-path", "non-dags-bundle-type"} { assert.Nil(t, cmd.Flags().Lookup(name), "--%s", name) } for _, letter := range []string{"s", "t", "e", "n"} { @@ -252,215 +248,16 @@ func TestDeployDropsThe1xOnlyFlags(t *testing.T) { } } -type NonDagsDeploySuite struct { - suite.Suite - mockV1Client *astrov1_mocks.ClientWithResponsesInterface - origV1Client astrov1.APIClient - origDeployBundle func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) - origWorkingPath string - origWd string - tmpWorkingDir string -} - -func (s *NonDagsDeploySuite) SetupTest() { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - // Run from an isolated temp directory that is not within an Astro project, so - // the non-DAG bundle path validation (which walks up to a .astro/config.yaml) - // is deterministic regardless of where the repo lives. - tmpDir, err := os.MkdirTemp("", "non-dags-test") - s.Require().NoError(err) - s.tmpWorkingDir = tmpDir - s.origWd, err = os.Getwd() - s.Require().NoError(err) - s.Require().NoError(os.Chdir(tmpDir)) - s.origWorkingPath = config.WorkingPath - config.WorkingPath = tmpDir - - s.origV1Client = astroV1Client - s.origDeployBundle = DeployBundle - s.mockV1Client = new(astrov1_mocks.ClientWithResponsesInterface) - astroV1Client = s.mockV1Client -} - -func (s *NonDagsDeploySuite) TearDownTest() { - s.mockV1Client.AssertExpectations(s.T()) - astroV1Client = s.origV1Client - DeployBundle = s.origDeployBundle - config.WorkingPath = s.origWorkingPath - if s.origWd != "" { - _ = os.Chdir(s.origWd) - } - if s.tmpWorkingDir != "" { - _ = os.RemoveAll(s.tmpWorkingDir) - } -} - -func TestNonDagsDeploy(t *testing.T) { - suite.Run(t, new(NonDagsDeploySuite)) -} - -func (s *NonDagsDeploySuite) TestRequiresMountPath() { - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags") - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "--non-dags-mount-path is required") -} - -func (s *NonDagsDeploySuite) TestBundleTypeDefaultsToNone() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = deployInput - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") - assert.NoError(s.T(), err) - s.Require().NotNil(captured) - assert.Equal(s.T(), "none", captured.BundleType) -} - -func (s *NonDagsDeploySuite) TestRejectsIncompatibleFlag() { - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt", "--dags") - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "cannot use --dags with --non-dags") -} - -func (s *NonDagsDeploySuite) TestProvidedDeploymentId() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = deployInput - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt") - assert.NoError(s.T(), err) - s.Require().NotNil(captured) - assert.Equal(s.T(), "test-deployment-id", captured.DeploymentID) - assert.Equal(s.T(), "/usr/local/airflow/x", captured.MountPath) - assert.Equal(s.T(), "dbt", captured.BundleType) - assert.Equal(s.T(), s.tmpWorkingDir, captured.BundlePath) -} - -// A bundle path in or below a project, by the walk every deploy decides by, -// is refused: a 1.x project's Dockerfile beside .astro (with no -// .astro/config.yaml), and a pyproject.toml project. -func (s *NonDagsDeploySuite) TestWithinAstroProject() { - oneX := s.T().TempDir() - s.Require().NoError(os.WriteFile(filepath.Join(oneX, "Dockerfile"), []byte("FROM x\n"), 0o600)) - s.Require().NoError(os.MkdirAll(filepath.Join(oneX, ".astro"), 0o755)) - s.Require().NoError(os.WriteFile(filepath.Join(oneX, ".astro", "config.yaml"), []byte("project:\n name: demo\n"), 0o600)) - manifestDir := s.T().TempDir() - s.Require().NoError(os.WriteFile(filepath.Join(manifestDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) - s.Require().NoError(os.MkdirAll(filepath.Join(manifestDir, "include", "dbt"), 0o755)) - - for _, bundle := range []string{oneX, filepath.Join(manifestDir, "include", "dbt")} { - resetDeployFlagVars() - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt", "--non-dags-local-path", bundle) - s.Require().Error(err, bundle) - s.Contains(err.Error(), "within an Astro project", bundle) - } -} - -func (s *NonDagsDeploySuite) TestBundlePathDoesNotExist() { - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", filepath.Join(s.tmpWorkingDir, "missing")) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "does not exist") -} - -func (s *NonDagsDeploySuite) TestBundlePathNotADirectory() { - file := filepath.Join(s.tmpWorkingDir, "a-file") - s.Require().NoError(os.WriteFile(file, []byte("x"), 0o600)) - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", file) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "is not a directory") -} - -// --deployment names a non-Dag bundle's Deployment as the argument does. -func (s *NonDagsDeploySuite) TestDeploymentFlag() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = deployInput - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "--deployment", "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") - assert.NoError(s.T(), err) - s.Require().NotNil(captured) - assert.Equal(s.T(), "test-deployment-id", captured.DeploymentID) -} - -// --non-dags runs before the project decides anything: from a pyproject.toml -// project it deploys a bundle, not the project, so the project directory -// itself is refused as the bundle. It used to run a whole project deploy, -// dropping --non-dags. -func (s *NonDagsDeploySuite) TestFromAManifestProject() { - s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) - DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - s.Fail("a bundle inside the project is not deployed") - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") - assert.ErrorContains(s.T(), err, "within an Astro project") -} - -// --output json publishes the bundle deploy as one object; a bad --output is -// a usage error before anything is read. -func (s *NonDagsDeploySuite) TestOutput() { - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - return astrodeploy.BundleDeploy{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath, BundleVersion: "v1"}, nil +func testExecCmd(cmd *cobra.Command, args ...string) error { + if args == nil { + args = []string{} } - resetDeployFlagVars() - out, err := execDeployCapture("test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--output", "json") - s.Require().NoError(err, out) - var got nonDagsDeployJSON - s.Require().NoError(json.Unmarshal([]byte(out), &got), out) - s.Equal(nonDagsDeployJSON{Deployment: "test-deployment-id", DeployID: "dep-1", BundleType: "none", BundlePath: s.tmpWorkingDir, MountPath: "/usr/local/airflow/x", BundleVersion: "v1"}, got) - - resetDeployFlagVars() - _, err = execDeployCapture("test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--output", "yaml") - s.True(cliout.IsUsage(err), "%v", err) -} - -// The target is named as a project deploy names it: the argument and -// --deployment must agree, and a link name of the project here is its -// Deployment. --workspace is the workspace whose Deployments are offered. -func (s *NonDagsDeploySuite) TestTarget() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - bundle := s.T().TempDir() - - resetDeployFlagVars() - err := testExecCmd(NewDeployCmd(), "dep-a", "--deployment", "dep-b", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) - s.ErrorContains(err, "name one") - s.True(cliout.IsUsage(err)) - - s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\ndependencies = [\"apache-airflow==3.1.*\"]\n\n[tool.astro]\n\n[tool.astro.deployments.prod]\ntarget = \"astro\"\nworkspace = \"clw-ws\"\ndeployment = \"clx-prod\"\n"), 0o600)) - resetDeployFlagVars() - err = testExecCmd(NewDeployCmd(), "prod", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) - s.Require().NoError(err) - s.Equal("clx-prod", captured.DeploymentID) - s.Nil(captured.Login, "a project naming no host deploys under the current context") -} - -// A Deployment id needs no manifest: a pyproject.toml here that does not load -// does not stop a deploy that names its target by id. -func (s *NonDagsDeploySuite) TestIDNeedsNoManifest() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[tool.astro]\nruntime = [\n"), 0o600)) - bundle := s.T().TempDir() - - resetDeployFlagVars() - err := testExecCmd(NewDeployCmd(), "--deployment", "clx-by-id", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) - s.Require().NoError(err) - s.Equal("clx-by-id", captured.DeploymentID) - s.Nil(captured.Login) + // deploy sets the package's workspaceID from --workspace-id or the one it + // resolves, and coalesceWorkspace reads it before the context. Left set, + // it would choose the Workspace of whatever runs next. + defer func() { workspaceID = "" }() + testUtil.SetupOSArgsForGinkgo() + cmd.SetArgs(args) + _, err := cmd.ExecuteC() + return err } diff --git a/cmd/astro/deployment_token_json_test.go b/cmd/astro/deployment_token_json_test.go index d1d33bb4d..62d263cd4 100644 --- a/cmd/astro/deployment_token_json_test.go +++ b/cmd/astro/deployment_token_json_test.go @@ -166,7 +166,7 @@ func execAstroCmd(t *testing.T, client astrov1.APIClient, answers string, newRoo // The token ids are package variables a positional argument sets and no // flag registration resets, so one run's id would answer the next. tokenID, orgTokenID, workspaceTokenID = "", "", "" - // --workspace-id, and deploy and dbt when they resolve one, set the + // --workspace-id, and deploy when it resolves one, set the // package's workspaceID, which coalesceWorkspace reads before the context. // Clear what an earlier run left, and what this one leaves. workspaceID = "" diff --git a/cmd/astro/output_binding_test.go b/cmd/astro/output_binding_test.go index 93509c225..6eab3f88f 100644 --- a/cmd/astro/output_binding_test.go +++ b/cmd/astro/output_binding_test.go @@ -38,8 +38,9 @@ func execUnboundRootWith(t *testing.T, client astrov1.APIClient, alpha astrov1al testUtil.InitTestConfig(testUtil.LocalPlatform) testUtil.SetupOSArgsForGinkgo() prevClient, prevAlpha := astroV1Client, astroV1Alpha1Client - // dbt deploy resolves a Workspace into the package's workspaceID, which - // coalesceWorkspace reads before the context; see execDeployCmd. + // A bundle deploy may resolve a Workspace into the package's + // workspaceID, which coalesceWorkspace reads before the context; see + // execDeployCmd. t.Cleanup(func() { astroV1Client, astroV1Alpha1Client, workspaceID = prevClient, prevAlpha, "" }) outR, outW, perr := os.Pipe() @@ -75,20 +76,20 @@ func execUnboundRootWith(t *testing.T, client astrov1.APIClient, alpha astrov1al // Under --output json the result is on stdout and nothing of it on stderr, // in the tree production builds. func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { - t.Run("dbt deploy", func(t *testing.T) { + t.Run("deploy bundle", func(t *testing.T) { project := dbtProject(t) - stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "dbt", "deploy", dbtTestDeploymentID, "--project-path", project, "-o", "json") + stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", "bundle", project, "--deployment", dbtTestDeploymentID, "-o", "json") require.NoError(t, err, "stderr:\n%s", stderr) - var got dbtDeployJSON + var got bundleDeployJSON decodeOne(t, stdout, &got) assert.Equal(t, dbtTestVersion, got.BundleVersion) assert.NotContains(t, stderr, dbtTestVersion, "the result went to stderr") - assert.Contains(t, stderr, "Initiating dbt deploy", "the notes go to stderr") + assert.Contains(t, stderr, "mount path /usr/local/airflow/dbt/jaffle_shop", "the notes go to stderr") }) // The bundle sits in a git checkout with uncommitted changes, so the // platform code prints its note on the way: under json that goes to // stderr, and stdout is the one result object. - t.Run("deploy --non-dags from a dirty checkout", func(t *testing.T) { + t.Run("deploy bundle from a dirty checkout", func(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("needs git") } @@ -103,11 +104,10 @@ func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { gitIn("add", ".") gitIn("commit", "-q", "-m", "first") require.NoError(t, os.WriteFile(filepath.Join(bundle, "dbt_project.yml"), []byte("name: changed\n"), 0o600)) - resetDeployFlagVars() - stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", bundle, "--output", "json") + stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", "bundle", bundle, "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/x", "--output", "json") require.NoError(t, err, "stderr:\n%s", stderr) - var got nonDagsDeployJSON + var got bundleDeployJSON decodeOne(t, stdout, &got) assert.Equal(t, dbtTestVersion, got.BundleVersion) assert.Contains(t, stderr, "uncommitted changes", "the platform's note goes to stderr") diff --git a/cmd/astro/remote.go b/cmd/astro/remote.go index ad6441124..b0b1ceac8 100644 --- a/cmd/astro/remote.go +++ b/cmd/astro/remote.go @@ -38,7 +38,9 @@ func newRemoteRootCmd(out io.Writer) *cobra.Command { Short: "Manage remote deploys and images", Long: "Commands for interacting with remote registries and deploying client images", } - // Bound for the same reason as dbt: see newDbtCmd. + // The results leave by cmd.OutOrStdout(), which must be the stdout the + // tree was built with: under json strayStdoutToStderr points os.Stdout + // at stderr, and an unbound command would follow it there. cmd.SetOut(out) cmd.AddCommand(newRemoteDeployCmd()) diff --git a/cmd/astro/remote_render.go b/cmd/astro/remote_render.go new file mode 100644 index 000000000..83c01d4b6 --- /dev/null +++ b/cmd/astro/remote_render.go @@ -0,0 +1,76 @@ +package astro + +// Rendering for astro remote deploy. The platform package returns what +// happened; this file decides how it looks, in text and in json. + +import ( + "bufio" + "fmt" + "io" + + "github.com/astronomer/astro-cli/cmd/cliout" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/pkg/ansi" +) + +// remoteDeployJSON is what `astro remote deploy --output json` publishes: the +// client image it pushed, which is what the agents' Helm values then name. +type remoteDeployJSON struct { + Image string `json:"image"` + Registry string `json:"registry"` + Tag string `json:"tag"` + // SourceImage is the local image pushed (--image-name), omitted when the + // run built one. + SourceImage string `json:"source_image,omitempty"` + // Platforms are the --platform the image was built for, [] for the host's + // or when nothing was built. + Platforms []string `json:"platforms"` + // RuntimeCheck is the check against --deployment, omitted without one. + RuntimeCheck *remoteRuntimeCheckJSON `json:"runtime_check,omitempty"` +} + +type remoteRuntimeCheckJSON struct { + Deployment string `json:"deployment"` + ClientRuntimeVersion string `json:"client_runtime_version"` + DeploymentRuntimeVersion string `json:"deployment_runtime_version"` +} + +func newRemoteDeployJSON(res *astrodeploy.ClientDeploy) remoteDeployJSON { + obj := remoteDeployJSON{ + Image: res.Image, + Registry: res.Registry, + Tag: res.Tag, + SourceImage: res.SourceImage, + Platforms: nonNil(res.Platforms), + } + if c := res.RuntimeCheck; c != nil { + obj.RuntimeCheck = &remoteRuntimeCheckJSON{ + Deployment: c.DeploymentID, + ClientRuntimeVersion: c.ClientRuntimeVersion, + DeploymentRuntimeVersion: c.DeploymentRuntimeVersion, + } + } + return obj +} + +// renderRemoteDeploy is the text a finished client image push has always +// printed: where it went, and what to do with it next. +func renderRemoteDeploy(image string) func(io.Writer) error { + return cliout.Text(func(b *bufio.Writer) { + fmt.Fprintf(b, "Successfully pushed client image to %s\n", ansi.Bold(image)) + fmt.Fprintf(b, "\n--------------------------------\n") + fmt.Fprintln(b, "The client image has been pushed to your private registry.") + fmt.Fprintln(b, "Your next step would be to update the agent component to use the new client image.") + fmt.Fprintln(b, "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.") + fmt.Fprintf(b, "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: %s\n", image) + fmt.Fprintln(b, "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components") + }) +} + +// nonNil is s, or [] for nil, so a list publishes as [] and never null. +func nonNil(s []string) []string { + if s == nil { + return []string{} + } + return s +} diff --git a/cmd/astro/root.go b/cmd/astro/root.go index 40bbd1f93..29ef43da0 100644 --- a/cmd/astro/root.go +++ b/cmd/astro/root.go @@ -29,7 +29,7 @@ func AddCmds(v1Client astrov1.APIClient, v1Alpha1Client astrov1alpha1.APIClient, newEnvRootCmd(out), newWorkspaceCmd(out), newOrganizationCmd(out), - newDbtCmd(out), + newDbtRemovedCmd(), newIDECommand(out), newRemoteRootCmd(out), } diff --git a/cmd/astro/schema_test.go b/cmd/astro/schema_test.go index 46113d3de..5eea888e8 100644 --- a/cmd/astro/schema_test.go +++ b/cmd/astro/schema_test.go @@ -153,11 +153,10 @@ var publishedPayloads = []cliouttest.Case{ // astro deploy --output json, in a project with a manifest: the one // object a finished deploy prints. {Name: "deploy", Value: deployJSON{}}, - {Name: "deploy-non-dags", Value: nonDagsDeployJSON{}}, - // astro dbt deploy: what it deployed where. delete: the bundle it + // astro deploy bundle: what it deployed where. --delete: the bundle it // removed. - {Name: "dbt-deploy", Value: dbtDeployJSON{}}, - {Name: "dbt-delete", Value: dbtDeleteJSON{}}, + {Name: "deploy-bundle", Value: bundleDeployJSON{}}, + {Name: "deploy-bundle-delete", Value: bundleDeleteJSON{}}, // astro remote deploy: the client image it pushed. {Name: "remote-deploy", Value: remoteDeployJSON{}}, @@ -234,7 +233,7 @@ func TestEveryGoldenHasACase(t *testing.T) { // minWatchedPayloads is a floor under the tally, not a target: an empty tally // reads exactly like a clean one, so without it the observer coming unwired -// would be silent. Fifty-two shapes reach Emit in this package's tests +// would be silent. Fifty-one shapes reach Emit in this package's tests // today: the deployment variable results (2) and the six API token results; // the deployment, workspace and organization user and team results, with the // organization's invite and team membership (10); the pkg/output lists their @@ -243,14 +242,16 @@ func TestEveryGoldenHasACase(t *testing.T) { // deployment inspect's deployment (which create and update publish too), // delete's removal, hibernate's override, the worker-queue result and the // log entry (6); the bundle a bundle create or update leaves and what a -// bundle delete did (2); the dbt deploy, delete and cleanup results and the remote -// deploy's pushed image (4); the Workspace a create, update or switch +// bundle delete did (2); astro deploy bundle's deploy and delete results and the +// remote deploy's pushed image (3); the Workspace a create, update or switch // publishes and what a Workspace delete did, what an Organization switch left // current, the role list and the file an audit-log export wrote (5); the // Astro IDE project list and what an import or an export moved (3); and the // error object. -// Raise it as conversions land; lower it only saying why. -const minWatchedPayloads = 52 +// Raise it as conversions land; lower it only saying why. It went from 52 +// to 51 when astro dbt deploy, astro dbt delete and astro deploy --non-dags, +// three shapes, became astro deploy bundle's two. +const minWatchedPayloads = 51 // pinnedElsewhere names the shapes that reach Emit here and are pinned by // another tree's goldens, keyed by type, with where. Pinning one twice would diff --git a/cmd/astro/testdata/schema/dbt-deploy.json b/cmd/astro/testdata/schema/dbt-deploy.json deleted file mode 100644 index 10aff5029..000000000 --- a/cmd/astro/testdata/schema/dbt-deploy.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "deployment": "x", - "deployment_name": "x", - "workspace": "x", - "deploy_id": "x", - "project": "x", - "project_path": "x", - "mount_path": "x", - "bundle_version": "x", - "waited": true, - "wait_error": "x", - "git": { - "commit_sha": "x", - "branch": "x", - "commit_url": "x" - } -} diff --git a/cmd/astro/testdata/schema/dbt-delete.json b/cmd/astro/testdata/schema/deploy-bundle-delete.json similarity index 76% rename from cmd/astro/testdata/schema/dbt-delete.json rename to cmd/astro/testdata/schema/deploy-bundle-delete.json index 3360e5d06..3fceddc4f 100644 --- a/cmd/astro/testdata/schema/dbt-delete.json +++ b/cmd/astro/testdata/schema/deploy-bundle-delete.json @@ -3,6 +3,8 @@ "deployment": "x", "workspace": "x", "deploy_id": "x", + "bundle_type": "x", + "bundle_name": "x", "mount_path": "x", "waited": true, "wait_error": "x" diff --git a/cmd/astro/testdata/schema/deploy-non-dags.json b/cmd/astro/testdata/schema/deploy-bundle.json similarity index 93% rename from cmd/astro/testdata/schema/deploy-non-dags.json rename to cmd/astro/testdata/schema/deploy-bundle.json index 4b06085e0..8f697e1cb 100644 --- a/cmd/astro/testdata/schema/deploy-non-dags.json +++ b/cmd/astro/testdata/schema/deploy-bundle.json @@ -4,6 +4,7 @@ "workspace": "x", "deploy_id": "x", "bundle_type": "x", + "bundle_name": "x", "bundle_path": "x", "mount_path": "x", "bundle_version": "x", diff --git a/cmd/local/link.go b/cmd/local/link.go index ce027b6d6..9fb95cc4d 100644 --- a/cmd/local/link.go +++ b/cmd/local/link.go @@ -340,6 +340,12 @@ func (c *cli) runLinkAdd(cmd *cobra.Command, in *linkAddInput, name string, comm return err } name = strings.TrimSpace(name) + // bundle is astro deploy bundle, so a new link by that name could not be + // deployed to by name: refused before anything is asked. One that exists + // is the --replace check's. + if _, exists := m.Astro.Deployments[name]; name == scaffold.BundleLinkName && !exists { + return cliout.Usage(scaffold.ErrBundleLinkName) + } // A NAME already linked is refused before anything asks a question or // calls the platform. if err := refuseExisting(m, name, in.replace); err != nil { @@ -448,7 +454,7 @@ func (c *cli) pickAstroDeployment(link *scaffold.Link, m *manifest.Manifest, nam return name, nil } name = scaffold.LinkNameFor(dep.Name) - if name == "" || name == manifest.ReservedLinkName { + if name == "" || name == manifest.ReservedLinkName || name == scaffold.BundleLinkName { return "", fmt.Errorf("the Deployment's name does not make a link name: pass one, astro link add NAME --%s %s", flagLinkDeployment, dep.ID) } return name, nil diff --git a/cmd/local/link_test.go b/cmd/local/link_test.go index ece7ed3f1..e1322b891 100644 --- a/cmd/local/link_test.go +++ b/cmd/local/link_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/astronomer/astro-cli/cmd/cliout" "github.com/astronomer/astro-cli/internal/userstate" "github.com/astronomer/astro-cli/pkg/instances" "github.com/astronomer/astro-cli/pkg/manifest" @@ -161,6 +162,23 @@ func TestLinkAddRefusesAnExistingNameUnlessReplacing(t *testing.T) { assert.True(t, dev.Default, "the replace dropped the default") } +// bundle names astro deploy bundle, so astro deploy could never reach a link +// by that name: it is refused, before anything is asked or written. +func TestLinkAddRefusesBundleAsAName(t *testing.T) { + dir, _ := linkTestProject(t, linkManifest) + _, _, err := runLink(t, dir, "add", "bundle", "--deployment", "dep-new") + require.Error(t, err) + assert.True(t, cliout.IsUsage(err), "%v", err) + assert.Contains(t, err.Error(), "bundle cannot name a link: astro deploy bundle deploys a non-DAG bundle") + assert.Equal(t, linkManifest, readManifest(t, dir)) + + // One already called bundle is replaced as any other is. + dir, path := linkTestProject(t, linkManifest+"\n[tool.astro.deployments.bundle]\ndeployment = 'dep-old'\n") + _, _, err = runLink(t, dir, "add", "bundle", "--deployment", "dep-new", "--replace") + require.NoError(t, err) + assert.Equal(t, "dep-new", loadManifest(t, path).Astro.Deployments["bundle"].Deployment) +} + // An MWAA link writes the shared region when given one, and an empty --region // removes it. func TestLinkAddAnMWAALink(t *testing.T) { diff --git a/cmd/removed_flags.go b/cmd/removed_flags.go index 1e77d4fdd..0d68853f1 100644 --- a/cmd/removed_flags.go +++ b/cmd/removed_flags.go @@ -52,7 +52,14 @@ const ( errDeployDagsPathRemoved = "--dags-path was removed in Astro CLI v2: astro deploy --dags ships the project's dags directory, so run it from the project" errDeployBundleNameRemoved = "--dag-bundle-name was removed in Astro CLI v2: deploying to a named DAG bundle is not supported yet. Use Astro CLI 1.x for it" errDeployNameRemoved = "--deployment-name was removed in Astro CLI v2: name the Deployment as the argument or with --deployment, which take a link name or a Deployment id" - errAPCDeployNoCache = "--no-cache was removed in Astro CLI v2: Astro CLI v2 builds no image for Astro Private Cloud. " + + // astro deploy --non-dags became astro deploy bundle, a subcommand of + // its own with the bundle's directory as its argument. + errDeployNonDagsRemoved = "--non-dags was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment , " + + "which deploys a non-DAG bundle such as a dbt project from its directory" + errDeployNonDagsMountPathRemoved = "--non-dags-mount-path was removed in Astro CLI v2: use astro deploy bundle --mount-path" + errDeployNonDagsLocalPathRemoved = "--non-dags-local-path was removed in Astro CLI v2: pass the bundle's directory as the argument, astro deploy bundle " + errDeployNonDagsBundleTypeRemoved = "--non-dags-bundle-type was removed in Astro CLI v2: use astro deploy bundle --type, which detects a dbt project by itself" + errAPCDeployNoCache = "--no-cache was removed in Astro CLI v2: Astro CLI v2 builds no image for Astro Private Cloud. " + "Build and deploy the project with Astro CLI 1.x" ) @@ -132,6 +139,10 @@ var removedFlags = []removedFlag{ {name: "dag-bundle-name", under: []string{"deploy"}, msg: errDeployBundleNameRemoved}, {name: "deployment-name", under: []string{"deploy"}, msg: errDeployNameRemoved}, {name: "no-cache", under: []string{"deploy"}, msg: errAPCDeployNoCache}, + {name: "non-dags", under: []string{"deploy"}, msg: errDeployNonDagsRemoved}, + {name: "non-dags-mount-path", under: []string{"deploy"}, msg: errDeployNonDagsMountPathRemoved}, + {name: "non-dags-local-path", under: []string{"deploy"}, msg: errDeployNonDagsLocalPathRemoved}, + {name: "non-dags-bundle-type", under: []string{"deploy"}, msg: errDeployNonDagsBundleTypeRemoved}, } // removedFlagErr is a run refused for passing a removed 1.x flag. diff --git a/cmd/removed_flags_test.go b/cmd/removed_flags_test.go index bbda7cf73..62ad93cf8 100644 --- a/cmd/removed_flags_test.go +++ b/cmd/removed_flags_test.go @@ -150,6 +150,10 @@ var removedFlagCases = []removedFlagCase{ {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "dag-bundle-name", value: "nightly", want: errDeployBundleNameRemoved}, {tree: astroTree, args: []string{"deploy"}, flag: "deployment-name", value: "prod", shorthand: "n", want: errDeployNameRemoved}, {tree: apcTree, args: []string{"deploy", "dep-id"}, flag: "no-cache", want: errAPCDeployNoCache}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "non-dags", want: errDeployNonDagsRemoved}, + {tree: astroHostedTree, args: []string{"deploy"}, flag: "non-dags-mount-path", value: "/usr/local/airflow/x", want: errDeployNonDagsMountPathRemoved}, + {tree: astroTree, args: []string{"deploy"}, flag: "non-dags-local-path", value: "./analytics", want: errDeployNonDagsLocalPathRemoved}, + {tree: astroTree, args: []string{"deploy"}, flag: "non-dags-bundle-type", value: "dbt", want: errDeployNonDagsBundleTypeRemoved}, } // spellings is every way the case's flag can be typed: --name, --name=value diff --git a/cmd/root.go b/cmd/root.go index 9cac3137a..2e897443d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -230,7 +230,7 @@ var commandGroups = []struct { }{ {groupDevelop, "Develop locally:", []string{"init", "local", "start", "stop", "logs"}}, {groupInspect, "Inspect Airflow:", []string{"af", "use", "link", "api"}}, - {groupShip, "Ship:", []string{"package", "deploy", "remote", "dbt"}}, + {groupShip, "Ship:", []string{"package", "deploy", "remote"}}, {groupManage, "Manage Astro:", []string{ "login", "logout", "auth", "context", "organization", "workspace", "deployment", "env", diff --git a/docs/architecture.md b/docs/architecture.md index 61fbabfba..bca9bba1f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -205,7 +205,7 @@ A failure with no kind publishes no `kind` key rather than an `unknown` catch-al A script written for Astro CLI 1.x is told what replaced what it typed rather than only that it is unknown. Either way the failure is a usage error: exit 2, and the `usage` error object under `--output json`. -A removed command stays in the tree as a hidden stub that fails naming its replacement: `astro dev` ([`cmd/local/dev.go`](../cmd/local/dev.go)), `astro run`, `astro deployment airflow-variable|connection|pool`, and `astro env … create|update`. Every stub is built by `cliout.RemovedCommand` ([`cmd/cliout/removed.go`](../cmd/cliout/removed.go)): hidden, with flag parsing off so an old invocation's flags reach it, a `Short` starting `Removed in v2`, and the `removedCommand` annotation. A stub has a pre-run of its own that runs the root's and no group's, so it never reaches the project lookup of `astro deployment` or `astro env`. The root's pre-run reads the annotation. It reads `--verbosity` from the raw arguments, sets up logging and records the command (`telemetry.TrackRemovedCommand`), and runs no login or version check. So a machine that is logged out, or a project that does not parse, is still told what to run. The event is sent only once telemetry is set up, meaning the first-run notice was shown on that machine. A stub never shows the notice itself, because the core tree leaves no state behind. So the counts miss fresh machines and CI runners that start fresh: a count near zero is not on its own a reason to remove a stub. A stub fails with a usage error whose message says `` `astro …` was removed in Astro CLI v2 `` and names the command to use instead, or says there is none. `astro dev` publishes the error object (`cliout.ErrorObject`, embedded) under `--output json`, with its whole mapping of 1.x `dev` commands besides. +A removed command stays in the tree as a hidden stub that fails naming its replacement: `astro dev` ([`cmd/local/dev.go`](../cmd/local/dev.go)), `astro run`, `astro dbt` (whose `deploy` and `delete` moved to `astro deploy bundle`), `astro deployment airflow-variable|connection|pool`, and `astro env … create|update`. Every stub is built by `cliout.RemovedCommand` ([`cmd/cliout/removed.go`](../cmd/cliout/removed.go)): hidden, with flag parsing off so an old invocation's flags reach it, a `Short` starting `Removed in v2`, and the `removedCommand` annotation. A stub has a pre-run of its own that runs the root's and no group's, so it never reaches the project lookup of `astro deployment` or `astro env`. The root's pre-run reads the annotation. It reads `--verbosity` from the raw arguments, sets up logging and records the command (`telemetry.TrackRemovedCommand`), and runs no login or version check. So a machine that is logged out, or a project that does not parse, is still told what to run. The event is sent only once telemetry is set up, meaning the first-run notice was shown on that machine. A stub never shows the notice itself, because the core tree leaves no state behind. So the counts miss fresh machines and CI runners that start fresh: a count near zero is not on its own a reason to remove a stub. A stub fails with a usage error whose message says `` `astro …` was removed in Astro CLI v2 `` and names the command to use instead, or says there is none. `astro dev` publishes the error object (`cliout.ErrorObject`, embedded) under `--output json`, with its whole mapping of 1.x `dev` commands besides. [`cmd/testdata/v1_commands.tsv`](../cmd/testdata/v1_commands.tsv) is the inventory of 1.x commands, in the same three trees as the flag inventory below. Only the tests read it, since a stub needs no inventory to be found. `TestEveryV1CommandStillRunsOrSaysWhatReplacedIt` resolves every path in it against its v2 trees, APC at both ends of its version gates. It passes a path that names a command v2 has, at its own name or a 1.x name kept as an alias, or that reaches a stub, which it then runs, in text and under `--output json`, with every other hook disarmed. Below APC's version gates a path may also reach the stand-in for a command the platform is too old for; in any other tree that fails. It fails on a path that stops at a command with words left over, which is a command dropped without a stub or renamed without an alias. To remove a command, replace it with a stub; to rename one, keep its 1.x name as an alias. `TestEveryRemovedCommandStubIsA1xCommand` fails on a stub that no 1.x path reaches, and `TestRemovedCommandStubsNeedNoLoginAndAreRecorded` runs stubs on the real tree to check that none reaches the login or the project lookup and that each is recorded. diff --git a/docs/deploy.md b/docs/deploy.md index 6c80412bd..80feb43de 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -14,9 +14,9 @@ v2 builds and deploys only projects with a `pyproject.toml`, on Astro and on Ast `astro deploy` looks at the working directory. A `pyproject.toml` with a `[tool.astro]` table is deployed, even if the table fails to validate (the deploy then reports the manifest error). A Dockerfile is not evidence of a 1.x project: a project with a manifest may declare one (`[tool.astro] dockerfile`). The routing is in [`cmd/astro/deploy.go`](../cmd/astro/deploy.go). -Anywhere else, one walk decides (`utils.Locate`, used by `astro deploy`, `astro remote deploy`, APC's deploy, and the bundle-path checks of `--non-dags` and `astro dbt deploy`). It is `astro init`'s walk (`scaffold.Find1xProject`), so the deploy refuses as 1.x exactly where init and the local commands' advice say a 1.x project is. From the working directory up, at each directory: a `pyproject.toml` that is a project's stops it (`scaffold.HasManifest`): one whose `[tool.astro]` loads, or one that fails to validate, to parse or to be read, so the deploy reports why. Only a `pyproject.toml` that parses and has no `[tool.astro]` (a monorepo root's `[tool.ruff]`) is passed over; a broken one cannot be told apart from a broken manifest, so it stops the walk with its parse error, a 1.x checkout's included. Else a 1.x project stops it (`scaffold.Is1xProject`): a Dockerfile and a `.astro/config.yaml` that is not the CLI's own settings file (one with a `project` key, or no `context`, `contexts` or `telemetry`), so a stray Dockerfile in the home directory beside the global config is not one, while a home directory that is a 1.x project is. Else the walk goes on up. Each directory's `pyproject.toml` is loaded once, and the deploy uses the manifest the walk loaded rather than reading the file again. +Anywhere else, one walk decides (`utils.Locate`, used by `astro deploy`, `astro remote deploy`, APC's deploy, and the bundle-path check of `astro deploy bundle`). It is `astro init`'s walk (`scaffold.Find1xProject`), so the deploy refuses as 1.x exactly where init and the local commands' advice say a 1.x project is. From the working directory up, at each directory: a `pyproject.toml` that is a project's stops it (`scaffold.HasManifest`): one whose `[tool.astro]` loads, or one that fails to validate, to parse or to be read, so the deploy reports why. Only a `pyproject.toml` that parses and has no `[tool.astro]` (a monorepo root's `[tool.ruff]`) is passed over; a broken one cannot be told apart from a broken manifest, so it stops the walk with its parse error, a 1.x checkout's included. Else a 1.x project stops it (`scaffold.Is1xProject`): a Dockerfile and a `.astro/config.yaml` that is not the CLI's own settings file (one with a `project` key, or no `context`, `contexts` or `telemetry`), so a stray Dockerfile in the home directory beside the global config is not one, while a home directory that is a 1.x project is. Else the walk goes on up. Each directory's `pyproject.toml` is loaded once, and the deploy uses the manifest the walk loaded rather than reading the file again. -| the walk finds | `astro deploy`, any mode but `--non-dags` | +| the walk finds | `astro deploy` | | --- | --- | | a 1.x project, here or above | refused: `this project uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), and Astro CLI v2 deploys only pyproject.toml projects. Convert it with astro init, or deploy it with Astro CLI 1.x`. From below it, it names the project: `this directory is inside a project at that uses …. Convert it with astro init in , …` | | a `pyproject.toml` project above | refused: `this directory is inside the project at . Run the deploy from the project directory, ` | @@ -26,11 +26,17 @@ Every refusal is reported under `--output json` as the error object with kind `n **`--image-name ` outside any project** deploys an image already built on this machine, alone: an image-only deploy, with no `dags/` and no git commit. The target is the argument or `--deployment`, a Deployment id (there are no links without a manifest), or the workspace's Deployments to pick from; a run that cannot ask says to pass the id as the argument or with `--deployment`. The login is the current context's. The deploy warns, on stderr and in the result's `warnings`: a Deployment that takes DAG deploys keeps the DAGs it had, and one that does not now runs the DAGs inside the image, if any. At a `pyproject.toml` project's root, `--image-name` is the project's deploy with a prebuilt image, `dags/` included as usual. -**`--non-dags`** ships a separate directory as a non-DAG bundle, which must not be inside any project, so it is checked first and runs anywhere. A bundle path in or below any project the walk finds is refused, as `astro dbt deploy` refuses a dbt project there: the two use the same check. +**`astro deploy bundle [PATH]`** ships a directory of its own, PATH or the working directory, as a non-DAG bundle: a dbt project, or any other files the DAGs read, mounted at a path in Airflow. It carries no DAGs, and it is not a project deploy: PATH must be a directory, and one in or below any project the walk finds is refused. **`--delete`** removes the bundle mounted at a path instead. It replaces Astro CLI 1.x's `astro dbt deploy` and `astro dbt delete` and the 1.x `astro deploy --non-dags`, which did the same upload with different defaults and Deployment selection. `astro deployment bundle` is something else: it lists and changes the bundle registrations on a Deployment, and uploads nothing. -- **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project here whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. -- **The target** is the argument or `--deployment`, which must agree: a link of the project here, or a Deployment id. With neither, the Deployments of `--workspace` (or `--workspace-id`, or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id as the argument or with --deployment`, as `astro deploy` does. A project on another host is told to pass `--deployment ` or switch context. -- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-non-dags.json`. What the deploy prints on the way (the git note, for one) goes to stderr there. +Because `bundle` is a subcommand, `astro deploy bundle` never means a Deployment called bundle. No new link may be called `bundle`: `scaffold.SaveLink` refuses to add one for every writer (`astro link add` among them, and Astro Desktop), and a deploy that links a picked Deployment names it otherwise. A link already called `bundle` saves again as any other does, a link already called `bundle` is deployed to with `astro deploy --deployment bundle`. + +- **The type** decides the defaults. The types are a table in [`internal/platform/astro/deploy/bundletype.go`](../internal/platform/astro/deploy/bundletype.go), each with how to detect it, how to check a bundle of it, how to name it, and the directory its bundles mount below by default; adding a type is adding an entry. With no `--type`, the first type that detects PATH is used; else, given `--mount-path`, the type whose default it has the shape of (its directory and one name below it, after cleaning: `/usr/local/airflow/dbt/` is `dbt`); else `none`. A deploy and a delete of the same bundle so record the same type. Today the table has one entry, `dbt`: a directory holding `dbt_project.yml`, named by the project's `name` and mounted by default at `/usr/local/airflow/dbt/`. Any other `--type` (and `none`) is a free-form label the deploy records, with no defaults, so it needs `--mount-path`. A default mount path is said on stderr. +- **PATH is checked only when the mount path is derived from it.** Without `--mount-path`, `--type dbt` needs a dbt project at PATH. With `--mount-path`, `--type dbt` labels any directory, as 1.x's `--non-dags-bundle-type dbt` did. +- **Delete** uses `--mount-path` as given, or the mount path PATH's type would deploy to, so `astro deploy bundle --delete` in a dbt project removes that project. With neither, it is a usage error. Its type is found as the deploy's is. +- **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project in the working directory whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. +- **The target** is `--deployment`: a link of the project in the working directory, else a Deployment id, else the name of a Deployment in `--workspace` (or the project's `[tool.astro] workspace`, or the login's). The name lookup is this command's, as 1.x's `astro dbt deploy --deployment-name` had one; `astro deploy` takes a link or an id. A name that matches no Deployment, or several (listed by id), is a usage error, as a mistyped link name is. It has no `-d`: on `astro deploy`, `-d` is `--dags`. Without it, the Deployments of `--workspace` (or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass --deployment with a Deployment id, a Deployment name, or a link name`. A project on another host is told to pass `--deployment ` or switch context. +- **`--wait`** waits for the Deployment to take the deploy or the delete, under the same login. In text the result prints first and the wait's progress goes to stderr; under json the one object comes after the wait, with `waited` and, when it failed, `wait_error`, and the run exits 1. +- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-bundle.json` and, for `--delete`, `deploy-bundle-delete.json`. Both carry `bundle_type`, `mount_path`, `deployment` and `deploy_id`, and `bundle_name` when the mount path is the one the type derives from PATH (a dbt project's `name`; a mount path of its own names no bundle); the deploy adds `bundle_path`, `bundle_version` and `git`. 1.x's `astro dbt deploy -o json` keys `project` and `project_path` are `bundle_name` and `bundle_path` here, beside `bundle_type`. What the deploy prints on the way (the git note, the default mount path) goes to stderr there. **Flags that cannot all apply** are refused as usage errors before anything is read: `--wait-time` without `--wait`, `--no-dags-base-dir` with a deploy that ships no DAGs (`--image`, or `--image-name` outside a project), and `--dags` with `--image` or `--image-name` (the message names the ones given). @@ -251,7 +257,7 @@ astro package [TARGET] [flags] # astro (default), mwaa, composer, oss -o, --output text|json ``` -The flags only a 1.x project's deploy read are gone: `--save` (`-s`), `--pytest`, `--env` (`-e`), `--test` (`-t`), `--parse`, `--deployment-name` (`-n`), `--dags-path` and `--dag-bundle-name`, and on Astro Private Cloud `--save` and `--no-cache`. A run passing one fails as a usage error naming what replaced it (run the tests with `uv run pytest` first, check the DAGs with `astro local check`, name the Deployment with `--deployment`, and so on; see [Removed 1.x commands and flags](architecture.md#removed-1x-commands-and-flags)). `--force` (`-f`) and `--prompt` (`-p`) are accepted, hidden and read by nothing: astronomer/deploy-action passes `--force` on every deploy, and the deploy always asks unless a Deployment is named, which is what `--prompt` asked for. `--non-dags` and its `--non-dags-*` flags deploy a non-DAG bundle from a separate directory (see above). +The flags only a 1.x project's deploy read are gone: `--save` (`-s`), `--pytest`, `--env` (`-e`), `--test` (`-t`), `--parse`, `--deployment-name` (`-n`), `--dags-path` and `--dag-bundle-name`, and on Astro Private Cloud `--save` and `--no-cache`. A run passing one fails as a usage error naming what replaced it (run the tests with `uv run pytest` first, check the DAGs with `astro local check`, name the Deployment with `--deployment`, and so on; see [Removed 1.x commands and flags](architecture.md#removed-1x-commands-and-flags)). `--force` (`-f`) and `--prompt` (`-p`) are accepted, hidden and read by nothing: astronomer/deploy-action passes `--force` on every deploy, and the deploy always asks unless a Deployment is named, which is what `--prompt` asked for. `--non-dags` and its `--non-dags-*` flags are gone too, for `astro deploy bundle` (see above). **Git metadata.** With the `deploy.git_metadata` setting on (the default), the deploy records the HEAD commit on Astro, and the commit message becomes the description when `--description` is not given. `commit_url` is set only for a GitHub remote. A project with uncommitted changes to tracked files records no commit and prints `note: the project has uncommitted changes, so this deploy records no git commit` on stderr. `--image-name` records none. diff --git a/docs/upgrading-from-v1.md b/docs/upgrading-from-v1.md index 0936d8f68..374494f3c 100644 --- a/docs/upgrading-from-v1.md +++ b/docs/upgrading-from-v1.md @@ -104,6 +104,8 @@ Everything below fails with an error naming the replacement, instead of an "unkn | `astro deployment connection copy` | no per-Deployment copy: set it once in the workspace with `astro env connection set --workspace --auto-link`, or on each Deployment | | `astro deployment pool …` | the Airflow UI or API, until pools reach the Environment Manager | | `astro env … create`, `astro env … update` | `astro env … set`, which creates or updates | +| `astro dbt deploy [DEPLOYMENT_ID]` | `astro deploy bundle [PATH] --deployment `: PATH replaces `--project-path`, `--deployment` the argument and `--deployment-name`, `--workspace` `--workspace-id`; `-m`, `-w` and `-t` are unchanged. Under `-o json`, `project` and `project_path` are now `bundle_name` and `bundle_path`, beside `bundle_type` | +| `astro dbt delete [DEPLOYMENT_ID]` | `astro deploy bundle --delete [PATH] --deployment `, with the same flags | Run inside a 1.x project, an `astro dev` command also tells you to convert it with `astro init`. One with a replacement says so in its first line (convert, then use the replacement); bare `astro dev`, `upgrade-test` and `proxy` say it after the list of `astro local` commands. @@ -129,8 +131,12 @@ Run inside a 1.x project, an `astro dev` command also tells you to convert it wi | `--dags-path` | `astro deploy --dags` from the project, which ships its `dags/` | `astro deploy` | | `--dag-bundle-name` | no replacement yet: use Astro CLI 1.x | `astro deploy` | | `--no-cache` | no replacement: v2 builds no image for Astro Private Cloud | `astro deploy` on Astro Private Cloud | +| `--non-dags` | `astro deploy bundle [PATH]`, which detects a dbt project and gives it a mount path | `astro deploy` | +| `--non-dags-local-path` | the PATH argument of `astro deploy bundle` | `astro deploy` | +| `--non-dags-mount-path` | `astro deploy bundle --mount-path` | `astro deploy` | +| `--non-dags-bundle-type` | `astro deploy bundle --type` | `astro deploy` | -Some 1.x flags still work but are hidden from help in favor of a new spelling. On the `astro deployment` subcommands, `astro env` and `astro dbt deploy` / `delete`, `--deployment-name` (`-n`) and `--deployment-id`, whichever the command has, give way to `--deployment`. `--workspace-id` gives way to `--workspace` on those commands and the other commands that have both, `astro deploy` among them. +Some 1.x flags still work but are hidden from help in favor of a new spelling. On the `astro deployment` subcommands and `astro env`, `--deployment-name` (`-n`) and `--deployment-id`, whichever the command has, give way to `--deployment`. `--workspace-id` gives way to `--workspace` on those commands and the other commands that have both, `astro deploy` among them. ### Behavior diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 2147f2cd5..a55e4b3d5 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -633,7 +633,7 @@ func unlinkedDeployments(req Request, d Deployer, links map[string]manifest.Link } return deps[i].ID < deps[j].ID }) - taken := map[string]bool{manifest.ReservedLinkName: true} + taken := map[string]bool{manifest.ReservedLinkName: true, BundleCommand: true} for name := range links { taken[name] = true } @@ -663,7 +663,7 @@ func newLinkName(dep Deployment, taken map[string]bool) string { // A name of digits alone would shadow a row number: the prompt matches // names first, so a Deployment called "1" would take the answer "1" from // whatever row 1 is. - if base == "" || base == manifest.ReservedLinkName || isDigits(base) { + if base == "" || base == manifest.ReservedLinkName || base == BundleCommand || isDigits(base) { base = dep.ID } name := base @@ -715,7 +715,7 @@ func listWorkspace(req Request) string { // argument (linkName) and --deployment, which must agree, each a link of m // (nil for no manifest) or a Deployment id. named is false when neither names // one. It is for a command that ships something other than the project, such -// as astro deploy --non-dags, to name its Deployment the way a deploy does. +// as astro deploy bundle, to name its Deployment the way a deploy does. func ResolveNamed(m *manifest.Manifest, linkName, deployment, workspaceID, contextWorkspace string) (target Target, named bool, err error) { req := Request{Manifest: m, LinkName: linkName, Deployment: deployment, WorkspaceID: workspaceID, ContextWorkspace: contextWorkspace} var links map[string]manifest.Link @@ -776,9 +776,15 @@ func unlinkedTarget(req Request, d Deployer) (Target, error) { if dep.WorkspaceID == "" { dep.WorkspaceID = workspaceID } - return linkedTarget(req, newLinkName(dep, map[string]bool{manifest.ReservedLinkName: true}), dep), nil + return linkedTarget(req, newLinkName(dep, map[string]bool{manifest.ReservedLinkName: true, BundleCommand: true}), dep), nil } +// BundleCommand is the word astro deploy bundle is spelled with. astro +// deploy reads it as that subcommand, never as a link name, so no link is +// saved under it (scaffold.BundleLinkName): a link a deploy saves for a +// picked Deployment is named otherwise. +const BundleCommand = scaffold.BundleLinkName + // DefaultMark is the word that marks a project's default link in a list of // links: the deploy prompt's "linked, default", and the `astro use` and `astro // link` pickers. One word in every place, never the manifest's own diff --git a/internal/deploy/deploy_test.go b/internal/deploy/deploy_test.go index 649a70c5f..602d5c1f7 100644 --- a/internal/deploy/deploy_test.go +++ b/internal/deploy/deploy_test.go @@ -538,6 +538,7 @@ func TestResolveTarget_OffersTheWorkspacesUnlinkedDeployments(t *testing.T) { {ID: "dep-new", Name: "Orders (EU)"}, // offered as orders-eu {ID: "dep-clash", Name: "Staging"}, // "staging" is a link already {ID: "dep-local", Name: "Local"}, // "local" is reserved + {ID: "dep-bundle", Name: "Bundle"}, // "bundle" is astro deploy bundle {ID: "dep-other", Name: "Orders (EU)", WorkspaceID: "ws-1"}, // same name twice } req := Request{Interactive: true, Manifest: manifestWith(links), ContextWorkspace: "ws-1"} @@ -548,7 +549,7 @@ func TestResolveTarget_OffersTheWorkspacesUnlinkedDeployments(t *testing.T) { _, err := resolveTarget(req, d) require.NoError(t, err) assert.Equal(t, "ws-1", d.listedWS) - assert.Equal(t, []string{"dev", "prod", "staging", "dep-local", "orders-eu", "orders-eu-2", "staging-2"}, choiceNames(d.offered)) + assert.Equal(t, []string{"dev", "prod", "staging", "dep-bundle", "dep-local", "orders-eu", "orders-eu-2", "staging-2"}, choiceNames(d.offered)) for _, c := range d.offered { assert.Equal(t, !slices.Contains([]string{"dev", "prod", "staging"}, c.Name), c.Unlinked, c.Name) } diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index 787111646..0c096260a 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -171,6 +171,9 @@ type DeleteBundleInput struct { BundleType string Description string AstroV1Client astrov1.APIClient + // Login is the Astro login the delete runs under, whose host the + // Deployment lives on; nil is the current context. + Login *config.Context } // BundleDelete is what a bundle delete did: the deploy that removes the @@ -188,7 +191,7 @@ type BundleDelete struct { // It does not report the result: its caller renders the BundleDelete it // returns. func DeleteBundle(input *DeleteBundleInput) (BundleDelete, error) { - c, err := config.GetCurrentContext() + c, err := loginOrCurrent(input.Login) if err != nil { return BundleDelete{}, err } diff --git a/internal/platform/astro/deploy/bundletype.go b/internal/platform/astro/deploy/bundletype.go new file mode 100644 index 000000000..e0219942c --- /dev/null +++ b/internal/platform/astro/deploy/bundletype.go @@ -0,0 +1,159 @@ +package deploy + +import ( + "errors" + "fmt" + "os" + pathpkg "path" + "path/filepath" + "strings" + + "gopkg.in/yaml.v2" +) + +// BundleType is a kind of non-DAG bundle that astro deploy bundle knows by +// its contents: how to tell a directory holds one, how to check it, what the +// bundle is called, and where it mounts when the deploy names no mount path. +// Adding a type is adding an entry to bundleTypes. +type BundleType struct { + // Name is the type recorded on the deploy, and what --type selects. + Name string + // Detect reports whether dir holds a bundle of this type, for a deploy + // that names no type. nil detects nothing. + Detect func(dir string) bool + // Validate checks dir is a bundle of this type, before a mount path is + // derived from it. nil accepts any directory. + Validate func(dir string) error + // BundleName is the name of the bundle at dir (a dbt project's name). + // nil means bundles of this type have none. + BundleName func(dir string) (string, error) + // MountPathPrefix is the directory in Airflow a bundle of this type + // mounts below by default, at MountPathPrefix + its BundleName. A mount + // path below it says a bundle is of this type, for a delete that names + // no type and runs where no bundle is. Empty means no default. + MountPathPrefix string +} + +// HasDefaultMountPath reports whether a bundle of this type has a mount path +// without one being given. +func (t *BundleType) HasDefaultMountPath() bool { + return t.MountPathPrefix != "" && t.BundleName != nil +} + +// DefaultMountPath is where the bundle at dir mounts when no mount path is +// given: MountPathPrefix and the bundle's name. It is an error for a type +// with no default (HasDefaultMountPath). +func (t *BundleType) DefaultMountPath(dir string) (string, error) { + if !t.HasDefaultMountPath() { + return "", fmt.Errorf("a bundle of type %s has no default mount path", t.Name) + } + name, err := t.BundleName(dir) + if err != nil { + return "", err + } + return t.MountPathPrefix + name, nil +} + +// BundleTypeNone is the type of a bundle that names no type and that no +// registered type detects. It is a label alone, with no defaults. +const BundleTypeNone = "none" + +// DbtDefaultMountPathPrefix is where a dbt project mounts by default, below +// a directory named after the project. +const DbtDefaultMountPathPrefix = "/usr/local/airflow/dbt/" + +const dbtProjectFile = "dbt_project.yml" + +// bundleTypes is every bundle type with defaults, in the order detection +// tries them. +var bundleTypes = []BundleType{ + { + Name: "dbt", + Detect: isDbtProject, + Validate: validateDbtProject, + BundleName: dbtProjectName, + MountPathPrefix: DbtDefaultMountPathPrefix, + }, +} + +// ResolveBundleType is the type a bundle at dir deploys as. name is the type +// asked for: a registered type's name (in any case) selects that type, and +// any other is a free-form label with no defaults, as the deploy API takes +// any label. An empty name is the first registered type that detects dir, +// else BundleTypeNone. +func ResolveBundleType(name, dir string) BundleType { + if name == "" { + for _, t := range bundleTypes { + if t.Detect != nil && t.Detect(dir) { + return t + } + } + return BundleType{Name: BundleTypeNone} + } + for _, t := range bundleTypes { + if strings.EqualFold(t.Name, name) { + return t + } + } + return BundleType{Name: name} +} + +// BundleTypeForMountPath is the registered type whose default mount path +// mountPath has the shape of, MountPathPrefix and one path component, and +// whether there is one. The path is cleaned first, so .. cannot climb out of +// the prefix and a nested path is no default's. +func BundleTypeForMountPath(mountPath string) (BundleType, bool) { + clean := pathpkg.Clean(mountPath) + for _, t := range bundleTypes { + if t.MountPathPrefix == "" { + continue + } + rest, ok := strings.CutPrefix(clean, t.MountPathPrefix) + if ok && rest != "" && !strings.Contains(rest, "/") { + return t, true + } + } + return BundleType{}, false +} + +// BundleTypeNames is the name of every registered bundle type, in order. +func BundleTypeNames() []string { + names := make([]string, 0, len(bundleTypes)) + for _, t := range bundleTypes { + names = append(names, t.Name) + } + return names +} + +func isDbtProject(dir string) bool { + info, err := os.Stat(filepath.Join(dir, dbtProjectFile)) + return err == nil && info.Mode().IsRegular() +} + +func validateDbtProject(dir string) error { + path := filepath.Join(dir, dbtProjectFile) + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("no %s in %s: a dbt bundle is the directory that holds the dbt project's %s", dbtProjectFile, dir, dbtProjectFile) + } + return err +} + +// dbtProjectName is the project's name, from its dbt_project.yml. +func dbtProjectName(dir string) (string, error) { + path := filepath.Join(dir, dbtProjectFile) + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("dbt project name not found in %s: could not open %s: %w", dir, path, err) + } + defer file.Close() + var project map[string]any + if err := yaml.NewDecoder(file).Decode(&project); err != nil { + return "", fmt.Errorf("dbt project name not found in %s: could not decode %s: %w", dir, path, err) + } + name, ok := project["name"].(string) + if !ok || name == "" { + return "", fmt.Errorf("dbt project name not found in %s: invalid dbt project name", dir) + } + return name, nil +} diff --git a/internal/platform/astro/deploy/bundletype_test.go b/internal/platform/astro/deploy/bundletype_test.go new file mode 100644 index 000000000..7116483f8 --- /dev/null +++ b/internal/platform/astro/deploy/bundletype_test.go @@ -0,0 +1,87 @@ +package deploy + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func dbtDir(t *testing.T, project string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte(project), 0o600)) + return dir +} + +// The type a bundle deploys as, and the defaults it brings. +func TestResolveBundleType(t *testing.T) { + dbt := dbtDir(t, "name: jaffle_shop\n") + plain := t.TempDir() + + t.Run("a dbt project, detected", func(t *testing.T) { + bt := ResolveBundleType("", dbt) + assert.Equal(t, "dbt", bt.Name) + require.NoError(t, bt.Validate(dbt)) + name, err := bt.BundleName(dbt) + require.NoError(t, err) + assert.Equal(t, "jaffle_shop", name) + require.True(t, bt.HasDefaultMountPath()) + mount, err := bt.DefaultMountPath(dbt) + require.NoError(t, err) + assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", mount) + }) + t.Run("dbt asked for, in any case", func(t *testing.T) { + for _, name := range []string{"dbt", "DBT"} { + bt := ResolveBundleType(name, plain) + assert.Equal(t, "dbt", bt.Name, name) + assert.ErrorContains(t, bt.Validate(plain), "no dbt_project.yml in "+plain, name) + } + }) + t.Run("nothing detected", func(t *testing.T) { + bt := ResolveBundleType("", plain) + assert.Equal(t, BundleTypeNone, bt.Name) + assert.False(t, bt.HasDefaultMountPath()) + _, err := bt.DefaultMountPath(plain) + assert.ErrorContains(t, err, "has no default mount path") + }) + t.Run("any other type is a label with no defaults", func(t *testing.T) { + for _, name := range []string{"seeds", BundleTypeNone} { + bt := ResolveBundleType(name, dbt) + assert.Equal(t, name, bt.Name, "even where a dbt project is") + assert.False(t, bt.HasDefaultMountPath(), name) + assert.Nil(t, bt.Validate, name) + } + }) + t.Run("a dbt project with no name", func(t *testing.T) { + nameless := dbtDir(t, "version: 2\n") + bt := ResolveBundleType("", nameless) + _, err := bt.DefaultMountPath(nameless) + assert.ErrorContains(t, err, "dbt project name not found") + }) + t.Run("a directory called dbt_project.yml is no dbt project", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir, "dbt_project.yml"), 0o755)) + assert.Equal(t, BundleTypeNone, ResolveBundleType("", dir).Name) + }) + assert.Equal(t, []string{"dbt"}, BundleTypeNames()) +} + +// A mount path below a type's default directory says the type. +func TestBundleTypeForMountPath(t *testing.T) { + bt, ok := BundleTypeForMountPath("/usr/local/airflow/dbt/jaffle_shop") + require.True(t, ok) + assert.Equal(t, "dbt", bt.Name) + bt, ok = BundleTypeForMountPath("/usr/local/airflow/dbt/jaffle_shop/") + require.True(t, ok, "a trailing slash is cleaned away") + assert.Equal(t, "dbt", bt.Name) + for _, p := range []string{ + "/usr/local/airflow/dbt/", "/usr/local/airflow/dbt", "/usr/local/airflow/seeds", "", + "/usr/local/airflow/dbt/../seeds", "/usr/local/airflow/dbt/a/b", "/usr/local/airflow/dbt/a/..", + } { + _, ok := BundleTypeForMountPath(p) + assert.False(t, ok, p) + } +} diff --git a/internal/platform/astro/deploy/cosmosboost_hook_test.go b/internal/platform/astro/deploy/cosmosboost_hook_test.go index a560f2905..34e5c8dd0 100644 --- a/internal/platform/astro/deploy/cosmosboost_hook_test.go +++ b/internal/platform/astro/deploy/cosmosboost_hook_test.go @@ -193,7 +193,7 @@ func TestUploadBundleSkipsPreDeployByDefault(t *testing.T) { } // TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled drives the full -// `astro dbt deploy` bundle path (DeployBundle into UploadBundle). +// `astro deploy bundle` path (DeployBundle into UploadBundle). func TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled(t *testing.T) { setupCosmosBoostEnv(t) enableCosmosBoost(t) @@ -217,7 +217,7 @@ func TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled(t *testing.T) { }) require.NoError(t, err) - assert.Contains(t, uploaded(), cosmosBoostArtifact, "astro dbt deploy must ship the sidecar") + assert.Contains(t, uploaded(), cosmosBoostArtifact, "astro deploy bundle must ship the sidecar") assert.NoDirExists(t, filepath.Join(bundleDir, ".astro")) } @@ -300,7 +300,7 @@ func TestUploadBundleMissingDirIsUnchanged(t *testing.T) { } } -// TestUploadBundleFromTheWorkingDirectory: astro dbt deploy --project-path . +// TestUploadBundleFromTheWorkingDirectory: astro deploy bundle . // bundles ".", from the working directory, and an old sidecar there is still // replaced by the fresh one, at the same path, with no second copy under // another name. Nor does the tarball carry itself: it is written outside the diff --git a/internal/platform/astro/deployment/clone_run.go b/internal/platform/astro/deployment/clone_run.go index 94f9f1ce8..acba4f932 100644 --- a/internal/platform/astro/deployment/clone_run.go +++ b/internal/platform/astro/deployment/clone_run.go @@ -23,23 +23,43 @@ func CloneSource(ref, ws string, astroV1Client astrov1.APIClient) (astrov1.Deplo if ws == "" { return astrov1.Deployment{}, fmt.Errorf("no Workspace to look up the Deployment named %q in: pass --workspace, or its id to --clone", ref) } - deployments, err := ListDeployments(ws, "", astroV1Client) + named, err := DeploymentsNamed(ref, ws, "", astroV1Client) if err != nil { return astrov1.Deployment{}, err } - var ids []string - for i := range deployments { - if deployments[i].Name == ref { - ids = append(ids, deployments[i].Id) - } - } - switch len(ids) { + switch len(named) { case 0: return astrov1.Deployment{}, fmt.Errorf("no Deployment named %q in Workspace %s. Pass its id to --clone to copy one from another Workspace", ref, ws) case 1: - return GetDeploymentByID("", ids[0], astroV1Client) + return GetDeploymentByID("", named[0].Id, astroV1Client) + } + return astrov1.Deployment{}, fmt.Errorf("%d Deployments are named %q in Workspace %s: pass one of their ids to --clone instead: %s", len(named), ref, ws, strings.Join(DeploymentIDs(named), ", ")) +} + +// DeploymentsNamed is every Deployment in workspace ws called name, listed +// in organization orgID ("" for the current context's). A name is not +// unique, so a caller decides what none, or several, means. +func DeploymentsNamed(name, ws, orgID string, astroV1Client astrov1.APIClient) ([]astrov1.Deployment, error) { + deployments, err := ListDeployments(ws, orgID, astroV1Client) + if err != nil { + return nil, err + } + var named []astrov1.Deployment + for i := range deployments { + if deployments[i].Name == name { + named = append(named, deployments[i]) + } + } + return named, nil +} + +// DeploymentIDs is the id of each of ds, in order. +func DeploymentIDs(ds []astrov1.Deployment) []string { + ids := make([]string, len(ds)) + for i := range ds { + ids[i] = ds[i].Id } - return astrov1.Deployment{}, fmt.Errorf("%d Deployments are named %q in Workspace %s: pass one of their ids to --clone instead: %s", len(ids), ref, ws, strings.Join(ids, ", ")) + return ids } // Clone creates a copy of src named name, in workspaceID or else src's own diff --git a/pkg/scaffold/linkwrite.go b/pkg/scaffold/linkwrite.go index d46545e1e..d9a4fe34b 100644 --- a/pkg/scaffold/linkwrite.go +++ b/pkg/scaffold/linkwrite.go @@ -33,6 +33,18 @@ import ( // because the parser would refuse it. The message names the field. var ErrInvalidLink = errors.New("invalid link") +// BundleLinkName is a name no new link may have: astro deploy reads astro +// deploy bundle as its subcommand that deploys a non-DAG bundle, so a deploy +// could never reach a link by that name as its argument. A link already +// called that still loads, is reached with --deployment, and saves again: only +// adding one is refused. +const BundleLinkName = "bundle" + +// ErrBundleLinkName is SaveLink refusing BundleLinkName. It wraps +// ErrInvalidLink. +var ErrBundleLinkName = fmt.Errorf("%w: %s cannot name a link: astro deploy %s deploys a non-DAG bundle, "+ + "so astro deploy %s would never reach the link. Name the link something else", ErrInvalidLink, BundleLinkName, BundleLinkName, BundleLinkName) + // ErrComposerProjectRequired and ErrComposerLocationRequired report a Composer // link saved without the Google Cloud project or the region holding its // environment. The parser does not require them, but an environment name alone @@ -197,6 +209,9 @@ func SaveLink(dir string, wrap func(run func() error) error, l Link) error { //nolint:gocritic // hugeParam: by value, as SaveLink holds it func (l Link) edit() ManifestEdit { return func(before *manifest.Manifest, ed tomledit.Editor) error { + if _, exists := before.Astro.Deployments[l.Name]; l.Name == BundleLinkName && !exists { + return ErrBundleLinkName + } key := linkKey(l.Name) // Read from the raw TOML, before the entry is replaced: the parsed // link has the defaults folded in, so it cannot say which keys the link diff --git a/pkg/scaffold/linkwrite_test.go b/pkg/scaffold/linkwrite_test.go index 1da25329e..eed081fe7 100644 --- a/pkg/scaffold/linkwrite_test.go +++ b/pkg/scaffold/linkwrite_test.go @@ -189,6 +189,24 @@ func TestSaveLinkRefusesAnInvalidLink(t *testing.T) { } } +// A new link cannot be called bundle, the word astro deploy bundle takes; one +// already called that saves again. +func TestSaveLinkRefusesANewBundleLink(t *testing.T) { + dir, path := linkProject(t, linkFixture) + err := SaveLink(dir, nil, Link{Name: BundleLinkName, Kind: manifest.KindAstro, Workspace: "W", Deployment: "d"}) + require.ErrorIs(t, err, ErrBundleLinkName) + require.ErrorIs(t, err, ErrInvalidLink) + assert.Contains(t, err.Error(), "astro deploy bundle deploys a non-DAG bundle") + assert.Equal(t, linkFixture, readFile(t, path)) + + existing := linkFixture + "\n[tool.astro.deployments.bundle]\nworkspace = 'W'\ndeployment = 'old'\n" + dir, path = linkProject(t, existing) + require.NoError(t, SaveLink(dir, nil, Link{Name: BundleLinkName, Kind: manifest.KindAstro, Deployment: "new"})) + m, err := manifest.Load(path) + require.NoError(t, err) + assert.Equal(t, "new", m.Astro.Deployments[BundleLinkName].Deployment) +} + // A credential pasted where an env var name belongs is refused, and the error // does not quote it: the message may land in a terminal or a CI log, and the // manifest is committed.