From 9c1a3a595a532caf8ce7739713ca8ec2ba09f58f Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 12:11:07 -0400 Subject: [PATCH 1/7] Generate Cosmos Boost sidecars into the deploy bundle, not the project, and retire astro dbt cleanup The pre-deploy step produces its .astro/dbt_metadata.json and .astro/manifest.slim.json in memory, and the bundle tarball carries them at the same paths with the same bytes, through a TarOverlay on fileutil.Tar. Nothing is written into the user's tree. Every bundle deploy leaves out the sidecars an earlier CLI wrote into the tree (producer marker "astro"), enabled or not, so a stale one never ships. One from an unrecognized producer still stops an enabled deploy, and ships untouched when the step is off. With nothing left to clean, astro dbt cleanup becomes a removed-command stub, and its cleanup code, json shape and golden go. Co-Authored-By: Claude --- cmd/astro/dbt.go | 44 +- cmd/astro/dbt_output_test.go | 75 +-- cmd/astro/dbt_render.go | 26 +- cmd/astro/dbt_test.go | 16 - cmd/astro/output_binding_test.go | 8 - cmd/astro/schema_test.go | 3 +- cmd/astro/testdata/schema/dbt-cleanup.json | 12 - internal/platform/astro/deploy/bundle.go | 20 +- .../astro/deploy/cosmosboost_hook_test.go | 283 +++++---- pkg/cosmosboost/cleanup.go | 82 --- pkg/cosmosboost/cleanup_test.go | 186 ------ pkg/cosmosboost/precompute/cleanup.go | 198 ------- pkg/cosmosboost/precompute/cleanup_test.go | 546 ------------------ pkg/cosmosboost/precompute/existing.go | 92 +++ pkg/cosmosboost/precompute/existing_test.go | 126 ++++ pkg/cosmosboost/precompute/hash.go | 6 +- pkg/cosmosboost/precompute/metadata.go | 30 +- pkg/cosmosboost/precompute/metadata_test.go | 21 - pkg/cosmosboost/precompute/precompute.go | 124 ++-- pkg/cosmosboost/precompute/precompute_test.go | 189 ++++-- pkg/cosmosboost/predeploy.go | 146 +++-- pkg/cosmosboost/predeploy_test.go | 266 +++++---- pkg/fileutil/files.go | 108 +++- pkg/fileutil/files_test.go | 61 +- 24 files changed, 1031 insertions(+), 1637 deletions(-) delete mode 100644 cmd/astro/testdata/schema/dbt-cleanup.json delete mode 100644 pkg/cosmosboost/cleanup.go delete mode 100644 pkg/cosmosboost/cleanup_test.go delete mode 100644 pkg/cosmosboost/precompute/cleanup.go delete mode 100644 pkg/cosmosboost/precompute/cleanup_test.go create mode 100644 pkg/cosmosboost/precompute/existing.go create mode 100644 pkg/cosmosboost/precompute/existing_test.go delete mode 100644 pkg/cosmosboost/precompute/metadata_test.go diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index 4cb758bc3..b7e879058 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -17,7 +17,6 @@ import ( "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/internal/platform/astro/deployment" - "github.com/astronomer/astro-cli/pkg/cosmosboost" ) var ( @@ -47,36 +46,29 @@ func newDbtCmd(out io.Writer) *cobra.Command { cmd.AddCommand( newDbtDeployCmd(), newDbtDeleteCmd(), - newDbtCleanupCmd(), + newDbtCleanupRemovedCmd(), ) applyPreferredFlagsIn(cmd) return cmd } -func newDbtCleanupCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "cleanup [PATH]...", - Args: cobra.ArbitraryArgs, - Short: "Remove the Cosmos Boost artifacts under each path", - Long: "Remove the artifacts the Cosmos Boost pre-deploy step wrote under each path (default: the current directory). Run it after disabling cosmos_boost.pre_deploy, because a disabled deploy leaves earlier deploys' artifacts in place.", - RunE: cleanupDbt, - Example: ` # Remove the artifacts under the current directory - astro dbt cleanup - - # Remove them under two dbt projects - astro dbt cleanup `, - } - cliout.AddOutputFlag(cmd, &dbtOutput) - return cmd -} - -func cleanupDbt(cmd *cobra.Command, args []string) error { - cmd.SilenceUsage = true - report, err := cosmosboost.Cleanup(args...) - if err != nil { - return err - } - return cliout.Renderer{Format: dbtOutput, Out: cmd.OutOrStdout()}.Emit(newDbtCleanupJSON(&report), cliout.Text(renderDbtCleanup)) +// errDbtCleanupRemoved is what astro dbt cleanup says now. It removed the +// Cosmos Boost sidecars earlier deploys wrote into the project; deploys now +// generate them into the bundle they upload and leave out any an earlier +// version wrote, so there is nothing left for it to do. +const errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + + "Cosmos Boost no longer writes into the project, so there is nothing to clean up. " + + "A deploy generates its sidecars into the bundle it uploads, and never ships the " + + ".astro/dbt_metadata.json or .astro/manifest.slim.json files an earlier version wrote into the project; " + + "delete those by hand whenever you like" + +// newDbtCleanupRemovedCmd builds the astro dbt cleanup tombstone +// (cliout.RemovedCommand), so a script that still runs it is told why rather +// than shown the dbt group's help. +func newDbtCleanupRemovedCmd() *cobra.Command { + return removedCmdStub("cleanup", nil, "Removed in v2 — Cosmos Boost no longer writes into the project", func([]string) string { + return errDbtCleanupRemoved + }) } //nolint:dupl // the duplication is acceptable here diff --git a/cmd/astro/dbt_output_test.go b/cmd/astro/dbt_output_test.go index 1493acbdd..a20845a8b 100644 --- a/cmd/astro/dbt_output_test.go +++ b/cmd/astro/dbt_output_test.go @@ -132,15 +132,6 @@ func TestDbtTextOutputUnchanged(t *testing.T) { require.NoError(t, r.err, "stderr:\n%s", r.stderr) assert.Equal(t, "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) }) - t.Run("cleanup", func(t *testing.T) { - dir := t.TempDir() - artifact := filepath.Join(dir, ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(artifact), 0o755)) - require.NoError(t, os.WriteFile(artifact, []byte(`{"generated_by": {"application": "astro"}}`), 0o600)) - r := execDbtRun(t, nil, "cleanup", dir) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Removed the Cosmos Boost artifacts\n", r.stdout) - }) } // stubBundleWait stands in for the wait a --wait does, writing one line of @@ -331,52 +322,30 @@ func TestPublishThenWaitReportsBothFailures(t *testing.T) { assert.Equal(t, cliout.ExitFailure, cliout.ExitCode(context.Background(), err)) } -func TestDbtCleanupJSON(t *testing.T) { +// astro dbt cleanup is a tombstone: it touches nothing, and says why in text +// and as the one error object under json, whatever paths and flags it gets. +func TestDbtCleanupRemoved(t *testing.T) { dir := t.TempDir() - // The cleanup walks the directory with symlinks resolved (macOS keeps - // temp dirs under /private/var), and reports the files at those paths. - walked, err := filepath.EvalSymlinks(dir) - require.NoError(t, err) - ours := filepath.Join(walked, ".astro", "dbt_metadata.json") - foreign := filepath.Join(walked, "other", ".astro", "dbt_metadata.json") - for path, app := range map[string]string{ours: "astro", foreign: "someone-else"} { - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(`{"generated_by": {"application": "`+app+`"}}`), 0o600)) - } - - r := execDbtRun(t, nil, "cleanup", dir, "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got dbtCleanupJSON + artifact := filepath.Join(dir, ".astro", "dbt_metadata.json") + require.NoError(t, os.MkdirAll(filepath.Dir(artifact), 0o755)) + require.NoError(t, os.WriteFile(artifact, []byte(`{"generated_by": {"application": "astro"}}`), 0o600)) + + r := execDbtRun(t, nil, "cleanup", dir) + require.Error(t, r.err) + assert.Equal(t, cliout.ExitUsage, r.code) + assert.Equal(t, errDbtCleanupRemoved, r.err.Error()) + assert.Contains(t, r.err.Error(), "nothing to clean up") + assert.NotContains(t, r.err.Error(), "`", "command names are plain text") + assert.Empty(t, r.stdout) + assert.FileExists(t, artifact, "the tombstone removes nothing") + + r = execDbtRun(t, nil, "cleanup", dir, "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitUsage, r.code) + var got errorJSON decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtCleanupJSON{Action: "removed", Paths: []string{dir}, Removed: []string{ours}, Kept: []string{foreign}}, got) - - t.Run("nothing to remove", func(t *testing.T) { - r := execDbtRun(t, nil, "cleanup", t.TempDir(), "-o", "json") - require.NoError(t, r.err) - var got dbtCleanupJSON - fields := decodeOne(t, r.stdout, &got) - assert.JSONEq(t, "[]", string(fields["removed"])) - assert.JSONEq(t, "[]", string(fields["kept"])) - }) - t.Run("no path: the current directory, absolute", func(t *testing.T) { - here := t.TempDir() - t.Chdir(here) - r := execDbtRun(t, nil, "cleanup", "-o", "json") - require.NoError(t, r.err) - var got dbtCleanupJSON - decodeOne(t, r.stdout, &got) - wd, err := os.Getwd() - require.NoError(t, err) - assert.Equal(t, []string{wd}, got.Paths) - }) - t.Run("a path that does not exist", func(t *testing.T) { - r := execDbtRun(t, nil, "cleanup", filepath.Join(t.TempDir(), "missing"), "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "missing") - }) + assert.Equal(t, errDbtCleanupRemoved, got.Error) + assert.FileExists(t, artifact) } // Under --output json the deployment picker is refused as input_required, diff --git a/cmd/astro/dbt_render.go b/cmd/astro/dbt_render.go index 40eed0d0a..a6f82547d 100644 --- a/cmd/astro/dbt_render.go +++ b/cmd/astro/dbt_render.go @@ -1,6 +1,6 @@ package astro -// Rendering for `astro dbt deploy|delete|cleanup`, `astro deploy --non-dags` +// Rendering for `astro dbt deploy|delete`, `astro deploy --non-dags` // and `astro remote deploy`. The platform packages return what happened; this // file decides how it looks, in text and in json. @@ -16,7 +16,6 @@ import ( "github.com/astronomer/astro-cli/cmd/cliout" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/pkg/ansi" - "github.com/astronomer/astro-cli/pkg/cosmosboost" ) // dbtOutput is --output for the `dbt` family. @@ -65,16 +64,6 @@ type dbtDeleteJSON struct { WaitError string `json:"wait_error,omitempty"` } -// dbtCleanupJSON is what `astro dbt cleanup --output json` publishes: the -// paths it scanned, the artifacts it removed, and the ones it kept because -// something other than the CLI wrote them, each an absolute path. -type dbtCleanupJSON struct { - Action string `json:"action"` - Paths []string `json:"paths"` - Removed []string `json:"removed"` - Kept []string `json:"kept"` -} - // remoteDeployJSON is what `astro remote deploy --output json` publishes: the // client image it pushed, which is what the agents' Helm values then name. type remoteDeployJSON struct { @@ -200,19 +189,6 @@ func renderDbtDeleted(mountPath string) func(io.Writer) error { }) } -func newDbtCleanupJSON(report *cosmosboost.CleanupReport) dbtCleanupJSON { - return dbtCleanupJSON{ - Action: "removed", - Paths: nonNil(report.Roots), - Removed: nonNil(report.Removed), - Kept: nonNil(report.Kept), - } -} - -func renderDbtCleanup(b *bufio.Writer) { - fmt.Fprintln(b, "Removed the Cosmos Boost artifacts") -} - func newRemoteDeployJSON(res *astrodeploy.ClientDeploy) remoteDeployJSON { obj := remoteDeployJSON{ Image: res.Image, diff --git a/cmd/astro/dbt_test.go b/cmd/astro/dbt_test.go index a6106af3d..871f1d633 100644 --- a/cmd/astro/dbt_test.go +++ b/cmd/astro/dbt_test.go @@ -310,19 +310,3 @@ func (s *DbtSuite) mockGetTestDeployment() { }, }, nil) } - -func (s *DbtSuite) TestDbtCleanup_RemovesArtifacts() { - dir := s.T().TempDir() - artifact := filepath.Join(dir, ".astro", "dbt_metadata.json") - assert.NoError(s.T(), os.MkdirAll(filepath.Dir(artifact), 0o755)) - assert.NoError(s.T(), os.WriteFile(artifact, []byte(`{"generated_by": {"application": "astro"}}`), 0o644)) - - err := testExecCmd(newDbtCleanupCmd(), dir) - assert.NoError(s.T(), err) - assert.NoFileExists(s.T(), artifact) -} - -func (s *DbtSuite) TestDbtCleanup_NonexistentPath() { - err := testExecCmd(newDbtCleanupCmd(), filepath.Join(s.T().TempDir(), "does-not-exist")) - assert.Error(s.T(), err) -} diff --git a/cmd/astro/output_binding_test.go b/cmd/astro/output_binding_test.go index bd98c3f36..93509c225 100644 --- a/cmd/astro/output_binding_test.go +++ b/cmd/astro/output_binding_test.go @@ -85,14 +85,6 @@ func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { assert.NotContains(t, stderr, dbtTestVersion, "the result went to stderr") assert.Contains(t, stderr, "Initiating dbt deploy", "the notes go to stderr") }) - t.Run("dbt cleanup", func(t *testing.T) { - stdout, stderr, err := execUnboundRoot(t, nil, "dbt", "cleanup", t.TempDir(), "-o", "json") - require.NoError(t, err, "stderr:\n%s", stderr) - var got dbtCleanupJSON - decodeOne(t, stdout, &got) - assert.Equal(t, "removed", got.Action) - assert.Empty(t, stderr) - }) // The bundle sits in a git checkout with uncommitted changes, so the // platform code prints its note on the way: under json that goes to // stderr, and stdout is the one result object. diff --git a/cmd/astro/schema_test.go b/cmd/astro/schema_test.go index 4e42696f4..46113d3de 100644 --- a/cmd/astro/schema_test.go +++ b/cmd/astro/schema_test.go @@ -155,10 +155,9 @@ var publishedPayloads = []cliouttest.Case{ {Name: "deploy", Value: deployJSON{}}, {Name: "deploy-non-dags", Value: nonDagsDeployJSON{}}, // astro dbt deploy: what it deployed where. delete: the bundle it - // removed. cleanup: the artifacts it removed and kept. + // removed. {Name: "dbt-deploy", Value: dbtDeployJSON{}}, {Name: "dbt-delete", Value: dbtDeleteJSON{}}, - {Name: "dbt-cleanup", Value: dbtCleanupJSON{}}, // astro remote deploy: the client image it pushed. {Name: "remote-deploy", Value: remoteDeployJSON{}}, diff --git a/cmd/astro/testdata/schema/dbt-cleanup.json b/cmd/astro/testdata/schema/dbt-cleanup.json deleted file mode 100644 index c1813a7b0..000000000 --- a/cmd/astro/testdata/schema/dbt-cleanup.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "action": "x", - "paths": [ - "x" - ], - "removed": [ - "x" - ], - "kept": [ - "x" - ] -} diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index fa79ea9d3..b03471d5e 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -296,21 +296,17 @@ func UploadBundle(tarDirPath, bundlePath, uploadURL string, prependBaseDir bool, } }() - // Cosmos Boost pre-deploy step, opt-in via cosmos_boost.pre_deploy; with - // the setting off the deploy does not touch the tree at all. Cleanup runs - // first and is fatal on failure (a stale artifact must not ship inside the - // bundle), while stamping is best-effort (a missing artifact is safe). - // Artifacts left by earlier enabled deploys are removed with - // `astro dbt cleanup`. - if config.CFG.CosmosBoostPreDeploy.GetBool() { - if err := cosmosboost.EnsureClean(bundlePath); err != nil { - return "", err - } - cosmosboost.BestEffortPreDeploy(bundlePath) + // Cosmos Boost: its sidecars go into the tarball only, never into the + // tree. Ones an earlier CLI wrote into the tree are always left out (a + // stale one must not ship), and with cosmos_boost.pre_deploy on, fresh + // ones are generated into the bundle, best-effort (a missing one is safe). + overlay, err := cosmosboost.BundleOverlay(bundlePath, config.CFG.CosmosBoostPreDeploy.GetBool()) + if err != nil { + return "", err } // Generate the bundle tar - err := fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}) + err = fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}, overlay) if err != nil { return "", err } diff --git a/internal/platform/astro/deploy/cosmosboost_hook_test.go b/internal/platform/astro/deploy/cosmosboost_hook_test.go index 26c2bcea8..58d44a4a1 100644 --- a/internal/platform/astro/deploy/cosmosboost_hook_test.go +++ b/internal/platform/astro/deploy/cosmosboost_hook_test.go @@ -1,10 +1,13 @@ package deploy import ( + "archive/tar" + "compress/gzip" + "encoding/json" "io" + "io/fs" "os" "path/filepath" - "runtime" "testing" "github.com/stretchr/testify/assert" @@ -15,11 +18,15 @@ import ( testUtil "github.com/astronomer/astro-cli/pkg/testing" ) -// cosmosBoostArtifact is the sidecar the pre-deploy step writes next to a dbt -// project; these tests assert it is produced (or removed) at the right moment -// in each deploy path. +// cosmosBoostArtifact is the sidecar the pre-deploy step generates for a dbt +// project, at the path the Cosmos Boost plugin reads it from in the bundle. const cosmosBoostArtifact = ".astro/dbt_metadata.json" +const ( + staleOursSidecar = `{"schema":1,"version":{"algo":"sha256-tree-v2","hash":"stale"},"generated_by":{"application":"astro","version":"1.0.0"}}` + foreignSidecar = `{"schema":1,"version":{"hash":"abc"},"generated_by":{"application":"someone-else"}}` +) + // setupCosmosBoostEnv isolates ~/.astro for the test. func setupCosmosBoostEnv(t *testing.T) { t.Helper() @@ -29,6 +36,11 @@ func setupCosmosBoostEnv(t *testing.T) { t.Cleanup(func() { config.HomeConfigPath = origHome }) } +func enableCosmosBoost(t *testing.T) { + t.Helper() + require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) +} + func writeDbtBundle(t *testing.T) string { t.Helper() bundleDir := t.TempDir() @@ -36,26 +48,102 @@ func writeDbtBundle(t *testing.T) string { return bundleDir } -func TestUploadBundleRunsPreDeployWhenEnabled(t *testing.T) { - setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) - - bundleDir := writeDbtBundle(t) +func writeTreeFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + return path +} +// captureUploads points azureUploader at a recorder and returns the entries of +// the last tarball uploaded, by name. +func captureUploads(t *testing.T) func() map[string]string { + t.Helper() + var entries map[string]string azureUploader = func(sasLink string, file io.Reader) (string, error) { + entries = map[string]string{} + gz, err := gzip.NewReader(file) + require.NoError(t, err) + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + data, err := io.ReadAll(tr) + require.NoError(t, err) + entries[header.Name] = string(data) + } return "version-id", nil } + return func() map[string]string { return entries } +} + +// treeSnapshot maps every file under root to its contents. +func treeSnapshot(t *testing.T, root string) map[string]string { + t.Helper() + out := map[string]string{} + require.NoError(t, filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + out[path] = string(data) + return err + })) + return out +} + +// TestUploadBundleGeneratesSidecarsIntoTheBundle pins the change this step +// exists for: with the setting on, the sidecar ships in the tarball at the +// path the plugin reads, and the project tree is left exactly as it was. +func TestUploadBundleGeneratesSidecarsIntoTheBundle(t *testing.T) { + setupCosmosBoostEnv(t) + enableCosmosBoost(t) + bundleDir := writeDbtBundle(t) + uploaded := captureUploads(t) + before := treeSnapshot(t, bundleDir) _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) - assert.FileExists(t, filepath.Join(bundleDir, cosmosBoostArtifact), - "the pre-deploy step must stamp the bundle before it is tarred") + assert.Equal(t, before, treeSnapshot(t, bundleDir), "the deploy must not write into the project") + sidecar, ok := uploaded()[cosmosBoostArtifact] + require.True(t, ok, "the sidecar must ship in the bundle; got %v", uploaded()) + var meta struct { + Version struct { + Hash string `json:"hash"` + } `json:"version"` + GeneratedBy struct { + Application string `json:"application"` + } `json:"generated_by"` + } + require.NoError(t, json.Unmarshal([]byte(sidecar), &meta)) + assert.Len(t, meta.Version.Hash, 64, "a sha256 version.hash") + assert.Equal(t, "astro", meta.GeneratedBy.Application) +} + +// TestUploadBundleSidecarPathsUnderTheDagsBaseDir: a DAG deploy prepends the +// directory's own name, and the sidecars sit under it like every other file. +func TestUploadBundleSidecarPathsUnderTheDagsBaseDir(t *testing.T) { + setupCosmosBoostEnv(t) + enableCosmosBoost(t) + dagsDir := filepath.Join(t.TempDir(), "dags") + writeTreeFile(t, dagsDir, "dbt/shop/dbt_project.yml", "name: shop\n") + uploaded := captureUploads(t) + + _, err := UploadBundle(t.TempDir(), dagsDir, "http://upload-url", true, "0.0.0") + require.NoError(t, err) + + assert.Contains(t, uploaded(), "dags/dbt/shop/"+cosmosBoostArtifact) + assert.Contains(t, uploaded(), "dags/dbt/shop/dbt_project.yml") } // TestUploadBundleSlimManifest pins the slim manifest's gating end to end: on // by default once the step is enabled, and off when the per-feature env var -// says so. Either way the hash sidecar still lands - the two optimizations are +// says so. Either way the hash sidecar still ships - the two optimizations are // independent. func TestUploadBundleSlimManifest(t *testing.T) { for _, tc := range []struct { @@ -68,31 +156,25 @@ func TestUploadBundleSlimManifest(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) + enableCosmosBoost(t) if tc.envVar != "" { t.Setenv("ASTRO_COSMOS_BOOST_SLIM_MANIFEST_ENABLED", tc.envVar) } bundleDir := writeDbtBundle(t) - manifest := `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}` - require.NoError(t, os.MkdirAll(filepath.Join(bundleDir, "target"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(bundleDir, "target", "manifest.json"), []byte(manifest), 0o644)) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } + writeTreeFile(t, bundleDir, "target/manifest.json", `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}`) + uploaded := captureUploads(t) _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) - astroDir := filepath.Join(bundleDir, "target", ".astro") - assert.FileExists(t, filepath.Join(astroDir, "dbt_metadata.json"), "the manifest must still be stamped") - slim := filepath.Join(astroDir, "manifest.slim.json") + assert.Contains(t, uploaded(), "target/.astro/dbt_metadata.json", "the manifest must still be stamped") if tc.wantSlim { - assert.FileExists(t, slim) + assert.Contains(t, uploaded(), "target/.astro/manifest.slim.json") } else { - assert.NoFileExists(t, slim) + assert.NotContains(t, uploaded(), "target/.astro/manifest.slim.json") } + assert.NoDirExists(t, filepath.Join(bundleDir, "target", ".astro"), "nothing is written into the project") }) } } @@ -100,26 +182,21 @@ func TestUploadBundleSlimManifest(t *testing.T) { func TestUploadBundleSkipsPreDeployByDefault(t *testing.T) { setupCosmosBoostEnv(t) // cosmos_boost.pre_deploy defaults to false, so nothing may be produced. - bundleDir := writeDbtBundle(t) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } + uploaded := captureUploads(t) _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) - assert.NoFileExists(t, filepath.Join(bundleDir, cosmosBoostArtifact), - "opt-in gate is off by default; nothing may be written") + assert.NotContains(t, uploaded(), cosmosBoostArtifact, "opt-in gate is off by default") + assert.NoDirExists(t, filepath.Join(bundleDir, ".astro")) } -// TestDeployBundleDbtPathRunsPreDeployWhenEnabled drives the full -// `astro dbt deploy` bundle path (DeployBundle into UploadBundle) to pin that -// moving the hook out of cmd/astro/dbt.go did not lose dbt-deploy coverage. -func TestDeployBundleDbtPathRunsPreDeployWhenEnabled(t *testing.T) { +// TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled drives the full +// `astro dbt deploy` bundle path (DeployBundle into UploadBundle). +func TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled(t *testing.T) { setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) + enableCosmosBoost(t) canCiCdDeploy = func(token string) bool { return true } @@ -129,9 +206,7 @@ func TestDeployBundleDbtPathRunsPreDeployWhenEnabled(t *testing.T) { mockGetDeployment(mockV1Client, true, false) mockCreateDeploy(mockV1Client, "http://bundle-upload-url", nil) mockUpdateDeploy(mockV1Client, "version-id") - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } + uploaded := captureUploads(t) _, err := DeployBundle(&DeployBundleInput{ BundlePath: bundleDir, @@ -142,80 +217,66 @@ func TestDeployBundleDbtPathRunsPreDeployWhenEnabled(t *testing.T) { }) require.NoError(t, err) - assert.FileExists(t, filepath.Join(bundleDir, cosmosBoostArtifact), - "astro dbt deploy must still run the pre-deploy step after the hook moved into UploadBundle") + assert.Contains(t, uploaded(), cosmosBoostArtifact, "astro dbt deploy must ship the sidecar") + assert.NoDirExists(t, filepath.Join(bundleDir, ".astro")) } -// TestUploadBundleFailsWhenStaleArtifactUnremovable pins the safety property -// of ENABLED deploys: a deploy that cannot guarantee the payload is free of -// stale artifacts must fail rather than ship one, because consumers cannot -// tell fresh from stale. -func TestUploadBundleFailsWhenStaleArtifactUnremovable(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory write-permission semantics differ on windows") - } - setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) - - bundleDir := writeDbtBundle(t) - stale := filepath.Join(bundleDir, cosmosBoostArtifact) - require.NoError(t, os.MkdirAll(filepath.Dir(stale), 0o755)) - require.NoError(t, os.WriteFile(stale, []byte(`{"generated_by": {"application": "astro"}}`), 0o644)) - require.NoError(t, os.Chmod(filepath.Dir(stale), 0o555)) // file cannot be unlinked - t.Cleanup(func() { _ = os.Chmod(filepath.Dir(stale), 0o755) }) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil +// TestUploadBundleLeavesOutStaleSidecars: a sidecar an earlier CLI wrote into +// the project never ships, whether the step is on (a fresh one replaces it) +// or off (none ships), and the deploy leaves it in the tree. +func TestUploadBundleLeavesOutStaleSidecars(t *testing.T) { + for _, enabled := range []bool{true, false} { + setupCosmosBoostEnv(t) + if enabled { + enableCosmosBoost(t) + } + bundleDir := writeDbtBundle(t) + stale := writeTreeFile(t, bundleDir, cosmosBoostArtifact, staleOursSidecar) + writeTreeFile(t, bundleDir, "gone/.astro/manifest.slim.json", `{"_generated_by":{"application":"astro"}}`) + uploaded := captureUploads(t) + before := treeSnapshot(t, bundleDir) + + _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + require.NoError(t, err) + + assert.Equal(t, before, treeSnapshot(t, bundleDir), "enabled=%v: the tree must not change", enabled) + assert.FileExists(t, stale) + assert.NotContains(t, uploaded(), "gone/.astro/manifest.slim.json", "enabled=%v", enabled) + if enabled { + require.Contains(t, uploaded(), cosmosBoostArtifact) + assert.NotContains(t, uploaded()[cosmosBoostArtifact], `"stale"`, "the fresh sidecar replaces the stale one") + } else { + assert.NotContains(t, uploaded(), cosmosBoostArtifact, "a disabled deploy ships no sidecar, stale or fresh") + } } - - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") - - require.Error(t, err, "the deploy must not proceed with a stale artifact it could not remove") - assert.ErrorContains(t, err, "Cosmos Boost") } -// TestUploadBundleIsANoOpWhenDisabled pins the opt-out contract (review ask on -// astronomer/astro-cli#2236): with the gate off the deploy neither walks nor -// mutates the tree - even an artifact left by an earlier enabled deploy stays -// in place, and removing it is `astro dbt cleanup`'s job. -func TestUploadBundleIsANoOpWhenDisabled(t *testing.T) { - setupCosmosBoostEnv(t) - - bundleDir := writeDbtBundle(t) - leftover := filepath.Join(bundleDir, cosmosBoostArtifact) - require.NoError(t, os.MkdirAll(filepath.Dir(leftover), 0o755)) - require.NoError(t, os.WriteFile(leftover, []byte(`{"generated_by": {"application": "astro"}}`), 0o644)) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } - - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") - require.NoError(t, err) - - assert.FileExists(t, leftover, - "a disabled deploy must not touch the tree; leftovers are cleaned by astro dbt cleanup") -} - -// TestUploadBundleFailsOnForeignSidecar: an enabled deploy must not ship a -// sidecar from an unrecognized producer - cleanup will not delete it, the -// plugin would consume it, so the deploy stops and asks for manual removal. -func TestUploadBundleFailsOnForeignSidecar(t *testing.T) { - setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) - - bundleDir := writeDbtBundle(t) - foreign := filepath.Join(bundleDir, cosmosBoostArtifact) - require.NoError(t, os.MkdirAll(filepath.Dir(foreign), 0o755)) - require.NoError(t, os.WriteFile(foreign, []byte(`{"schema": 1, "version": {"hash": "abc"}, "generated_by": {"application": "someone-else"}}`), 0o644)) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } - - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") - - require.Error(t, err) - assert.ErrorContains(t, err, "unrecognized producer") - assert.FileExists(t, foreign, "the foreign file must be preserved") +// TestUploadBundleForeignSidecar: a sidecar from an unrecognized producer is +// the user's file. Disabled, it ships untouched; enabled, the deploy stops and +// asks for its removal, because the plugin would consume its version.hash. +func TestUploadBundleForeignSidecar(t *testing.T) { + t.Run("disabled ships it", func(t *testing.T) { + setupCosmosBoostEnv(t) + bundleDir := writeDbtBundle(t) + writeTreeFile(t, bundleDir, cosmosBoostArtifact, foreignSidecar) + uploaded := captureUploads(t) + + _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + require.NoError(t, err) + assert.Equal(t, foreignSidecar, uploaded()[cosmosBoostArtifact]) + }) + t.Run("enabled refuses", func(t *testing.T) { + setupCosmosBoostEnv(t) + enableCosmosBoost(t) + bundleDir := writeDbtBundle(t) + foreign := writeTreeFile(t, bundleDir, cosmosBoostArtifact, foreignSidecar) + uploaded := captureUploads(t) + + _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + + require.Error(t, err) + assert.ErrorContains(t, err, "unrecognized producer") + assert.Nil(t, uploaded(), "nothing may be uploaded") + assert.FileExists(t, foreign, "the foreign file must be preserved") + }) } diff --git a/pkg/cosmosboost/cleanup.go b/pkg/cosmosboost/cleanup.go deleted file mode 100644 index c0efec9ad..000000000 --- a/pkg/cosmosboost/cleanup.go +++ /dev/null @@ -1,82 +0,0 @@ -package cosmosboost - -import ( - "fmt" - "path/filepath" -) - -// CleanupReport is what Cleanup did: the roots it scanned, as given but -// absolute, the artifacts it removed, and the ones it kept because their -// producer marker is not this tool's. Every path is absolute. A root that is -// a symlink is scanned at its target, so the artifacts under it are reported -// at their resolved paths. -type CleanupReport struct { - Roots []string - Removed []string - Kept []string -} - -// Cleanup removes the Cosmos Boost artifacts under the given roots (default -// "."). It fails when any artifact it owns could not be removed. It does not -// report the result: its caller renders the CleanupReport it returns. -func Cleanup(roots ...string) (CleanupReport, error) { - return cleanup(roots, filepath.Abs) -} - -// cleanup is Cleanup with the way a root is made absolute handed in, so a -// test can make that fail without swapping a package variable. It fails only -// when the working directory cannot be read, which no platform reproduces -// the same way (Linux fails once the directory is removed; macOS still -// answers). -func cleanup(roots []string, abs func(string) (string, error)) (CleanupReport, error) { - if len(roots) == 0 { - roots = []string{"."} - } - given, walk, err := resolveRoots(roots, abs) - if err != nil { - return CleanupReport{}, err - } - summary, err := cleanupRoots(walk) - if err != nil { - return CleanupReport{}, err - } - report := CleanupReport{Roots: given} - for _, r := range summary.Results { - if r.Kept { - report.Kept = append(report.Kept, r.Path) - } else { - report.Removed = append(report.Removed, r.Path) - } - } - return report, nil -} - -// resolveRoots is each root as given made absolute (given, what the report -// names), and the directory it resolves to with symlinks followed (walk, what -// is scanned), each directory once however it was spelled: `. $(pwd)`, or a -// symlink and its target. Walking the resolved path is what makes a symlinked -// root work at all: the walk does not descend a root that is itself a link. -// -// A root that cannot be made absolute fails the cleanup, since the report -// could not say where it removed anything; one that cannot be resolved (it -// does not exist) fails naming the root as it was typed. -func resolveRoots(roots []string, abs func(string) (string, error)) (given, walk []string, err error) { - seen := map[string]bool{} - for _, root := range roots { - a, err := abs(root) - if err != nil { - return nil, nil, fmt.Errorf("removing the Cosmos Boost artifacts: resolving %q: %w", root, err) - } - resolved, err := filepath.EvalSymlinks(a) - if err != nil { - return nil, nil, fmt.Errorf("removing the Cosmos Boost artifacts: scanning %q: %w", root, err) - } - if seen[resolved] { - continue - } - seen[resolved] = true - given = append(given, a) - walk = append(walk, resolved) - } - return given, walk, nil -} diff --git a/pkg/cosmosboost/cleanup_test.go b/pkg/cosmosboost/cleanup_test.go deleted file mode 100644 index 6e5e84ee3..000000000 --- a/pkg/cosmosboost/cleanup_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package cosmosboost - -import ( - "errors" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCleanupDefaultRootRemovesArtifact(t *testing.T) { - dir := t.TempDir() - writeDbtProject(t, dir) - require.NoError(t, PreDeploy(dir)) - t.Chdir(dir) - - require.NoError(t, cleanupErr()) - - _, err := os.Stat(filepath.Join(dir, artifactRelPath)) - require.True(t, os.IsNotExist(err), "artifact under the default root must be removed") -} - -func TestCleanupNothingToRemove(t *testing.T) { - require.NoError(t, cleanupErr(t.TempDir()), "an already-clean tree is not an error") -} - -func TestCleanupNonexistentRoot(t *testing.T) { - require.Error(t, cleanupErr(filepath.Join(t.TempDir(), "does-not-exist"))) -} - -// TestCleanupTouchesOnlyTheRequestedPaths pins the command's scope: cleanup -// acts on the paths it was given and nothing else on the machine. -func TestCleanupTouchesOnlyTheRequestedPaths(t *testing.T) { - requested := t.TempDir() - writeDbtProject(t, requested) - require.NoError(t, PreDeploy(requested)) - - elsewhere := t.TempDir() - writeDbtProject(t, elsewhere) - require.NoError(t, PreDeploy(elsewhere)) - - require.NoError(t, cleanupErr(requested)) - - _, err := os.Stat(filepath.Join(requested, artifactRelPath)) - require.True(t, os.IsNotExist(err)) - require.FileExists(t, filepath.Join(elsewhere, artifactRelPath), - "a path that was not requested must not be touched") -} - -// cleanupErr is Cleanup for a test that reads only its error. -func cleanupErr(roots ...string) error { - _, err := Cleanup(roots...) - return err -} - -// resolved is path with symlinks followed, as the cleanup walks it (on macOS -// a temp dir under /var is really under /private/var). -func resolved(t *testing.T, path string) string { - t.Helper() - r, err := filepath.EvalSymlinks(path) - require.NoError(t, err) - return r -} - -// TestCleanupReportsWhatItDid: the report names each artifact removed and -// each one kept because another tool wrote it, which is what -// `astro dbt cleanup -o json` publishes. -func TestCleanupReportsWhatItDid(t *testing.T) { - dir := t.TempDir() - writeDbtProject(t, dir) - require.NoError(t, PreDeploy(dir)) - foreign := filepath.Join(dir, "other", ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(foreign), 0o755)) - require.NoError(t, os.WriteFile(foreign, []byte(`{"generated_by": {"application": "someone-else"}}`), 0o600)) - - report, err := Cleanup(dir) - - require.NoError(t, err) - require.Equal(t, []string{dir}, report.Roots) - require.Equal(t, []string{filepath.Join(resolved(t, dir), artifactRelPath)}, report.Removed) - require.Equal(t, []string{resolved(t, foreign)}, report.Kept) -} - -// TestCleanupReportsEachRootOnce: one directory given twice, as `.` and as -// its absolute path, is one root in the report. -func TestCleanupReportsEachRootOnce(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - wd, err := os.Getwd() - require.NoError(t, err) - - report, err := Cleanup(".", wd) - - require.NoError(t, err) - require.Equal(t, []string{wd}, report.Roots) -} - -// symlinkTo makes link point at target, skipping the test where this -// platform or account cannot make one (Windows without the privilege). -func symlinkTo(t *testing.T, target, link string) { - t.Helper() - if err := os.Symlink(target, link); err != nil { - t.Skipf("cannot make a symlink here: %v", err) - } -} - -// TestCleanupWalksASymlinkedRoot: a root that is a symlink to a project is -// cleaned at its target. The walk does not descend a root that is itself a -// link, so without resolving it first this removed nothing and succeeded. -// The report names the root as given. -func TestCleanupWalksASymlinkedRoot(t *testing.T) { - target := t.TempDir() - writeDbtProject(t, target) - require.NoError(t, PreDeploy(target)) - link := filepath.Join(t.TempDir(), "proj") - symlinkTo(t, target, link) - - report, err := Cleanup(link + string(filepath.Separator)) - - require.NoError(t, err) - require.Equal(t, []string{link}, report.Roots) - require.Equal(t, []string{filepath.Join(resolved(t, target), artifactRelPath)}, report.Removed) - require.NoFileExists(t, filepath.Join(target, artifactRelPath)) -} - -// TestCleanupWalksALinkAndItsTargetOnce: a symlink and its target given -// together are one directory, cleaned once, and named by the first spelling. -func TestCleanupWalksALinkAndItsTargetOnce(t *testing.T) { - target := t.TempDir() - writeDbtProject(t, target) - require.NoError(t, PreDeploy(target)) - link := filepath.Join(t.TempDir(), "proj") - symlinkTo(t, target, link) - - report, err := Cleanup(link, target) - - require.NoError(t, err) - require.Equal(t, []string{link}, report.Roots) - require.Len(t, report.Removed, 1) - require.NoFileExists(t, filepath.Join(target, artifactRelPath)) -} - -// TestCleanupNamesAMissingRootAsGiven: the failure for a root that does not -// exist names it as it was typed. -func TestCleanupNamesAMissingRootAsGiven(t *testing.T) { - t.Chdir(t.TempDir()) - - _, err := Cleanup("missing-dir") - - require.ErrorContains(t, err, `scanning "missing-dir"`) -} - -// TestCleanupFailsWhenItCannotSayWhere: a root that cannot be made absolute -// fails the cleanup, before anything is removed, rather than report a path -// relative to nothing. The failure is handed in through cleanup's parameter, -// not a package variable, so the test is safe to run in parallel. -func TestCleanupFailsWhenItCannotSayWhere(t *testing.T) { - t.Parallel() - dir := t.TempDir() - writeDbtProject(t, dir) - require.NoError(t, PreDeploy(dir)) - failing := func(string) (string, error) { return "", errors.New("getwd: no such file or directory") } - - _, err := cleanup([]string{dir}, failing) - - require.ErrorContains(t, err, "resolving") - require.FileExists(t, filepath.Join(dir, artifactRelPath), "nothing is removed") -} - -// TestCleanupReportsAbsolutePaths: a relative root, the default "." among -// them, is reported absolute, and so is what was removed under it. -func TestCleanupReportsAbsolutePaths(t *testing.T) { - dir := t.TempDir() - writeDbtProject(t, dir) - require.NoError(t, PreDeploy(dir)) - t.Chdir(dir) - wd, err := os.Getwd() - require.NoError(t, err) - - report, err := Cleanup() - - require.NoError(t, err) - require.Equal(t, []string{wd}, report.Roots) - require.Equal(t, []string{filepath.Join(resolved(t, wd), artifactRelPath)}, report.Removed) -} diff --git a/pkg/cosmosboost/precompute/cleanup.go b/pkg/cosmosboost/precompute/cleanup.go deleted file mode 100644 index 7a09c8df7..000000000 --- a/pkg/cosmosboost/precompute/cleanup.go +++ /dev/null @@ -1,198 +0,0 @@ -package precompute - -import ( - "encoding/json" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "time" -) - -// cleanupSkipDirs are not traversed below the walked root. Discovery never -// writes sidecars under them (both discovery walks skip these names at any -// depth), and generated directories like a root-owned logs/ left by a -// container bind mount may not even be readable - aborting a deploy over a -// directory we never write to helps nobody. The root itself is exempt, -// mirroring discovery: a project that happens to live in a directory named -// logs or dbt_packages can be stamped, so it must be cleanable too. .git is -// skipped unconditionally - VCS internals are never ours to mutate. target/ -// is deliberately absent: a compiled manifest's sidecar lives at -// target/.astro/. -var cleanupSkipDirs = map[string]bool{ - "logs": true, - defaultPackagesDir: true, -} - -// CleanupResult records what happened to one artifact found during Cleanup. -type CleanupResult struct { - Path string // artifact file path - Kept bool // left in place: the file was not written by this tool - Err error // non-nil if removal failed -} - -// CleanupSummary is the structured outcome of an cleanup run. -type CleanupSummary struct { - Duration time.Duration - Results []CleanupResult -} - -// Cleanup removes every .astro/dbt_metadata.json sidecar and every -// .astro/manifest.slim.json under the given roots that this tool wrote, -// pruning each containing .astro directory when removal leaves it empty. -// -// Each file is judged by its own producer marker, never by its neighbor's, -// because the mixed states are real: a slim manifest outlives its sidecar when -// an older astro-cli removed only the sidecar, and a sidecar we wrote can sit -// beside a slim manifest we did not. One whose marker isn't ours is reported -// kept and left alone. -// -// Per-file removal failures are recorded in their Result and do not stop the -// others; like Run, a non-nil error is returned only for a top-level problem -// such as a root that cannot be walked. -func Cleanup(roots []string) (CleanupSummary, error) { - start := time.Now() - - seen := map[string]bool{} - var results []CleanupResult - for _, root := range roots { - err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if d.Name() == gitDir || (path != root && cleanupSkipDirs[d.Name()]) { - return filepath.SkipDir - } - return nil - } - if filepath.Base(filepath.Dir(path)) != sidecarDir { - return nil - } - switch d.Name() { - case sidecarName: - recordOnce(seen, &results, path, removeSidecar) - case slimManifestName: - recordOnce(seen, &results, path, removeSlimManifest) - } - return nil - }) - if err != nil { - return CleanupSummary{}, fmt.Errorf("scanning %q for artifacts: %w", root, err) - } - } - - return CleanupSummary{Duration: time.Since(start), Results: results}, nil -} - -// recordOnce runs remove(path) and appends its result, deduplicated by -// canonical path so overlapping roots naming the same tree differently — a -// relative name and an absolute one, or a symlink and its target — report -// each file once rather than once per spelling. -func recordOnce(seen map[string]bool, results *[]CleanupResult, path string, remove func(string) CleanupResult) { - key := canonicalPath(path) - if seen[key] { // overlapping roots: report each file once - return - } - seen[key] = true - *results = append(*results, remove(path)) -} - -// canonicalPath returns a path suitable for identifying one file across roots -// that spell it differently: absolute first, then symlinks resolved. Both steps -// are needed. Abs alone leaves a symlinked directory and its target looking like -// different files, and EvalSymlinks alone preserves relativity, so a relative -// root and an absolute one would still not compare equal. Each step is skipped -// if it fails, leaving Clean as the floor. -func canonicalPath(path string) string { - if abs, err := filepath.Abs(path); err == nil { - path = abs - } - if resolved, err := filepath.EvalSymlinks(path); err == nil { - return resolved - } - return filepath.Clean(path) -} - -// removeArtifact deletes one artifact after producedByUs confirms this tool -// wrote it, then prunes the containing .astro directory if removal left it -// empty. Anything else under .astro (e.g. an Astro project's config.yaml) is -// never touched. -func removeArtifact(path string, producedByUs func(data []byte) bool) CleanupResult { - data, err := os.ReadFile(path) - if err != nil { - return CleanupResult{Path: path, Err: err} - } - if !producedByUs(data) { - return CleanupResult{Path: path, Kept: true} - } - if err := os.Remove(path); err != nil { - return CleanupResult{Path: path, Err: err} - } - _ = os.Remove(filepath.Dir(path)) //nolint:errcheck // best-effort rmdir; succeeds only when the dir is empty - return CleanupResult{Path: path} -} - -// removeSidecar removes one .astro/dbt_metadata.json, keeping it unless its -// generated_by names this tool. -func removeSidecar(path string) CleanupResult { - return removeArtifact(path, func(data []byte) bool { - var meta Metadata - return json.Unmarshal(data, &meta) == nil && meta.GeneratedBy.Application == application - }) -} - -// removeSlimManifest removes one .astro/manifest.slim.json, keeping it unless -// its own _generated_by marker names this tool. -func removeSlimManifest(path string) CleanupResult { - return removeArtifact(path, func(data []byte) bool { - var marker struct { - GeneratedBy GeneratedBy `json:"_generated_by"` - } - return json.Unmarshal(data, &marker) == nil && marker.GeneratedBy.Application == application - }) -} - -// CountKept returns the number of artifacts left in place because this tool -// did not write them. -func (s CleanupSummary) CountKept() int { - n := 0 - for _, r := range s.Results { - if r.Kept { - n++ - } - } - return n -} - -// CountFailed returns the number of artifacts that could not be removed. -func (s CleanupSummary) CountFailed() int { - n := 0 - for _, r := range s.Results { - if r.Err != nil { - n++ - } - } - return n -} - -// WriteReport prints a short, human-readable report of the run. It follows the -// same shape as Summary.WriteReport (precompute.go): a counts line, then one -// line per entry using the shared glyphs. The columns differ because the entries -// do — a cleanup has nothing to say about hashes or bytes. -func (s CleanupSummary) WriteReport(w io.Writer) { - removed := len(s.Results) - s.CountKept() - s.CountFailed() - fmt.Fprintf(w, "cosmos boost cleanup: %d removed, %d kept, %d failed in %s\n", - removed, s.CountKept(), s.CountFailed(), s.Duration.Round(time.Microsecond)) - for _, r := range s.Results { - switch { - case r.Err != nil: - fmt.Fprintf(w, " %s %s (%v)\n", glyphFail, r.Path, r.Err) - case r.Kept: - fmt.Fprintf(w, " %s %s (unrecognized producer; left in place)\n", glyphLeft, r.Path) - default: - fmt.Fprintf(w, " %s %s\n", glyphDone, r.Path) - } - } -} diff --git a/pkg/cosmosboost/precompute/cleanup_test.go b/pkg/cosmosboost/precompute/cleanup_test.go deleted file mode 100644 index ab25e75aa..000000000 --- a/pkg/cosmosboost/precompute/cleanup_test.go +++ /dev/null @@ -1,546 +0,0 @@ -package precompute - -import ( - "bytes" - "os" - "path/filepath" - "runtime" - "strings" - "testing" -) - -// stampProject creates a minimal dbt project under dir and runs the real -// pre-deploy over it, so cleanup tests exercise sidecars exactly as the -// producer writes them. -func stampProject(t *testing.T, dir string) { - t.Helper() - writeFiles(t, dir, map[string]string{ - "dbt_project.yml": "name: shop\n", - "models/a.sql": "select 1", - }) - summary, err := Run([]string{dir}, "test", Options{SlimManifest: true}) - if err != nil || summary.CountFailed() > 0 { - t.Fatalf("stamping fixture project failed: err=%v failed=%d", err, summary.CountFailed()) - } -} - -// stampManifest is stampProject's sibling for the other unit kind: it drops a -// minimal dbt manifest.json under dir and runs the real pre-deploy over it, so -// the tests below exercise a sidecar *and* a slim manifest exactly as the -// producer writes them. (stampProject writes dbt_project.yml, which produces a -// project sidecar and no manifest, so it can't stand in here.) -func stampManifest(t *testing.T, dir string) { - t.Helper() - writeFiles(t, dir, map[string]string{ - "manifest.json": `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}`, - }) - summary, err := Run([]string{dir}, "test", Options{SlimManifest: true}) - if err != nil || summary.CountFailed() > 0 { - t.Fatalf("stamping fixture manifest failed: err=%v failed=%d", err, summary.CountFailed()) - } -} - -func TestCleanupRemovesSidecarAndPrunesDir(t *testing.T) { - dir := t.TempDir() - stampProject(t, dir) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := len(summary.Results); got != 1 { - t.Fatalf("results = %d, want 1", got) - } - if summary.CountFailed() != 0 || summary.CountKept() != 0 { - t.Fatalf("failed=%d kept=%d, want 0/0", summary.CountFailed(), summary.CountKept()) - } - if _, err := os.Stat(filepath.Join(dir, sidecarDir, sidecarName)); !os.IsNotExist(err) { - t.Fatalf("sidecar still present: %v", err) - } - // Removal emptied .astro, so the directory itself is pruned too. - if _, err := os.Stat(filepath.Join(dir, sidecarDir)); !os.IsNotExist(err) { - t.Fatalf(".astro dir not pruned: %v", err) - } -} - -// TestCleanupRemovesSlimManifestAlongsideSidecar: a slim manifest is only -// meaningful alongside a fresh hash sidecar, so cleanup must remove a stale -// one too - otherwise a later deploy (with a changed manifest, or with -// cosmos_boost.pre_deploy disabled) could still ship an outdated -// manifest.slim.json. -func TestCleanupRemovesSlimManifestAlongsideSidecar(t *testing.T) { - dir := t.TempDir() - stampManifest(t, dir) - slimPath := filepath.Join(dir, sidecarDir, slimManifestName) - if _, err := os.Stat(slimPath); err != nil { - t.Fatalf("fixture setup: slim manifest not written: %v", err) - } - - cleanupSummary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - // Each artifact is removed on its own terms, so each gets its own entry. - if got := len(cleanupSummary.Results); got != 2 { - t.Fatalf("results = %d, want 2 (one per artifact removed)", got) - } - if cleanupSummary.CountFailed() != 0 || cleanupSummary.CountKept() != 0 { - t.Fatalf("failed=%d kept=%d, want 0/0", cleanupSummary.CountFailed(), cleanupSummary.CountKept()) - } - - if _, err := os.Stat(slimPath); !os.IsNotExist(err) { - t.Fatalf("slim manifest still present after cleanup: %v", err) - } - if _, err := os.Stat(filepath.Join(dir, sidecarDir)); !os.IsNotExist(err) { - t.Fatalf(".astro dir not pruned after removing both files: %v", err) - } -} - -// TestCleanupJudgesEachArtifactSeparately: provenance is read per file, never -// inferred from the neighbor. Deleting on the neighbor's marker would destroy -// a file we do not own in one direction, and strand a stale artifact of ours - -// unremoved and unreported - in the other. -func TestCleanupJudgesEachArtifactSeparately(t *testing.T) { - for _, tc := range []struct { - name string - foreign string // artifact to overwrite with another producer's marker - marker string - wantKept string // artifact that must survive - wantGone string - }{ - { - name: "foreign slim manifest beside our sidecar", - foreign: slimManifestName, marker: `{"_generated_by": {"application": "someone-else"}}`, - wantKept: slimManifestName, wantGone: sidecarName, - }, - { - name: "our slim manifest beside a foreign sidecar", - foreign: sidecarName, marker: `{"generated_by": {"application": "someone-else"}}`, - wantKept: sidecarName, wantGone: slimManifestName, - }, - } { - t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() - stampManifest(t, dir) - astroDir := filepath.Join(dir, sidecarDir) - if err := os.WriteFile(filepath.Join(astroDir, tc.foreign), []byte(tc.marker), 0o644); err != nil { - t.Fatal(err) - } - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if summary.CountKept() != 1 || summary.CountFailed() != 0 { - t.Fatalf("kept=%d failed=%d, want 1/0", summary.CountKept(), summary.CountFailed()) - } - if _, err := os.Stat(filepath.Join(astroDir, tc.wantKept)); err != nil { - t.Fatalf("%s must survive: %v", tc.wantKept, err) - } - if _, err := os.Stat(filepath.Join(astroDir, tc.wantGone)); !os.IsNotExist(err) { - t.Fatalf("%s must be removed: %v", tc.wantGone, err) - } - }) - } -} - -// TestCleanupRemovesOrphanedSlimManifest: a slim manifest can end up without -// its sidecar - an older astro-cli's `cleanup`/EnsureClean deletes only the -// sidecar it knows about, or EnsureClean leaves a foreign sidecar in place - -// and nothing walking for dbt_metadata.json would ever find it again. Cleanup -// must check and remove it on its own terms, via its own _generated_by -// marker. -func TestCleanupRemovesOrphanedSlimManifest(t *testing.T) { - dir := t.TempDir() - stampManifest(t, dir) - astroDir := filepath.Join(dir, sidecarDir) - slimPath := filepath.Join(astroDir, slimManifestName) - if _, err := os.Stat(slimPath); err != nil { - t.Fatalf("fixture setup: slim manifest not written: %v", err) - } - // Simulate the sidecar being gone without this file's knowledge: an older - // astro-cli's cleanup, or a hand removal. - if err := os.Remove(filepath.Join(astroDir, sidecarName)); err != nil { - t.Fatal(err) - } - - cleanupSummary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if cleanupSummary.CountFailed() != 0 || cleanupSummary.CountKept() != 0 { - t.Fatalf("failed=%d kept=%d, want 0/0", cleanupSummary.CountFailed(), cleanupSummary.CountKept()) - } - if got := len(cleanupSummary.Results); got != 1 { - t.Fatalf("results = %d, want 1 (the orphaned slim manifest)", got) - } - if _, err := os.Stat(slimPath); !os.IsNotExist(err) { - t.Fatalf("orphaned slim manifest still present after cleanup: %v", err) - } - if _, err := os.Stat(astroDir); !os.IsNotExist(err) { - t.Fatalf(".astro dir not pruned after removing the orphaned slim manifest: %v", err) - } -} - -// TestCleanupKeepsForeignOrphanedSlimManifest: an orphaned slim manifest -// checks its own provenance the same way a sidecar does - one this tool did -// not write (or that isn't valid JSON) must never be deleted. -func TestCleanupKeepsForeignOrphanedSlimManifest(t *testing.T) { - dir := t.TempDir() - writeFiles(t, dir, map[string]string{ - ".astro/manifest.slim.json": `{"_generated_by": {"application": "someone-else"}}`, - }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := summary.CountKept(); got != 1 { - t.Fatalf("kept = %d, want 1", got) - } - if _, err := os.Stat(filepath.Join(dir, sidecarDir, slimManifestName)); err != nil { - t.Fatalf("foreign orphaned slim manifest was removed: %v", err) - } -} - -// TestCleanupAttributesRemovalFailuresPerArtifact: a failure must name the -// file that actually could not be removed. Reporting one artifact's error -// against its neighbor's path sends the user to fix the wrong file, and the -// next deploy fails on the same unnamed one. -func TestCleanupAttributesRemovalFailuresPerArtifact(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory write-permission semantics differ on windows") - } - dir := t.TempDir() - stampManifest(t, dir) - astroDir := filepath.Join(dir, sidecarDir) - sidecarPath := filepath.Join(astroDir, sidecarName) - slimPath := filepath.Join(astroDir, slimManifestName) - if _, err := os.Stat(sidecarPath); err != nil { - t.Fatalf("fixture setup: sidecar not written: %v", err) - } - - // A read-only .astro makes both removals fail. - if err := os.Chmod(astroDir, 0o555); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(astroDir, 0o755) }) - - cleanupSummary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if cleanupSummary.CountFailed() != 2 { - t.Fatalf("failed = %d, want 2 (neither artifact removable)", cleanupSummary.CountFailed()) - } - failed := map[string]bool{} - for _, r := range cleanupSummary.Results { - if r.Err != nil { - failed[r.Path] = true - } - } - for _, want := range []string{sidecarPath, slimPath} { - if !failed[want] { - t.Fatalf("no failure reported against %s; got %+v", want, cleanupSummary.Results) - } - } - if err := os.Chmod(astroDir, 0o755); err != nil { // restore before the stats below - t.Fatal(err) - } - for _, path := range []string{sidecarPath, slimPath} { - if _, err := os.Stat(path); err != nil { - t.Fatalf("%s must survive a failed removal so Cleanup can retry it: %v", path, err) - } - } -} - -func TestCleanupLeavesNonEmptyAstroDir(t *testing.T) { - dir := t.TempDir() - stampProject(t, dir) - writeFiles(t, dir, map[string]string{".astro/config.yaml": "project:\n name: shop\n"}) - - if _, err := Cleanup([]string{dir}); err != nil { - t.Fatalf("Cleanup: %v", err) - } - if _, err := os.Stat(filepath.Join(dir, sidecarDir, sidecarName)); !os.IsNotExist(err) { - t.Fatalf("sidecar still present: %v", err) - } - if _, err := os.Stat(filepath.Join(dir, sidecarDir, "config.yaml")); err != nil { - t.Fatalf("unrelated .astro content was touched: %v", err) - } -} - -// TestCleanupKeepsForeignSidecars pins the safety property: a -// .astro/dbt_metadata.json this tool did not write — another producer's, or -// one that isn't JSON at all — is never deleted. -func TestCleanupKeepsForeignSidecars(t *testing.T) { - dir := t.TempDir() - writeFiles(t, dir, map[string]string{ - "other/.astro/dbt_metadata.json": `{"generated_by": {"application": "someone-else"}}`, - "invalid/.astro/dbt_metadata.json": "not json", - }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := summary.CountKept(); got != 2 { - t.Fatalf("kept = %d, want 2", got) - } - for _, rel := range []string{"other", "invalid"} { - if _, err := os.Stat(filepath.Join(dir, rel, sidecarDir, sidecarName)); err != nil { - t.Fatalf("foreign sidecar under %s was removed: %v", rel, err) - } - } -} - -// TestCleanupIgnoresLookalikes: only dbt_metadata.json directly inside a -// .astro directory is a sidecar; same-named files elsewhere are untouched. -func TestCleanupIgnoresLookalikes(t *testing.T) { - dir := t.TempDir() - writeFiles(t, dir, map[string]string{ - "data/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`, - }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := len(summary.Results); got != 0 { - t.Fatalf("results = %d, want 0", got) - } - if _, err := os.Stat(filepath.Join(dir, "data", sidecarName)); err != nil { - t.Fatalf("lookalike outside .astro was removed: %v", err) - } -} - -func TestCleanupOverlappingRoots(t *testing.T) { - dir := t.TempDir() - stampProject(t, dir) - - summary, err := Cleanup([]string{dir, dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := len(summary.Results); got != 1 { - t.Fatalf("overlapping roots reported %d results, want 1", got) - } -} - -// TestCleanupRootsSpelledDifferently covers one tree named two ways: an -// absolute path and a relative one. A sidecar we remove cannot be double-counted -// (it is gone by the second walk), but a foreign one is deliberately left in -// place, so keying the seen-set on the raw walked path finds it again under the -// other spelling and reports it kept twice. -func TestCleanupRootsSpelledDifferently(t *testing.T) { - parent := t.TempDir() - project := filepath.Join(parent, "shop") - writeFiles(t, project, map[string]string{ - ".astro/dbt_metadata.json": `{"generated_by": {"application": "someone-else"}}`, - }) - - t.Chdir(parent) // so "shop" and the absolute path name one tree - - summary, err := Cleanup([]string{project, "shop"}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := len(summary.Results); got != 1 { - t.Fatalf("roots spelled differently reported %d results, want 1", got) - } - if got := summary.CountKept(); got != 1 { - t.Fatalf("kept = %d, want 1: one file must be reported once per run", got) - } -} - -func TestCleanupNonexistentRoot(t *testing.T) { - missing := filepath.Join(t.TempDir(), "does-not-exist") - if _, err := Cleanup([]string{missing}); err == nil { - t.Fatal("Cleanup on a nonexistent root: error = nil, want non-nil") - } -} - -func TestCleanupWriteReport(t *testing.T) { - dir := t.TempDir() - stampProject(t, dir) - writeFiles(t, dir, map[string]string{"other/.astro/dbt_metadata.json": "not json"}) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - var out bytes.Buffer - summary.WriteReport(&out) - got := out.String() - if !strings.Contains(got, "1 removed, 1 kept, 0 failed") { - t.Fatalf("report summary line missing: %q", got) - } - if !strings.Contains(got, "left in place") { - t.Fatalf("kept sidecar not explained in report: %q", got) - } -} - -// TestCleanupSkipsGitInternals pins that cleanup never traverses .git: even a -// sidecar this tool's own (buggy or interrupted) run left there is not touched, -// because mutating VCS internals is worse than leaving an undeployed file. -func TestCleanupSkipsGitInternals(t *testing.T) { - dir := t.TempDir() - writeFiles(t, dir, map[string]string{ - ".git/trap/.astro/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`, - }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if got := len(summary.Results); got != 0 { - t.Fatalf("results = %d, want 0 (nothing under .git may be visited)", got) - } - if _, err := os.Stat(filepath.Join(dir, ".git", "trap", ".astro", "dbt_metadata.json")); err != nil { - t.Fatalf("file under .git was touched: %v", err) - } -} - -func TestCanonicalPathFallsBackOnMissingPath(t *testing.T) { - missing := filepath.Join(t.TempDir(), "does-not-exist") - if got := canonicalPath(missing); got != filepath.Clean(missing) { - t.Fatalf("canonicalPath(%q) = %q, want the cleaned path", missing, got) - } -} - -// TestCleanupReportsUnreadableSidecar: a sidecar whose content cannot be read -// cannot be provenance-checked, so it is a failure (not silently kept). -func TestCleanupReportsUnreadableSidecar(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("file permission semantics differ on windows") - } - dir := t.TempDir() - writeFiles(t, dir, map[string]string{"proj/.astro/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`}) - sidecar := filepath.Join(dir, "proj", ".astro", "dbt_metadata.json") - if err := os.Chmod(sidecar, 0o000); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(sidecar, 0o644) }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if summary.CountFailed() != 1 { - t.Fatalf("failed = %d, want 1", summary.CountFailed()) - } - var out bytes.Buffer - summary.WriteReport(&out) - if !strings.Contains(out.String(), "✗") { - t.Fatalf("failure missing from report: %q", out.String()) - } -} - -// TestCleanupReportsUnremovableSidecar: a sidecar that passes the provenance -// check but cannot be unlinked is recorded as failed, not dropped. -func TestCleanupReportsUnremovableSidecar(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory write-permission semantics differ on windows") - } - dir := t.TempDir() - writeFiles(t, dir, map[string]string{"proj/.astro/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`}) - astroDir := filepath.Join(dir, "proj", ".astro") - if err := os.Chmod(astroDir, 0o555); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(astroDir, 0o755) }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if summary.CountFailed() != 1 { - t.Fatalf("failed = %d, want 1", summary.CountFailed()) - } -} - -// TestCleanupSkipsUnreadableGeneratedDirs: logs/ and dbt_packages/ never hold -// sidecars (discovery skips them by name), so cleanup must not fail over -// them even when they are unreadable, e.g. a root-owned bind-mount leftover. -func TestCleanupSkipsUnreadableGeneratedDirs(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory permission semantics differ on windows") - } - dir := t.TempDir() - for _, name := range []string{"logs", "dbt_packages"} { - locked := filepath.Join(dir, "proj", name) - if err := os.MkdirAll(locked, 0o755); err != nil { - t.Fatal(err) - } - if err := os.Chmod(locked, 0o000); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) - } - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup must not fail over generated dirs it never writes to: %v", err) - } - if len(summary.Results) != 0 { - t.Fatalf("results = %d, want 0", len(summary.Results)) - } -} - -// TestCleanupStillVisitsTarget: target/ can legitimately hold a compiled -// manifest's sidecar at target/.astro/, so it is NOT among the skipped names. -func TestCleanupStillVisitsTarget(t *testing.T) { - dir := t.TempDir() - writeFiles(t, dir, map[string]string{ - "proj/target/.astro/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`, - }) - - summary, err := Cleanup([]string{dir}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if len(summary.Results) != 1 || summary.CountFailed() != 0 || summary.CountKept() != 0 { - t.Fatalf("results = %+v, want the target/.astro sidecar removed", summary.Results) - } -} - -// TestCleanupCleansRootsNamedLikeSkipDirs mirrors discovery's root exemption: -// a project living in a directory named logs or dbt_packages can be stamped, -// so cleanup of that same root must not skip it - otherwise a sidecar could -// survive EnsureClean and ship stale. .git stays skipped even as the root. -func TestCleanupCleansRootsNamedLikeSkipDirs(t *testing.T) { - for _, name := range []string{"logs", "dbt_packages"} { - t.Run(name, func(t *testing.T) { - root := filepath.Join(t.TempDir(), name) - writeFiles(t, root, map[string]string{ - "proj/.astro/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`, - }) - - summary, err := Cleanup([]string{root}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if len(summary.Results) != 1 || summary.CountFailed() != 0 || summary.CountKept() != 0 { - t.Fatalf("results = %+v, want the sidecar under the %s root removed", summary.Results, name) - } - }) - } - - t.Run(".git stays protected", func(t *testing.T) { - root := filepath.Join(t.TempDir(), ".git") - writeFiles(t, root, map[string]string{ - "trap/.astro/dbt_metadata.json": `{"generated_by": {"application": "astro"}}`, - }) - - summary, err := Cleanup([]string{root}) - if err != nil { - t.Fatalf("Cleanup: %v", err) - } - if len(summary.Results) != 0 { - t.Fatalf("results = %+v, want nothing visited under a .git root", summary.Results) - } - if _, err := os.Stat(filepath.Join(root, "trap", ".astro", "dbt_metadata.json")); err != nil { - t.Fatalf("file under .git was touched: %v", err) - } - }) -} diff --git a/pkg/cosmosboost/precompute/existing.go b/pkg/cosmosboost/precompute/existing.go new file mode 100644 index 000000000..18786343f --- /dev/null +++ b/pkg/cosmosboost/precompute/existing.go @@ -0,0 +1,92 @@ +package precompute + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// existingSkipDirs are not traversed below the walked root. Discovery never +// produces artifacts under them (both discovery walks skip these names at any +// depth), and generated directories like a root-owned logs/ left by a +// container bind mount may not even be readable - failing a deploy over a +// directory no artifact is ever produced for helps nobody. The root itself is +// exempt, mirroring discovery: a project that happens to live in a directory +// named logs or dbt_packages can be stamped. .git is skipped unconditionally. +// target/ is deliberately absent: a compiled manifest's sidecar belongs at +// target/.astro/. +var existingSkipDirs = map[string]bool{ + "logs": true, + defaultPackagesDir: true, +} + +// Existing is a Cosmos Boost artifact already on disk under a walked root: a +// .astro/dbt_metadata.json or .astro/manifest.slim.json. Ours reports whether +// its own producer marker names this tool. Earlier astro-cli versions wrote +// these into the project; this one never does. +type Existing struct { + Path string + Ours bool +} + +// FindExisting returns every .astro/dbt_metadata.json and +// .astro/manifest.slim.json under root, each judged by its own producer +// marker, never by its neighbor's, because the mixed states are real: a slim +// manifest outlives its sidecar when an older astro-cli removed only the +// sidecar, and a sidecar this tool wrote can sit beside a slim manifest it did +// not. A file that cannot be read fails the scan, since nobody could say +// whose it is. +func FindExisting(root string) ([]Existing, error) { + var found []Existing + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == gitDir || (path != root && existingSkipDirs[d.Name()]) { + return filepath.SkipDir + } + return nil + } + if filepath.Base(filepath.Dir(path)) != sidecarDir { + return nil + } + var ours func([]byte) bool + switch d.Name() { + case sidecarName: + ours = sidecarIsOurs + case slimManifestName: + ours = slimManifestIsOurs + default: + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + found = append(found, Existing{Path: path, Ours: ours(data)}) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scanning %q for Cosmos Boost files: %w", root, err) + } + return found, nil +} + +// sidecarIsOurs reports whether a .astro/dbt_metadata.json's generated_by +// names this tool. +func sidecarIsOurs(data []byte) bool { + var meta Metadata + return json.Unmarshal(data, &meta) == nil && meta.GeneratedBy.Application == application +} + +// slimManifestIsOurs reports whether a .astro/manifest.slim.json's own +// _generated_by marker names this tool. +func slimManifestIsOurs(data []byte) bool { + var marker struct { + GeneratedBy GeneratedBy `json:"_generated_by"` + } + return json.Unmarshal(data, &marker) == nil && marker.GeneratedBy.Application == application +} diff --git a/pkg/cosmosboost/precompute/existing_test.go b/pkg/cosmosboost/precompute/existing_test.go new file mode 100644 index 000000000..ba12204b8 --- /dev/null +++ b/pkg/cosmosboost/precompute/existing_test.go @@ -0,0 +1,126 @@ +package precompute + +import ( + "os" + "path/filepath" + "runtime" + "sort" + "testing" +) + +const ( + oursSidecar = `{"schema":1,"version":{"algo":"sha256-tree-v2","hash":"old"},"generated_by":{"application":"astro","version":"1.0.0"}}` + foreignSlim = `{"_schema":1,"_generated_by":{"application":"someone-else"}}` + oursSlim = `{"_schema":1,"_generated_by":{"application":"astro","version":"1.0.0"}}` + foreignNoTag = `{"schema":1,"version":{"hash":"abc"}}` +) + +// existingByRel is FindExisting's result keyed by root-relative slash path. +func existingByRel(t *testing.T, root string) map[string]bool { + t.Helper() + found, err := FindExisting(root) + if err != nil { + t.Fatal(err) + } + out := map[string]bool{} + for _, e := range found { + rel, err := filepath.Rel(root, e.Path) + if err != nil { + t.Fatal(err) + } + out[filepath.ToSlash(rel)] = e.Ours + } + return out +} + +// TestFindExistingJudgesEachFileByItsOwnMarker: provenance is read per file, +// never inferred from a neighbor, and a file with no marker is not ours. +func TestFindExistingJudgesEachFileByItsOwnMarker(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "proj/.astro/dbt_metadata.json": oursSidecar, + "proj/.astro/manifest.slim.json": foreignSlim, + "proj/target/.astro/dbt_metadata.json": foreignNoTag, + "proj/target/.astro/manifest.slim.json": oursSlim, + "orphan/.astro/manifest.slim.json": oursSlim, + "proj/.astro/config.yaml": "not ours to judge", + "proj/dbt_metadata.json": oursSidecar, // not under .astro + "proj/.astro/nested/dbt_metadata.json": oursSidecar, // not directly under .astro + }) + + got := existingByRel(t, root) + want := map[string]bool{ + "proj/.astro/dbt_metadata.json": true, + "proj/.astro/manifest.slim.json": false, + "proj/target/.astro/dbt_metadata.json": false, + "proj/target/.astro/manifest.slim.json": true, + "orphan/.astro/manifest.slim.json": true, + } + if len(got) != len(want) { + t.Fatalf("found %v, want %v", got, want) + } + for path, ours := range want { + if g, ok := got[path]; !ok || g != ours { + t.Fatalf("%s: found=%v ours=%v, want ours=%v (all: %v)", path, ok, g, ours, got) + } + } +} + +// TestFindExistingSkipsGitAndGeneratedDirs: .git is never walked, and logs/ +// and dbt_packages/ below the root are skipped even when unreadable, while +// target/ is visited and a root named like a skipped directory is still +// scanned. +func TestFindExistingSkipsGitAndGeneratedDirs(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "logs") + writeFiles(t, root, map[string]string{ + ".git/x/.astro/dbt_metadata.json": oursSidecar, + "proj/logs/.astro/dbt_metadata.json": oursSidecar, + "proj/dbt_packages/.astro/dbt_metadata.json": oursSidecar, + "proj/target/.astro/dbt_metadata.json": oursSidecar, + ".astro/dbt_metadata.json": oursSidecar, + }) + if runtime.GOOS != "windows" { + locked := filepath.Join(root, "proj", "logs") + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) + } + + got := existingByRel(t, root) + var paths []string + for p := range got { + paths = append(paths, p) + } + sort.Strings(paths) + want := []string{".astro/dbt_metadata.json", "proj/target/.astro/dbt_metadata.json"} + if len(paths) != len(want) || paths[0] != want[0] || paths[1] != want[1] { + t.Fatalf("found %v, want %v", paths, want) + } +} + +// TestFindExistingFailsOnUnreadableArtifact: a file nobody can read cannot be +// attributed, so the scan fails rather than guessing. +func TestFindExistingFailsOnUnreadableArtifact(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("file permission semantics differ on windows") + } + root := t.TempDir() + writeFiles(t, root, map[string]string{"proj/.astro/dbt_metadata.json": oursSidecar}) + path := filepath.Join(root, "proj", ".astro", "dbt_metadata.json") + if err := os.Chmod(path, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(path, 0o644) }) + + if _, err := FindExisting(root); err == nil { + t.Fatal("FindExisting over an unreadable sidecar: error = nil, want non-nil") + } +} + +func TestFindExistingNonexistentRoot(t *testing.T) { + if _, err := FindExisting(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("FindExisting on a missing root: error = nil, want non-nil") + } +} diff --git a/pkg/cosmosboost/precompute/hash.go b/pkg/cosmosboost/precompute/hash.go index d2c87c355..9c747fbfe 100644 --- a/pkg/cosmosboost/precompute/hash.go +++ b/pkg/cosmosboost/precompute/hash.go @@ -98,9 +98,9 @@ func hashProject(dir string, cfg dbtConfig) (hash string, files int, totalBytes if d.IsDir() { // Our own sidecar directory must never affect the project hash, at ANY // depth. A dbt manifest.json in a subdirectory (e.g. docs/manifest.json) - // gets its own sidecar written into the walked tree (docs/.astro/), and - // because manifest and project units run concurrently, hashing it would - // make the project hash depend on that sidecar and vary between runs. + // gets its own sidecar at docs/.astro/ in the bundle, and earlier + // astro-cli versions wrote those into the tree itself, so hashing it + // would make the project hash depend on whether one was lying there. // .astro is always our output, never dbt source, so skipping it by name // everywhere is safe (unlike target/logs, which are matched root-relative // to avoid over-excluding a same-named source dir). diff --git a/pkg/cosmosboost/precompute/metadata.go b/pkg/cosmosboost/precompute/metadata.go index 4f7b73152..0c6554fd9 100644 --- a/pkg/cosmosboost/precompute/metadata.go +++ b/pkg/cosmosboost/precompute/metadata.go @@ -2,6 +2,7 @@ package precompute import ( "encoding/json" + "fmt" "os" "path/filepath" ) @@ -30,11 +31,6 @@ const ( sidecarDir = ".astro" sidecarName = "dbt_metadata.json" - - sidecarDirPerm = 0o755 - // sidecarPerm is deliberately world-readable: the sidecar ships inside the - // deploy bundle or image and the Airflow runtime user must be able to read it. - sidecarPerm = 0o644 ) // Metadata is the content of the .astro/dbt_metadata.json sidecar. The Cosmos @@ -45,7 +41,7 @@ type Metadata struct { Version ProjectVersion `json:"version"` GeneratedBy GeneratedBy `json:"generated_by"` // FilteredManifest points at the slim manifest beside this sidecar, absent - // when none was written. The sidecar is the consumer's entry point: it + // when none was produced. The sidecar is the consumer's entry point: it // already reads and schema-gates this file, so it discovers the slim // manifest here and falls back to the full one when the section is missing. FilteredManifest *FilteredManifest `json:"filtered_manifest,omitempty"` @@ -74,10 +70,10 @@ type GeneratedBy struct { Version string `json:"version"` } -// writeSidecar writes .astro/dbt_metadata.json inside dir. version is the +// sidecarArtifact is the .astro/dbt_metadata.json for dir. version is the // producer's version, recorded in generated_by. filtered describes a slim -// manifest written beside it, or is nil when none was. -func writeSidecar(dir, algo, hash, version string, filtered *FilteredManifest) error { +// manifest produced beside it, or is nil when none was. +func sidecarArtifact(dir, algo, hash, version string, filtered *FilteredManifest) Artifact { meta := Metadata{ Schema: schemaVersion, Version: ProjectVersion{Algo: algo, Hash: hash}, @@ -89,15 +85,15 @@ func writeSidecar(dir, algo, hash, version string, filtered *FilteredManifest) e data, _ := json.MarshalIndent(meta, "", " ") //nolint:errcheck // see above: Metadata holds only strings and ints, so this cannot fail data = append(data, '\n') - return writeArtifact(dir, sidecarName, data) + return Artifact{Path: filepath.Join(dir, sidecarDir, sidecarName), Data: data} } -// writeArtifact writes data to dir/.astro/name, creating the directory if -// needed. -func writeArtifact(dir, name string, data []byte) error { - out := filepath.Join(dir, sidecarDir) - if err := os.MkdirAll(out, sidecarDirPerm); err != nil { - return err +// checkSidecarDir fails when something other than a directory holds dir's +// .astro: an artifact under it could not be unpacked from the bundle. +func checkSidecarDir(dir string) error { + info, err := os.Lstat(filepath.Join(dir, sidecarDir)) + if err != nil || info.IsDir() { + return nil //nolint:nilerr // absent is fine: the bundle creates it } - return os.WriteFile(filepath.Join(out, name), data, sidecarPerm) + return fmt.Errorf("%s is not a directory, so its Cosmos Boost files cannot sit under it", filepath.Join(dir, sidecarDir)) } diff --git a/pkg/cosmosboost/precompute/metadata_test.go b/pkg/cosmosboost/precompute/metadata_test.go deleted file mode 100644 index e33d5c33e..000000000 --- a/pkg/cosmosboost/precompute/metadata_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package precompute - -import ( - "os" - "path/filepath" - "testing" -) - -// TestWriteSidecarError covers the best-effort failure branch: when a plain file -// already occupies the .astro path, MkdirAll can't create the sidecar directory, so -// writeSidecar returns an error rather than panicking. -func TestWriteSidecarError(t *testing.T) { - dir := t.TempDir() - // Block the ".astro" directory with a regular file of the same name. - if err := os.WriteFile(filepath.Join(dir, sidecarDir), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - if err := writeSidecar(dir, algoProjectTree, "deadbeef", "test", nil); err == nil { - t.Fatal("writeSidecar should fail when .astro is a file, not a directory") - } -} diff --git a/pkg/cosmosboost/precompute/precompute.go b/pkg/cosmosboost/precompute/precompute.go index 743488c34..a5b829708 100644 --- a/pkg/cosmosboost/precompute/precompute.go +++ b/pkg/cosmosboost/precompute/precompute.go @@ -27,9 +27,21 @@ type Result struct { Files int // files hashed (1 for a manifest) Bytes int64 // total bytes hashed Duration time.Duration // time spent on this unit - Skipped bool // a manifest.json that isn't a dbt manifest (no sidecar written) - Warning string // non-fatal note (sidecar still written), e.g. an unresolved template - Err error // non-nil if hashing, writing the sidecar, or writing the slim manifest failed + Skipped bool // a manifest.json that isn't a dbt manifest (no sidecar produced) + Warning string // non-fatal note (sidecar still produced), e.g. an unresolved template + Err error // non-nil if reading or hashing the unit failed (no artifacts then) + // Artifacts are the files this unit produced: its sidecar, and the slim + // manifest beside it when one was asked for. Empty when Err or Skipped. + Artifacts []Artifact +} + +// Artifact is one file a run produced, held in memory: Run writes nothing to +// disk. Path is where it belongs in the walked tree, /.astro/, +// which is where the Cosmos Boost plugin looks for it in a deployed bundle; the +// caller decides how the bytes get there. +type Artifact struct { + Path string + Data []byte } // Summary is the structured outcome of a precompute run. It backs both the @@ -39,17 +51,18 @@ type Summary struct { Results []Result } -// Options selects which artifacts a run writes; the zero value writes only the -// hash sidecars. +// Options selects which artifacts a run produces; the zero value produces only +// the hash sidecars. type Options struct { - // SlimManifest also writes a slim, field-filtered copy of each discovered + // SlimManifest also produces a slim, field-filtered copy of each discovered // manifest.json (see buildSlimManifest) next to its sidecar. SlimManifest bool } // Run finds every dbt project (a directory with dbt_project.yml) and standalone -// dbt manifest.json under the given roots, and writes a .astro/dbt_metadata.json -// hash sidecar next to each. Units are processed concurrently — one worker each, +// dbt manifest.json under the given roots, and produces a .astro/dbt_metadata.json +// hash sidecar for each, in memory (Result.Artifacts): nothing under the roots +// is written. Units are processed concurrently — one worker each, // bounded by GOMAXPROCS — and each is hashed over sorted input, so results are // deterministic with no cross-worker coordination. // @@ -59,8 +72,7 @@ type Options struct { // sidecar's generated_by. // // With opts.SlimManifest set, every dbt manifest.json also gets a slim, -// field-filtered copy (see buildSlimManifest) written into the .astro/ beside -// it, for the Cosmos Boost plugin to load in place of the full manifest at +// field-filtered copy (see buildSlimManifest) for the .astro/ beside it, for the Cosmos Boost plugin to load in place of the full manifest at // DAG-parse time. That includes one in a project's own root, which is not a // discovery unit of its own and is handled by processProject. func Run(roots []string, version string, opts Options) (Summary, error) { @@ -123,16 +135,20 @@ func Run(roots []string, version string, opts Options) (Summary, error) { return Summary{Duration: time.Since(start), Results: results}, nil } -// processProject hashes one dbt project directory and writes its sidecar. It reads +// processProject hashes one dbt project directory and produces its sidecar. It reads // dbt_project.yml once (readDbtConfig) and threads the result through hashing and the // templated-packages warning, so the file isn't parsed more than once per project. func processProject(dir, version string, opts Options) Result { start := time.Now() cfg := readDbtConfig(dir) hash, files, totalBytes, err := hashProject(dir, cfg) - r := Result{Kind: kindProject, Path: dir, Hash: hash, Files: files, Bytes: totalBytes, Duration: time.Since(start)} + r := Result{Kind: kindProject, Path: dir, Hash: hash, Files: files, Bytes: totalBytes} + if err == nil { + err = checkSidecarDir(dir) + } if err != nil { r.Err = err + r.Duration = time.Since(start) return r } if len(cfg.templatedSettings) > 0 { @@ -141,30 +157,29 @@ func processProject(dir, version string, opts Options) Result { } // A manifest.json in the project root is not a unit of its own - its .astro/ - // is this project's, so it would collide with the sidecar written below - and - // findManifests skips it for that reason. Slim it here instead, leaving the - // project's own hash as the anchor the pointer hangs off. + // is this project's, so it would collide with the sidecar produced below - + // and findManifests skips it for that reason. Slim it here instead, leaving + // the project's own hash as the anchor the pointer hangs off. var filtered *FilteredManifest if opts.SlimManifest { if doc, _, isDbt, readErr := readManifestDoc(filepath.Join(dir, manifestFile)); readErr == nil && isDbt { // Nothing mutates doc afterward here, unlike processManifest. data, _ := json.Marshal(buildSlimManifest(doc, version)) //nolint:errcheck // JSON-native types only, so this cannot fail - if filtered, r.Err = writeSlimManifest(dir, data); r.Err != nil { - r.Duration = time.Since(start) - return r - } + var slim Artifact + slim, filtered = slimManifestArtifact(dir, data) + r.Artifacts = append(r.Artifacts, slim) } } - r.Err = writeSidecar(dir, algoProjectTree, hash, version, filtered) + r.Artifacts = append(r.Artifacts, sidecarArtifact(dir, algoProjectTree, hash, version, filtered)) r.Duration = time.Since(start) return r } -// processManifest hashes one manifest.json and writes a sidecar next to it, -// plus a slim, field-filtered copy of the manifest when opts asks for one (see +// processManifest hashes one manifest.json and produces a sidecar for it, plus +// a slim, field-filtered copy of the manifest when opts asks for one (see // buildSlimManifest). A file that isn't a dbt manifest is skipped (nothing is -// written) so unrelated manifest.json files in the project aren't stamped. +// produced) so unrelated manifest.json files in the project aren't stamped. func processManifest(path, version string, opts Options) Result { start := time.Now() doc, bytes, isDbt, err := readManifestDoc(path) @@ -179,47 +194,58 @@ func processManifest(path, version string, opts Options) Result { } hash = hashDocument(doc) } - r := Result{Kind: kindManifest, Path: path, Hash: hash, Files: 1, Bytes: bytes, Duration: time.Since(start)} + r := Result{Kind: kindManifest, Path: path, Hash: hash, Files: 1, Bytes: bytes} + dir := filepath.Dir(path) switch { case err != nil: r.Err = err case !isDbt: r.Skipped = true default: - dir := filepath.Dir(path) - // The sidecar goes last: it carries the filtered_manifest pointer, so it - // must never exist before the file it points at. Stopping short of it - // looks like "nothing was stamped", which BestEffortPreDeploy treats as - // safe. + if r.Err = checkSidecarDir(dir); r.Err != nil { + break + } var filtered *FilteredManifest if slimData != nil { - filtered, r.Err = writeSlimManifest(dir, slimData) - } - if r.Err == nil { - r.Err = writeSidecar(dir, algoManifestJSON, hash, version, filtered) + var slim Artifact + slim, filtered = slimManifestArtifact(dir, slimData) + r.Artifacts = append(r.Artifacts, slim) } + r.Artifacts = append(r.Artifacts, sidecarArtifact(dir, algoManifestJSON, hash, version, filtered)) } + r.Duration = time.Since(start) return r } -// writeSlimManifest writes data as dir's slim manifest and returns the sidecar -// pointer describing it. data must already be marshaled, so a caller that later -// mutates the source doc cannot leak into it (see processManifest). -func writeSlimManifest(dir string, data []byte) (*FilteredManifest, error) { - if err := writeArtifact(dir, slimManifestName, data); err != nil { - return nil, err - } - return &FilteredManifest{ +// slimManifestArtifact is dir's slim manifest holding data, and the sidecar +// pointer describing it. data must already be marshaled, so a caller that +// later mutates the source doc cannot leak into it (see processManifest). +func slimManifestArtifact(dir string, data []byte) (Artifact, *FilteredManifest) { + return Artifact{Path: filepath.Join(dir, sidecarDir, slimManifestName), Data: data}, &FilteredManifest{ Schema: slimSchemaVersion, Path: slimManifestName, Version: ProjectVersion{Algo: algoFilteredManifest, Hash: sha256Hex(data)}, - }, nil + } +} + +// Artifacts returns every artifact the run produced, from the units that did +// not fail, in the order of Results. +func (s Summary) Artifacts() []Artifact { + var out []Artifact + for i := range s.Results { + r := &s.Results[i] + if r.Err == nil { + out = append(out, r.Artifacts...) + } + } + return out } // CountFailed returns the number of units that errored. func (s Summary) CountFailed() int { n := 0 - for _, r := range s.Results { + for i := range s.Results { + r := &s.Results[i] if r.Err != nil { n++ } @@ -231,7 +257,8 @@ func (s Summary) CountFailed() int { // dbt manifests). func (s Summary) CountSkipped() int { n := 0 - for _, r := range s.Results { + for i := range s.Results { + r := &s.Results[i] if r.Skipped { n++ } @@ -240,10 +267,8 @@ func (s Summary) CountSkipped() int { } // WriteReport prints a short, human-readable report of the run. -// Per-entry glyphs, shared by every command's WriteReport (see also -// CleanupSummary.WriteReport in cleanup.go) so the two reports keep one -// convention: acted on, deliberately left alone, failed, and a note attached to -// an entry that otherwise succeeded. +// Per-entry glyphs: acted on, deliberately left alone, failed, and a note +// attached to an entry that otherwise succeeded. const ( glyphDone = "✓" glyphLeft = "⊘" @@ -255,7 +280,8 @@ func (s Summary) WriteReport(w io.Writer) { stamped := len(s.Results) - s.CountFailed() - s.CountSkipped() fmt.Fprintf(w, "cosmos boost pre-deploy: %d stamped, %d skipped, %d failed in %s total (incl. discovery)\n", stamped, s.CountSkipped(), s.CountFailed(), s.Duration.Round(time.Microsecond)) - for _, r := range s.Results { + for i := range s.Results { + r := &s.Results[i] switch { case r.Err != nil: fmt.Fprintf(w, " %s %-8s %s (%v)\n", glyphFail, r.Kind, r.Path, r.Err) diff --git a/pkg/cosmosboost/precompute/precompute_test.go b/pkg/cosmosboost/precompute/precompute_test.go index 1177db052..2484f3437 100644 --- a/pkg/cosmosboost/precompute/precompute_test.go +++ b/pkg/cosmosboost/precompute/precompute_test.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "errors" + "io/fs" + "maps" "os" "path/filepath" "runtime" @@ -41,7 +43,7 @@ func TestRunDiscoversProjectsAndWritesSidecars(t *testing.T) { } sidecar := filepath.Join(r.Path, sidecarDir, sidecarName) var m Metadata - readJSON(t, sidecar, &m) + readJSON(t, summary, sidecar, &m) if m.Schema != schemaVersion || m.Version.Algo != algoProjectTree || m.Version.Hash != r.Hash { t.Fatalf("sidecar mismatch for %s: %+v (result hash %s)", r.Path, m, r.Hash) } @@ -68,7 +70,7 @@ func TestRunManifestOnly(t *testing.T) { } var m Metadata - readJSON(t, filepath.Join(root, "shipped", sidecarDir, sidecarName), &m) + readJSON(t, summary, filepath.Join(root, "shipped", sidecarDir, sidecarName), &m) if m.Version.Algo != algoManifestJSON || m.Version.Hash == "" { t.Fatalf("manifest sidecar mismatch: %+v", m) } @@ -99,8 +101,8 @@ func TestRunProjectWithTargetManifest(t *testing.T) { if kinds["project"] != 1 || kinds["manifest"] != 1 { t.Fatalf("want 1 project + 1 manifest, got %+v (%+v)", kinds, summary.Results) } - mustExist(t, filepath.Join(root, "proj", sidecarDir, sidecarName)) - mustExist(t, filepath.Join(root, "proj", "target", sidecarDir, sidecarName)) + mustProduce(t, summary, filepath.Join(root, "proj", sidecarDir, sidecarName)) + mustProduce(t, summary, filepath.Join(root, "proj", "target", sidecarDir, sidecarName)) } // TestRunSkipsNonDBTManifest verifies an unrelated manifest.json (no dbt shape) @@ -121,9 +123,7 @@ func TestRunSkipsNonDBTManifest(t *testing.T) { if summary.CountSkipped() != 1 { t.Fatalf("Skipped() = %d, want 1", summary.CountSkipped()) } - if _, err := os.Stat(filepath.Join(root, "webapp", sidecarDir, sidecarName)); !os.IsNotExist(err) { - t.Fatalf("a sidecar was written beside a non-dbt manifest (err=%v)", err) - } + mustNotProduce(t, summary, filepath.Join(root, "webapp", sidecarDir, sidecarName)) } // TestRunWarnsOnTemplatedPackagesPath verifies a project whose packages-install-path @@ -149,7 +149,7 @@ func TestRunWarnsOnTemplatedPackagesPath(t *testing.T) { if r.Warning == "" { t.Fatal("expected a warning for a templated packages-install-path") } - mustExist(t, filepath.Join(root, "proj", sidecarDir, sidecarName)) + mustProduce(t, summary, filepath.Join(root, "proj", sidecarDir, sidecarName)) } func TestRunIsDeterministicAcrossRuns(t *testing.T) { @@ -211,7 +211,8 @@ func hashesByPath(t *testing.T, root string) map[string]string { t.Fatal(err) } m := make(map[string]string, len(s.Results)) - for _, r := range s.Results { + for i := range s.Results { + r := &s.Results[i] if r.Err != nil { t.Fatalf("unexpected error for %s: %v", r.Path, r.Err) } @@ -220,29 +221,45 @@ func hashesByPath(t *testing.T, root string) map[string]string { return m } -func readJSON(t *testing.T, path string, v any) { +// artifact returns the bytes s produced for path, and whether it produced any. +func artifact(s Summary, path string) ([]byte, bool) { + for _, a := range s.Artifacts() { + if a.Path == path { + return a.Data, true + } + } + return nil, false +} + +func readJSON(t *testing.T, s Summary, path string, v any) { t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %s: %v", path, err) + data, ok := artifact(s, path) + if !ok { + t.Fatalf("no artifact produced for %s", path) } if err := json.Unmarshal(data, v); err != nil { t.Fatalf("unmarshal %s: %v", path, err) } } -func mustExist(t *testing.T, path string) { +func mustProduce(t *testing.T, s Summary, path string) { t.Helper() - if _, err := os.Stat(path); err != nil { - t.Fatalf("expected file %s: %v", path, err) + if _, ok := artifact(s, path); !ok { + t.Fatalf("expected an artifact for %s", path) + } +} + +func mustNotProduce(t *testing.T, s Summary, path string) { + t.Helper() + if _, ok := artifact(s, path); ok { + t.Fatalf("unexpected artifact for %s", path) } } // TestRunProjectHashStableWithNestedManifestSidecar guards the hash-stability // concern: a dbt manifest.json in a subdirectory (e.g. docs/) gets its own sidecar -// written into the walked tree (docs/.astro/). Since manifest and project units run -// concurrently, the enclosing project's hash must not depend on that sidecar — so it -// must be identical before and after the sidecar exists on disk. +// at docs/.astro/ in the bundle. The enclosing project's hash must not depend on +// that sidecar, so it must be identical whether or not one exists on disk. func TestRunProjectHashStableWithNestedManifestSidecar(t *testing.T) { root := t.TempDir() writeFiles(t, root, map[string]string{ @@ -256,7 +273,8 @@ func TestRunProjectHashStableWithNestedManifestSidecar(t *testing.T) { if err != nil { t.Fatal(err) } - for _, r := range s.Results { + for i := range s.Results { + r := &s.Results[i] if r.Kind == "project" { if r.Err != nil { t.Fatalf("unexpected error: %v", r.Err) @@ -268,14 +286,14 @@ func TestRunProjectHashStableWithNestedManifestSidecar(t *testing.T) { return "" } - first := projectHash() // writes proj/.astro and proj/docs/.astro - second := projectHash() // the nested sidecar now exists on disk + first := projectHash() + // Put the nested sidecar on disk, as a stale one from an earlier astro-cli + // (or the unpacked bundle) would be. + writeFiles(t, root, map[string]string{"proj/docs/.astro/dbt_metadata.json": oursSidecar}) + second := projectHash() if first != second { t.Fatalf("project hash changed once the nested docs/.astro sidecar existed:\n first %s\n second %s", first, second) } - // Sanity: the nested docs/manifest.json did get its own sidecar (so the walk - // really does encounter docs/.astro and correctly skips it). - mustExist(t, filepath.Join(root, "proj", "docs", sidecarDir, sidecarName)) } func TestShortHash(t *testing.T) { @@ -306,9 +324,9 @@ func TestRunSkipsGitInternals(t *testing.T) { if got := len(summary.Results); got != 1 { t.Fatalf("results = %d, want 1 (only the real project)", got) } - for _, rel := range []string{".git/trap/.astro", ".git/deep/.astro", ".git/.astro"} { - if _, err := os.Stat(filepath.Join(dir, filepath.FromSlash(rel))); !os.IsNotExist(err) { - t.Fatalf("wrote into VCS internals: %s exists (%v)", rel, err) + for _, a := range summary.Artifacts() { + if strings.Contains(filepath.ToSlash(a.Path), "/.git/") { + t.Fatalf("produced an artifact inside VCS internals: %s", a.Path) } } } @@ -459,8 +477,8 @@ func TestRunReportsFailedUnits(t *testing.T) { // discovered manifest.json gets a slim copy next to its hash sidecar, and the // sidecar carries the filtered_manifest pointer at it. The pointer hashes the // slim file's own bytes, so a consumer can confirm the two are a matched pair - -// which adjacency alone no longer implies, now that cleanup judges each -// artifact independently. +// which adjacency alone does not imply, since each file on disk is judged by +// its own producer marker. func TestRunWritesSlimManifestAlongsideSidecar(t *testing.T) { root := t.TempDir() writeFiles(t, root, map[string]string{ @@ -477,13 +495,13 @@ func TestRunWritesSlimManifestAlongsideSidecar(t *testing.T) { astroDir := filepath.Join(root, "shipped", sidecarDir) var slim map[string]any - readJSON(t, filepath.Join(astroDir, slimManifestName), &slim) + readJSON(t, summary, filepath.Join(astroDir, slimManifestName), &slim) if _, ok := slim["nodes"].(map[string]any)["model.shop.orders"]; !ok { t.Fatalf("slim manifest missing expected node: %+v", slim) } var meta Metadata - readJSON(t, filepath.Join(astroDir, sidecarName), &meta) + readJSON(t, summary, filepath.Join(astroDir, sidecarName), &meta) fm := meta.FilteredManifest if fm == nil { t.Fatalf("sidecar has no filtered_manifest section: %+v", meta) @@ -491,10 +509,7 @@ func TestRunWritesSlimManifestAlongsideSidecar(t *testing.T) { if fm.Path != slimManifestName || fm.Schema != slimSchemaVersion || fm.Version.Algo != algoFilteredManifest { t.Fatalf("filtered_manifest = %+v", fm) } - slimData, err := os.ReadFile(filepath.Join(astroDir, slimManifestName)) - if err != nil { - t.Fatal(err) - } + slimData, _ := artifact(summary, filepath.Join(astroDir, slimManifestName)) if want := sha256Hex(slimData); fm.Version.Hash != want { t.Fatalf("filtered_manifest hash = %q, want the slim file's own hash %q", fm.Version.Hash, want) } @@ -512,17 +527,16 @@ func TestRunSkipsSlimManifestWhenNotRequested(t *testing.T) { "shipped/manifest.json": `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json","project_name":"shop"},"nodes":{}}`, }) - if _, err := Run([]string{root}, "test", Options{}); err != nil { + summary, err := Run([]string{root}, "test", Options{}) + if err != nil { t.Fatal(err) } astroDir := filepath.Join(root, "shipped", sidecarDir) - if _, err := os.Stat(filepath.Join(astroDir, slimManifestName)); !os.IsNotExist(err) { - t.Fatalf("slim manifest written without being requested: %v", err) - } - raw, err := os.ReadFile(filepath.Join(astroDir, sidecarName)) - if err != nil { - t.Fatal(err) + mustNotProduce(t, summary, filepath.Join(astroDir, slimManifestName)) + raw, ok := artifact(summary, filepath.Join(astroDir, sidecarName)) + if !ok { + t.Fatal("no sidecar produced") } if strings.Contains(string(raw), "filtered_manifest") { t.Fatalf("filtered_manifest emitted with the slim manifest disabled: %s", raw) @@ -553,13 +567,13 @@ func TestRunSlimsManifestInProjectRoot(t *testing.T) { astroDir := filepath.Join(root, "proj", sidecarDir) var slim map[string]any - readJSON(t, filepath.Join(astroDir, slimManifestName), &slim) + readJSON(t, summary, filepath.Join(astroDir, slimManifestName), &slim) if _, ok := slim["nodes"].(map[string]any)["model.shop.orders"]; !ok { t.Fatalf("slim manifest missing expected node: %+v", slim) } var meta Metadata - readJSON(t, filepath.Join(astroDir, sidecarName), &meta) + readJSON(t, summary, filepath.Join(astroDir, sidecarName), &meta) if meta.Version.Algo != algoProjectTree { t.Fatalf("project sidecar algo = %q, want %q", meta.Version.Algo, algoProjectTree) } @@ -602,23 +616,22 @@ func TestRunSkipsNonDbtManifestInProjectRoot(t *testing.T) { "proj/manifest.json": `{"name":"My App","short_name":"App","icons":[]}`, }) - if _, err := Run([]string{root}, "test", Options{SlimManifest: true}); err != nil { + summary, err := Run([]string{root}, "test", Options{SlimManifest: true}) + if err != nil { t.Fatal(err) } astroDir := filepath.Join(root, "proj", sidecarDir) - if _, err := os.Stat(filepath.Join(astroDir, slimManifestName)); !os.IsNotExist(err) { - t.Fatalf("a non-dbt manifest.json must not be slimmed: %v", err) - } + mustNotProduce(t, summary, filepath.Join(astroDir, slimManifestName)) var meta Metadata - readJSON(t, filepath.Join(astroDir, sidecarName), &meta) + readJSON(t, summary, filepath.Join(astroDir, sidecarName), &meta) if meta.FilteredManifest != nil { t.Fatalf("project sidecar points at a slim manifest that was never written: %+v", meta.FilteredManifest) } } -// TestRunLeavesManifestUnchanged: the slim manifest is a copy written into -// .astro/, never an edit of what dbt produced. Worth pinning by bytes, since +// TestRunLeavesManifestUnchanged: the slim manifest is a copy for .astro/, +// never an edit of what dbt produced. Worth pinning by bytes, since // processManifest parses the manifest and hashDocument then mutates that doc. func TestRunLeavesManifestUnchanged(t *testing.T) { root := t.TempDir() @@ -628,12 +641,13 @@ func TestRunLeavesManifestUnchanged(t *testing.T) { writeFiles(t, root, map[string]string{"shipped/manifest.json": manifest}) manifestPath := filepath.Join(root, "shipped", "manifest.json") - if _, err := Run([]string{root}, "test", Options{SlimManifest: true}); err != nil { + summary, err := Run([]string{root}, "test", Options{SlimManifest: true}) + if err != nil { t.Fatal(err) } // Both artifacts must exist, or this would pass trivially. - mustExist(t, filepath.Join(root, "shipped", sidecarDir, sidecarName)) - mustExist(t, filepath.Join(root, "shipped", sidecarDir, slimManifestName)) + mustProduce(t, summary, filepath.Join(root, "shipped", sidecarDir, sidecarName)) + mustProduce(t, summary, filepath.Join(root, "shipped", sidecarDir, slimManifestName)) after, err := os.ReadFile(manifestPath) if err != nil { @@ -669,12 +683,73 @@ func TestRunSlimManifestUnaffectedByHashDocumentMutation(t *testing.T) { } var slim map[string]any - readJSON(t, filepath.Join(root, "shipped", sidecarDir, slimManifestName), &slim) + readJSON(t, summary, filepath.Join(root, "shipped", sidecarDir, slimManifestName), &slim) selector, ok := slim["selectors"].(map[string]any)["my_selector"].(map[string]any) if !ok { t.Fatalf("slim manifest missing expected selector: %+v", slim) } if _, ok := selector["created_at"]; !ok { - t.Fatalf("selector lost created_at: hashDocument's mutation of doc leaked into the already-written slim manifest, got %+v", selector) + t.Fatalf("selector lost created_at: hashDocument's mutation of doc leaked into the already-marshaled slim manifest, got %+v", selector) + } +} + +// TestRunWritesNothing pins the contract the deploy relies on: the artifacts +// live in memory only, and the walked tree is byte-for-byte what it was. +func TestRunWritesNothing(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "proj/dbt_project.yml": "name: p\n", + "proj/models/a.sql": "select 1", + "proj/manifest.json": `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}`, + "proj/target/manifest.json": `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}`, + }) + before := treeSnapshot(t, root) + + summary, err := Run([]string{root}, "test", Options{SlimManifest: true}) + if err != nil { + t.Fatal(err) + } + if got := len(summary.Artifacts()); got != 4 { + t.Fatalf("artifacts = %d, want 4 (two sidecars, two slim manifests)", got) + } + if after := treeSnapshot(t, root); !maps.Equal(before, after) { + t.Fatalf("Run changed the tree:\n before %v\n after %v", before, after) + } +} + +// TestRunFailsUnitWhenAstroIsAFile: a file named .astro where the artifacts +// belong would make the bundle unpackable, so the unit fails and produces +// nothing rather than shipping a path collision. +func TestRunFailsUnitWhenAstroIsAFile(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "proj/dbt_project.yml": "name: p\n", + "proj/.astro": "x", + }) + + summary, err := Run([]string{root}, "test", Options{SlimManifest: true}) + if err != nil { + t.Fatal(err) + } + if summary.CountFailed() != 1 || len(summary.Artifacts()) != 0 { + t.Fatalf("want one failed unit and no artifacts, got %+v", summary.Results) + } +} + +// treeSnapshot maps every file under root to its contents. +func treeSnapshot(t *testing.T, root string) map[string]string { + t.Helper() + out := map[string]string{} + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + out[path] = string(data) + return err + }) + if err != nil { + t.Fatal(err) } + return out } diff --git a/pkg/cosmosboost/predeploy.go b/pkg/cosmosboost/predeploy.go index 1a4321dee..441821335 100644 --- a/pkg/cosmosboost/predeploy.go +++ b/pkg/cosmosboost/predeploy.go @@ -5,9 +5,11 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "github.com/astronomer/astro-cli/pkg/cosmosboost/precompute" + "github.com/astronomer/astro-cli/pkg/fileutil" "github.com/astronomer/astro-cli/pkg/logger" "github.com/astronomer/astro-cli/pkg/util" "github.com/astronomer/astro-cli/version" @@ -19,90 +21,108 @@ import ( // convention for its own optimizations. const slimManifestEnvVar = "ASTRO_COSMOS_BOOST_SLIM_MANIFEST_ENABLED" -// slimManifestEnabled reports whether to write the slim manifest: on when unset, -// otherwise only for a value util.CheckEnvBool reads as true. The plugin keeps -// an unrecognized value on; here it turns the optimization off, because reusing -// this repo's only bool-env helper beats hand-rolling a second parser. +// slimManifestEnabled reports whether to produce the slim manifest: on when +// unset, otherwise only for a value util.CheckEnvBool reads as true. The plugin +// keeps an unrecognized value on; here it turns the optimization off, because +// reusing this repo's only bool-env helper beats hand-rolling a second parser. func slimManifestEnabled() bool { value := os.Getenv(slimManifestEnvVar) return value == "" || util.CheckEnvBool(value) } -// PreDeploy runs the Cosmos Boost pre-deploy step over path: every dbt project -// (dbt_project.yml) gets a .astro/dbt_metadata.json sidecar carrying its -// content hash, which the Cosmos Boost plugin uses as a cache version key at -// parse time instead of hashing the project tree itself. Every standalone dbt -// manifest.json gets a hash sidecar too, plus a slim, field-filtered copy for -// the plugin to load in place of the full manifest at DAG-parse time. -func PreDeploy(path string) error { - opts := precompute.Options{SlimManifest: slimManifestEnabled()} - summary, err := precompute.Run([]string{path}, version.Current(), opts) +// BundleOverlay is what a deploy bundle of root must change for Cosmos Boost, +// as an overlay on the tarball: root itself is never written. +// +// Whatever enabled says, every .astro/dbt_metadata.json and +// .astro/manifest.slim.json under root that an earlier astro-cli wrote into +// the project (its producer marker names this tool) is left out. The plugin +// cannot tell a fresh sidecar from a stale one, and this one never writes +// there, so nothing a deploy ships comes from the tree. +// +// With enabled set (cosmos_boost.pre_deploy), the pre-deploy step then runs +// over root and its sidecars are added to the bundle at the same paths, with +// the same contents, the plugin has always read: every dbt project +// (dbt_project.yml) gets a .astro/dbt_metadata.json carrying its content hash, +// which the plugin uses as a cache version key at parse time instead of +// hashing the project tree itself, and every standalone dbt manifest.json gets +// a hash sidecar too, plus a slim, field-filtered copy for the plugin to load +// in place of the full manifest. That step is best-effort - a failure is a +// warning, never an error - because without a sidecar the plugin simply falls +// back to hashing at parse time. +// +// A file from an unrecognized producer is the user's, and ships untouched +// when the step is off. With it on, the deploy fails instead: the plugin would +// consume that file's version.hash all the same, and this tool will not judge +// a file it did not write fresh. +func BundleOverlay(root string, enabled bool) (fileutil.TarOverlay, error) { + overlay := fileutil.TarOverlay{Skip: map[string]bool{}, Extra: map[string][]byte{}} + + existing, err := precompute.FindExisting(root) if err != nil { - return fmt.Errorf("running the Cosmos Boost pre-deploy step: %w", err) + return overlay, err } - debugReport("pre-deploy", summary) - if failed := summary.CountFailed(); failed > 0 { - return fmt.Errorf("the Cosmos Boost pre-deploy step failed for %d unit(s) (re-run with --verbosity debug for details)", failed) + var foreign []string + for _, e := range existing { + if !e.Ours { + foreign = append(foreign, e.Path) + continue + } + rel, err := relSlash(root, e.Path) + if err != nil { + return overlay, err + } + logger.Debugf("leaving out a Cosmos Boost file an earlier Astro CLI wrote into the project: %s", e.Path) + overlay.Skip[rel] = true + } + if !enabled { + return overlay, nil + } + if len(foreign) > 0 { + return overlay, fmt.Errorf("found Cosmos Boost file(s) from an unrecognized producer: %s; the deploy cannot prove they are fresh and will not ship them for the Cosmos Boost plugin to read. Remove them and retry", strings.Join(foreign, ", ")) } - return nil -} -// cleanupRoots deletes the sidecars earlier pre-deploy runs wrote under -// roots. Files this integration did not write are left in place; the summary -// is returned so callers with stricter needs can inspect what was kept. -func cleanupRoots(roots []string) (precompute.CleanupSummary, error) { - summary, err := precompute.Cleanup(roots) - if err != nil { - return summary, fmt.Errorf("removing the Cosmos Boost artifacts: %w", err) + artifacts, err := preDeploy(root) + for _, a := range artifacts { + rel, relErr := relSlash(root, a.Path) + if relErr != nil { + return overlay, relErr + } + overlay.Extra[rel] = a.Data } - debugReport("cleanup", summary) - if failed := summary.CountFailed(); failed > 0 { - return summary, fmt.Errorf("could not remove %d Cosmos Boost artifact(s) (re-run with --verbosity debug for details)", failed) + if err != nil { + fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) + return overlay, nil } - return summary, nil + fmt.Println("Cosmos Boost pre-deploy step complete") + return overlay, nil } -// EnsureClean removes the Cosmos Boost artifacts earlier deploys left under -// path, and fails when their absence cannot be guaranteed. Consumers cannot -// tell fresh output from stale, so a deploy must not proceed while a stale -// artifact may still be in the payload. -// -// That includes sidecars from an unrecognized producer: cleanup deliberately -// never deletes a file it does not own, but the plugin would consume its -// version.hash all the same if the deploy shipped it - so an enabled deploy -// stops and asks for manual removal instead. `astro dbt cleanup` keeps -// preserving such files. -func EnsureClean(path string) error { - summary, err := cleanupRoots([]string{path}) +// preDeploy runs the Cosmos Boost pre-deploy step over path and returns the +// artifacts it produced, in memory. It fails when any unit failed, still +// returning the artifacts of the ones that did not. +func preDeploy(path string) ([]precompute.Artifact, error) { + opts := precompute.Options{SlimManifest: slimManifestEnabled()} + summary, err := precompute.Run([]string{path}, version.Current(), opts) if err != nil { - return fmt.Errorf("%w; remove the reported files (or run 'astro dbt cleanup %s') and retry", err, path) + return nil, fmt.Errorf("running the Cosmos Boost pre-deploy step: %w", err) } - var kept []string - for _, r := range summary.Results { - if r.Kept { - kept = append(kept, r.Path) - } - } - if len(kept) > 0 { - return fmt.Errorf("found Cosmos Boost artifact(s) from an unrecognized producer: %s; the deploy cannot prove they are fresh, and cleanup will not delete files it does not own - remove them manually and retry", strings.Join(kept, ", ")) + debugReport("pre-deploy", summary) + if failed := summary.CountFailed(); failed > 0 { + return summary.Artifacts(), fmt.Errorf("the Cosmos Boost pre-deploy step failed for %d unit(s) (re-run with --verbosity debug for details)", failed) } - return nil + return summary.Artifacts(), nil } -// BestEffortPreDeploy runs the Cosmos Boost pre-deploy step over path ahead of -// a deploy. Stamping is best-effort — failures are warnings, never errors — -// because without a sidecar the plugin simply falls back to hashing at parse -// time. Callers must run EnsureClean first: writing nothing is safe, leaving -// something stale is not. -func BestEffortPreDeploy(path string) { - if err := PreDeploy(path); err != nil { - fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) - return +// relSlash is path relative to root, slash-separated, as TarOverlay keys it. +func relSlash(root, path string) (string, error) { + rel, err := filepath.Rel(root, path) + if err != nil { + return "", err } - fmt.Println("Cosmos Boost pre-deploy step complete") + return filepath.ToSlash(rel), nil } -// reporter is the common shape of the precompute summaries. +// reporter is the shape of the precompute summary. type reporter interface{ WriteReport(io.Writer) } // debugReport keeps the step's detailed report behind --verbosity debug; the diff --git a/pkg/cosmosboost/predeploy_test.go b/pkg/cosmosboost/predeploy_test.go index 9de7c18a0..dd73ebe48 100644 --- a/pkg/cosmosboost/predeploy_test.go +++ b/pkg/cosmosboost/predeploy_test.go @@ -2,6 +2,7 @@ package cosmosboost import ( "encoding/json" + "io/fs" "os" "path/filepath" "runtime" @@ -9,30 +10,80 @@ import ( "github.com/stretchr/testify/require" + "github.com/astronomer/astro-cli/pkg/cosmosboost/precompute" "github.com/astronomer/astro-cli/version" ) -const artifactRelPath = ".astro/dbt_metadata.json" +const ( + sidecarRel = ".astro/dbt_metadata.json" + oursSidecar = `{"schema":1,"version":{"algo":"sha256-tree-v2","hash":"stale"},"generated_by":{"application":"astro","version":"1.0.0"}}` + foreignFile = `{"schema":1,"version":{"hash":"abc"},"generated_by":{"application":"someone-else"}}` + dbtManifest = `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}` +) + +func writeFiles(t *testing.T, dir string, files map[string]string) { + t.Helper() + for rel, content := range files { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + } +} func writeDbtProject(t *testing.T, dir string) { t.Helper() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "models"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte("name: shop\n"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "models", "a.sql"), []byte("select 1"), 0o644)) + writeFiles(t, dir, map[string]string{"dbt_project.yml": "name: shop\n", "models/a.sql": "select 1"}) +} + +// snapshot maps every file under root to its contents, to prove a call left +// the tree exactly as it found it. +func snapshot(t *testing.T, root string) map[string]string { + t.Helper() + out := map[string]string{} + require.NoError(t, filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + out[path] = string(data) + return err + })) + return out } -func TestPreDeployWritesArtifact(t *testing.T) { - origVersion := version.CurrVersion - version.CurrVersion = "1.2.3" - t.Cleanup(func() { version.CurrVersion = origVersion }) +func setVersion(t *testing.T, v string) { + t.Helper() + orig := version.CurrVersion + version.CurrVersion = v + t.Cleanup(func() { version.CurrVersion = orig }) +} +// TestBundleOverlayEnabledAddsSidecars pins the plugin contract: the same +// files at the same paths with the same contents earlier versions wrote into +// the project, now only in the bundle - the tree is untouched. +func TestBundleOverlayEnabledAddsSidecars(t *testing.T) { + setVersion(t, "1.2.3") dir := t.TempDir() writeDbtProject(t, dir) + writeFiles(t, dir, map[string]string{"target/manifest.json": dbtManifest}) + before := snapshot(t, dir) + + overlay, err := BundleOverlay(dir, true) + require.NoError(t, err) - require.NoError(t, PreDeploy(dir)) + require.Equal(t, before, snapshot(t, dir), "the tree must not be written") + require.Empty(t, overlay.Skip) + require.ElementsMatch(t, []string{sidecarRel, "target/" + sidecarRel, "target/.astro/manifest.slim.json"}, keys(overlay.Extra)) - data, err := os.ReadFile(filepath.Join(dir, artifactRelPath)) + // The contents are exactly what the pre-deploy step produces for the tree. + summary, err := precompute.Run([]string{dir}, "1.2.3", precompute.Options{SlimManifest: true}) require.NoError(t, err) + for _, a := range summary.Artifacts() { + rel, err := filepath.Rel(dir, a.Path) + require.NoError(t, err) + require.Equal(t, string(a.Data), string(overlay.Extra[filepath.ToSlash(rel)]), rel) + } + var meta struct { Schema int `json:"schema"` Version struct { @@ -44,40 +95,28 @@ func TestPreDeployWritesArtifact(t *testing.T) { Version string `json:"version"` } `json:"generated_by"` } - require.NoError(t, json.Unmarshal(data, &meta)) + require.NoError(t, json.Unmarshal(overlay.Extra[sidecarRel], &meta)) require.Equal(t, 1, meta.Schema, "schema is the plugin's compatibility gate and must stay 1") - require.NotEmpty(t, meta.Version.Hash, "version.hash is what the plugin consumes") - require.NotEmpty(t, meta.Version.Algo) + require.Equal(t, "sha256-tree-v2", meta.Version.Algo) + for _, r := range summary.Results { + if r.Path == dir { + require.Equal(t, r.Hash, meta.Version.Hash, "version.hash is what the plugin consumes") + } + } require.Equal(t, "astro", meta.GeneratedBy.Application) - require.Equal(t, "1.2.3", meta.GeneratedBy.Version, "provenance records the CLI version now that the step runs in-process") -} - -// TestPreDeployWritesSlimManifest pins that PreDeploy writes a slim manifest -// next to a discovered manifest.json's hash sidecar. It is on by default: the -// only switch is the step itself (cosmos_boost.pre_deploy), with a per-feature -// env var below. -func TestPreDeployWritesSlimManifest(t *testing.T) { - dir := t.TempDir() - manifest := `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}` - require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0o644)) - - require.NoError(t, PreDeploy(dir)) - - require.FileExists(t, filepath.Join(dir, ".astro", "manifest.slim.json")) + require.Equal(t, "1.2.3", meta.GeneratedBy.Version) } -// TestPreDeployRespectsSlimManifestEnvVar: the env var disables this one -// optimization without disabling the step, so the hash sidecar still lands. -func TestPreDeployRespectsSlimManifestEnvVar(t *testing.T) { +// TestBundleOverlayRespectsSlimManifestEnvVar: the env var disables this one +// optimization without disabling the step, so the hash sidecar still ships. +func TestBundleOverlayRespectsSlimManifestEnvVar(t *testing.T) { dir := t.TempDir() - manifest := `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}` - require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0o644)) + writeFiles(t, dir, map[string]string{"manifest.json": dbtManifest}) t.Setenv(slimManifestEnvVar, "false") - require.NoError(t, PreDeploy(dir)) - - require.FileExists(t, filepath.Join(dir, ".astro", "dbt_metadata.json")) - require.NoFileExists(t, filepath.Join(dir, ".astro", "manifest.slim.json")) + overlay, err := BundleOverlay(dir, true) + require.NoError(t, err) + require.Equal(t, []string{sidecarRel}, keys(overlay.Extra)) } // TestSlimManifestEnabled: unset leaves the optimization on; otherwise only a @@ -101,118 +140,99 @@ func TestSlimManifestEnabled(t *testing.T) { } } -func TestPreDeployNoDbtContentIsANoOp(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "app.py"), []byte("print('hi')"), 0o644)) - - require.NoError(t, PreDeploy(dir)) - - _, err := os.Stat(filepath.Join(dir, ".astro")) - require.True(t, os.IsNotExist(err), "nothing dbt-shaped means nothing written") -} - -func TestPreDeployNonexistentPath(t *testing.T) { - require.Error(t, PreDeploy(filepath.Join(t.TempDir(), "does-not-exist"))) -} - -// TestEnsureCleanRemovesStaleArtifacts covers the sequence the deploy hooks -// run: EnsureClean first, then (opt-in) BestEffortPreDeploy — so an artifact -// from an earlier deploy, even one whose project no longer exists, cannot -// survive into the payload. -func TestEnsureCleanRemovesStaleArtifacts(t *testing.T) { - dir := t.TempDir() - writeDbtProject(t, dir) - - // A stale artifact from an earlier deploy, in a spot the current tree no - // longer produces one for (its project was deleted since). - stale := filepath.Join(dir, "removed-project", ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(stale), 0o755)) - require.NoError(t, os.WriteFile(stale, []byte(`{"generated_by": {"application": "astro"}}`), 0o644)) - - require.NoError(t, EnsureClean(dir)) - BestEffortPreDeploy(dir) - - _, err := os.Stat(stale) - require.True(t, os.IsNotExist(err), "stale artifacts must not survive a pre-deploy run") - require.FileExists(t, filepath.Join(dir, artifactRelPath), "the current project must be stamped") +// TestBundleOverlayLeavesOutStaleSidecars: whether the step is on or off, a +// sidecar an earlier astro-cli wrote into the project never ships - including +// one whose project no longer exists - and the tree keeps it. +func TestBundleOverlayLeavesOutStaleSidecars(t *testing.T) { + for _, enabled := range []bool{true, false} { + dir := t.TempDir() + writeDbtProject(t, dir) + writeFiles(t, dir, map[string]string{ + sidecarRel: oursSidecar, + "removed-project/" + sidecarRel: oursSidecar, + "removed-project/.astro/manifest.slim.json": `{"_generated_by":{"application":"astro"}}`, + }) + before := snapshot(t, dir) + + overlay, err := BundleOverlay(dir, enabled) + require.NoError(t, err) + + require.Equal(t, before, snapshot(t, dir), "enabled=%v: the tree must not be written", enabled) + require.ElementsMatch(t, []string{sidecarRel, "removed-project/" + sidecarRel, "removed-project/.astro/manifest.slim.json"}, keys(overlay.Skip), "enabled=%v", enabled) + if enabled { + require.Equal(t, []string{sidecarRel}, keys(overlay.Extra), "the current project gets a fresh sidecar") + require.NotContains(t, string(overlay.Extra[sidecarRel]), "stale") + } else { + require.Empty(t, overlay.Extra, "a disabled deploy adds nothing") + } + } } -// TestEnsureCleanFailsOnForeignSidecar pins the strict-ownership contract for -// deploys: cleanup never deletes a file it does not own, but the plugin would -// consume its version.hash all the same - so an enabled deploy must stop and -// ask for manual removal instead of shipping it. -func TestEnsureCleanFailsOnForeignSidecar(t *testing.T) { +// TestBundleOverlayForeignSidecar: a file from an unrecognized producer is +// the user's. Disabled, it ships untouched; enabled, the deploy stops and asks +// for its removal, because the plugin would consume its version.hash. +func TestBundleOverlayForeignSidecar(t *testing.T) { dir := t.TempDir() writeDbtProject(t, dir) - require.NoError(t, PreDeploy(dir)) - + writeFiles(t, dir, map[string]string{"other/" + sidecarRel: foreignFile}) foreign := filepath.Join(dir, "other", ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(foreign), 0o755)) - require.NoError(t, os.WriteFile(foreign, []byte(`{"generated_by": {"application": "someone-else"}}`), 0o644)) - err := EnsureClean(dir) + overlay, err := BundleOverlay(dir, false) + require.NoError(t, err) + require.Empty(t, overlay.Skip, "disabled: a foreign file ships") + require.Empty(t, overlay.Extra) - require.Error(t, err, "an unrecognized sidecar must fail the clean") + _, err = BundleOverlay(dir, true) + require.Error(t, err) + require.ErrorContains(t, err, "unrecognized producer") require.ErrorContains(t, err, foreign) - require.ErrorContains(t, err, "remove them manually") - _, statErr := os.Stat(filepath.Join(dir, artifactRelPath)) - require.True(t, os.IsNotExist(statErr), "our artifact must still be removed") + require.ErrorContains(t, err, "Remove them and retry") require.FileExists(t, foreign, "a file another tool wrote must never be deleted") } -// TestCleanupKeepsForeignSidecarWithoutError: the explicit command keeps -// preserving unrecognized files and reports success, unlike EnsureClean. -func TestCleanupKeepsForeignSidecarWithoutError(t *testing.T) { +func TestBundleOverlayNoDbtContentIsEmpty(t *testing.T) { dir := t.TempDir() - foreign := filepath.Join(dir, "other", ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(foreign), 0o755)) - require.NoError(t, os.WriteFile(foreign, []byte(`{"generated_by": {"application": "someone-else"}}`), 0o644)) + writeFiles(t, dir, map[string]string{"app.py": "print('hi')"}) - require.NoError(t, cleanupErr(dir)) - require.FileExists(t, foreign) + overlay, err := BundleOverlay(dir, true) + require.NoError(t, err) + require.Empty(t, overlay.Skip) + require.Empty(t, overlay.Extra) } -// TestEnsureCleanFailsWhenArtifactUnremovable pins the safety property behind -// the hooks: when a stale artifact's absence cannot be guaranteed, EnsureClean -// errors so the deploy stops instead of shipping it. -func TestEnsureCleanFailsWhenArtifactUnremovable(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory write-permission semantics differ on windows") - } - dir := t.TempDir() - astroDir := filepath.Join(dir, "proj", ".astro") - require.NoError(t, os.MkdirAll(astroDir, 0o755)) - artifact := filepath.Join(astroDir, "dbt_metadata.json") - require.NoError(t, os.WriteFile(artifact, []byte(`{"generated_by": {"application": "astro"}}`), 0o644)) - require.NoError(t, os.Chmod(astroDir, 0o555)) // file cannot be unlinked - t.Cleanup(func() { _ = os.Chmod(astroDir, 0o755) }) - - err := EnsureClean(dir) - - require.Error(t, err, "an artifact that cannot be removed must fail the clean") - require.ErrorContains(t, err, "astro dbt cleanup") - require.FileExists(t, artifact) +// TestBundleOverlayMissingRoot: a root that is not there cannot be checked +// for stale files, so the deploy fails rather than guessing. +func TestBundleOverlayMissingRoot(t *testing.T) { + _, err := BundleOverlay(filepath.Join(t.TempDir(), "does-not-exist"), false) + require.Error(t, err) } -// TestPreDeployReportsFailedUnits: a unit that cannot be hashed turns into an -// error naming the failure count, so the hook can warn with substance. -func TestPreDeployReportsFailedUnits(t *testing.T) { +// TestBundleOverlayStepFailureIsAWarning: stamping is best-effort - a unit +// that cannot be hashed ships no sidecar, the others still do, and the +// deploy goes on. +func TestBundleOverlayStepFailureIsAWarning(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("file permission semantics differ on windows") } dir := t.TempDir() - writeDbtProject(t, dir) - model := filepath.Join(dir, "models", "a.sql") + writeDbtProject(t, filepath.Join(dir, "broken")) + writeDbtProject(t, filepath.Join(dir, "fine")) + model := filepath.Join(dir, "broken", "models", "a.sql") require.NoError(t, os.Chmod(model, 0o000)) t.Cleanup(func() { _ = os.Chmod(model, 0o644) }) - err := PreDeploy(dir) + overlay, err := BundleOverlay(dir, true) + require.NoError(t, err) + require.Equal(t, []string{"fine/" + sidecarRel}, keys(overlay.Extra)) - require.Error(t, err) + _, err = preDeploy(dir) require.ErrorContains(t, err, "failed for 1 unit(s)") } -// TestBestEffortPreDeployWarnsAndContinues: stamping failures never propagate. -func TestBestEffortPreDeployWarnsAndContinues(t *testing.T) { - BestEffortPreDeploy(filepath.Join(t.TempDir(), "does-not-exist")) +func keys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out } diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index e13f27bd5..005a47d20 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -13,6 +13,7 @@ import ( "path" "path/filepath" "regexp" + "sort" "strings" "time" @@ -105,15 +106,28 @@ func WriteToFile(filePath string, r io.Reader) error { return err } +// TarOverlay changes what Tar writes without touching source. Both maps are +// keyed by slash-separated paths relative to source (before any base +// directory is prepended): Skip leaves those files out, and Extra adds a +// regular file with those bytes, in place of any file at the same path. +type TarOverlay struct { + Skip map[string]bool + Extra map[string][]byte +} + +// extraFileMode is an Extra file's mode in the tarball: world-readable, since +// whatever unpacks it (the Airflow runtime user) must be able to read it. +const extraFileMode = 0o644 + // Tar writes the directory source to the tarball target, its paths -// relative to source, or to source's parent with prependBaseDir. A source -// that is not there makes an empty tarball, not an error. -func Tar(source, target string, prependBaseDir bool, excludePathPrefixes []string) error { +// relative to source, or to source's parent with prependBaseDir, changed by +// overlay. A source that is not there makes an empty tarball, not an error. +func Tar(source, target string, prependBaseDir bool, excludePathPrefixes []string, overlay TarOverlay) error { baseDir := "" if prependBaseDir { baseDir = filepath.Base(source) } - return writeTar(source, target, baseDir, excludePathPrefixes, false) + return writeTar(source, target, baseDir, excludePathPrefixes, overlay, false) } // TarDir writes the directory source to the tarball target, each path in it @@ -121,7 +135,7 @@ func Tar(source, target string, prependBaseDir bool, excludePathPrefixes []strin // source is called. Unlike Tar it fails on a source that is not there, or // that goes while it is read, rather than writing a tarball short of it. func TarDir(source, target, baseDir string) error { - return writeTar(source, target, baseDir, nil, true) + return writeTar(source, target, baseDir, nil, TarOverlay{}, true) } // createTarFile creates the tarball writeTar writes; a test fails its Close. @@ -129,7 +143,7 @@ var createTarFile = func(name string) (io.WriteCloser, error) { return os.Create // writeTar writes the tarball. A tarball whose footer or file did not close // is not one, so failing to close either fails it. -func writeTar(source, target, baseDir string, excludePathPrefixes []string, mustExist bool) (err error) { +func writeTar(source, target, baseDir string, excludePathPrefixes []string, overlay TarOverlay, mustExist bool) (err error) { tarfile, err := createTarFile(target) if err != nil { return err @@ -147,11 +161,11 @@ func writeTar(source, target, baseDir string, excludePathPrefixes []string, must err = cerr } }() - return addToTar(tarball, source, target, baseDir, excludePathPrefixes, mustExist) + return addToTar(tarball, source, target, baseDir, excludePathPrefixes, overlay, mustExist) } // addToTar writes the directory source into tarball, as writeTar says. -func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPrefixes []string, mustExist bool) error { +func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPrefixes []string, overlay TarOverlay, mustExist bool) error { sourceInfo, err := os.Stat(source) if err != nil { if mustExist { @@ -164,7 +178,7 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr return errors.New("source is not a directory") } - return filepath.Walk(source, + err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -190,22 +204,10 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr return err } - // set the tar file path to be relative to the source directory - headerName := strings.TrimPrefix(path, filepath.Clean(source)) - headerName = strings.TrimPrefix(headerName, string(filepath.Separator)) - // prepend the base directory to the tar file path, e.g. "dags/" to "my_dag.py" - headerName = filepath.Join(baseDir, headerName) - // force use forward slashes in tar files - headerName = filepath.ToSlash(headerName) - - // ignore excluded paths - for _, excludePathPrefix := range excludePathPrefixes { - if strings.HasPrefix(headerName, excludePathPrefix) { - logger.Debugf("Excluding tarball path: %s", headerName) - return nil - } + headerName, ok := tarEntryName(path, source, baseDir, excludePathPrefixes, overlay) + if !ok { + return nil } - header.Name = headerName logger.Debugf("Adding to tarball: %s", header.Name) @@ -225,6 +227,64 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr _, err = io.Copy(tarball, file) return err }) + if err != nil { + return err + } + return addExtrasToTar(tarball, baseDir, overlay.Extra) +} + +// tarEntryName is the name file goes into the tarball under, or +// false when it stays out: one the overlay replaces or skips, or one under an +// excluded prefix. +func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, overlay TarOverlay) (string, bool) { + // set the tar file path to be relative to the source directory + relName := strings.TrimPrefix(file, filepath.Clean(source)) + relName = strings.TrimPrefix(relName, string(filepath.Separator)) + if rel := filepath.ToSlash(relName); overlay.Skip[rel] || overlay.Extra[rel] != nil { + logger.Debugf("Leaving out of the tarball: %s", rel) + return "", false + } + // prepend the base directory to the tar file path, e.g. "dags/" to "my_dag.py", + // with forward slashes whatever the platform + headerName := filepath.ToSlash(filepath.Join(baseDir, relName)) + + // ignore excluded paths + for _, excludePathPrefix := range excludePathPrefixes { + if strings.HasPrefix(headerName, excludePathPrefix) { + logger.Debugf("Excluding tarball path: %s", headerName) + return "", false + } + } + return headerName, true +} + +// addExtrasToTar writes each of extra into tarball under baseDir, in path +// order so the same overlay makes the same tarball. +func addExtrasToTar(tarball *tar.Writer, baseDir string, extra map[string][]byte) error { + names := make([]string, 0, len(extra)) + for name := range extra { + names = append(names, name) + } + sort.Strings(names) + now := time.Now() + for _, name := range names { + data := extra[name] + header := &tar.Header{ + Typeflag: tar.TypeReg, + Name: path.Join(filepath.ToSlash(baseDir), name), + Mode: extraFileMode, + Size: int64(len(data)), + ModTime: now, + } + logger.Debugf("Adding to tarball: %s", header.Name) + if err := tarball.WriteHeader(header); err != nil { + return err + } + if _, err := tarball.Write(data); err != nil { + return err + } + } + return nil } // skipPycache keeps bytecode caches out of a tarball. Only the directory is diff --git a/pkg/fileutil/files_test.go b/pkg/fileutil/files_test.go index 9c91960f3..0acd37fbe 100644 --- a/pkg/fileutil/files_test.go +++ b/pkg/fileutil/files_test.go @@ -240,7 +240,7 @@ func (s *Suite) TestTar() { for _, tt := range tests { s.Run(tt.name, func() { // check that the tar operation was successful - assert.True(s.T(), tt.errAssertion(s.T(), Tar(tt.args.source, tt.args.target, tt.args.prependBaseDir, tt.args.excludePaths))) + assert.True(s.T(), tt.errAssertion(s.T(), Tar(tt.args.source, tt.args.target, tt.args.prependBaseDir, tt.args.excludePaths, TarOverlay{}))) // check that all the files are in the tar at the correct paths file, err := os.Open(tt.args.target) @@ -292,7 +292,62 @@ func (s *Suite) TestTarDir() { missing := filepath.Join(dir, "missing") s.ErrorIs(TarDir(missing, filepath.Join(dir, "missing.tar"), "dags"), os.ErrNotExist) - s.NoError(Tar(missing, filepath.Join(dir, "missing-tar.tar"), true, nil), "Tar is unchanged") + s.NoError(Tar(missing, filepath.Join(dir, "missing-tar.tar"), true, nil, TarOverlay{}), "Tar is unchanged") +} + +// TarOverlay leaves Skip's files out, adds Extra's under the base directory +// (replacing a file at the same path), and never touches the source. +func (s *Suite) TestTarOverlay() { + dir := s.T().TempDir() + source := filepath.Join(dir, "proj") + s.Require().NoError(os.MkdirAll(filepath.Join(source, ".astro"), 0o755)) + s.Require().NoError(os.WriteFile(filepath.Join(source, "a.sql"), []byte("a"), 0o600)) + s.Require().NoError(os.WriteFile(filepath.Join(source, ".astro", "stale.json"), []byte("stale"), 0o600)) + s.Require().NoError(os.WriteFile(filepath.Join(source, ".astro", "replaced.json"), []byte("old"), 0o600)) + overlay := TarOverlay{ + Skip: map[string]bool{".astro/stale.json": true}, + Extra: map[string][]byte{ + ".astro/replaced.json": []byte("new"), + "sub/.astro/added.json": []byte("added"), + }, + } + + for _, tc := range []struct { + prepend bool + prefix string + }{{false, ""}, {true, "proj/"}} { + target := filepath.Join(dir, fmt.Sprintf("overlay-%v.tar", tc.prepend)) + s.Require().NoError(Tar(source, target, tc.prepend, nil, overlay)) + + file, err := os.Open(target) + s.Require().NoError(err) + got := map[string]string{} + tr := tar.NewReader(file) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + s.Require().NoError(err) + data, err := io.ReadAll(tr) + s.Require().NoError(err) + got[header.Name] = string(data) + if header.Name == tc.prefix+"sub/.astro/added.json" { + s.Equal(int64(0o644), header.Mode) + } + } + file.Close() + s.Equal(map[string]string{ + tc.prefix + "a.sql": "a", + tc.prefix + ".astro/replaced.json": "new", + tc.prefix + "sub/.astro/added.json": "added", + }, got) + } + + stale, err := os.ReadFile(filepath.Join(source, ".astro", "replaced.json")) + s.Require().NoError(err) + s.Equal("old", string(stale), "the source is never written") + s.NoDirExists(filepath.Join(source, "sub")) } // failingClose is a tarball file whose writes land and whose Close fails. @@ -314,7 +369,7 @@ func (s *Suite) TestTarFailsOnAFailedClose() { s.EqualError(TarDir(source, "dags.tar", "dags"), "close failed") s.Positive(written.Len(), "the footer was flushed before the file was closed") - s.EqualError(Tar(source, "dags.tar", true, nil), "close failed") + s.EqualError(Tar(source, "dags.tar", true, nil, TarOverlay{}), "close failed") } func (s *Suite) TestReadFileToString() { From eff03eb5fbbe79c20e99dee9054fa6627916db03 Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 12:33:53 -0400 Subject: [PATCH 2/7] Leave old Cosmos Boost sidecars out inside the tarball's walk, and out of generated images The bundle judges each .astro/dbt_metadata.json and manifest.slim.json by its producer marker inside Tar's own walk (TarOverlay.Skip), rather than in a separate pre-scan: what it leaves out is exactly what the tarball would have carried (one under logs/ too), and a deploy with the step off never fails over Cosmos Boost - a missing dags/ makes the empty tarball it always did. A foreign file still refuses an enabled deploy, now even where a fresh sidecar would replace it. The predicate lives in pkg/airflowrt, a leaf both the root module and pkg/imagebuild already require. A generated image's project copy (and its survey and tag digest) leaves the same files out; a declared Dockerfile's context carrying them is warned about, naming them. Tar names entries with filepath.Rel, so a source of "." (astro dbt deploy --project-path .) names .astro/ and other dotfiles right. A directory or link where a sidecar goes fails that unit instead of colliding with it in the bundle. The dbt cleanup stub says what v2 does now, and why old files are still worth deleting by hand. Co-Authored-By: Claude --- cmd/astro/dbt.go | 13 +- cmd/astro/dbt_output_test.go | 2 +- internal/platform/astro/deploy/bundle.go | 10 +- .../astro/deploy/cosmosboost_hook_test.go | 41 ++++ internal/shipcheck/shipcheck.go | 17 ++ internal/shipcheck/shipcheck_test.go | 25 +++ pkg/airflowrt/cosmosboost.go | 61 ++++++ pkg/airflowrt/cosmosboost_test.go | 46 +++++ pkg/cosmosboost/precompute/existing.go | 92 --------- pkg/cosmosboost/precompute/existing_test.go | 126 ------------ pkg/cosmosboost/precompute/metadata.go | 22 ++- pkg/cosmosboost/precompute/precompute.go | 12 +- pkg/cosmosboost/precompute/precompute_test.go | 25 ++- pkg/cosmosboost/predeploy.go | 127 ++++++------ pkg/cosmosboost/predeploy_test.go | 181 +++++++++++------- pkg/fileutil/files.go | 59 ++++-- pkg/fileutil/files_test.go | 45 ++++- pkg/imagebuild/projectfiles.go | 60 +++++- pkg/imagebuild/projectsurvey_test.go | 48 +++++ 19 files changed, 623 insertions(+), 389 deletions(-) create mode 100644 pkg/airflowrt/cosmosboost.go create mode 100644 pkg/airflowrt/cosmosboost_test.go delete mode 100644 pkg/cosmosboost/precompute/existing.go delete mode 100644 pkg/cosmosboost/precompute/existing_test.go diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index b7e879058..dc6e4ba97 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -53,14 +53,13 @@ func newDbtCmd(out io.Writer) *cobra.Command { } // errDbtCleanupRemoved is what astro dbt cleanup says now. It removed the -// Cosmos Boost sidecars earlier deploys wrote into the project; deploys now -// generate them into the bundle they upload and leave out any an earlier -// version wrote, so there is nothing left for it to do. +// Cosmos Boost sidecars earlier deploys wrote into the project; v2 writes +// none there, and leaves out the old ones from what it ships. const errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + - "Cosmos Boost no longer writes into the project, so there is nothing to clean up. " + - "A deploy generates its sidecars into the bundle it uploads, and never ships the " + - ".astro/dbt_metadata.json or .astro/manifest.slim.json files an earlier version wrote into the project; " + - "delete those by hand whenever you like" + "Astro CLI v2 no longer writes Cosmos Boost files into the project, and leaves out the ones earlier versions wrote " + + "from every bundle it deploys and every image it generates. " + + "Delete any old .astro/dbt_metadata.json and .astro/manifest.slim.json files by hand: " + + "a local Airflow running the Cosmos Boost plugin, or the build of a declared Dockerfile, may still read them" // newDbtCleanupRemovedCmd builds the astro dbt cleanup tombstone // (cliout.RemovedCommand), so a script that still runs it is told why rather diff --git a/cmd/astro/dbt_output_test.go b/cmd/astro/dbt_output_test.go index a20845a8b..12ea1a6c0 100644 --- a/cmd/astro/dbt_output_test.go +++ b/cmd/astro/dbt_output_test.go @@ -334,7 +334,7 @@ func TestDbtCleanupRemoved(t *testing.T) { require.Error(t, r.err) assert.Equal(t, cliout.ExitUsage, r.code) assert.Equal(t, errDbtCleanupRemoved, r.err.Error()) - assert.Contains(t, r.err.Error(), "nothing to clean up") + assert.Contains(t, r.err.Error(), "no longer writes Cosmos Boost files") assert.NotContains(t, r.err.Error(), "`", "command names are plain text") assert.Empty(t, r.stdout) assert.FileExists(t, artifact, "the tombstone removes nothing") diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index b03471d5e..579cf1618 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -300,16 +300,16 @@ func UploadBundle(tarDirPath, bundlePath, uploadURL string, prependBaseDir bool, // tree. Ones an earlier CLI wrote into the tree are always left out (a // stale one must not ship), and with cosmos_boost.pre_deploy on, fresh // ones are generated into the bundle, best-effort (a missing one is safe). - overlay, err := cosmosboost.BundleOverlay(bundlePath, config.CFG.CosmosBoostPreDeploy.GetBool()) - if err != nil { - return "", err - } + cosmosBoost := cosmosboost.NewBundle(bundlePath, config.CFG.CosmosBoostPreDeploy.GetBool()) // Generate the bundle tar - err = fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}, overlay) + err := fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}, cosmosBoost.Overlay) if err != nil { return "", err } + if err := cosmosBoost.Err(); err != nil { + return "", err + } // Gzip the tar err = fileutil.GzipFile(tarFilePath, tarGzFilePath) diff --git a/internal/platform/astro/deploy/cosmosboost_hook_test.go b/internal/platform/astro/deploy/cosmosboost_hook_test.go index 58d44a4a1..0cffba63b 100644 --- a/internal/platform/astro/deploy/cosmosboost_hook_test.go +++ b/internal/platform/astro/deploy/cosmosboost_hook_test.go @@ -280,3 +280,44 @@ func TestUploadBundleForeignSidecar(t *testing.T) { assert.FileExists(t, foreign, "the foreign file must be preserved") }) } + +// TestUploadBundleMissingDirIsUnchanged: a DAG deploy of a project with no +// dags/ uploads the empty tarball it always did, with Cosmos Boost on or off: +// the step never fails a deploy that had nothing for it. +func TestUploadBundleMissingDirIsUnchanged(t *testing.T) { + for _, enabled := range []bool{false, true} { + setupCosmosBoostEnv(t) + if enabled { + enableCosmosBoost(t) + } + dagsDir := filepath.Join(t.TempDir(), "dags") + uploaded := captureUploads(t) + + _, err := UploadBundle(t.TempDir(), dagsDir, "http://upload-url", true, "0.0.0") + require.NoError(t, err, "enabled=%v", enabled) + assert.Empty(t, uploaded(), "enabled=%v", enabled) + assert.NoDirExists(t, dagsDir) + } +} + +// TestUploadBundleFromTheWorkingDirectory: astro dbt deploy --project-path . +// bundles ".", and an old sidecar there is still replaced by the fresh one, +// at the same path, with no second copy under another name. +func TestUploadBundleFromTheWorkingDirectory(t *testing.T) { + setupCosmosBoostEnv(t) + enableCosmosBoost(t) + bundleDir := writeDbtBundle(t) + writeTreeFile(t, bundleDir, cosmosBoostArtifact, staleOursSidecar) + t.Chdir(bundleDir) + uploaded := captureUploads(t) + + _, err := UploadBundle(t.TempDir(), ".", "http://upload-url", false, "0.0.0") + require.NoError(t, err) + + names := make([]string, 0, len(uploaded())) + for name := range uploaded() { + names = append(names, name) + } + assert.ElementsMatch(t, []string{"dbt_project.yml", cosmosBoostArtifact}, names) + assert.NotContains(t, uploaded()[cosmosBoostArtifact], `"stale"`) +} diff --git a/internal/shipcheck/shipcheck.go b/internal/shipcheck/shipcheck.go index 66fefe1bc..d3dbf5cab 100644 --- a/internal/shipcheck/shipcheck.go +++ b/internal/shipcheck/shipcheck.go @@ -119,6 +119,20 @@ func (r *Report) Err() error { return nil } +// CosmosBoostWarning is the warning for the Cosmos Boost files an earlier +// Astro CLI wrote into the project that a declared Dockerfile's context +// carries (imagebuild.ProjectSurvey.CosmosBoost), or "" when there are none. +// A generated image leaves them out; a declared Dockerfile decides for itself +// what it copies, so the build goes ahead, but the plugin would read any it +// copies as current. +func CosmosBoostWarning(survey *imagebuild.ProjectSurvey) string { + if len(survey.CosmosBoost) == 0 { + return "" + } + return fmt.Sprintf("the build context carries Cosmos Boost files an earlier Astro CLI wrote into the project: %s. The Cosmos Boost plugin would read them as current if the Dockerfile copies them into the image. This CLI no longer writes them; delete them", + cappedList(survey.CosmosBoost)) +} + // Survey is the check a command runs before it builds an image to push. It // surveys what the image built from the project at dir would carry // (imagebuild.SurveyProject; dockerfile is the manifest's [tool.astro] @@ -136,6 +150,9 @@ func Survey(dir, dockerfile string, podman bool, warn func(string)) (*imagebuild if w := scaffold.LocalFilesWarningFor(dockerfile, survey.IgnoreFile, slices.Concat(survey.Files, survey.VCS)); w != "" { warn(w) } + if w := CosmosBoostWarning(&survey); w != "" { + warn(w) + } r := Check(dir, &survey) for _, w := range r.Warnings() { warn(w) diff --git a/internal/shipcheck/shipcheck_test.go b/internal/shipcheck/shipcheck_test.go index 680385225..07ce2d55a 100644 --- a/internal/shipcheck/shipcheck_test.go +++ b/internal/shipcheck/shipcheck_test.go @@ -346,6 +346,31 @@ func TestSurvey(t *testing.T) { assert.Contains(t, fmt.Sprint(warned), "local or per-machine files: .git, .astro/standalone") } +// A declared Dockerfile's context carrying Cosmos Boost files an earlier CLI +// wrote is warned about, naming them, and the build goes ahead; a file from +// another producer, and a generated build, which leaves them out, are not. +func TestSurveyWarnsAboutStaleCosmosBoostFilesInADeclaredContext(t *testing.T) { + dir := t.TempDir() + writeFiles(t, dir, map[string]string{ + "Dockerfile": "FROM python:3.12\n", + "dbt/shop/.astro/dbt_metadata.json": `{"generated_by":{"application":"astro"}}`, + "dbt/shop/.astro/manifest.slim.json": `{"_generated_by":{"application":"astro"}}`, + "dbt/other/.astro/dbt_metadata.json": `{"generated_by":{"application":"someone-else"}}`, + }) + var warned []string + _, err := Survey(dir, "Dockerfile", false, func(w string) { warned = append(warned, w) }) + require.NoError(t, err) + all := fmt.Sprint(warned) + assert.Contains(t, all, "Cosmos Boost files an earlier Astro CLI wrote into the project: dbt/shop/.astro/dbt_metadata.json, dbt/shop/.astro/manifest.slim.json.") + assert.Contains(t, all, "delete them") + assert.NotContains(t, all, "dbt/other") + + warned = nil + _, err = Survey(dir, "", false, func(w string) { warned = append(warned, w) }) + require.NoError(t, err) + assert.NotContains(t, fmt.Sprint(warned), "Cosmos Boost") +} + func TestLooksSecret(t *testing.T) { for _, p := range []string{ "certs/server.pem", "tls.KEY", "client.p12", "client.pfx", "id_rsa", "id_rsa.pub", diff --git a/pkg/airflowrt/cosmosboost.go b/pkg/airflowrt/cosmosboost.go new file mode 100644 index 000000000..8e39cad7f --- /dev/null +++ b/pkg/airflowrt/cosmosboost.go @@ -0,0 +1,61 @@ +package airflowrt + +import ( + "encoding/json" + "path" +) + +// Earlier Astro CLIs wrote the Cosmos Boost sidecars into the user's dbt +// project: a .astro/dbt_metadata.json beside each dbt project or manifest, +// and a .astro/manifest.slim.json beside it. The Cosmos Boost plugin reads +// them as current, so one left behind is stale the moment the project +// changes. The CLI now generates them into what it ships and never into the +// project, and everything it ships - a bundle, a generated image - leaves out +// the ones an earlier CLI left there. These say which those are, here so the +// bundle (root module) and the generated image (pkg/imagebuild) use one rule. + +const ( + cosmosBoostDir = ".astro" + cosmosBoostSidecar = "dbt_metadata.json" + cosmosBoostSlimManifest = "manifest.slim.json" + // cosmosBoostProducer is the producer an Astro CLI records in each. + cosmosBoostProducer = "astro" +) + +// IsCosmosBoostFile reports whether rel, slash-separated, is where a Cosmos +// Boost sidecar goes: a dbt_metadata.json or manifest.slim.json directly +// inside a .astro directory, at any depth. +func IsCosmosBoostFile(rel string) bool { + switch path.Base(rel) { + case cosmosBoostSidecar, cosmosBoostSlimManifest: + return path.Base(path.Dir(rel)) == cosmosBoostDir + } + return false +} + +// CosmosBoostFileWrittenByAstro reports whether data, the contents of the +// file at rel (one IsCosmosBoostFile accepts), carries an Astro CLI's +// producer marker: generated_by.application in a dbt_metadata.json, +// _generated_by.application in a manifest.slim.json. Each file is judged by +// its own marker, never by its neighbor's, and one that does not parse or +// names another producer is not the CLI's: it is the user's. +func CosmosBoostFileWrittenByAstro(rel string, data []byte) bool { + var marker struct { + GeneratedBy struct { + Application string `json:"application"` + } `json:"generated_by"` + SlimGeneratedBy struct { + Application string `json:"application"` + } `json:"_generated_by"` + } + if json.Unmarshal(data, &marker) != nil { + return false + } + switch path.Base(rel) { + case cosmosBoostSidecar: + return marker.GeneratedBy.Application == cosmosBoostProducer + case cosmosBoostSlimManifest: + return marker.SlimGeneratedBy.Application == cosmosBoostProducer + } + return false +} diff --git a/pkg/airflowrt/cosmosboost_test.go b/pkg/airflowrt/cosmosboost_test.go new file mode 100644 index 000000000..fd94a5321 --- /dev/null +++ b/pkg/airflowrt/cosmosboost_test.go @@ -0,0 +1,46 @@ +package airflowrt + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsCosmosBoostFile(t *testing.T) { + for rel, want := range map[string]bool{ + ".astro/dbt_metadata.json": true, + ".astro/manifest.slim.json": true, + "dags/dbt/shop/target/.astro/dbt_metadata.json": true, + "dbt_metadata.json": false, + ".astro/config.yaml": false, + ".astro/nested/dbt_metadata.json": false, + "x.astro/dbt_metadata.json": false, + } { + assert.Equal(t, want, IsCosmosBoostFile(rel), rel) + } +} + +// Each file is judged by its own marker: a sidecar by generated_by, a slim +// manifest by _generated_by, and anything else is the user's. +func TestCosmosBoostFileWrittenByAstro(t *testing.T) { + const ( + sidecar = ".astro/dbt_metadata.json" + slim = ".astro/manifest.slim.json" + ) + for _, tc := range []struct { + rel, data string + want bool + }{ + {sidecar, `{"generated_by":{"application":"astro","version":"1.0.0"}}`, true}, + {sidecar, `{"generated_by":{"application":"someone-else"}}`, false}, + {sidecar, `{"schema":1,"version":{"hash":"abc"}}`, false}, + {sidecar, `{"_generated_by":{"application":"astro"}}`, false}, + {sidecar, `not json`, false}, + {slim, `{"_generated_by":{"application":"astro"}}`, true}, + {slim, `{"generated_by":{"application":"astro"}}`, false}, + {slim, `{"_generated_by":{"application":"someone-else"}}`, false}, + {".astro/config.yaml", `{"generated_by":{"application":"astro"}}`, false}, + } { + assert.Equal(t, tc.want, CosmosBoostFileWrittenByAstro(tc.rel, []byte(tc.data)), "%s %s", tc.rel, tc.data) + } +} diff --git a/pkg/cosmosboost/precompute/existing.go b/pkg/cosmosboost/precompute/existing.go deleted file mode 100644 index 18786343f..000000000 --- a/pkg/cosmosboost/precompute/existing.go +++ /dev/null @@ -1,92 +0,0 @@ -package precompute - -import ( - "encoding/json" - "fmt" - "io/fs" - "os" - "path/filepath" -) - -// existingSkipDirs are not traversed below the walked root. Discovery never -// produces artifacts under them (both discovery walks skip these names at any -// depth), and generated directories like a root-owned logs/ left by a -// container bind mount may not even be readable - failing a deploy over a -// directory no artifact is ever produced for helps nobody. The root itself is -// exempt, mirroring discovery: a project that happens to live in a directory -// named logs or dbt_packages can be stamped. .git is skipped unconditionally. -// target/ is deliberately absent: a compiled manifest's sidecar belongs at -// target/.astro/. -var existingSkipDirs = map[string]bool{ - "logs": true, - defaultPackagesDir: true, -} - -// Existing is a Cosmos Boost artifact already on disk under a walked root: a -// .astro/dbt_metadata.json or .astro/manifest.slim.json. Ours reports whether -// its own producer marker names this tool. Earlier astro-cli versions wrote -// these into the project; this one never does. -type Existing struct { - Path string - Ours bool -} - -// FindExisting returns every .astro/dbt_metadata.json and -// .astro/manifest.slim.json under root, each judged by its own producer -// marker, never by its neighbor's, because the mixed states are real: a slim -// manifest outlives its sidecar when an older astro-cli removed only the -// sidecar, and a sidecar this tool wrote can sit beside a slim manifest it did -// not. A file that cannot be read fails the scan, since nobody could say -// whose it is. -func FindExisting(root string) ([]Existing, error) { - var found []Existing - err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if d.Name() == gitDir || (path != root && existingSkipDirs[d.Name()]) { - return filepath.SkipDir - } - return nil - } - if filepath.Base(filepath.Dir(path)) != sidecarDir { - return nil - } - var ours func([]byte) bool - switch d.Name() { - case sidecarName: - ours = sidecarIsOurs - case slimManifestName: - ours = slimManifestIsOurs - default: - return nil - } - data, err := os.ReadFile(path) - if err != nil { - return err - } - found = append(found, Existing{Path: path, Ours: ours(data)}) - return nil - }) - if err != nil { - return nil, fmt.Errorf("scanning %q for Cosmos Boost files: %w", root, err) - } - return found, nil -} - -// sidecarIsOurs reports whether a .astro/dbt_metadata.json's generated_by -// names this tool. -func sidecarIsOurs(data []byte) bool { - var meta Metadata - return json.Unmarshal(data, &meta) == nil && meta.GeneratedBy.Application == application -} - -// slimManifestIsOurs reports whether a .astro/manifest.slim.json's own -// _generated_by marker names this tool. -func slimManifestIsOurs(data []byte) bool { - var marker struct { - GeneratedBy GeneratedBy `json:"_generated_by"` - } - return json.Unmarshal(data, &marker) == nil && marker.GeneratedBy.Application == application -} diff --git a/pkg/cosmosboost/precompute/existing_test.go b/pkg/cosmosboost/precompute/existing_test.go deleted file mode 100644 index ba12204b8..000000000 --- a/pkg/cosmosboost/precompute/existing_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package precompute - -import ( - "os" - "path/filepath" - "runtime" - "sort" - "testing" -) - -const ( - oursSidecar = `{"schema":1,"version":{"algo":"sha256-tree-v2","hash":"old"},"generated_by":{"application":"astro","version":"1.0.0"}}` - foreignSlim = `{"_schema":1,"_generated_by":{"application":"someone-else"}}` - oursSlim = `{"_schema":1,"_generated_by":{"application":"astro","version":"1.0.0"}}` - foreignNoTag = `{"schema":1,"version":{"hash":"abc"}}` -) - -// existingByRel is FindExisting's result keyed by root-relative slash path. -func existingByRel(t *testing.T, root string) map[string]bool { - t.Helper() - found, err := FindExisting(root) - if err != nil { - t.Fatal(err) - } - out := map[string]bool{} - for _, e := range found { - rel, err := filepath.Rel(root, e.Path) - if err != nil { - t.Fatal(err) - } - out[filepath.ToSlash(rel)] = e.Ours - } - return out -} - -// TestFindExistingJudgesEachFileByItsOwnMarker: provenance is read per file, -// never inferred from a neighbor, and a file with no marker is not ours. -func TestFindExistingJudgesEachFileByItsOwnMarker(t *testing.T) { - root := t.TempDir() - writeFiles(t, root, map[string]string{ - "proj/.astro/dbt_metadata.json": oursSidecar, - "proj/.astro/manifest.slim.json": foreignSlim, - "proj/target/.astro/dbt_metadata.json": foreignNoTag, - "proj/target/.astro/manifest.slim.json": oursSlim, - "orphan/.astro/manifest.slim.json": oursSlim, - "proj/.astro/config.yaml": "not ours to judge", - "proj/dbt_metadata.json": oursSidecar, // not under .astro - "proj/.astro/nested/dbt_metadata.json": oursSidecar, // not directly under .astro - }) - - got := existingByRel(t, root) - want := map[string]bool{ - "proj/.astro/dbt_metadata.json": true, - "proj/.astro/manifest.slim.json": false, - "proj/target/.astro/dbt_metadata.json": false, - "proj/target/.astro/manifest.slim.json": true, - "orphan/.astro/manifest.slim.json": true, - } - if len(got) != len(want) { - t.Fatalf("found %v, want %v", got, want) - } - for path, ours := range want { - if g, ok := got[path]; !ok || g != ours { - t.Fatalf("%s: found=%v ours=%v, want ours=%v (all: %v)", path, ok, g, ours, got) - } - } -} - -// TestFindExistingSkipsGitAndGeneratedDirs: .git is never walked, and logs/ -// and dbt_packages/ below the root are skipped even when unreadable, while -// target/ is visited and a root named like a skipped directory is still -// scanned. -func TestFindExistingSkipsGitAndGeneratedDirs(t *testing.T) { - parent := t.TempDir() - root := filepath.Join(parent, "logs") - writeFiles(t, root, map[string]string{ - ".git/x/.astro/dbt_metadata.json": oursSidecar, - "proj/logs/.astro/dbt_metadata.json": oursSidecar, - "proj/dbt_packages/.astro/dbt_metadata.json": oursSidecar, - "proj/target/.astro/dbt_metadata.json": oursSidecar, - ".astro/dbt_metadata.json": oursSidecar, - }) - if runtime.GOOS != "windows" { - locked := filepath.Join(root, "proj", "logs") - if err := os.Chmod(locked, 0o000); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) - } - - got := existingByRel(t, root) - var paths []string - for p := range got { - paths = append(paths, p) - } - sort.Strings(paths) - want := []string{".astro/dbt_metadata.json", "proj/target/.astro/dbt_metadata.json"} - if len(paths) != len(want) || paths[0] != want[0] || paths[1] != want[1] { - t.Fatalf("found %v, want %v", paths, want) - } -} - -// TestFindExistingFailsOnUnreadableArtifact: a file nobody can read cannot be -// attributed, so the scan fails rather than guessing. -func TestFindExistingFailsOnUnreadableArtifact(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("file permission semantics differ on windows") - } - root := t.TempDir() - writeFiles(t, root, map[string]string{"proj/.astro/dbt_metadata.json": oursSidecar}) - path := filepath.Join(root, "proj", ".astro", "dbt_metadata.json") - if err := os.Chmod(path, 0o000); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(path, 0o644) }) - - if _, err := FindExisting(root); err == nil { - t.Fatal("FindExisting over an unreadable sidecar: error = nil, want non-nil") - } -} - -func TestFindExistingNonexistentRoot(t *testing.T) { - if _, err := FindExisting(filepath.Join(t.TempDir(), "missing")); err == nil { - t.Fatal("FindExisting on a missing root: error = nil, want non-nil") - } -} diff --git a/pkg/cosmosboost/precompute/metadata.go b/pkg/cosmosboost/precompute/metadata.go index 0c6554fd9..0526a62d5 100644 --- a/pkg/cosmosboost/precompute/metadata.go +++ b/pkg/cosmosboost/precompute/metadata.go @@ -88,12 +88,20 @@ func sidecarArtifact(dir, algo, hash, version string, filtered *FilteredManifest return Artifact{Path: filepath.Join(dir, sidecarDir, sidecarName), Data: data} } -// checkSidecarDir fails when something other than a directory holds dir's -// .astro: an artifact under it could not be unpacked from the bundle. -func checkSidecarDir(dir string) error { - info, err := os.Lstat(filepath.Join(dir, sidecarDir)) - if err != nil || info.IsDir() { - return nil //nolint:nilerr // absent is fine: the bundle creates it +// checkArtifactPaths fails when the tree holds something the artifacts could +// not replace in a bundle: a .astro under dir that is not a directory (a file, +// or a link), or anything but a regular file where an artifact goes (a +// directory, a link). Unpacked beside the artifact, either would collide with +// it. A regular file there is fine: the bundle carries the artifact instead. +func checkArtifactPaths(dir string, artifacts []Artifact) error { + astroDir := filepath.Join(dir, sidecarDir) + if info, err := os.Lstat(astroDir); err == nil && !info.IsDir() { + return fmt.Errorf("%s is not a directory, so its Cosmos Boost files cannot sit under it", astroDir) } - return fmt.Errorf("%s is not a directory, so its Cosmos Boost files cannot sit under it", filepath.Join(dir, sidecarDir)) + for _, a := range artifacts { + if info, err := os.Lstat(a.Path); err == nil && !info.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file, so the Cosmos Boost file cannot take its place", a.Path) + } + } + return nil } diff --git a/pkg/cosmosboost/precompute/precompute.go b/pkg/cosmosboost/precompute/precompute.go index a5b829708..d2210eee6 100644 --- a/pkg/cosmosboost/precompute/precompute.go +++ b/pkg/cosmosboost/precompute/precompute.go @@ -143,9 +143,6 @@ func processProject(dir, version string, opts Options) Result { cfg := readDbtConfig(dir) hash, files, totalBytes, err := hashProject(dir, cfg) r := Result{Kind: kindProject, Path: dir, Hash: hash, Files: files, Bytes: totalBytes} - if err == nil { - err = checkSidecarDir(dir) - } if err != nil { r.Err = err r.Duration = time.Since(start) @@ -172,6 +169,9 @@ func processProject(dir, version string, opts Options) Result { } r.Artifacts = append(r.Artifacts, sidecarArtifact(dir, algoProjectTree, hash, version, filtered)) + if r.Err = checkArtifactPaths(dir, r.Artifacts); r.Err != nil { + r.Artifacts = nil + } r.Duration = time.Since(start) return r } @@ -202,9 +202,6 @@ func processManifest(path, version string, opts Options) Result { case !isDbt: r.Skipped = true default: - if r.Err = checkSidecarDir(dir); r.Err != nil { - break - } var filtered *FilteredManifest if slimData != nil { var slim Artifact @@ -212,6 +209,9 @@ func processManifest(path, version string, opts Options) Result { r.Artifacts = append(r.Artifacts, slim) } r.Artifacts = append(r.Artifacts, sidecarArtifact(dir, algoManifestJSON, hash, version, filtered)) + if r.Err = checkArtifactPaths(dir, r.Artifacts); r.Err != nil { + r.Artifacts = nil + } } r.Duration = time.Since(start) return r diff --git a/pkg/cosmosboost/precompute/precompute_test.go b/pkg/cosmosboost/precompute/precompute_test.go index 2484f3437..1c1603784 100644 --- a/pkg/cosmosboost/precompute/precompute_test.go +++ b/pkg/cosmosboost/precompute/precompute_test.go @@ -289,7 +289,7 @@ func TestRunProjectHashStableWithNestedManifestSidecar(t *testing.T) { first := projectHash() // Put the nested sidecar on disk, as a stale one from an earlier astro-cli // (or the unpacked bundle) would be. - writeFiles(t, root, map[string]string{"proj/docs/.astro/dbt_metadata.json": oursSidecar}) + writeFiles(t, root, map[string]string{"proj/docs/.astro/dbt_metadata.json": `{"generated_by":{"application":"astro"}}`}) second := projectHash() if first != second { t.Fatalf("project hash changed once the nested docs/.astro sidecar existed:\n first %s\n second %s", first, second) @@ -736,6 +736,29 @@ func TestRunFailsUnitWhenAstroIsAFile(t *testing.T) { } } +// TestRunFailsUnitWhenArtifactPathIsNotAFile: a directory where a sidecar +// goes would collide with it in the bundle, so that unit fails and produces +// nothing, while a regular file there (an old sidecar) is fine. +func TestRunFailsUnitWhenArtifactPathIsNotAFile(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "blocked/dbt_project.yml": "name: blocked\n", + "blocked/.astro/dbt_metadata.json/inside.txt": "x", + "fine/dbt_project.yml": "name: fine\n", + "fine/.astro/dbt_metadata.json": "old", + }) + + summary, err := Run([]string{root}, "test", Options{SlimManifest: true}) + if err != nil { + t.Fatal(err) + } + if summary.CountFailed() != 1 { + t.Fatalf("want one failed unit, got %+v", summary.Results) + } + mustNotProduce(t, summary, filepath.Join(root, "blocked", sidecarDir, sidecarName)) + mustProduce(t, summary, filepath.Join(root, "fine", sidecarDir, sidecarName)) +} + // treeSnapshot maps every file under root to its contents. func treeSnapshot(t *testing.T, root string) map[string]string { t.Helper() diff --git a/pkg/cosmosboost/predeploy.go b/pkg/cosmosboost/predeploy.go index 441821335..e3010a94c 100644 --- a/pkg/cosmosboost/predeploy.go +++ b/pkg/cosmosboost/predeploy.go @@ -2,12 +2,14 @@ package cosmosboost import ( "bytes" + "errors" "fmt" "io" "os" "path/filepath" "strings" + "github.com/astronomer/astro-cli/pkg/airflowrt" "github.com/astronomer/astro-cli/pkg/cosmosboost/precompute" "github.com/astronomer/astro-cli/pkg/fileutil" "github.com/astronomer/astro-cli/pkg/logger" @@ -30,71 +32,93 @@ func slimManifestEnabled() bool { return value == "" || util.CheckEnvBool(value) } -// BundleOverlay is what a deploy bundle of root must change for Cosmos Boost, -// as an overlay on the tarball: root itself is never written. +// Bundle is what a deploy bundle of a directory changes for Cosmos Boost, as +// an overlay on the tarball (Overlay): the directory itself is never written. // -// Whatever enabled says, every .astro/dbt_metadata.json and -// .astro/manifest.slim.json under root that an earlier astro-cli wrote into -// the project (its producer marker names this tool) is left out. The plugin -// cannot tell a fresh sidecar from a stale one, and this one never writes -// there, so nothing a deploy ships comes from the tree. +// Whatever the setting says, every .astro/dbt_metadata.json and +// .astro/manifest.slim.json that an earlier Astro CLI wrote into the tree (its +// own producer marker says so, airflowrt.CosmosBoostFileWrittenByAstro) is +// left out, judged inside the tarball's own walk so the files left out are +// exactly the ones it would have carried. The plugin cannot tell a fresh +// sidecar from a stale one, and this CLI never writes there. // -// With enabled set (cosmos_boost.pre_deploy), the pre-deploy step then runs -// over root and its sidecars are added to the bundle at the same paths, with -// the same contents, the plugin has always read: every dbt project -// (dbt_project.yml) gets a .astro/dbt_metadata.json carrying its content hash, -// which the plugin uses as a cache version key at parse time instead of -// hashing the project tree itself, and every standalone dbt manifest.json gets -// a hash sidecar too, plus a slim, field-filtered copy for the plugin to load -// in place of the full manifest. That step is best-effort - a failure is a -// warning, never an error - because without a sidecar the plugin simply falls -// back to hashing at parse time. +// With the setting on (cosmos_boost.pre_deploy), the pre-deploy step runs over +// the directory and its sidecars are added at the same paths, with the same +// contents, the plugin has always read: every dbt project (dbt_project.yml) +// gets a .astro/dbt_metadata.json carrying its content hash, which the plugin +// uses as a cache version key at parse time instead of hashing the project +// tree itself, and every standalone dbt manifest.json gets a hash sidecar too, +// plus a slim, field-filtered copy for the plugin to load in place of the full +// manifest. That step is best-effort - a failure is a warning, never an error +// - because without a sidecar the plugin simply falls back to hashing at parse +// time. // // A file from an unrecognized producer is the user's, and ships untouched -// when the step is off. With it on, the deploy fails instead: the plugin would -// consume that file's version.hash all the same, and this tool will not judge +// when the step is off. With it on, Err refuses the deploy: the plugin would +// consume that file's version.hash all the same, and this CLI will not judge // a file it did not write fresh. -func BundleOverlay(root string, enabled bool) (fileutil.TarOverlay, error) { - overlay := fileutil.TarOverlay{Skip: map[string]bool{}, Extra: map[string][]byte{}} +type Bundle struct { + Overlay fileutil.TarOverlay + enabled bool + foreign []string +} - existing, err := precompute.FindExisting(root) - if err != nil { - return overlay, err - } - var foreign []string - for _, e := range existing { - if !e.Ours { - foreign = append(foreign, e.Path) - continue - } - rel, err := relSlash(root, e.Path) - if err != nil { - return overlay, err - } - logger.Debugf("leaving out a Cosmos Boost file an earlier Astro CLI wrote into the project: %s", e.Path) - overlay.Skip[rel] = true - } +// NewBundle prepares the overlay for a bundle of root, generating the +// sidecars into it when enabled. A root that cannot be read generates none, +// and is left for the tarball to deal with as it always has. +func NewBundle(root string, enabled bool) *Bundle { + b := &Bundle{enabled: enabled} + b.Overlay = fileutil.TarOverlay{Skip: b.skip, Extra: map[string][]byte{}} if !enabled { - return overlay, nil + return b } - if len(foreign) > 0 { - return overlay, fmt.Errorf("found Cosmos Boost file(s) from an unrecognized producer: %s; the deploy cannot prove they are fresh and will not ship them for the Cosmos Boost plugin to read. Remove them and retry", strings.Join(foreign, ", ")) + if _, err := os.Stat(root); err != nil { + return b } - artifacts, err := preDeploy(root) for _, a := range artifacts { - rel, relErr := relSlash(root, a.Path) + rel, relErr := filepath.Rel(root, a.Path) if relErr != nil { - return overlay, relErr + err = errors.Join(err, relErr) + continue } - overlay.Extra[rel] = a.Data + b.Overlay.Extra[filepath.ToSlash(rel)] = a.Data } if err != nil { fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) - return overlay, nil + return b } fmt.Println("Cosmos Boost pre-deploy step complete") - return overlay, nil + return b +} + +// skip is Overlay.Skip: it leaves out each Cosmos Boost file an Astro CLI +// wrote into the tree, and notes every other one for Err. Only files at a +// sidecar's path are read. +func (b *Bundle) skip(rel, file string) (bool, error) { + if !airflowrt.IsCosmosBoostFile(rel) { + return false, nil + } + data, err := os.ReadFile(file) + if err != nil { + return false, err + } + if airflowrt.CosmosBoostFileWrittenByAstro(rel, data) { + logger.Debugf("leaving out a Cosmos Boost file an earlier Astro CLI wrote into the project: %s", file) + return true, nil + } + b.foreign = append(b.foreign, file) + return false, nil +} + +// Err, once the tarball is written, refuses an enabled deploy that found a +// Cosmos Boost file from an unrecognized producer, which the plugin would read +// as current. nil otherwise. +func (b *Bundle) Err() error { + if !b.enabled || len(b.foreign) == 0 { + return nil + } + return fmt.Errorf("found Cosmos Boost file(s) from an unrecognized producer: %s; the deploy cannot prove they are fresh and will not ship them for the Cosmos Boost plugin to read. Remove them and retry", strings.Join(b.foreign, ", ")) } // preDeploy runs the Cosmos Boost pre-deploy step over path and returns the @@ -113,15 +137,6 @@ func preDeploy(path string) ([]precompute.Artifact, error) { return summary.Artifacts(), nil } -// relSlash is path relative to root, slash-separated, as TarOverlay keys it. -func relSlash(root, path string) (string, error) { - rel, err := filepath.Rel(root, path) - if err != nil { - return "", err - } - return filepath.ToSlash(rel), nil -} - // reporter is the shape of the precompute summary. type reporter interface{ WriteReport(io.Writer) } diff --git a/pkg/cosmosboost/predeploy_test.go b/pkg/cosmosboost/predeploy_test.go index dd73ebe48..d2e71ba97 100644 --- a/pkg/cosmosboost/predeploy_test.go +++ b/pkg/cosmosboost/predeploy_test.go @@ -1,7 +1,10 @@ package cosmosboost import ( + "archive/tar" "encoding/json" + "errors" + "io" "io/fs" "os" "path/filepath" @@ -10,13 +13,17 @@ import ( "github.com/stretchr/testify/require" + "github.com/astronomer/astro-cli/pkg/airflowrt" "github.com/astronomer/astro-cli/pkg/cosmosboost/precompute" + "github.com/astronomer/astro-cli/pkg/fileutil" "github.com/astronomer/astro-cli/version" ) const ( sidecarRel = ".astro/dbt_metadata.json" + slimRel = ".astro/manifest.slim.json" oursSidecar = `{"schema":1,"version":{"algo":"sha256-tree-v2","hash":"stale"},"generated_by":{"application":"astro","version":"1.0.0"}}` + oursSlim = `{"_schema":1,"_generated_by":{"application":"astro","version":"1.0.0"}}` foreignFile = `{"schema":1,"version":{"hash":"abc"},"generated_by":{"application":"someone-else"}}` dbtManifest = `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}` ) @@ -36,18 +43,21 @@ func writeDbtProject(t *testing.T, dir string) { } // snapshot maps every file under root to its contents, to prove a call left -// the tree exactly as it found it. +// the tree exactly as it found it. A root that is not there is empty. func snapshot(t *testing.T, root string) map[string]string { t.Helper() out := map[string]string{} - require.NoError(t, filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { return err } data, err := os.ReadFile(path) out[path] = string(data) return err - })) + }) + if !errors.Is(err, fs.ErrNotExist) { + require.NoError(t, err) + } return out } @@ -58,22 +68,48 @@ func setVersion(t *testing.T, v string) { t.Cleanup(func() { version.CurrVersion = orig }) } -// TestBundleOverlayEnabledAddsSidecars pins the plugin contract: the same -// files at the same paths with the same contents earlier versions wrote into -// the project, now only in the bundle - the tree is untouched. -func TestBundleOverlayEnabledAddsSidecars(t *testing.T) { +// bundle tars dir as a deploy does, through NewBundle's overlay, and returns +// the tarball's entries by name, and the error the deploy would stop on. +func bundle(t *testing.T, dir string, enabled bool) (map[string]string, error) { + t.Helper() + b := NewBundle(dir, enabled) + target := filepath.Join(t.TempDir(), "bundle.tar") + if err := fileutil.Tar(dir, target, false, []string{".git/"}, b.Overlay); err != nil { + return nil, err + } + file, err := os.Open(target) + require.NoError(t, err) + defer file.Close() + entries := map[string]string{} + tr := tar.NewReader(file) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + data, err := io.ReadAll(tr) + require.NoError(t, err) + entries[header.Name] = string(data) + } + return entries, b.Err() +} + +// TestBundleEnabledAddsSidecars pins the plugin contract: the same files at +// the same paths with the same contents earlier versions wrote into the +// project, now only in the bundle - the tree is untouched. +func TestBundleEnabledAddsSidecars(t *testing.T) { setVersion(t, "1.2.3") dir := t.TempDir() writeDbtProject(t, dir) writeFiles(t, dir, map[string]string{"target/manifest.json": dbtManifest}) before := snapshot(t, dir) - overlay, err := BundleOverlay(dir, true) + entries, err := bundle(t, dir, true) require.NoError(t, err) require.Equal(t, before, snapshot(t, dir), "the tree must not be written") - require.Empty(t, overlay.Skip) - require.ElementsMatch(t, []string{sidecarRel, "target/" + sidecarRel, "target/.astro/manifest.slim.json"}, keys(overlay.Extra)) + require.ElementsMatch(t, []string{"dbt_project.yml", "models/a.sql", "target/manifest.json", sidecarRel, "target/" + sidecarRel, "target/" + slimRel}, keys(entries)) // The contents are exactly what the pre-deploy step produces for the tree. summary, err := precompute.Run([]string{dir}, "1.2.3", precompute.Options{SlimManifest: true}) @@ -81,7 +117,9 @@ func TestBundleOverlayEnabledAddsSidecars(t *testing.T) { for _, a := range summary.Artifacts() { rel, err := filepath.Rel(dir, a.Path) require.NoError(t, err) - require.Equal(t, string(a.Data), string(overlay.Extra[filepath.ToSlash(rel)]), rel) + require.Equal(t, string(a.Data), entries[filepath.ToSlash(rel)], rel) + // What it generates is what every later deploy recognizes as its own. + require.True(t, airflowrt.CosmosBoostFileWrittenByAstro(filepath.ToSlash(rel), a.Data), rel) } var meta struct { @@ -95,7 +133,7 @@ func TestBundleOverlayEnabledAddsSidecars(t *testing.T) { Version string `json:"version"` } `json:"generated_by"` } - require.NoError(t, json.Unmarshal(overlay.Extra[sidecarRel], &meta)) + require.NoError(t, json.Unmarshal([]byte(entries[sidecarRel]), &meta)) require.Equal(t, 1, meta.Schema, "schema is the plugin's compatibility gate and must stay 1") require.Equal(t, "sha256-tree-v2", meta.Version.Algo) for _, r := range summary.Results { @@ -107,16 +145,16 @@ func TestBundleOverlayEnabledAddsSidecars(t *testing.T) { require.Equal(t, "1.2.3", meta.GeneratedBy.Version) } -// TestBundleOverlayRespectsSlimManifestEnvVar: the env var disables this one +// TestBundleRespectsSlimManifestEnvVar: the env var disables this one // optimization without disabling the step, so the hash sidecar still ships. -func TestBundleOverlayRespectsSlimManifestEnvVar(t *testing.T) { +func TestBundleRespectsSlimManifestEnvVar(t *testing.T) { dir := t.TempDir() writeFiles(t, dir, map[string]string{"manifest.json": dbtManifest}) t.Setenv(slimManifestEnvVar, "false") - overlay, err := BundleOverlay(dir, true) + entries, err := bundle(t, dir, true) require.NoError(t, err) - require.Equal(t, []string{sidecarRel}, keys(overlay.Extra)) + require.ElementsMatch(t, []string{"manifest.json", sidecarRel}, keys(entries)) } // TestSlimManifestEnabled: unset leaves the optimization on; otherwise only a @@ -140,93 +178,106 @@ func TestSlimManifestEnabled(t *testing.T) { } } -// TestBundleOverlayLeavesOutStaleSidecars: whether the step is on or off, a -// sidecar an earlier astro-cli wrote into the project never ships - including -// one whose project no longer exists - and the tree keeps it. -func TestBundleOverlayLeavesOutStaleSidecars(t *testing.T) { +// TestBundleLeavesOutStaleSidecars: whether the step is on or off, a sidecar +// an earlier CLI wrote into the project never ships - including one whose +// project no longer exists, and one under logs/, which discovery never visits +// but the tarball would carry - and the tree keeps it. +func TestBundleLeavesOutStaleSidecars(t *testing.T) { for _, enabled := range []bool{true, false} { dir := t.TempDir() writeDbtProject(t, dir) writeFiles(t, dir, map[string]string{ - sidecarRel: oursSidecar, - "removed-project/" + sidecarRel: oursSidecar, - "removed-project/.astro/manifest.slim.json": `{"_generated_by":{"application":"astro"}}`, + sidecarRel: oursSidecar, + "removed-project/" + sidecarRel: oursSidecar, + "removed-project/" + slimRel: oursSlim, + "logs/" + sidecarRel: oursSidecar, }) before := snapshot(t, dir) - overlay, err := BundleOverlay(dir, enabled) + entries, err := bundle(t, dir, enabled) require.NoError(t, err) require.Equal(t, before, snapshot(t, dir), "enabled=%v: the tree must not be written", enabled) - require.ElementsMatch(t, []string{sidecarRel, "removed-project/" + sidecarRel, "removed-project/.astro/manifest.slim.json"}, keys(overlay.Skip), "enabled=%v", enabled) + for _, gone := range []string{"removed-project/" + sidecarRel, "removed-project/" + slimRel, "logs/" + sidecarRel} { + require.NotContains(t, entries, gone, "enabled=%v", enabled) + } if enabled { - require.Equal(t, []string{sidecarRel}, keys(overlay.Extra), "the current project gets a fresh sidecar") - require.NotContains(t, string(overlay.Extra[sidecarRel]), "stale") + require.Contains(t, entries, sidecarRel, "the current project gets a fresh sidecar") + require.NotContains(t, entries[sidecarRel], "stale") } else { - require.Empty(t, overlay.Extra, "a disabled deploy adds nothing") + require.NotContains(t, entries, sidecarRel, "a disabled deploy ships no sidecar, stale or fresh") } } } -// TestBundleOverlayForeignSidecar: a file from an unrecognized producer is -// the user's. Disabled, it ships untouched; enabled, the deploy stops and asks -// for its removal, because the plugin would consume its version.hash. -func TestBundleOverlayForeignSidecar(t *testing.T) { - dir := t.TempDir() - writeDbtProject(t, dir) - writeFiles(t, dir, map[string]string{"other/" + sidecarRel: foreignFile}) - foreign := filepath.Join(dir, "other", ".astro", "dbt_metadata.json") +// TestBundleForeignSidecar: a file from an unrecognized producer is the +// user's. Disabled, it ships untouched; enabled, the deploy stops and asks for +// its removal, because the plugin would consume its version.hash - even when +// it sits where a fresh sidecar would replace it. +func TestBundleForeignSidecar(t *testing.T) { + for _, rel := range []string{"other/" + sidecarRel, sidecarRel} { + dir := t.TempDir() + writeDbtProject(t, dir) + writeFiles(t, dir, map[string]string{rel: foreignFile}) + foreign := filepath.Join(dir, filepath.FromSlash(rel)) - overlay, err := BundleOverlay(dir, false) - require.NoError(t, err) - require.Empty(t, overlay.Skip, "disabled: a foreign file ships") - require.Empty(t, overlay.Extra) - - _, err = BundleOverlay(dir, true) - require.Error(t, err) - require.ErrorContains(t, err, "unrecognized producer") - require.ErrorContains(t, err, foreign) - require.ErrorContains(t, err, "Remove them and retry") - require.FileExists(t, foreign, "a file another tool wrote must never be deleted") + entries, err := bundle(t, dir, false) + require.NoError(t, err) + require.Equal(t, foreignFile, entries[rel], "disabled: a foreign file ships") + + _, err = bundle(t, dir, true) + require.Error(t, err, rel) + require.ErrorContains(t, err, "unrecognized producer") + require.ErrorContains(t, err, foreign) + require.ErrorContains(t, err, "Remove them and retry") + require.FileExists(t, foreign, "a file another tool wrote must never be deleted") + } } -func TestBundleOverlayNoDbtContentIsEmpty(t *testing.T) { +func TestBundleNoDbtContentAddsNothing(t *testing.T) { dir := t.TempDir() writeFiles(t, dir, map[string]string{"app.py": "print('hi')"}) - overlay, err := BundleOverlay(dir, true) + entries, err := bundle(t, dir, true) require.NoError(t, err) - require.Empty(t, overlay.Skip) - require.Empty(t, overlay.Extra) + require.Equal(t, []string{"app.py"}, keys(entries)) } -// TestBundleOverlayMissingRoot: a root that is not there cannot be checked -// for stale files, so the deploy fails rather than guessing. -func TestBundleOverlayMissingRoot(t *testing.T) { - _, err := BundleOverlay(filepath.Join(t.TempDir(), "does-not-exist"), false) - require.Error(t, err) +// TestBundleMissingRoot: a directory that is not there (a project with no +// dags/) makes the empty tarball it always did, with the step on or off, and +// nothing is created. +func TestBundleMissingRoot(t *testing.T) { + for _, enabled := range []bool{true, false} { + dir := filepath.Join(t.TempDir(), "dags") + entries, err := bundle(t, dir, enabled) + require.NoError(t, err, "enabled=%v", enabled) + require.Empty(t, entries) + require.NoDirExists(t, dir) + } } -// TestBundleOverlayStepFailureIsAWarning: stamping is best-effort - a unit -// that cannot be hashed ships no sidecar, the others still do, and the -// deploy goes on. -func TestBundleOverlayStepFailureIsAWarning(t *testing.T) { +// TestBundleStepFailureIsAWarning: stamping is best-effort - a unit that +// cannot be hashed ships no sidecar, the others still do, and the deploy +// goes on. +func TestBundleStepFailureIsAWarning(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("file permission semantics differ on windows") } dir := t.TempDir() writeDbtProject(t, filepath.Join(dir, "broken")) writeDbtProject(t, filepath.Join(dir, "fine")) + writeFiles(t, dir, map[string]string{"blocked/dbt_project.yml": "name: blocked\n", "blocked/.astro/dbt_metadata.json/x": "x"}) model := filepath.Join(dir, "broken", "models", "a.sql") require.NoError(t, os.Chmod(model, 0o000)) t.Cleanup(func() { _ = os.Chmod(model, 0o644) }) - overlay, err := BundleOverlay(dir, true) - require.NoError(t, err) - require.Equal(t, []string{"fine/" + sidecarRel}, keys(overlay.Extra)) + b := NewBundle(dir, true) + require.NoError(t, b.Err()) + require.Equal(t, []string{"fine/" + sidecarRel}, keys(b.Overlay.Extra), + "neither the unreadable project nor the one with a directory where its sidecar goes gets one") - _, err = preDeploy(dir) - require.ErrorContains(t, err, "failed for 1 unit(s)") + _, err := preDeploy(dir) + require.ErrorContains(t, err, "failed for 2 unit(s)") } func keys[V any](m map[string]V) []string { diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index 005a47d20..546c15ac8 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -106,12 +106,16 @@ func WriteToFile(filePath string, r io.Reader) error { return err } -// TarOverlay changes what Tar writes without touching source. Both maps are -// keyed by slash-separated paths relative to source (before any base -// directory is prepended): Skip leaves those files out, and Extra adds a -// regular file with those bytes, in place of any file at the same path. +// TarOverlay changes what Tar writes without touching source. Paths are +// slash-separated and relative to source, before any base directory is +// prepended. type TarOverlay struct { - Skip map[string]bool + // Skip, when set, is asked about each regular file the walk would add, by + // rel and by its path on disk, and leaves it out on true. An error fails + // the tarball. + Skip func(rel, file string) (bool, error) + // Extra adds a regular file with these bytes at each path, in place of + // anything the walk finds there. Extra map[string][]byte } @@ -204,9 +208,9 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr return err } - headerName, ok := tarEntryName(path, source, baseDir, excludePathPrefixes, overlay) - if !ok { - return nil + headerName, ok, err := tarEntryName(path, source, baseDir, excludePathPrefixes, overlay, info.Mode().IsRegular()) + if err != nil || !ok { + return err } header.Name = headerName logger.Debugf("Adding to tarball: %s", header.Name) @@ -233,16 +237,15 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr return addExtrasToTar(tarball, baseDir, overlay.Extra) } -// tarEntryName is the name file goes into the tarball under, or -// false when it stays out: one the overlay replaces or skips, or one under an -// excluded prefix. -func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, overlay TarOverlay) (string, bool) { - // set the tar file path to be relative to the source directory - relName := strings.TrimPrefix(file, filepath.Clean(source)) - relName = strings.TrimPrefix(relName, string(filepath.Separator)) - if rel := filepath.ToSlash(relName); overlay.Skip[rel] || overlay.Extra[rel] != nil { - logger.Debugf("Leaving out of the tarball: %s", rel) - return "", false +// tarEntryName is the name file goes into the tarball under, or false when +// it stays out: one under an excluded prefix, or one the overlay skips or +// replaces. regular says file is a regular file, the only kind Skip is asked +// about. +func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, overlay TarOverlay, regular bool) (name string, ok bool, err error) { + // the tar file path is relative to the source directory, which may be "." + relName, err := filepath.Rel(source, file) + if err != nil { + return "", false, err } // prepend the base directory to the tar file path, e.g. "dags/" to "my_dag.py", // with forward slashes whatever the platform @@ -252,10 +255,26 @@ func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, ov for _, excludePathPrefix := range excludePathPrefixes { if strings.HasPrefix(headerName, excludePathPrefix) { logger.Debugf("Excluding tarball path: %s", headerName) - return "", false + return "", false, nil } } - return headerName, true + + rel := filepath.ToSlash(relName) + if regular && overlay.Skip != nil { + skip, err := overlay.Skip(rel, file) + if err != nil { + return "", false, err + } + if skip { + logger.Debugf("Leaving out of the tarball: %s", rel) + return "", false, nil + } + } + if overlay.Extra[rel] != nil { + logger.Debugf("Replacing in the tarball: %s", rel) + return "", false, nil + } + return headerName, true, nil } // addExtrasToTar writes each of extra into tarball under baseDir, in path diff --git a/pkg/fileutil/files_test.go b/pkg/fileutil/files_test.go index 0acd37fbe..d7b285619 100644 --- a/pkg/fileutil/files_test.go +++ b/pkg/fileutil/files_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sort" "strings" "testing" @@ -304,8 +305,12 @@ func (s *Suite) TestTarOverlay() { s.Require().NoError(os.WriteFile(filepath.Join(source, "a.sql"), []byte("a"), 0o600)) s.Require().NoError(os.WriteFile(filepath.Join(source, ".astro", "stale.json"), []byte("stale"), 0o600)) s.Require().NoError(os.WriteFile(filepath.Join(source, ".astro", "replaced.json"), []byte("old"), 0o600)) + var asked []string overlay := TarOverlay{ - Skip: map[string]bool{".astro/stale.json": true}, + Skip: func(rel, file string) (bool, error) { + asked = append(asked, rel) + return rel == ".astro/stale.json", nil + }, Extra: map[string][]byte{ ".astro/replaced.json": []byte("new"), "sub/.astro/added.json": []byte("added"), @@ -344,6 +349,24 @@ func (s *Suite) TestTarOverlay() { }, got) } + s.ElementsMatch([]string{ + "a.sql", ".astro/stale.json", ".astro/replaced.json", + "a.sql", ".astro/stale.json", ".astro/replaced.json", + }, asked, "Skip is asked about every regular file, once per tarball") + + s.Run("source is the working directory", func() { + s.T().Chdir(source) + target := filepath.Join(dir, "dot.tar") + s.Require().NoError(Tar(".", target, false, nil, overlay)) + s.Equal([]string{".astro/replaced.json", "a.sql", "sub/.astro/added.json"}, tarNames(s.T(), target)) + }) + + s.Run("a Skip error fails the tarball", func() { + boom := errors.New("boom") + err := Tar(source, filepath.Join(dir, "boom.tar"), false, nil, TarOverlay{Skip: func(string, string) (bool, error) { return false, boom }}) + s.ErrorIs(err, boom) + }) + stale, err := os.ReadFile(filepath.Join(source, ".astro", "replaced.json")) s.Require().NoError(err) s.Equal("old", string(stale), "the source is never written") @@ -961,3 +984,23 @@ func (s *Suite) TestDefaultPermissions() { s.Equal(os.FileMode(0o755), perm) s.Equal(0o755, openPermissions) } + +// tarNames lists the entries of the tarball at target, sorted. +func tarNames(t *testing.T, target string) []string { + t.Helper() + file, err := os.Open(target) + require.NoError(t, err) + defer file.Close() + var names []string + tr := tar.NewReader(file) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + names = append(names, header.Name) + } + sort.Strings(names) + return names +} diff --git a/pkg/imagebuild/projectfiles.go b/pkg/imagebuild/projectfiles.go index e9e92d7ee..b759b1d83 100644 --- a/pkg/imagebuild/projectfiles.go +++ b/pkg/imagebuild/projectfiles.go @@ -185,6 +185,10 @@ type walker struct { // then neither entered nor visited. A generated build's filter leaves all // of it out. vcs func(rel string) + // cosmosBoost, when set, takes each Cosmos Boost file an earlier Astro CLI + // wrote into the project (airflowrt.CosmosBoostFileWrittenByAstro) that + // the walk carries. A generated build's filter leaves them out instead. + cosmosBoost func(rel string) // link, when set, takes each symlink a following walk goes through: the // link's own path, for a file carried as what it points at or a directory // entered through it. @@ -266,7 +270,7 @@ func (w walker) walkDir(dir, rel string, onPath map[string]bool) error { case info.Mode().IsRegular(), info.Mode()&fs.ModeSymlink != 0: // A symlink gets here only when links are not followed: it is // carried as itself. - if err := w.visit(childRel, src, info.Mode()); err != nil && !errors.Is(err, fs.ErrNotExist) { + if err := w.carry(childRel, src, info.Mode()); err != nil { return err } } @@ -275,6 +279,48 @@ func (w walker) walkDir(dir, rel string, onPath map[string]bool) error { return nil } +// carry visits a file the walk carries, unless it is a Cosmos Boost file an +// earlier Astro CLI wrote (staleCosmosBoost), which a generated build leaves +// out and a declared one reports. +func (w walker) carry(rel, src string, mode fs.FileMode) error { + if mode.IsRegular() { + stale, err := w.staleCosmosBoost(rel, src) + if err != nil { + return err + } + if stale && w.filter.generated { + return nil + } + if stale && w.cosmosBoost != nil { + w.cosmosBoost(rel) + } + } + if err := w.visit(rel, src, mode); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +// staleCosmosBoost reports whether the regular file at src is a Cosmos Boost +// sidecar an earlier Astro CLI wrote into the project, which the plugin would +// read as current in the image (airflowrt.CosmosBoostFileWrittenByAstro). The +// CLI no longer writes them; a deploy's bundle leaves them out, and so does a +// generated image. Only a file at a sidecar's path is read, and only when the +// answer is used. +func (w walker) staleCosmosBoost(rel, src string) (bool, error) { + if (!w.filter.generated && w.cosmosBoost == nil) || !airflowrt.IsCosmosBoostFile(rel) { + return false, nil + } + data, err := os.ReadFile(src) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + return airflowrt.CosmosBoostFileWrittenByAstro(rel, data), nil +} + // enter walks the directory at src, unless it is one already being walked // (a link back up the tree). func (w walker) enter(src, rel string, onPath map[string]bool) error { @@ -306,6 +352,11 @@ type ProjectSurvey struct { // Dockerfile's context carries, which its ignore file leaves in: each // one's path, not what is in it. A generated build carries none. VCS []string + // CosmosBoost are the Cosmos Boost files an earlier Astro CLI wrote into + // the project that a declared Dockerfile's context carries, which the + // plugin would read as current in the image. A generated build leaves + // them out and lists none. + CosmosBoost []string // IgnoreFile is the ignore file a declared Dockerfile's build reads // (declaredIgnoreFile), "" for none or for a generated build. IgnoreFile string @@ -349,7 +400,12 @@ func SurveyProject(dir, dockerfile string, podman bool) (ProjectSurvey, error) { if filter, err = newDeclaredFilter(resolved, s.IgnoreFile); err != nil { return ProjectSurvey{}, err } - w := walker{filter: filter, vcs: func(rel string) { s.VCS = append(s.VCS, rel) }, visit: collect} + w := walker{ + filter: filter, + vcs: func(rel string) { s.VCS = append(s.VCS, rel) }, + cosmosBoost: func(rel string) { s.CosmosBoost = append(s.CosmosBoost, rel) }, + visit: collect, + } err = w.walkDir(resolved, "", map[string]bool{resolved: true}) } if err != nil { diff --git a/pkg/imagebuild/projectsurvey_test.go b/pkg/imagebuild/projectsurvey_test.go index fbd7e5ba8..f73ee896b 100644 --- a/pkg/imagebuild/projectsurvey_test.go +++ b/pkg/imagebuild/projectsurvey_test.go @@ -36,6 +36,54 @@ func TestSurveyProjectGenerated(t *testing.T) { assert.ElementsMatch(t, s.Files, contextFiles(t, dst), "the survey and the copy agree") } +// A generated image leaves out the Cosmos Boost files an earlier Astro CLI +// wrote into the project, at any depth and dags/ included, judged by each +// file's own marker: the plugin would read them as current. One from another +// producer is the user's and is carried. The survey, the copy and the digest +// agree. +func TestGeneratedImageLeavesOutStaleCosmosBoostFiles(t *testing.T) { + project := t.TempDir() + writeTree(t, project, map[string]string{ + "dags/dbt/shop/dbt_project.yml": "name: shop\n", + "dags/dbt/shop/.astro/dbt_metadata.json": `{"generated_by":{"application":"astro"}}`, + "dags/dbt/shop/target/.astro/manifest.slim.json": `{"_generated_by":{"application":"astro"}}`, + "include/dbt/.astro/dbt_metadata.json": `{"generated_by":{"application":"someone-else"}}`, + "include/dbt/.astro/notes.json": `{"generated_by":{"application":"astro"}}`, + }) + want := []string{"dags/dbt/shop/dbt_project.yml", "include/dbt/.astro/dbt_metadata.json", "include/dbt/.astro/notes.json"} + + s, err := SurveyProject(project, "", false) + require.NoError(t, err) + assert.ElementsMatch(t, want, s.Files) + assert.Empty(t, s.CosmosBoost, "a generated build lists none: it carries none") + + dst := t.TempDir() + require.NoError(t, copyProjectFiles(project, dst)) + assert.ElementsMatch(t, want, contextFiles(t, dst)) + + before, err := ProjectFilesDigest(project) + require.NoError(t, err) + writeTree(t, project, map[string]string{"dags/dbt/shop/.astro/dbt_metadata.json": `{"generated_by":{"application":"astro"},"version":{"hash":"changed"}}`}) + after, err := ProjectFilesDigest(project) + require.NoError(t, err) + assert.Equal(t, before, after, "a file the image leaves out does not move its tag") +} + +// A declared Dockerfile's context carries them, and the survey names them for +// the warning. +func TestSurveyProjectDeclaredNamesStaleCosmosBoostFiles(t *testing.T) { + project := t.TempDir() + writeTree(t, project, map[string]string{ + "Dockerfile": "FROM x\n", + "dbt/.astro/dbt_metadata.json": `{"generated_by":{"application":"astro"}}`, + "dbt/other/.astro/dbt_metadata.json": `{"generated_by":{"application":"someone-else"}}`, + }) + s, err := SurveyProject(project, "Dockerfile", false) + require.NoError(t, err) + assert.Contains(t, s.Files, "dbt/.astro/dbt_metadata.json", "the declared context carries it") + assert.Equal(t, []string{"dbt/.astro/dbt_metadata.json"}, s.CosmosBoost) +} + // A .dockerignore that drops every DAG leaves none carried, which a deploy // that bakes DAGs into the image refuses. func TestSurveyProjectDagsDropped(t *testing.T) { From f5f24935f47bbde779296d5547b35ed8ef9417bc Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 13:06:08 -0400 Subject: [PATCH 3/7] Refuse only foreign sidecars the plugin reads, never ship links at sidecar paths, and keep the tarball out of the bundle UploadBundle writes the tarball to a temporary directory of its own. It used to land in the working directory, which under astro dbt deploy --project-path . is the directory being bundled, so it could carry itself. Tar asks the overlay's Skip about every entry, links and directories too, and anything at a sidecar's path that is not a regular file is left out with a warning. With the step on, a link there is replaced by the fresh sidecar; a directory cannot be, so that unit fails. A generated image leaves them out the same way, and a declared Dockerfile's survey names them for its warning. The foreign-producer refusal now covers only the .astro/ of a unit the step stamps, where the plugin reads, and runs on those few paths before anything is hashed, so it comes before any "step complete". One under dbt_packages/ or logs/ is the project's file and ships. precompute takes the sidecar directory, names and producer from airflowrt, and the marker check stream-decodes only the marker. Run splits into Discover and Units.Run so the check sees the units first. Co-Authored-By: Claude --- internal/platform/astro/deploy/bundle.go | 38 +++--- .../astro/deploy/cosmosboost_hook_test.go | 24 ++-- internal/platform/astro/deploy/deploy.go | 4 +- .../platform/astro/deploy/manifest_image.go | 2 +- internal/shipcheck/shipcheck.go | 9 +- internal/shipcheck/shipcheck_test.go | 2 +- pkg/airflowrt/cosmosboost.go | 87 +++++++++---- pkg/airflowrt/cosmosboost_test.go | 7 + pkg/cosmosboost/precompute/metadata.go | 26 ++-- pkg/cosmosboost/precompute/precompute.go | 71 +++++++---- pkg/cosmosboost/precompute/precompute_test.go | 47 +++++++ pkg/cosmosboost/precompute/run_test.go | 16 +++ pkg/cosmosboost/precompute/slim.go | 4 +- pkg/cosmosboost/predeploy.go | 120 ++++++++++-------- pkg/cosmosboost/predeploy_test.go | 119 ++++++++++++++--- pkg/fileutil/files.go | 47 +++++-- pkg/fileutil/files_test.go | 14 +- pkg/imagebuild/projectfiles.go | 42 +++++- pkg/imagebuild/projectsurvey_test.go | 30 +++++ 19 files changed, 520 insertions(+), 189 deletions(-) create mode 100644 pkg/cosmosboost/precompute/run_test.go diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index 579cf1618..6f716b269 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -107,7 +107,7 @@ func DeployBundle(input *DeployBundleInput) (BundleDeploy, error) { } // upload the bundle - tarballVersion, err := UploadBundle(config.WorkingPath, input.BundlePath, *deploy.BundleUploadUrl, false, currentDeployment.AstroRuntimeVersion) + tarballVersion, err := UploadBundle(input.BundlePath, *deploy.BundleUploadUrl, false, currentDeployment.AstroRuntimeVersion) if err != nil { return BundleDeploy{}, err } @@ -271,7 +271,10 @@ func ValidateBundleSymlinks(bundlePath string) error { return nil } -func UploadBundle(tarDirPath, bundlePath, uploadURL string, prependBaseDir bool, currentRuntimeVersion string) (string, error) { +// UploadBundle tars and gzips the directory bundlePath and uploads it to +// uploadURL. The tarball is written to a temporary directory of its own, never +// under bundlePath, so it cannot end up inside itself. +func UploadBundle(bundlePath, uploadURL string, prependBaseDir bool, currentRuntimeVersion string) (string, error) { // If Airflow 3.x, check for symlinks pointing outside the bundle directory if airflowversions.AirflowMajorVersionForRuntimeVersion(currentRuntimeVersion) == "3" { err := ValidateBundleSymlinks(bundlePath) @@ -280,34 +283,31 @@ func UploadBundle(tarDirPath, bundlePath, uploadURL string, prependBaseDir bool, } } - tarFilePath := filepath.Join(tarDirPath, "bundle.tar") - tarGzFilePath := tarFilePath + ".gz" + tarDir, err := os.MkdirTemp("", "astro-bundle-") + if err != nil { + return "", err + } defer func() { - tarFiles := []string{tarFilePath, tarGzFilePath} - for _, file := range tarFiles { - err := os.Remove(file) - if err != nil { - if os.IsNotExist(err) { - continue - } - fmt.Println("\nFailed to delete archived file: ", err.Error()) - fmt.Println("\nPlease delete the archived file manually from path: " + file) - } + if err := os.RemoveAll(tarDir); err != nil { + fmt.Println("\nFailed to delete archived file: ", err.Error()) + fmt.Println("\nPlease delete the archived file manually from path: " + tarDir) } }() + tarFilePath := filepath.Join(tarDir, "bundle.tar") + tarGzFilePath := tarFilePath + ".gz" // Cosmos Boost: its sidecars go into the tarball only, never into the // tree. Ones an earlier CLI wrote into the tree are always left out (a // stale one must not ship), and with cosmos_boost.pre_deploy on, fresh // ones are generated into the bundle, best-effort (a missing one is safe). - cosmosBoost := cosmosboost.NewBundle(bundlePath, config.CFG.CosmosBoostPreDeploy.GetBool()) - - // Generate the bundle tar - err := fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}, cosmosBoost.Overlay) + overlay, err := cosmosboost.BundleOverlay(bundlePath, config.CFG.CosmosBoostPreDeploy.GetBool()) if err != nil { return "", err } - if err := cosmosBoost.Err(); err != nil { + + // Generate the bundle tar + err = fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}, overlay) + if err != nil { return "", err } diff --git a/internal/platform/astro/deploy/cosmosboost_hook_test.go b/internal/platform/astro/deploy/cosmosboost_hook_test.go index 0cffba63b..a560f2905 100644 --- a/internal/platform/astro/deploy/cosmosboost_hook_test.go +++ b/internal/platform/astro/deploy/cosmosboost_hook_test.go @@ -106,7 +106,7 @@ func TestUploadBundleGeneratesSidecarsIntoTheBundle(t *testing.T) { uploaded := captureUploads(t) before := treeSnapshot(t, bundleDir) - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + _, err := UploadBundle(bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) assert.Equal(t, before, treeSnapshot(t, bundleDir), "the deploy must not write into the project") @@ -134,7 +134,7 @@ func TestUploadBundleSidecarPathsUnderTheDagsBaseDir(t *testing.T) { writeTreeFile(t, dagsDir, "dbt/shop/dbt_project.yml", "name: shop\n") uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), dagsDir, "http://upload-url", true, "0.0.0") + _, err := UploadBundle(dagsDir, "http://upload-url", true, "0.0.0") require.NoError(t, err) assert.Contains(t, uploaded(), "dags/dbt/shop/"+cosmosBoostArtifact) @@ -165,7 +165,7 @@ func TestUploadBundleSlimManifest(t *testing.T) { writeTreeFile(t, bundleDir, "target/manifest.json", `{"metadata":{"dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v12.json"},"nodes":{}}`) uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + _, err := UploadBundle(bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) assert.Contains(t, uploaded(), "target/.astro/dbt_metadata.json", "the manifest must still be stamped") @@ -185,7 +185,7 @@ func TestUploadBundleSkipsPreDeployByDefault(t *testing.T) { bundleDir := writeDbtBundle(t) uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + _, err := UploadBundle(bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) assert.NotContains(t, uploaded(), cosmosBoostArtifact, "opt-in gate is off by default") @@ -236,7 +236,7 @@ func TestUploadBundleLeavesOutStaleSidecars(t *testing.T) { uploaded := captureUploads(t) before := treeSnapshot(t, bundleDir) - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + _, err := UploadBundle(bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) assert.Equal(t, before, treeSnapshot(t, bundleDir), "enabled=%v: the tree must not change", enabled) @@ -261,7 +261,7 @@ func TestUploadBundleForeignSidecar(t *testing.T) { writeTreeFile(t, bundleDir, cosmosBoostArtifact, foreignSidecar) uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + _, err := UploadBundle(bundleDir, "http://upload-url", false, "0.0.0") require.NoError(t, err) assert.Equal(t, foreignSidecar, uploaded()[cosmosBoostArtifact]) }) @@ -272,7 +272,7 @@ func TestUploadBundleForeignSidecar(t *testing.T) { foreign := writeTreeFile(t, bundleDir, cosmosBoostArtifact, foreignSidecar) uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), bundleDir, "http://upload-url", false, "0.0.0") + _, err := UploadBundle(bundleDir, "http://upload-url", false, "0.0.0") require.Error(t, err) assert.ErrorContains(t, err, "unrecognized producer") @@ -293,7 +293,7 @@ func TestUploadBundleMissingDirIsUnchanged(t *testing.T) { dagsDir := filepath.Join(t.TempDir(), "dags") uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), dagsDir, "http://upload-url", true, "0.0.0") + _, err := UploadBundle(dagsDir, "http://upload-url", true, "0.0.0") require.NoError(t, err, "enabled=%v", enabled) assert.Empty(t, uploaded(), "enabled=%v", enabled) assert.NoDirExists(t, dagsDir) @@ -301,8 +301,10 @@ func TestUploadBundleMissingDirIsUnchanged(t *testing.T) { } // TestUploadBundleFromTheWorkingDirectory: astro dbt deploy --project-path . -// bundles ".", and an old sidecar there is still replaced by the fresh one, -// at the same path, with no second copy under another name. +// bundles ".", from the working directory, and an old sidecar there is still +// replaced by the fresh one, at the same path, with no second copy under +// another name. Nor does the tarball carry itself: it is written outside the +// directory it bundles, where it used to land in the working directory. func TestUploadBundleFromTheWorkingDirectory(t *testing.T) { setupCosmosBoostEnv(t) enableCosmosBoost(t) @@ -311,7 +313,7 @@ func TestUploadBundleFromTheWorkingDirectory(t *testing.T) { t.Chdir(bundleDir) uploaded := captureUploads(t) - _, err := UploadBundle(t.TempDir(), ".", "http://upload-url", false, "0.0.0") + _, err := UploadBundle(".", "http://upload-url", false, "0.0.0") require.NoError(t, err) names := make([]string, 0, len(uploaded())) diff --git a/internal/platform/astro/deploy/deploy.go b/internal/platform/astro/deploy/deploy.go index c699c463d..4b996eecb 100644 --- a/internal/platform/astro/deploy/deploy.go +++ b/internal/platform/astro/deploy/deploy.go @@ -75,7 +75,7 @@ func includeMonitoringDag(orgHosted bool, deploymentType astrov1.DeploymentType) return !orgHosted && !deployment.IsDeploymentDedicated(deploymentType) && !deployment.IsDeploymentStandard(deploymentType) } -func uploadDags(path, dagsPath, dagsUploadURL, currentRuntimeVersion string, monitoringDag, noDagsBaseDir bool) (string, error) { +func uploadDags(dagsPath, dagsUploadURL, currentRuntimeVersion string, monitoringDag, noDagsBaseDir bool) (string, error) { if monitoringDag { monitoringDagPath := filepath.Join(dagsPath, "astronomer_monitoring_dag.py") @@ -102,7 +102,7 @@ func uploadDags(path, dagsPath, dagsUploadURL, currentRuntimeVersion string, mon // By default, prepend dags/ directory prefix. Use --no-dags-base-dir to place files at bundle root // (needed for some Airflow 3.x deployments where sys.path includes the bundle root, not dags/). prependBaseDir := !noDagsBaseDir - versionID, err := UploadBundle(path, dagsPath, dagsUploadURL, prependBaseDir, currentRuntimeVersion) + versionID, err := UploadBundle(dagsPath, dagsUploadURL, prependBaseDir, currentRuntimeVersion) if err != nil { return "", err } diff --git a/internal/platform/astro/deploy/manifest_image.go b/internal/platform/astro/deploy/manifest_image.go index ec0120ad6..e981af149 100644 --- a/internal/platform/astro/deploy/manifest_image.go +++ b/internal/platform/astro/deploy/manifest_image.go @@ -361,7 +361,7 @@ func uploadDeployDags(c *config.Context, projectDir, deploymentID string, dep *a } dagsPath := filepath.Join(projectDir, "dags") monitoringDag := includeMonitoringDag(c.OrganizationProduct == string(astrov1.OrganizationProductHOSTED), deploymentType) - tarballVersion, err := uploadDags(projectDir, dagsPath, uploadURL, dep.AstroRuntimeVersion, monitoringDag, noDagsBaseDir) + tarballVersion, err := uploadDags(dagsPath, uploadURL, dep.AstroRuntimeVersion, monitoringDag, noDagsBaseDir) if err != nil { if strings.Contains(err.Error(), dagDeployDisabled) { return "", fmt.Errorf(enableDagDeployMsg, deploymentID) diff --git a/internal/shipcheck/shipcheck.go b/internal/shipcheck/shipcheck.go index d3dbf5cab..d6194d3bf 100644 --- a/internal/shipcheck/shipcheck.go +++ b/internal/shipcheck/shipcheck.go @@ -119,9 +119,10 @@ func (r *Report) Err() error { return nil } -// CosmosBoostWarning is the warning for the Cosmos Boost files an earlier -// Astro CLI wrote into the project that a declared Dockerfile's context -// carries (imagebuild.ProjectSurvey.CosmosBoost), or "" when there are none. +// CosmosBoostWarning is the warning for the entries at Cosmos Boost sidecar +// paths a declared Dockerfile's context carries - old sidecars an earlier +// Astro CLI wrote, links, directories (imagebuild.ProjectSurvey.CosmosBoost) - +// or "" when there are none. // A generated image leaves them out; a declared Dockerfile decides for itself // what it copies, so the build goes ahead, but the plugin would read any it // copies as current. @@ -129,7 +130,7 @@ func CosmosBoostWarning(survey *imagebuild.ProjectSurvey) string { if len(survey.CosmosBoost) == 0 { return "" } - return fmt.Sprintf("the build context carries Cosmos Boost files an earlier Astro CLI wrote into the project: %s. The Cosmos Boost plugin would read them as current if the Dockerfile copies them into the image. This CLI no longer writes them; delete them", + return fmt.Sprintf("the build context carries old Cosmos Boost sidecars, written into the project by an earlier Astro CLI or linked there: %s. The Cosmos Boost plugin would read them as current if the Dockerfile copies them into the image. This CLI no longer writes them; delete them", cappedList(survey.CosmosBoost)) } diff --git a/internal/shipcheck/shipcheck_test.go b/internal/shipcheck/shipcheck_test.go index 07ce2d55a..c337fbf1b 100644 --- a/internal/shipcheck/shipcheck_test.go +++ b/internal/shipcheck/shipcheck_test.go @@ -361,7 +361,7 @@ func TestSurveyWarnsAboutStaleCosmosBoostFilesInADeclaredContext(t *testing.T) { _, err := Survey(dir, "Dockerfile", false, func(w string) { warned = append(warned, w) }) require.NoError(t, err) all := fmt.Sprint(warned) - assert.Contains(t, all, "Cosmos Boost files an earlier Astro CLI wrote into the project: dbt/shop/.astro/dbt_metadata.json, dbt/shop/.astro/manifest.slim.json.") + assert.Contains(t, all, "old Cosmos Boost sidecars, written into the project by an earlier Astro CLI or linked there: dbt/shop/.astro/dbt_metadata.json, dbt/shop/.astro/manifest.slim.json.") assert.Contains(t, all, "delete them") assert.NotContains(t, all, "dbt/other") diff --git a/pkg/airflowrt/cosmosboost.go b/pkg/airflowrt/cosmosboost.go index 8e39cad7f..062cee81c 100644 --- a/pkg/airflowrt/cosmosboost.go +++ b/pkg/airflowrt/cosmosboost.go @@ -1,6 +1,7 @@ package airflowrt import ( + "bytes" "encoding/json" "path" ) @@ -11,15 +12,19 @@ import ( // them as current, so one left behind is stale the moment the project // changes. The CLI now generates them into what it ships and never into the // project, and everything it ships - a bundle, a generated image - leaves out -// the ones an earlier CLI left there. These say which those are, here so the -// bundle (root module) and the generated image (pkg/imagebuild) use one rule. - +// the ones an earlier CLI left there. These are the one definition of where +// they go and whose they are, for the bundle and the pre-deploy step (root +// module) and the generated image (pkg/imagebuild) alike. const ( - cosmosBoostDir = ".astro" - cosmosBoostSidecar = "dbt_metadata.json" - cosmosBoostSlimManifest = "manifest.slim.json" - // cosmosBoostProducer is the producer an Astro CLI records in each. - cosmosBoostProducer = "astro" + // CosmosBoostDir is the directory, beside a dbt project or manifest, the + // sidecars go in. + CosmosBoostDir = ".astro" + // CosmosBoostSidecar carries the content hash the plugin keys its cache on. + CosmosBoostSidecar = "dbt_metadata.json" + // CosmosBoostSlimManifest is the field-filtered manifest beside it. + CosmosBoostSlimManifest = "manifest.slim.json" + // CosmosBoostProducer is the producer an Astro CLI records in each. + CosmosBoostProducer = "astro" ) // IsCosmosBoostFile reports whether rel, slash-separated, is where a Cosmos @@ -27,8 +32,8 @@ const ( // inside a .astro directory, at any depth. func IsCosmosBoostFile(rel string) bool { switch path.Base(rel) { - case cosmosBoostSidecar, cosmosBoostSlimManifest: - return path.Base(path.Dir(rel)) == cosmosBoostDir + case CosmosBoostSidecar, CosmosBoostSlimManifest: + return path.Base(path.Dir(rel)) == CosmosBoostDir } return false } @@ -39,23 +44,59 @@ func IsCosmosBoostFile(rel string) bool { // _generated_by.application in a manifest.slim.json. Each file is judged by // its own marker, never by its neighbor's, and one that does not parse or // names another producer is not the CLI's: it is the user's. +// +// It reads only as far as the marker: a slim manifest can be large, and the +// CLI writes the marker first. func CosmosBoostFileWrittenByAstro(rel string, data []byte) bool { - var marker struct { - GeneratedBy struct { - Application string `json:"application"` - } `json:"generated_by"` - SlimGeneratedBy struct { - Application string `json:"application"` - } `json:"_generated_by"` + var key string + switch path.Base(rel) { + case CosmosBoostSidecar: + key = "generated_by" + case CosmosBoostSlimManifest: + key = "_generated_by" + default: + return false } - if json.Unmarshal(data, &marker) != nil { + dec := json.NewDecoder(bytes.NewReader(data)) + if tok, err := dec.Token(); err != nil || tok != json.Delim('{') { return false } - switch path.Base(rel) { - case cosmosBoostSidecar: - return marker.GeneratedBy.Application == cosmosBoostProducer - case cosmosBoostSlimManifest: - return marker.SlimGeneratedBy.Application == cosmosBoostProducer + for dec.More() { + name, err := dec.Token() + if err != nil { + return false + } + if name != key { + if skipJSONValue(dec) != nil { + return false + } + continue + } + var marker struct { + Application string `json:"application"` + } + return dec.Decode(&marker) == nil && marker.Application == CosmosBoostProducer } return false } + +// skipJSONValue reads past the next value in dec, token by token, without +// holding it. +func skipJSONValue(dec *json.Decoder) error { + depth := 0 + for { + tok, err := dec.Token() + if err != nil { + return err + } + switch tok { + case json.Delim('{'), json.Delim('['): + depth++ + case json.Delim('}'), json.Delim(']'): + depth-- + } + if depth == 0 { + return nil + } + } +} diff --git a/pkg/airflowrt/cosmosboost_test.go b/pkg/airflowrt/cosmosboost_test.go index fd94a5321..9fde1eb49 100644 --- a/pkg/airflowrt/cosmosboost_test.go +++ b/pkg/airflowrt/cosmosboost_test.go @@ -40,6 +40,13 @@ func TestCosmosBoostFileWrittenByAstro(t *testing.T) { {slim, `{"generated_by":{"application":"astro"}}`, false}, {slim, `{"_generated_by":{"application":"someone-else"}}`, false}, {".astro/config.yaml", `{"generated_by":{"application":"astro"}}`, false}, + // The marker after other keys, nested ones skipped whole. + {sidecar, `{"schema":1,"version":{"algo":"x","hash":"h","n":[1,{"a":[]}]},"generated_by":{"application":"astro"}}`, true}, + {slim, `{"nodes":{"a":{"generated_by":{"application":"astro"}}},"_generated_by":{"application":"astro"}}`, true}, + {slim, `{"nodes":{"a":{"_generated_by":{"application":"astro"}}}}`, false}, + {sidecar, `["generated_by",{"application":"astro"}]`, false}, + {sidecar, `{"generated_by":"astro"}`, false}, + {sidecar, `{"schema":1,`, false}, } { assert.Equal(t, tc.want, CosmosBoostFileWrittenByAstro(tc.rel, []byte(tc.data)), "%s %s", tc.rel, tc.data) } diff --git a/pkg/cosmosboost/precompute/metadata.go b/pkg/cosmosboost/precompute/metadata.go index 0526a62d5..54ed58499 100644 --- a/pkg/cosmosboost/precompute/metadata.go +++ b/pkg/cosmosboost/precompute/metadata.go @@ -5,11 +5,15 @@ import ( "fmt" "os" "path/filepath" + + "github.com/astronomer/astro-cli/pkg/airflowrt" ) // application is the producer name recorded in the sidecar. (The version is owned -// by the caller and passed in to Run.) -const application = "astro" +// by the caller and passed in to Run.) It, and where the artifacts go, are +// airflowrt's: the bundle and the generated image judge what an Astro CLI +// wrote by the same definition. +const application = airflowrt.CosmosBoostProducer const ( // schemaVersion lets the read-side cope with future format changes. @@ -29,8 +33,8 @@ const ( // manifest it came from (that is version.hash's job). algoFilteredManifest = "sha256-filtered-manifest-v1" - sidecarDir = ".astro" - sidecarName = "dbt_metadata.json" + sidecarDir = airflowrt.CosmosBoostDir + sidecarName = airflowrt.CosmosBoostSidecar ) // Metadata is the content of the .astro/dbt_metadata.json sidecar. The Cosmos @@ -88,19 +92,19 @@ func sidecarArtifact(dir, algo, hash, version string, filtered *FilteredManifest return Artifact{Path: filepath.Join(dir, sidecarDir, sidecarName), Data: data} } -// checkArtifactPaths fails when the tree holds something the artifacts could -// not replace in a bundle: a .astro under dir that is not a directory (a file, -// or a link), or anything but a regular file where an artifact goes (a -// directory, a link). Unpacked beside the artifact, either would collide with -// it. A regular file there is fine: the bundle carries the artifact instead. +// checkArtifactPaths fails when the tree holds something an artifact could +// not cleanly replace in a bundle: a .astro under dir that is not a directory +// (a file, or a link), or a directory where an artifact goes, whose contents +// would be unpacked beside it. Anything else there - an old sidecar, a link - +// the bundle leaves out and carries the artifact in its place. func checkArtifactPaths(dir string, artifacts []Artifact) error { astroDir := filepath.Join(dir, sidecarDir) if info, err := os.Lstat(astroDir); err == nil && !info.IsDir() { return fmt.Errorf("%s is not a directory, so its Cosmos Boost files cannot sit under it", astroDir) } for _, a := range artifacts { - if info, err := os.Lstat(a.Path); err == nil && !info.Mode().IsRegular() { - return fmt.Errorf("%s is not a regular file, so the Cosmos Boost file cannot take its place", a.Path) + if info, err := os.Lstat(a.Path); err == nil && info.IsDir() { + return fmt.Errorf("%s is a directory, so the Cosmos Boost file cannot take its place", a.Path) } } return nil diff --git a/pkg/cosmosboost/precompute/precompute.go b/pkg/cosmosboost/precompute/precompute.go index d2210eee6..ca348bd06 100644 --- a/pkg/cosmosboost/precompute/precompute.go +++ b/pkg/cosmosboost/precompute/precompute.go @@ -59,30 +59,23 @@ type Options struct { SlimManifest bool } -// Run finds every dbt project (a directory with dbt_project.yml) and standalone -// dbt manifest.json under the given roots, and produces a .astro/dbt_metadata.json -// hash sidecar for each, in memory (Result.Artifacts): nothing under the roots -// is written. Units are processed concurrently — one worker each, -// bounded by GOMAXPROCS — and each is hashed over sorted input, so results are -// deterministic with no cross-worker coordination. -// -// Per-unit failures are best-effort: a unit that fails is recorded in its Result -// and does not stop the others. Run only returns a non-nil error for a top-level -// problem, such as a root that cannot be walked. version is recorded in each -// sidecar's generated_by. -// -// With opts.SlimManifest set, every dbt manifest.json also gets a slim, -// field-filtered copy (see buildSlimManifest) for the .astro/ beside it, for the Cosmos Boost plugin to load in place of the full manifest at -// DAG-parse time. That includes one in a project's own root, which is not a -// discovery unit of its own and is handled by processProject. -func Run(roots []string, version string, opts Options) (Summary, error) { - start := time.Now() +// Units are what a run processes: every dbt project (a directory with +// dbt_project.yml) and standalone manifest.json under its roots, in a fixed +// order. Discover finds them; Units.Run processes them. +type Units struct { + units []unit +} + +type unit struct{ kind, path string } +// Discover finds the units under roots. It fails only when a root cannot be +// walked. +func Discover(roots []string) (Units, error) { projectDirs := map[string]bool{} for _, root := range roots { found, err := findProjects(root) if err != nil { - return Summary{}, fmt.Errorf("scanning %q for dbt projects: %w", root, err) + return Units{}, fmt.Errorf("scanning %q for dbt projects: %w", root, err) } for _, d := range found { projectDirs[d] = true @@ -93,14 +86,13 @@ func Run(roots []string, version string, opts Options) (Summary, error) { for _, root := range roots { found, err := findManifests(root, projectDirs) if err != nil { - return Summary{}, fmt.Errorf("scanning %q for manifests: %w", root, err) + return Units{}, fmt.Errorf("scanning %q for manifests: %w", root, err) } for _, m := range found { manifests[m] = true } } - type unit struct{ kind, path string } var units []unit for d := range projectDirs { units = append(units, unit{kindProject, d}) @@ -113,7 +105,42 @@ func Run(roots []string, version string, opts Options) (Summary, error) { sort.Slice(units, func(i, j int) bool { return units[i].path+"\x00"+units[i].kind < units[j].path+"\x00"+units[j].kind }) + return Units{units: units}, nil +} +// SidecarDirs are the .astro directories a run of u produces its artifacts +// in, one per unit, without reading any unit: a manifest.json that turns out +// not to be a dbt manifest is listed too, though nothing is produced for it. +func (u Units) SidecarDirs() []string { + dirs := make([]string, 0, len(u.units)) + for _, un := range u.units { + dir := un.path + if un.kind == kindManifest { + dir = filepath.Dir(un.path) + } + dirs = append(dirs, filepath.Join(dir, sidecarDir)) + } + return dirs +} + +// Run produces a .astro/dbt_metadata.json hash sidecar for each of u's units, +// in memory (Result.Artifacts): nothing is written. Units are processed +// concurrently — one worker each, bounded by GOMAXPROCS — and each is hashed +// over sorted input, so results are deterministic with no cross-worker +// coordination. +// +// Per-unit failures are best-effort: a unit that fails is recorded in its +// Result and does not stop the others. version is recorded in each sidecar's +// generated_by. +// +// With opts.SlimManifest set, every dbt manifest.json also gets a slim, +// field-filtered copy (see buildSlimManifest) for the .astro/ beside it, for +// the Cosmos Boost plugin to load in place of the full manifest at DAG-parse +// time. That includes one in a project's own root, which is not a discovery +// unit of its own and is handled by processProject. +func (u Units) Run(version string, opts Options) Summary { + start := time.Now() + units := u.units results := make([]Result, len(units)) sem := make(chan struct{}, max(1, runtime.GOMAXPROCS(0))) var wg sync.WaitGroup @@ -132,7 +159,7 @@ func Run(roots []string, version string, opts Options) (Summary, error) { } wg.Wait() - return Summary{Duration: time.Since(start), Results: results}, nil + return Summary{Duration: time.Since(start), Results: results} } // processProject hashes one dbt project directory and produces its sidecar. It reads diff --git a/pkg/cosmosboost/precompute/precompute_test.go b/pkg/cosmosboost/precompute/precompute_test.go index 1c1603784..5e0bc1e06 100644 --- a/pkg/cosmosboost/precompute/precompute_test.go +++ b/pkg/cosmosboost/precompute/precompute_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "testing" "time" @@ -759,6 +760,52 @@ func TestRunFailsUnitWhenArtifactPathIsNotAFile(t *testing.T) { mustProduce(t, summary, filepath.Join(root, "fine", sidecarDir, sidecarName)) } +// TestRunReplacesALinkAtAnArtifactPath: a link where a sidecar goes is left +// out of the bundle, so the fresh sidecar takes its place and the unit +// succeeds. +func TestRunReplacesALinkAtAnArtifactPath(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{"proj/dbt_project.yml": "name: p\n", "elsewhere.json": "{}"}) + if err := os.MkdirAll(filepath.Join(root, "proj", sidecarDir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "elsewhere.json"), filepath.Join(root, "proj", sidecarDir, sidecarName)); err != nil { + t.Skipf("cannot create a symlink here: %v", err) + } + + summary, err := Run([]string{root}, "test", Options{SlimManifest: true}) + if err != nil { + t.Fatal(err) + } + if summary.CountFailed() != 0 { + t.Fatalf("want no failures, got %+v", summary.Results) + } + mustProduce(t, summary, filepath.Join(root, "proj", sidecarDir, sidecarName)) +} + +// TestUnitsSidecarDirs: one .astro per unit, beside a project or a manifest, +// known before any unit is read. +func TestUnitsSidecarDirs(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "proj/dbt_project.yml": "name: p\n", + "proj/target/manifest.json": "not even json", + "shipped/manifest.json": "{}", + }) + units, err := Discover([]string{root}) + if err != nil { + t.Fatal(err) + } + want := []string{ + filepath.Join(root, "proj", sidecarDir), + filepath.Join(root, "proj", "target", sidecarDir), + filepath.Join(root, "shipped", sidecarDir), + } + if got := units.SidecarDirs(); !slices.Equal(got, want) { + t.Fatalf("SidecarDirs = %v, want %v", got, want) + } +} + // treeSnapshot maps every file under root to its contents. func treeSnapshot(t *testing.T, root string) map[string]string { t.Helper() diff --git a/pkg/cosmosboost/precompute/run_test.go b/pkg/cosmosboost/precompute/run_test.go new file mode 100644 index 000000000..f362cb3d6 --- /dev/null +++ b/pkg/cosmosboost/precompute/run_test.go @@ -0,0 +1,16 @@ +package precompute + +import "time" + +// Run discovers the units under roots and runs them, as a deploy does in two +// steps, timing both. +func Run(roots []string, version string, opts Options) (Summary, error) { + start := time.Now() + units, err := Discover(roots) + if err != nil { + return Summary{}, err + } + summary := units.Run(version, opts) + summary.Duration = time.Since(start) + return summary, nil +} diff --git a/pkg/cosmosboost/precompute/slim.go b/pkg/cosmosboost/precompute/slim.go index c202c95ac..59d17a05d 100644 --- a/pkg/cosmosboost/precompute/slim.go +++ b/pkg/cosmosboost/precompute/slim.go @@ -1,6 +1,8 @@ package precompute -const slimManifestName = "manifest.slim.json" +import "github.com/astronomer/astro-cli/pkg/airflowrt" + +const slimManifestName = airflowrt.CosmosBoostSlimManifest // slimSchemaVersion identifies the allowlist that produced a slim manifest, so // a reader that doesn't recognize it can fall back to the full manifest. diff --git a/pkg/cosmosboost/predeploy.go b/pkg/cosmosboost/predeploy.go index e3010a94c..21eec78d4 100644 --- a/pkg/cosmosboost/predeploy.go +++ b/pkg/cosmosboost/predeploy.go @@ -32,18 +32,20 @@ func slimManifestEnabled() bool { return value == "" || util.CheckEnvBool(value) } -// Bundle is what a deploy bundle of a directory changes for Cosmos Boost, as -// an overlay on the tarball (Overlay): the directory itself is never written. +// BundleOverlay is what a deploy bundle of root changes for Cosmos Boost, as +// an overlay on the tarball: root itself is never written. // -// Whatever the setting says, every .astro/dbt_metadata.json and -// .astro/manifest.slim.json that an earlier Astro CLI wrote into the tree (its -// own producer marker says so, airflowrt.CosmosBoostFileWrittenByAstro) is -// left out, judged inside the tarball's own walk so the files left out are -// exactly the ones it would have carried. The plugin cannot tell a fresh -// sidecar from a stale one, and this CLI never writes there. +// Whatever the setting says, the tarball's own walk (Skip) leaves out every +// .astro/dbt_metadata.json and .astro/manifest.slim.json that an earlier +// Astro CLI wrote into the tree (its own producer marker says so, +// airflowrt.CosmosBoostFileWrittenByAstro), so the files left out are exactly +// ones the tarball would have carried; and anything at such a path that is +// not a regular file, with a warning. The plugin cannot tell a fresh sidecar +// from a stale one, and this CLI never writes there. One from another +// producer is the user's file and ships untouched. // -// With the setting on (cosmos_boost.pre_deploy), the pre-deploy step runs over -// the directory and its sidecars are added at the same paths, with the same +// With the setting on (cosmos_boost.pre_deploy), the pre-deploy step runs +// over root and its sidecars are added at the same paths, with the same // contents, the plugin has always read: every dbt project (dbt_project.yml) // gets a .astro/dbt_metadata.json carrying its content hash, which the plugin // uses as a cache version key at parse time instead of hashing the project @@ -51,54 +53,57 @@ func slimManifestEnabled() bool { // plus a slim, field-filtered copy for the plugin to load in place of the full // manifest. That step is best-effort - a failure is a warning, never an error // - because without a sidecar the plugin simply falls back to hashing at parse -// time. +// time. Before it hashes anything, it refuses the deploy when another +// producer's file sits where it would put a sidecar: the plugin reads that +// directory, and this CLI will not judge a file it did not write fresh. // -// A file from an unrecognized producer is the user's, and ships untouched -// when the step is off. With it on, Err refuses the deploy: the plugin would -// consume that file's version.hash all the same, and this CLI will not judge -// a file it did not write fresh. -type Bundle struct { - Overlay fileutil.TarOverlay - enabled bool - foreign []string -} - -// NewBundle prepares the overlay for a bundle of root, generating the -// sidecars into it when enabled. A root that cannot be read generates none, -// and is left for the tarball to deal with as it always has. -func NewBundle(root string, enabled bool) *Bundle { - b := &Bundle{enabled: enabled} - b.Overlay = fileutil.TarOverlay{Skip: b.skip, Extra: map[string][]byte{}} +// A root that cannot be read gets no sidecars, and is left for the tarball to +// deal with as it always has. +func BundleOverlay(root string, enabled bool) (fileutil.TarOverlay, error) { + overlay := fileutil.TarOverlay{Skip: skipOldSidecar, Extra: map[string][]byte{}} if !enabled { - return b + return overlay, nil } if _, err := os.Stat(root); err != nil { - return b + return overlay, nil //nolint:nilerr // the tarball reports a root it cannot read, as it always has } - artifacts, err := preDeploy(root) + units, err := precompute.Discover([]string{root}) + if err != nil { + fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) + return overlay, nil + } + if err := refuseForeignSidecars(units.SidecarDirs()); err != nil { + return overlay, err + } + + artifacts, err := preDeploy(units) for _, a := range artifacts { rel, relErr := filepath.Rel(root, a.Path) if relErr != nil { err = errors.Join(err, relErr) continue } - b.Overlay.Extra[filepath.ToSlash(rel)] = a.Data + overlay.Extra[filepath.ToSlash(rel)] = a.Data } if err != nil { fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) - return b + return overlay, nil } fmt.Println("Cosmos Boost pre-deploy step complete") - return b + return overlay, nil } -// skip is Overlay.Skip: it leaves out each Cosmos Boost file an Astro CLI -// wrote into the tree, and notes every other one for Err. Only files at a -// sidecar's path are read. -func (b *Bundle) skip(rel, file string) (bool, error) { +// skipOldSidecar is the overlay's Skip: it leaves out each Cosmos Boost file +// an Astro CLI wrote into the tree, and anything at a sidecar's path that is +// not a regular file. Only a regular file at a sidecar's path is read. +func skipOldSidecar(rel, file string, info os.FileInfo) (bool, error) { if !airflowrt.IsCosmosBoostFile(rel) { return false, nil } + if !info.Mode().IsRegular() { + fmt.Printf("Warning: leaving %s out of the bundle: Cosmos Boost keeps a sidecar file at that path, and this is not a regular file\n", file) + return true, nil + } data, err := os.ReadFile(file) if err != nil { return false, err @@ -107,29 +112,42 @@ func (b *Bundle) skip(rel, file string) (bool, error) { logger.Debugf("leaving out a Cosmos Boost file an earlier Astro CLI wrote into the project: %s", file) return true, nil } - b.foreign = append(b.foreign, file) return false, nil } -// Err, once the tarball is written, refuses an enabled deploy that found a -// Cosmos Boost file from an unrecognized producer, which the plugin would read -// as current. nil otherwise. -func (b *Bundle) Err() error { - if !b.enabled || len(b.foreign) == 0 { - return nil +// refuseForeignSidecars fails when a regular file another producer wrote sits +// where the pre-deploy step puts a sidecar, in one of dirs. Only those few +// paths are read. +func refuseForeignSidecars(dirs []string) error { + var foreign []string + for _, dir := range dirs { + for _, name := range []string{airflowrt.CosmosBoostSidecar, airflowrt.CosmosBoostSlimManifest} { + file := filepath.Join(dir, name) + info, err := os.Lstat(file) + if err != nil || !info.Mode().IsRegular() { + continue // absent, or left out by skipOldSidecar + } + data, err := os.ReadFile(file) + if err != nil { + return err + } + if !airflowrt.CosmosBoostFileWrittenByAstro(name, data) { + foreign = append(foreign, file) + } + } } - return fmt.Errorf("found Cosmos Boost file(s) from an unrecognized producer: %s; the deploy cannot prove they are fresh and will not ship them for the Cosmos Boost plugin to read. Remove them and retry", strings.Join(b.foreign, ", ")) + if len(foreign) > 0 { + return fmt.Errorf("found Cosmos Boost file(s) from an unrecognized producer: %s; the deploy cannot prove they are fresh and will not ship them for the Cosmos Boost plugin to read. Remove them and retry", strings.Join(foreign, ", ")) + } + return nil } -// preDeploy runs the Cosmos Boost pre-deploy step over path and returns the +// preDeploy runs the Cosmos Boost pre-deploy step over units and returns the // artifacts it produced, in memory. It fails when any unit failed, still // returning the artifacts of the ones that did not. -func preDeploy(path string) ([]precompute.Artifact, error) { +func preDeploy(units precompute.Units) ([]precompute.Artifact, error) { opts := precompute.Options{SlimManifest: slimManifestEnabled()} - summary, err := precompute.Run([]string{path}, version.Current(), opts) - if err != nil { - return nil, fmt.Errorf("running the Cosmos Boost pre-deploy step: %w", err) - } + summary := units.Run(version.Current(), opts) debugReport("pre-deploy", summary) if failed := summary.CountFailed(); failed > 0 { return summary.Artifacts(), fmt.Errorf("the Cosmos Boost pre-deploy step failed for %d unit(s) (re-run with --verbosity debug for details)", failed) diff --git a/pkg/cosmosboost/predeploy_test.go b/pkg/cosmosboost/predeploy_test.go index d2e71ba97..545317fe4 100644 --- a/pkg/cosmosboost/predeploy_test.go +++ b/pkg/cosmosboost/predeploy_test.go @@ -68,13 +68,16 @@ func setVersion(t *testing.T, v string) { t.Cleanup(func() { version.CurrVersion = orig }) } -// bundle tars dir as a deploy does, through NewBundle's overlay, and returns -// the tarball's entries by name, and the error the deploy would stop on. +// bundle tars dir as a deploy does, through BundleOverlay, and returns the +// tarball's entries by name, or the error the deploy would stop on. func bundle(t *testing.T, dir string, enabled bool) (map[string]string, error) { t.Helper() - b := NewBundle(dir, enabled) + overlay, err := BundleOverlay(dir, enabled) + if err != nil { + return nil, err + } target := filepath.Join(t.TempDir(), "bundle.tar") - if err := fileutil.Tar(dir, target, false, []string{".git/"}, b.Overlay); err != nil { + if err := fileutil.Tar(dir, target, false, []string{".git/"}, overlay); err != nil { return nil, err } file, err := os.Open(target) @@ -92,7 +95,7 @@ func bundle(t *testing.T, dir string, enabled bool) (map[string]string, error) { require.NoError(t, err) entries[header.Name] = string(data) } - return entries, b.Err() + return entries, nil } // TestBundleEnabledAddsSidecars pins the plugin contract: the same files at @@ -112,8 +115,9 @@ func TestBundleEnabledAddsSidecars(t *testing.T) { require.ElementsMatch(t, []string{"dbt_project.yml", "models/a.sql", "target/manifest.json", sidecarRel, "target/" + sidecarRel, "target/" + slimRel}, keys(entries)) // The contents are exactly what the pre-deploy step produces for the tree. - summary, err := precompute.Run([]string{dir}, "1.2.3", precompute.Options{SlimManifest: true}) + units, err := precompute.Discover([]string{dir}) require.NoError(t, err) + summary := units.Run("1.2.3", precompute.Options{SlimManifest: true}) for _, a := range summary.Artifacts() { rel, err := filepath.Rel(dir, a.Path) require.NoError(t, err) @@ -211,29 +215,106 @@ func TestBundleLeavesOutStaleSidecars(t *testing.T) { } // TestBundleForeignSidecar: a file from an unrecognized producer is the -// user's. Disabled, it ships untouched; enabled, the deploy stops and asks for -// its removal, because the plugin would consume its version.hash - even when -// it sits where a fresh sidecar would replace it. +// user's. Disabled, it ships untouched. Enabled, where the step would put a +// sidecar the plugin reads - a discovered project's or manifest's .astro/ - +// the deploy stops before anything is hashed and asks for its removal; the +// replacement would hide it, and the plugin would have read it. func TestBundleForeignSidecar(t *testing.T) { - for _, rel := range []string{"other/" + sidecarRel, sidecarRel} { + for _, rel := range []string{sidecarRel, slimRel, "target/" + sidecarRel} { dir := t.TempDir() writeDbtProject(t, dir) - writeFiles(t, dir, map[string]string{rel: foreignFile}) + writeFiles(t, dir, map[string]string{"target/manifest.json": dbtManifest, rel: foreignFile}) foreign := filepath.Join(dir, filepath.FromSlash(rel)) entries, err := bundle(t, dir, false) require.NoError(t, err) require.Equal(t, foreignFile, entries[rel], "disabled: a foreign file ships") - _, err = bundle(t, dir, true) + out := captureStdout(t, func() { _, err = bundle(t, dir, true) }) require.Error(t, err, rel) require.ErrorContains(t, err, "unrecognized producer") require.ErrorContains(t, err, foreign) require.ErrorContains(t, err, "Remove them and retry") + require.NotContains(t, out, "pre-deploy step complete", "the refusal comes before the step") require.FileExists(t, foreign, "a file another tool wrote must never be deleted") } } +// TestBundleShipsForeignSidecarsTheStepDoesNotWrite: one where no sidecar +// would go - an installed package's own, under dbt_packages/ - is the +// project's file, and an enabled deploy ships it. +func TestBundleShipsForeignSidecarsTheStepDoesNotWrite(t *testing.T) { + dir := t.TempDir() + writeDbtProject(t, dir) + writeFiles(t, dir, map[string]string{ + "dbt_packages/utils/dbt_project.yml": "name: utils\n", + "dbt_packages/utils/" + sidecarRel: foreignFile, + "logs/" + sidecarRel: foreignFile, + }) + + entries, err := bundle(t, dir, true) + require.NoError(t, err) + require.Equal(t, foreignFile, entries["dbt_packages/utils/"+sidecarRel]) + require.Equal(t, foreignFile, entries["logs/"+sidecarRel]) + require.Contains(t, entries, sidecarRel, "the project itself is stamped") +} + +// TestBundleLeavesOutNonRegularEntriesAtSidecarPaths: a link or a directory +// where a sidecar goes never ships, with a warning naming it. Enabled, a link +// is replaced by the fresh sidecar; a directory cannot be, so that unit fails +// (a warning) and nothing is shipped there. +func TestBundleLeavesOutNonRegularEntriesAtSidecarPaths(t *testing.T) { + for _, enabled := range []bool{false, true} { + dir := t.TempDir() + writeDbtProject(t, filepath.Join(dir, "linked")) + writeDbtProject(t, filepath.Join(dir, "blocked")) + writeFiles(t, dir, map[string]string{ + "target.json": oursSidecar, + "blocked/" + sidecarRel + "/x": "x", + }) + link := filepath.Join(dir, "linked", ".astro", "dbt_metadata.json") + require.NoError(t, os.MkdirAll(filepath.Dir(link), 0o755)) + if err := os.Symlink(filepath.Join(dir, "target.json"), link); err != nil { + t.Skipf("cannot create a symlink here: %v", err) + } + + var entries map[string]string + var err error + out := captureStdout(t, func() { entries, err = bundle(t, dir, enabled) }) + require.NoError(t, err, "enabled=%v", enabled) + require.Contains(t, out, "leaving "+link+" out of the bundle", "enabled=%v", enabled) + require.Contains(t, out, "leaving "+filepath.Join(dir, "blocked", ".astro", "dbt_metadata.json")+" out of the bundle", "enabled=%v", enabled) + require.NotContains(t, entries, "blocked/"+sidecarRel+"/x", "enabled=%v", enabled) + require.NotContains(t, entries, "blocked/"+sidecarRel, "enabled=%v: a directory cannot be replaced", enabled) + if enabled { + require.Contains(t, out, "pre-deploy step failed") + require.Contains(t, entries, "linked/"+sidecarRel) + require.NotContains(t, entries["linked/"+sidecarRel], "stale", "the fresh sidecar replaces the link") + } else { + require.NotContains(t, entries, "linked/"+sidecarRel) + } + } +} + +// captureStdout returns what fn prints to stdout. +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + require.NoError(t, err) + orig := os.Stdout + os.Stdout = w + done := make(chan string) + go func() { + data, _ := io.ReadAll(r) + done <- string(data) + }() + defer func() { os.Stdout = orig }() + fn() + os.Stdout = orig + require.NoError(t, w.Close()) + return <-done +} + func TestBundleNoDbtContentAddsNothing(t *testing.T) { dir := t.TempDir() writeFiles(t, dir, map[string]string{"app.py": "print('hi')"}) @@ -266,18 +347,18 @@ func TestBundleStepFailureIsAWarning(t *testing.T) { dir := t.TempDir() writeDbtProject(t, filepath.Join(dir, "broken")) writeDbtProject(t, filepath.Join(dir, "fine")) - writeFiles(t, dir, map[string]string{"blocked/dbt_project.yml": "name: blocked\n", "blocked/.astro/dbt_metadata.json/x": "x"}) model := filepath.Join(dir, "broken", "models", "a.sql") require.NoError(t, os.Chmod(model, 0o000)) t.Cleanup(func() { _ = os.Chmod(model, 0o644) }) - b := NewBundle(dir, true) - require.NoError(t, b.Err()) - require.Equal(t, []string{"fine/" + sidecarRel}, keys(b.Overlay.Extra), - "neither the unreadable project nor the one with a directory where its sidecar goes gets one") + overlay, err := BundleOverlay(dir, true) + require.NoError(t, err) + require.Equal(t, []string{"fine/" + sidecarRel}, keys(overlay.Extra)) - _, err := preDeploy(dir) - require.ErrorContains(t, err, "failed for 2 unit(s)") + units, err := precompute.Discover([]string{dir}) + require.NoError(t, err) + _, err = preDeploy(units) + require.ErrorContains(t, err, "failed for 1 unit(s)") } func keys[V any](m map[string]V) []string { diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index 546c15ac8..99a4e86c6 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -110,12 +110,14 @@ func WriteToFile(filePath string, r io.Reader) error { // slash-separated and relative to source, before any base directory is // prepended. type TarOverlay struct { - // Skip, when set, is asked about each regular file the walk would add, by - // rel and by its path on disk, and leaves it out on true. An error fails - // the tarball. - Skip func(rel, file string) (bool, error) - // Extra adds a regular file with these bytes at each path, in place of - // anything the walk finds there. + // Skip, when set, is asked about each entry the walk reaches below source - + // a file, a link, a directory - by rel, its path on disk and what Lstat + // says of it, and leaves it out on true: a directory with everything + // under it. An error fails the tarball. + Skip func(rel, file string, info os.FileInfo) (bool, error) + // Extra adds a regular file with these bytes at each path, in place of a + // file or link the walk finds there. A directory there is not replaced: + // callers must not add an extra where one exists. Extra map[string][]byte } @@ -189,7 +191,7 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr } if info.IsDir() { - return skipPycache(info, path, source) + return tarDir(path, source, info, overlay) } if path == target { @@ -208,7 +210,7 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr return err } - headerName, ok, err := tarEntryName(path, source, baseDir, excludePathPrefixes, overlay, info.Mode().IsRegular()) + headerName, ok, err := tarEntryName(path, source, baseDir, excludePathPrefixes, overlay, info) if err != nil || !ok { return err } @@ -237,11 +239,30 @@ func addToTar(tarball *tar.Writer, source, target, baseDir string, excludePathPr return addExtrasToTar(tarball, baseDir, overlay.Extra) } +// tarDir decides whether the walk descends into the directory at dir: +// filepath.SkipDir for one the overlay skips, or a bytecode cache. +func tarDir(dir, source string, info os.FileInfo, overlay TarOverlay) error { + if dir != source && overlay.Skip != nil { + rel, err := filepath.Rel(source, dir) + if err != nil { + return err + } + skip, err := overlay.Skip(filepath.ToSlash(rel), dir, info) + if err != nil { + return err + } + if skip { + logger.Debugf("Leaving out of the tarball: %s", rel) + return filepath.SkipDir + } + } + return skipPycache(info, dir, source) +} + // tarEntryName is the name file goes into the tarball under, or false when // it stays out: one under an excluded prefix, or one the overlay skips or -// replaces. regular says file is a regular file, the only kind Skip is asked -// about. -func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, overlay TarOverlay, regular bool) (name string, ok bool, err error) { +// replaces. info is what Lstat says of file. +func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, overlay TarOverlay, info os.FileInfo) (name string, ok bool, err error) { // the tar file path is relative to the source directory, which may be "." relName, err := filepath.Rel(source, file) if err != nil { @@ -260,8 +281,8 @@ func tarEntryName(file, source, baseDir string, excludePathPrefixes []string, ov } rel := filepath.ToSlash(relName) - if regular && overlay.Skip != nil { - skip, err := overlay.Skip(rel, file) + if overlay.Skip != nil { + skip, err := overlay.Skip(rel, file, info) if err != nil { return "", false, err } diff --git a/pkg/fileutil/files_test.go b/pkg/fileutil/files_test.go index d7b285619..3f391c822 100644 --- a/pkg/fileutil/files_test.go +++ b/pkg/fileutil/files_test.go @@ -305,11 +305,13 @@ func (s *Suite) TestTarOverlay() { s.Require().NoError(os.WriteFile(filepath.Join(source, "a.sql"), []byte("a"), 0o600)) s.Require().NoError(os.WriteFile(filepath.Join(source, ".astro", "stale.json"), []byte("stale"), 0o600)) s.Require().NoError(os.WriteFile(filepath.Join(source, ".astro", "replaced.json"), []byte("old"), 0o600)) + s.Require().NoError(os.MkdirAll(filepath.Join(source, "gone", "deeper"), 0o755)) + s.Require().NoError(os.WriteFile(filepath.Join(source, "gone", "deeper", "x"), []byte("x"), 0o600)) var asked []string overlay := TarOverlay{ - Skip: func(rel, file string) (bool, error) { + Skip: func(rel, file string, info os.FileInfo) (bool, error) { asked = append(asked, rel) - return rel == ".astro/stale.json", nil + return rel == ".astro/stale.json" || rel == "gone", nil }, Extra: map[string][]byte{ ".astro/replaced.json": []byte("new"), @@ -350,9 +352,9 @@ func (s *Suite) TestTarOverlay() { } s.ElementsMatch([]string{ - "a.sql", ".astro/stale.json", ".astro/replaced.json", - "a.sql", ".astro/stale.json", ".astro/replaced.json", - }, asked, "Skip is asked about every regular file, once per tarball") + "a.sql", ".astro", ".astro/stale.json", ".astro/replaced.json", "gone", + "a.sql", ".astro", ".astro/stale.json", ".astro/replaced.json", "gone", + }, asked, "Skip is asked about every entry below the source, once per tarball, and not below one it skips") s.Run("source is the working directory", func() { s.T().Chdir(source) @@ -363,7 +365,7 @@ func (s *Suite) TestTarOverlay() { s.Run("a Skip error fails the tarball", func() { boom := errors.New("boom") - err := Tar(source, filepath.Join(dir, "boom.tar"), false, nil, TarOverlay{Skip: func(string, string) (bool, error) { return false, boom }}) + err := Tar(source, filepath.Join(dir, "boom.tar"), false, nil, TarOverlay{Skip: func(string, string, os.FileInfo) (bool, error) { return false, boom }}) s.ErrorIs(err, boom) }) diff --git a/pkg/imagebuild/projectfiles.go b/pkg/imagebuild/projectfiles.go index b759b1d83..43f227ef7 100644 --- a/pkg/imagebuild/projectfiles.go +++ b/pkg/imagebuild/projectfiles.go @@ -246,7 +246,7 @@ func (w walker) walkDir(dir, rel string, onPath map[string]bool) error { if err != nil { return err } - skip, descend := w.filter.skip(childRel, info.IsDir()) + skip, descend := w.decide(childRel, e.Type(), info) if w.vcs != nil && vcsNames[e.Name()] { if !skip { w.vcs(childRel) @@ -301,6 +301,37 @@ func (w walker) carry(rel, src string, mode fs.FileMode) error { return nil } +// decide is the filter's word on the entry at rel (projectFilter.skip), and +// oddSidecar's on one at a Cosmos Boost sidecar's path. entry is the +// directory entry's own type, info what the walk's stat says. +func (w walker) decide(rel string, entry fs.FileMode, info fs.FileInfo) (skip, descend bool) { + skip, descend = w.filter.skip(rel, info.IsDir()) + if !skip && w.oddSidecar(rel, entry, info.Mode()) { + return true, false + } + return skip, descend +} + +// oddSidecar handles a link, a directory or anything else that is not a +// plain regular file at a Cosmos Boost sidecar's path (airflowrt.IsCosmosBoostFile): +// the plugin would read whatever it leads to as a sidecar. A generated build +// leaves it out, which oddSidecar reports by returning true; a declared one +// carries it and reports it, as staleCosmosBoost does an old sidecar. entry +// is the directory entry's own type, mode what the walk's stat says. +func (w walker) oddSidecar(rel string, entry, mode fs.FileMode) bool { + if (!w.filter.generated && w.cosmosBoost == nil) || !airflowrt.IsCosmosBoostFile(rel) { + return false + } + if entry&fs.ModeSymlink == 0 && mode.IsRegular() { + return false + } + if w.filter.generated { + return true + } + w.cosmosBoost(rel) + return false +} + // staleCosmosBoost reports whether the regular file at src is a Cosmos Boost // sidecar an earlier Astro CLI wrote into the project, which the plugin would // read as current in the image (airflowrt.CosmosBoostFileWrittenByAstro). The @@ -352,10 +383,11 @@ type ProjectSurvey struct { // Dockerfile's context carries, which its ignore file leaves in: each // one's path, not what is in it. A generated build carries none. VCS []string - // CosmosBoost are the Cosmos Boost files an earlier Astro CLI wrote into - // the project that a declared Dockerfile's context carries, which the - // plugin would read as current in the image. A generated build leaves - // them out and lists none. + // CosmosBoost are the entries at a Cosmos Boost sidecar's path that a + // declared Dockerfile's context carries and the plugin would read as + // current in the image: the sidecars an earlier Astro CLI wrote into the + // project, and any link or directory there. A generated build leaves them + // out and lists none. CosmosBoost []string // IgnoreFile is the ignore file a declared Dockerfile's build reads // (declaredIgnoreFile), "" for none or for a generated build. diff --git a/pkg/imagebuild/projectsurvey_test.go b/pkg/imagebuild/projectsurvey_test.go index f73ee896b..71a84fb80 100644 --- a/pkg/imagebuild/projectsurvey_test.go +++ b/pkg/imagebuild/projectsurvey_test.go @@ -84,6 +84,36 @@ func TestSurveyProjectDeclaredNamesStaleCosmosBoostFiles(t *testing.T) { assert.Equal(t, []string{"dbt/.astro/dbt_metadata.json"}, s.CosmosBoost) } +// A link or a directory at a sidecar's path is not a sidecar this CLI wrote, +// but the plugin would read whatever it leads to as one: a generated image +// leaves it out, and a declared context's survey names it. +func TestNonRegularEntriesAtSidecarPaths(t *testing.T) { + project := t.TempDir() + writeTree(t, project, map[string]string{ + "Dockerfile": "FROM x\n", + "dags/dbt/shop/dbt_project.yml": "name: shop\n", + "include/stale.json": `{"generated_by":{"application":"someone-else"}}`, + "dags/dbt/dir/.astro/dbt_metadata.json/x": "x", + }) + link := filepath.Join(project, "dags", "dbt", "shop", ".astro", "dbt_metadata.json") + require.NoError(t, os.MkdirAll(filepath.Dir(link), 0o755)) + if err := os.Symlink(filepath.Join(project, "include", "stale.json"), link); err != nil { + t.Skipf("cannot create a symlink here: %v", err) + } + + s, err := SurveyProject(project, "", false) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"Dockerfile", "dags/dbt/shop/dbt_project.yml", "include/stale.json"}, s.Files) + assert.Empty(t, s.CosmosBoost) + dst := t.TempDir() + require.NoError(t, copyProjectFiles(project, dst)) + assert.ElementsMatch(t, s.Files, contextFiles(t, dst), "the survey and the copy agree") + + s, err = SurveyProject(project, "Dockerfile", false) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"dags/dbt/shop/.astro/dbt_metadata.json", "dags/dbt/dir/.astro/dbt_metadata.json"}, s.CosmosBoost) +} + // A .dockerignore that drops every DAG leaves none carried, which a deploy // that bakes DAGs into the image refuses. func TestSurveyProjectDagsDropped(t *testing.T) { From e2dc32bf85a7ec0cdc2072c1c108c3358382c379 Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 13:25:15 -0400 Subject: [PATCH 4/7] Simplify the Cosmos Boost bundle rules, and stream the bundle into one gzipped file The foreign-producer refusal is decided from the paths this deploy actually writes a fresh sidecar to, after the step runs and before it reports complete; a foreign file anywhere else ships. That drops the separate pre-hash check and the Discover/SidecarDirs split. With the step off, only regular files an Astro CLI wrote are left out; links and everything else at a sidecar path ship as they are, in the bundle and in a generated image. With it on, whatever is at an artifact's path, a link or a directory too, is left out for the fresh sidecar; a unit fails only when its .astro is not a directory. The producer check goes back to unmarshaling the bytes already read. UploadBundle streams tar into gzip into one file (fileutil.TarGz) and checks every close, so a full disk fails the deploy instead of uploading a truncated bundle. Co-Authored-By: Claude --- internal/platform/astro/deploy/bundle.go | 13 +- internal/shipcheck/shipcheck.go | 9 +- internal/shipcheck/shipcheck_test.go | 2 +- pkg/airflowrt/cosmosboost.go | 67 +++------- pkg/airflowrt/cosmosboost_test.go | 10 +- pkg/cosmosboost/precompute/metadata.go | 16 +-- pkg/cosmosboost/precompute/precompute.go | 59 ++++----- pkg/cosmosboost/precompute/precompute_test.go | 38 +----- pkg/cosmosboost/precompute/run_test.go | 16 --- pkg/cosmosboost/predeploy.go | 121 +++++++++--------- pkg/cosmosboost/predeploy_test.go | 89 +++++++------ pkg/fileutil/files.go | 44 +++++-- pkg/fileutil/files_test.go | 52 ++++++-- pkg/imagebuild/projectfiles.go | 42 +----- pkg/imagebuild/projectsurvey_test.go | 30 ----- 15 files changed, 245 insertions(+), 363 deletions(-) delete mode 100644 pkg/cosmosboost/precompute/run_test.go diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index 6f716b269..787111646 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -293,8 +293,7 @@ func UploadBundle(bundlePath, uploadURL string, prependBaseDir bool, currentRunt fmt.Println("\nPlease delete the archived file manually from path: " + tarDir) } }() - tarFilePath := filepath.Join(tarDir, "bundle.tar") - tarGzFilePath := tarFilePath + ".gz" + tarGzFilePath := filepath.Join(tarDir, "bundle.tar.gz") // Cosmos Boost: its sidecars go into the tarball only, never into the // tree. Ones an earlier CLI wrote into the tree are always left out (a @@ -305,14 +304,8 @@ func UploadBundle(bundlePath, uploadURL string, prependBaseDir bool, currentRunt return "", err } - // Generate the bundle tar - err = fileutil.Tar(bundlePath, tarFilePath, prependBaseDir, []string{".git/"}, overlay) - if err != nil { - return "", err - } - - // Gzip the tar - err = fileutil.GzipFile(tarFilePath, tarGzFilePath) + // Generate the gzipped bundle tarball + err = fileutil.TarGz(bundlePath, tarGzFilePath, prependBaseDir, []string{".git/"}, overlay) if err != nil { return "", err } diff --git a/internal/shipcheck/shipcheck.go b/internal/shipcheck/shipcheck.go index d6194d3bf..d3dbf5cab 100644 --- a/internal/shipcheck/shipcheck.go +++ b/internal/shipcheck/shipcheck.go @@ -119,10 +119,9 @@ func (r *Report) Err() error { return nil } -// CosmosBoostWarning is the warning for the entries at Cosmos Boost sidecar -// paths a declared Dockerfile's context carries - old sidecars an earlier -// Astro CLI wrote, links, directories (imagebuild.ProjectSurvey.CosmosBoost) - -// or "" when there are none. +// CosmosBoostWarning is the warning for the Cosmos Boost files an earlier +// Astro CLI wrote into the project that a declared Dockerfile's context +// carries (imagebuild.ProjectSurvey.CosmosBoost), or "" when there are none. // A generated image leaves them out; a declared Dockerfile decides for itself // what it copies, so the build goes ahead, but the plugin would read any it // copies as current. @@ -130,7 +129,7 @@ func CosmosBoostWarning(survey *imagebuild.ProjectSurvey) string { if len(survey.CosmosBoost) == 0 { return "" } - return fmt.Sprintf("the build context carries old Cosmos Boost sidecars, written into the project by an earlier Astro CLI or linked there: %s. The Cosmos Boost plugin would read them as current if the Dockerfile copies them into the image. This CLI no longer writes them; delete them", + return fmt.Sprintf("the build context carries Cosmos Boost files an earlier Astro CLI wrote into the project: %s. The Cosmos Boost plugin would read them as current if the Dockerfile copies them into the image. This CLI no longer writes them; delete them", cappedList(survey.CosmosBoost)) } diff --git a/internal/shipcheck/shipcheck_test.go b/internal/shipcheck/shipcheck_test.go index c337fbf1b..07ce2d55a 100644 --- a/internal/shipcheck/shipcheck_test.go +++ b/internal/shipcheck/shipcheck_test.go @@ -361,7 +361,7 @@ func TestSurveyWarnsAboutStaleCosmosBoostFilesInADeclaredContext(t *testing.T) { _, err := Survey(dir, "Dockerfile", false, func(w string) { warned = append(warned, w) }) require.NoError(t, err) all := fmt.Sprint(warned) - assert.Contains(t, all, "old Cosmos Boost sidecars, written into the project by an earlier Astro CLI or linked there: dbt/shop/.astro/dbt_metadata.json, dbt/shop/.astro/manifest.slim.json.") + assert.Contains(t, all, "Cosmos Boost files an earlier Astro CLI wrote into the project: dbt/shop/.astro/dbt_metadata.json, dbt/shop/.astro/manifest.slim.json.") assert.Contains(t, all, "delete them") assert.NotContains(t, all, "dbt/other") diff --git a/pkg/airflowrt/cosmosboost.go b/pkg/airflowrt/cosmosboost.go index 062cee81c..495211df2 100644 --- a/pkg/airflowrt/cosmosboost.go +++ b/pkg/airflowrt/cosmosboost.go @@ -1,7 +1,6 @@ package airflowrt import ( - "bytes" "encoding/json" "path" ) @@ -42,61 +41,25 @@ func IsCosmosBoostFile(rel string) bool { // file at rel (one IsCosmosBoostFile accepts), carries an Astro CLI's // producer marker: generated_by.application in a dbt_metadata.json, // _generated_by.application in a manifest.slim.json. Each file is judged by -// its own marker, never by its neighbor's, and one that does not parse or -// names another producer is not the CLI's: it is the user's. -// -// It reads only as far as the marker: a slim manifest can be large, and the -// CLI writes the marker first. +// its own marker, never by its neighbor's, and one that does not parse as a +// JSON object or names another producer is not the CLI's: it is the user's. func CosmosBoostFileWrittenByAstro(rel string, data []byte) bool { - var key string - switch path.Base(rel) { - case CosmosBoostSidecar: - key = "generated_by" - case CosmosBoostSlimManifest: - key = "_generated_by" - default: - return false + var marker struct { + GeneratedBy struct { + Application string `json:"application"` + } `json:"generated_by"` + SlimGeneratedBy struct { + Application string `json:"application"` + } `json:"_generated_by"` } - dec := json.NewDecoder(bytes.NewReader(data)) - if tok, err := dec.Token(); err != nil || tok != json.Delim('{') { + if json.Unmarshal(data, &marker) != nil { return false } - for dec.More() { - name, err := dec.Token() - if err != nil { - return false - } - if name != key { - if skipJSONValue(dec) != nil { - return false - } - continue - } - var marker struct { - Application string `json:"application"` - } - return dec.Decode(&marker) == nil && marker.Application == CosmosBoostProducer + switch path.Base(rel) { + case CosmosBoostSidecar: + return marker.GeneratedBy.Application == CosmosBoostProducer + case CosmosBoostSlimManifest: + return marker.SlimGeneratedBy.Application == CosmosBoostProducer } return false } - -// skipJSONValue reads past the next value in dec, token by token, without -// holding it. -func skipJSONValue(dec *json.Decoder) error { - depth := 0 - for { - tok, err := dec.Token() - if err != nil { - return err - } - switch tok { - case json.Delim('{'), json.Delim('['): - depth++ - case json.Delim('}'), json.Delim(']'): - depth-- - } - if depth == 0 { - return nil - } - } -} diff --git a/pkg/airflowrt/cosmosboost_test.go b/pkg/airflowrt/cosmosboost_test.go index 9fde1eb49..16fb12eec 100644 --- a/pkg/airflowrt/cosmosboost_test.go +++ b/pkg/airflowrt/cosmosboost_test.go @@ -40,13 +40,15 @@ func TestCosmosBoostFileWrittenByAstro(t *testing.T) { {slim, `{"generated_by":{"application":"astro"}}`, false}, {slim, `{"_generated_by":{"application":"someone-else"}}`, false}, {".astro/config.yaml", `{"generated_by":{"application":"astro"}}`, false}, - // The marker after other keys, nested ones skipped whole. - {sidecar, `{"schema":1,"version":{"algo":"x","hash":"h","n":[1,{"a":[]}]},"generated_by":{"application":"astro"}}`, true}, - {slim, `{"nodes":{"a":{"generated_by":{"application":"astro"}}},"_generated_by":{"application":"astro"}}`, true}, + // The whole document is read: the marker after other keys, the last + // of a repeated key, and nothing below the top level. + {sidecar, `{"schema":1,"version":{"algo":"x","hash":"h"},"generated_by":{"application":"astro"}}`, true}, + {sidecar, `{"generated_by":{"application":"astro"},"generated_by":{"application":"someone-else"}}`, false}, {slim, `{"nodes":{"a":{"_generated_by":{"application":"astro"}}}}`, false}, + // Malformed is not ours. {sidecar, `["generated_by",{"application":"astro"}]`, false}, {sidecar, `{"generated_by":"astro"}`, false}, - {sidecar, `{"schema":1,`, false}, + {sidecar, `{"generated_by":{"application":"astro"}},`, false}, } { assert.Equal(t, tc.want, CosmosBoostFileWrittenByAstro(tc.rel, []byte(tc.data)), "%s %s", tc.rel, tc.data) } diff --git a/pkg/cosmosboost/precompute/metadata.go b/pkg/cosmosboost/precompute/metadata.go index 54ed58499..296512c33 100644 --- a/pkg/cosmosboost/precompute/metadata.go +++ b/pkg/cosmosboost/precompute/metadata.go @@ -92,20 +92,14 @@ func sidecarArtifact(dir, algo, hash, version string, filtered *FilteredManifest return Artifact{Path: filepath.Join(dir, sidecarDir, sidecarName), Data: data} } -// checkArtifactPaths fails when the tree holds something an artifact could -// not cleanly replace in a bundle: a .astro under dir that is not a directory -// (a file, or a link), or a directory where an artifact goes, whose contents -// would be unpacked beside it. Anything else there - an old sidecar, a link - -// the bundle leaves out and carries the artifact in its place. -func checkArtifactPaths(dir string, artifacts []Artifact) error { +// checkSidecarDir fails when something other than a directory holds dir's +// .astro (a file, or a link): a bundle cannot carry the artifacts under it. +// Whatever is at an artifact's own path - an old sidecar, a link, a +// directory - the bundle leaves out and carries the artifact in its place. +func checkSidecarDir(dir string) error { astroDir := filepath.Join(dir, sidecarDir) if info, err := os.Lstat(astroDir); err == nil && !info.IsDir() { return fmt.Errorf("%s is not a directory, so its Cosmos Boost files cannot sit under it", astroDir) } - for _, a := range artifacts { - if info, err := os.Lstat(a.Path); err == nil && info.IsDir() { - return fmt.Errorf("%s is a directory, so the Cosmos Boost file cannot take its place", a.Path) - } - } return nil } diff --git a/pkg/cosmosboost/precompute/precompute.go b/pkg/cosmosboost/precompute/precompute.go index ca348bd06..58c2f4a2f 100644 --- a/pkg/cosmosboost/precompute/precompute.go +++ b/pkg/cosmosboost/precompute/precompute.go @@ -59,23 +59,17 @@ type Options struct { SlimManifest bool } -// Units are what a run processes: every dbt project (a directory with -// dbt_project.yml) and standalone manifest.json under its roots, in a fixed -// order. Discover finds them; Units.Run processes them. -type Units struct { - units []unit -} - type unit struct{ kind, path string } -// Discover finds the units under roots. It fails only when a root cannot be -// walked. -func Discover(roots []string) (Units, error) { +// discover finds every dbt project (a directory with dbt_project.yml) and +// standalone manifest.json under roots, in a fixed order. It fails only when a +// root cannot be walked. +func discover(roots []string) ([]unit, error) { projectDirs := map[string]bool{} for _, root := range roots { found, err := findProjects(root) if err != nil { - return Units{}, fmt.Errorf("scanning %q for dbt projects: %w", root, err) + return nil, fmt.Errorf("scanning %q for dbt projects: %w", root, err) } for _, d := range found { projectDirs[d] = true @@ -86,7 +80,7 @@ func Discover(roots []string) (Units, error) { for _, root := range roots { found, err := findManifests(root, projectDirs) if err != nil { - return Units{}, fmt.Errorf("scanning %q for manifests: %w", root, err) + return nil, fmt.Errorf("scanning %q for manifests: %w", root, err) } for _, m := range found { manifests[m] = true @@ -105,42 +99,33 @@ func Discover(roots []string) (Units, error) { sort.Slice(units, func(i, j int) bool { return units[i].path+"\x00"+units[i].kind < units[j].path+"\x00"+units[j].kind }) - return Units{units: units}, nil -} - -// SidecarDirs are the .astro directories a run of u produces its artifacts -// in, one per unit, without reading any unit: a manifest.json that turns out -// not to be a dbt manifest is listed too, though nothing is produced for it. -func (u Units) SidecarDirs() []string { - dirs := make([]string, 0, len(u.units)) - for _, un := range u.units { - dir := un.path - if un.kind == kindManifest { - dir = filepath.Dir(un.path) - } - dirs = append(dirs, filepath.Join(dir, sidecarDir)) - } - return dirs + return units, nil } -// Run produces a .astro/dbt_metadata.json hash sidecar for each of u's units, -// in memory (Result.Artifacts): nothing is written. Units are processed +// Run finds every dbt project (a directory with dbt_project.yml) and +// standalone dbt manifest.json under the given roots, and produces a +// .astro/dbt_metadata.json hash sidecar for each, in memory +// (Result.Artifacts): nothing under the roots is written. Units are processed // concurrently — one worker each, bounded by GOMAXPROCS — and each is hashed // over sorted input, so results are deterministic with no cross-worker // coordination. // // Per-unit failures are best-effort: a unit that fails is recorded in its -// Result and does not stop the others. version is recorded in each sidecar's -// generated_by. +// Result and does not stop the others. Run only returns a non-nil error for a +// top-level problem, such as a root that cannot be walked. version is +// recorded in each sidecar's generated_by. // // With opts.SlimManifest set, every dbt manifest.json also gets a slim, // field-filtered copy (see buildSlimManifest) for the .astro/ beside it, for // the Cosmos Boost plugin to load in place of the full manifest at DAG-parse // time. That includes one in a project's own root, which is not a discovery // unit of its own and is handled by processProject. -func (u Units) Run(version string, opts Options) Summary { +func Run(roots []string, version string, opts Options) (Summary, error) { start := time.Now() - units := u.units + units, err := discover(roots) + if err != nil { + return Summary{}, err + } results := make([]Result, len(units)) sem := make(chan struct{}, max(1, runtime.GOMAXPROCS(0))) var wg sync.WaitGroup @@ -159,7 +144,7 @@ func (u Units) Run(version string, opts Options) Summary { } wg.Wait() - return Summary{Duration: time.Since(start), Results: results} + return Summary{Duration: time.Since(start), Results: results}, nil } // processProject hashes one dbt project directory and produces its sidecar. It reads @@ -196,7 +181,7 @@ func processProject(dir, version string, opts Options) Result { } r.Artifacts = append(r.Artifacts, sidecarArtifact(dir, algoProjectTree, hash, version, filtered)) - if r.Err = checkArtifactPaths(dir, r.Artifacts); r.Err != nil { + if r.Err = checkSidecarDir(dir); r.Err != nil { r.Artifacts = nil } r.Duration = time.Since(start) @@ -236,7 +221,7 @@ func processManifest(path, version string, opts Options) Result { r.Artifacts = append(r.Artifacts, slim) } r.Artifacts = append(r.Artifacts, sidecarArtifact(dir, algoManifestJSON, hash, version, filtered)) - if r.Err = checkArtifactPaths(dir, r.Artifacts); r.Err != nil { + if r.Err = checkSidecarDir(dir); r.Err != nil { r.Artifacts = nil } } diff --git a/pkg/cosmosboost/precompute/precompute_test.go b/pkg/cosmosboost/precompute/precompute_test.go index 5e0bc1e06..5b5c7c185 100644 --- a/pkg/cosmosboost/precompute/precompute_test.go +++ b/pkg/cosmosboost/precompute/precompute_test.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "runtime" - "slices" "strings" "testing" "time" @@ -737,10 +736,10 @@ func TestRunFailsUnitWhenAstroIsAFile(t *testing.T) { } } -// TestRunFailsUnitWhenArtifactPathIsNotAFile: a directory where a sidecar -// goes would collide with it in the bundle, so that unit fails and produces -// nothing, while a regular file there (an old sidecar) is fine. -func TestRunFailsUnitWhenArtifactPathIsNotAFile(t *testing.T) { +// TestRunReplacesWhateverIsAtAnArtifactPath: an old sidecar, or a directory, +// where a sidecar goes is left out of the bundle and the fresh sidecar takes +// its place, so the unit succeeds. +func TestRunReplacesWhateverIsAtAnArtifactPath(t *testing.T) { root := t.TempDir() writeFiles(t, root, map[string]string{ "blocked/dbt_project.yml": "name: blocked\n", @@ -753,10 +752,10 @@ func TestRunFailsUnitWhenArtifactPathIsNotAFile(t *testing.T) { if err != nil { t.Fatal(err) } - if summary.CountFailed() != 1 { - t.Fatalf("want one failed unit, got %+v", summary.Results) + if summary.CountFailed() != 0 { + t.Fatalf("want no failures, got %+v", summary.Results) } - mustNotProduce(t, summary, filepath.Join(root, "blocked", sidecarDir, sidecarName)) + mustProduce(t, summary, filepath.Join(root, "blocked", sidecarDir, sidecarName)) mustProduce(t, summary, filepath.Join(root, "fine", sidecarDir, sidecarName)) } @@ -783,29 +782,6 @@ func TestRunReplacesALinkAtAnArtifactPath(t *testing.T) { mustProduce(t, summary, filepath.Join(root, "proj", sidecarDir, sidecarName)) } -// TestUnitsSidecarDirs: one .astro per unit, beside a project or a manifest, -// known before any unit is read. -func TestUnitsSidecarDirs(t *testing.T) { - root := t.TempDir() - writeFiles(t, root, map[string]string{ - "proj/dbt_project.yml": "name: p\n", - "proj/target/manifest.json": "not even json", - "shipped/manifest.json": "{}", - }) - units, err := Discover([]string{root}) - if err != nil { - t.Fatal(err) - } - want := []string{ - filepath.Join(root, "proj", sidecarDir), - filepath.Join(root, "proj", "target", sidecarDir), - filepath.Join(root, "shipped", sidecarDir), - } - if got := units.SidecarDirs(); !slices.Equal(got, want) { - t.Fatalf("SidecarDirs = %v, want %v", got, want) - } -} - // treeSnapshot maps every file under root to its contents. func treeSnapshot(t *testing.T, root string) map[string]string { t.Helper() diff --git a/pkg/cosmosboost/precompute/run_test.go b/pkg/cosmosboost/precompute/run_test.go deleted file mode 100644 index f362cb3d6..000000000 --- a/pkg/cosmosboost/precompute/run_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package precompute - -import "time" - -// Run discovers the units under roots and runs them, as a deploy does in two -// steps, timing both. -func Run(roots []string, version string, opts Options) (Summary, error) { - start := time.Now() - units, err := Discover(roots) - if err != nil { - return Summary{}, err - } - summary := units.Run(version, opts) - summary.Duration = time.Since(start) - return summary, nil -} diff --git a/pkg/cosmosboost/predeploy.go b/pkg/cosmosboost/predeploy.go index 21eec78d4..0bf2309a2 100644 --- a/pkg/cosmosboost/predeploy.go +++ b/pkg/cosmosboost/predeploy.go @@ -37,73 +37,67 @@ func slimManifestEnabled() bool { // // Whatever the setting says, the tarball's own walk (Skip) leaves out every // .astro/dbt_metadata.json and .astro/manifest.slim.json that an earlier -// Astro CLI wrote into the tree (its own producer marker says so, -// airflowrt.CosmosBoostFileWrittenByAstro), so the files left out are exactly -// ones the tarball would have carried; and anything at such a path that is -// not a regular file, with a warning. The plugin cannot tell a fresh sidecar -// from a stale one, and this CLI never writes there. One from another -// producer is the user's file and ships untouched. +// Astro CLI wrote into the tree as a regular file (its own producer marker +// says so, airflowrt.CosmosBoostFileWrittenByAstro), so the files left out +// are exactly ones the tarball would have carried. The plugin cannot tell a +// fresh sidecar from a stale one, and this CLI never writes there. Anything +// else at such a path - another producer's file, a link - is the user's and +// ships untouched. // // With the setting on (cosmos_boost.pre_deploy), the pre-deploy step runs // over root and its sidecars are added at the same paths, with the same -// contents, the plugin has always read: every dbt project (dbt_project.yml) -// gets a .astro/dbt_metadata.json carrying its content hash, which the plugin -// uses as a cache version key at parse time instead of hashing the project -// tree itself, and every standalone dbt manifest.json gets a hash sidecar too, -// plus a slim, field-filtered copy for the plugin to load in place of the full -// manifest. That step is best-effort - a failure is a warning, never an error -// - because without a sidecar the plugin simply falls back to hashing at parse -// time. Before it hashes anything, it refuses the deploy when another -// producer's file sits where it would put a sidecar: the plugin reads that -// directory, and this CLI will not judge a file it did not write fresh. -// -// A root that cannot be read gets no sidecars, and is left for the tarball to -// deal with as it always has. +// contents, the plugin has always read, in place of whatever is there: every +// dbt project (dbt_project.yml) gets a .astro/dbt_metadata.json carrying its +// content hash, which the plugin uses as a cache version key at parse time +// instead of hashing the project tree itself, and every standalone dbt +// manifest.json gets a hash sidecar too, plus a slim, field-filtered copy for +// the plugin to load in place of the full manifest. That step is best-effort +// - a failure is a warning, never an error - because without a sidecar the +// plugin simply falls back to hashing at parse time. The one thing it refuses +// is to replace another producer's file: the plugin reads that path, and this +// CLI will not judge a file it did not write stale. func BundleOverlay(root string, enabled bool) (fileutil.TarOverlay, error) { - overlay := fileutil.TarOverlay{Skip: skipOldSidecar, Extra: map[string][]byte{}} + extra := map[string][]byte{} + overlay := fileutil.TarOverlay{Extra: extra, Skip: func(rel, file string, info os.FileInfo) (bool, error) { + if _, replaced := extra[rel]; replaced { + return true, nil + } + return oldSidecar(rel, file, info) + }} if !enabled { return overlay, nil } if _, err := os.Stat(root); err != nil { return overlay, nil //nolint:nilerr // the tarball reports a root it cannot read, as it always has } - units, err := precompute.Discover([]string{root}) - if err != nil { - fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) - return overlay, nil - } - if err := refuseForeignSidecars(units.SidecarDirs()); err != nil { + + artifacts, stepErr := preDeploy(root) + if err := refuseToReplaceForeign(artifacts); err != nil { return overlay, err } - - artifacts, err := preDeploy(units) for _, a := range artifacts { - rel, relErr := filepath.Rel(root, a.Path) - if relErr != nil { - err = errors.Join(err, relErr) + rel, err := filepath.Rel(root, a.Path) + if err != nil { + stepErr = errors.Join(stepErr, err) continue } - overlay.Extra[filepath.ToSlash(rel)] = a.Data + extra[filepath.ToSlash(rel)] = a.Data } - if err != nil { - fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", err) + if stepErr != nil { + fmt.Printf("Warning: the Cosmos Boost pre-deploy step failed, continuing deploy: %s\n", stepErr) return overlay, nil } fmt.Println("Cosmos Boost pre-deploy step complete") return overlay, nil } -// skipOldSidecar is the overlay's Skip: it leaves out each Cosmos Boost file -// an Astro CLI wrote into the tree, and anything at a sidecar's path that is -// not a regular file. Only a regular file at a sidecar's path is read. -func skipOldSidecar(rel, file string, info os.FileInfo) (bool, error) { - if !airflowrt.IsCosmosBoostFile(rel) { +// oldSidecar reports whether the entry at rel is a Cosmos Boost file an +// earlier Astro CLI wrote into the tree: a regular file at a sidecar's path +// whose own producer marker names this CLI. Only such a file is read. +func oldSidecar(rel, file string, info os.FileInfo) (bool, error) { + if !info.Mode().IsRegular() || !airflowrt.IsCosmosBoostFile(rel) { return false, nil } - if !info.Mode().IsRegular() { - fmt.Printf("Warning: leaving %s out of the bundle: Cosmos Boost keeps a sidecar file at that path, and this is not a regular file\n", file) - return true, nil - } data, err := os.ReadFile(file) if err != nil { return false, err @@ -115,25 +109,21 @@ func skipOldSidecar(rel, file string, info os.FileInfo) (bool, error) { return false, nil } -// refuseForeignSidecars fails when a regular file another producer wrote sits -// where the pre-deploy step puts a sidecar, in one of dirs. Only those few -// paths are read. -func refuseForeignSidecars(dirs []string) error { +// refuseToReplaceForeign fails when a regular file another producer wrote +// sits where one of artifacts goes. Only those paths are read. +func refuseToReplaceForeign(artifacts []precompute.Artifact) error { var foreign []string - for _, dir := range dirs { - for _, name := range []string{airflowrt.CosmosBoostSidecar, airflowrt.CosmosBoostSlimManifest} { - file := filepath.Join(dir, name) - info, err := os.Lstat(file) - if err != nil || !info.Mode().IsRegular() { - continue // absent, or left out by skipOldSidecar - } - data, err := os.ReadFile(file) - if err != nil { - return err - } - if !airflowrt.CosmosBoostFileWrittenByAstro(name, data) { - foreign = append(foreign, file) - } + for _, a := range artifacts { + info, err := os.Lstat(a.Path) + if err != nil || !info.Mode().IsRegular() { + continue // absent, or a link or directory the artifact replaces + } + data, err := os.ReadFile(a.Path) + if err != nil { + return err + } + if !airflowrt.CosmosBoostFileWrittenByAstro(filepath.ToSlash(a.Path), data) { + foreign = append(foreign, a.Path) } } if len(foreign) > 0 { @@ -142,12 +132,15 @@ func refuseForeignSidecars(dirs []string) error { return nil } -// preDeploy runs the Cosmos Boost pre-deploy step over units and returns the +// preDeploy runs the Cosmos Boost pre-deploy step over path and returns the // artifacts it produced, in memory. It fails when any unit failed, still // returning the artifacts of the ones that did not. -func preDeploy(units precompute.Units) ([]precompute.Artifact, error) { +func preDeploy(path string) ([]precompute.Artifact, error) { opts := precompute.Options{SlimManifest: slimManifestEnabled()} - summary := units.Run(version.Current(), opts) + summary, err := precompute.Run([]string{path}, version.Current(), opts) + if err != nil { + return nil, fmt.Errorf("running the Cosmos Boost pre-deploy step: %w", err) + } debugReport("pre-deploy", summary) if failed := summary.CountFailed(); failed > 0 { return summary.Artifacts(), fmt.Errorf("the Cosmos Boost pre-deploy step failed for %d unit(s) (re-run with --verbosity debug for details)", failed) diff --git a/pkg/cosmosboost/predeploy_test.go b/pkg/cosmosboost/predeploy_test.go index 545317fe4..2bc9f4b23 100644 --- a/pkg/cosmosboost/predeploy_test.go +++ b/pkg/cosmosboost/predeploy_test.go @@ -2,6 +2,7 @@ package cosmosboost import ( "archive/tar" + "compress/gzip" "encoding/json" "errors" "io" @@ -76,15 +77,17 @@ func bundle(t *testing.T, dir string, enabled bool) (map[string]string, error) { if err != nil { return nil, err } - target := filepath.Join(t.TempDir(), "bundle.tar") - if err := fileutil.Tar(dir, target, false, []string{".git/"}, overlay); err != nil { + target := filepath.Join(t.TempDir(), "bundle.tar.gz") + if err := fileutil.TarGz(dir, target, false, []string{".git/"}, overlay); err != nil { return nil, err } file, err := os.Open(target) require.NoError(t, err) defer file.Close() + gz, err := gzip.NewReader(file) + require.NoError(t, err) entries := map[string]string{} - tr := tar.NewReader(file) + tr := tar.NewReader(gz) for { header, err := tr.Next() if err == io.EOF { @@ -115,9 +118,8 @@ func TestBundleEnabledAddsSidecars(t *testing.T) { require.ElementsMatch(t, []string{"dbt_project.yml", "models/a.sql", "target/manifest.json", sidecarRel, "target/" + sidecarRel, "target/" + slimRel}, keys(entries)) // The contents are exactly what the pre-deploy step produces for the tree. - units, err := precompute.Discover([]string{dir}) + summary, err := precompute.Run([]string{dir}, "1.2.3", precompute.Options{SlimManifest: true}) require.NoError(t, err) - summary := units.Run("1.2.3", precompute.Options{SlimManifest: true}) for _, a := range summary.Artifacts() { rel, err := filepath.Rel(dir, a.Path) require.NoError(t, err) @@ -215,12 +217,11 @@ func TestBundleLeavesOutStaleSidecars(t *testing.T) { } // TestBundleForeignSidecar: a file from an unrecognized producer is the -// user's. Disabled, it ships untouched. Enabled, where the step would put a -// sidecar the plugin reads - a discovered project's or manifest's .astro/ - -// the deploy stops before anything is hashed and asks for its removal; the -// replacement would hide it, and the plugin would have read it. +// user's. Disabled, it ships untouched. Enabled, where the step puts a fresh +// sidecar, the deploy stops and asks for its removal, before saying the step +// is complete: the plugin reads that path, and the replacement would hide it. func TestBundleForeignSidecar(t *testing.T) { - for _, rel := range []string{sidecarRel, slimRel, "target/" + sidecarRel} { + for _, rel := range []string{sidecarRel, "target/" + sidecarRel, "target/" + slimRel} { dir := t.TempDir() writeDbtProject(t, dir) writeFiles(t, dir, map[string]string{"target/manifest.json": dbtManifest, rel: foreignFile}) @@ -235,35 +236,45 @@ func TestBundleForeignSidecar(t *testing.T) { require.ErrorContains(t, err, "unrecognized producer") require.ErrorContains(t, err, foreign) require.ErrorContains(t, err, "Remove them and retry") - require.NotContains(t, out, "pre-deploy step complete", "the refusal comes before the step") + require.NotContains(t, out, "pre-deploy step complete", "the refusal comes before the step says it is done") require.FileExists(t, foreign, "a file another tool wrote must never be deleted") } } -// TestBundleShipsForeignSidecarsTheStepDoesNotWrite: one where no sidecar -// would go - an installed package's own, under dbt_packages/ - is the -// project's file, and an enabled deploy ships it. -func TestBundleShipsForeignSidecarsTheStepDoesNotWrite(t *testing.T) { +// TestBundleShipsForeignSidecarsTheStepDoesNotReplace: where the step writes +// nothing - an installed package's own sidecar under dbt_packages/, logs/, +// beside a manifest.json that is not dbt's, a slim manifest with the slim +// optimization off - another producer's file is the project's, and an enabled +// deploy ships it. +func TestBundleShipsForeignSidecarsTheStepDoesNotReplace(t *testing.T) { + t.Setenv(slimManifestEnvVar, "false") dir := t.TempDir() writeDbtProject(t, dir) + shipped := map[string]string{ + "dbt_packages/utils/" + sidecarRel: foreignFile, + "logs/" + sidecarRel: foreignFile, + "webapp/" + sidecarRel: foreignFile, + slimRel: foreignFile, + } + writeFiles(t, dir, shipped) writeFiles(t, dir, map[string]string{ "dbt_packages/utils/dbt_project.yml": "name: utils\n", - "dbt_packages/utils/" + sidecarRel: foreignFile, - "logs/" + sidecarRel: foreignFile, + "webapp/manifest.json": `{"name":"My App"}`, }) entries, err := bundle(t, dir, true) require.NoError(t, err) - require.Equal(t, foreignFile, entries["dbt_packages/utils/"+sidecarRel]) - require.Equal(t, foreignFile, entries["logs/"+sidecarRel]) + for rel, content := range shipped { + require.Equal(t, content, entries[rel], rel) + } require.Contains(t, entries, sidecarRel, "the project itself is stamped") } -// TestBundleLeavesOutNonRegularEntriesAtSidecarPaths: a link or a directory -// where a sidecar goes never ships, with a warning naming it. Enabled, a link -// is replaced by the fresh sidecar; a directory cannot be, so that unit fails -// (a warning) and nothing is shipped there. -func TestBundleLeavesOutNonRegularEntriesAtSidecarPaths(t *testing.T) { +// TestBundleLinksAndDirectoriesAtSidecarPaths: disabled, a link at a +// sidecar's path is the user's and ships as it is. Enabled, a link or a +// directory where the step puts a sidecar is left out and the fresh sidecar +// takes its place. +func TestBundleLinksAndDirectoriesAtSidecarPaths(t *testing.T) { for _, enabled := range []bool{false, true} { dir := t.TempDir() writeDbtProject(t, filepath.Join(dir, "linked")) @@ -278,20 +289,18 @@ func TestBundleLeavesOutNonRegularEntriesAtSidecarPaths(t *testing.T) { t.Skipf("cannot create a symlink here: %v", err) } - var entries map[string]string - var err error - out := captureStdout(t, func() { entries, err = bundle(t, dir, enabled) }) + entries, err := bundle(t, dir, enabled) require.NoError(t, err, "enabled=%v", enabled) - require.Contains(t, out, "leaving "+link+" out of the bundle", "enabled=%v", enabled) - require.Contains(t, out, "leaving "+filepath.Join(dir, "blocked", ".astro", "dbt_metadata.json")+" out of the bundle", "enabled=%v", enabled) - require.NotContains(t, entries, "blocked/"+sidecarRel+"/x", "enabled=%v", enabled) - require.NotContains(t, entries, "blocked/"+sidecarRel, "enabled=%v: a directory cannot be replaced", enabled) if enabled { - require.Contains(t, out, "pre-deploy step failed") - require.Contains(t, entries, "linked/"+sidecarRel) - require.NotContains(t, entries["linked/"+sidecarRel], "stale", "the fresh sidecar replaces the link") + require.NotContains(t, entries, "blocked/"+sidecarRel+"/x") + for _, rel := range []string{"linked/" + sidecarRel, "blocked/" + sidecarRel} { + require.Contains(t, entries, rel) + require.NotContains(t, entries[rel], "stale", "the fresh sidecar replaces it") + } } else { - require.NotContains(t, entries, "linked/"+sidecarRel) + require.Contains(t, entries, "linked/"+sidecarRel, "a link ships as it is") + require.Empty(t, entries["linked/"+sidecarRel], "as a link, not as what it points at") + require.Contains(t, entries, "blocked/"+sidecarRel+"/x") } } } @@ -351,14 +360,12 @@ func TestBundleStepFailureIsAWarning(t *testing.T) { require.NoError(t, os.Chmod(model, 0o000)) t.Cleanup(func() { _ = os.Chmod(model, 0o644) }) - overlay, err := BundleOverlay(dir, true) + var overlay fileutil.TarOverlay + var err error + out := captureStdout(t, func() { overlay, err = BundleOverlay(dir, true) }) require.NoError(t, err) + require.Contains(t, out, "pre-deploy step failed for 1 unit(s)") require.Equal(t, []string{"fine/" + sidecarRel}, keys(overlay.Extra)) - - units, err := precompute.Discover([]string{dir}) - require.NoError(t, err) - _, err = preDeploy(units) - require.ErrorContains(t, err, "failed for 1 unit(s)") } func keys[V any](m map[string]V) []string { diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index 99a4e86c6..a826cef36 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -116,8 +116,9 @@ type TarOverlay struct { // under it. An error fails the tarball. Skip func(rel, file string, info os.FileInfo) (bool, error) // Extra adds a regular file with these bytes at each path, in place of a - // file or link the walk finds there. A directory there is not replaced: - // callers must not add an extra where one exists. + // file or link the walk finds there. A directory there is still walked + // unless Skip leaves it out, and its entries would collide with the file: + // a caller adding an extra where one may exist skips it too. Extra map[string][]byte } @@ -125,31 +126,33 @@ type TarOverlay struct { // whatever unpacks it (the Airflow runtime user) must be able to read it. const extraFileMode = 0o644 -// Tar writes the directory source to the tarball target, its paths -// relative to source, or to source's parent with prependBaseDir, changed by -// overlay. A source that is not there makes an empty tarball, not an error. -func Tar(source, target string, prependBaseDir bool, excludePathPrefixes []string, overlay TarOverlay) error { +// TarGz writes the directory source to the gzip-compressed tarball target, +// in one pass, its paths relative to source, or to source's parent with +// prependBaseDir, changed by overlay. A source that is not there makes an +// empty tarball, not an error. +func TarGz(source, target string, prependBaseDir bool, excludePathPrefixes []string, overlay TarOverlay) error { baseDir := "" if prependBaseDir { baseDir = filepath.Base(source) } - return writeTar(source, target, baseDir, excludePathPrefixes, overlay, false) + return writeTar(source, target, baseDir, excludePathPrefixes, overlay, true, false) } // TarDir writes the directory source to the tarball target, each path in it // under baseDir: "dags" makes dags/my_dag.py of source/my_dag.py, whatever -// source is called. Unlike Tar it fails on a source that is not there, or +// source is called. Unlike TarGz it fails on a source that is not there, or // that goes while it is read, rather than writing a tarball short of it. func TarDir(source, target, baseDir string) error { - return writeTar(source, target, baseDir, nil, TarOverlay{}, true) + return writeTar(source, target, baseDir, nil, TarOverlay{}, false, true) } // createTarFile creates the tarball writeTar writes; a test fails its Close. var createTarFile = func(name string) (io.WriteCloser, error) { return os.Create(name) } -// writeTar writes the tarball. A tarball whose footer or file did not close -// is not one, so failing to close either fails it. -func writeTar(source, target, baseDir string, excludePathPrefixes []string, overlay TarOverlay, mustExist bool) (err error) { +// writeTar writes the tarball, gzip-compressed with gzipped. A tarball whose +// footer, compressed stream or file did not close is not one - a full disk +// shows up there - so failing to close any of them fails it. +func writeTar(source, target, baseDir string, excludePathPrefixes []string, overlay TarOverlay, gzipped, mustExist bool) (err error) { tarfile, err := createTarFile(target) if err != nil { return err @@ -160,8 +163,21 @@ func writeTar(source, target, baseDir string, excludePathPrefixes []string, over } }() - tarball := tar.NewWriter(tarfile) - // Closing writes the footer; deferred after the file's, it runs first. + var out io.Writer = tarfile + if gzipped { + gz := gzip.NewWriter(tarfile) + // Closing flushes what is still compressed in memory; deferred after + // the file's, it runs before it. + defer func() { + if cerr := gz.Close(); err == nil { + err = cerr + } + }() + out = gz + } + + tarball := tar.NewWriter(out) + // Closing writes the footer; deferred last, it runs first. defer func() { if cerr := tarball.Close(); err == nil { err = cerr diff --git a/pkg/fileutil/files_test.go b/pkg/fileutil/files_test.go index 3f391c822..bac92b4e2 100644 --- a/pkg/fileutil/files_test.go +++ b/pkg/fileutil/files_test.go @@ -241,7 +241,7 @@ func (s *Suite) TestTar() { for _, tt := range tests { s.Run(tt.name, func() { // check that the tar operation was successful - assert.True(s.T(), tt.errAssertion(s.T(), Tar(tt.args.source, tt.args.target, tt.args.prependBaseDir, tt.args.excludePaths, TarOverlay{}))) + assert.True(s.T(), tt.errAssertion(s.T(), TarGz(tt.args.source, tt.args.target, tt.args.prependBaseDir, tt.args.excludePaths, TarOverlay{}))) // check that all the files are in the tar at the correct paths file, err := os.Open(tt.args.target) @@ -249,7 +249,9 @@ func (s *Suite) TestTar() { s.Fail("Error opening file %s", tt.args.target) } defer file.Close() - tarReader := tar.NewReader(file) + gz, err := gzip.NewReader(file) + require.NoError(s.T(), err) + tarReader := tar.NewReader(gz) numIteratedFiles := 0 for { header, err := tarReader.Next() @@ -293,7 +295,7 @@ func (s *Suite) TestTarDir() { missing := filepath.Join(dir, "missing") s.ErrorIs(TarDir(missing, filepath.Join(dir, "missing.tar"), "dags"), os.ErrNotExist) - s.NoError(Tar(missing, filepath.Join(dir, "missing-tar.tar"), true, nil, TarOverlay{}), "Tar is unchanged") + s.NoError(TarGz(missing, filepath.Join(dir, "missing-tar.tar"), true, nil, TarOverlay{}), "Tar is unchanged") } // TarOverlay leaves Skip's files out, adds Extra's under the base directory @@ -324,12 +326,14 @@ func (s *Suite) TestTarOverlay() { prefix string }{{false, ""}, {true, "proj/"}} { target := filepath.Join(dir, fmt.Sprintf("overlay-%v.tar", tc.prepend)) - s.Require().NoError(Tar(source, target, tc.prepend, nil, overlay)) + s.Require().NoError(TarGz(source, target, tc.prepend, nil, overlay)) file, err := os.Open(target) s.Require().NoError(err) got := map[string]string{} - tr := tar.NewReader(file) + gz, err := gzip.NewReader(file) + s.Require().NoError(err) + tr := tar.NewReader(gz) for { header, err := tr.Next() if err == io.EOF { @@ -359,13 +363,13 @@ func (s *Suite) TestTarOverlay() { s.Run("source is the working directory", func() { s.T().Chdir(source) target := filepath.Join(dir, "dot.tar") - s.Require().NoError(Tar(".", target, false, nil, overlay)) + s.Require().NoError(TarGz(".", target, false, nil, overlay)) s.Equal([]string{".astro/replaced.json", "a.sql", "sub/.astro/added.json"}, tarNames(s.T(), target)) }) s.Run("a Skip error fails the tarball", func() { boom := errors.New("boom") - err := Tar(source, filepath.Join(dir, "boom.tar"), false, nil, TarOverlay{Skip: func(string, string, os.FileInfo) (bool, error) { return false, boom }}) + err := TarGz(source, filepath.Join(dir, "boom.tar"), false, nil, TarOverlay{Skip: func(string, string, os.FileInfo) (bool, error) { return false, boom }}) s.ErrorIs(err, boom) }) @@ -375,12 +379,38 @@ func (s *Suite) TestTarOverlay() { s.NoDirExists(filepath.Join(source, "sub")) } +// failingWrite is a tarball file that takes the first write - a gzip +// header - and fails every one after it, as a full disk does. +type failingWrite struct{ writes int } + +func (f *failingWrite) Write(p []byte) (int, error) { + f.writes++ + if f.writes > 1 { + return 0, errors.New("no space left on device") + } + return len(p), nil +} + +func (*failingWrite) Close() error { return nil } + +// A compressed stream that could not be flushed fails TarGz: gzip holds the +// data until it closes, so the write that fails is the close's. +func (s *Suite) TestTarGzFailsWhenTheCompressedStreamCannotBeWritten() { + prev := createTarFile + defer func() { createTarFile = prev }() + createTarFile = func(string) (io.WriteCloser, error) { return &failingWrite{}, nil } + source := s.T().TempDir() + s.Require().NoError(os.WriteFile(filepath.Join(source, "a.py"), []byte("a"), 0o600)) + + s.EqualError(TarGz(source, "bundle.tar.gz", false, nil, TarOverlay{}), "no space left on device") +} + // failingClose is a tarball file whose writes land and whose Close fails. type failingClose struct{ bytes.Buffer } func (*failingClose) Close() error { return errors.New("close failed") } -// A tarball that did not close is not one: TarDir and Tar both fail on it. +// A tarball that did not close is not one: TarDir and TarGz both fail on it. func (s *Suite) TestTarFailsOnAFailedClose() { prev := createTarFile defer func() { createTarFile = prev }() @@ -394,7 +424,7 @@ func (s *Suite) TestTarFailsOnAFailedClose() { s.EqualError(TarDir(source, "dags.tar", "dags"), "close failed") s.Positive(written.Len(), "the footer was flushed before the file was closed") - s.EqualError(Tar(source, "dags.tar", true, nil, TarOverlay{}), "close failed") + s.EqualError(TarGz(source, "dags.tar", true, nil, TarOverlay{}), "close failed") } func (s *Suite) TestReadFileToString() { @@ -994,7 +1024,9 @@ func tarNames(t *testing.T, target string) []string { require.NoError(t, err) defer file.Close() var names []string - tr := tar.NewReader(file) + gz, err := gzip.NewReader(file) + require.NoError(t, err) + tr := tar.NewReader(gz) for { header, err := tr.Next() if err == io.EOF { diff --git a/pkg/imagebuild/projectfiles.go b/pkg/imagebuild/projectfiles.go index 43f227ef7..b759b1d83 100644 --- a/pkg/imagebuild/projectfiles.go +++ b/pkg/imagebuild/projectfiles.go @@ -246,7 +246,7 @@ func (w walker) walkDir(dir, rel string, onPath map[string]bool) error { if err != nil { return err } - skip, descend := w.decide(childRel, e.Type(), info) + skip, descend := w.filter.skip(childRel, info.IsDir()) if w.vcs != nil && vcsNames[e.Name()] { if !skip { w.vcs(childRel) @@ -301,37 +301,6 @@ func (w walker) carry(rel, src string, mode fs.FileMode) error { return nil } -// decide is the filter's word on the entry at rel (projectFilter.skip), and -// oddSidecar's on one at a Cosmos Boost sidecar's path. entry is the -// directory entry's own type, info what the walk's stat says. -func (w walker) decide(rel string, entry fs.FileMode, info fs.FileInfo) (skip, descend bool) { - skip, descend = w.filter.skip(rel, info.IsDir()) - if !skip && w.oddSidecar(rel, entry, info.Mode()) { - return true, false - } - return skip, descend -} - -// oddSidecar handles a link, a directory or anything else that is not a -// plain regular file at a Cosmos Boost sidecar's path (airflowrt.IsCosmosBoostFile): -// the plugin would read whatever it leads to as a sidecar. A generated build -// leaves it out, which oddSidecar reports by returning true; a declared one -// carries it and reports it, as staleCosmosBoost does an old sidecar. entry -// is the directory entry's own type, mode what the walk's stat says. -func (w walker) oddSidecar(rel string, entry, mode fs.FileMode) bool { - if (!w.filter.generated && w.cosmosBoost == nil) || !airflowrt.IsCosmosBoostFile(rel) { - return false - } - if entry&fs.ModeSymlink == 0 && mode.IsRegular() { - return false - } - if w.filter.generated { - return true - } - w.cosmosBoost(rel) - return false -} - // staleCosmosBoost reports whether the regular file at src is a Cosmos Boost // sidecar an earlier Astro CLI wrote into the project, which the plugin would // read as current in the image (airflowrt.CosmosBoostFileWrittenByAstro). The @@ -383,11 +352,10 @@ type ProjectSurvey struct { // Dockerfile's context carries, which its ignore file leaves in: each // one's path, not what is in it. A generated build carries none. VCS []string - // CosmosBoost are the entries at a Cosmos Boost sidecar's path that a - // declared Dockerfile's context carries and the plugin would read as - // current in the image: the sidecars an earlier Astro CLI wrote into the - // project, and any link or directory there. A generated build leaves them - // out and lists none. + // CosmosBoost are the Cosmos Boost files an earlier Astro CLI wrote into + // the project that a declared Dockerfile's context carries, which the + // plugin would read as current in the image. A generated build leaves + // them out and lists none. CosmosBoost []string // IgnoreFile is the ignore file a declared Dockerfile's build reads // (declaredIgnoreFile), "" for none or for a generated build. diff --git a/pkg/imagebuild/projectsurvey_test.go b/pkg/imagebuild/projectsurvey_test.go index 71a84fb80..f73ee896b 100644 --- a/pkg/imagebuild/projectsurvey_test.go +++ b/pkg/imagebuild/projectsurvey_test.go @@ -84,36 +84,6 @@ func TestSurveyProjectDeclaredNamesStaleCosmosBoostFiles(t *testing.T) { assert.Equal(t, []string{"dbt/.astro/dbt_metadata.json"}, s.CosmosBoost) } -// A link or a directory at a sidecar's path is not a sidecar this CLI wrote, -// but the plugin would read whatever it leads to as one: a generated image -// leaves it out, and a declared context's survey names it. -func TestNonRegularEntriesAtSidecarPaths(t *testing.T) { - project := t.TempDir() - writeTree(t, project, map[string]string{ - "Dockerfile": "FROM x\n", - "dags/dbt/shop/dbt_project.yml": "name: shop\n", - "include/stale.json": `{"generated_by":{"application":"someone-else"}}`, - "dags/dbt/dir/.astro/dbt_metadata.json/x": "x", - }) - link := filepath.Join(project, "dags", "dbt", "shop", ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(link), 0o755)) - if err := os.Symlink(filepath.Join(project, "include", "stale.json"), link); err != nil { - t.Skipf("cannot create a symlink here: %v", err) - } - - s, err := SurveyProject(project, "", false) - require.NoError(t, err) - assert.ElementsMatch(t, []string{"Dockerfile", "dags/dbt/shop/dbt_project.yml", "include/stale.json"}, s.Files) - assert.Empty(t, s.CosmosBoost) - dst := t.TempDir() - require.NoError(t, copyProjectFiles(project, dst)) - assert.ElementsMatch(t, s.Files, contextFiles(t, dst), "the survey and the copy agree") - - s, err = SurveyProject(project, "Dockerfile", false) - require.NoError(t, err) - assert.ElementsMatch(t, []string{"dags/dbt/shop/.astro/dbt_metadata.json", "dags/dbt/dir/.astro/dbt_metadata.json"}, s.CosmosBoost) -} - // A .dockerignore that drops every DAG leaves none carried, which a deploy // that bakes DAGs into the image refuses. func TestSurveyProjectDagsDropped(t *testing.T) { From f71d5df2b908cf2d1b24bcd8ad8a757f69b779b5 Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 15:10:17 -0400 Subject: [PATCH 5/7] Deploy dbt projects and other non-DAG bundles with astro deploy bundle astro dbt deploy, astro dbt delete and astro deploy --non-dags did the same upload with different defaults, target selection and JSON. They become one subcommand, astro deploy bundle [PATH], with --delete for the removal. - A table of bundle types (internal/platform/astro/deploy/bundletype.go) gives each type its detection, check and default mount path. dbt is the one entry: a directory with dbt_project.yml, mounted at /usr/local/airflow/dbt/. Any other --type is a label that needs --mount-path. - One target path for deploy and delete: --deployment (a link of the project here, or an id), the picker on the current host only, and input_required when the run cannot be asked. The project's host login now reaches DeleteBundle too. - One JSON shape each for deploy and delete, pinned as deploy-bundle and deploy-bundle-delete; dbt-deploy, dbt-delete and deploy-non-dags go. - astro dbt is one hidden stub naming the replacement for each subcommand; --non-dags and its --non-dags-* flags join the removed-flag registry. - bundle is refused as a link name, and a deploy never auto-links a picked Deployment under it, since astro deploy bundle is the subcommand. Co-Authored-By: Claude --- cmd/astro/bundle_fixtures_test.go | 220 ++++++++ cmd/astro/dbt.go | 368 ++------------ cmd/astro/dbt_output_test.go | 471 ----------------- cmd/astro/dbt_render.go | 230 --------- cmd/astro/dbt_test.go | 312 ------------ cmd/astro/deploy.go | 219 +------- cmd/astro/deploy_bundle.go | 443 ++++++++++++++++ cmd/astro/deploy_bundle_test.go | 479 ++++++++++++++++++ cmd/astro/deploy_help.go | 13 +- cmd/astro/deploy_manifest_test.go | 92 ---- cmd/astro/deploy_test.go | 229 +-------- cmd/astro/deployment_token_json_test.go | 2 +- cmd/astro/output_binding_test.go | 20 +- cmd/astro/remote.go | 4 +- cmd/astro/remote_render.go | 76 +++ cmd/astro/root.go | 2 +- cmd/astro/schema_test.go | 19 +- cmd/astro/testdata/schema/dbt-deploy.json | 17 - ...-delete.json => deploy-bundle-delete.json} | 1 + ...eploy-non-dags.json => deploy-bundle.json} | 0 cmd/local/link.go | 7 +- cmd/local/link_test.go | 12 + cmd/removed_flags.go | 13 +- cmd/removed_flags_test.go | 4 + cmd/root.go | 2 +- docs/architecture.md | 2 +- docs/deploy.md | 19 +- docs/upgrading-from-v1.md | 8 +- internal/deploy/deploy.go | 18 +- internal/deploy/deploy_test.go | 3 +- internal/platform/astro/deploy/bundle.go | 5 +- internal/platform/astro/deploy/bundletype.go | 124 +++++ .../platform/astro/deploy/bundletype_test.go | 60 +++ .../astro/deploy/cosmosboost_hook_test.go | 6 +- 34 files changed, 1568 insertions(+), 1932 deletions(-) create mode 100644 cmd/astro/bundle_fixtures_test.go delete mode 100644 cmd/astro/dbt_output_test.go delete mode 100644 cmd/astro/dbt_render.go delete mode 100644 cmd/astro/dbt_test.go create mode 100644 cmd/astro/deploy_bundle.go create mode 100644 cmd/astro/deploy_bundle_test.go create mode 100644 cmd/astro/remote_render.go delete mode 100644 cmd/astro/testdata/schema/dbt-deploy.json rename cmd/astro/testdata/schema/{dbt-delete.json => deploy-bundle-delete.json} (86%) rename cmd/astro/testdata/schema/{deploy-non-dags.json => deploy-bundle.json} (100%) create mode 100644 internal/platform/astro/deploy/bundletype.go create mode 100644 internal/platform/astro/deploy/bundletype_test.go diff --git a/cmd/astro/bundle_fixtures_test.go b/cmd/astro/bundle_fixtures_test.go new file mode 100644 index 000000000..5dd00f2fe --- /dev/null +++ b/cmd/astro/bundle_fixtures_test.go @@ -0,0 +1,220 @@ +package astro + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/config" + "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" + astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" +) + +const ( + dbtTestDeploymentID = "clxdbtdeployment000000000" + dbtTestDeployID = "clxdbtdeploy0000000000000" + dbtTestVersion = "2026-10-07T00:00:00.0000000Z" +) + +// fakeBlobStore answers the bundle upload the way Azure does: every request +// succeeds, and the commit reports the version the upload created. +func fakeBlobStore(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("x-ms-version-id", dbtTestVersion) + w.WriteHeader(http.StatusCreated) + })) + t.Cleanup(srv.Close) + return srv.URL + "/container/bundle.tar.gz?sig=x" +} + +// dbtProject writes a dbt project named jaffle_shop in a fresh directory outside +// any Astro project, and makes a second fresh directory the working path, so +// the bundle's tarball is written beside it rather than inside it. +func dbtProject(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte("name: jaffle_shop\n"), 0o600)) + prev := config.WorkingPath + config.WorkingPath = t.TempDir() + t.Cleanup(func() { config.WorkingPath = prev }) + return dir +} + +func dbtTestDeployment() astrov1.Deployment { + return astrov1.Deployment{ + Id: dbtTestDeploymentID, + Name: "analytics", + WorkspaceId: "clxdbtworkspace0000000000", + IsDagDeployEnabled: true, + Status: astrov1.DeploymentStatusHEALTHY, + } +} + +// dbtBundleMock is the API a bundle deploy or delete talks to. +func dbtBundleMock(t *testing.T, uploadURL string) *astrov1_mocks.ClientWithResponsesInterface { + t.Helper() + m := new(astrov1_mocks.ClientWithResponsesInterface) + t.Cleanup(func() { m.AssertExpectations(t) }) + d := dbtTestDeployment() + m.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID).Return(&astrov1.GetDeploymentResponse{ + HTTPResponse: &http.Response{StatusCode: http.StatusOK}, + JSON200: &d, + }, nil).Maybe() + deploy := astrov1.Deploy{Id: dbtTestDeployID} + if uploadURL != "" { + deploy.BundleUploadUrl = &uploadURL + } + m.On("CreateDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, mock.Anything).Return(&astrov1.CreateDeployResponse{ + HTTPResponse: &http.Response{StatusCode: http.StatusOK}, + JSON200: &deploy, + }, nil).Once() + m.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, dbtTestDeployID, mock.Anything).Return(&astrov1.FinalizeDeployResponse{ + HTTPResponse: &http.Response{StatusCode: http.StatusOK}, + JSON200: &deploy, + }, nil).Once() + return m +} + +// stubBundleWait stands in for the wait a --wait does, writing one line of +// progress where the real one writes its own and answering err. +func stubBundleWait(t *testing.T, err error) *[]string { + t.Helper() + var waited []string + prev := waitForBundle + waitForBundle = func(progress io.Writer, deploymentID string, _ time.Duration, _ astrov1.APIClient) error { + waited = append(waited, deploymentID) + fmt.Fprintln(progress, "Waiting for the deployment to become healthy…") + return err + } + t.Cleanup(func() { waitForBundle = prev }) + return &waited +} + +// A wait that fails and a result that then cannot be written are both +// reported: neither error hides the other. +func TestPublishThenWaitReportsBothFailures(t *testing.T) { + timedOut := errors.New("timed out") + stubBundleWait(t, timedOut) + writeFailed := errors.New("broken pipe") + + var stdout, stderr bytes.Buffer + root := &cobra.Command{Use: "astro"} + var format cliout.Format + cmd := &cobra.Command{Use: "x", RunE: func(cmd *cobra.Command, _ []string) error { + wait := bundleWait(cmd, &deployLogin{current: true}, dbtTestDeploymentID, time.Second) + return publishThenWaitWith(cmd, cliout.FormatJSON, true, wait, func(waitErr error) error { + assert.Same(t, timedOut, waitErr) + return writeFailed + }) + }} + cliout.AddOutputFlag(cmd, &format) + root.AddCommand(cmd) + root.SetOut(&stdout) + root.SetErr(&stderr) + + err := cliout.Execute(context.Background(), root, []string{"x", "-o", "json"}, &stdout, nil) + + require.ErrorIs(t, err, writeFailed) + require.ErrorIs(t, err, timedOut) + assert.Empty(t, stdout.String(), "the write to stdout failed, so no error object is written there after it") + assert.Equal(t, "Waiting for the deployment to become healthy…\nError: broken pipe\ntimed out\n", stderr.String(), "after the wait's progress, the write's failure first, then the wait's") + assert.Equal(t, cliout.ExitFailure, cliout.ExitCode(context.Background(), err)) +} + +// remote deploy's closing lines, as v2 printed them when the push succeeded. +const remoteDeployTail = "Successfully pushed client image to registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + + "\n--------------------------------\n" + + "The client image has been pushed to your private registry.\n" + + "Your next step would be to update the agent component to use the new client image.\n" + + "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.\n" + + "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + + "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components\n" + +// stubClientDeploy stands in for the build and the push, printing one of the +// progress lines the real one prints on the way. +func stubClientDeploy(t *testing.T, res *astrodeploy.ClientDeploy, err error) { + t.Helper() + prev := deployClientImage + deployClientImage = func(astrodeploy.InputClientDeploy, astrov1.APIClient) (astrodeploy.ClientDeploy, error) { + fmt.Println("Pushing client image to configured remote registry") + return *res, err + } + t.Cleanup(func() { deployClientImage = prev }) +} + +func execRemoteRun(t *testing.T, args ...string) tokenRun { + t.Helper() + // remote deploy runs in a project. + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + prev := config.WorkingPath + config.WorkingPath = dir + t.Cleanup(func() { config.WorkingPath = prev }) + return execAstroCmd(t, nil, "", newRemoteRootCmd, append([]string{"remote"}, args...)...) +} + +func TestRemoteDeployOutput(t *testing.T) { + pushed := astrodeploy.ClientDeploy{ + Image: "registry.example.com/agents/client:deploy-2026-10-07T14-53", + Registry: "registry.example.com/agents/client", + Tag: "deploy-2026-10-07T14-53", + Platforms: []string{"linux/amd64", "linux/arm64"}, + RuntimeCheck: &astrodeploy.ClientRuntimeCheck{ + DeploymentID: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2", + }, + } + t.Run("text", func(t *testing.T) { + stubClientDeploy(t, &pushed, nil) + r := execRemoteRun(t, "deploy") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + assert.Equal(t, "Pushing client image to configured remote registry\n"+remoteDeployTail, r.stdout) + }) + t.Run("json", func(t *testing.T) { + stubClientDeploy(t, &pushed, nil) + r := execRemoteRun(t, "deploy", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got remoteDeployJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, "registry.example.com/agents/client:deploy-2026-10-07T14-53", got.Image) + assert.Equal(t, "deploy-2026-10-07T14-53", got.Tag) + assert.Equal(t, []string{"linux/amd64", "linux/arm64"}, got.Platforms) + require.NotNil(t, got.RuntimeCheck) + assert.Equal(t, remoteRuntimeCheckJSON{Deployment: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2"}, *got.RuntimeCheck) + }) + t.Run("json, a prebuilt image", func(t *testing.T) { + stubClientDeploy(t, &astrodeploy.ClientDeploy{Image: "r/c:deploy-x", Registry: "r/c", Tag: "deploy-x", SourceImage: "local:1"}, nil) + r := execRemoteRun(t, "deploy", "--image-name", "local:1", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got remoteDeployJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, "local:1", got.SourceImage) + assert.JSONEq(t, "[]", string(fields["platforms"])) + assert.NotContains(t, fields, "runtime_check") + }) + t.Run("json, the push failing", func(t *testing.T) { + stubClientDeploy(t, &astrodeploy.ClientDeploy{}, errors.New("failed to push client image: denied")) + r := execRemoteRun(t, "deploy", "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitFailure, r.code) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Contains(t, got.Error, "denied") + }) +} diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index dc6e4ba97..024364c0d 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -1,348 +1,52 @@ package astro import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "time" - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" - - "github.com/astronomer/astro-cli/cmd/cliout" - "github.com/astronomer/astro-cli/cmd/utils" - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - "github.com/astronomer/astro-cli/internal/platform/astro/deployment" -) - -var ( - mountPath string - dbtProjectPath string - - DeployBundle = astrodeploy.DeployBundle - DeleteBundle = astrodeploy.DeleteBundle ) +// What the astro dbt group's commands say now. deploy and delete moved to +// astro deploy bundle, which deploys a dbt project, or any other non-DAG +// bundle, from its own directory. cleanup removed the Cosmos Boost sidecars +// earlier deploys wrote into the project; v2 writes none there, and leaves +// out the old ones from what it ships. const ( - dbtDefaultMountPathPrefix = "/usr/local/airflow/dbt/" - dbtProjectYmlFilename = "dbt_project.yml" - dbtBundleType = "dbt" - dbtWaitTime = 300 * time.Second + errDbtRemoved = "astro dbt was removed in Astro CLI v2: use astro deploy bundle, which deploys a dbt project " + + "(or any other non-DAG bundle) from its directory, and astro deploy bundle --delete, which removes one" + errDbtDeployRemoved = "astro dbt deploy was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment . " + + "PATH replaces --project-path, --deployment replaces the argument and --deployment-name, and --workspace replaces --workspace-id; " + + "--mount-path, --description, --wait and --wait-time are unchanged" + errDbtDeleteRemoved = "astro dbt delete was removed in Astro CLI v2: use astro deploy bundle --delete [PATH] --deployment . " + + "PATH replaces --project-path, --deployment replaces the argument and --deployment-name, and --workspace replaces --workspace-id; " + + "--mount-path, --description, --wait and --wait-time are unchanged" + errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + + "Astro CLI v2 no longer writes Cosmos Boost files into the project, and leaves out the ones earlier versions wrote " + + "from every bundle it deploys and every image it generates. " + + "Delete any old .astro/dbt_metadata.json and .astro/manifest.slim.json files by hand: " + + "a local Airflow running the Cosmos Boost plugin, or the build of a declared Dockerfile, may still read them" ) -func newDbtCmd(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "dbt", - Short: "Manage dbt projects deployed to your Deployments", - } - // The results leave by cmd.OutOrStdout(), which must be the stdout the - // tree was built with: under json strayStdoutToStderr points os.Stdout - // at stderr, and an unbound command would follow it there. - cmd.SetOut(out) - cmd.AddCommand( - newDbtDeployCmd(), - newDbtDeleteCmd(), - newDbtCleanupRemovedCmd(), - ) - applyPreferredFlagsIn(cmd) - return cmd +// newDbtRemovedCmd builds the astro dbt tombstone (cliout.RemovedCommand): +// one hidden stub that takes any subcommand, so astro dbt deploy and its +// 1.x flags reach the guidance for what was typed rather than dying on a +// flag. +func newDbtRemovedCmd() *cobra.Command { + return removedCmdStub("dbt", nil, "Removed in v2 — use astro deploy bundle", dbtRemovedGuidance) } -// errDbtCleanupRemoved is what astro dbt cleanup says now. It removed the -// Cosmos Boost sidecars earlier deploys wrote into the project; v2 writes -// none there, and leaves out the old ones from what it ships. -const errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + - "Astro CLI v2 no longer writes Cosmos Boost files into the project, and leaves out the ones earlier versions wrote " + - "from every bundle it deploys and every image it generates. " + - "Delete any old .astro/dbt_metadata.json and .astro/manifest.slim.json files by hand: " + - "a local Airflow running the Cosmos Boost plugin, or the build of a declared Dockerfile, may still read them" - -// newDbtCleanupRemovedCmd builds the astro dbt cleanup tombstone -// (cliout.RemovedCommand), so a script that still runs it is told why rather -// than shown the dbt group's help. -func newDbtCleanupRemovedCmd() *cobra.Command { - return removedCmdStub("cleanup", nil, "Removed in v2 — Cosmos Boost no longer writes into the project", func([]string) string { - return errDbtCleanupRemoved - }) -} - -//nolint:dupl // the duplication is acceptable here -func newDbtDeployCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "deploy [DEPLOYMENT_ID]", - Short: "Deploy your dbt project to a Deployment on Astro", - Long: "Deploy your dbt project to a Deployment on Astro. This command bundles your dbt project files and uploads it to your Deployment.", - Args: cobra.MaximumNArgs(1), - RunE: deployDbt, - Example: ` # Deploy the dbt project in the current directory, picking the Deployment from a list - astro dbt deploy - - # Deploy a dbt project elsewhere to a given Deployment - astro dbt deploy --project-path `, - } - - cmd.Flags().StringVarP(&mountPath, "mount-path", "m", "", fmt.Sprintf("Path to mount dbt project in Airflow, for reference by Dags. Default %s{dbt project name}", dbtDefaultMountPathPrefix)) - cmd.Flags().StringVarP(&dbtProjectPath, "project-path", "p", "", "Path to the dbt project to deploy. Default current directory") - cmd.Flags().StringVar(&workspaceID, "workspace-id", "", "Workspace for your Deployment") - cmd.Flags().StringVarP(&deploymentName, "deployment-name", "n", "", "Name of the Deployment to deploy to") - addWorkspaceFlag(cmd.Flags(), "", "Workspace for your Deployment") - addDeploymentFlag(cmd.Flags(), "Deployment to deploy to: a Deployment id, or a Deployment name") - cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Description to store on the deploy") - cmd.Flags().BoolVarP(&waitForDeploy, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") - cmd.Flags().DurationVarP(&waitTime, "wait-time", "t", dbtWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") - cliout.AddOutputFlag(cmd, &dbtOutput) - - return cmd -} - -func deployDbt(cmd *cobra.Command, args []string) error { - format := dbtOutput - cmd.SilenceUsage = true - // The notes below, and the ones the platform code prints on the way, are - // the output text mode has always shown; under json they go to stderr. - defer strayStdoutToStderr(format)() - - // if the dbt project path is not provided, use the current directory - if dbtProjectPath == "" { - dbtProjectPath = config.WorkingPath - } - - // check that the dbt project path is not within an Astro project, by the - // walk every deploy decides by (astro deploy --non-dags checks the same) - if !utils.Locate(dbtProjectPath).None() { - return fmt.Errorf("dbt project is within an Astro project. Use 'astro deploy' to deploy your Astro project") - } - - // check that there is a valid dbt project at the dbt project path - err := validateDbtProjectExists(dbtProjectPath) - if err != nil { - return err - } - - // extract the dbt project's name - dbtProjectName, err := extractDbtProjectName(dbtProjectPath) - if err != nil { - return fmt.Errorf("dbt project name not found in %s: %w", dbtProjectPath, err) - } - - // if the workspace ID is not provided, try to find a valid workspace - if workspaceID == "" { - var err error - workspaceID, err = coalesceWorkspace() - if err != nil { - return fmt.Errorf("failed to find a valid workspace: %w", err) - } - } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return errors.New("cannot use --wait-time with --wait=false") - } - - // get the deployment id to deploy the dbt project to - deploymentID, read, err := resolveBundleDeployment(args, workspaceID, deploymentName, noCreateUnderJSON(format, "to deploy to")) - if err != nil { - return err - } - fmt.Println("Initiating dbt deploy for deployment ID: " + deploymentID) - - // if the mount path is not provided, derive it from the dbt project name - if mountPath == "" { - mountPath = dbtDefaultMountPathPrefix + dbtProjectName - fmt.Printf("Generated mount path from dbt project name: %s\n", mountPath) - } - - // deploy the dbt project as a bundle - deployBundleInput := &astrodeploy.DeployBundleInput{ - BundlePath: dbtProjectPath, - MountPath: mountPath, - DeploymentID: deploymentID, - Deployment: read, - BundleType: dbtBundleType, - Description: deployDescription, - AstroV1Client: astroV1Client, - } - res, err := DeployBundle(deployBundleInput) - if err != nil { - return err - } - projectPath, err := filepath.Abs(dbtProjectPath) - if err != nil { - projectPath = dbtProjectPath - } - r := cliout.Renderer{Format: format, Out: cmd.OutOrStdout()} - return publishThenWait(cmd, format, waitForDeploy, res.DeploymentID, waitTime, func(waitErr error) error { - return r.Emit(newDbtDeployJSON(&res, dbtProjectName, projectPath, waitForDeploy, waitErr), renderBundleUploaded(res.BundleVersion)) - }) -} - -//nolint:dupl // the duplication is acceptable here -func newDbtDeleteCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "delete [DEPLOYMENT_ID]", - Short: "Delete a dbt project from a Deployment on Astro", - Long: "Delete a dbt project bundle from a Deployment. This removes the uploaded dbt project files but does not affect Dags that were generated from the project.", - Args: cobra.MaximumNArgs(1), - RunE: deleteDbt, - Example: ` # Delete the dbt project in the current directory, picking the Deployment from a list - astro dbt delete - - # Delete it from a given Deployment, by its mount path - astro dbt delete --mount-path `, - } - - cmd.Flags().StringVarP(&mountPath, "mount-path", "m", "", fmt.Sprintf("Mount path of the dbt project to be deleted from the Deployment. Default %s{dbt project name}", dbtDefaultMountPathPrefix)) - cmd.Flags().StringVarP(&dbtProjectPath, "project-path", "p", "", "Path to the dbt project to delete from the Deployment. Default current directory") - cmd.Flags().StringVar(&workspaceID, "workspace-id", "", "Workspace for your Deployment") - cmd.Flags().StringVarP(&deploymentName, "deployment-name", "n", "", "Name of the Deployment to delete the dbt project from") - addWorkspaceFlag(cmd.Flags(), "", "Workspace for your Deployment") - addDeploymentFlag(cmd.Flags(), "Deployment to delete the dbt project from: a Deployment id, or a Deployment name") - cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Description to store on the deploy") - cmd.Flags().BoolVarP(&waitForDeploy, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") - cmd.Flags().DurationVarP(&waitTime, "wait-time", "t", dbtWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") - cliout.AddOutputFlag(cmd, &dbtOutput) - return cmd -} - -func deleteDbt(cmd *cobra.Command, args []string) error { - format := dbtOutput - cmd.SilenceUsage = true - defer strayStdoutToStderr(format)() - - // if the workspace ID is not provided, try to find a valid workspace - if workspaceID == "" { - var err error - workspaceID, err = coalesceWorkspace() - if err != nil { - return fmt.Errorf("failed to find a valid workspace: %w", err) - } - } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return errors.New("cannot use --wait-time with --wait=false") - } - - // get the deployment id to delete the dbt project - deploymentID, read, err := resolveBundleDeployment(args, workspaceID, deploymentName, noCreateUnderJSON(format, "to delete from")) - if err != nil { - return err - } - fmt.Println("Initiating dbt delete deploy for deployment ID: " + deploymentID) - - // if the mount path is not provided, derive it from the dbt project name - if mountPath == "" { - // if the dbt project path is not provided, use the current directory - if dbtProjectPath == "" { - dbtProjectPath = config.WorkingPath - } - - // check that there is a valid dbt project at the dbt project path - err := validateDbtProjectExists(dbtProjectPath) - if err != nil { - return err - } - - // extract the dbt project's name - dbtProjectName, err := extractDbtProjectName(dbtProjectPath) - if err != nil { - return fmt.Errorf("dbt project name not found in %s: %w", dbtProjectPath, err) - } - - mountPath = dbtDefaultMountPathPrefix + dbtProjectName - } - - deleteBundleInput := &astrodeploy.DeleteBundleInput{ - MountPath: mountPath, - DeploymentID: deploymentID, - Deployment: read, - BundleType: dbtBundleType, - Description: deployDescription, - AstroV1Client: astroV1Client, - } - res, err := DeleteBundle(deleteBundleInput) - if err != nil { - return err - } - r := cliout.Renderer{Format: format, Out: cmd.OutOrStdout()} - return publishThenWait(cmd, format, waitForDeploy, res.DeploymentID, waitTime, func(waitErr error) error { - return r.Emit(newDbtDeleteJSON(&res, waitForDeploy, waitErr), renderDbtDeleted(res.MountPath)) - }) -} - -func validateDbtProjectExists(dbtProjectPath string) error { - dbtProjectYamlPath := filepath.Join(dbtProjectPath, dbtProjectYmlFilename) - - _, err := os.Stat(dbtProjectYamlPath) - if os.IsNotExist(err) { - return fmt.Errorf("dbt project file not found at %s. Please run this command in the root of your dbt project, or use --project-path to specify the dbt project path", dbtProjectYamlPath) - } - return err -} - -func extractDbtProjectName(dbtProjectPath string) (string, error) { - dbtProjectYamlPath := filepath.Join(dbtProjectPath, dbtProjectYmlFilename) - - var dbtProject map[string]interface{} - file, err := os.Open(dbtProjectYamlPath) - if err != nil { - return "", fmt.Errorf("could not open %s: %w", dbtProjectYamlPath, err) - } - defer file.Close() - decoder := yaml.NewDecoder(file) - err = decoder.Decode(&dbtProject) - if err != nil { - return "", fmt.Errorf("could not decode %s: %w", dbtProjectYamlPath, err) - } - - dbtProjectName, ok := dbtProject["name"].(string) - if !ok || dbtProjectName == "" { - return "", errors.New("invalid dbt project name") - } - - return dbtProjectName, nil -} - -// resolveBundleDeployment is the Deployment a bundle command acts on: the -// argument, --deployment, the one --deployment-name names, or the one picked -// from the Workspace's. It returns the id, and the Deployment itself when it -// read it (picked, or found by name), so the command need not read it again; -// read is nil for one named by id. -// -// noCreateFor decides what a Workspace with no Deployment does. Empty, the -// run walks through creating one, as text mode always has. Set (under -// --output json, where nothing may be asked), it is what the command would -// have done with the Deployment ("to deploy to"), and the run fails saying -// so, rather than asking a question answered by a --name this command does -// not have. -func resolveBundleDeployment(args []string, workspaceID, deploymentName, noCreateFor string) (id string, read *astrov1.Deployment, err error) { - // if provided, use the deployment ID from the command argument +// dbtRemovedGuidance names the replacement for the subcommand typed, the +// first argument that is not a flag. +func dbtRemovedGuidance(args []string) string { if len(args) > 0 { - return args[0], nil, nil - } - // or the one given as --deployment - if deploymentArg != "" { - return deploymentArg, nil, nil - } - - // otherwise, prompt the user to select a deployment - selectedDeployment, err := deployment.GetDeployment(workspaceID, "", deploymentName, noCreateFor != "", nil, astroV1Client) - if err != nil { - return "", nil, err - } - if selectedDeployment.Id == "" { - return "", nil, deployment.ErrNothingTo(workspaceID, "", deploymentName, noCreateFor) + if msg, ok := dbtSubcommandGuidance[args[0]]; ok { + return msg + } } - return selectedDeployment.Id, &selectedDeployment, nil + return errDbtRemoved } -// noCreateUnderJSON is resolveBundleDeployment's noCreateFor for a command -// in format: what under json, and nothing (the create walk-through) in text. -func noCreateUnderJSON(format cliout.Format, what string) string { - if format == cliout.FormatJSON { - return what - } - return "" +// dbtSubcommandGuidance is what each 1.x astro dbt subcommand says now. +var dbtSubcommandGuidance = map[string]string{ + "deploy": errDbtDeployRemoved, + "delete": errDbtDeleteRemoved, + "cleanup": errDbtCleanupRemoved, } diff --git a/cmd/astro/dbt_output_test.go b/cmd/astro/dbt_output_test.go deleted file mode 100644 index c43b1b673..000000000 --- a/cmd/astro/dbt_output_test.go +++ /dev/null @@ -1,471 +0,0 @@ -package astro - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - "time" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/astronomer/astro-cli/cmd/cliout" - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" -) - -const ( - dbtTestDeploymentID = "clxdbtdeployment000000000" - dbtTestDeployID = "clxdbtdeploy0000000000000" - dbtTestVersion = "2026-10-07T00:00:00.0000000Z" -) - -// fakeBlobStore answers the bundle upload the way Azure does: every request -// succeeds, and the commit reports the version the upload created. -func fakeBlobStore(t *testing.T) string { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) - w.Header().Set("x-ms-version-id", dbtTestVersion) - w.WriteHeader(http.StatusCreated) - })) - t.Cleanup(srv.Close) - return srv.URL + "/container/bundle.tar.gz?sig=x" -} - -// dbtProject writes a dbt project named jaffle_shop in a fresh directory outside -// any Astro project, and makes a second fresh directory the working path, so -// the bundle's tarball is written beside it rather than inside it. -func dbtProject(t *testing.T) string { - t.Helper() - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte("name: jaffle_shop\n"), 0o600)) - prev := config.WorkingPath - config.WorkingPath = t.TempDir() - t.Cleanup(func() { config.WorkingPath = prev }) - return dir -} - -func dbtTestDeployment() astrov1.Deployment { - return astrov1.Deployment{ - Id: dbtTestDeploymentID, - Name: "analytics", - WorkspaceId: "clxdbtworkspace0000000000", - IsDagDeployEnabled: true, - Status: astrov1.DeploymentStatusHEALTHY, - } -} - -// dbtBundleMock is the API a bundle deploy or delete talks to. -func dbtBundleMock(t *testing.T, uploadURL string) *astrov1_mocks.ClientWithResponsesInterface { - t.Helper() - m := new(astrov1_mocks.ClientWithResponsesInterface) - t.Cleanup(func() { m.AssertExpectations(t) }) - d := dbtTestDeployment() - m.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID).Return(&astrov1.GetDeploymentResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &d, - }, nil).Maybe() - deploy := astrov1.Deploy{Id: dbtTestDeployID} - if uploadURL != "" { - deploy.BundleUploadUrl = &uploadURL - } - m.On("CreateDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, mock.Anything).Return(&astrov1.CreateDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &deploy, - }, nil).Once() - m.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, dbtTestDeploymentID, dbtTestDeployID, mock.Anything).Return(&astrov1.FinalizeDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &deploy, - }, nil).Once() - return m -} - -func execDbtRun(t *testing.T, client astrov1.APIClient, args ...string) tokenRun { - t.Helper() - return execAstroCmd(t, client, "", newDbtCmd, append([]string{"dbt"}, args...)...) -} - -// What v2 printed on stdout in text mode, recorded before --output json -// existed on these commands. Text output may not change, so these are pinned -// byte for byte. -func TestDbtTextOutputUnchanged(t *testing.T) { - t.Run("deploy", func(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--project-path", project) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Initiating dbt deploy for deployment ID: "+dbtTestDeploymentID+"\n"+ - "Generated mount path from dbt project name: /usr/local/airflow/dbt/jaffle_shop\n"+ - "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) - }) - t.Run("deploy --mount-path", func(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--project-path", project, "--mount-path", "/usr/local/airflow/dbt/custom") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Initiating dbt deploy for deployment ID: "+dbtTestDeploymentID+"\n"+ - "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) - }) - t.Run("delete", func(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, ""), "delete", dbtTestDeploymentID, "--project-path", project) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Initiating dbt delete deploy for deployment ID: "+dbtTestDeploymentID+"\n"+ - "Successfully requested bundle delete for mount path /usr/local/airflow/dbt/jaffle_shop from Astro.\n", r.stdout) - }) - // astro deploy --non-dags uploads a bundle the same way, and prints the - // same line. - t.Run("astro deploy --non-dags", func(t *testing.T) { - bundle := dbtProject(t) - r := execAstroCmd(t, dbtBundleMock(t, fakeBlobStore(t)), "", func(io.Writer) *cobra.Command { return NewDeployCmd() }, - "deploy", dbtTestDeploymentID, "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", bundle) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Successfully uploaded bundle with version "+dbtTestVersion+" to Astro.\n", r.stdout) - }) -} - -// stubBundleWait stands in for the wait a --wait does, writing one line of -// progress where the real one writes its own and answering err. -func stubBundleWait(t *testing.T, err error) *[]string { - t.Helper() - var waited []string - prev := waitForBundle - waitForBundle = func(progress io.Writer, deploymentID string, _ time.Duration, _ astrov1.APIClient) error { - waited = append(waited, deploymentID) - fmt.Fprintln(progress, "Waiting for the deployment to become healthy…") - return err - } - t.Cleanup(func() { waitForBundle = prev }) - return &waited -} - -func TestDbtDeployJSON(t *testing.T) { - project := dbtProject(t) - r := execDbtRun(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--project-path", project, "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - - var got dbtDeployJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeploymentID, got.Deployment) - assert.Equal(t, "analytics", got.DeploymentName) - assert.Equal(t, "clxdbtworkspace0000000000", got.Workspace) - assert.Equal(t, dbtTestDeployID, got.DeployID) - assert.Equal(t, "jaffle_shop", got.Project) - assert.Equal(t, project, got.ProjectPath) - assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", got.MountPath) - assert.Equal(t, dbtTestVersion, got.BundleVersion) - assert.False(t, got.Waited) - assert.Nil(t, got.Git, "the project is in no git checkout") -} - -// A Deployment found by name is read once, by the lookup, and the deploy -// uses what the lookup read rather than reading it again. -func TestDbtDeployReadsANamedDeploymentOnce(t *testing.T) { - project := dbtProject(t) - m := dbtBundleMock(t, fakeBlobStore(t)) - m.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(listDeploymentsResp(dbtTestDeployment()), nil).Once() - - r := execDbtRun(t, m, "deploy", "--deployment-name", "analytics", "--project-path", project, "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got dbtDeployJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeploymentID, got.Deployment) - m.AssertNumberOfCalls(t, "GetDeploymentWithResponse", 1) -} - -// A --wait's progress is on stderr in both formats. In text the result comes -// first, as it always has, so stdout is what it is without --wait; under json -// the object comes after the wait it reports, and says it waited. -func TestDbtWait(t *testing.T) { - cases := []struct { - name string - upload bool - }{ - {"deploy", true}, - {"delete", false}, - } - for _, tc := range cases { - args := func(project string, more ...string) []string { - return append([]string{tc.name, dbtTestDeploymentID, "--project-path", project}, more...) - } - client := func(t *testing.T) *astrov1_mocks.ClientWithResponsesInterface { - if tc.upload { - return dbtBundleMock(t, fakeBlobStore(t)) - } - return dbtBundleMock(t, "") - } - t.Run(tc.name+" text", func(t *testing.T) { - project := dbtProject(t) - without := execDbtRun(t, client(t), args(project)...) - require.NoError(t, without.err) - - waited := stubBundleWait(t, nil) - r := execDbtRun(t, client(t), args(project, "--wait")...) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, []string{dbtTestDeploymentID}, *waited) - assert.Equal(t, without.stdout, r.stdout, "stdout is the result, the same with --wait as without") - assert.Contains(t, r.stderr, "Waiting for the deployment to become healthy…") - }) - t.Run(tc.name+" text, the wait failing", func(t *testing.T) { - project := dbtProject(t) - stubBundleWait(t, errors.New("timed out")) - r := execDbtRun(t, client(t), args(project, "--wait")...) - require.Error(t, r.err) - assert.Contains(t, r.stdout, "Successfully", "the result is printed before the wait begins") - }) - t.Run(tc.name+" json", func(t *testing.T) { - project := dbtProject(t) - waited := stubBundleWait(t, nil) - r := execDbtRun(t, client(t), args(project, "--wait", "-o", "json")...) - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, []string{dbtTestDeploymentID}, *waited) - var got struct { - Waited bool `json:"waited"` - } - fields := decodeOne(t, r.stdout, &got) - assert.True(t, got.Waited) - assert.NotContains(t, fields, "wait_error", "a wait that succeeded has no error") - assert.Contains(t, r.stderr, "Waiting for the deployment to become healthy…") - }) - t.Run(tc.name+" json, the wait failing", func(t *testing.T) { - project := dbtProject(t) - stubBundleWait(t, errors.New("timed out")) - r := execDbtRun(t, client(t), args(project, "--wait", "-o", "json")...) - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - // The upload happened, so the result is still published, saying - // it waited and why the wait failed, and the run fails. - var got struct { - DeployID string `json:"deploy_id"` - Waited bool `json:"waited"` - WaitError string `json:"wait_error"` - } - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeployID, got.DeployID) - assert.True(t, got.Waited, "the wait was attempted") - assert.Equal(t, "timed out", got.WaitError) - assert.Contains(t, r.stderr, "Error: timed out") - }) - } -} - -func TestDbtDeleteJSON(t *testing.T) { - // Named by id, the Deployment is never read, so its Workspace is not - // known and not published. - t.Run("by id", func(t *testing.T) { - r := execDbtRun(t, dbtBundleMock(t, ""), "delete", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/dbt/custom", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - - var got dbtDeleteJSON - fields := decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtDeleteJSON{ - Action: "deleted", - Deployment: dbtTestDeploymentID, - DeployID: dbtTestDeployID, - MountPath: "/usr/local/airflow/dbt/custom", - }, got) - assert.NotContains(t, fields, "workspace") - assert.NotContains(t, fields, "wait_error") - }) - // Named by name, it was read from the Workspace, and its Workspace is - // published. - t.Run("by name", func(t *testing.T) { - m := dbtBundleMock(t, "") - m.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(listDeploymentsResp(dbtTestDeployment()), nil).Once() - r := execDbtRun(t, m, "delete", "--deployment-name", "analytics", "--mount-path", "/usr/local/airflow/dbt/custom", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - - var got dbtDeleteJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, dbtTestDeploymentID, got.Deployment) - assert.Equal(t, "clxdbtworkspace0000000000", got.Workspace) - }) -} - -// A wait that fails and a result that then cannot be written are both -// reported: neither error hides the other. -func TestPublishThenWaitReportsBothFailures(t *testing.T) { - timedOut := errors.New("timed out") - stubBundleWait(t, timedOut) - writeFailed := errors.New("broken pipe") - - var stdout, stderr bytes.Buffer - root := &cobra.Command{Use: "astro"} - var format cliout.Format - cmd := &cobra.Command{Use: "x", RunE: func(cmd *cobra.Command, _ []string) error { - return publishThenWait(cmd, cliout.FormatJSON, true, dbtTestDeploymentID, time.Second, func(waitErr error) error { - assert.Same(t, timedOut, waitErr) - return writeFailed - }) - }} - cliout.AddOutputFlag(cmd, &format) - root.AddCommand(cmd) - root.SetOut(&stdout) - root.SetErr(&stderr) - - err := cliout.Execute(context.Background(), root, []string{"x", "-o", "json"}, &stdout, nil) - - require.ErrorIs(t, err, writeFailed) - require.ErrorIs(t, err, timedOut) - assert.Empty(t, stdout.String(), "the write to stdout failed, so no error object is written there after it") - assert.Equal(t, "Waiting for the deployment to become healthy…\nError: broken pipe\ntimed out\n", stderr.String(), "after the wait's progress, the write's failure first, then the wait's") - assert.Equal(t, cliout.ExitFailure, cliout.ExitCode(context.Background(), err)) -} - -// astro dbt cleanup is a tombstone: it touches nothing, and says why in text -// and as the one error object under json, whatever paths and flags it gets. -func TestDbtCleanupRemoved(t *testing.T) { - dir := t.TempDir() - artifact := filepath.Join(dir, ".astro", "dbt_metadata.json") - require.NoError(t, os.MkdirAll(filepath.Dir(artifact), 0o755)) - require.NoError(t, os.WriteFile(artifact, []byte(`{"generated_by": {"application": "astro"}}`), 0o600)) - - r := execDbtRun(t, nil, "cleanup", dir) - require.Error(t, r.err) - assert.Equal(t, cliout.ExitUsage, r.code) - assert.Equal(t, errDbtCleanupRemoved, r.err.Error()) - assert.Contains(t, r.err.Error(), "no longer writes Cosmos Boost files") - assert.NotContains(t, r.err.Error(), "`", "command names are plain text") - assert.Empty(t, r.stdout) - assert.FileExists(t, artifact, "the tombstone removes nothing") - - r = execDbtRun(t, nil, "cleanup", dir, "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitUsage, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, errDbtCleanupRemoved, got.Error) - assert.FileExists(t, artifact) -} - -// Under --output json the deployment picker is refused as input_required, -// naming --deployment, before anything is uploaded or deleted. A Workspace -// with no Deployment fails instead of walking through a create, whose name -// question would name a --name these commands do not have. -func TestDbtJSONNeverAsks(t *testing.T) { - two := []astrov1.Deployment{dbtTestDeployment(), dbtTestDeployment()} - two[1].Id, two[1].Name = "clxdbtother00000000000000", "other" - for _, sub := range []string{"deploy", "delete"} { - t.Run(sub+" with several Deployments", func(t *testing.T) { - project := dbtProject(t) - r := execAstroCmd(t, coreMock(t, two...), "1\n", newDbtCmd, "dbt", sub, "--project-path", project, "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, string(cliout.KindInputRequired), got.Kind) - assert.Contains(t, got.Error, "pass --deployment") - }) - t.Run(sub+" with no Deployment", func(t *testing.T) { - project := dbtProject(t) - r := execAstroCmd(t, coreMock(t), "", newDbtCmd, "dbt", sub, "--project-path", project, "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "no Deployments found in workspace") - }) - // Named, it is a Deployment that was not found, as in a Workspace - // that has others. - t.Run(sub+" naming a Deployment, with none", func(t *testing.T) { - project := dbtProject(t) - r := execAstroCmd(t, coreMock(t), "", newDbtCmd, "dbt", sub, "--project-path", project, "--deployment-name", "analytics", "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "the Deployment specified was not found") - }) - } -} - -// remote deploy's closing lines, as v2 printed them when the push succeeded. -const remoteDeployTail = "Successfully pushed client image to registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + - "\n--------------------------------\n" + - "The client image has been pushed to your private registry.\n" + - "Your next step would be to update the agent component to use the new client image.\n" + - "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.\n" + - "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: registry.example.com/agents/client:deploy-2026-10-07T14-53\n" + - "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components\n" - -// stubClientDeploy stands in for the build and the push, printing one of the -// progress lines the real one prints on the way. -func stubClientDeploy(t *testing.T, res *astrodeploy.ClientDeploy, err error) { - t.Helper() - prev := deployClientImage - deployClientImage = func(astrodeploy.InputClientDeploy, astrov1.APIClient) (astrodeploy.ClientDeploy, error) { - fmt.Println("Pushing client image to configured remote registry") - return *res, err - } - t.Cleanup(func() { deployClientImage = prev }) -} - -func execRemoteRun(t *testing.T, args ...string) tokenRun { - t.Helper() - // remote deploy runs in a project. - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) - prev := config.WorkingPath - config.WorkingPath = dir - t.Cleanup(func() { config.WorkingPath = prev }) - return execAstroCmd(t, nil, "", newRemoteRootCmd, append([]string{"remote"}, args...)...) -} - -func TestRemoteDeployOutput(t *testing.T) { - pushed := astrodeploy.ClientDeploy{ - Image: "registry.example.com/agents/client:deploy-2026-10-07T14-53", - Registry: "registry.example.com/agents/client", - Tag: "deploy-2026-10-07T14-53", - Platforms: []string{"linux/amd64", "linux/arm64"}, - RuntimeCheck: &astrodeploy.ClientRuntimeCheck{ - DeploymentID: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2", - }, - } - t.Run("text", func(t *testing.T) { - stubClientDeploy(t, &pushed, nil) - r := execRemoteRun(t, "deploy") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - assert.Equal(t, "Pushing client image to configured remote registry\n"+remoteDeployTail, r.stdout) - }) - t.Run("json", func(t *testing.T) { - stubClientDeploy(t, &pushed, nil) - r := execRemoteRun(t, "deploy", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got remoteDeployJSON - decodeOne(t, r.stdout, &got) - assert.Equal(t, "registry.example.com/agents/client:deploy-2026-10-07T14-53", got.Image) - assert.Equal(t, "deploy-2026-10-07T14-53", got.Tag) - assert.Equal(t, []string{"linux/amd64", "linux/arm64"}, got.Platforms) - require.NotNil(t, got.RuntimeCheck) - assert.Equal(t, remoteRuntimeCheckJSON{Deployment: dbtTestDeploymentID, ClientRuntimeVersion: "3.1-1", DeploymentRuntimeVersion: "3.1-2"}, *got.RuntimeCheck) - }) - t.Run("json, a prebuilt image", func(t *testing.T) { - stubClientDeploy(t, &astrodeploy.ClientDeploy{Image: "r/c:deploy-x", Registry: "r/c", Tag: "deploy-x", SourceImage: "local:1"}, nil) - r := execRemoteRun(t, "deploy", "--image-name", "local:1", "-o", "json") - require.NoError(t, r.err, "stderr:\n%s", r.stderr) - var got remoteDeployJSON - fields := decodeOne(t, r.stdout, &got) - assert.Equal(t, "local:1", got.SourceImage) - assert.JSONEq(t, "[]", string(fields["platforms"])) - assert.NotContains(t, fields, "runtime_check") - }) - t.Run("json, the push failing", func(t *testing.T) { - stubClientDeploy(t, &astrodeploy.ClientDeploy{}, errors.New("failed to push client image: denied")) - r := execRemoteRun(t, "deploy", "-o", "json") - require.Error(t, r.err) - assert.Equal(t, cliout.ExitFailure, r.code) - var got errorJSON - decodeOne(t, r.stdout, &got) - assert.Contains(t, got.Error, "denied") - }) -} diff --git a/cmd/astro/dbt_render.go b/cmd/astro/dbt_render.go deleted file mode 100644 index a0b2991b8..000000000 --- a/cmd/astro/dbt_render.go +++ /dev/null @@ -1,230 +0,0 @@ -package astro - -// Rendering for `astro dbt deploy|delete`, `astro deploy --non-dags` -// and `astro remote deploy`. The platform packages return what happened; this -// file decides how it looks, in text and in json. - -import ( - "bufio" - "errors" - "fmt" - "io" - "time" - - "github.com/spf13/cobra" - - "github.com/astronomer/astro-cli/cmd/cliout" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - "github.com/astronomer/astro-cli/pkg/ansi" -) - -// dbtOutput is --output for the `dbt` family. -var dbtOutput cliout.Format - -// waitForBundle waits for a Deployment to take a bundle deploy or delete. A -// var so a test need not sit through the poll's sleeps. -var waitForBundle = astrodeploy.WaitForBundle - -// dbtDeployJSON is what `astro dbt deploy --output json` publishes: what was -// deployed where. Its deployment, workspace and git keys are the ones -// `astro deploy`'s result uses for the same facts. -type dbtDeployJSON struct { - Deployment string `json:"deployment"` - DeploymentName string `json:"deployment_name"` - Workspace string `json:"workspace"` - // DeployID is the deploy record the upload created on the Deployment. - DeployID string `json:"deploy_id"` - // Project is the dbt project's name, from its dbt_project.yml, and - // ProjectPath the directory bundled. - Project string `json:"project"` - ProjectPath string `json:"project_path"` - MountPath string `json:"mount_path"` - // BundleVersion is the version of the uploaded bundle. - BundleVersion string `json:"bundle_version"` - // Waited is true when the run waited for the Deployment to become - // healthy (--wait), and WaitError says why that wait failed; it is - // omitted when the wait succeeded or none was asked for. - Waited bool `json:"waited"` - WaitError string `json:"wait_error,omitempty"` - Git *deployGitJSON `json:"git,omitempty"` -} - -// dbtDeleteJSON is what `astro dbt delete --output json` publishes: the -// bundle it removed from the Deployment. Astro removes the files as the -// Deployment takes the deploy, which a wait (waited, and wait_error when it -// failed) watches for. workspace is the Deployment's, omitted when the -// Deployment was named by id and so never read. -type dbtDeleteJSON struct { - Action string `json:"action"` - Deployment string `json:"deployment"` - Workspace string `json:"workspace,omitempty"` - DeployID string `json:"deploy_id"` - MountPath string `json:"mount_path"` - Waited bool `json:"waited"` - WaitError string `json:"wait_error,omitempty"` -} - -// remoteDeployJSON is what `astro remote deploy --output json` publishes: the -// client image it pushed, which is what the agents' Helm values then name. -type remoteDeployJSON struct { - Image string `json:"image"` - Registry string `json:"registry"` - Tag string `json:"tag"` - // SourceImage is the local image pushed (--image-name), omitted when the - // run built one. - SourceImage string `json:"source_image,omitempty"` - // Platforms are the --platform the image was built for, [] for the host's - // or when nothing was built. - Platforms []string `json:"platforms"` - // RuntimeCheck is the check against --deployment, omitted without one. - RuntimeCheck *remoteRuntimeCheckJSON `json:"runtime_check,omitempty"` -} - -type remoteRuntimeCheckJSON struct { - Deployment string `json:"deployment"` - ClientRuntimeVersion string `json:"client_runtime_version"` - DeploymentRuntimeVersion string `json:"deployment_runtime_version"` -} - -func newDbtDeployJSON(res *astrodeploy.BundleDeploy, project, projectPath string, waited bool, waitErr error) dbtDeployJSON { - return dbtDeployJSON{ - Deployment: res.DeploymentID, - DeploymentName: res.DeploymentName, - Workspace: res.WorkspaceID, - DeployID: res.DeployID, - Project: project, - ProjectPath: projectPath, - MountPath: res.MountPath, - BundleVersion: res.BundleVersion, - Waited: waited, - WaitError: errText(waitErr), - Git: bundleGitJSON(res.Git), - } -} - -// bundleGitJSON is the commit a bundle deploy recorded, as a deploy result -// publishes it; nil when it recorded none. -func bundleGitJSON(g *astrodeploy.BundleGit) *deployGitJSON { - if g == nil { - return nil - } - return &deployGitJSON{CommitSHA: g.CommitSHA, Branch: g.Branch, CommitURL: g.CommitURL} -} - -// renderBundleUploaded is the text a finished bundle upload has always -// printed. -func renderBundleUploaded(version string) func(io.Writer) error { - return cliout.Text(func(b *bufio.Writer) { - fmt.Fprintf(b, "Successfully uploaded bundle with version %s to Astro.\n", version) - }) -} - -// publishThenWait publishes a bundle deploy's or delete's result and, with -// --wait, waits for the Deployment to take it. publish is handed the wait's -// failure, nil when it succeeded or there was no wait. -// -// The order depends on the format. In text the result comes first, as it -// always has, so a person sees the upload succeed before the wait begins; -// the wait's progress goes to stderr, and its failure is the run's error. -// Under json the one object is the last thing the run writes, after the wait -// it reports on. A wait that fails there still publishes the result, saying -// why, because the upload happened and a script needs its deploy id and -// version, and then exits 1 (failedAfterResult). -func publishThenWait(cmd *cobra.Command, format cliout.Format, wait bool, deploymentID string, waitTime time.Duration, publish func(waitErr error) error) error { - return publishThenWaitWith(cmd, format, wait, func() error { - return waitForBundle(cmd.ErrOrStderr(), deploymentID, waitTime, astroV1Client) - }, publish) -} - -// publishThenWaitWith is publishThenWait with the wait its caller's, for a -// Deployment reached under a login other than the current context's. -func publishThenWaitWith(cmd *cobra.Command, format cliout.Format, wait bool, waitFor func() error, publish func(waitErr error) error) error { - waitDone := func() error { - if !wait { - return nil - } - return waitFor() - } - if format == cliout.FormatJSON { - waitErr := waitDone() - if err := publish(waitErr); err != nil { - // The write failed, so stdout is what broke: the root must not - // try to write the error object there too. failedAfterResult - // marks it shown and puts the words on stderr, the write's - // failure first and the wait's after it, so neither hides the - // other. - return failedAfterResult(cmd, format, errors.Join(err, waitErr)) - } - return failedAfterResult(cmd, format, waitErr) - } - if err := publish(nil); err != nil { - return err - } - return waitDone() -} - -// errText is err's message, or "" for nil. -func errText(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func newDbtDeleteJSON(res *astrodeploy.BundleDelete, waited bool, waitErr error) dbtDeleteJSON { - return dbtDeleteJSON{ - Action: "deleted", - Deployment: res.DeploymentID, - Workspace: res.WorkspaceID, - DeployID: res.DeployID, - MountPath: res.MountPath, - Waited: waited, - WaitError: errText(waitErr), - } -} - -func renderDbtDeleted(mountPath string) func(io.Writer) error { - return cliout.Text(func(b *bufio.Writer) { - fmt.Fprintf(b, "Successfully requested bundle delete for mount path %s from Astro.\n", mountPath) - }) -} - -func newRemoteDeployJSON(res *astrodeploy.ClientDeploy) remoteDeployJSON { - obj := remoteDeployJSON{ - Image: res.Image, - Registry: res.Registry, - Tag: res.Tag, - SourceImage: res.SourceImage, - Platforms: nonNil(res.Platforms), - } - if c := res.RuntimeCheck; c != nil { - obj.RuntimeCheck = &remoteRuntimeCheckJSON{ - Deployment: c.DeploymentID, - ClientRuntimeVersion: c.ClientRuntimeVersion, - DeploymentRuntimeVersion: c.DeploymentRuntimeVersion, - } - } - return obj -} - -// renderRemoteDeploy is the text a finished client image push has always -// printed: where it went, and what to do with it next. -func renderRemoteDeploy(image string) func(io.Writer) error { - return cliout.Text(func(b *bufio.Writer) { - fmt.Fprintf(b, "Successfully pushed client image to %s\n", ansi.Bold(image)) - fmt.Fprintf(b, "\n--------------------------------\n") - fmt.Fprintln(b, "The client image has been pushed to your private registry.") - fmt.Fprintln(b, "Your next step would be to update the agent component to use the new client image.") - fmt.Fprintln(b, "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.") - fmt.Fprintf(b, "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: %s\n", image) - fmt.Fprintln(b, "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components") - }) -} - -// nonNil is s, or [] for nil, so a list publishes as [] and never null. -func nonNil(s []string) []string { - if s == nil { - return []string{} - } - return s -} diff --git a/cmd/astro/dbt_test.go b/cmd/astro/dbt_test.go deleted file mode 100644 index 871f1d633..000000000 --- a/cmd/astro/dbt_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package astro - -import ( - "net/http" - "os" - "path/filepath" - "testing" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - testUtil "github.com/astronomer/astro-cli/pkg/testing" -) - -type DbtSuite struct { - suite.Suite - mockV1Client *astrov1_mocks.ClientWithResponsesInterface - origV1Client astrov1.APIClient - origDeployBundle func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) - origDeleteBundle func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) - origWorkingPath string - origWd string - tmpWorkingDir string -} - -func (s *DbtSuite) SetupTest() { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - // Run each test from an isolated temp directory that is not within an Astro - // project. dbt deploy/delete default the project path to config.WorkingPath - // and reject paths nested under an Astro project (detected by walking up the - // real filesystem for a .astro/config.yaml). Relying on the package's own - // directory makes the suite fail whenever an ancestor of the repo happens to - // be an Astro project, so we point WorkingPath/cwd at a clean temp dir. - tmpDir, err := os.MkdirTemp("", "dbt-test") - s.Require().NoError(err) - s.tmpWorkingDir = tmpDir - s.origWd, err = os.Getwd() - s.Require().NoError(err) - s.Require().NoError(os.Chdir(tmpDir)) - s.origWorkingPath = config.WorkingPath - config.WorkingPath = tmpDir - - // the package depends on global variables so we need to manage overriding those - s.origV1Client = astroV1Client - s.origDeployBundle = DeployBundle - s.origDeleteBundle = DeleteBundle - s.mockV1Client = new(astrov1_mocks.ClientWithResponsesInterface) - astroV1Client = s.mockV1Client -} - -func (s *DbtSuite) TearDownTest() { - s.mockV1Client.AssertExpectations(s.T()) - astroV1Client = s.origV1Client - DeployBundle = s.origDeployBundle - DeleteBundle = s.origDeleteBundle - - config.WorkingPath = s.origWorkingPath - if s.origWd != "" { - _ = os.Chdir(s.origWd) - } - if s.tmpWorkingDir != "" { - _ = os.RemoveAll(s.tmpWorkingDir) - } -} - -func TestDbt(t *testing.T) { - suite.Run(t, new(DbtSuite)) -} - -func (s *DbtSuite) TestDbtDeploy_PickDeployment() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - return astrodeploy.BundleDeploy{}, nil - } - - s.mockListTestDeployments() - s.mockGetTestDeployment() - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeployCmd()) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDeploy_ProvidedDeploymentId() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDeploy_CustomProjectPath() { - projectPath, err := os.MkdirTemp("", "") - assert.NoError(s.T(), err) - defer os.RemoveAll(projectPath) - - s.createDbtProjectFile(filepath.Join(projectPath, "dbt_project.yml")) - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - if deployInput.BundlePath != projectPath { - return astrodeploy.BundleDeploy{}, assert.AnError - } - return astrodeploy.BundleDeploy{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err = testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", projectPath) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDeploy_CustomMountPath() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - if deployInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDeploy{}, assert.AnError - } - return astrodeploy.BundleDeploy{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--mount-path", dbtDefaultMountPathPrefix+"test_dbt_project") - assert.NoError(s.T(), err) -} - -// A dbt project in or below a 1.x project, by the walk every deploy decides -// by (a Dockerfile beside .astro, with or without .astro/config.yaml), is -// refused, as astro deploy --non-dags refuses the same bundle path. -func (s *DbtSuite) TestDbtDeploy_WithinAstroProject() { - projectDir := s.T().TempDir() - s.Require().NoError(os.WriteFile(filepath.Join(projectDir, "Dockerfile"), []byte("FROM x\n"), 0o600)) - s.Require().NoError(os.MkdirAll(filepath.Join(projectDir, ".astro"), 0o755)) - s.Require().NoError(os.WriteFile(filepath.Join(projectDir, ".astro", "config.yaml"), []byte("project:\n name: demo\n"), 0o600)) - dbtDir := filepath.Join(projectDir, "include", "dbt") - s.Require().NoError(os.MkdirAll(dbtDir, 0o755)) - - for _, path := range []string{projectDir, dbtDir} { - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", path) - s.Require().Error(err, path) - s.Contains(err.Error(), "dbt project is within an Astro project", path) - - resetDeployFlagVars() - nonDags := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", path) - s.Require().Error(nonDags, path) - s.Contains(nonDags.Error(), "within an Astro project", "dbt deploy and --non-dags agree: %s", path) - } -} - -// A project with a pyproject.toml carries no .astro/config.yaml, so the 1.x walk answers false at -// every level of it. Before this, a dbt project nested inside one bundled and -// deployed instead of being refused. -func (s *DbtSuite) TestDbtDeploy_WithinManifestProject() { - projectDir := s.T().TempDir() - manifest := "[project]\nname = \"demo\"\n\n[tool.astro]\n" - assert.NoError(s.T(), os.WriteFile(filepath.Join(projectDir, "pyproject.toml"), []byte(manifest), 0o600)) - nested := filepath.Join(projectDir, "analytics") - assert.NoError(s.T(), os.MkdirAll(nested, 0o755)) - - err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", nested) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "dbt project is within an Astro project") -} - -func (s *DbtSuite) TestDbtDelete_PickDeployment() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - return astrodeploy.BundleDelete{}, nil - } - - s.mockListTestDeployments() - s.mockGetTestDeployment() - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd()) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_ProvidedDeploymentId() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - return astrodeploy.BundleDelete{}, nil - } - - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_CustomProjectPath() { - projectPath, err := os.MkdirTemp("", "") - assert.NoError(s.T(), err) - defer os.RemoveAll(projectPath) - - s.createDbtProjectFile(filepath.Join(projectPath, "dbt_project.yml")) - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - if deleteInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDelete{}, assert.AnError - } - return astrodeploy.BundleDelete{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err = testExecCmd(newDbtDeleteCmd(), "test-deployment-id", "--project-path", projectPath) - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_NoMountPathOrProjectPath() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - if deleteInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDelete{}, assert.AnError - } - return astrodeploy.BundleDelete{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) TestDbtDelete_NoMountPathOrProjectPath_MissingProject() { - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id") - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "dbt project file not found") -} - -func (s *DbtSuite) TestDbtDelete_CustomMountPath() { - s.createDbtProjectFile("dbt_project.yml") - defer os.Remove("dbt_project.yml") - - DeleteBundle = func(deleteInput *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { - if deleteInput.MountPath != dbtDefaultMountPathPrefix+"test_dbt_project" { - return astrodeploy.BundleDelete{}, assert.AnError - } - return astrodeploy.BundleDelete{}, nil - } - - defer testUtil.MockUserInput(s.T(), "1")() - err := testExecCmd(newDbtDeleteCmd(), "test-deployment-id", "--mount-path", dbtDefaultMountPathPrefix+"test_dbt_project") - assert.NoError(s.T(), err) -} - -func testExecCmd(cmd *cobra.Command, args ...string) error { - if args == nil { - args = []string{} - } - // deploy and dbt set the package's workspaceID from --workspace-id or the - // one they resolve, and coalesceWorkspace reads it before the context. - // Left set, it would choose the Workspace of whatever runs next. - defer func() { workspaceID = "" }() - testUtil.SetupOSArgsForGinkgo() - cmd.SetArgs(args) - _, err := cmd.ExecuteC() - return err -} - -func (s *DbtSuite) createDbtProjectFile(path string) { - file, err := os.Create(path) - assert.NoError(s.T(), err) - defer file.Close() - _, err = file.WriteString("name: test_dbt_project") - assert.NoError(s.T(), err) -} - -func (s *DbtSuite) mockListTestDeployments() { - s.mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&astrov1.ListDeploymentsResponse{ - HTTPResponse: &http.Response{ - StatusCode: http.StatusOK, - }, - JSON200: &astrov1.DeploymentsPaginated{ - Deployments: []astrov1.Deployment{ - { - Id: "test-deployment-id", - }, - }, - }, - }, nil) -} - -func (s *DbtSuite) mockGetTestDeployment() { - s.mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&astrov1.GetDeploymentResponse{ - HTTPResponse: &http.Response{ - StatusCode: http.StatusOK, - }, - JSON200: &astrov1.Deployment{ - Id: "test-deployment-id", - }, - }, nil) -} diff --git a/cmd/astro/deploy.go b/cmd/astro/deploy.go index 0177127b5..7bed0a6c6 100644 --- a/cmd/astro/deploy.go +++ b/cmd/astro/deploy.go @@ -2,7 +2,6 @@ package astro import ( "bufio" - "cmp" "context" goerrors "errors" "fmt" @@ -46,10 +45,6 @@ var ( imageName string deployDescription string noDagsBaseDir bool - nonDags bool - nonDagsMountPath string - nonDagsBundleType string - nonDagsBundlePath string manifestDeployment string manifestWorkspace string deployOutput string @@ -63,7 +58,10 @@ var ( astro deploy --dags # Deploy an image built on this machine, from any directory - astro deploy --image-name ` + astro deploy --image-name + + # Deploy a dbt project, or another non-DAG bundle, from its own directory + astro deploy bundle --deployment ` buildSecrets = []string{} ) @@ -72,14 +70,13 @@ const ( deployWaitTime = 300 * time.Second imageNameFlag = "image-name" - nonDagsFlag = "non-dags" ) func NewDeployCmd() *cobra.Command { cmd := &cobra.Command{ Use: "deploy [DEPLOYMENT_ID]", Short: "Ship this project's code to a Deployment", - Long: "Deploy your project to a Deployment on Astro. Run it in a project with a pyproject.toml ([tool.astro]): it builds the project into a Docker image and pushes that image, with the project's DAGs, to Astronomer. A project in the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml) is converted with astro init, or deployed with Astro CLI 1.x. With --image-name it deploys an image built on this machine, from any directory. In Deployments with Remote Execution enabled, this only updates the Orchestration Plane components (the API Server and Scheduler). For all other components, use astro remote deploy instead. It does not include any metadata associated with your local Airflow environment.", + Long: "Deploy your project to a Deployment on Astro. Run it in a project with a pyproject.toml ([tool.astro]): it builds the project into a Docker image and pushes that image, with the project's DAGs, to Astronomer. A project in the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml) is converted with astro init, or deployed with Astro CLI 1.x. With --image-name it deploys an image built on this machine, from any directory. In Deployments with Remote Execution enabled, this only updates the Orchestration Plane components (the API Server and Scheduler). For all other components, use astro remote deploy instead. It does not include any metadata associated with your local Airflow environment. To deploy a dbt project or another non-DAG bundle, use astro deploy bundle.", Args: cobra.MaximumNArgs(1), RunE: deploy, Example: deployExample, @@ -111,22 +108,17 @@ func NewDeployCmd() *cobra.Command { cmd.Flags().StringVar(&deployOutput, "output", string(cliout.FormatText), "Output format: text or json") cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Add a description for more context on this deploy") utils.AddBuildSecretFlag(cmd.Flags(), &buildSecrets) - cmd.Flags().BoolVar(&nonDags, nonDagsFlag, false, "Deploy a non-Dag bundle from a separate directory, instead of your Astro project. Requires --non-dags-mount-path") - cmd.Flags().StringVar(&nonDagsMountPath, "non-dags-mount-path", "", "Path to mount the non-Dag bundle in Airflow, for reference by Dags. Used with --non-dags") - cmd.Flags().StringVar(&nonDagsBundleType, "non-dags-bundle-type", "none", "Free-form label identifying the kind of non-Dag bundle (e.g. dbt). Any value is accepted. Defaults to \"none\". Used with --non-dags") - cmd.Flags().StringVar(&nonDagsBundlePath, "non-dags-local-path", "", "Path to the non-Dag bundle to deploy. Default current directory. Used with --non-dags") annotateDeployFlag(cmd, "image", deployGroupImage) annotateDeployFlag(cmd, imageNameFlag, deployGroupImage) annotateDeployFlag(cmd, "build-secret", deployGroupImage) annotateDeployFlag(cmd, "dags", deployGroupDAG) annotateDeployFlag(cmd, "no-dags-base-dir", deployGroupDAG) - annotateDeployFlag(cmd, nonDagsFlag, deployGroupNonDAG) - annotateDeployFlag(cmd, "non-dags-mount-path", deployGroupNonDAG) - annotateDeployFlag(cmd, "non-dags-bundle-type", deployGroupNonDAG) - annotateDeployFlag(cmd, "non-dags-local-path", deployGroupNonDAG) orderDeployFlagGroups(cmd) utils.MarkPreferredFlag(cmd.Flags(), "workspace", "workspace-id") + // astro deploy bundle is the subcommand, so the word bundle is never a + // Deployment argument: a link of that name is reached with --deployment. + cmd.AddCommand(newDeployBundleCmd()) applyPreferredFlagsIn(cmd) return cmd } @@ -134,10 +126,7 @@ func NewDeployCmd() *cobra.Command { // deploy ships a project with a pyproject.toml ([tool.astro]), the one kind // of project v2 deploys (deployManifest), run from the project's root. // -// --non-dags deploys a directory that must not be inside a project at all, so -// where it runs from says nothing about the project. It is checked first. -// -// Everywhere else the project decides, by utils.Locate's walk. In or below a +// The project decides, by utils.Locate's walk. In or below a // project in the Astro CLI 1.x layout every mode is refused, naming the two // ways forward: convert it with astro init, or deploy it with Astro CLI 1.x, // which keeps deploying that layout. --image-name is refused there too: a @@ -150,9 +139,6 @@ func NewDeployCmd() *cobra.Command { // Deployment that takes DAG deploys keeps the DAGs it had, and the deploy // says so. func deploy(cmd *cobra.Command, args []string) error { - if nonDags { - return deployNonDagsBundle(cmd, args) - } // One walk routes the deploy and words its refusal. where := utils.Locate(config.WorkingPath) if where.AtRoot(config.WorkingPath) { @@ -195,193 +181,6 @@ func refuseFlagCombinations(cmd *cobra.Command, inProject bool) error { return nil } -// nonDagsDeployJSON is what `astro deploy --non-dags --output json` -// publishes: the bundle deployed and where. Its keys are the ones -// `astro dbt deploy`'s result uses for the same facts. -type nonDagsDeployJSON struct { - Deployment string `json:"deployment"` - DeploymentName string `json:"deployment_name"` - Workspace string `json:"workspace"` - DeployID string `json:"deploy_id"` - // BundleType is --non-dags-bundle-type, BundlePath the directory bundled. - BundleType string `json:"bundle_type"` - BundlePath string `json:"bundle_path"` - MountPath string `json:"mount_path"` - BundleVersion string `json:"bundle_version"` - // Waited is true when the run waited for the Deployment to become - // healthy (--wait), and WaitError says why that wait failed. - Waited bool `json:"waited"` - WaitError string `json:"wait_error,omitempty"` - Git *deployGitJSON `json:"git,omitempty"` -} - -func newNonDagsDeployJSON(res *astrodeploy.BundleDeploy, bundleType, bundlePath string, waited bool, waitErr error) nonDagsDeployJSON { - return nonDagsDeployJSON{ - Deployment: res.DeploymentID, - DeploymentName: res.DeploymentName, - Workspace: res.WorkspaceID, - DeployID: res.DeployID, - BundleType: bundleType, - BundlePath: bundlePath, - MountPath: res.MountPath, - BundleVersion: res.BundleVersion, - Waited: waited, - WaitError: errText(waitErr), - Git: bundleGitJSON(res.Git), - } -} - -func deployNonDagsBundle(cmd *cobra.Command, args []string) error { - // Read first, so a bad --output is the usage error reported. - format, err := cliout.ParseFormat(deployOutput) - if err != nil { - return err - } - // The result goes where stdout is now. What the platform code prints on - // the way (the git note, the archive's) is the output text mode has always - // shown; under json it goes to stderr, as astro dbt deploy's does. - out := cmd.OutOrStdout() - defer strayStdoutToStderr(format)() - - for _, f := range []string{"dags", "image", imageNameFlag, "build-secret", "no-dags-base-dir"} { - if cmd.Flags().Changed(f) { - return cliout.Usage(fmt.Errorf("cannot use --%s with --non-dags; --non-dags performs a non-Dag bundle deploy", f)) - } - } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return cliout.Usage(errors.New("cannot use --wait-time with --wait=false")) - } - - if nonDagsMountPath == "" { - return cliout.Usage(errors.New("--non-dags-mount-path is required with --non-dags")) - } - - if nonDagsBundlePath == "" { - nonDagsBundlePath = config.WorkingPath - } - - info, err := os.Stat(nonDagsBundlePath) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("bundle path %s does not exist", nonDagsBundlePath) - } - return fmt.Errorf("failed to access bundle path %s: %w", nonDagsBundlePath, err) - } - if !info.IsDir() { - return fmt.Errorf("bundle path %s is not a directory", nonDagsBundlePath) - } - - // The walk every deploy decides by: a pyproject.toml project or a 1.x - // one at or above the bundle path. - if !utils.Locate(nonDagsBundlePath).None() { - return errors.New("bundle path is within an Astro project. Non-Dag bundles must be a separate directory") - } - - targetID, target, login, err := nonDagsTarget(cmd.Context(), args, format) - if err != nil { - return err - } - - cmd.SilenceUsage = true - - deployBundleInput := &astrodeploy.DeployBundleInput{ - BundlePath: nonDagsBundlePath, - MountPath: nonDagsMountPath, - DeploymentID: targetID, - Deployment: target, - BundleType: nonDagsBundleType, - Description: deployDescription, - AstroV1Client: login.client, - } - if !login.current { - deployBundleInput.Login = &login.context - } - res, err := DeployBundle(deployBundleInput) - if err != nil { - return err - } - waitDone := func() error { - if login.current { - return waitForBundle(cmd.ErrOrStderr(), res.DeploymentID, waitTime, login.client) - } - return astrodeploy.WaitForBundleIn(cmd.ErrOrStderr(), &login.context, res.DeploymentID, waitTime, login.client) - } - return publishThenWaitWith(cmd, format, waitForDeploy, waitDone, func(waitErr error) error { - if format == cliout.FormatJSON { - return cliout.Renderer{Format: format, Out: out}.Emit(newNonDagsDeployJSON(&res, nonDagsBundleType, nonDagsBundlePath, waitForDeploy, waitErr), nil) - } - return renderBundleUploaded(res.BundleVersion)(out) - }) -} - -// nonDagsTarget is the Deployment a non-Dag bundle deploys to, and the login -// it deploys under, the way astro deploy from the same directory picks them. -// -// The login: in a pyproject.toml project here whose manifest loads, the -// project's (its Astro host's), for everything; anywhere else, or when the -// manifest does not load, the current context's. -// -// The target: the argument or --deployment, which must agree, a link name of -// the project here or a Deployment id. With neither, the workspace's -// Deployments are offered (--workspace, else --workspace-id, else the -// login's): only to a run that can be asked, and only on the current -// context's host, as astro deploy offers them. A run that cannot be asked, -// under --output json or without a terminal, is refused as astro deploy's is, -// with kind input_required. -func nonDagsTarget(ctx context.Context, args []string, format cliout.Format) (string, *astrov1.Deployment, deployLogin, error) { - linkName := "" - if len(args) > 0 { - linkName = args[0] - } - ws := cmp.Or(manifestWorkspace, workspaceID) - // Agreement first, before anything is read. - if _, _, err := manifestdeploy.ResolveNamed(nil, linkName, manifestDeployment, ws, ""); err != nil { - current, _ := loginForDeploy(ctx, "") //nolint:errcheck // with no domain it is the current context, and never fails - return "", nil, current, cliout.Usage(err) - } - - m := manifestHere() - domain := "" - if m != nil { - domain = m.Astro.LoginDomain() - } - login, err := loginForDeploy(ctx, domain) - if err != nil { - return "", nil, login, err - } - - if name := cmp.Or(linkName, manifestDeployment); name != "" { - if m == nil || !hasLink(m, name) { - return name, nil, login, nil - } - named, _, err := manifestdeploy.ResolveNamed(m, name, "", ws, "") - if err != nil { - return "", nil, login, cliout.Usage(err) - } - return named.DeploymentID, nil, login, nil - } - - if format != cliout.FormatText || !stdinIsTerminal() { - return "", nil, login, input.Required(errNonDagsNoTarget) - } - if !login.current { - return "", nil, login, errPickerOtherHost(login.context.Domain) - } - if ws == "" { - ws, err = coalesceWorkspace() - if err != nil { - return "", nil, login, errors.Wrap(err, "failed to find a valid workspace") - } - } - id, read, err := resolveBundleDeployment(nil, ws, "", noCreateUnderJSON(format, "to deploy to")) - return id, read, login, err -} - -// errNonDagsNoTarget is a --non-dags deploy that names no Deployment and -// cannot be asked for one, in astro deploy's words for the same refusal. -var errNonDagsNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id as the argument or with --deployment") - // errPickerOtherHost is the workspace picker refused on a login other than // the current context's: it lists only the current context's Deployments. func errPickerOtherHost(domain string) error { diff --git a/cmd/astro/deploy_bundle.go b/cmd/astro/deploy_bundle.go new file mode 100644 index 000000000..3241a26a5 --- /dev/null +++ b/cmd/astro/deploy_bundle.go @@ -0,0 +1,443 @@ +package astro + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/cmd/utils" + "github.com/astronomer/astro-cli/config" + manifestdeploy "github.com/astronomer/astro-cli/internal/deploy" + astrov1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/internal/platform/astro/deployment" + "github.com/astronomer/astro-cli/pkg/input" +) + +// The transport astro deploy bundle calls, vars so a test can stand in for +// the upload and the delete. +var ( + DeployBundle = astrodeploy.DeployBundle + DeleteBundle = astrodeploy.DeleteBundle +) + +// waitForBundle waits for a Deployment to take a bundle deploy or delete. A +// var so a test need not sit through the poll's sleeps. +var waitForBundle = astrodeploy.WaitForBundle + +// deployBundleOptions are astro deploy bundle's flags. +type deployBundleOptions struct { + mountPath string + bundleType string + deployment string + workspace string + description string + wait bool + waitTime time.Duration + delete bool + output cliout.Format +} + +func newDeployBundleCmd() *cobra.Command { + o := &deployBundleOptions{} + types := strings.Join(astrodeploy.BundleTypeNames(), ", ") + cmd := &cobra.Command{ + Use: manifestdeploy.BundleCommand + " [PATH]", + Short: "Deploy a non-DAG bundle (for example a dbt project) to a Deployment", + Long: "Deploy a non-DAG bundle (for example a dbt project) to a Deployment on Astro. " + + "The bundle is a directory of its own, outside any Astro project: PATH, or the current directory. " + + "It is uploaded as it is and mounted at --mount-path in Airflow, for your DAGs to read; it carries no DAGs. " + + "With --delete it removes the bundle mounted at that path instead.\n\n" + + "A bundle's type decides its defaults. With no --type, the type is the first one PATH looks like (" + types + "), else none. " + + "A dbt project (a directory holding dbt_project.yml) mounts at " + astrodeploy.DbtDefaultMountPathPrefix + " unless --mount-path says otherwise. " + + "Any other --type is recorded as a label, with no defaults, so it needs --mount-path.\n\n" + + "This deploys the bundle's files; astro deploy deploys the project and its DAGs. " + + "To list or change the bundles a Deployment has registered, use astro deployment bundle. " + + "A project link called bundle is not reachable as astro deploy bundle: pass it as astro deploy --deployment bundle.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDeployBundle(cmd, args, o) + }, + Example: ` # Deploy the dbt project in the current directory, picking the Deployment from a list + astro deploy bundle + + # Deploy a dbt project elsewhere to a given Deployment + astro deploy bundle --deployment + + # Deploy another kind of bundle, which needs a mount path + astro deploy bundle --type --mount-path /usr/local/airflow/ \ + --deployment + + # Remove the dbt project in the current directory from a Deployment + astro deploy bundle --delete --deployment `, + } + f := cmd.Flags() + f.StringVarP(&o.mountPath, "mount-path", "m", "", "Path the bundle is mounted at in Airflow, for reference by Dags. Default: the type's, "+astrodeploy.DbtDefaultMountPathPrefix+" for a dbt project") + f.StringVar(&o.bundleType, "type", "", "Type of the bundle: "+types+", or any other label, which has no defaults. Default: detected from PATH, else none") + f.StringVarP(&o.deployment, "deployment", "d", "", "Deployment to deploy to: a link name from the project here, or a Deployment id") + f.StringVar(&o.workspace, "workspace", "", "Workspace whose Deployments are offered when no --deployment is given") + f.StringVar(&o.description, "description", "", "Description to store on the deploy") + f.BoolVarP(&o.wait, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") + f.DurationVar(&o.waitTime, "wait-time", deployWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") + f.BoolVar(&o.delete, "delete", false, "Remove the bundle mounted at --mount-path (by default the one PATH's type would mount at) instead of deploying") + cliout.AddOutputFlag(cmd, &o.output) + return cmd +} + +func runDeployBundle(cmd *cobra.Command, args []string, o *deployBundleOptions) error { + format := o.output + // The result goes where stdout is now. What the platform code prints on + // the way (the git note, the archive's) is output text mode has always + // shown; under json it goes to stderr. + out := cmd.OutOrStdout() + defer strayStdoutToStderr(format)() + + if cmd.Flags().Changed("wait-time") && !o.wait { + return cliout.Usage(errors.New("cannot use --wait-time with --wait=false")) + } + path := config.WorkingPath + if len(args) > 0 { + path = args[0] + } + if o.delete { + return deleteBundle(cmd, out, path, o) + } + return deployBundle(cmd, out, path, o) +} + +// deployBundle uploads the bundle at path. +func deployBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundleOptions) error { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("bundle path %s does not exist", path) + } + return fmt.Errorf("failed to access bundle path %s: %w", path, err) + } + if !info.IsDir() { + return fmt.Errorf("bundle path %s is not a directory", path) + } + // The walk every deploy decides by: a pyproject.toml project or a 1.x one + // at or above the bundle path. + if !utils.Locate(path).None() { + return fmt.Errorf("bundle path %s is within an Astro project: a non-DAG bundle is a directory of its own. To deploy the project, run astro deploy", path) + } + bt := astrodeploy.ResolveBundleType(o.bundleType, path) + if bt.Validate != nil { + if err := bt.Validate(path); err != nil { + return err + } + } + mountPath, err := resolveBundleMountPath(cmd, path, bt, o.mountPath) + if err != nil { + return err + } + targetID, target, login, err := bundleTarget(cmd.Context(), o.deployment, o.workspace, o.output) + if err != nil { + return err + } + cmd.SilenceUsage = true + + in := &astrodeploy.DeployBundleInput{ + BundlePath: path, + MountPath: mountPath, + DeploymentID: targetID, + Deployment: target, + BundleType: bt.Name, + Description: o.description, + AstroV1Client: login.client, + } + if !login.current { + in.Login = &login.context + } + res, err := DeployBundle(in) + if err != nil { + return err + } + bundlePath, err := filepath.Abs(path) + if err != nil { + bundlePath = path + } + return publishThenWaitWith(cmd, o.output, o.wait, bundleWait(cmd, &login, res.DeploymentID, o.waitTime), func(waitErr error) error { + r := cliout.Renderer{Format: o.output, Out: out} + return r.Emit(newBundleDeployJSON(&res, bt.Name, bundlePath, o.wait, waitErr), renderBundleUploaded(res.BundleVersion)) + }) +} + +// deleteBundle removes the bundle mounted at --mount-path, or at the mount +// path the bundle at path would deploy to. +func deleteBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundleOptions) error { + bt := astrodeploy.ResolveBundleType(o.bundleType, path) + mountPath := o.mountPath + if mountPath == "" { + if bt.Validate != nil { + if err := bt.Validate(path); err != nil { + return err + } + } + var err error + if mountPath, err = resolveBundleMountPath(cmd, path, bt, ""); err != nil { + return err + } + } + targetID, target, login, err := bundleTarget(cmd.Context(), o.deployment, o.workspace, o.output) + if err != nil { + return err + } + cmd.SilenceUsage = true + + in := &astrodeploy.DeleteBundleInput{ + MountPath: mountPath, + DeploymentID: targetID, + Deployment: target, + BundleType: bt.Name, + Description: o.description, + AstroV1Client: login.client, + } + if !login.current { + in.Login = &login.context + } + res, err := DeleteBundle(in) + if err != nil { + return err + } + return publishThenWaitWith(cmd, o.output, o.wait, bundleWait(cmd, &login, res.DeploymentID, o.waitTime), func(waitErr error) error { + r := cliout.Renderer{Format: o.output, Out: out} + return r.Emit(newBundleDeleteJSON(&res, bt.Name, o.wait, waitErr), renderBundleDeleted(res.MountPath)) + }) +} + +// resolveBundleMountPath is given, or else the default of the bundle's type, which +// it says on stderr it chose. A type with no default needs one given. +func resolveBundleMountPath(cmd *cobra.Command, path string, bt astrodeploy.BundleType, given string) (string, error) { + if given != "" { + return given, nil + } + if bt.DefaultMountPath == nil { + return "", cliout.Usage(fmt.Errorf("--mount-path is required for a bundle of type %s: only a %s bundle has a default", bt.Name, strings.Join(astrodeploy.BundleTypeNames(), " or "))) + } + mountPath, err := bt.DefaultMountPath(path) + if err != nil { + return "", err + } + fmt.Fprintf(cmd.ErrOrStderr(), "mount path %s: the default for a %s bundle\n", mountPath, bt.Name) + return mountPath, nil +} + +// bundleWait waits for the Deployment under the login the bundle went to. +func bundleWait(cmd *cobra.Command, login *deployLogin, deploymentID string, waitTime time.Duration) func() error { + return func() error { + if login.current { + return waitForBundle(cmd.ErrOrStderr(), deploymentID, waitTime, login.client) + } + return astrodeploy.WaitForBundleIn(cmd.ErrOrStderr(), &login.context, deploymentID, waitTime, login.client) + } +} + +// bundleTarget is the Deployment a bundle deploys to or is deleted from, and +// the login it does so under, the way astro deploy from the same directory +// picks them. +// +// The login: in a pyproject.toml project here whose manifest loads, the +// project's (its Astro host's), for everything; anywhere else, or when the +// manifest does not load, the current context's. +// +// The target: name (--deployment), a link name of the project here or a +// Deployment id. Without one, the workspace's Deployments are offered +// (workspace, else the login's): only to a run that can be asked, and only on +// the current context's host, as astro deploy offers them. A run that cannot +// be asked, under --output json or without a terminal, is refused as astro +// deploy's is, with kind input_required. read is the Deployment when it was +// picked, so the deploy need not read it again. +func bundleTarget(ctx context.Context, name, workspace string, format cliout.Format) (id string, read *astrov1.Deployment, login deployLogin, err error) { + m := manifestHere() + domain := "" + if m != nil { + domain = m.Astro.LoginDomain() + } + login, err = loginForDeploy(ctx, domain) + if err != nil { + return "", nil, login, err + } + + if name != "" { + if m == nil || !hasLink(m, name) { + return name, nil, login, nil + } + named, _, err := manifestdeploy.ResolveNamed(m, name, "", workspace, "") + if err != nil { + return "", nil, login, cliout.Usage(err) + } + return named.DeploymentID, nil, login, nil + } + + if format != cliout.FormatText || !stdinIsTerminal() { + return "", nil, login, input.Required(errBundleNoTarget) + } + if !login.current { + return "", nil, login, errPickerOtherHost(login.context.Domain) + } + ws := workspace + if ws == "" { + if ws, err = coalesceWorkspace(); err != nil { + return "", nil, login, fmt.Errorf("failed to find a valid workspace: %w", err) + } + } + picked, err := deployment.GetDeployment(ws, "", "", false, nil, login.client) + if err != nil { + return "", nil, login, err + } + if picked.Id == "" { + return "", nil, login, deployment.ErrNothingTo(ws, "", "", "") + } + return picked.Id, &picked, login, nil +} + +// errBundleNoTarget is a bundle deploy or delete that names no Deployment and +// cannot be asked for one, in astro deploy's words for the same refusal. +var errBundleNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id or a link name with --deployment") + +// bundleDeployJSON is what astro deploy bundle --output json publishes: the +// bundle deployed and where. Its deployment, workspace and git keys are the +// ones astro deploy's result uses for the same facts. +type bundleDeployJSON struct { + Deployment string `json:"deployment"` + DeploymentName string `json:"deployment_name"` + Workspace string `json:"workspace"` + // DeployID is the deploy record the upload created on the Deployment. + DeployID string `json:"deploy_id"` + // BundleType is the type the bundle deployed as (--type, or detected), + // BundlePath the directory bundled, as an absolute path. + BundleType string `json:"bundle_type"` + BundlePath string `json:"bundle_path"` + MountPath string `json:"mount_path"` + // BundleVersion is the version of the uploaded bundle. + BundleVersion string `json:"bundle_version"` + // Waited is true when the run waited for the Deployment to become + // healthy (--wait), and WaitError says why that wait failed; it is + // omitted when the wait succeeded or none was asked for. + Waited bool `json:"waited"` + WaitError string `json:"wait_error,omitempty"` + Git *deployGitJSON `json:"git,omitempty"` +} + +func newBundleDeployJSON(res *astrodeploy.BundleDeploy, bundleType, bundlePath string, waited bool, waitErr error) bundleDeployJSON { + return bundleDeployJSON{ + Deployment: res.DeploymentID, + DeploymentName: res.DeploymentName, + Workspace: res.WorkspaceID, + DeployID: res.DeployID, + BundleType: bundleType, + BundlePath: bundlePath, + MountPath: res.MountPath, + BundleVersion: res.BundleVersion, + Waited: waited, + WaitError: errText(waitErr), + Git: bundleGitJSON(res.Git), + } +} + +// bundleDeleteJSON is what astro deploy bundle --delete --output json +// publishes: the bundle it removed from the Deployment. Astro removes the +// files as the Deployment takes the deploy, which a wait (waited, and +// wait_error when it failed) watches for. workspace is the Deployment's, +// omitted when the Deployment was named and so never read. +type bundleDeleteJSON struct { + Action string `json:"action"` + Deployment string `json:"deployment"` + Workspace string `json:"workspace,omitempty"` + DeployID string `json:"deploy_id"` + BundleType string `json:"bundle_type"` + MountPath string `json:"mount_path"` + Waited bool `json:"waited"` + WaitError string `json:"wait_error,omitempty"` +} + +func newBundleDeleteJSON(res *astrodeploy.BundleDelete, bundleType string, waited bool, waitErr error) bundleDeleteJSON { + return bundleDeleteJSON{ + Action: "deleted", + Deployment: res.DeploymentID, + Workspace: res.WorkspaceID, + DeployID: res.DeployID, + BundleType: bundleType, + MountPath: res.MountPath, + Waited: waited, + WaitError: errText(waitErr), + } +} + +// bundleGitJSON is the commit a bundle deploy recorded, as a deploy result +// publishes it; nil when it recorded none. +func bundleGitJSON(g *astrodeploy.BundleGit) *deployGitJSON { + if g == nil { + return nil + } + return &deployGitJSON{CommitSHA: g.CommitSHA, Branch: g.Branch, CommitURL: g.CommitURL} +} + +// renderBundleUploaded is the text a finished bundle upload has always +// printed. +func renderBundleUploaded(version string) func(io.Writer) error { + return cliout.Text(func(b *bufio.Writer) { + fmt.Fprintf(b, "Successfully uploaded bundle with version %s to Astro.\n", version) + }) +} + +func renderBundleDeleted(mountPath string) func(io.Writer) error { + return cliout.Text(func(b *bufio.Writer) { + fmt.Fprintf(b, "Successfully requested bundle delete for mount path %s from Astro.\n", mountPath) + }) +} + +// publishThenWait publishes a bundle deploy's or delete's result and, with +// --wait, waits for the Deployment to take it (waitFor). publish is handed +// the wait's failure, nil when it succeeded or there was no wait. +// +// The order depends on the format. In text the result comes first, as it +// always has, so a person sees the upload succeed before the wait begins; +// the wait's progress goes to stderr, and its failure is the run's error. +// Under json the one object is the last thing the run writes, after the wait +// it reports on. A wait that fails there still publishes the result, saying +// why, because the upload happened and a script needs its deploy id and +// version, and then exits 1 (failedAfterResult). +func publishThenWaitWith(cmd *cobra.Command, format cliout.Format, wait bool, waitFor func() error, publish func(waitErr error) error) error { + waitDone := func() error { + if !wait { + return nil + } + return waitFor() + } + if format == cliout.FormatJSON { + waitErr := waitDone() + if err := publish(waitErr); err != nil { + // The write failed, so stdout is what broke: the root must not + // try to write the error object there too. failedAfterResult + // marks it shown and puts the words on stderr, the write's + // failure first and the wait's after it, so neither hides the + // other. + return failedAfterResult(cmd, format, errors.Join(err, waitErr)) + } + return failedAfterResult(cmd, format, waitErr) + } + if err := publish(nil); err != nil { + return err + } + return waitDone() +} + +// errText is err's message, or "" for nil. +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/cmd/astro/deploy_bundle_test.go b/cmd/astro/deploy_bundle_test.go new file mode 100644 index 000000000..3fee9719c --- /dev/null +++ b/cmd/astro/deploy_bundle_test.go @@ -0,0 +1,479 @@ +package astro + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/config" + "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/pkg/input" +) + +// execBundleRun runs astro deploy bundle with args under a root, as a user +// runs it. +func execBundleRun(t *testing.T, client astrov1.APIClient, answers string, args ...string) tokenRun { + t.Helper() + return execAstroCmd(t, client, answers, func(io.Writer) *cobra.Command { return NewDeployCmd() }, append([]string{"deploy", "bundle"}, args...)...) +} + +// fakeDeployBundle stands in for the upload, recording what it was asked. +func fakeDeployBundle(t *testing.T) **astrodeploy.DeployBundleInput { + t.Helper() + var captured *astrodeploy.DeployBundleInput + prev := DeployBundle + t.Cleanup(func() { DeployBundle = prev }) + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = in + return astrodeploy.BundleDeploy{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath, BundleVersion: "v1"}, nil + } + return &captured +} + +// fakeDeleteBundle stands in for the delete, recording what it was asked. +func fakeDeleteBundle(t *testing.T) **astrodeploy.DeleteBundleInput { + t.Helper() + var captured *astrodeploy.DeleteBundleInput + prev := DeleteBundle + t.Cleanup(func() { DeleteBundle = prev }) + DeleteBundle = func(in *astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { + captured = in + return astrodeploy.BundleDelete{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath}, nil + } + return &captured +} + +// noBundleTransport fails the test if the run reaches the upload or the +// delete. +func noBundleTransport(t *testing.T) { + t.Helper() + prevDeploy, prevDelete := DeployBundle, DeleteBundle + t.Cleanup(func() { DeployBundle, DeleteBundle = prevDeploy, prevDelete }) + DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + t.Error("nothing is deployed") + return astrodeploy.BundleDeploy{}, nil + } + DeleteBundle = func(*astrodeploy.DeleteBundleInput) (astrodeploy.BundleDelete, error) { + t.Error("nothing is deleted") + return astrodeploy.BundleDelete{}, nil + } +} + +// plainBundle is a directory outside any project that no bundle type +// detects, with the working path a fresh directory beside it. +func plainBundle(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seeds.csv"), []byte("a,b\n"), 0o600)) + prev := config.WorkingPath + config.WorkingPath = t.TempDir() + t.Cleanup(func() { config.WorkingPath = prev }) + return dir +} + +// astro deploy bundle is the subcommand, and any other word after deploy is +// still the Deployment the project deploys to. +func TestDeployBundleRouting(t *testing.T) { + root := deployUnderRoot() + cmd, rest, err := root.Find([]string{"deploy", "bundle", "some/path"}) + require.NoError(t, err) + assert.Equal(t, "astro deploy bundle", cmd.CommandPath()) + assert.Equal(t, []string{"some/path"}, rest) + + cmd, rest, err = root.Find([]string{"deploy", "clx-dep"}) + require.NoError(t, err) + assert.Equal(t, "astro deploy", cmd.CommandPath()) + assert.Equal(t, []string{"clx-dep"}, rest) +} + +// The type decides the defaults: a dbt project is detected and mounts below +// /usr/local/airflow/dbt by its name; --type dbt asks for one; any other type +// is a label that needs --mount-path. +func TestDeployBundleTypes(t *testing.T) { + t.Run("dbt, detected", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeployBundle(t) + r := execBundleRun(t, nil, "", project, "--deployment", "clx-dep", "--description", "nightly") + require.NoError(t, r.err, r.stderr) + in := *captured + require.NotNil(t, in) + assert.Equal(t, project, in.BundlePath) + assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", in.MountPath) + assert.Equal(t, "dbt", in.BundleType) + assert.Equal(t, "clx-dep", in.DeploymentID) + assert.Equal(t, "nightly", in.Description) + assert.Nil(t, in.Deployment, "named by id, so not read") + assert.Nil(t, in.Login, "the current context") + assert.Contains(t, r.stderr, "mount path /usr/local/airflow/dbt/jaffle_shop: the default for a dbt bundle") + assert.Equal(t, "Successfully uploaded bundle with version v1 to Astro.\n", r.stdout) + }) + t.Run("dbt, the working directory and a mount path given", func(t *testing.T) { + project := dbtProject(t) + config.WorkingPath = project + captured := fakeDeployBundle(t) + r := execBundleRun(t, nil, "", "-d", "clx-dep", "-m", "/usr/local/airflow/dbt/custom") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, project, (*captured).BundlePath) + assert.Equal(t, "/usr/local/airflow/dbt/custom", (*captured).MountPath) + assert.Equal(t, "dbt", (*captured).BundleType) + assert.NotContains(t, r.stderr, "the default for") + }) + t.Run("--type dbt, without a dbt project", func(t *testing.T) { + bundle := plainBundle(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", bundle, "--type", "dbt", "--deployment", "clx-dep", "--mount-path", "/x") + require.Error(t, r.err) + assert.Contains(t, r.err.Error(), "no dbt_project.yml in "+bundle) + }) + t.Run("--type of another kind needs a mount path", func(t *testing.T) { + bundle := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", "clx-dep") + require.Error(t, r.err) + assert.True(t, cliout.IsUsage(r.err), "%v", r.err) + assert.Equal(t, "--mount-path is required for a bundle of type seeds: only a dbt bundle has a default", r.err.Error()) + }) + t.Run("--type of another kind, with a mount path", func(t *testing.T) { + bundle := dbtProject(t) + captured := fakeDeployBundle(t) + r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", "clx-dep", "--mount-path", "/usr/local/airflow/seeds") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "seeds", (*captured).BundleType, "a free-form label") + assert.Equal(t, "/usr/local/airflow/seeds", (*captured).MountPath) + }) + t.Run("nothing detected", func(t *testing.T) { + bundle := plainBundle(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", bundle, "--deployment", "clx-dep") + require.Error(t, r.err) + assert.Equal(t, "--mount-path is required for a bundle of type none: only a dbt bundle has a default", r.err.Error()) + + captured := fakeDeployBundle(t) + r = execBundleRun(t, nil, "", bundle, "--deployment", "clx-dep", "--mount-path", "/usr/local/airflow/x") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "none", (*captured).BundleType) + }) +} + +// PATH is a directory of its own: one that does not exist, a file, and one in +// or below a project are refused before anything is read. +func TestDeployBundleRefusesItsPath(t *testing.T) { + oneX := make1xProject(t) + manifestDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(manifestDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + nested := filepath.Join(manifestDir, "include", "dbt") + require.NoError(t, os.MkdirAll(nested, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nested, "dbt_project.yml"), []byte("name: x\n"), 0o600)) + file := filepath.Join(t.TempDir(), "a-file") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o600)) + + for path, want := range map[string]string{ + oneX: "is within an Astro project", + manifestDir: "is within an Astro project", + nested: "is within an Astro project", + filepath.Join(manifestDir, "..", "absent"): "does not exist", + file: "is not a directory", + } { + noBundleTransport(t) + r := execBundleRun(t, nil, "", path, "--deployment", "clx-dep", "--mount-path", "/x") + require.Error(t, r.err, path) + assert.Contains(t, r.err.Error(), want, path) + } + + noBundleTransport(t) + r := execBundleRun(t, nil, "", t.TempDir(), "--deployment", "clx-dep", "--mount-path", "/x", "--wait-time", "1m") + require.Error(t, r.err) + assert.True(t, cliout.IsUsage(r.err)) + assert.Contains(t, r.err.Error(), "cannot use --wait-time with --wait=false") +} + +// --delete removes the bundle at --mount-path, or at the mount path PATH's +// type gives it. +func TestDeployBundleDelete(t *testing.T) { + t.Run("the dbt project's default", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeleteBundle(t) + r := execBundleRun(t, nil, "", "--delete", project, "--deployment", "clx-dep") + require.NoError(t, r.err, r.stderr) + in := *captured + require.NotNil(t, in) + assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", in.MountPath) + assert.Equal(t, "dbt", in.BundleType) + assert.Equal(t, "clx-dep", in.DeploymentID) + assert.Nil(t, in.Login) + assert.Equal(t, "Successfully requested bundle delete for mount path /usr/local/airflow/dbt/jaffle_shop from Astro.\n", r.stdout) + }) + t.Run("a mount path needs no bundle here", func(t *testing.T) { + plainBundle(t) + captured := fakeDeleteBundle(t) + r := execBundleRun(t, nil, "", "--delete", "--deployment", "clx-dep", "--mount-path", "/usr/local/airflow/x", "--type", "seeds") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "/usr/local/airflow/x", (*captured).MountPath) + assert.Equal(t, "seeds", (*captured).BundleType) + }) + t.Run("no mount path and nothing to derive one from", func(t *testing.T) { + bundle := plainBundle(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", "--delete", bundle, "--deployment", "clx-dep") + require.Error(t, r.err) + assert.True(t, cliout.IsUsage(r.err)) + assert.Contains(t, r.err.Error(), "--mount-path is required for a bundle of type none") + + r = execBundleRun(t, nil, "", "--delete", bundle, "--type", "dbt", "--deployment", "clx-dep") + require.Error(t, r.err) + assert.Contains(t, r.err.Error(), "no dbt_project.yml in") + }) +} + +// The target: --deployment, a link of the project here or an id. Naming +// none, a run that cannot be asked is refused as input_required before +// anything is uploaded or deleted. +func TestDeployBundleTarget(t *testing.T) { + for _, del := range []bool{false, true} { + name := "deploy" + if del { + name = "delete" + } + with := func(args ...string) []string { + if del { + return append([]string{"--delete"}, args...) + } + return args + } + t.Run(name+": a link of the project here", func(t *testing.T) { + bundle := dbtProject(t) + require.NoError(t, os.WriteFile(filepath.Join(config.WorkingPath, "pyproject.toml"), []byte("[project]\nname = \"demo\"\ndependencies = [\"apache-airflow==3.1.*\"]\n\n[tool.astro]\n\n[tool.astro.deployments.prod]\ntarget = \"astro\"\nworkspace = \"clw-ws\"\ndeployment = \"clx-prod\"\n"), 0o600)) + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + r := execBundleRun(t, nil, "", with(bundle, "--deployment", "prod")...) + require.NoError(t, r.err, r.stderr) + if del { + assert.Equal(t, "clx-prod", (*deleted).DeploymentID) + } else { + assert.Equal(t, "clx-prod", (*deployed).DeploymentID) + } + }) + t.Run(name+": none named, under json", func(t *testing.T) { + bundle := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", with(bundle, "-o", "json")...) + require.Error(t, r.err) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, string(cliout.KindInputRequired), got.Kind) + assert.Contains(t, got.Error, "pass the Deployment id or a link name with --deployment") + }) + t.Run(name+": none named, without a terminal", func(t *testing.T) { + bundle := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, nil, "", with(bundle)...) + require.Error(t, r.err) + assert.True(t, input.IsRequired(r.err), "%v", r.err) + }) + t.Run(name+": none named, picked", func(t *testing.T) { + bundle := dbtProject(t) + interactiveDeploy(t) + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "1\n", with(bundle, "--workspace", "clxdbtworkspace0000000000")...) + require.NoError(t, r.err, r.stderr) + var read *astrov1.Deployment + if del { + read = (*deleted).Deployment + } else { + read = (*deployed).Deployment + } + require.NotNil(t, read, "the picked Deployment is handed on, not read again") + assert.Equal(t, dbtTestDeploymentID, read.Id) + }) + } +} + +// From a project on another host, a bundle goes under that host's login, by +// link and by id alike, and the picker, which lists only the current +// context's Deployments, is refused. +func TestDeployBundleUsesTheManifestDomainsLogin(t *testing.T) { + for _, del := range []bool{false, true} { + for _, target := range []string{"test", "clx-bare"} { + t.Run(fmt.Sprintf("%s, delete %t", target, del), func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + args := []string{"bundle", t.TempDir(), "--deployment", target, "--mount-path", "/x"} + if del { + args = append(args, "--delete") + } + _, err := execDeployCapture(args...) + require.NoError(t, err) + var login *config.Context + if del { + login = (*deleted).Login + } else { + login = (*deployed).Login + } + require.NotNil(t, login, "under the project's host") + assert.Equal(t, "astronomer.io", login.Domain) + assert.Equal(t, "prod-org", login.Organization) + }) + } + } + + t.Run("the picker on another host", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + interactiveDeploy(t) + noBundleTransport(t) + _, err := execDeployCapture("bundle", t.TempDir(), "--mount-path", "/x") + require.Error(t, err) + assert.Contains(t, err.Error(), "the deployment picker lists only the current context's Deployments") + assert.Contains(t, err.Error(), "astro context switch astronomer.io") + }) + + t.Run("outside a project", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + require.NoError(t, os.Remove(filepath.Join(config.WorkingPath, "pyproject.toml"))) + captured := fakeDeployBundle(t) + _, err := execDeployCapture("bundle", t.TempDir(), "--deployment", "clx-bare", "--mount-path", "/x") + require.NoError(t, err) + assert.Nil(t, (*captured).Login, "the current context") + }) +} + +func TestDeployBundleJSON(t *testing.T) { + t.Run("deploy", func(t *testing.T) { + project := dbtProject(t) + r := execBundleRun(t, dbtBundleMock(t, fakeBlobStore(t)), "", project, "--deployment", dbtTestDeploymentID, "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got bundleDeployJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, bundleDeployJSON{ + Deployment: dbtTestDeploymentID, + DeploymentName: "analytics", + Workspace: "clxdbtworkspace0000000000", + DeployID: dbtTestDeployID, + BundleType: "dbt", + BundlePath: project, + MountPath: "/usr/local/airflow/dbt/jaffle_shop", + BundleVersion: dbtTestVersion, + }, got) + assert.NotContains(t, fields, "git", "the project is in no git checkout") + assert.NotContains(t, fields, "wait_error") + }) + // Named by id, the Deployment is never read, so its Workspace is not + // known and not published. + t.Run("delete", func(t *testing.T) { + plainBundle(t) + r := execBundleRun(t, dbtBundleMock(t, ""), "", "--delete", "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/dbt/custom", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + var got bundleDeleteJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, bundleDeleteJSON{ + Action: "deleted", + Deployment: dbtTestDeploymentID, + DeployID: dbtTestDeployID, + BundleType: "none", + MountPath: "/usr/local/airflow/dbt/custom", + }, got) + assert.NotContains(t, fields, "workspace") + }) +} + +// A --wait's progress is on stderr in both formats. In text the result comes +// first, so stdout is what it is without --wait; under json the object comes +// after the wait it reports, and says it waited, and why the wait failed. +func TestDeployBundleWait(t *testing.T) { + for _, del := range []bool{false, true} { + name := "deploy" + if del { + name = "delete" + } + run := func(t *testing.T, more ...string) tokenRun { + project := dbtProject(t) + args := []string{project, "--deployment", dbtTestDeploymentID} + var client astrov1.APIClient + if del { + client = dbtBundleMock(t, "") + args = append(args, "--delete") + } else { + client = dbtBundleMock(t, fakeBlobStore(t)) + } + return execBundleRun(t, client, "", append(args, more...)...) + } + t.Run(name+" text", func(t *testing.T) { + without := run(t) + require.NoError(t, without.err) + waited := stubBundleWait(t, nil) + r := run(t, "--wait") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + assert.Equal(t, []string{dbtTestDeploymentID}, *waited) + assert.Equal(t, without.stdout, r.stdout) + assert.Contains(t, r.stderr, "Waiting for the deployment to become healthy…") + }) + t.Run(name+" json, the wait failing", func(t *testing.T) { + stubBundleWait(t, errors.New("timed out")) + r := run(t, "--wait", "-o", "json") + require.Error(t, r.err) + assert.Equal(t, cliout.ExitFailure, r.code) + var got struct { + DeployID string `json:"deploy_id"` + Waited bool `json:"waited"` + WaitError string `json:"wait_error"` + } + decodeOne(t, r.stdout, &got) + assert.Equal(t, dbtTestDeployID, got.DeployID) + assert.True(t, got.Waited) + assert.Equal(t, "timed out", got.WaitError) + }) + t.Run(name+" json", func(t *testing.T) { + waited := stubBundleWait(t, nil) + r := run(t, "--wait", "-o", "json") + require.NoError(t, r.err, "stderr:\n%s", r.stderr) + assert.Equal(t, []string{dbtTestDeploymentID}, *waited) + var got struct { + Waited bool `json:"waited"` + } + fields := decodeOne(t, r.stdout, &got) + assert.True(t, got.Waited) + assert.NotContains(t, fields, "wait_error") + }) + } +} + +// astro dbt is a tombstone: every subcommand, and the group bare, says what +// replaced it, in text and as the one error object under json, and runs +// nothing, whatever 1.x flags it is passed. +func TestDbtRemoved(t *testing.T) { + cases := map[string]string{ + "": errDbtRemoved, + "deploy": errDbtDeployRemoved, + "delete": errDbtDeleteRemoved, + "cleanup": errDbtCleanupRemoved, + } + for sub, want := range cases { + args := []string{"dbt"} + if sub != "" { + args = append(args, sub, "clx-dep", "--project-path", t.TempDir(), "-n", "analytics") + } + noBundleTransport(t) + r := execAstroCmd(t, nil, "", func(io.Writer) *cobra.Command { return newDbtRemovedCmd() }, args...) + require.Error(t, r.err, sub) + assert.Equal(t, cliout.ExitUsage, r.code, sub) + assert.Equal(t, want, r.err.Error(), sub) + assert.NotContains(t, want, "`", "command names are plain text") + assert.Empty(t, r.stdout, sub) + + r = execAstroCmd(t, nil, "", func(io.Writer) *cobra.Command { return newDbtRemovedCmd() }, append(args, "-o", "json")...) + require.Error(t, r.err, sub) + assert.Equal(t, cliout.ExitUsage, r.code, sub) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, want, got.Error, sub) + } +} diff --git a/cmd/astro/deploy_help.go b/cmd/astro/deploy_help.go index 8ebbdfaec..a7d60f37e 100644 --- a/cmd/astro/deploy_help.go +++ b/cmd/astro/deploy_help.go @@ -1,8 +1,6 @@ package astro import ( - "strings" - "github.com/spf13/cobra" ) @@ -10,9 +8,8 @@ import ( // apply to which kind of deploy. Each value is the section's title: the root's // help (cmd/help.go) lists a flag annotated "Image" under "Image Flags:". const ( - deployGroupImage = "Image" - deployGroupDAG = "Dag" - deployGroupNonDAG = "Non-Dag Bundle" + deployGroupImage = "Image" + deployGroupDAG = "Dag" ) // The annotation keys cmd/help.go reads a flag's section and the sections' @@ -29,12 +26,10 @@ func annotateDeployFlag(cmd *cobra.Command, name, group string) { } // orderDeployFlagGroups lists deploy's sections in the order a deploy narrows: -// what image ships, which DAGs, and what rides beside. +// what image ships, then which DAGs. func orderDeployFlagGroups(cmd *cobra.Command) { if cmd.Annotations == nil { cmd.Annotations = map[string]string{} } - cmd.Annotations[flagGroupOrderAnnotation] = strings.Join([]string{ - deployGroupImage, deployGroupDAG, deployGroupNonDAG, - }, ",") + cmd.Annotations[flagGroupOrderAnnotation] = deployGroupImage + "," + deployGroupDAG } diff --git a/cmd/astro/deploy_manifest_test.go b/cmd/astro/deploy_manifest_test.go index b8b67dd9f..070bf263b 100644 --- a/cmd/astro/deploy_manifest_test.go +++ b/cmd/astro/deploy_manifest_test.go @@ -54,7 +54,6 @@ func resetDeployFlagVars() { workspaceID = "" deploymentName = "" deployDescription = "" - nonDags = false deployOutput = string(cliout.FormatText) } @@ -1167,97 +1166,6 @@ func TestDeployManifestUsesTheManifestDomainsLogin(t *testing.T) { } } -// --non-dags from a project deploys under the project's host as the project's -// deploy does, a link and a bare Deployment id alike; outside a project a -// bare id goes under the current context. -func TestDeployNonDagsUsesTheManifestDomainsLogin(t *testing.T) { - for _, args := range [][]string{{"test"}, {"--deployment", "clx-bare"}} { - t.Run(strings.Join(args, " "), func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - var captured *astrodeploy.DeployBundleInput - prev := DeployBundle - t.Cleanup(func() { DeployBundle = prev }) - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - - _, err := execDeployCapture(append(args, "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", t.TempDir())...) - require.NoError(t, err) - require.NotNil(t, captured.Login, "deployed under the project's host") - assert.Equal(t, "astronomer.io", captured.Login.Domain) - assert.Equal(t, "prod-org", captured.Login.Organization) - }) - } - - t.Run("outside a project", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - require.NoError(t, os.Remove(filepath.Join(config.WorkingPath, "pyproject.toml"))) - var captured *astrodeploy.DeployBundleInput - prev := DeployBundle - t.Cleanup(func() { DeployBundle = prev }) - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - _, err := execDeployCapture("--deployment", "clx-bare", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", t.TempDir()) - require.NoError(t, err) - assert.Nil(t, captured.Login, "the current context") - }) -} - -// --non-dags naming no Deployment: a run that cannot be asked is refused as -// astro deploy's is, input_required, before any picker; one that can is -// offered the picker only on the current context's host, so from a project -// on another host it is refused with the same words astro deploy uses. -func TestDeployNonDagsWithNoTarget(t *testing.T) { - noBundle := func(t *testing.T) { - prev := DeployBundle - t.Cleanup(func() { DeployBundle = prev }) - DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - t.Fatal("no target, so nothing is deployed") - return astrodeploy.BundleDeploy{}, nil - } - } - nonDags := []string{"--non-dags", "--non-dags-mount-path", "/x"} - - t.Run("under --output json", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - noBundle(t) - out, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir(), "--output", "json")...) - require.Error(t, err) - var obj struct { - Error string `json:"error"` - Kind string `json:"kind"` - } - require.NoError(t, json.Unmarshal([]byte(out), &obj), out) - assert.Equal(t, "input_required", obj.Kind) - assert.Contains(t, obj.Error, "pass the Deployment id as the argument or with --deployment") - }) - - t.Run("without a terminal", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - noBundle(t) - _, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir())...) - require.Error(t, err) - assert.True(t, input.IsRequired(err), "%v", err) - }) - - t.Run("the picker on another host", func(t *testing.T) { - setupDeployOnStage(t, manifestOnProd) - logInToProd(t) - interactiveDeploy(t) - noBundle(t) - _, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir())...) - require.Error(t, err) - assert.Contains(t, err.Error(), "the deployment picker lists only the current context's Deployments") - assert.Contains(t, err.Error(), "astro context switch astronomer.io") - }) -} - // ASTRO_API_TOKEN outranks the stored login, and goes out with its scheme like // a stored token, since the deploy's CI/CD check splits it off. func TestDeployManifestSendsTheAPITokenToTheManifestDomain(t *testing.T) { diff --git a/cmd/astro/deploy_test.go b/cmd/astro/deploy_test.go index 526a95133..eaf10cd41 100644 --- a/cmd/astro/deploy_test.go +++ b/cmd/astro/deploy_test.go @@ -1,7 +1,6 @@ package astro import ( - "encoding/json" "errors" "fmt" "io" @@ -9,17 +8,14 @@ import ( "path/filepath" "testing" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "github.com/astronomer/astro-cli/cmd/cliout" "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/config" manifestdeploy "github.com/astronomer/astro-cli/internal/deploy" - "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/internal/project" testUtil "github.com/astronomer/astro-cli/pkg/testing" ) @@ -239,7 +235,7 @@ func TestDeployRefusesFlagsThatCannotAllApply(t *testing.T) { // passes --force, and a pyproject.toml project's deploy always took both. func TestDeployDropsThe1xOnlyFlags(t *testing.T) { cmd := NewDeployCmd() - for _, name := range []string{"save", "pytest", "parse", "test", "env", "dags-path", "dag-bundle-name", "deployment-name"} { + for _, name := range []string{"save", "pytest", "parse", "test", "env", "dags-path", "dag-bundle-name", "deployment-name", "non-dags", "non-dags-mount-path", "non-dags-local-path", "non-dags-bundle-type"} { assert.Nil(t, cmd.Flags().Lookup(name), "--%s", name) } for _, letter := range []string{"s", "t", "e", "n"} { @@ -252,215 +248,16 @@ func TestDeployDropsThe1xOnlyFlags(t *testing.T) { } } -type NonDagsDeploySuite struct { - suite.Suite - mockV1Client *astrov1_mocks.ClientWithResponsesInterface - origV1Client astrov1.APIClient - origDeployBundle func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) - origWorkingPath string - origWd string - tmpWorkingDir string -} - -func (s *NonDagsDeploySuite) SetupTest() { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - // Run from an isolated temp directory that is not within an Astro project, so - // the non-DAG bundle path validation (which walks up to a .astro/config.yaml) - // is deterministic regardless of where the repo lives. - tmpDir, err := os.MkdirTemp("", "non-dags-test") - s.Require().NoError(err) - s.tmpWorkingDir = tmpDir - s.origWd, err = os.Getwd() - s.Require().NoError(err) - s.Require().NoError(os.Chdir(tmpDir)) - s.origWorkingPath = config.WorkingPath - config.WorkingPath = tmpDir - - s.origV1Client = astroV1Client - s.origDeployBundle = DeployBundle - s.mockV1Client = new(astrov1_mocks.ClientWithResponsesInterface) - astroV1Client = s.mockV1Client -} - -func (s *NonDagsDeploySuite) TearDownTest() { - s.mockV1Client.AssertExpectations(s.T()) - astroV1Client = s.origV1Client - DeployBundle = s.origDeployBundle - config.WorkingPath = s.origWorkingPath - if s.origWd != "" { - _ = os.Chdir(s.origWd) - } - if s.tmpWorkingDir != "" { - _ = os.RemoveAll(s.tmpWorkingDir) - } -} - -func TestNonDagsDeploy(t *testing.T) { - suite.Run(t, new(NonDagsDeploySuite)) -} - -func (s *NonDagsDeploySuite) TestRequiresMountPath() { - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags") - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "--non-dags-mount-path is required") -} - -func (s *NonDagsDeploySuite) TestBundleTypeDefaultsToNone() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = deployInput - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") - assert.NoError(s.T(), err) - s.Require().NotNil(captured) - assert.Equal(s.T(), "none", captured.BundleType) -} - -func (s *NonDagsDeploySuite) TestRejectsIncompatibleFlag() { - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt", "--dags") - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "cannot use --dags with --non-dags") -} - -func (s *NonDagsDeploySuite) TestProvidedDeploymentId() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = deployInput - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt") - assert.NoError(s.T(), err) - s.Require().NotNil(captured) - assert.Equal(s.T(), "test-deployment-id", captured.DeploymentID) - assert.Equal(s.T(), "/usr/local/airflow/x", captured.MountPath) - assert.Equal(s.T(), "dbt", captured.BundleType) - assert.Equal(s.T(), s.tmpWorkingDir, captured.BundlePath) -} - -// A bundle path in or below a project, by the walk every deploy decides by, -// is refused: a 1.x project's Dockerfile beside .astro (with no -// .astro/config.yaml), and a pyproject.toml project. -func (s *NonDagsDeploySuite) TestWithinAstroProject() { - oneX := s.T().TempDir() - s.Require().NoError(os.WriteFile(filepath.Join(oneX, "Dockerfile"), []byte("FROM x\n"), 0o600)) - s.Require().NoError(os.MkdirAll(filepath.Join(oneX, ".astro"), 0o755)) - s.Require().NoError(os.WriteFile(filepath.Join(oneX, ".astro", "config.yaml"), []byte("project:\n name: demo\n"), 0o600)) - manifestDir := s.T().TempDir() - s.Require().NoError(os.WriteFile(filepath.Join(manifestDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) - s.Require().NoError(os.MkdirAll(filepath.Join(manifestDir, "include", "dbt"), 0o755)) - - for _, bundle := range []string{oneX, filepath.Join(manifestDir, "include", "dbt")} { - resetDeployFlagVars() - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt", "--non-dags-local-path", bundle) - s.Require().Error(err, bundle) - s.Contains(err.Error(), "within an Astro project", bundle) - } -} - -func (s *NonDagsDeploySuite) TestBundlePathDoesNotExist() { - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", filepath.Join(s.tmpWorkingDir, "missing")) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "does not exist") -} - -func (s *NonDagsDeploySuite) TestBundlePathNotADirectory() { - file := filepath.Join(s.tmpWorkingDir, "a-file") - s.Require().NoError(os.WriteFile(file, []byte("x"), 0o600)) - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", file) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "is not a directory") -} - -// --deployment names a non-Dag bundle's Deployment as the argument does. -func (s *NonDagsDeploySuite) TestDeploymentFlag() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = deployInput - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "--deployment", "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") - assert.NoError(s.T(), err) - s.Require().NotNil(captured) - assert.Equal(s.T(), "test-deployment-id", captured.DeploymentID) -} - -// --non-dags runs before the project decides anything: from a pyproject.toml -// project it deploys a bundle, not the project, so the project directory -// itself is refused as the bundle. It used to run a whole project deploy, -// dropping --non-dags. -func (s *NonDagsDeploySuite) TestFromAManifestProject() { - s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) - DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - s.Fail("a bundle inside the project is not deployed") - return astrodeploy.BundleDeploy{}, nil - } - - err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") - assert.ErrorContains(s.T(), err, "within an Astro project") -} - -// --output json publishes the bundle deploy as one object; a bad --output is -// a usage error before anything is read. -func (s *NonDagsDeploySuite) TestOutput() { - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - return astrodeploy.BundleDeploy{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath, BundleVersion: "v1"}, nil +func testExecCmd(cmd *cobra.Command, args ...string) error { + if args == nil { + args = []string{} } - resetDeployFlagVars() - out, err := execDeployCapture("test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--output", "json") - s.Require().NoError(err, out) - var got nonDagsDeployJSON - s.Require().NoError(json.Unmarshal([]byte(out), &got), out) - s.Equal(nonDagsDeployJSON{Deployment: "test-deployment-id", DeployID: "dep-1", BundleType: "none", BundlePath: s.tmpWorkingDir, MountPath: "/usr/local/airflow/x", BundleVersion: "v1"}, got) - - resetDeployFlagVars() - _, err = execDeployCapture("test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--output", "yaml") - s.True(cliout.IsUsage(err), "%v", err) -} - -// The target is named as a project deploy names it: the argument and -// --deployment must agree, and a link name of the project here is its -// Deployment. --workspace is the workspace whose Deployments are offered. -func (s *NonDagsDeploySuite) TestTarget() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - bundle := s.T().TempDir() - - resetDeployFlagVars() - err := testExecCmd(NewDeployCmd(), "dep-a", "--deployment", "dep-b", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) - s.ErrorContains(err, "name one") - s.True(cliout.IsUsage(err)) - - s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\ndependencies = [\"apache-airflow==3.1.*\"]\n\n[tool.astro]\n\n[tool.astro.deployments.prod]\ntarget = \"astro\"\nworkspace = \"clw-ws\"\ndeployment = \"clx-prod\"\n"), 0o600)) - resetDeployFlagVars() - err = testExecCmd(NewDeployCmd(), "prod", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) - s.Require().NoError(err) - s.Equal("clx-prod", captured.DeploymentID) - s.Nil(captured.Login, "a project naming no host deploys under the current context") -} - -// A Deployment id needs no manifest: a pyproject.toml here that does not load -// does not stop a deploy that names its target by id. -func (s *NonDagsDeploySuite) TestIDNeedsNoManifest() { - var captured *astrodeploy.DeployBundleInput - DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { - captured = in - return astrodeploy.BundleDeploy{}, nil - } - s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[tool.astro]\nruntime = [\n"), 0o600)) - bundle := s.T().TempDir() - - resetDeployFlagVars() - err := testExecCmd(NewDeployCmd(), "--deployment", "clx-by-id", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) - s.Require().NoError(err) - s.Equal("clx-by-id", captured.DeploymentID) - s.Nil(captured.Login) + // deploy sets the package's workspaceID from --workspace-id or the one it + // resolves, and coalesceWorkspace reads it before the context. Left set, + // it would choose the Workspace of whatever runs next. + defer func() { workspaceID = "" }() + testUtil.SetupOSArgsForGinkgo() + cmd.SetArgs(args) + _, err := cmd.ExecuteC() + return err } diff --git a/cmd/astro/deployment_token_json_test.go b/cmd/astro/deployment_token_json_test.go index d1d33bb4d..62d263cd4 100644 --- a/cmd/astro/deployment_token_json_test.go +++ b/cmd/astro/deployment_token_json_test.go @@ -166,7 +166,7 @@ func execAstroCmd(t *testing.T, client astrov1.APIClient, answers string, newRoo // The token ids are package variables a positional argument sets and no // flag registration resets, so one run's id would answer the next. tokenID, orgTokenID, workspaceTokenID = "", "", "" - // --workspace-id, and deploy and dbt when they resolve one, set the + // --workspace-id, and deploy when it resolves one, set the // package's workspaceID, which coalesceWorkspace reads before the context. // Clear what an earlier run left, and what this one leaves. workspaceID = "" diff --git a/cmd/astro/output_binding_test.go b/cmd/astro/output_binding_test.go index 93509c225..6eab3f88f 100644 --- a/cmd/astro/output_binding_test.go +++ b/cmd/astro/output_binding_test.go @@ -38,8 +38,9 @@ func execUnboundRootWith(t *testing.T, client astrov1.APIClient, alpha astrov1al testUtil.InitTestConfig(testUtil.LocalPlatform) testUtil.SetupOSArgsForGinkgo() prevClient, prevAlpha := astroV1Client, astroV1Alpha1Client - // dbt deploy resolves a Workspace into the package's workspaceID, which - // coalesceWorkspace reads before the context; see execDeployCmd. + // A bundle deploy may resolve a Workspace into the package's + // workspaceID, which coalesceWorkspace reads before the context; see + // execDeployCmd. t.Cleanup(func() { astroV1Client, astroV1Alpha1Client, workspaceID = prevClient, prevAlpha, "" }) outR, outW, perr := os.Pipe() @@ -75,20 +76,20 @@ func execUnboundRootWith(t *testing.T, client astrov1.APIClient, alpha astrov1al // Under --output json the result is on stdout and nothing of it on stderr, // in the tree production builds. func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { - t.Run("dbt deploy", func(t *testing.T) { + t.Run("deploy bundle", func(t *testing.T) { project := dbtProject(t) - stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "dbt", "deploy", dbtTestDeploymentID, "--project-path", project, "-o", "json") + stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", "bundle", project, "--deployment", dbtTestDeploymentID, "-o", "json") require.NoError(t, err, "stderr:\n%s", stderr) - var got dbtDeployJSON + var got bundleDeployJSON decodeOne(t, stdout, &got) assert.Equal(t, dbtTestVersion, got.BundleVersion) assert.NotContains(t, stderr, dbtTestVersion, "the result went to stderr") - assert.Contains(t, stderr, "Initiating dbt deploy", "the notes go to stderr") + assert.Contains(t, stderr, "mount path /usr/local/airflow/dbt/jaffle_shop", "the notes go to stderr") }) // The bundle sits in a git checkout with uncommitted changes, so the // platform code prints its note on the way: under json that goes to // stderr, and stdout is the one result object. - t.Run("deploy --non-dags from a dirty checkout", func(t *testing.T) { + t.Run("deploy bundle from a dirty checkout", func(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("needs git") } @@ -103,11 +104,10 @@ func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { gitIn("add", ".") gitIn("commit", "-q", "-m", "first") require.NoError(t, os.WriteFile(filepath.Join(bundle, "dbt_project.yml"), []byte("name: changed\n"), 0o600)) - resetDeployFlagVars() - stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", bundle, "--output", "json") + stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", "bundle", bundle, "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/x", "--output", "json") require.NoError(t, err, "stderr:\n%s", stderr) - var got nonDagsDeployJSON + var got bundleDeployJSON decodeOne(t, stdout, &got) assert.Equal(t, dbtTestVersion, got.BundleVersion) assert.Contains(t, stderr, "uncommitted changes", "the platform's note goes to stderr") diff --git a/cmd/astro/remote.go b/cmd/astro/remote.go index ad6441124..b0b1ceac8 100644 --- a/cmd/astro/remote.go +++ b/cmd/astro/remote.go @@ -38,7 +38,9 @@ func newRemoteRootCmd(out io.Writer) *cobra.Command { Short: "Manage remote deploys and images", Long: "Commands for interacting with remote registries and deploying client images", } - // Bound for the same reason as dbt: see newDbtCmd. + // The results leave by cmd.OutOrStdout(), which must be the stdout the + // tree was built with: under json strayStdoutToStderr points os.Stdout + // at stderr, and an unbound command would follow it there. cmd.SetOut(out) cmd.AddCommand(newRemoteDeployCmd()) diff --git a/cmd/astro/remote_render.go b/cmd/astro/remote_render.go new file mode 100644 index 000000000..83c01d4b6 --- /dev/null +++ b/cmd/astro/remote_render.go @@ -0,0 +1,76 @@ +package astro + +// Rendering for astro remote deploy. The platform package returns what +// happened; this file decides how it looks, in text and in json. + +import ( + "bufio" + "fmt" + "io" + + "github.com/astronomer/astro-cli/cmd/cliout" + astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/pkg/ansi" +) + +// remoteDeployJSON is what `astro remote deploy --output json` publishes: the +// client image it pushed, which is what the agents' Helm values then name. +type remoteDeployJSON struct { + Image string `json:"image"` + Registry string `json:"registry"` + Tag string `json:"tag"` + // SourceImage is the local image pushed (--image-name), omitted when the + // run built one. + SourceImage string `json:"source_image,omitempty"` + // Platforms are the --platform the image was built for, [] for the host's + // or when nothing was built. + Platforms []string `json:"platforms"` + // RuntimeCheck is the check against --deployment, omitted without one. + RuntimeCheck *remoteRuntimeCheckJSON `json:"runtime_check,omitempty"` +} + +type remoteRuntimeCheckJSON struct { + Deployment string `json:"deployment"` + ClientRuntimeVersion string `json:"client_runtime_version"` + DeploymentRuntimeVersion string `json:"deployment_runtime_version"` +} + +func newRemoteDeployJSON(res *astrodeploy.ClientDeploy) remoteDeployJSON { + obj := remoteDeployJSON{ + Image: res.Image, + Registry: res.Registry, + Tag: res.Tag, + SourceImage: res.SourceImage, + Platforms: nonNil(res.Platforms), + } + if c := res.RuntimeCheck; c != nil { + obj.RuntimeCheck = &remoteRuntimeCheckJSON{ + Deployment: c.DeploymentID, + ClientRuntimeVersion: c.ClientRuntimeVersion, + DeploymentRuntimeVersion: c.DeploymentRuntimeVersion, + } + } + return obj +} + +// renderRemoteDeploy is the text a finished client image push has always +// printed: where it went, and what to do with it next. +func renderRemoteDeploy(image string) func(io.Writer) error { + return cliout.Text(func(b *bufio.Writer) { + fmt.Fprintf(b, "Successfully pushed client image to %s\n", ansi.Bold(image)) + fmt.Fprintf(b, "\n--------------------------------\n") + fmt.Fprintln(b, "The client image has been pushed to your private registry.") + fmt.Fprintln(b, "Your next step would be to update the agent component to use the new client image.") + fmt.Fprintln(b, "For that you would either need to update the helm chart values.yaml file or update your CI/CD pipeline to use the new client image.") + fmt.Fprintf(b, "If you are using Astronomer provided Agent Helm chart, you would need to update the image field for each of the workers, dagProcessor, and triggerer component sections to the new image: %s\n", image) + fmt.Fprintln(b, "Once you have updated the helm chart values.yaml file, you can run 'helm upgrade' or update via your CI/CD pipeline to update the agent components") + }) +} + +// nonNil is s, or [] for nil, so a list publishes as [] and never null. +func nonNil(s []string) []string { + if s == nil { + return []string{} + } + return s +} diff --git a/cmd/astro/root.go b/cmd/astro/root.go index 40bbd1f93..29ef43da0 100644 --- a/cmd/astro/root.go +++ b/cmd/astro/root.go @@ -29,7 +29,7 @@ func AddCmds(v1Client astrov1.APIClient, v1Alpha1Client astrov1alpha1.APIClient, newEnvRootCmd(out), newWorkspaceCmd(out), newOrganizationCmd(out), - newDbtCmd(out), + newDbtRemovedCmd(), newIDECommand(out), newRemoteRootCmd(out), } diff --git a/cmd/astro/schema_test.go b/cmd/astro/schema_test.go index 46113d3de..5eea888e8 100644 --- a/cmd/astro/schema_test.go +++ b/cmd/astro/schema_test.go @@ -153,11 +153,10 @@ var publishedPayloads = []cliouttest.Case{ // astro deploy --output json, in a project with a manifest: the one // object a finished deploy prints. {Name: "deploy", Value: deployJSON{}}, - {Name: "deploy-non-dags", Value: nonDagsDeployJSON{}}, - // astro dbt deploy: what it deployed where. delete: the bundle it + // astro deploy bundle: what it deployed where. --delete: the bundle it // removed. - {Name: "dbt-deploy", Value: dbtDeployJSON{}}, - {Name: "dbt-delete", Value: dbtDeleteJSON{}}, + {Name: "deploy-bundle", Value: bundleDeployJSON{}}, + {Name: "deploy-bundle-delete", Value: bundleDeleteJSON{}}, // astro remote deploy: the client image it pushed. {Name: "remote-deploy", Value: remoteDeployJSON{}}, @@ -234,7 +233,7 @@ func TestEveryGoldenHasACase(t *testing.T) { // minWatchedPayloads is a floor under the tally, not a target: an empty tally // reads exactly like a clean one, so without it the observer coming unwired -// would be silent. Fifty-two shapes reach Emit in this package's tests +// would be silent. Fifty-one shapes reach Emit in this package's tests // today: the deployment variable results (2) and the six API token results; // the deployment, workspace and organization user and team results, with the // organization's invite and team membership (10); the pkg/output lists their @@ -243,14 +242,16 @@ func TestEveryGoldenHasACase(t *testing.T) { // deployment inspect's deployment (which create and update publish too), // delete's removal, hibernate's override, the worker-queue result and the // log entry (6); the bundle a bundle create or update leaves and what a -// bundle delete did (2); the dbt deploy, delete and cleanup results and the remote -// deploy's pushed image (4); the Workspace a create, update or switch +// bundle delete did (2); astro deploy bundle's deploy and delete results and the +// remote deploy's pushed image (3); the Workspace a create, update or switch // publishes and what a Workspace delete did, what an Organization switch left // current, the role list and the file an audit-log export wrote (5); the // Astro IDE project list and what an import or an export moved (3); and the // error object. -// Raise it as conversions land; lower it only saying why. -const minWatchedPayloads = 52 +// Raise it as conversions land; lower it only saying why. It went from 52 +// to 51 when astro dbt deploy, astro dbt delete and astro deploy --non-dags, +// three shapes, became astro deploy bundle's two. +const minWatchedPayloads = 51 // pinnedElsewhere names the shapes that reach Emit here and are pinned by // another tree's goldens, keyed by type, with where. Pinning one twice would diff --git a/cmd/astro/testdata/schema/dbt-deploy.json b/cmd/astro/testdata/schema/dbt-deploy.json deleted file mode 100644 index 10aff5029..000000000 --- a/cmd/astro/testdata/schema/dbt-deploy.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "deployment": "x", - "deployment_name": "x", - "workspace": "x", - "deploy_id": "x", - "project": "x", - "project_path": "x", - "mount_path": "x", - "bundle_version": "x", - "waited": true, - "wait_error": "x", - "git": { - "commit_sha": "x", - "branch": "x", - "commit_url": "x" - } -} diff --git a/cmd/astro/testdata/schema/dbt-delete.json b/cmd/astro/testdata/schema/deploy-bundle-delete.json similarity index 86% rename from cmd/astro/testdata/schema/dbt-delete.json rename to cmd/astro/testdata/schema/deploy-bundle-delete.json index 3360e5d06..81d630849 100644 --- a/cmd/astro/testdata/schema/dbt-delete.json +++ b/cmd/astro/testdata/schema/deploy-bundle-delete.json @@ -3,6 +3,7 @@ "deployment": "x", "workspace": "x", "deploy_id": "x", + "bundle_type": "x", "mount_path": "x", "waited": true, "wait_error": "x" diff --git a/cmd/astro/testdata/schema/deploy-non-dags.json b/cmd/astro/testdata/schema/deploy-bundle.json similarity index 100% rename from cmd/astro/testdata/schema/deploy-non-dags.json rename to cmd/astro/testdata/schema/deploy-bundle.json diff --git a/cmd/local/link.go b/cmd/local/link.go index ce027b6d6..f36b381dd 100644 --- a/cmd/local/link.go +++ b/cmd/local/link.go @@ -340,6 +340,11 @@ func (c *cli) runLinkAdd(cmd *cobra.Command, in *linkAddInput, name string, comm return err } name = strings.TrimSpace(name) + // bundle is astro deploy bundle, so a link by that name could not be + // deployed to by name: refused before anything is asked. + if name == deploy.BundleCommand { + return cliout.Usage(deploy.ErrBundleLinkName) + } // A NAME already linked is refused before anything asks a question or // calls the platform. if err := refuseExisting(m, name, in.replace); err != nil { @@ -448,7 +453,7 @@ func (c *cli) pickAstroDeployment(link *scaffold.Link, m *manifest.Manifest, nam return name, nil } name = scaffold.LinkNameFor(dep.Name) - if name == "" || name == manifest.ReservedLinkName { + if name == "" || name == manifest.ReservedLinkName || name == deploy.BundleCommand { return "", fmt.Errorf("the Deployment's name does not make a link name: pass one, astro link add NAME --%s %s", flagLinkDeployment, dep.ID) } return name, nil diff --git a/cmd/local/link_test.go b/cmd/local/link_test.go index ece7ed3f1..3d3c1a569 100644 --- a/cmd/local/link_test.go +++ b/cmd/local/link_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/astronomer/astro-cli/cmd/cliout" "github.com/astronomer/astro-cli/internal/userstate" "github.com/astronomer/astro-cli/pkg/instances" "github.com/astronomer/astro-cli/pkg/manifest" @@ -161,6 +162,17 @@ func TestLinkAddRefusesAnExistingNameUnlessReplacing(t *testing.T) { assert.True(t, dev.Default, "the replace dropped the default") } +// bundle names astro deploy bundle, so astro deploy could never reach a link +// by that name: it is refused, before anything is asked or written. +func TestLinkAddRefusesBundleAsAName(t *testing.T) { + dir, _ := linkTestProject(t, linkManifest) + _, _, err := runLink(t, dir, "add", "bundle", "--deployment", "dep-new") + require.Error(t, err) + assert.True(t, cliout.IsUsage(err), "%v", err) + assert.Contains(t, err.Error(), "bundle cannot name a link: astro deploy bundle deploys a non-DAG bundle") + assert.Equal(t, linkManifest, readManifest(t, dir)) +} + // An MWAA link writes the shared region when given one, and an empty --region // removes it. func TestLinkAddAnMWAALink(t *testing.T) { diff --git a/cmd/removed_flags.go b/cmd/removed_flags.go index 1e77d4fdd..adb405e16 100644 --- a/cmd/removed_flags.go +++ b/cmd/removed_flags.go @@ -52,7 +52,14 @@ const ( errDeployDagsPathRemoved = "--dags-path was removed in Astro CLI v2: astro deploy --dags ships the project's dags directory, so run it from the project" errDeployBundleNameRemoved = "--dag-bundle-name was removed in Astro CLI v2: deploying to a named DAG bundle is not supported yet. Use Astro CLI 1.x for it" errDeployNameRemoved = "--deployment-name was removed in Astro CLI v2: name the Deployment as the argument or with --deployment, which take a link name or a Deployment id" - errAPCDeployNoCache = "--no-cache was removed in Astro CLI v2: Astro CLI v2 builds no image for Astro Private Cloud. " + + // astro deploy --non-dags became astro deploy bundle, a subcommand of + // its own with the bundle's directory as its argument. + errDeployNonDagsRemoved = "--non-dags was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment , " + + "which deploys a non-DAG bundle such as a dbt project from its directory" + errDeployNonDagsMountPathRemoved = "--non-dags-mount-path was removed in Astro CLI v2: use astro deploy bundle --mount-path" + errDeployNonDagsLocalPathRemoved = "--non-dags-local-path was removed in Astro CLI v2: pass the bundle's directory as the argument, astro deploy bundle " + errDeployNonDagsBundleTypeRemoved = "--non-dags-bundle-type was removed in Astro CLI v2: use astro deploy bundle --type, which detects a dbt project by itself" + errAPCDeployNoCache = "--no-cache was removed in Astro CLI v2: Astro CLI v2 builds no image for Astro Private Cloud. " + "Build and deploy the project with Astro CLI 1.x" ) @@ -132,6 +139,10 @@ var removedFlags = []removedFlag{ {name: "dag-bundle-name", under: []string{"deploy"}, msg: errDeployBundleNameRemoved}, {name: "deployment-name", under: []string{"deploy"}, msg: errDeployNameRemoved}, {name: "no-cache", under: []string{"deploy"}, msg: errAPCDeployNoCache}, + {name: "non-dags", under: []string{"deploy"}, msg: errDeployNonDagsRemoved}, + {name: "non-dags-mount-path", under: []string{"deploy"}, msg: errDeployNonDagsMountPathRemoved}, + {name: "non-dags-local-path", under: []string{"deploy"}, msg: errDeployNonDagsLocalPathRemoved}, + {name: "non-dags-bundle-type", under: []string{"deploy"}, msg: errDeployNonDagsBundleTypeRemoved}, } // removedFlagErr is a run refused for passing a removed 1.x flag. diff --git a/cmd/removed_flags_test.go b/cmd/removed_flags_test.go index bbda7cf73..62ad93cf8 100644 --- a/cmd/removed_flags_test.go +++ b/cmd/removed_flags_test.go @@ -150,6 +150,10 @@ var removedFlagCases = []removedFlagCase{ {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "dag-bundle-name", value: "nightly", want: errDeployBundleNameRemoved}, {tree: astroTree, args: []string{"deploy"}, flag: "deployment-name", value: "prod", shorthand: "n", want: errDeployNameRemoved}, {tree: apcTree, args: []string{"deploy", "dep-id"}, flag: "no-cache", want: errAPCDeployNoCache}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "non-dags", want: errDeployNonDagsRemoved}, + {tree: astroHostedTree, args: []string{"deploy"}, flag: "non-dags-mount-path", value: "/usr/local/airflow/x", want: errDeployNonDagsMountPathRemoved}, + {tree: astroTree, args: []string{"deploy"}, flag: "non-dags-local-path", value: "./analytics", want: errDeployNonDagsLocalPathRemoved}, + {tree: astroTree, args: []string{"deploy"}, flag: "non-dags-bundle-type", value: "dbt", want: errDeployNonDagsBundleTypeRemoved}, } // spellings is every way the case's flag can be typed: --name, --name=value diff --git a/cmd/root.go b/cmd/root.go index 9cac3137a..2e897443d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -230,7 +230,7 @@ var commandGroups = []struct { }{ {groupDevelop, "Develop locally:", []string{"init", "local", "start", "stop", "logs"}}, {groupInspect, "Inspect Airflow:", []string{"af", "use", "link", "api"}}, - {groupShip, "Ship:", []string{"package", "deploy", "remote", "dbt"}}, + {groupShip, "Ship:", []string{"package", "deploy", "remote"}}, {groupManage, "Manage Astro:", []string{ "login", "logout", "auth", "context", "organization", "workspace", "deployment", "env", diff --git a/docs/architecture.md b/docs/architecture.md index 61fbabfba..bca9bba1f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -205,7 +205,7 @@ A failure with no kind publishes no `kind` key rather than an `unknown` catch-al A script written for Astro CLI 1.x is told what replaced what it typed rather than only that it is unknown. Either way the failure is a usage error: exit 2, and the `usage` error object under `--output json`. -A removed command stays in the tree as a hidden stub that fails naming its replacement: `astro dev` ([`cmd/local/dev.go`](../cmd/local/dev.go)), `astro run`, `astro deployment airflow-variable|connection|pool`, and `astro env … create|update`. Every stub is built by `cliout.RemovedCommand` ([`cmd/cliout/removed.go`](../cmd/cliout/removed.go)): hidden, with flag parsing off so an old invocation's flags reach it, a `Short` starting `Removed in v2`, and the `removedCommand` annotation. A stub has a pre-run of its own that runs the root's and no group's, so it never reaches the project lookup of `astro deployment` or `astro env`. The root's pre-run reads the annotation. It reads `--verbosity` from the raw arguments, sets up logging and records the command (`telemetry.TrackRemovedCommand`), and runs no login or version check. So a machine that is logged out, or a project that does not parse, is still told what to run. The event is sent only once telemetry is set up, meaning the first-run notice was shown on that machine. A stub never shows the notice itself, because the core tree leaves no state behind. So the counts miss fresh machines and CI runners that start fresh: a count near zero is not on its own a reason to remove a stub. A stub fails with a usage error whose message says `` `astro …` was removed in Astro CLI v2 `` and names the command to use instead, or says there is none. `astro dev` publishes the error object (`cliout.ErrorObject`, embedded) under `--output json`, with its whole mapping of 1.x `dev` commands besides. +A removed command stays in the tree as a hidden stub that fails naming its replacement: `astro dev` ([`cmd/local/dev.go`](../cmd/local/dev.go)), `astro run`, `astro dbt` (whose `deploy` and `delete` moved to `astro deploy bundle`), `astro deployment airflow-variable|connection|pool`, and `astro env … create|update`. Every stub is built by `cliout.RemovedCommand` ([`cmd/cliout/removed.go`](../cmd/cliout/removed.go)): hidden, with flag parsing off so an old invocation's flags reach it, a `Short` starting `Removed in v2`, and the `removedCommand` annotation. A stub has a pre-run of its own that runs the root's and no group's, so it never reaches the project lookup of `astro deployment` or `astro env`. The root's pre-run reads the annotation. It reads `--verbosity` from the raw arguments, sets up logging and records the command (`telemetry.TrackRemovedCommand`), and runs no login or version check. So a machine that is logged out, or a project that does not parse, is still told what to run. The event is sent only once telemetry is set up, meaning the first-run notice was shown on that machine. A stub never shows the notice itself, because the core tree leaves no state behind. So the counts miss fresh machines and CI runners that start fresh: a count near zero is not on its own a reason to remove a stub. A stub fails with a usage error whose message says `` `astro …` was removed in Astro CLI v2 `` and names the command to use instead, or says there is none. `astro dev` publishes the error object (`cliout.ErrorObject`, embedded) under `--output json`, with its whole mapping of 1.x `dev` commands besides. [`cmd/testdata/v1_commands.tsv`](../cmd/testdata/v1_commands.tsv) is the inventory of 1.x commands, in the same three trees as the flag inventory below. Only the tests read it, since a stub needs no inventory to be found. `TestEveryV1CommandStillRunsOrSaysWhatReplacedIt` resolves every path in it against its v2 trees, APC at both ends of its version gates. It passes a path that names a command v2 has, at its own name or a 1.x name kept as an alias, or that reaches a stub, which it then runs, in text and under `--output json`, with every other hook disarmed. Below APC's version gates a path may also reach the stand-in for a command the platform is too old for; in any other tree that fails. It fails on a path that stops at a command with words left over, which is a command dropped without a stub or renamed without an alias. To remove a command, replace it with a stub; to rename one, keep its 1.x name as an alias. `TestEveryRemovedCommandStubIsA1xCommand` fails on a stub that no 1.x path reaches, and `TestRemovedCommandStubsNeedNoLoginAndAreRecorded` runs stubs on the real tree to check that none reaches the login or the project lookup and that each is recorded. diff --git a/docs/deploy.md b/docs/deploy.md index 6c80412bd..318c09993 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -14,9 +14,9 @@ v2 builds and deploys only projects with a `pyproject.toml`, on Astro and on Ast `astro deploy` looks at the working directory. A `pyproject.toml` with a `[tool.astro]` table is deployed, even if the table fails to validate (the deploy then reports the manifest error). A Dockerfile is not evidence of a 1.x project: a project with a manifest may declare one (`[tool.astro] dockerfile`). The routing is in [`cmd/astro/deploy.go`](../cmd/astro/deploy.go). -Anywhere else, one walk decides (`utils.Locate`, used by `astro deploy`, `astro remote deploy`, APC's deploy, and the bundle-path checks of `--non-dags` and `astro dbt deploy`). It is `astro init`'s walk (`scaffold.Find1xProject`), so the deploy refuses as 1.x exactly where init and the local commands' advice say a 1.x project is. From the working directory up, at each directory: a `pyproject.toml` that is a project's stops it (`scaffold.HasManifest`): one whose `[tool.astro]` loads, or one that fails to validate, to parse or to be read, so the deploy reports why. Only a `pyproject.toml` that parses and has no `[tool.astro]` (a monorepo root's `[tool.ruff]`) is passed over; a broken one cannot be told apart from a broken manifest, so it stops the walk with its parse error, a 1.x checkout's included. Else a 1.x project stops it (`scaffold.Is1xProject`): a Dockerfile and a `.astro/config.yaml` that is not the CLI's own settings file (one with a `project` key, or no `context`, `contexts` or `telemetry`), so a stray Dockerfile in the home directory beside the global config is not one, while a home directory that is a 1.x project is. Else the walk goes on up. Each directory's `pyproject.toml` is loaded once, and the deploy uses the manifest the walk loaded rather than reading the file again. +Anywhere else, one walk decides (`utils.Locate`, used by `astro deploy`, `astro remote deploy`, APC's deploy, and the bundle-path check of `astro deploy bundle`). It is `astro init`'s walk (`scaffold.Find1xProject`), so the deploy refuses as 1.x exactly where init and the local commands' advice say a 1.x project is. From the working directory up, at each directory: a `pyproject.toml` that is a project's stops it (`scaffold.HasManifest`): one whose `[tool.astro]` loads, or one that fails to validate, to parse or to be read, so the deploy reports why. Only a `pyproject.toml` that parses and has no `[tool.astro]` (a monorepo root's `[tool.ruff]`) is passed over; a broken one cannot be told apart from a broken manifest, so it stops the walk with its parse error, a 1.x checkout's included. Else a 1.x project stops it (`scaffold.Is1xProject`): a Dockerfile and a `.astro/config.yaml` that is not the CLI's own settings file (one with a `project` key, or no `context`, `contexts` or `telemetry`), so a stray Dockerfile in the home directory beside the global config is not one, while a home directory that is a 1.x project is. Else the walk goes on up. Each directory's `pyproject.toml` is loaded once, and the deploy uses the manifest the walk loaded rather than reading the file again. -| the walk finds | `astro deploy`, any mode but `--non-dags` | +| the walk finds | `astro deploy` | | --- | --- | | a 1.x project, here or above | refused: `this project uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), and Astro CLI v2 deploys only pyproject.toml projects. Convert it with astro init, or deploy it with Astro CLI 1.x`. From below it, it names the project: `this directory is inside a project at that uses …. Convert it with astro init in , …` | | a `pyproject.toml` project above | refused: `this directory is inside the project at . Run the deploy from the project directory, ` | @@ -26,11 +26,16 @@ Every refusal is reported under `--output json` as the error object with kind `n **`--image-name ` outside any project** deploys an image already built on this machine, alone: an image-only deploy, with no `dags/` and no git commit. The target is the argument or `--deployment`, a Deployment id (there are no links without a manifest), or the workspace's Deployments to pick from; a run that cannot ask says to pass the id as the argument or with `--deployment`. The login is the current context's. The deploy warns, on stderr and in the result's `warnings`: a Deployment that takes DAG deploys keeps the DAGs it had, and one that does not now runs the DAGs inside the image, if any. At a `pyproject.toml` project's root, `--image-name` is the project's deploy with a prebuilt image, `dags/` included as usual. -**`--non-dags`** ships a separate directory as a non-DAG bundle, which must not be inside any project, so it is checked first and runs anywhere. A bundle path in or below any project the walk finds is refused, as `astro dbt deploy` refuses a dbt project there: the two use the same check. +**`astro deploy bundle [PATH]`** ships a directory of its own, PATH or the working directory, as a non-DAG bundle: a dbt project, or any other files the DAGs read, mounted at a path in Airflow. It carries no DAGs, and it is not a project deploy: PATH must be a directory, and one in or below any project the walk finds is refused. **`--delete`** removes the bundle mounted at a path instead. It replaces Astro CLI 1.x's `astro dbt deploy` and `astro dbt delete` and the 1.x `astro deploy --non-dags`, which did the same upload with different defaults and Deployment selection. `astro deployment bundle` is something else: it lists and changes the bundle registrations on a Deployment, and uploads nothing. -- **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project here whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. -- **The target** is the argument or `--deployment`, which must agree: a link of the project here, or a Deployment id. With neither, the Deployments of `--workspace` (or `--workspace-id`, or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id as the argument or with --deployment`, as `astro deploy` does. A project on another host is told to pass `--deployment ` or switch context. -- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-non-dags.json`. What the deploy prints on the way (the git note, for one) goes to stderr there. +Because `bundle` is a subcommand, `astro deploy bundle` never means a Deployment called bundle. `astro link add` refuses `bundle` as a link name, and a deploy that links a picked Deployment names it otherwise; a link already called `bundle` is deployed to with `astro deploy --deployment bundle`. + +- **The type** decides the defaults. The types are a table in [`internal/platform/astro/deploy/bundletype.go`](../internal/platform/astro/deploy/bundletype.go), each with how to detect it, how to check a bundle of it, and its default mount path; adding a type is adding an entry. With no `--type`, the first type that detects PATH is used, else `none`. Today the table has one entry, `dbt`: a directory holding `dbt_project.yml`, mounted by default at `/usr/local/airflow/dbt/` from the project's `name`. `--type dbt` asks for a dbt project and fails without one. Any other `--type` (and `none`) is a free-form label the deploy records, with no defaults, so it needs `--mount-path`. A default mount path is said on stderr. +- **Delete** uses `--mount-path` as given, or the mount path PATH's type would deploy to, so `astro deploy bundle --delete` in a dbt project removes that project. With neither, it is a usage error. +- **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project in the working directory whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. +- **The target** is `--deployment` (`-d`): a link of the project in the working directory, or a Deployment id. Without it, the Deployments of `--workspace` (or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id or a link name with --deployment`. A project on another host is told to pass `--deployment ` or switch context. +- **`--wait`** waits for the Deployment to take the deploy or the delete, under the same login. In text the result prints first and the wait's progress goes to stderr; under json the one object comes after the wait, with `waited` and, when it failed, `wait_error`, and the run exits 1. +- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-bundle.json` and, for `--delete`, `deploy-bundle-delete.json`. Both carry `bundle_type`, `mount_path`, `deployment` and `deploy_id`; the deploy adds `bundle_path`, `bundle_version` and `git`. What the deploy prints on the way (the git note, the default mount path) goes to stderr there. **Flags that cannot all apply** are refused as usage errors before anything is read: `--wait-time` without `--wait`, `--no-dags-base-dir` with a deploy that ships no DAGs (`--image`, or `--image-name` outside a project), and `--dags` with `--image` or `--image-name` (the message names the ones given). @@ -251,7 +256,7 @@ astro package [TARGET] [flags] # astro (default), mwaa, composer, oss -o, --output text|json ``` -The flags only a 1.x project's deploy read are gone: `--save` (`-s`), `--pytest`, `--env` (`-e`), `--test` (`-t`), `--parse`, `--deployment-name` (`-n`), `--dags-path` and `--dag-bundle-name`, and on Astro Private Cloud `--save` and `--no-cache`. A run passing one fails as a usage error naming what replaced it (run the tests with `uv run pytest` first, check the DAGs with `astro local check`, name the Deployment with `--deployment`, and so on; see [Removed 1.x commands and flags](architecture.md#removed-1x-commands-and-flags)). `--force` (`-f`) and `--prompt` (`-p`) are accepted, hidden and read by nothing: astronomer/deploy-action passes `--force` on every deploy, and the deploy always asks unless a Deployment is named, which is what `--prompt` asked for. `--non-dags` and its `--non-dags-*` flags deploy a non-DAG bundle from a separate directory (see above). +The flags only a 1.x project's deploy read are gone: `--save` (`-s`), `--pytest`, `--env` (`-e`), `--test` (`-t`), `--parse`, `--deployment-name` (`-n`), `--dags-path` and `--dag-bundle-name`, and on Astro Private Cloud `--save` and `--no-cache`. A run passing one fails as a usage error naming what replaced it (run the tests with `uv run pytest` first, check the DAGs with `astro local check`, name the Deployment with `--deployment`, and so on; see [Removed 1.x commands and flags](architecture.md#removed-1x-commands-and-flags)). `--force` (`-f`) and `--prompt` (`-p`) are accepted, hidden and read by nothing: astronomer/deploy-action passes `--force` on every deploy, and the deploy always asks unless a Deployment is named, which is what `--prompt` asked for. `--non-dags` and its `--non-dags-*` flags are gone too, for `astro deploy bundle` (see above). **Git metadata.** With the `deploy.git_metadata` setting on (the default), the deploy records the HEAD commit on Astro, and the commit message becomes the description when `--description` is not given. `commit_url` is set only for a GitHub remote. A project with uncommitted changes to tracked files records no commit and prints `note: the project has uncommitted changes, so this deploy records no git commit` on stderr. `--image-name` records none. diff --git a/docs/upgrading-from-v1.md b/docs/upgrading-from-v1.md index 0936d8f68..5d9b74c60 100644 --- a/docs/upgrading-from-v1.md +++ b/docs/upgrading-from-v1.md @@ -104,6 +104,8 @@ Everything below fails with an error naming the replacement, instead of an "unkn | `astro deployment connection copy` | no per-Deployment copy: set it once in the workspace with `astro env connection set --workspace --auto-link`, or on each Deployment | | `astro deployment pool …` | the Airflow UI or API, until pools reach the Environment Manager | | `astro env … create`, `astro env … update` | `astro env … set`, which creates or updates | +| `astro dbt deploy [DEPLOYMENT_ID]` | `astro deploy bundle [PATH] --deployment `: PATH replaces `--project-path`, `--deployment` the argument and `--deployment-name`, `--workspace` `--workspace-id` | +| `astro dbt delete [DEPLOYMENT_ID]` | `astro deploy bundle --delete [PATH] --deployment `, with the same flags | Run inside a 1.x project, an `astro dev` command also tells you to convert it with `astro init`. One with a replacement says so in its first line (convert, then use the replacement); bare `astro dev`, `upgrade-test` and `proxy` say it after the list of `astro local` commands. @@ -129,8 +131,12 @@ Run inside a 1.x project, an `astro dev` command also tells you to convert it wi | `--dags-path` | `astro deploy --dags` from the project, which ships its `dags/` | `astro deploy` | | `--dag-bundle-name` | no replacement yet: use Astro CLI 1.x | `astro deploy` | | `--no-cache` | no replacement: v2 builds no image for Astro Private Cloud | `astro deploy` on Astro Private Cloud | +| `--non-dags` | `astro deploy bundle [PATH]`, which detects a dbt project and gives it a mount path | `astro deploy` | +| `--non-dags-local-path` | the PATH argument of `astro deploy bundle` | `astro deploy` | +| `--non-dags-mount-path` | `astro deploy bundle --mount-path` | `astro deploy` | +| `--non-dags-bundle-type` | `astro deploy bundle --type` | `astro deploy` | -Some 1.x flags still work but are hidden from help in favor of a new spelling. On the `astro deployment` subcommands, `astro env` and `astro dbt deploy` / `delete`, `--deployment-name` (`-n`) and `--deployment-id`, whichever the command has, give way to `--deployment`. `--workspace-id` gives way to `--workspace` on those commands and the other commands that have both, `astro deploy` among them. +Some 1.x flags still work but are hidden from help in favor of a new spelling. On the `astro deployment` subcommands and `astro env`, `--deployment-name` (`-n`) and `--deployment-id`, whichever the command has, give way to `--deployment`. `--workspace-id` gives way to `--workspace` on those commands and the other commands that have both, `astro deploy` among them. ### Behavior diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 2147f2cd5..4b635786b 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -633,7 +633,7 @@ func unlinkedDeployments(req Request, d Deployer, links map[string]manifest.Link } return deps[i].ID < deps[j].ID }) - taken := map[string]bool{manifest.ReservedLinkName: true} + taken := map[string]bool{manifest.ReservedLinkName: true, BundleCommand: true} for name := range links { taken[name] = true } @@ -663,7 +663,7 @@ func newLinkName(dep Deployment, taken map[string]bool) string { // A name of digits alone would shadow a row number: the prompt matches // names first, so a Deployment called "1" would take the answer "1" from // whatever row 1 is. - if base == "" || base == manifest.ReservedLinkName || isDigits(base) { + if base == "" || base == manifest.ReservedLinkName || base == BundleCommand || isDigits(base) { base = dep.ID } name := base @@ -715,7 +715,7 @@ func listWorkspace(req Request) string { // argument (linkName) and --deployment, which must agree, each a link of m // (nil for no manifest) or a Deployment id. named is false when neither names // one. It is for a command that ships something other than the project, such -// as astro deploy --non-dags, to name its Deployment the way a deploy does. +// as astro deploy bundle, to name its Deployment the way a deploy does. func ResolveNamed(m *manifest.Manifest, linkName, deployment, workspaceID, contextWorkspace string) (target Target, named bool, err error) { req := Request{Manifest: m, LinkName: linkName, Deployment: deployment, WorkspaceID: workspaceID, ContextWorkspace: contextWorkspace} var links map[string]manifest.Link @@ -776,9 +776,19 @@ func unlinkedTarget(req Request, d Deployer) (Target, error) { if dep.WorkspaceID == "" { dep.WorkspaceID = workspaceID } - return linkedTarget(req, newLinkName(dep, map[string]bool{manifest.ReservedLinkName: true}), dep), nil + return linkedTarget(req, newLinkName(dep, map[string]bool{manifest.ReservedLinkName: true, BundleCommand: true}), dep), nil } +// BundleCommand is the word astro deploy bundle is spelled with. astro +// deploy reads it as that subcommand, never as a link name, so no link is +// saved under it: astro link add refuses it, and a link a deploy saves for a +// picked Deployment is named otherwise. A link by that name already in a +// manifest is still reached with --deployment. +const BundleCommand = "bundle" + +// ErrBundleLinkName is astro link add refusing BundleCommand as a link name. +var ErrBundleLinkName = fmt.Errorf("%[1]s cannot name a link: astro deploy %[1]s deploys a non-DAG bundle, so astro deploy %[1]s would never reach the link. Name the link something else", BundleCommand) + // DefaultMark is the word that marks a project's default link in a list of // links: the deploy prompt's "linked, default", and the `astro use` and `astro // link` pickers. One word in every place, never the manifest's own diff --git a/internal/deploy/deploy_test.go b/internal/deploy/deploy_test.go index 649a70c5f..602d5c1f7 100644 --- a/internal/deploy/deploy_test.go +++ b/internal/deploy/deploy_test.go @@ -538,6 +538,7 @@ func TestResolveTarget_OffersTheWorkspacesUnlinkedDeployments(t *testing.T) { {ID: "dep-new", Name: "Orders (EU)"}, // offered as orders-eu {ID: "dep-clash", Name: "Staging"}, // "staging" is a link already {ID: "dep-local", Name: "Local"}, // "local" is reserved + {ID: "dep-bundle", Name: "Bundle"}, // "bundle" is astro deploy bundle {ID: "dep-other", Name: "Orders (EU)", WorkspaceID: "ws-1"}, // same name twice } req := Request{Interactive: true, Manifest: manifestWith(links), ContextWorkspace: "ws-1"} @@ -548,7 +549,7 @@ func TestResolveTarget_OffersTheWorkspacesUnlinkedDeployments(t *testing.T) { _, err := resolveTarget(req, d) require.NoError(t, err) assert.Equal(t, "ws-1", d.listedWS) - assert.Equal(t, []string{"dev", "prod", "staging", "dep-local", "orders-eu", "orders-eu-2", "staging-2"}, choiceNames(d.offered)) + assert.Equal(t, []string{"dev", "prod", "staging", "dep-bundle", "dep-local", "orders-eu", "orders-eu-2", "staging-2"}, choiceNames(d.offered)) for _, c := range d.offered { assert.Equal(t, !slices.Contains([]string{"dev", "prod", "staging"}, c.Name), c.Unlinked, c.Name) } diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index 787111646..0c096260a 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -171,6 +171,9 @@ type DeleteBundleInput struct { BundleType string Description string AstroV1Client astrov1.APIClient + // Login is the Astro login the delete runs under, whose host the + // Deployment lives on; nil is the current context. + Login *config.Context } // BundleDelete is what a bundle delete did: the deploy that removes the @@ -188,7 +191,7 @@ type BundleDelete struct { // It does not report the result: its caller renders the BundleDelete it // returns. func DeleteBundle(input *DeleteBundleInput) (BundleDelete, error) { - c, err := config.GetCurrentContext() + c, err := loginOrCurrent(input.Login) if err != nil { return BundleDelete{}, err } diff --git a/internal/platform/astro/deploy/bundletype.go b/internal/platform/astro/deploy/bundletype.go new file mode 100644 index 000000000..d97a85460 --- /dev/null +++ b/internal/platform/astro/deploy/bundletype.go @@ -0,0 +1,124 @@ +package deploy + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v2" +) + +// BundleType is a kind of non-DAG bundle that astro deploy bundle knows by +// its contents: how to tell a directory holds one, how to check it before +// anything is uploaded, and where it mounts when the deploy names no mount +// path. Adding a type is adding an entry to bundleTypes. +type BundleType struct { + // Name is the type recorded on the deploy, and what --type selects. + Name string + // Detect reports whether dir holds a bundle of this type, for a deploy + // that names no type. nil detects nothing. + Detect func(dir string) bool + // Validate checks dir is a bundle of this type. nil accepts any + // directory. + Validate func(dir string) error + // DefaultMountPath is where the bundle at dir mounts when no mount path + // is given. nil means the type has no default, so a mount path is + // required. + DefaultMountPath func(dir string) (string, error) +} + +// BundleTypeNone is the type of a bundle that names no type and that no +// registered type detects. It is a label alone, with no defaults. +const BundleTypeNone = "none" + +// DbtDefaultMountPathPrefix is where a dbt project mounts by default, below +// a directory named after the project. +const DbtDefaultMountPathPrefix = "/usr/local/airflow/dbt/" + +const dbtProjectFile = "dbt_project.yml" + +// bundleTypes is every bundle type with defaults, in the order detection +// tries them. +var bundleTypes = []BundleType{ + { + Name: "dbt", + Detect: isDbtProject, + Validate: validateDbtProject, + DefaultMountPath: dbtDefaultMountPath, + }, +} + +// ResolveBundleType is the type a bundle at dir deploys as. name is the type +// asked for: a registered type's name (in any case) selects that type, and +// any other is a free-form label with no defaults, as the deploy API takes +// any label. An empty name is the first registered type that detects dir, +// else BundleTypeNone. +func ResolveBundleType(name, dir string) BundleType { + if name == "" { + for _, t := range bundleTypes { + if t.Detect != nil && t.Detect(dir) { + return t + } + } + return BundleType{Name: BundleTypeNone} + } + for _, t := range bundleTypes { + if strings.EqualFold(t.Name, name) { + return t + } + } + return BundleType{Name: name} +} + +// BundleTypeNames is the name of every registered bundle type, in order. +func BundleTypeNames() []string { + names := make([]string, 0, len(bundleTypes)) + for _, t := range bundleTypes { + names = append(names, t.Name) + } + return names +} + +func isDbtProject(dir string) bool { + info, err := os.Stat(filepath.Join(dir, dbtProjectFile)) + return err == nil && info.Mode().IsRegular() +} + +func validateDbtProject(dir string) error { + path := filepath.Join(dir, dbtProjectFile) + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("no %s in %s: a dbt bundle is the directory that holds the dbt project's %s", dbtProjectFile, dir, dbtProjectFile) + } + return err +} + +// dbtDefaultMountPath is DbtDefaultMountPathPrefix and the project's name, +// from its dbt_project.yml. +func dbtDefaultMountPath(dir string) (string, error) { + name, err := dbtProjectName(dir) + if err != nil { + return "", fmt.Errorf("dbt project name not found in %s: %w", dir, err) + } + return DbtDefaultMountPathPrefix + name, nil +} + +func dbtProjectName(dir string) (string, error) { + path := filepath.Join(dir, dbtProjectFile) + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("could not open %s: %w", path, err) + } + defer file.Close() + var project map[string]any + if err := yaml.NewDecoder(file).Decode(&project); err != nil { + return "", fmt.Errorf("could not decode %s: %w", path, err) + } + name, ok := project["name"].(string) + if !ok || name == "" { + return "", errors.New("invalid dbt project name") + } + return name, nil +} diff --git a/internal/platform/astro/deploy/bundletype_test.go b/internal/platform/astro/deploy/bundletype_test.go new file mode 100644 index 000000000..aadbbb5d4 --- /dev/null +++ b/internal/platform/astro/deploy/bundletype_test.go @@ -0,0 +1,60 @@ +package deploy + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func dbtDir(t *testing.T, project string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "dbt_project.yml"), []byte(project), 0o600)) + return dir +} + +// The type a bundle deploys as, and the defaults it brings. +func TestResolveBundleType(t *testing.T) { + dbt := dbtDir(t, "name: jaffle_shop\n") + plain := t.TempDir() + + t.Run("a dbt project, detected", func(t *testing.T) { + bt := ResolveBundleType("", dbt) + assert.Equal(t, "dbt", bt.Name) + require.NoError(t, bt.Validate(dbt)) + mount, err := bt.DefaultMountPath(dbt) + require.NoError(t, err) + assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", mount) + }) + t.Run("dbt asked for, in any case", func(t *testing.T) { + for _, name := range []string{"dbt", "DBT"} { + bt := ResolveBundleType(name, plain) + assert.Equal(t, "dbt", bt.Name, name) + assert.ErrorContains(t, bt.Validate(plain), "no dbt_project.yml in "+plain, name) + } + }) + t.Run("nothing detected", func(t *testing.T) { + bt := ResolveBundleType("", plain) + assert.Equal(t, BundleType{Name: BundleTypeNone}, bt) + }) + t.Run("any other type is a label with no defaults", func(t *testing.T) { + for _, name := range []string{"seeds", BundleTypeNone} { + bt := ResolveBundleType(name, dbt) + assert.Equal(t, BundleType{Name: name}, bt, "even where a dbt project is") + } + }) + t.Run("a dbt project with no name", func(t *testing.T) { + nameless := dbtDir(t, "version: 2\n") + _, err := ResolveBundleType("", nameless).DefaultMountPath(nameless) + assert.ErrorContains(t, err, "dbt project name not found") + }) + t.Run("a directory called dbt_project.yml is no dbt project", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir, "dbt_project.yml"), 0o755)) + assert.Equal(t, BundleTypeNone, ResolveBundleType("", dir).Name) + }) + assert.Equal(t, []string{"dbt"}, BundleTypeNames()) +} diff --git a/internal/platform/astro/deploy/cosmosboost_hook_test.go b/internal/platform/astro/deploy/cosmosboost_hook_test.go index a560f2905..34e5c8dd0 100644 --- a/internal/platform/astro/deploy/cosmosboost_hook_test.go +++ b/internal/platform/astro/deploy/cosmosboost_hook_test.go @@ -193,7 +193,7 @@ func TestUploadBundleSkipsPreDeployByDefault(t *testing.T) { } // TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled drives the full -// `astro dbt deploy` bundle path (DeployBundle into UploadBundle). +// `astro deploy bundle` path (DeployBundle into UploadBundle). func TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled(t *testing.T) { setupCosmosBoostEnv(t) enableCosmosBoost(t) @@ -217,7 +217,7 @@ func TestDeployBundleDbtPathGeneratesSidecarsWhenEnabled(t *testing.T) { }) require.NoError(t, err) - assert.Contains(t, uploaded(), cosmosBoostArtifact, "astro dbt deploy must ship the sidecar") + assert.Contains(t, uploaded(), cosmosBoostArtifact, "astro deploy bundle must ship the sidecar") assert.NoDirExists(t, filepath.Join(bundleDir, ".astro")) } @@ -300,7 +300,7 @@ func TestUploadBundleMissingDirIsUnchanged(t *testing.T) { } } -// TestUploadBundleFromTheWorkingDirectory: astro dbt deploy --project-path . +// TestUploadBundleFromTheWorkingDirectory: astro deploy bundle . // bundles ".", from the working directory, and an old sidecar there is still // replaced by the fresh one, at the same path, with no second copy under // another name. Nor does the tarball carry itself: it is written outside the From b163dbfba53318a761bab860be26e5b54d8f7161 Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 15:32:22 -0400 Subject: [PATCH 6/7] Resolve deploy bundle's Deployment by name, infer a delete's type, and reserve bundle in scaffold Review of the first cut: - --deployment resolves as the other commands do: a link of the project here, else a Deployment id, else a Deployment's name in the workspace (--workspace, the project's, the login's). No match, or several (listed by id), is an error. - One resolveBundle settles type, check and mount path for deploy and delete. PATH is checked only when the mount path is derived from it, so --type dbt with --mount-path labels any directory, as 1.x's --non-dags-bundle-type dbt did. A delete with no bundle at PATH takes its type from the mount path (BundleType.MountPathPrefix, also the default's directory). - The JSON gains bundle_name (a dbt project's name), from the table's BundleName; the docs map 1.x's project/project_path to it. - bundle is refused as a link name by scaffold's link validation, so every writer, Astro Desktop included, refuses it; astro link add keeps its early usage error. - No -d on deploy bundle (it is --dags on astro deploy); -t is --wait-time again, as on 1.x's astro dbt. - The astro dbt stub reads the subcommand past flags and their values. Co-Authored-By: Claude --- cmd/astro/dbt.go | 50 ++++- cmd/astro/deploy_bundle.go | 186 +++++++++++++----- cmd/astro/deploy_bundle_test.go | 117 +++++++++-- .../testdata/schema/deploy-bundle-delete.json | 1 + cmd/astro/testdata/schema/deploy-bundle.json | 1 + cmd/local/link.go | 6 +- cmd/removed_flags.go | 2 +- docs/deploy.md | 11 +- docs/upgrading-from-v1.md | 4 +- internal/deploy/deploy.go | 10 +- internal/platform/astro/deploy/bundletype.go | 79 +++++--- .../platform/astro/deploy/bundletype_test.go | 27 ++- pkg/scaffold/linkwrite.go | 13 ++ pkg/scaffold/linkwrite_test.go | 1 + 14 files changed, 382 insertions(+), 126 deletions(-) diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index 024364c0d..88aa0f4e5 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -1,6 +1,8 @@ package astro import ( + "strings" + "github.com/spf13/cobra" ) @@ -12,10 +14,10 @@ import ( const ( errDbtRemoved = "astro dbt was removed in Astro CLI v2: use astro deploy bundle, which deploys a dbt project " + "(or any other non-DAG bundle) from its directory, and astro deploy bundle --delete, which removes one" - errDbtDeployRemoved = "astro dbt deploy was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment . " + + errDbtDeployRemoved = "astro dbt deploy was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment . " + "PATH replaces --project-path, --deployment replaces the argument and --deployment-name, and --workspace replaces --workspace-id; " + "--mount-path, --description, --wait and --wait-time are unchanged" - errDbtDeleteRemoved = "astro dbt delete was removed in Astro CLI v2: use astro deploy bundle --delete [PATH] --deployment . " + + errDbtDeleteRemoved = "astro dbt delete was removed in Astro CLI v2: use astro deploy bundle --delete [PATH] --deployment . " + "PATH replaces --project-path, --deployment replaces the argument and --deployment-name, and --workspace replaces --workspace-id; " + "--mount-path, --description, --wait and --wait-time are unchanged" errDbtCleanupRemoved = "astro dbt cleanup was removed in Astro CLI v2, with no replacement: " + @@ -36,10 +38,8 @@ func newDbtRemovedCmd() *cobra.Command { // dbtRemovedGuidance names the replacement for the subcommand typed, the // first argument that is not a flag. func dbtRemovedGuidance(args []string) string { - if len(args) > 0 { - if msg, ok := dbtSubcommandGuidance[args[0]]; ok { - return msg - } + if msg, ok := dbtSubcommandGuidance[dbtTypedSubcommand(args)]; ok { + return msg } return errDbtRemoved } @@ -50,3 +50,41 @@ var dbtSubcommandGuidance = map[string]string{ "delete": errDbtDeleteRemoved, "cleanup": errDbtCleanupRemoved, } + +// dbtValueFlags are the flags that took a value on 1.x's astro dbt commands +// (v1_flags.tsv), and the --output and --verbosity v2 reads anywhere, so a +// flag's value typed before the subcommand is not read as the subcommand. +var dbtValueFlags = map[string]bool{ + "deployment-name": true, "n": true, + "description": true, + "mount-path": true, "m": true, + "project-path": true, "p": true, + "wait-time": true, "t": true, + "workspace-id": true, + "output": true, "o": true, + "verbosity": true, +} + +// dbtTypedSubcommand is the first argument that is neither a flag nor a +// value one took: astro dbt -n prod delete names delete. It stops at "--". +func dbtTypedSubcommand(args []string) string { + for i := 0; i < len(args); i++ { + a := args[i] + switch { + case a == "--": + return "" + case strings.HasPrefix(a, "--"): + if name := a[2:]; !strings.Contains(name, "=") && dbtValueFlags[name] { + i++ + } + case strings.HasPrefix(a, "-") && len(a) > 1: + // -n prod takes the next word; -nprod carries its value. + if len(a) == 2 && dbtValueFlags[a[1:]] { + i++ + } + default: + return a + } + } + return "" +} diff --git a/cmd/astro/deploy_bundle.go b/cmd/astro/deploy_bundle.go index 3241a26a5..81687b0be 100644 --- a/cmd/astro/deploy_bundle.go +++ b/cmd/astro/deploy_bundle.go @@ -2,6 +2,7 @@ package astro import ( "bufio" + "cmp" "context" "errors" "fmt" @@ -21,6 +22,7 @@ import ( astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/internal/platform/astro/deployment" "github.com/astronomer/astro-cli/pkg/input" + "github.com/astronomer/astro-cli/pkg/util" ) // The transport astro deploy bundle calls, vars so a test can stand in for @@ -59,10 +61,12 @@ func newDeployBundleCmd() *cobra.Command { "With --delete it removes the bundle mounted at that path instead.\n\n" + "A bundle's type decides its defaults. With no --type, the type is the first one PATH looks like (" + types + "), else none. " + "A dbt project (a directory holding dbt_project.yml) mounts at " + astrodeploy.DbtDefaultMountPathPrefix + " unless --mount-path says otherwise. " + - "Any other --type is recorded as a label, with no defaults, so it needs --mount-path.\n\n" + + "Any other --type is recorded as a label, with no defaults, so it needs --mount-path. " + + "With --delete and no bundle at PATH, a mount path below a type's default directory says the type.\n\n" + + "--deployment takes a link name of the project in the current directory, a Deployment id, or the name of a Deployment in the workspace. " + "This deploys the bundle's files; astro deploy deploys the project and its DAGs. " + "To list or change the bundles a Deployment has registered, use astro deployment bundle. " + - "A project link called bundle is not reachable as astro deploy bundle: pass it as astro deploy --deployment bundle.", + "A project link called bundle is not reachable as astro deploy bundle: deploy the project to it with astro deploy --deployment bundle.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runDeployBundle(cmd, args, o) @@ -73,6 +77,9 @@ func newDeployBundleCmd() *cobra.Command { # Deploy a dbt project elsewhere to a given Deployment astro deploy bundle --deployment + # Deploy it to a Deployment named analytics in a given Workspace + astro deploy bundle --deployment analytics --workspace + # Deploy another kind of bundle, which needs a mount path astro deploy bundle --type --mount-path /usr/local/airflow/ \ --deployment @@ -83,11 +90,11 @@ func newDeployBundleCmd() *cobra.Command { f := cmd.Flags() f.StringVarP(&o.mountPath, "mount-path", "m", "", "Path the bundle is mounted at in Airflow, for reference by Dags. Default: the type's, "+astrodeploy.DbtDefaultMountPathPrefix+" for a dbt project") f.StringVar(&o.bundleType, "type", "", "Type of the bundle: "+types+", or any other label, which has no defaults. Default: detected from PATH, else none") - f.StringVarP(&o.deployment, "deployment", "d", "", "Deployment to deploy to: a link name from the project here, or a Deployment id") - f.StringVar(&o.workspace, "workspace", "", "Workspace whose Deployments are offered when no --deployment is given") + f.StringVar(&o.deployment, "deployment", "", "Deployment to deploy to: a link name from the project here, a Deployment id, or a Deployment name") + f.StringVar(&o.workspace, "workspace", "", "Workspace a Deployment name is looked up in, and whose Deployments are offered when no --deployment is given") f.StringVar(&o.description, "description", "", "Description to store on the deploy") f.BoolVarP(&o.wait, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") - f.DurationVar(&o.waitTime, "wait-time", deployWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") + f.DurationVarP(&o.waitTime, "wait-time", "t", deployWaitTime, "Time to wait for the Deployment to become healthy before ending the command. Can only be used with --wait=true") f.BoolVar(&o.delete, "delete", false, "Remove the bundle mounted at --mount-path (by default the one PATH's type would mount at) instead of deploying") cliout.AddOutputFlag(cmd, &o.output) return cmd @@ -131,13 +138,7 @@ func deployBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundl if !utils.Locate(path).None() { return fmt.Errorf("bundle path %s is within an Astro project: a non-DAG bundle is a directory of its own. To deploy the project, run astro deploy", path) } - bt := astrodeploy.ResolveBundleType(o.bundleType, path) - if bt.Validate != nil { - if err := bt.Validate(path); err != nil { - return err - } - } - mountPath, err := resolveBundleMountPath(cmd, path, bt, o.mountPath) + b, err := resolveBundle(cmd, path, o) if err != nil { return err } @@ -149,10 +150,10 @@ func deployBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundl in := &astrodeploy.DeployBundleInput{ BundlePath: path, - MountPath: mountPath, + MountPath: b.mountPath, DeploymentID: targetID, Deployment: target, - BundleType: bt.Name, + BundleType: b.bundleType, Description: o.description, AstroV1Client: login.client, } @@ -169,25 +170,16 @@ func deployBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundl } return publishThenWaitWith(cmd, o.output, o.wait, bundleWait(cmd, &login, res.DeploymentID, o.waitTime), func(waitErr error) error { r := cliout.Renderer{Format: o.output, Out: out} - return r.Emit(newBundleDeployJSON(&res, bt.Name, bundlePath, o.wait, waitErr), renderBundleUploaded(res.BundleVersion)) + return r.Emit(newBundleDeployJSON(&res, &b, bundlePath, o.wait, waitErr), renderBundleUploaded(res.BundleVersion)) }) } // deleteBundle removes the bundle mounted at --mount-path, or at the mount // path the bundle at path would deploy to. func deleteBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundleOptions) error { - bt := astrodeploy.ResolveBundleType(o.bundleType, path) - mountPath := o.mountPath - if mountPath == "" { - if bt.Validate != nil { - if err := bt.Validate(path); err != nil { - return err - } - } - var err error - if mountPath, err = resolveBundleMountPath(cmd, path, bt, ""); err != nil { - return err - } + b, err := resolveBundle(cmd, path, o) + if err != nil { + return err } targetID, target, login, err := bundleTarget(cmd.Context(), o.deployment, o.workspace, o.output) if err != nil { @@ -196,10 +188,10 @@ func deleteBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundl cmd.SilenceUsage = true in := &astrodeploy.DeleteBundleInput{ - MountPath: mountPath, + MountPath: b.mountPath, DeploymentID: targetID, Deployment: target, - BundleType: bt.Name, + BundleType: b.bundleType, Description: o.description, AstroV1Client: login.client, } @@ -212,25 +204,62 @@ func deleteBundle(cmd *cobra.Command, out io.Writer, path string, o *deployBundl } return publishThenWaitWith(cmd, o.output, o.wait, bundleWait(cmd, &login, res.DeploymentID, o.waitTime), func(waitErr error) error { r := cliout.Renderer{Format: o.output, Out: out} - return r.Emit(newBundleDeleteJSON(&res, bt.Name, o.wait, waitErr), renderBundleDeleted(res.MountPath)) + return r.Emit(newBundleDeleteJSON(&res, &b, o.wait, waitErr), renderBundleDeleted(res.MountPath)) }) } -// resolveBundleMountPath is given, or else the default of the bundle's type, which -// it says on stderr it chose. A type with no default needs one given. -func resolveBundleMountPath(cmd *cobra.Command, path string, bt astrodeploy.BundleType, given string) (string, error) { - if given != "" { - return given, nil - } - if bt.DefaultMountPath == nil { - return "", cliout.Usage(fmt.Errorf("--mount-path is required for a bundle of type %s: only a %s bundle has a default", bt.Name, strings.Join(astrodeploy.BundleTypeNames(), " or "))) +// resolvedBundle is what a bundle deploy or delete acts on: the type it is +// recorded as, the mount path, and the bundle's name, when its type names +// it. +type resolvedBundle struct { + bundleType string + mountPath string + name string +} + +// resolveBundle settles the bundle at path, for a deploy and a delete alike. +// +// The type is --type when given; else the first registered type that +// detects path; else, for a delete given a mount path, the type whose +// default directory that mount path is below; else none. +// +// The mount path is --mount-path when given, and then path is not checked: +// --type dbt with a mount path deploys any directory, as 1.x's +// --non-dags-bundle-type dbt did. Otherwise the type checks path and derives +// the mount path from it, which the run says on stderr; a type with no +// default needs --mount-path. +func resolveBundle(cmd *cobra.Command, path string, o *deployBundleOptions) (resolvedBundle, error) { + bt := astrodeploy.ResolveBundleType(o.bundleType, path) + if o.bundleType == "" && bt.Name == astrodeploy.BundleTypeNone && o.delete && o.mountPath != "" { + if inferred, ok := astrodeploy.BundleTypeForMountPath(o.mountPath); ok { + bt = inferred + } } - mountPath, err := bt.DefaultMountPath(path) - if err != nil { - return "", err + b := resolvedBundle{bundleType: bt.Name, mountPath: o.mountPath} + if b.mountPath == "" { + if !bt.HasDefaultMountPath() { + return b, cliout.Usage(fmt.Errorf("--mount-path is required for a bundle of type %s: only a %s bundle has a default", bt.Name, strings.Join(astrodeploy.BundleTypeNames(), " or "))) + } + if bt.Validate != nil { + if err := bt.Validate(path); err != nil { + return b, err + } + } + mountPath, err := bt.DefaultMountPath(path) + if err != nil { + return b, err + } + b.mountPath = mountPath + fmt.Fprintf(cmd.ErrOrStderr(), "mount path %s: the default for a %s bundle\n", mountPath, bt.Name) + } + // The name is the bundle's when path holds one this type names: a + // --type dbt deploy of another directory has none. + if bt.BundleName != nil && (bt.Detect == nil || bt.Detect(path)) { + if name, err := bt.BundleName(path); err == nil { + b.name = name + } } - fmt.Fprintf(cmd.ErrOrStderr(), "mount path %s: the default for a %s bundle\n", mountPath, bt.Name) - return mountPath, nil + return b, nil } // bundleWait waits for the Deployment under the login the bundle went to. @@ -251,8 +280,9 @@ func bundleWait(cmd *cobra.Command, login *deployLogin, deploymentID string, wai // project's (its Astro host's), for everything; anywhere else, or when the // manifest does not load, the current context's. // -// The target: name (--deployment), a link name of the project here or a -// Deployment id. Without one, the workspace's Deployments are offered +// The target: name (--deployment), a link name of the project here, else a +// Deployment id (a CUID), else the name of a Deployment in the workspace +// (deploymentNamed). Without one, the workspace's Deployments are offered // (workspace, else the login's): only to a run that can be asked, and only on // the current context's host, as astro deploy offers them. A run that cannot // be asked, under --output json or without a terminal, is refused as astro @@ -270,14 +300,25 @@ func bundleTarget(ctx context.Context, name, workspace string, format cliout.For } if name != "" { - if m == nil || !hasLink(m, name) { + if m != nil && hasLink(m, name) { + named, _, err := manifestdeploy.ResolveNamed(m, name, "", workspace, "") + if err != nil { + return "", nil, login, cliout.Usage(err) + } + return named.DeploymentID, nil, login, nil + } + if util.IsCUID(name) { return name, nil, login, nil } - named, _, err := manifestdeploy.ResolveNamed(m, name, "", workspace, "") + projectWorkspace := "" + if m != nil { + projectWorkspace = m.Astro.Workspace + } + dep, err := deploymentNamed(name, cmp.Or(workspace, projectWorkspace, login.context.Workspace), &login) if err != nil { - return "", nil, login, cliout.Usage(err) + return "", nil, login, err } - return named.DeploymentID, nil, login, nil + return dep.Id, dep, login, nil } if format != cliout.FormatText || !stdinIsTerminal() { @@ -302,6 +343,36 @@ func bundleTarget(ctx context.Context, name, workspace string, format cliout.For return picked.Id, &picked, login, nil } +// deploymentNamed is the one Deployment called name in workspace ws, listed +// under login. None, or more than one, is an error that says so, naming the +// ids of the ones that share the name. +func deploymentNamed(name, ws string, login *deployLogin) (*astrov1.Deployment, error) { + if ws == "" { + return nil, fmt.Errorf("--deployment %s is not a link of the project here or a Deployment id, and there is no Workspace to look it up by name in: pass --workspace, or the Deployment id", name) + } + deps, err := deployment.ListDeployments(ws, login.context.Organization, login.client) + if err != nil { + return nil, err + } + var found []astrov1.Deployment + for i := range deps { + if deps[i].Name == name { + found = append(found, deps[i]) + } + } + switch len(found) { + case 0: + return nil, fmt.Errorf("no Deployment named %s in workspace %s: pass a Deployment id, a link name of the project here, or the name of a Deployment in that workspace", name, ws) + case 1: + return &found[0], nil + } + ids := make([]string, len(found)) + for i := range found { + ids[i] = found[i].Id + } + return nil, fmt.Errorf("%d Deployments are named %s in workspace %s (%s): pass the id of the one you mean with --deployment", len(found), name, ws, strings.Join(ids, ", ")) +} + // errBundleNoTarget is a bundle deploy or delete that names no Deployment and // cannot be asked for one, in astro deploy's words for the same refusal. var errBundleNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id or a link name with --deployment") @@ -316,8 +387,10 @@ type bundleDeployJSON struct { // DeployID is the deploy record the upload created on the Deployment. DeployID string `json:"deploy_id"` // BundleType is the type the bundle deployed as (--type, or detected), - // BundlePath the directory bundled, as an absolute path. + // BundleName the bundle's name when its type names it (a dbt project's + // name), and BundlePath the directory bundled, as an absolute path. BundleType string `json:"bundle_type"` + BundleName string `json:"bundle_name,omitempty"` BundlePath string `json:"bundle_path"` MountPath string `json:"mount_path"` // BundleVersion is the version of the uploaded bundle. @@ -330,13 +403,14 @@ type bundleDeployJSON struct { Git *deployGitJSON `json:"git,omitempty"` } -func newBundleDeployJSON(res *astrodeploy.BundleDeploy, bundleType, bundlePath string, waited bool, waitErr error) bundleDeployJSON { +func newBundleDeployJSON(res *astrodeploy.BundleDeploy, b *resolvedBundle, bundlePath string, waited bool, waitErr error) bundleDeployJSON { return bundleDeployJSON{ Deployment: res.DeploymentID, DeploymentName: res.DeploymentName, Workspace: res.WorkspaceID, DeployID: res.DeployID, - BundleType: bundleType, + BundleType: b.bundleType, + BundleName: b.name, BundlePath: bundlePath, MountPath: res.MountPath, BundleVersion: res.BundleVersion, @@ -357,18 +431,22 @@ type bundleDeleteJSON struct { Workspace string `json:"workspace,omitempty"` DeployID string `json:"deploy_id"` BundleType string `json:"bundle_type"` + // BundleName is the name of the bundle at PATH when its type names it, + // omitted when the delete was given a mount path and no bundle. + BundleName string `json:"bundle_name,omitempty"` MountPath string `json:"mount_path"` Waited bool `json:"waited"` WaitError string `json:"wait_error,omitempty"` } -func newBundleDeleteJSON(res *astrodeploy.BundleDelete, bundleType string, waited bool, waitErr error) bundleDeleteJSON { +func newBundleDeleteJSON(res *astrodeploy.BundleDelete, b *resolvedBundle, waited bool, waitErr error) bundleDeleteJSON { return bundleDeleteJSON{ Action: "deleted", Deployment: res.DeploymentID, Workspace: res.WorkspaceID, DeployID: res.DeployID, - BundleType: bundleType, + BundleType: b.bundleType, + BundleName: b.name, MountPath: res.MountPath, Waited: waited, WaitError: errText(waitErr), diff --git a/cmd/astro/deploy_bundle_test.go b/cmd/astro/deploy_bundle_test.go index 3fee9719c..6d11e53c9 100644 --- a/cmd/astro/deploy_bundle_test.go +++ b/cmd/astro/deploy_bundle_test.go @@ -68,6 +68,9 @@ func noBundleTransport(t *testing.T) { } } +// bundleTestID is a Deployment id, which --deployment takes as it is. +const bundleTestID = "clxbundle0000000000000000" + // plainBundle is a directory outside any project that no bundle type // detects, with the working path a fresh directory beside it. func plainBundle(t *testing.T) string { @@ -102,14 +105,14 @@ func TestDeployBundleTypes(t *testing.T) { t.Run("dbt, detected", func(t *testing.T) { project := dbtProject(t) captured := fakeDeployBundle(t) - r := execBundleRun(t, nil, "", project, "--deployment", "clx-dep", "--description", "nightly") + r := execBundleRun(t, nil, "", project, "--deployment", bundleTestID, "--description", "nightly") require.NoError(t, r.err, r.stderr) in := *captured require.NotNil(t, in) assert.Equal(t, project, in.BundlePath) assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", in.MountPath) assert.Equal(t, "dbt", in.BundleType) - assert.Equal(t, "clx-dep", in.DeploymentID) + assert.Equal(t, bundleTestID, in.DeploymentID) assert.Equal(t, "nightly", in.Description) assert.Nil(t, in.Deployment, "named by id, so not read") assert.Nil(t, in.Login, "the current context") @@ -120,24 +123,33 @@ func TestDeployBundleTypes(t *testing.T) { project := dbtProject(t) config.WorkingPath = project captured := fakeDeployBundle(t) - r := execBundleRun(t, nil, "", "-d", "clx-dep", "-m", "/usr/local/airflow/dbt/custom") + r := execBundleRun(t, nil, "", "--deployment", bundleTestID, "-m", "/usr/local/airflow/dbt/custom") require.NoError(t, r.err, r.stderr) assert.Equal(t, project, (*captured).BundlePath) assert.Equal(t, "/usr/local/airflow/dbt/custom", (*captured).MountPath) assert.Equal(t, "dbt", (*captured).BundleType) assert.NotContains(t, r.stderr, "the default for") }) + // With no mount path, --type dbt derives one from the project, so it + // needs one; given a mount path, it labels any directory dbt, as 1.x's + // --non-dags-bundle-type dbt did. t.Run("--type dbt, without a dbt project", func(t *testing.T) { bundle := plainBundle(t) noBundleTransport(t) - r := execBundleRun(t, nil, "", bundle, "--type", "dbt", "--deployment", "clx-dep", "--mount-path", "/x") + r := execBundleRun(t, nil, "", bundle, "--type", "dbt", "--deployment", bundleTestID) require.Error(t, r.err) assert.Contains(t, r.err.Error(), "no dbt_project.yml in "+bundle) + + captured := fakeDeployBundle(t) + r = execBundleRun(t, nil, "", bundle, "--type", "dbt", "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/dbt/seeds") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, "dbt", (*captured).BundleType) + assert.Equal(t, "/usr/local/airflow/dbt/seeds", (*captured).MountPath) }) t.Run("--type of another kind needs a mount path", func(t *testing.T) { bundle := dbtProject(t) noBundleTransport(t) - r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", "clx-dep") + r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", bundleTestID) require.Error(t, r.err) assert.True(t, cliout.IsUsage(r.err), "%v", r.err) assert.Equal(t, "--mount-path is required for a bundle of type seeds: only a dbt bundle has a default", r.err.Error()) @@ -145,7 +157,7 @@ func TestDeployBundleTypes(t *testing.T) { t.Run("--type of another kind, with a mount path", func(t *testing.T) { bundle := dbtProject(t) captured := fakeDeployBundle(t) - r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", "clx-dep", "--mount-path", "/usr/local/airflow/seeds") + r := execBundleRun(t, nil, "", bundle, "--type", "seeds", "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/seeds") require.NoError(t, r.err, r.stderr) assert.Equal(t, "seeds", (*captured).BundleType, "a free-form label") assert.Equal(t, "/usr/local/airflow/seeds", (*captured).MountPath) @@ -153,12 +165,12 @@ func TestDeployBundleTypes(t *testing.T) { t.Run("nothing detected", func(t *testing.T) { bundle := plainBundle(t) noBundleTransport(t) - r := execBundleRun(t, nil, "", bundle, "--deployment", "clx-dep") + r := execBundleRun(t, nil, "", bundle, "--deployment", bundleTestID) require.Error(t, r.err) assert.Equal(t, "--mount-path is required for a bundle of type none: only a dbt bundle has a default", r.err.Error()) captured := fakeDeployBundle(t) - r = execBundleRun(t, nil, "", bundle, "--deployment", "clx-dep", "--mount-path", "/usr/local/airflow/x") + r = execBundleRun(t, nil, "", bundle, "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/x") require.NoError(t, r.err, r.stderr) assert.Equal(t, "none", (*captured).BundleType) }) @@ -184,13 +196,13 @@ func TestDeployBundleRefusesItsPath(t *testing.T) { file: "is not a directory", } { noBundleTransport(t) - r := execBundleRun(t, nil, "", path, "--deployment", "clx-dep", "--mount-path", "/x") + r := execBundleRun(t, nil, "", path, "--deployment", bundleTestID, "--mount-path", "/x") require.Error(t, r.err, path) assert.Contains(t, r.err.Error(), want, path) } noBundleTransport(t) - r := execBundleRun(t, nil, "", t.TempDir(), "--deployment", "clx-dep", "--mount-path", "/x", "--wait-time", "1m") + r := execBundleRun(t, nil, "", t.TempDir(), "--deployment", bundleTestID, "--mount-path", "/x", "--wait-time", "1m") require.Error(t, r.err) assert.True(t, cliout.IsUsage(r.err)) assert.Contains(t, r.err.Error(), "cannot use --wait-time with --wait=false") @@ -202,33 +214,44 @@ func TestDeployBundleDelete(t *testing.T) { t.Run("the dbt project's default", func(t *testing.T) { project := dbtProject(t) captured := fakeDeleteBundle(t) - r := execBundleRun(t, nil, "", "--delete", project, "--deployment", "clx-dep") + r := execBundleRun(t, nil, "", "--delete", project, "--deployment", bundleTestID) require.NoError(t, r.err, r.stderr) in := *captured require.NotNil(t, in) assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", in.MountPath) assert.Equal(t, "dbt", in.BundleType) - assert.Equal(t, "clx-dep", in.DeploymentID) + assert.Equal(t, bundleTestID, in.DeploymentID) assert.Nil(t, in.Login) assert.Equal(t, "Successfully requested bundle delete for mount path /usr/local/airflow/dbt/jaffle_shop from Astro.\n", r.stdout) }) t.Run("a mount path needs no bundle here", func(t *testing.T) { plainBundle(t) captured := fakeDeleteBundle(t) - r := execBundleRun(t, nil, "", "--delete", "--deployment", "clx-dep", "--mount-path", "/usr/local/airflow/x", "--type", "seeds") + r := execBundleRun(t, nil, "", "--delete", "--deployment", bundleTestID, "--mount-path", "/usr/local/airflow/x", "--type", "seeds") require.NoError(t, r.err, r.stderr) assert.Equal(t, "/usr/local/airflow/x", (*captured).MountPath) assert.Equal(t, "seeds", (*captured).BundleType) }) + // From a directory that holds no bundle, a mount path below the dbt + // default directory is a dbt bundle's. + t.Run("a dbt mount path from elsewhere", func(t *testing.T) { + plainBundle(t) + r := execBundleRun(t, dbtBundleMock(t, ""), "", "--delete", "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/dbt/jaffle_shop", "-o", "json") + require.NoError(t, r.err, r.stderr) + var got bundleDeleteJSON + fields := decodeOne(t, r.stdout, &got) + assert.Equal(t, "dbt", got.BundleType) + assert.NotContains(t, fields, "bundle_name", "no bundle here to name") + }) t.Run("no mount path and nothing to derive one from", func(t *testing.T) { bundle := plainBundle(t) noBundleTransport(t) - r := execBundleRun(t, nil, "", "--delete", bundle, "--deployment", "clx-dep") + r := execBundleRun(t, nil, "", "--delete", bundle, "--deployment", bundleTestID) require.Error(t, r.err) assert.True(t, cliout.IsUsage(r.err)) assert.Contains(t, r.err.Error(), "--mount-path is required for a bundle of type none") - r = execBundleRun(t, nil, "", "--delete", bundle, "--type", "dbt", "--deployment", "clx-dep") + r = execBundleRun(t, nil, "", "--delete", bundle, "--type", "dbt", "--deployment", bundleTestID) require.Error(t, r.err) assert.Contains(t, r.err.Error(), "no dbt_project.yml in") }) @@ -296,12 +319,49 @@ func TestDeployBundleTarget(t *testing.T) { } } +// --deployment that is neither a link nor an id is a Deployment's name in +// the workspace: one match is used, as read; none, or several, are errors +// that say so. +func TestDeployBundleDeploymentByName(t *testing.T) { + other := dbtTestDeployment() + other.Id = "clxdbtother00000000000000" + t.Run("one", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeployBundle(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, dbtTestDeploymentID, (*captured).DeploymentID) + require.NotNil(t, (*captured).Deployment, "the listed Deployment is handed on") + }) + t.Run("one, deleted", func(t *testing.T) { + project := dbtProject(t) + captured := fakeDeleteBundle(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", "--delete", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000") + require.NoError(t, r.err, r.stderr) + assert.Equal(t, dbtTestDeploymentID, (*captured).DeploymentID) + }) + t.Run("none", func(t *testing.T) { + project := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", project, "--deployment", "reporting", "--workspace", "clxdbtworkspace0000000000") + require.Error(t, r.err) + assert.Contains(t, r.err.Error(), "no Deployment named reporting in workspace clxdbtworkspace0000000000") + }) + t.Run("two", func(t *testing.T) { + project := dbtProject(t) + noBundleTransport(t) + r := execBundleRun(t, coreMock(t, dbtTestDeployment(), other), "", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000") + require.Error(t, r.err) + assert.Contains(t, r.err.Error(), "2 Deployments are named analytics in workspace clxdbtworkspace0000000000 ("+dbtTestDeploymentID+", clxdbtother00000000000000)") + }) +} + // From a project on another host, a bundle goes under that host's login, by // link and by id alike, and the picker, which lists only the current // context's Deployments, is refused. func TestDeployBundleUsesTheManifestDomainsLogin(t *testing.T) { for _, del := range []bool{false, true} { - for _, target := range []string{"test", "clx-bare"} { + for _, target := range []string{"test", bundleTestID} { t.Run(fmt.Sprintf("%s, delete %t", target, del), func(t *testing.T) { setupDeployOnStage(t, manifestOnProd) logInToProd(t) @@ -340,7 +400,7 @@ func TestDeployBundleUsesTheManifestDomainsLogin(t *testing.T) { setupDeployOnStage(t, manifestOnProd) require.NoError(t, os.Remove(filepath.Join(config.WorkingPath, "pyproject.toml"))) captured := fakeDeployBundle(t) - _, err := execDeployCapture("bundle", t.TempDir(), "--deployment", "clx-bare", "--mount-path", "/x") + _, err := execDeployCapture("bundle", t.TempDir(), "--deployment", bundleTestID, "--mount-path", "/x") require.NoError(t, err) assert.Nil(t, (*captured).Login, "the current context") }) @@ -359,6 +419,7 @@ func TestDeployBundleJSON(t *testing.T) { Workspace: "clxdbtworkspace0000000000", DeployID: dbtTestDeployID, BundleType: "dbt", + BundleName: "jaffle_shop", BundlePath: project, MountPath: "/usr/local/airflow/dbt/jaffle_shop", BundleVersion: dbtTestVersion, @@ -370,7 +431,7 @@ func TestDeployBundleJSON(t *testing.T) { // known and not published. t.Run("delete", func(t *testing.T) { plainBundle(t) - r := execBundleRun(t, dbtBundleMock(t, ""), "", "--delete", "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/dbt/custom", "-o", "json") + r := execBundleRun(t, dbtBundleMock(t, ""), "", "--delete", "--deployment", dbtTestDeploymentID, "--mount-path", "/usr/local/airflow/seeds", "-o", "json") require.NoError(t, r.err, "stderr:\n%s", r.stderr) var got bundleDeleteJSON fields := decodeOne(t, r.stdout, &got) @@ -379,7 +440,7 @@ func TestDeployBundleJSON(t *testing.T) { Deployment: dbtTestDeploymentID, DeployID: dbtTestDeployID, BundleType: "none", - MountPath: "/usr/local/airflow/dbt/custom", + MountPath: "/usr/local/airflow/seeds", }, got) assert.NotContains(t, fields, "workspace") }) @@ -446,6 +507,24 @@ func TestDeployBundleWait(t *testing.T) { } } +// The guidance is for the subcommand typed, wherever the flags stand, a +// flag's value never taken for it. +func TestDbtRemovedReadsTheSubcommand(t *testing.T) { + for _, tc := range []struct { + args []string + want string + }{ + {[]string{"--workspace-id", "ws1", "deploy"}, errDbtDeployRemoved}, + {[]string{"-n", "prod", "delete"}, errDbtDeleteRemoved}, + {[]string{"--project-path=.", "cleanup"}, errDbtCleanupRemoved}, + {[]string{"-w", "deploy"}, errDbtDeployRemoved}, + {[]string{"--workspace-id", "deploy"}, errDbtRemoved}, + {[]string{"--", "deploy"}, errDbtRemoved}, + } { + assert.Equal(t, tc.want, dbtRemovedGuidance(tc.args), "%v", tc.args) + } +} + // astro dbt is a tombstone: every subcommand, and the group bare, says what // replaced it, in text and as the one error object under json, and runs // nothing, whatever 1.x flags it is passed. diff --git a/cmd/astro/testdata/schema/deploy-bundle-delete.json b/cmd/astro/testdata/schema/deploy-bundle-delete.json index 81d630849..3fceddc4f 100644 --- a/cmd/astro/testdata/schema/deploy-bundle-delete.json +++ b/cmd/astro/testdata/schema/deploy-bundle-delete.json @@ -4,6 +4,7 @@ "workspace": "x", "deploy_id": "x", "bundle_type": "x", + "bundle_name": "x", "mount_path": "x", "waited": true, "wait_error": "x" diff --git a/cmd/astro/testdata/schema/deploy-bundle.json b/cmd/astro/testdata/schema/deploy-bundle.json index 4b06085e0..8f697e1cb 100644 --- a/cmd/astro/testdata/schema/deploy-bundle.json +++ b/cmd/astro/testdata/schema/deploy-bundle.json @@ -4,6 +4,7 @@ "workspace": "x", "deploy_id": "x", "bundle_type": "x", + "bundle_name": "x", "bundle_path": "x", "mount_path": "x", "bundle_version": "x", diff --git a/cmd/local/link.go b/cmd/local/link.go index f36b381dd..c92a210e1 100644 --- a/cmd/local/link.go +++ b/cmd/local/link.go @@ -342,8 +342,8 @@ func (c *cli) runLinkAdd(cmd *cobra.Command, in *linkAddInput, name string, comm name = strings.TrimSpace(name) // bundle is astro deploy bundle, so a link by that name could not be // deployed to by name: refused before anything is asked. - if name == deploy.BundleCommand { - return cliout.Usage(deploy.ErrBundleLinkName) + if name == scaffold.BundleLinkName { + return cliout.Usage(scaffold.ErrBundleLinkName) } // A NAME already linked is refused before anything asks a question or // calls the platform. @@ -453,7 +453,7 @@ func (c *cli) pickAstroDeployment(link *scaffold.Link, m *manifest.Manifest, nam return name, nil } name = scaffold.LinkNameFor(dep.Name) - if name == "" || name == manifest.ReservedLinkName || name == deploy.BundleCommand { + if name == "" || name == manifest.ReservedLinkName || name == scaffold.BundleLinkName { return "", fmt.Errorf("the Deployment's name does not make a link name: pass one, astro link add NAME --%s %s", flagLinkDeployment, dep.ID) } return name, nil diff --git a/cmd/removed_flags.go b/cmd/removed_flags.go index adb405e16..0d68853f1 100644 --- a/cmd/removed_flags.go +++ b/cmd/removed_flags.go @@ -54,7 +54,7 @@ const ( errDeployNameRemoved = "--deployment-name was removed in Astro CLI v2: name the Deployment as the argument or with --deployment, which take a link name or a Deployment id" // astro deploy --non-dags became astro deploy bundle, a subcommand of // its own with the bundle's directory as its argument. - errDeployNonDagsRemoved = "--non-dags was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment , " + + errDeployNonDagsRemoved = "--non-dags was removed in Astro CLI v2: use astro deploy bundle [PATH] --deployment , " + "which deploys a non-DAG bundle such as a dbt project from its directory" errDeployNonDagsMountPathRemoved = "--non-dags-mount-path was removed in Astro CLI v2: use astro deploy bundle --mount-path" errDeployNonDagsLocalPathRemoved = "--non-dags-local-path was removed in Astro CLI v2: pass the bundle's directory as the argument, astro deploy bundle " diff --git a/docs/deploy.md b/docs/deploy.md index 318c09993..82ee23014 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -28,14 +28,15 @@ Every refusal is reported under `--output json` as the error object with kind `n **`astro deploy bundle [PATH]`** ships a directory of its own, PATH or the working directory, as a non-DAG bundle: a dbt project, or any other files the DAGs read, mounted at a path in Airflow. It carries no DAGs, and it is not a project deploy: PATH must be a directory, and one in or below any project the walk finds is refused. **`--delete`** removes the bundle mounted at a path instead. It replaces Astro CLI 1.x's `astro dbt deploy` and `astro dbt delete` and the 1.x `astro deploy --non-dags`, which did the same upload with different defaults and Deployment selection. `astro deployment bundle` is something else: it lists and changes the bundle registrations on a Deployment, and uploads nothing. -Because `bundle` is a subcommand, `astro deploy bundle` never means a Deployment called bundle. `astro link add` refuses `bundle` as a link name, and a deploy that links a picked Deployment names it otherwise; a link already called `bundle` is deployed to with `astro deploy --deployment bundle`. +Because `bundle` is a subcommand, `astro deploy bundle` never means a Deployment called bundle. `bundle` is no link's name: `scaffold.SaveLink` refuses it for every writer (`astro link add` among them, and Astro Desktop), and a deploy that links a picked Deployment names it otherwise; a link already called `bundle` is deployed to with `astro deploy --deployment bundle`. -- **The type** decides the defaults. The types are a table in [`internal/platform/astro/deploy/bundletype.go`](../internal/platform/astro/deploy/bundletype.go), each with how to detect it, how to check a bundle of it, and its default mount path; adding a type is adding an entry. With no `--type`, the first type that detects PATH is used, else `none`. Today the table has one entry, `dbt`: a directory holding `dbt_project.yml`, mounted by default at `/usr/local/airflow/dbt/` from the project's `name`. `--type dbt` asks for a dbt project and fails without one. Any other `--type` (and `none`) is a free-form label the deploy records, with no defaults, so it needs `--mount-path`. A default mount path is said on stderr. -- **Delete** uses `--mount-path` as given, or the mount path PATH's type would deploy to, so `astro deploy bundle --delete` in a dbt project removes that project. With neither, it is a usage error. +- **The type** decides the defaults. The types are a table in [`internal/platform/astro/deploy/bundletype.go`](../internal/platform/astro/deploy/bundletype.go), each with how to detect it, how to check a bundle of it, how to name it, and the directory its bundles mount below by default; adding a type is adding an entry. With no `--type`, the first type that detects PATH is used, else `none`. Today the table has one entry, `dbt`: a directory holding `dbt_project.yml`, named by the project's `name` and mounted by default at `/usr/local/airflow/dbt/`. Any other `--type` (and `none`) is a free-form label the deploy records, with no defaults, so it needs `--mount-path`. A default mount path is said on stderr. +- **PATH is checked only when the mount path is derived from it.** Without `--mount-path`, `--type dbt` needs a dbt project at PATH. With `--mount-path`, `--type dbt` labels any directory, as 1.x's `--non-dags-bundle-type dbt` did. +- **Delete** uses `--mount-path` as given, or the mount path PATH's type would deploy to, so `astro deploy bundle --delete` in a dbt project removes that project. With neither, it is a usage error. Its type is `--type`, else PATH's, else, from a directory with no bundle, the type whose default directory `--mount-path` is below (`/usr/local/airflow/dbt/…` is `dbt`), else `none`. - **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project in the working directory whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. -- **The target** is `--deployment` (`-d`): a link of the project in the working directory, or a Deployment id. Without it, the Deployments of `--workspace` (or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id or a link name with --deployment`. A project on another host is told to pass `--deployment ` or switch context. +- **The target** is `--deployment`, resolved in the order the other commands use: a link of the project in the working directory, else a Deployment id, else the name of a Deployment in `--workspace` (or the project's `[tool.astro] workspace`, or the login's). A name that matches no Deployment, or several, is an error; several are listed by id. It has no `-d`: on `astro deploy`, `-d` is `--dags`. Without it, the Deployments of `--workspace` (or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id or a link name with --deployment`. A project on another host is told to pass `--deployment ` or switch context. - **`--wait`** waits for the Deployment to take the deploy or the delete, under the same login. In text the result prints first and the wait's progress goes to stderr; under json the one object comes after the wait, with `waited` and, when it failed, `wait_error`, and the run exits 1. -- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-bundle.json` and, for `--delete`, `deploy-bundle-delete.json`. Both carry `bundle_type`, `mount_path`, `deployment` and `deploy_id`; the deploy adds `bundle_path`, `bundle_version` and `git`. What the deploy prints on the way (the git note, the default mount path) goes to stderr there. +- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-bundle.json` and, for `--delete`, `deploy-bundle-delete.json`. Both carry `bundle_type`, `mount_path`, `deployment` and `deploy_id`, and `bundle_name` when the type names the bundle at PATH (a dbt project's `name`); the deploy adds `bundle_path`, `bundle_version` and `git`. 1.x's `astro dbt deploy -o json` keys `project` and `project_path` are `bundle_name` and `bundle_path` here, beside `bundle_type`. What the deploy prints on the way (the git note, the default mount path) goes to stderr there. **Flags that cannot all apply** are refused as usage errors before anything is read: `--wait-time` without `--wait`, `--no-dags-base-dir` with a deploy that ships no DAGs (`--image`, or `--image-name` outside a project), and `--dags` with `--image` or `--image-name` (the message names the ones given). diff --git a/docs/upgrading-from-v1.md b/docs/upgrading-from-v1.md index 5d9b74c60..374494f3c 100644 --- a/docs/upgrading-from-v1.md +++ b/docs/upgrading-from-v1.md @@ -104,8 +104,8 @@ Everything below fails with an error naming the replacement, instead of an "unkn | `astro deployment connection copy` | no per-Deployment copy: set it once in the workspace with `astro env connection set --workspace --auto-link`, or on each Deployment | | `astro deployment pool …` | the Airflow UI or API, until pools reach the Environment Manager | | `astro env … create`, `astro env … update` | `astro env … set`, which creates or updates | -| `astro dbt deploy [DEPLOYMENT_ID]` | `astro deploy bundle [PATH] --deployment `: PATH replaces `--project-path`, `--deployment` the argument and `--deployment-name`, `--workspace` `--workspace-id` | -| `astro dbt delete [DEPLOYMENT_ID]` | `astro deploy bundle --delete [PATH] --deployment `, with the same flags | +| `astro dbt deploy [DEPLOYMENT_ID]` | `astro deploy bundle [PATH] --deployment `: PATH replaces `--project-path`, `--deployment` the argument and `--deployment-name`, `--workspace` `--workspace-id`; `-m`, `-w` and `-t` are unchanged. Under `-o json`, `project` and `project_path` are now `bundle_name` and `bundle_path`, beside `bundle_type` | +| `astro dbt delete [DEPLOYMENT_ID]` | `astro deploy bundle --delete [PATH] --deployment `, with the same flags | Run inside a 1.x project, an `astro dev` command also tells you to convert it with `astro init`. One with a replacement says so in its first line (convert, then use the replacement); bare `astro dev`, `upgrade-test` and `proxy` say it after the list of `astro local` commands. diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 4b635786b..a55e4b3d5 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -781,13 +781,9 @@ func unlinkedTarget(req Request, d Deployer) (Target, error) { // BundleCommand is the word astro deploy bundle is spelled with. astro // deploy reads it as that subcommand, never as a link name, so no link is -// saved under it: astro link add refuses it, and a link a deploy saves for a -// picked Deployment is named otherwise. A link by that name already in a -// manifest is still reached with --deployment. -const BundleCommand = "bundle" - -// ErrBundleLinkName is astro link add refusing BundleCommand as a link name. -var ErrBundleLinkName = fmt.Errorf("%[1]s cannot name a link: astro deploy %[1]s deploys a non-DAG bundle, so astro deploy %[1]s would never reach the link. Name the link something else", BundleCommand) +// saved under it (scaffold.BundleLinkName): a link a deploy saves for a +// picked Deployment is named otherwise. +const BundleCommand = scaffold.BundleLinkName // DefaultMark is the word that marks a project's default link in a list of // links: the deploy prompt's "linked, default", and the `astro use` and `astro diff --git a/internal/platform/astro/deploy/bundletype.go b/internal/platform/astro/deploy/bundletype.go index d97a85460..e7cadbc2e 100644 --- a/internal/platform/astro/deploy/bundletype.go +++ b/internal/platform/astro/deploy/bundletype.go @@ -11,22 +11,46 @@ import ( ) // BundleType is a kind of non-DAG bundle that astro deploy bundle knows by -// its contents: how to tell a directory holds one, how to check it before -// anything is uploaded, and where it mounts when the deploy names no mount -// path. Adding a type is adding an entry to bundleTypes. +// its contents: how to tell a directory holds one, how to check it, what the +// bundle is called, and where it mounts when the deploy names no mount path. +// Adding a type is adding an entry to bundleTypes. type BundleType struct { // Name is the type recorded on the deploy, and what --type selects. Name string // Detect reports whether dir holds a bundle of this type, for a deploy // that names no type. nil detects nothing. Detect func(dir string) bool - // Validate checks dir is a bundle of this type. nil accepts any - // directory. + // Validate checks dir is a bundle of this type, before a mount path is + // derived from it. nil accepts any directory. Validate func(dir string) error - // DefaultMountPath is where the bundle at dir mounts when no mount path - // is given. nil means the type has no default, so a mount path is - // required. - DefaultMountPath func(dir string) (string, error) + // BundleName is the name of the bundle at dir (a dbt project's name). + // nil means bundles of this type have none. + BundleName func(dir string) (string, error) + // MountPathPrefix is the directory in Airflow a bundle of this type + // mounts below by default, at MountPathPrefix + its BundleName. A mount + // path below it says a bundle is of this type, for a delete that names + // no type and runs where no bundle is. Empty means no default. + MountPathPrefix string +} + +// HasDefaultMountPath reports whether a bundle of this type has a mount path +// without one being given. +func (t *BundleType) HasDefaultMountPath() bool { + return t.MountPathPrefix != "" && t.BundleName != nil +} + +// DefaultMountPath is where the bundle at dir mounts when no mount path is +// given: MountPathPrefix and the bundle's name. It is an error for a type +// with no default (HasDefaultMountPath). +func (t *BundleType) DefaultMountPath(dir string) (string, error) { + if !t.HasDefaultMountPath() { + return "", fmt.Errorf("a bundle of type %s has no default mount path", t.Name) + } + name, err := t.BundleName(dir) + if err != nil { + return "", err + } + return t.MountPathPrefix + name, nil } // BundleTypeNone is the type of a bundle that names no type and that no @@ -43,10 +67,11 @@ const dbtProjectFile = "dbt_project.yml" // tries them. var bundleTypes = []BundleType{ { - Name: "dbt", - Detect: isDbtProject, - Validate: validateDbtProject, - DefaultMountPath: dbtDefaultMountPath, + Name: "dbt", + Detect: isDbtProject, + Validate: validateDbtProject, + BundleName: dbtProjectName, + MountPathPrefix: DbtDefaultMountPathPrefix, }, } @@ -72,6 +97,17 @@ func ResolveBundleType(name, dir string) BundleType { return BundleType{Name: name} } +// BundleTypeForMountPath is the registered type whose bundles mount below +// mountPath's directory by default, and whether there is one. +func BundleTypeForMountPath(mountPath string) (BundleType, bool) { + for _, t := range bundleTypes { + if t.MountPathPrefix != "" && strings.HasPrefix(mountPath, t.MountPathPrefix) && len(mountPath) > len(t.MountPathPrefix) { + return t, true + } + } + return BundleType{}, false +} + // BundleTypeNames is the name of every registered bundle type, in order. func BundleTypeNames() []string { names := make([]string, 0, len(bundleTypes)) @@ -95,30 +131,21 @@ func validateDbtProject(dir string) error { return err } -// dbtDefaultMountPath is DbtDefaultMountPathPrefix and the project's name, -// from its dbt_project.yml. -func dbtDefaultMountPath(dir string) (string, error) { - name, err := dbtProjectName(dir) - if err != nil { - return "", fmt.Errorf("dbt project name not found in %s: %w", dir, err) - } - return DbtDefaultMountPathPrefix + name, nil -} - +// dbtProjectName is the project's name, from its dbt_project.yml. func dbtProjectName(dir string) (string, error) { path := filepath.Join(dir, dbtProjectFile) file, err := os.Open(path) if err != nil { - return "", fmt.Errorf("could not open %s: %w", path, err) + return "", fmt.Errorf("dbt project name not found in %s: could not open %s: %w", dir, path, err) } defer file.Close() var project map[string]any if err := yaml.NewDecoder(file).Decode(&project); err != nil { - return "", fmt.Errorf("could not decode %s: %w", path, err) + return "", fmt.Errorf("dbt project name not found in %s: could not decode %s: %w", dir, path, err) } name, ok := project["name"].(string) if !ok || name == "" { - return "", errors.New("invalid dbt project name") + return "", fmt.Errorf("dbt project name not found in %s: invalid dbt project name", dir) } return name, nil } diff --git a/internal/platform/astro/deploy/bundletype_test.go b/internal/platform/astro/deploy/bundletype_test.go index aadbbb5d4..b86101ac1 100644 --- a/internal/platform/astro/deploy/bundletype_test.go +++ b/internal/platform/astro/deploy/bundletype_test.go @@ -25,6 +25,10 @@ func TestResolveBundleType(t *testing.T) { bt := ResolveBundleType("", dbt) assert.Equal(t, "dbt", bt.Name) require.NoError(t, bt.Validate(dbt)) + name, err := bt.BundleName(dbt) + require.NoError(t, err) + assert.Equal(t, "jaffle_shop", name) + require.True(t, bt.HasDefaultMountPath()) mount, err := bt.DefaultMountPath(dbt) require.NoError(t, err) assert.Equal(t, "/usr/local/airflow/dbt/jaffle_shop", mount) @@ -38,17 +42,23 @@ func TestResolveBundleType(t *testing.T) { }) t.Run("nothing detected", func(t *testing.T) { bt := ResolveBundleType("", plain) - assert.Equal(t, BundleType{Name: BundleTypeNone}, bt) + assert.Equal(t, BundleTypeNone, bt.Name) + assert.False(t, bt.HasDefaultMountPath()) + _, err := bt.DefaultMountPath(plain) + assert.ErrorContains(t, err, "has no default mount path") }) t.Run("any other type is a label with no defaults", func(t *testing.T) { for _, name := range []string{"seeds", BundleTypeNone} { bt := ResolveBundleType(name, dbt) - assert.Equal(t, BundleType{Name: name}, bt, "even where a dbt project is") + assert.Equal(t, name, bt.Name, "even where a dbt project is") + assert.False(t, bt.HasDefaultMountPath(), name) + assert.Nil(t, bt.Validate, name) } }) t.Run("a dbt project with no name", func(t *testing.T) { nameless := dbtDir(t, "version: 2\n") - _, err := ResolveBundleType("", nameless).DefaultMountPath(nameless) + bt := ResolveBundleType("", nameless) + _, err := bt.DefaultMountPath(nameless) assert.ErrorContains(t, err, "dbt project name not found") }) t.Run("a directory called dbt_project.yml is no dbt project", func(t *testing.T) { @@ -58,3 +68,14 @@ func TestResolveBundleType(t *testing.T) { }) assert.Equal(t, []string{"dbt"}, BundleTypeNames()) } + +// A mount path below a type's default directory says the type. +func TestBundleTypeForMountPath(t *testing.T) { + bt, ok := BundleTypeForMountPath("/usr/local/airflow/dbt/jaffle_shop") + require.True(t, ok) + assert.Equal(t, "dbt", bt.Name) + for _, p := range []string{"/usr/local/airflow/dbt/", "/usr/local/airflow/dbt", "/usr/local/airflow/seeds", ""} { + _, ok := BundleTypeForMountPath(p) + assert.False(t, ok, p) + } +} diff --git a/pkg/scaffold/linkwrite.go b/pkg/scaffold/linkwrite.go index d46545e1e..18268358e 100644 --- a/pkg/scaffold/linkwrite.go +++ b/pkg/scaffold/linkwrite.go @@ -33,6 +33,17 @@ import ( // because the parser would refuse it. The message names the field. var ErrInvalidLink = errors.New("invalid link") +// BundleLinkName is a name no link may have: astro deploy reads astro deploy +// bundle as its subcommand that deploys a non-DAG bundle, so a deploy could +// never reach a link by that name as its argument. A manifest that already +// has one still loads, and it is reached with --deployment. +const BundleLinkName = "bundle" + +// ErrBundleLinkName is SaveLink refusing BundleLinkName. It wraps +// ErrInvalidLink. +var ErrBundleLinkName = fmt.Errorf("%w: %s cannot name a link: astro deploy %s deploys a non-DAG bundle, "+ + "so astro deploy %s would never reach the link. Name the link something else", ErrInvalidLink, BundleLinkName, BundleLinkName, BundleLinkName) + // ErrComposerProjectRequired and ErrComposerLocationRequired report a Composer // link saved without the Google Cloud project or the region holding its // environment. The parser does not require them, but an environment name alone @@ -615,6 +626,8 @@ func (l *Link) validate() error { return invalidLink("a link needs a name") case manifest.ReservedLinkName: return invalidLink("%q is reserved: it always means the Airflow running on this machine", manifest.ReservedLinkName) + case BundleLinkName: + return ErrBundleLinkName } switch l.Kind { case manifest.KindAstro: diff --git a/pkg/scaffold/linkwrite_test.go b/pkg/scaffold/linkwrite_test.go index 1da25329e..eaac67594 100644 --- a/pkg/scaffold/linkwrite_test.go +++ b/pkg/scaffold/linkwrite_test.go @@ -169,6 +169,7 @@ func TestSaveLinkRefusesAnInvalidLink(t *testing.T) { }{ {"no name", Link{Name: " ", Kind: manifest.KindAstro, Workspace: "W", Deployment: "d"}, "needs a name"}, {"the reserved name", Link{Name: manifest.ReservedLinkName, Kind: manifest.KindMWAA, Environment: "env"}, "reserved"}, + {"the deploy bundle subcommand's name", Link{Name: BundleLinkName, Kind: manifest.KindAstro, Workspace: "W", Deployment: "d"}, "astro deploy bundle deploys a non-DAG bundle"}, {"an astro link with no deployment", Link{Name: "prod", Kind: manifest.KindAstro, Workspace: "W"}, "needs a deployment"}, {"an mwaa link with no environment", Link{Name: "aws", Kind: manifest.KindMWAA}, "needs an environment"}, {"an endpoint with no url", Link{Name: "oss", Kind: manifest.KindEndpoint, Auth: manifest.Auth{Method: manifest.AuthNone}}, "needs a url"}, From 56ab0e62c57471d0de4e133f7f39727dd274f69d Mon Sep 17 00:00:00 2001 From: Greg Neiheisel Date: Wed, 14 Oct 2026 15:56:53 -0400 Subject: [PATCH 7/7] Settle deploy bundle's type, name and lookup errors one way for deploy and delete Second review: - One type rule for deploy and delete: --type, else what PATH holds, else the type whose default mount path --mount-path has the shape of (prefix and one component, after path.Clean), so the pair records the same type. - bundle_name only when the mount path is the one derived from PATH. - A Deployment name with no match, or several, is a usage error, as a mistyped link name is; the lookup (DeploymentsNamed) is shared with astro deployment create --clone. - scaffold refuses only a new link called bundle: one already in the manifest saves again. - The astro dbt stub reads a shorthand group's value (-wt 5m, -nprod). - The no-target message names ids, names and link names; the docs say the name lookup is astro deploy bundle's, as astro deploy takes links or ids. Co-Authored-By: Claude --- cmd/astro/dbt.go | 13 +++- cmd/astro/deploy_bundle.go | 56 ++++++------- cmd/astro/deploy_bundle_test.go | 78 +++++++++++++++++-- cmd/local/link.go | 7 +- cmd/local/link_test.go | 6 ++ docs/deploy.md | 10 +-- internal/platform/astro/deploy/bundletype.go | 14 +++- .../platform/astro/deploy/bundletype_test.go | 8 +- .../platform/astro/deployment/clone_run.go | 40 +++++++--- pkg/scaffold/linkwrite.go | 14 ++-- pkg/scaffold/linkwrite_test.go | 19 ++++- 11 files changed, 202 insertions(+), 63 deletions(-) diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index 88aa0f4e5..4ac03ea31 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -78,9 +78,16 @@ func dbtTypedSubcommand(args []string) string { i++ } case strings.HasPrefix(a, "-") && len(a) > 1: - // -n prod takes the next word; -nprod carries its value. - if len(a) == 2 && dbtValueFlags[a[1:]] { - i++ + // A group of shorthands: the first that takes a value takes the + // rest of the group (-nprod), or the next word when it is the + // group's last letter (-n prod, -wt 5m). + for j := 1; j < len(a); j++ { + if dbtValueFlags[a[j:j+1]] { + if j == len(a)-1 { + i++ + } + break + } } default: return a diff --git a/cmd/astro/deploy_bundle.go b/cmd/astro/deploy_bundle.go index 81687b0be..2c8a7ceb3 100644 --- a/cmd/astro/deploy_bundle.go +++ b/cmd/astro/deploy_bundle.go @@ -219,9 +219,10 @@ type resolvedBundle struct { // resolveBundle settles the bundle at path, for a deploy and a delete alike. // -// The type is --type when given; else the first registered type that -// detects path; else, for a delete given a mount path, the type whose -// default directory that mount path is below; else none. +// The type, for a deploy and a delete alike, is --type when given; else the +// first registered type that detects path; else, given a mount path, the +// type whose default directory that mount path is directly below; else +// none. So a deploy and a delete of the same bundle record the same type. // // The mount path is --mount-path when given, and then path is not checked: // --type dbt with a mount path deploys any directory, as 1.x's @@ -230,12 +231,29 @@ type resolvedBundle struct { // default needs --mount-path. func resolveBundle(cmd *cobra.Command, path string, o *deployBundleOptions) (resolvedBundle, error) { bt := astrodeploy.ResolveBundleType(o.bundleType, path) - if o.bundleType == "" && bt.Name == astrodeploy.BundleTypeNone && o.delete && o.mountPath != "" { + if o.bundleType == "" && bt.Name == astrodeploy.BundleTypeNone && o.mountPath != "" { if inferred, ok := astrodeploy.BundleTypeForMountPath(o.mountPath); ok { bt = inferred } } b := resolvedBundle{bundleType: bt.Name, mountPath: o.mountPath} + // The name is the bundle's when the mount path is the one this type + // derives from path: a mount path of its own names some other bundle, + // or none, and PATH says nothing of it. + named := func() string { + if !bt.HasDefaultMountPath() || (bt.Detect != nil && !bt.Detect(path)) { + return "" + } + derived, err := bt.DefaultMountPath(path) + if err != nil || derived != b.mountPath { + return "" + } + name, err := bt.BundleName(path) + if err != nil { + return "" + } + return name + } if b.mountPath == "" { if !bt.HasDefaultMountPath() { return b, cliout.Usage(fmt.Errorf("--mount-path is required for a bundle of type %s: only a %s bundle has a default", bt.Name, strings.Join(astrodeploy.BundleTypeNames(), " or "))) @@ -252,13 +270,7 @@ func resolveBundle(cmd *cobra.Command, path string, o *deployBundleOptions) (res b.mountPath = mountPath fmt.Fprintf(cmd.ErrOrStderr(), "mount path %s: the default for a %s bundle\n", mountPath, bt.Name) } - // The name is the bundle's when path holds one this type names: a - // --type dbt deploy of another directory has none. - if bt.BundleName != nil && (bt.Detect == nil || bt.Detect(path)) { - if name, err := bt.BundleName(path); err == nil { - b.name = name - } - } + b.name = named() return b, nil } @@ -348,34 +360,26 @@ func bundleTarget(ctx context.Context, name, workspace string, format cliout.For // ids of the ones that share the name. func deploymentNamed(name, ws string, login *deployLogin) (*astrov1.Deployment, error) { if ws == "" { - return nil, fmt.Errorf("--deployment %s is not a link of the project here or a Deployment id, and there is no Workspace to look it up by name in: pass --workspace, or the Deployment id", name) + return nil, cliout.Usage(fmt.Errorf("--deployment %s is not a link of the project here or a Deployment id, and there is no Workspace to look it up by name in: pass --workspace, or the Deployment id", name)) } - deps, err := deployment.ListDeployments(ws, login.context.Organization, login.client) + found, err := deployment.DeploymentsNamed(name, ws, login.context.Organization, login.client) if err != nil { return nil, err } - var found []astrov1.Deployment - for i := range deps { - if deps[i].Name == name { - found = append(found, deps[i]) - } - } switch len(found) { case 0: - return nil, fmt.Errorf("no Deployment named %s in workspace %s: pass a Deployment id, a link name of the project here, or the name of a Deployment in that workspace", name, ws) + // Named wrong, as a link name that is not one would be: a usage + // mistake, not a failure of the deploy. + return nil, cliout.Usage(fmt.Errorf("no Deployment named %s in workspace %s: pass a Deployment id, a link name of the project here, or the name of a Deployment in that workspace", name, ws)) case 1: return &found[0], nil } - ids := make([]string, len(found)) - for i := range found { - ids[i] = found[i].Id - } - return nil, fmt.Errorf("%d Deployments are named %s in workspace %s (%s): pass the id of the one you mean with --deployment", len(found), name, ws, strings.Join(ids, ", ")) + return nil, cliout.Usage(fmt.Errorf("%d Deployments are named %s in workspace %s (%s): pass the id of the one you mean with --deployment", len(found), name, ws, strings.Join(deployment.DeploymentIDs(found), ", "))) } // errBundleNoTarget is a bundle deploy or delete that names no Deployment and // cannot be asked for one, in astro deploy's words for the same refusal. -var errBundleNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id or a link name with --deployment") +var errBundleNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass --deployment with a Deployment id, a Deployment name, or a link name") // bundleDeployJSON is what astro deploy bundle --output json publishes: the // bundle deployed and where. Its deployment, workspace and git keys are the diff --git a/cmd/astro/deploy_bundle_test.go b/cmd/astro/deploy_bundle_test.go index 6d11e53c9..24c09741c 100644 --- a/cmd/astro/deploy_bundle_test.go +++ b/cmd/astro/deploy_bundle_test.go @@ -292,7 +292,7 @@ func TestDeployBundleTarget(t *testing.T) { var got errorJSON decodeOne(t, r.stdout, &got) assert.Equal(t, string(cliout.KindInputRequired), got.Kind) - assert.Contains(t, got.Error, "pass the Deployment id or a link name with --deployment") + assert.Contains(t, got.Error, "pass --deployment with a Deployment id, a Deployment name, or a link name") }) t.Run(name+": none named, without a terminal", func(t *testing.T) { bundle := dbtProject(t) @@ -319,6 +319,62 @@ func TestDeployBundleTarget(t *testing.T) { } } +// A deploy and a delete of the same bundle record the same type: --type, +// else what PATH holds, else what the mount path's shape says. +func TestDeployBundleDeployAndDeleteAgreeOnTheType(t *testing.T) { + for _, tc := range []struct { + name, mountPath, want string + args []string + dbtHere bool + }{ + {name: "a dbt mount path, no bundle here", mountPath: "/usr/local/airflow/dbt/jaffle_shop", want: "dbt"}, + {name: "a nested mount path", mountPath: "/usr/local/airflow/dbt/a/b", want: "none"}, + {name: "a mount path climbing out", mountPath: "/usr/local/airflow/dbt/../x", want: "none"}, + {name: "a dbt project here, another mount path", mountPath: "/usr/local/airflow/x", want: "dbt", dbtHere: true}, + {name: "--type wins", mountPath: "/usr/local/airflow/dbt/jaffle_shop", want: "seeds", args: []string{"--type", "seeds"}}, + } { + t.Run(tc.name, func(t *testing.T) { + bundle := plainBundle(t) + if tc.dbtHere { + bundle = dbtProject(t) + } + deployed, deleted := fakeDeployBundle(t), fakeDeleteBundle(t) + args := append([]string{bundle, "--deployment", bundleTestID, "--mount-path", tc.mountPath}, tc.args...) + r := execBundleRun(t, nil, "", args...) + require.NoError(t, r.err, r.stderr) + r = execBundleRun(t, nil, "", append([]string{"--delete"}, args...)...) + require.NoError(t, r.err, r.stderr) + assert.Equal(t, tc.want, (*deployed).BundleType) + assert.Equal(t, tc.want, (*deleted).BundleType) + }) + } +} + +// bundle_name names the bundle the mount path was derived from: a mount path +// of its own is some other bundle's, or none. +func TestDeployBundleNameOnlyForTheDerivedMountPath(t *testing.T) { + for mountPath, named := range map[string]bool{ + "": true, + "/usr/local/airflow/dbt/jaffle_shop": true, + "/usr/local/airflow/dbt/custom": false, + } { + project := dbtProject(t) + args := []string{project, "--deployment", dbtTestDeploymentID, "-o", "json"} + if mountPath != "" { + args = append(args, "--mount-path", mountPath) + } + r := execBundleRun(t, dbtBundleMock(t, fakeBlobStore(t)), "", args...) + require.NoError(t, r.err, r.stderr) + var got bundleDeployJSON + fields := decodeOne(t, r.stdout, &got) + if named { + assert.Equal(t, "jaffle_shop", got.BundleName, mountPath) + } else { + assert.NotContains(t, fields, "bundle_name", mountPath) + } + } +} + // --deployment that is neither a link nor an id is a Deployment's name in // the workspace: one match is used, as read; none, or several, are errors // that say so. @@ -340,19 +396,29 @@ func TestDeployBundleDeploymentByName(t *testing.T) { require.NoError(t, r.err, r.stderr) assert.Equal(t, dbtTestDeploymentID, (*captured).DeploymentID) }) + // None, or several, is a Deployment named wrong: a usage error, under + // json the usage error object. t.Run("none", func(t *testing.T) { project := dbtProject(t) noBundleTransport(t) - r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", project, "--deployment", "reporting", "--workspace", "clxdbtworkspace0000000000") + r := execBundleRun(t, coreMock(t, dbtTestDeployment()), "", project, "--deployment", "reporting", "--workspace", "clxdbtworkspace0000000000", "-o", "json") require.Error(t, r.err) - assert.Contains(t, r.err.Error(), "no Deployment named reporting in workspace clxdbtworkspace0000000000") + assert.Equal(t, cliout.ExitUsage, r.code) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, string(cliout.KindUsage), got.Kind) + assert.Contains(t, got.Error, "no Deployment named reporting in workspace clxdbtworkspace0000000000") }) t.Run("two", func(t *testing.T) { project := dbtProject(t) noBundleTransport(t) - r := execBundleRun(t, coreMock(t, dbtTestDeployment(), other), "", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000") + r := execBundleRun(t, coreMock(t, dbtTestDeployment(), other), "", project, "--deployment", "analytics", "--workspace", "clxdbtworkspace0000000000", "-o", "json") require.Error(t, r.err) - assert.Contains(t, r.err.Error(), "2 Deployments are named analytics in workspace clxdbtworkspace0000000000 ("+dbtTestDeploymentID+", clxdbtother00000000000000)") + assert.Equal(t, cliout.ExitUsage, r.code) + var got errorJSON + decodeOne(t, r.stdout, &got) + assert.Equal(t, string(cliout.KindUsage), got.Kind) + assert.Contains(t, got.Error, "2 Deployments are named analytics in workspace clxdbtworkspace0000000000 ("+dbtTestDeploymentID+", clxdbtother00000000000000)") }) } @@ -518,6 +584,8 @@ func TestDbtRemovedReadsTheSubcommand(t *testing.T) { {[]string{"-n", "prod", "delete"}, errDbtDeleteRemoved}, {[]string{"--project-path=.", "cleanup"}, errDbtCleanupRemoved}, {[]string{"-w", "deploy"}, errDbtDeployRemoved}, + {[]string{"-wt", "5m", "deploy"}, errDbtDeployRemoved}, + {[]string{"-nprod", "delete"}, errDbtDeleteRemoved}, {[]string{"--workspace-id", "deploy"}, errDbtRemoved}, {[]string{"--", "deploy"}, errDbtRemoved}, } { diff --git a/cmd/local/link.go b/cmd/local/link.go index c92a210e1..9fb95cc4d 100644 --- a/cmd/local/link.go +++ b/cmd/local/link.go @@ -340,9 +340,10 @@ func (c *cli) runLinkAdd(cmd *cobra.Command, in *linkAddInput, name string, comm return err } name = strings.TrimSpace(name) - // bundle is astro deploy bundle, so a link by that name could not be - // deployed to by name: refused before anything is asked. - if name == scaffold.BundleLinkName { + // bundle is astro deploy bundle, so a new link by that name could not be + // deployed to by name: refused before anything is asked. One that exists + // is the --replace check's. + if _, exists := m.Astro.Deployments[name]; name == scaffold.BundleLinkName && !exists { return cliout.Usage(scaffold.ErrBundleLinkName) } // A NAME already linked is refused before anything asks a question or diff --git a/cmd/local/link_test.go b/cmd/local/link_test.go index 3d3c1a569..e1322b891 100644 --- a/cmd/local/link_test.go +++ b/cmd/local/link_test.go @@ -171,6 +171,12 @@ func TestLinkAddRefusesBundleAsAName(t *testing.T) { assert.True(t, cliout.IsUsage(err), "%v", err) assert.Contains(t, err.Error(), "bundle cannot name a link: astro deploy bundle deploys a non-DAG bundle") assert.Equal(t, linkManifest, readManifest(t, dir)) + + // One already called bundle is replaced as any other is. + dir, path := linkTestProject(t, linkManifest+"\n[tool.astro.deployments.bundle]\ndeployment = 'dep-old'\n") + _, _, err = runLink(t, dir, "add", "bundle", "--deployment", "dep-new", "--replace") + require.NoError(t, err) + assert.Equal(t, "dep-new", loadManifest(t, path).Astro.Deployments["bundle"].Deployment) } // An MWAA link writes the shared region when given one, and an empty --region diff --git a/docs/deploy.md b/docs/deploy.md index 82ee23014..80feb43de 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -28,15 +28,15 @@ Every refusal is reported under `--output json` as the error object with kind `n **`astro deploy bundle [PATH]`** ships a directory of its own, PATH or the working directory, as a non-DAG bundle: a dbt project, or any other files the DAGs read, mounted at a path in Airflow. It carries no DAGs, and it is not a project deploy: PATH must be a directory, and one in or below any project the walk finds is refused. **`--delete`** removes the bundle mounted at a path instead. It replaces Astro CLI 1.x's `astro dbt deploy` and `astro dbt delete` and the 1.x `astro deploy --non-dags`, which did the same upload with different defaults and Deployment selection. `astro deployment bundle` is something else: it lists and changes the bundle registrations on a Deployment, and uploads nothing. -Because `bundle` is a subcommand, `astro deploy bundle` never means a Deployment called bundle. `bundle` is no link's name: `scaffold.SaveLink` refuses it for every writer (`astro link add` among them, and Astro Desktop), and a deploy that links a picked Deployment names it otherwise; a link already called `bundle` is deployed to with `astro deploy --deployment bundle`. +Because `bundle` is a subcommand, `astro deploy bundle` never means a Deployment called bundle. No new link may be called `bundle`: `scaffold.SaveLink` refuses to add one for every writer (`astro link add` among them, and Astro Desktop), and a deploy that links a picked Deployment names it otherwise. A link already called `bundle` saves again as any other does, a link already called `bundle` is deployed to with `astro deploy --deployment bundle`. -- **The type** decides the defaults. The types are a table in [`internal/platform/astro/deploy/bundletype.go`](../internal/platform/astro/deploy/bundletype.go), each with how to detect it, how to check a bundle of it, how to name it, and the directory its bundles mount below by default; adding a type is adding an entry. With no `--type`, the first type that detects PATH is used, else `none`. Today the table has one entry, `dbt`: a directory holding `dbt_project.yml`, named by the project's `name` and mounted by default at `/usr/local/airflow/dbt/`. Any other `--type` (and `none`) is a free-form label the deploy records, with no defaults, so it needs `--mount-path`. A default mount path is said on stderr. +- **The type** decides the defaults. The types are a table in [`internal/platform/astro/deploy/bundletype.go`](../internal/platform/astro/deploy/bundletype.go), each with how to detect it, how to check a bundle of it, how to name it, and the directory its bundles mount below by default; adding a type is adding an entry. With no `--type`, the first type that detects PATH is used; else, given `--mount-path`, the type whose default it has the shape of (its directory and one name below it, after cleaning: `/usr/local/airflow/dbt/` is `dbt`); else `none`. A deploy and a delete of the same bundle so record the same type. Today the table has one entry, `dbt`: a directory holding `dbt_project.yml`, named by the project's `name` and mounted by default at `/usr/local/airflow/dbt/`. Any other `--type` (and `none`) is a free-form label the deploy records, with no defaults, so it needs `--mount-path`. A default mount path is said on stderr. - **PATH is checked only when the mount path is derived from it.** Without `--mount-path`, `--type dbt` needs a dbt project at PATH. With `--mount-path`, `--type dbt` labels any directory, as 1.x's `--non-dags-bundle-type dbt` did. -- **Delete** uses `--mount-path` as given, or the mount path PATH's type would deploy to, so `astro deploy bundle --delete` in a dbt project removes that project. With neither, it is a usage error. Its type is `--type`, else PATH's, else, from a directory with no bundle, the type whose default directory `--mount-path` is below (`/usr/local/airflow/dbt/…` is `dbt`), else `none`. +- **Delete** uses `--mount-path` as given, or the mount path PATH's type would deploy to, so `astro deploy bundle --delete` in a dbt project removes that project. With neither, it is a usage error. Its type is found as the deploy's is. - **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project in the working directory whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. -- **The target** is `--deployment`, resolved in the order the other commands use: a link of the project in the working directory, else a Deployment id, else the name of a Deployment in `--workspace` (or the project's `[tool.astro] workspace`, or the login's). A name that matches no Deployment, or several, is an error; several are listed by id. It has no `-d`: on `astro deploy`, `-d` is `--dags`. Without it, the Deployments of `--workspace` (or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id or a link name with --deployment`. A project on another host is told to pass `--deployment ` or switch context. +- **The target** is `--deployment`: a link of the project in the working directory, else a Deployment id, else the name of a Deployment in `--workspace` (or the project's `[tool.astro] workspace`, or the login's). The name lookup is this command's, as 1.x's `astro dbt deploy --deployment-name` had one; `astro deploy` takes a link or an id. A name that matches no Deployment, or several (listed by id), is a usage error, as a mistyped link name is. It has no `-d`: on `astro deploy`, `-d` is `--dags`. Without it, the Deployments of `--workspace` (or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass --deployment with a Deployment id, a Deployment name, or a link name`. A project on another host is told to pass `--deployment ` or switch context. - **`--wait`** waits for the Deployment to take the deploy or the delete, under the same login. In text the result prints first and the wait's progress goes to stderr; under json the one object comes after the wait, with `waited` and, when it failed, `wait_error`, and the run exits 1. -- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-bundle.json` and, for `--delete`, `deploy-bundle-delete.json`. Both carry `bundle_type`, `mount_path`, `deployment` and `deploy_id`, and `bundle_name` when the type names the bundle at PATH (a dbt project's `name`); the deploy adds `bundle_path`, `bundle_version` and `git`. 1.x's `astro dbt deploy -o json` keys `project` and `project_path` are `bundle_name` and `bundle_path` here, beside `bundle_type`. What the deploy prints on the way (the git note, the default mount path) goes to stderr there. +- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-bundle.json` and, for `--delete`, `deploy-bundle-delete.json`. Both carry `bundle_type`, `mount_path`, `deployment` and `deploy_id`, and `bundle_name` when the mount path is the one the type derives from PATH (a dbt project's `name`; a mount path of its own names no bundle); the deploy adds `bundle_path`, `bundle_version` and `git`. 1.x's `astro dbt deploy -o json` keys `project` and `project_path` are `bundle_name` and `bundle_path` here, beside `bundle_type`. What the deploy prints on the way (the git note, the default mount path) goes to stderr there. **Flags that cannot all apply** are refused as usage errors before anything is read: `--wait-time` without `--wait`, `--no-dags-base-dir` with a deploy that ships no DAGs (`--image`, or `--image-name` outside a project), and `--dags` with `--image` or `--image-name` (the message names the ones given). diff --git a/internal/platform/astro/deploy/bundletype.go b/internal/platform/astro/deploy/bundletype.go index e7cadbc2e..e0219942c 100644 --- a/internal/platform/astro/deploy/bundletype.go +++ b/internal/platform/astro/deploy/bundletype.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + pathpkg "path" "path/filepath" "strings" @@ -97,11 +98,18 @@ func ResolveBundleType(name, dir string) BundleType { return BundleType{Name: name} } -// BundleTypeForMountPath is the registered type whose bundles mount below -// mountPath's directory by default, and whether there is one. +// BundleTypeForMountPath is the registered type whose default mount path +// mountPath has the shape of, MountPathPrefix and one path component, and +// whether there is one. The path is cleaned first, so .. cannot climb out of +// the prefix and a nested path is no default's. func BundleTypeForMountPath(mountPath string) (BundleType, bool) { + clean := pathpkg.Clean(mountPath) for _, t := range bundleTypes { - if t.MountPathPrefix != "" && strings.HasPrefix(mountPath, t.MountPathPrefix) && len(mountPath) > len(t.MountPathPrefix) { + if t.MountPathPrefix == "" { + continue + } + rest, ok := strings.CutPrefix(clean, t.MountPathPrefix) + if ok && rest != "" && !strings.Contains(rest, "/") { return t, true } } diff --git a/internal/platform/astro/deploy/bundletype_test.go b/internal/platform/astro/deploy/bundletype_test.go index b86101ac1..7116483f8 100644 --- a/internal/platform/astro/deploy/bundletype_test.go +++ b/internal/platform/astro/deploy/bundletype_test.go @@ -74,7 +74,13 @@ func TestBundleTypeForMountPath(t *testing.T) { bt, ok := BundleTypeForMountPath("/usr/local/airflow/dbt/jaffle_shop") require.True(t, ok) assert.Equal(t, "dbt", bt.Name) - for _, p := range []string{"/usr/local/airflow/dbt/", "/usr/local/airflow/dbt", "/usr/local/airflow/seeds", ""} { + bt, ok = BundleTypeForMountPath("/usr/local/airflow/dbt/jaffle_shop/") + require.True(t, ok, "a trailing slash is cleaned away") + assert.Equal(t, "dbt", bt.Name) + for _, p := range []string{ + "/usr/local/airflow/dbt/", "/usr/local/airflow/dbt", "/usr/local/airflow/seeds", "", + "/usr/local/airflow/dbt/../seeds", "/usr/local/airflow/dbt/a/b", "/usr/local/airflow/dbt/a/..", + } { _, ok := BundleTypeForMountPath(p) assert.False(t, ok, p) } diff --git a/internal/platform/astro/deployment/clone_run.go b/internal/platform/astro/deployment/clone_run.go index 94f9f1ce8..acba4f932 100644 --- a/internal/platform/astro/deployment/clone_run.go +++ b/internal/platform/astro/deployment/clone_run.go @@ -23,23 +23,43 @@ func CloneSource(ref, ws string, astroV1Client astrov1.APIClient) (astrov1.Deplo if ws == "" { return astrov1.Deployment{}, fmt.Errorf("no Workspace to look up the Deployment named %q in: pass --workspace, or its id to --clone", ref) } - deployments, err := ListDeployments(ws, "", astroV1Client) + named, err := DeploymentsNamed(ref, ws, "", astroV1Client) if err != nil { return astrov1.Deployment{}, err } - var ids []string - for i := range deployments { - if deployments[i].Name == ref { - ids = append(ids, deployments[i].Id) - } - } - switch len(ids) { + switch len(named) { case 0: return astrov1.Deployment{}, fmt.Errorf("no Deployment named %q in Workspace %s. Pass its id to --clone to copy one from another Workspace", ref, ws) case 1: - return GetDeploymentByID("", ids[0], astroV1Client) + return GetDeploymentByID("", named[0].Id, astroV1Client) + } + return astrov1.Deployment{}, fmt.Errorf("%d Deployments are named %q in Workspace %s: pass one of their ids to --clone instead: %s", len(named), ref, ws, strings.Join(DeploymentIDs(named), ", ")) +} + +// DeploymentsNamed is every Deployment in workspace ws called name, listed +// in organization orgID ("" for the current context's). A name is not +// unique, so a caller decides what none, or several, means. +func DeploymentsNamed(name, ws, orgID string, astroV1Client astrov1.APIClient) ([]astrov1.Deployment, error) { + deployments, err := ListDeployments(ws, orgID, astroV1Client) + if err != nil { + return nil, err + } + var named []astrov1.Deployment + for i := range deployments { + if deployments[i].Name == name { + named = append(named, deployments[i]) + } + } + return named, nil +} + +// DeploymentIDs is the id of each of ds, in order. +func DeploymentIDs(ds []astrov1.Deployment) []string { + ids := make([]string, len(ds)) + for i := range ds { + ids[i] = ds[i].Id } - return astrov1.Deployment{}, fmt.Errorf("%d Deployments are named %q in Workspace %s: pass one of their ids to --clone instead: %s", len(ids), ref, ws, strings.Join(ids, ", ")) + return ids } // Clone creates a copy of src named name, in workspaceID or else src's own diff --git a/pkg/scaffold/linkwrite.go b/pkg/scaffold/linkwrite.go index 18268358e..d9a4fe34b 100644 --- a/pkg/scaffold/linkwrite.go +++ b/pkg/scaffold/linkwrite.go @@ -33,10 +33,11 @@ import ( // because the parser would refuse it. The message names the field. var ErrInvalidLink = errors.New("invalid link") -// BundleLinkName is a name no link may have: astro deploy reads astro deploy -// bundle as its subcommand that deploys a non-DAG bundle, so a deploy could -// never reach a link by that name as its argument. A manifest that already -// has one still loads, and it is reached with --deployment. +// BundleLinkName is a name no new link may have: astro deploy reads astro +// deploy bundle as its subcommand that deploys a non-DAG bundle, so a deploy +// could never reach a link by that name as its argument. A link already +// called that still loads, is reached with --deployment, and saves again: only +// adding one is refused. const BundleLinkName = "bundle" // ErrBundleLinkName is SaveLink refusing BundleLinkName. It wraps @@ -208,6 +209,9 @@ func SaveLink(dir string, wrap func(run func() error) error, l Link) error { //nolint:gocritic // hugeParam: by value, as SaveLink holds it func (l Link) edit() ManifestEdit { return func(before *manifest.Manifest, ed tomledit.Editor) error { + if _, exists := before.Astro.Deployments[l.Name]; l.Name == BundleLinkName && !exists { + return ErrBundleLinkName + } key := linkKey(l.Name) // Read from the raw TOML, before the entry is replaced: the parsed // link has the defaults folded in, so it cannot say which keys the link @@ -626,8 +630,6 @@ func (l *Link) validate() error { return invalidLink("a link needs a name") case manifest.ReservedLinkName: return invalidLink("%q is reserved: it always means the Airflow running on this machine", manifest.ReservedLinkName) - case BundleLinkName: - return ErrBundleLinkName } switch l.Kind { case manifest.KindAstro: diff --git a/pkg/scaffold/linkwrite_test.go b/pkg/scaffold/linkwrite_test.go index eaac67594..eed081fe7 100644 --- a/pkg/scaffold/linkwrite_test.go +++ b/pkg/scaffold/linkwrite_test.go @@ -169,7 +169,6 @@ func TestSaveLinkRefusesAnInvalidLink(t *testing.T) { }{ {"no name", Link{Name: " ", Kind: manifest.KindAstro, Workspace: "W", Deployment: "d"}, "needs a name"}, {"the reserved name", Link{Name: manifest.ReservedLinkName, Kind: manifest.KindMWAA, Environment: "env"}, "reserved"}, - {"the deploy bundle subcommand's name", Link{Name: BundleLinkName, Kind: manifest.KindAstro, Workspace: "W", Deployment: "d"}, "astro deploy bundle deploys a non-DAG bundle"}, {"an astro link with no deployment", Link{Name: "prod", Kind: manifest.KindAstro, Workspace: "W"}, "needs a deployment"}, {"an mwaa link with no environment", Link{Name: "aws", Kind: manifest.KindMWAA}, "needs an environment"}, {"an endpoint with no url", Link{Name: "oss", Kind: manifest.KindEndpoint, Auth: manifest.Auth{Method: manifest.AuthNone}}, "needs a url"}, @@ -190,6 +189,24 @@ func TestSaveLinkRefusesAnInvalidLink(t *testing.T) { } } +// A new link cannot be called bundle, the word astro deploy bundle takes; one +// already called that saves again. +func TestSaveLinkRefusesANewBundleLink(t *testing.T) { + dir, path := linkProject(t, linkFixture) + err := SaveLink(dir, nil, Link{Name: BundleLinkName, Kind: manifest.KindAstro, Workspace: "W", Deployment: "d"}) + require.ErrorIs(t, err, ErrBundleLinkName) + require.ErrorIs(t, err, ErrInvalidLink) + assert.Contains(t, err.Error(), "astro deploy bundle deploys a non-DAG bundle") + assert.Equal(t, linkFixture, readFile(t, path)) + + existing := linkFixture + "\n[tool.astro.deployments.bundle]\nworkspace = 'W'\ndeployment = 'old'\n" + dir, path = linkProject(t, existing) + require.NoError(t, SaveLink(dir, nil, Link{Name: BundleLinkName, Kind: manifest.KindAstro, Deployment: "new"})) + m, err := manifest.Load(path) + require.NoError(t, err) + assert.Equal(t, "new", m.Astro.Deployments[BundleLinkName].Deployment) +} + // A credential pasted where an env var name belongs is refused, and the error // does not quote it: the message may land in a terminal or a CI log, and the // manifest is committed.