diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..344684db --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Cobra owns this complete generated reference; authored command docs remain visible. +/docs/cli-reference.md linguist-generated=true diff --git a/.github/workflows/generate-qualification.yml b/.github/workflows/generate-qualification.yml index 5feae773..9fb0c07e 100644 --- a/.github/workflows/generate-qualification.yml +++ b/.github/workflows/generate-qualification.yml @@ -101,6 +101,11 @@ jobs: "$RUNNER_TEMP/generate-bin/buf" --version echo "$RUNNER_TEMP/generate-bin" >> "$GITHUB_PATH" - name: Run tagged and qualification tests in isolated processes + env: + # Core uses this read-only token for release lookups and downloads. + # Each test has its own CODEFLY_HOME, so unauthenticated lookups + # otherwise exhaust the runner's shared GitHub API quota. + GITHUB_TOKEN: ${{ github.token }} run: ./scripts/test-generate-isolated.sh "${{ matrix.shard }}" "${{ strategy.job-total }}" - name: Retain test output, timings and conformance receipts if: always() diff --git a/cmd/environment/configuration_profile.go b/cmd/environment/configuration_profile.go index 23d01315..e63a3185 100644 --- a/cmd/environment/configuration_profile.go +++ b/cmd/environment/configuration_profile.go @@ -3,6 +3,7 @@ package environment import ( "context" "fmt" + "io/fs" "os" "path/filepath" "regexp" @@ -55,14 +56,19 @@ func materializeConfigurationProfile(ctx context.Context, directory string, env } } output := filepath.Join(destination, configurations.ProfileDerivationFile) - retired, err := retiredGeneratedConfigurations(destination, output) + confined, err := os.OpenRoot(configurationRoot) + if err != nil { + return err + } + defer confined.Close() + retired, err := retiredGeneratedConfigurations(confined, target) if err != nil { return err } if dryRun { return nil } - if err := os.MkdirAll(destination, 0o755); err != nil { + if err := confined.MkdirAll(target, 0o755); err != nil { return err } body := profileMarker + env.ConfigurationSource + "\nderives-from: " + env.ConfigurationSource + "\n" @@ -70,7 +76,7 @@ func materializeConfigurationProfile(ctx context.Context, directory string, env return err } for _, path := range retired { - if err := os.Remove(path); err != nil { + if err := confined.Remove(path); err != nil { return err } } @@ -89,7 +95,12 @@ func validateConfigurationSourceLayer(layer, destination string) error { if err != nil || resolved != layer { return fmt.Errorf("configuration source must be an ordinary profile directory") } - entries, err := os.ReadDir(layer) + confined, err := os.OpenRoot(layer) + if err != nil { + return err + } + defer confined.Close() + entries, err := fs.ReadDir(confined.FS(), ".") if err != nil { return err } @@ -108,7 +119,7 @@ func validateConfigurationSourceLayer(layer, destination string) error { return fmt.Errorf("configuration-source must not contain environment-specific runnable bindings") } if strings.HasSuffix(entry.Name(), ".secret.env") { - if err := validateSecretPlaceholders(filepath.Join(layer, entry.Name()), entry.Name()); err != nil { + if err := validateSecretPlaceholders(confined, entry.Name()); err != nil { return err } } @@ -118,8 +129,8 @@ func validateConfigurationSourceLayer(layer, destination string) error { // validateSecretPlaceholders refuses a source secret file whose keys carry a value: // a configuration source declares which secrets exist, never what they hold. -func validateSecretPlaceholders(path, name string) error { - body, err := os.ReadFile(path) +func validateSecretPlaceholders(confined *os.Root, name string) error { + body, err := confined.ReadFile(name) if err != nil { return err } @@ -139,13 +150,13 @@ func validateSecretPlaceholders(path, name string) error { // retiredGeneratedConfigurations lists the generated files an earlier import left in // the destination, which a re-import removes; it refuses to overwrite an authored one // and a destination that is not an ordinary directory. -func retiredGeneratedConfigurations(destination, output string) ([]string, error) { - if info, err := os.Lstat(destination); err == nil && !info.IsDir() { +func retiredGeneratedConfigurations(confined *os.Root, target string) ([]string, error) { + if info, err := confined.Lstat(target); err == nil && !info.IsDir() { return nil, fmt.Errorf("generated profile must be an ordinary directory") } else if err != nil && !os.IsNotExist(err) { return nil, err } - previous, err := os.ReadDir(destination) + previous, err := fs.ReadDir(confined.FS(), target) if err != nil && !os.IsNotExist(err) { return nil, err } @@ -154,12 +165,12 @@ func retiredGeneratedConfigurations(destination, output string) ([]string, error if !strings.HasSuffix(entry.Name(), ".env") && entry.Name() != configurations.ProfileDerivationFile { continue } - path := filepath.Join(destination, entry.Name()) + path := filepath.Join(target, entry.Name()) info, infoErr := entry.Info() if infoErr != nil || !info.Mode().IsRegular() { return nil, fmt.Errorf("generated configuration %s must be a regular file", entry.Name()) } - body, readErr := os.ReadFile(path) + body, readErr := confined.ReadFile(path) if readErr != nil { return nil, readErr } @@ -167,7 +178,7 @@ func retiredGeneratedConfigurations(destination, output string) ([]string, error if !owned && entry.Name() != "runnable-bindings.env" { return nil, fmt.Errorf("refusing to overwrite authored configuration %s", entry.Name()) } - if path != output && owned { + if entry.Name() != configurations.ProfileDerivationFile && owned { retired = append(retired, path) } } diff --git a/cmd/environment/configuration_profile_test.go b/cmd/environment/configuration_profile_test.go index 1c9f485f..d619bf25 100644 --- a/cmd/environment/configuration_profile_test.go +++ b/cmd/environment/configuration_profile_test.go @@ -1,6 +1,7 @@ package environment import ( + "bytes" "os" "path/filepath" "strings" @@ -11,6 +12,46 @@ import ( "github.com/stretchr/testify/require" ) +func TestImportProductProfileIntoExistingPlacement(t *testing.T) { + root := profileSource(t) + workspace := "name: acme\nlayout: modules\nenvironments:\n- name: azure\n namespace: acme\n service-identity:\n services:\n model/model:\n principal: applied-client\n" + require.NoError(t, os.WriteFile(filepath.Join(root, "workspace.codefly.yaml"), []byte(workspace), 0o644)) + opts := importOptions{configurationSource: "hosted", configurationProfiles: []string{"azure", "local"}, dryRun: true} + doImport(t, root, opts) + require.Equal(t, workspace, readFile(t, filepath.Join(root, "workspace.codefly.yaml"))) + _, err := os.Stat(filepath.Join(root, "configurations", "azure")) + require.True(t, os.IsNotExist(err)) + opts.dryRun = false + doImport(t, root, opts) + env := loadWorkspace(t, root).FindEnvironment("azure") + require.Equal(t, "hosted", env.ConfigurationSource) + require.Equal(t, []string{"azure", "local"}, env.ConfigurationProfiles) + require.Contains(t, readFile(t, filepath.Join(root, "workspace.codefly.yaml")), "principal: applied-client") + profile := filepath.Join(root, "configurations", "azure", configurations.ProfileDerivationFile) + require.Contains(t, readFile(t, profile), "derives-from: hosted") + // A later placement refresh keeps and regenerates the product selection. + require.NoError(t, os.Remove(profile)) + doImport(t, root, importOptions{}) + require.Contains(t, readFile(t, profile), "derives-from: hosted") +} + +func TestImportProductProfileRefusesCoordinateConflict(t *testing.T) { + for _, declared := range []string{ + `"configuration-source":"other"`, + `"configuration-profile":"other"`, + `"configuration-profiles":["other"]`, + } { + t.Run(declared, func(t *testing.T) { + root := profileSource(t) + workspace := "name: acme\nlayout: modules\n" + require.NoError(t, os.WriteFile(filepath.Join(root, "workspace.codefly.yaml"), []byte(workspace), 0o644)) + opts := importOptions{dir: root, envName: "azure", configurationSource: "hosted", configurationProfiles: []string{"azure", "local"}, contractData: []byte(`{"schema":"codefly/coordinate/v1","environment":{"name":"azure",` + declared + `}}`), stdout: &bytes.Buffer{}} + require.Error(t, runImport(t.Context(), &opts)) + require.Equal(t, workspace, readFile(t, filepath.Join(root, "workspace.codefly.yaml"))) + }) + } +} + func profileSource(t *testing.T) string { t.Helper() root := t.TempDir() diff --git a/cmd/environment/import.go b/cmd/environment/import.go index f94fb096..b5a628ef 100644 --- a/cmd/environment/import.go +++ b/cmd/environment/import.go @@ -86,6 +86,8 @@ full delivery.intent.yaml is not interpreted.`, if err != nil { return err } + configurationSource, _ := cmd.Flags().GetString("configuration-source") + configurationProfiles, _ := cmd.Flags().GetStringSlice("configuration-profiles") productPath, err := cmd.Flags().GetString("product-intent") if err != nil { @@ -108,18 +110,23 @@ full delivery.intent.yaml is not interpreted.`, } opts := importOptions{ - dir: *dir, - envName: args[0], - contractData: data, - productIntentData: productData, - namespace: importNamespace, - namespaceSet: cmd.Flags().Changed("namespace"), - dryRun: importDryRun, + dir: *dir, + envName: args[0], + contractData: data, + productIntentData: productData, + namespace: importNamespace, + namespaceSet: cmd.Flags().Changed("namespace"), + dryRun: importDryRun, + configurationSource: configurationSource, + configurationProfiles: configurationProfiles, // UTC so the provenance timestamp does not vary by the operator's // machine timezone and churn the diff on re-import across a team. now: time.Now().UTC(), stdout: cmd.OutOrStdout(), } + if err := common.ResolvePinnedModules(ctx); err != nil { + return err + } if err := runImport(ctx, &opts); err != nil { return err } @@ -170,15 +177,17 @@ func readContract(stdin io.Reader, path string) ([]byte, error) { } type importOptions struct { - dir string - envName string - contractData []byte - productIntentData []byte - namespace string - namespaceSet bool - dryRun bool - now time.Time - stdout io.Writer + dir string + envName string + contractData []byte + productIntentData []byte + namespace string + namespaceSet bool + dryRun bool + now time.Time + stdout io.Writer + configurationSource string + configurationProfiles []string } // runImport parses the contract, merges its owned fields into the named @@ -202,6 +211,9 @@ func runImport(ctx context.Context, opts *importOptions) error { return err } declaredEnvironment := yamledit.MapValue(declaration, "environment") + if profileErr := applyConfigurationProfiles(opts, contract, declaredEnvironment); profileErr != nil { + return profileErr + } file := filepath.Join(opts.dir, resources.WorkspaceConfigurationName) original, err := os.ReadFile(file) @@ -267,14 +279,14 @@ func runImport(ctx context.Context, opts *importOptions) error { if err != nil { return err } - if _, err := environments.FromWorkspace(merged); err != nil { + if _, err = environments.FromWorkspace(merged); err != nil { return fmt.Errorf("merged environment declarations: %w", err) } + selected, err := environments.Select(merged, opts.envName) + if err != nil { + return err + } if intent != nil { - selected, err := environments.Select(merged, opts.envName) - if err != nil { - return err - } workspace, err := resources.LoadWorkspaceFromDir(ctx, opts.dir) if err != nil { return err @@ -286,7 +298,7 @@ func runImport(ctx context.Context, opts *importOptions) error { return fmt.Errorf("product intent: %w", err) } } - if err := materializeConfigurationProfile(ctx, opts.dir, env, opts.dryRun); err != nil { + if err := materializeConfigurationProfile(ctx, opts.dir, selected, opts.dryRun); err != nil { return err } if opts.dryRun { @@ -608,9 +620,40 @@ func stampProvenance(envNode *yaml.Node, contract *environments.CoordinateContra func init() { importCmd.Flags().String("coordinate-contract", "", "Path to a codefly/coordinate/v1 descriptor, or - for stdin") + importCmd.Flags().String("configuration-source", "", "Authored product profile inherited by the imported environment") + importCmd.Flags().StringSlice("configuration-profiles", nil, "Product configuration profile chain when absent from the coordinate declaration") importCmd.Flags().String("product-intent", "", "Authored product decisions: environments..exclude-services and service-secrets generate / may-be-empty") importCmd.Flags().String("cell-contract", "", "Former spelling of --coordinate-contract") _ = importCmd.Flags().MarkDeprecated("cell-contract", "use --coordinate-contract") importCmd.Flags().StringVar(&importNamespace, "namespace", "", "Assert the namespace declared by the producer") importCmd.Flags().BoolVar(&importDryRun, "dry-run", false, "Print the unified diff of workspace.codefly.yaml and write nothing") } + +func applyConfigurationProfiles(opts *importOptions, contract *environments.CoordinateContract, declaredEnvironment *yaml.Node) error { + // Product profiles are owned by the consuming workspace. A placement-only + // producer need not acquire product configuration to import its target. + if opts.configurationSource != "" { + if existing := contract.Environment.ConfigurationSource; existing != "" && existing != opts.configurationSource { + return fmt.Errorf("--configuration-source conflicts with the coordinate declaration") + } + contract.Environment.ConfigurationSource = opts.configurationSource + node, encodeErr := yamledit.Encode(opts.configurationSource) + if encodeErr != nil { + return encodeErr + } + yamledit.SetMapValue(declaredEnvironment, "configuration-source", node) + } + if len(opts.configurationProfiles) > 0 { + if contract.Environment.ConfigurationProfile != "" || len(contract.Environment.ConfigurationProfiles) > 0 { + return fmt.Errorf("--configuration-profiles cannot replace coordinate-declared profiles") + } + contract.Environment.ConfigurationProfiles = opts.configurationProfiles + node, encodeErr := yamledit.Encode(opts.configurationProfiles) + if encodeErr != nil { + return encodeErr + } + yamledit.SetMapValue(declaredEnvironment, "configuration-profiles", node) + } + + return nil +} diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 966ccb69..efc48c49 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -2702,10 +2702,12 @@ codefly environment import --coordinate-contract Flags: ``` - --coordinate-contract string Path to a codefly/coordinate/v1 descriptor, or - for stdin - --dry-run Print the unified diff of workspace.codefly.yaml and write nothing - --namespace string Assert the namespace declared by the producer - --product-intent string Authored product decisions: environments..exclude-services and service-secrets generate / may-be-empty + --configuration-profiles strings Product configuration profile chain when absent from the coordinate declaration + --configuration-source string Authored product profile inherited by the imported environment + --coordinate-contract string Path to a codefly/coordinate/v1 descriptor, or - for stdin + --dry-run Print the unified diff of workspace.codefly.yaml and write nothing + --namespace string Assert the namespace declared by the producer + --product-intent string Authored product decisions: environments..exclude-services and service-secrets generate / may-be-empty ``` ## `codefly environment show` diff --git a/docs/commands.md b/docs/commands.md index 554ae783..eead739b 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -2758,6 +2758,19 @@ outputs, preserving the environment's separately generated runnable bindings. Run `codefly generate runnable-bindings --env blue` after import so addresses are resolved for blue rather than copied from a sibling. +For a placement-only coordinate, the product may select its own source at import: + +```bash +codefly environment import staging --coordinate-contract coordinate.json \ + --configuration-source hosted --configuration-profiles staging,local +``` + +These options use the same importer and profile inheritance. A source conflicting +with the coordinate is refused; an explicit profile chain cannot replace a chain +or single profile declared by the coordinate. Without a profile chain, the target +defaults to the environment name. Later placement-only imports preserve the +product selection and regenerate its inherited profile. + The cell inventory also carries `source.path` when publication uses a root other than `deployments`. Consumers must use that root to discover the published module trees. The local render directory remains a qualification tree; `publish` writes diff --git a/docs/module-manifests.md b/docs/module-manifests.md index 752035ef..928fb7a8 100644 --- a/docs/module-manifests.md +++ b/docs/module-manifests.md @@ -92,7 +92,10 @@ digest and media type, verbatim. A deployed render of a release declaring none builds the bundle from the release's own source, as it builds the release's images: `npm ci && npm run build` in the remote's npm project (`services/fe-remote`, else the component root), inside a digest-pinned Node -image, on the committed tree (HEAD) of the repository the component is +image, using the same environment-selected, host-networked BuildKit builder and +registry mirrors as service images. This reuses the existing builder provisioner; +the default Docker bridge can otherwise lose the operator's DNS path during npm +installation. The build runs on the committed tree (HEAD) of the repository the component is released from: nothing untracked or ignored reaches the builder (nor do uncommitted edits), and the module cache is never written. A component in no repository builds from its own directory alone. It archives `services/fe-remote/dist` in core's diff --git a/pkg/environments/environment.go b/pkg/environments/environment.go index 18fefcc4..989500b4 100644 --- a/pkg/environments/environment.go +++ b/pkg/environments/environment.go @@ -1236,6 +1236,18 @@ type SecretScope struct { Service string } +// MappingFor selects a module-qualified declaration before the unqualified fallback, +// matching service configuration and identity selection. +func (s *EnvironmentServiceSecrets) MappingFor(module, service string) EnvironmentServiceSecretMapping { + if s == nil { + return EnvironmentServiceSecretMapping{} + } + if mapping, ok := s.Services[module+"/"+service]; module != "" && ok { + return mapping + } + return s.Services[service] +} + // RemoteRef locates one secret key of a service in the remote store. An explicit // per-service RemoteKeys entry wins; else the service's own Defaults template; // else the environment-wide Defaults template; else the "/" store @@ -1244,7 +1256,7 @@ type SecretScope struct { // fallback path, the same as an environment that declares nothing. func (s *EnvironmentServiceSecrets) RemoteRef(scope SecretScope, key string) EnvironmentSecretRemoteRef { if s != nil { - mapping := s.Services[scope.Service] + mapping := s.MappingFor(scope.Module, scope.Service) if remote, ok := mapping.RemoteKeys[key]; ok { return remote } diff --git a/pkg/environments/environment_test.go b/pkg/environments/environment_test.go index b641f10c..4a504f54 100644 --- a/pkg/environments/environment_test.go +++ b/pkg/environments/environment_test.go @@ -90,10 +90,11 @@ func serviceIdentityDeclaration(serviceName string) string { // service passes the graph check. func TestValidateEnvironmentsAllowsKnownService(t *testing.T) { for name, declaration := range map[string]string{ - "service-secrets": serviceSecretsDeclaration("accounts"), - "service-config": serviceConfigDeclaration("accounts"), - "qualified service-config": serviceConfigDeclaration("saas/accounts"), - "service-identity": serviceIdentityDeclaration("accounts"), + "service-secrets": serviceSecretsDeclaration("accounts"), + "qualified service-secrets": serviceSecretsDeclaration("saas/accounts"), + "service-config": serviceConfigDeclaration("accounts"), + "qualified service-config": serviceConfigDeclaration("saas/accounts"), + "service-identity": serviceIdentityDeclaration("accounts"), } { t.Run(name, func(t *testing.T) { ctx := context.Background() @@ -113,10 +114,11 @@ func TestValidateEnvironmentsAllowsKnownService(t *testing.T) { // projection time; the graph check must reject it. func TestValidateEnvironmentsRejectsUnknownService(t *testing.T) { for name, declaration := range map[string]string{ - "service-secrets": serviceSecretsDeclaration("accunts"), - "service-config": serviceConfigDeclaration("accunts"), - "qualified service-config": serviceConfigDeclaration("wrong/accounts"), - "service-identity": serviceIdentityDeclaration("accunts"), + "service-secrets": serviceSecretsDeclaration("accunts"), + "qualified service-secrets": serviceSecretsDeclaration("wrong/accounts"), + "service-config": serviceConfigDeclaration("accunts"), + "qualified service-config": serviceConfigDeclaration("wrong/accounts"), + "service-identity": serviceIdentityDeclaration("accunts"), } { t.Run(name, func(t *testing.T) { ctx := context.Background() diff --git a/pkg/environments/workspace.go b/pkg/environments/workspace.go index 8e5807bb..7cae37bb 100644 --- a/pkg/environments/workspace.go +++ b/pkg/environments/workspace.go @@ -191,7 +191,7 @@ func validateResourcePathComponent(kind, name string) error { // it earlier. // Loading the graph is why this is a pass separate from postLoad, mirroring // ValidateServiceDependencies. -func ValidateWorkspace(ctx context.Context, workspace *resources.Workspace) error { +func ValidateWorkspace(ctx context.Context, workspace *resources.Workspace, selected ...*Environment) error { environments, err := FromWorkspace(workspace) if err != nil { return err @@ -200,6 +200,17 @@ func ValidateWorkspace(ctx context.Context, workspace *resources.Workspace) erro if nsErr := validateModuleNamespaces(workspace, environments); nsErr != nil { return w.Wrap(nsErr) } + if len(selected) > 0 { + for _, env := range selected { + if env == nil { + return fmt.Errorf("selected environment is nil") + } + if validationErr := env.Validate(); validationErr != nil { + return validationErr + } + } + environments = selected + } for _, env := range environments { if env == nil { continue @@ -235,7 +246,7 @@ func ValidateWorkspace(ctx context.Context, workspace *resources.Workspace) erro // A module-qualified service-identity key names exactly one // service of that module, or nothing (a typo that would leave // the workload on the default identity with no report). - if _, service, qualified := strings.Cut(name, "/"); qualified && (block == "service-identity" || block == "service-mesh" || block == "service-config") { + if _, service, qualified := strings.Cut(name, "/"); qualified { if !slices.Contains(byName[service], name) { return w.Wrap(fmt.Errorf("environment %q %s references unknown service %q", env.Name, block, name)) } diff --git a/pkg/gitops/configuration_projection.go b/pkg/gitops/configuration_projection.go index 97a7b0bb..45e45e97 100644 --- a/pkg/gitops/configuration_projection.go +++ b/pkg/gitops/configuration_projection.go @@ -35,7 +35,7 @@ func projectConfigurationValuesInTree(ctx context.Context, tree *renderRoot, roo values := env.ServiceConfig.ValuesFor(module, service) refs := map[string]environments.EnvironmentSecretRemoteRef{} if env.ServiceSecrets != nil { - refs = env.ServiceSecrets.Services[service].RemoteKeys + refs = env.ServiceSecrets.MappingFor(module, service).RemoteKeys } if len(values) == 0 && len(refs) == 0 { return nil @@ -190,7 +190,7 @@ func validateProjectedConfigurationInTree(tree *renderRoot, root string, service values := env.ServiceConfig.ValuesFor(scope.Module, service.Name) refs := map[string]environments.EnvironmentSecretRemoteRef{} if env.ServiceSecrets != nil { - refs = env.ServiceSecrets.Services[service.Name].RemoteKeys + refs = env.ServiceSecrets.MappingFor(scope.Module, service.Name).RemoteKeys } identity, err := soleWorkloadIdentity(scope.Module, service.Name, consumedManagedServices(service, scope.Module, env), env) if err != nil { diff --git a/pkg/gitops/configuration_projection_test.go b/pkg/gitops/configuration_projection_test.go index 2baab6e8..58eae6d5 100644 --- a/pkg/gitops/configuration_projection_test.go +++ b/pkg/gitops/configuration_projection_test.go @@ -439,3 +439,46 @@ func TestRejectedConfigurationCannotReplacePublishedTree(t *testing.T) { require.NoError(t, err) require.Len(t, entries, 1) } + +func TestQualifiedSecretsReachOnlyTheirModule(t *testing.T) { + env := &environments.Environment{Name: "blue", Namespace: "product", ServiceSecrets: &environments.EnvironmentServiceSecrets{ + SecretStore: environments.EnvironmentSecretStoreReference{Name: "default-store", Kind: "ClusterSecretStore"}, + Services: map[string]environments.EnvironmentServiceSecretMapping{ + "saas/accounts": { + SecretStore: &environments.EnvironmentSecretStoreReference{Name: "accounts-store", Kind: "ClusterSecretStore"}, + RefreshInterval: "2m", + RemoteKeys: map[string]environments.EnvironmentSecretRemoteRef{"SIGNING_KEY": {Key: "accounts-signing", Property: "wrapped"}}, + }, + }, + }} + for _, module := range []string{"saas", "other"} { + t.Run(module, func(t *testing.T) { + root := t.TempDir() + writeConsumerTree(t, root, env.Name, env.Namespace, "accounts", "declared.example") + scope := scopeOf(env) + scope.Module = module + require.NoError(t, projectServiceConfiguration(t.Context(), testRenderRoot(t, root), root, &resources.Service{Name: "accounts"}, env, scope, serviceInjection{})) + rendered := buildOverlay(t, root, env.Name) + values := containerEnvironment(t, rendered) + if module == "other" { + require.NotContains(t, values, "SIGNING_KEY") + return + } + require.Equal(t, map[string]any{"secretKeyRef": map[string]any{"name": "secret-accounts", "key": "SIGNING_KEY"}}, values["SIGNING_KEY"]["valueFrom"]) + found := false + for _, doc := range rendered { + if doc.kind != "ExternalSecret" { + continue + } + spec := doc.value["spec"].(map[string]any) + require.Equal(t, "accounts-store", spec["secretStoreRef"].(map[string]any)["name"]) + require.Equal(t, "2m", spec["refreshInterval"]) + found = true + } + require.True(t, found) + remote := env.ServiceSecrets.RemoteRef(environments.SecretScope{Module: module, Service: "accounts"}, "SIGNING_KEY") + require.Equal(t, "accounts-signing", remote.Key) + require.Equal(t, "wrapped", remote.Property) + }) + } +} diff --git a/pkg/gitops/configuration_templates.go b/pkg/gitops/configuration_templates.go index cd79daa6..feb3a005 100644 --- a/pkg/gitops/configuration_templates.go +++ b/pkg/gitops/configuration_templates.go @@ -196,7 +196,7 @@ func producerPrimitiveRemote( // The producer's own projection takes its per-service override the same // way (serviceSecretProjection), so this is the store its ExternalSecret // actually reads from. - if override := secrets.Services[service].SecretStore; override != nil { + if override := secrets.MappingFor(module, service).SecretStore; override != nil { store = *override } } diff --git a/pkg/gitops/dev.go b/pkg/gitops/dev.go index 1d0416a0..d9dc6402 100644 --- a/pkg/gitops/dev.go +++ b/pkg/gitops/dev.go @@ -134,7 +134,7 @@ func buildRenderedServiceImages( rebuild bool, sink orchestration.OutputSink, ) ([]string, error) { - if err := environments.ValidateWorkspace(ctx, workspace); err != nil { + if err := environments.ValidateWorkspace(ctx, workspace, env); err != nil { return nil, err } if err := prepareSnapshotRegistry(ctx, env); err != nil { diff --git a/pkg/gitops/frontend_payload.go b/pkg/gitops/frontend_payload.go index ba1c7eaf..f1ea2536 100644 --- a/pkg/gitops/frontend_payload.go +++ b/pkg/gitops/frontend_payload.go @@ -11,6 +11,7 @@ import ( clicomposition "github.com/codefly-dev/cli/pkg/composition" "github.com/codefly-dev/cli/pkg/environments" + "github.com/codefly-dev/cli/pkg/orchestration" "github.com/codefly-dev/cli/pkg/solutionrun" "github.com/codefly-dev/core/composition" basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" @@ -109,7 +110,13 @@ func artifactCacheInjection(service *resources.Service, env *environments.Enviro // buildFrontendBundle and publishRuntimeArtifact are the build and the push a // test stands in for, the way stubBuild stands in for an image build. var ( - buildFrontendBundle = solutionrun.BuildFrontendBundle + buildFrontendBundle = func(ctx context.Context, module *resources.Module, env *environments.Environment, destination string) error { + builder, err := orchestration.EnsureEnvironmentBuilder(ctx, env) + if err != nil { + return err + } + return solutionrun.BuildFrontendBundleWithBuilder(ctx, module, destination, builder) + } publishRuntimeArtifact = clicomposition.PublishRuntimeArtifact ) @@ -141,7 +148,7 @@ func publishFrontendBundle(ctx context.Context, workspace *resources.Workspace, } defer func() { _ = os.RemoveAll(scratch) }() bundle := filepath.Join(scratch, "bundle") - if err = buildFrontendBundle(ctx, module, bundle); err != nil { + if err = buildFrontendBundle(ctx, module, env, bundle); err != nil { return nil, err } archive, _, err := composition.CanonicalArchive(bundle) diff --git a/pkg/gitops/frontend_publish_test.go b/pkg/gitops/frontend_publish_test.go index 440d1fc8..17a2d9bb 100644 --- a/pkg/gitops/frontend_publish_test.go +++ b/pkg/gitops/frontend_publish_test.go @@ -24,7 +24,7 @@ func stubFrontendPublication(t *testing.T) (*int, *string) { builds, published := 0, "" previousBuild, previousPublish := buildFrontendBundle, publishRuntimeArtifact t.Cleanup(func() { buildFrontendBundle, publishRuntimeArtifact = previousBuild, previousPublish }) - buildFrontendBundle = func(_ context.Context, _ *resources.Module, destination string) error { + buildFrontendBundle = func(_ context.Context, _ *resources.Module, _ *environments.Environment, destination string) error { builds++ require.NoError(t, os.MkdirAll(destination, 0o755)) return os.WriteFile(filepath.Join(destination, solutionrun.FrontendBundleManifest), []byte("{}"), 0o600) diff --git a/pkg/gitops/orchestrate.go b/pkg/gitops/orchestrate.go index 45ae2375..1b99a542 100644 --- a/pkg/gitops/orchestrate.go +++ b/pkg/gitops/orchestrate.go @@ -46,7 +46,7 @@ func renderModuleTree( if err := selectionguard.RejectUnboundExecution(workspace.Dir(), module.Dir()); err != nil { return RenderResult{}, err } - if err := environments.ValidateWorkspace(ctx, workspace); err != nil { + if err := environments.ValidateWorkspace(ctx, workspace, env); err != nil { return RenderResult{}, err } if env.WithdrawsModule(module) { @@ -618,7 +618,7 @@ func RenderService(ctx context.Context, workspace *resources.Workspace, module * } func renderService(ctx context.Context, workspace *resources.Workspace, module *resources.Module, service *resources.Service, env *environments.Environment, project string, build renderBuild, sink orchestration.OutputSink) (RenderResult, error) { - if err := environments.ValidateWorkspace(ctx, workspace); err != nil { + if err := environments.ValidateWorkspace(ctx, workspace, env); err != nil { return RenderResult{}, err } if err := CheckCompositionPermissions(ctx, workspace, env, true, module.Name); err != nil { diff --git a/pkg/gitops/secret_projection.go b/pkg/gitops/secret_projection.go index 7865995e..c7d8977c 100644 --- a/pkg/gitops/secret_projection.go +++ b/pkg/gitops/secret_projection.go @@ -140,7 +140,7 @@ func serviceSecretProjection(scope unitScope, service string, secrets *environme return nil, nil } namespace := scope.Namespace - mapping := secrets.Services[service] + mapping := secrets.MappingFor(scope.Module, service) // A service may resolve from a different store than the environment default, // mirroring EnvironmentManagedSecretReference's per-reference store; without // this the per-service secret-store declaration would load, validate, and then diff --git a/pkg/gitops/secret_projection_test.go b/pkg/gitops/secret_projection_test.go index 0aba59b3..abe24cd2 100644 --- a/pkg/gitops/secret_projection_test.go +++ b/pkg/gitops/secret_projection_test.go @@ -248,6 +248,35 @@ func TestServiceSecretProjectionDefaultsAndOverridesRemoteKeys(t *testing.T) { } } +func TestServiceSecretProjectionQualifiedMappingDoesNotLeakAcrossModules(t *testing.T) { + secrets := azureServiceSecrets() + secrets.Services["saas/accounts"] = environments.EnvironmentServiceSecretMapping{ + SecretStore: &environments.EnvironmentSecretStoreReference{Name: "signing", Kind: "ClusterSecretStore"}, + RemoteKeys: map[string]environments.EnvironmentSecretRemoteRef{ + "workos-client-secret": {Key: "accounts-signing", Property: "wrapped"}, + }, + Defaults: &environments.EnvironmentSecretRemoteRef{Key: "{module}-{service}", Property: "{key}"}, + RefreshInterval: "5m", + } + for _, module := range []string{"saas", "other", "saas"} { + projection, err := serviceSecretProjection(unitScope{Namespace: "product-" + module, Module: module}, + "accounts", secrets, []string{"workos-api-key", "workos-client-secret"}) + if err != nil { + t.Fatal(err) + } + if module == "saas" { + if projection.Spec.SecretStoreRef.Name != "signing" || projection.Spec.RefreshInterval != "5m" || + projection.Spec.Data[0].RemoteRef.Key != "saas-accounts" || + projection.Spec.Data[1].RemoteRef != (externalSecretRemote{Key: "accounts-signing", Property: "wrapped"}) { + t.Fatalf("qualified mapping lost: %+v", projection.Spec) + } + } else if projection.Spec.SecretStoreRef.Name != "azure-keyvault-prod" || + projection.Spec.Data[1].RemoteRef.Key != "workos/prod/client-secret" { + t.Fatalf("qualified mapping leaked to another module: %+v", projection.Spec) + } + } +} + func TestServiceSecretProjectionRendersNothingWithoutStoreOrKeys(t *testing.T) { if projection, err := serviceSecretProjection(unitScope{Namespace: "payments"}, "accounts", nil, []string{"api-key"}); err != nil || projection != nil { t.Fatalf("nil store rendered %v (err %v)", projection, err) diff --git a/pkg/gitops/solution.go b/pkg/gitops/solution.go index ad7b3e72..ab82414b 100644 --- a/pkg/gitops/solution.go +++ b/pkg/gitops/solution.go @@ -59,7 +59,7 @@ func RenderSolution(ctx context.Context, req *SolutionRenderRequest) (RenderResu if err := selectionguard.RejectUnboundExecution(req.Workspace.Dir(), req.Source); err != nil { return RenderResult{}, err } - if err := environments.ValidateWorkspace(ctx, req.Workspace); err != nil { + if err := environments.ValidateWorkspace(ctx, req.Workspace, req.Environment); err != nil { return RenderResult{}, err } env := req.Environment diff --git a/pkg/orchestration/build_plan.go b/pkg/orchestration/build_plan.go index 7b1665d9..2b27517a 100644 --- a/pkg/orchestration/build_plan.go +++ b/pkg/orchestration/build_plan.go @@ -564,12 +564,10 @@ func (b *Builder) buildRecipe( // CLI's builder, provisioned by dockerBuildContext before the agent's Build // RPC, so it is named rather than provisioned again here. // -// Beyond a multi-platform build, which the default "docker" driver cannot -// produce at all, a declared registry mirror also requires the CLI's own -// builder whatever the platforms: the default builder takes no buildkitd -// configuration, so a single-platform build left on it would read its base from -// the registry the recipe names and the mirror would reach the resolution -// alone. +// Hosted image builds use the same host-networked builder as mirrored builds: +// even a single-platform image without mirrors must resolve package repositories +// through the operator's network. Local builds without a registry keep Docker's +// default builder. func (b *Builder) selectImageBuilder(ctx context.Context, multiArch bool, mirrors map[string]string) (string, error) { if name := b.world.BuildxBuilder; name != "" { if len(mirrors) > 0 { @@ -590,7 +588,8 @@ func (b *Builder) selectImageBuilder(ctx context.Context, multiArch bool, mirror } return buildxBuilderFor(mirrors), nil } - if !multiArch && len(mirrors) == 0 { + hostedRegistry := b.world.Env != nil && b.world.Env.Registry != nil + if !multiArch && len(mirrors) == 0 && !hostedRegistry { return "", nil } return ensureBuildxBuilder(ctx, mirrors) diff --git a/pkg/orchestration/image_cache.go b/pkg/orchestration/image_cache.go index 5e0ba2fa..b4861996 100644 --- a/pkg/orchestration/image_cache.go +++ b/pkg/orchestration/image_cache.go @@ -726,17 +726,15 @@ func hashContextEntry(hasher hash.Hash, relative, path string, entry fs.DirEntry writeImageCacheRecord(hasher, "symlink", relative, filepath.ToSlash(target)) return nil case info.IsDir(): - writeImageCacheRecord(hasher, "directory", relative) + writeImageCacheRecord(hasher, "directory", relative, strconv.FormatUint(uint64(info.Mode().Perm()), 8)) return nil case !info.Mode().IsRegular(): writeImageCacheRecord(hasher, "special", relative, info.Mode().String()) return nil } - executable := "0" - if info.Mode().Perm()&0o111 != 0 { - executable = "1" - } - writeImageCacheRecord(hasher, "file", relative, executable, strconv.FormatInt(info.Size(), 10)) + // COPY preserves read and traversal permissions too: identical bytes can + // still be unreadable to the image's non-root user. + writeImageCacheRecord(hasher, "file", relative, strconv.FormatUint(uint64(info.Mode().Perm()), 8), strconv.FormatInt(info.Size(), 10)) file, err := os.Open(path) if err != nil { return err diff --git a/pkg/orchestration/image_cache_test.go b/pkg/orchestration/image_cache_test.go index 41f1d22d..268eab7c 100644 --- a/pkg/orchestration/image_cache_test.go +++ b/pkg/orchestration/image_cache_test.go @@ -370,6 +370,12 @@ func TestEveryContextChangeMovesTheImageIdentity(t *testing.T) { "mode changed": func(t *testing.T, harness *imageCacheHarness) { require.NoError(t, os.Chmod(filepath.Join(harness.serviceDir, "code", "main.go"), 0o755)) }, + "file readability changed": func(t *testing.T, harness *imageCacheHarness) { + require.NoError(t, os.Chmod(filepath.Join(harness.serviceDir, "code", "main.go"), 0o600)) + }, + "directory traversal changed": func(t *testing.T, harness *imageCacheHarness) { + require.NoError(t, os.Chmod(filepath.Join(harness.serviceDir, "code"), 0o700)) + }, "file replaced by a symlink": func(t *testing.T, harness *imageCacheHarness) { path := filepath.Join(harness.serviceDir, "code", "main.go") require.NoError(t, os.Remove(path)) diff --git a/pkg/orchestration/registry_mirror.go b/pkg/orchestration/registry_mirror.go index 7dbc394b..f440492c 100644 --- a/pkg/orchestration/registry_mirror.go +++ b/pkg/orchestration/registry_mirror.go @@ -1,6 +1,7 @@ package orchestration import ( + "context" "crypto/sha256" "encoding/hex" "fmt" @@ -48,9 +49,13 @@ func registryMirrors(env *environments.Environment) map[string]string { return env.Registry.Mirrors } -// mirroredReference is the reference to read `reference` from. It is -// environments.MirroredReference, the one spelling a cell's pull repository -// (EnvironmentRegistry.PullRepository) is mirrored with too. +// EnsureEnvironmentBuilder reuses the environment's registry configuration and +// host-networked builder for auxiliary builds such as frontend bundles. +func EnsureEnvironmentBuilder(ctx context.Context, env *environments.Environment) (string, error) { + return ensureBuildxBuilder(ctx, registryMirrors(env)) +} + +// mirroredReference uses the environment's declared read repository. func mirroredReference(ref string, mirrors map[string]string) string { return environments.MirroredReference(ref, mirrors) } diff --git a/pkg/solutionrun/frontend_build.go b/pkg/solutionrun/frontend_build.go index b958a8ad..19656751 100644 --- a/pkg/solutionrun/frontend_build.go +++ b/pkg/solutionrun/frontend_build.go @@ -22,6 +22,8 @@ import ( // mirror serves it like any other base. const FrontendBuilderImage = "docker.io/library/node:24-bookworm-slim@sha256:a5f4366e834695d5f363b920c104953a535d3a9864d1bdf74566a0ebe01daee5" +const frontendBuildCommand = "build" + // FrontendProject is the npm project a component's frontend build runs in, // slash-separated and relative to the module: the remote's own directory // (fe-remote beside the services) when it holds a package.json, else the @@ -111,7 +113,16 @@ func BuildFrontendBundle(ctx context.Context, module *resources.Module, destinat return buildFrontendBundle(ctx, module, destination, frontendBuildContext) } -func buildFrontendBundle(ctx context.Context, module *resources.Module, destination string, stage func(context.Context, *resources.Module, string) (string, string, error)) error { +// BuildFrontendBundleWithBuilder uses the deployment environment's builder. +func BuildFrontendBundleWithBuilder(ctx context.Context, module *resources.Module, destination, builder string) error { + return buildFrontendBundle(ctx, module, destination, frontendBuildContext, builder) +} + +func buildFrontendBundle(ctx context.Context, module *resources.Module, destination string, stage func(context.Context, *resources.Module, string) (string, string, error), builders ...string) error { + builder := "" + if len(builders) > 0 { + builder = builders[0] + } project, err := FrontendProject(module) if err != nil { return err @@ -140,7 +151,15 @@ func buildFrontendBundle(ctx context.Context, module *resources.Module, destinat dockerfile := fmt.Sprintf("FROM %s AS build\nWORKDIR /source\nCOPY [\".\", \".\"]\nWORKDIR %s\nRUN [\"sh\", \"-c\", \"npm ci && npm run build\"]\nFROM scratch\nCOPY --from=build [\"%s/\", \"/\"]\n", FrontendBuilderImage, workdir, bundle) //nolint:gosec // G204: fixed docker executable, separate arguments, no shell; the Dockerfile arrives on stdin. - command := exec.CommandContext(ctx, "docker", "build", "--progress=plain", "--file", "-", "--output", "type=local,dest="+destination, root) + args := make([]string, 0, 10) + if builder != "" { + args = append(args, "buildx", frontendBuildCommand, "--builder", builder) + } else { + args = append(args, frontendBuildCommand) + } + args = append(args, "--progress=plain", "--file", "-", "--output", "type=local,dest="+destination, root) + //nolint:gosec // G204: fixed docker executable, separate arguments, no shell. + command := exec.CommandContext(ctx, "docker", args...) command.Stdin = strings.NewReader(dockerfile) var output bytes.Buffer command.Stdout, command.Stderr = &output, &output diff --git a/pkg/solutionrun/frontend_local.go b/pkg/solutionrun/frontend_local.go index 6a3676ad..554384a2 100644 --- a/pkg/solutionrun/frontend_local.go +++ b/pkg/solutionrun/frontend_local.go @@ -580,7 +580,7 @@ func (s *frontendSources) buildNatively(ctx context.Context, module *resources.M if _, err := os.Stat(filepath.Join(projectDir, "node_modules")); err != nil { install = true } - steps := [][]string{{"npm", "run", "build"}} + steps := [][]string{{"npm", "run", frontendBuildCommand}} if install { steps = append([][]string{{"npm", "ci"}}, steps...) }