diff --git a/cmd/generate/runnable_bindings.go b/cmd/generate/runnable_bindings.go index b3655ff4..9759b8d4 100644 --- a/cmd/generate/runnable_bindings.go +++ b/cmd/generate/runnable_bindings.go @@ -102,8 +102,9 @@ owner must still serve it. In workspace.codefly.yaml, environments[].runnable-scope-selections maps each binding key (MODULE__OPERATION) to Core ScopeSelection messages: slot, invoke, and lookup. Every required slot needs exact resource kinds, actions and ids; -missing selections, wildcards and overlapping kinds are refused. Tool exposure -is carried from the owner. See docs/runnable.md for the selection YAML. +missing selections, wildcards and overlapping kinds are refused. Every +operation is a tool; its presentation and effect are carried from the derived +policy. See docs/runnable.md for the selection YAML. An environment that installs an operation for no resource answers its key with an explicitly empty list, KEY: []. That operation is declined: its key is absent @@ -528,6 +529,15 @@ func operationSpec(declared *runnablespkg.Operation, spelling string, codes core return nil, fmt.Errorf("completion %q is not a completion mode; %s and %s are the two", declared.Completion, basev0.RunnableExecution_COMPLETION_CALL, basev0.RunnableExecution_COMPLETION_SUBMIT) } + var recovery basev0.RunnableExecution_Recovery + if declared.Recovery != "" { + value, known := basev0.RunnableExecution_Recovery_value[declared.Recovery] + if !known || basev0.RunnableExecution_Recovery(value) == basev0.RunnableExecution_RECOVERY_UNKNOWN { + return nil, fmt.Errorf("recovery %q is not a recovery; %s and %s are the two", + declared.Recovery, basev0.RunnableExecution_RECOVERY_RECOMPUTE, basev0.RunnableExecution_RECOVERY_RECEIPT) + } + recovery = basev0.RunnableExecution_Recovery(value) + } return &corerunnable.OperationSpec{ Method: spelling, Codes: codes, @@ -545,6 +555,7 @@ func operationSpec(declared *runnablespkg.Operation, spelling string, codes core Completion: basev0.RunnableExecution_Completion(completion), Tool: declared.Tool, RequiredScopeSlots: declared.RequiredScopeSlots, + Recovery: recovery, }, nil } diff --git a/cmd/generate/runnable_scopes_test.go b/cmd/generate/runnable_scopes_test.go index 0e28ba1a..f1ed17b6 100644 --- a/cmd/generate/runnable_scopes_test.go +++ b/cmd/generate/runnable_scopes_test.go @@ -27,6 +27,7 @@ func slotOperation(fixed bool) *runnablev0.Operation { } declared.RequiredScopeSlots = []*runnablev0.ScopeSlot{{Name: "source", RequiredActions: []string{"invoke", "read"}, Lookup: true}} declared.Tool = &runnablev0.ToolExposure{Name: "invoke_source", Description: "Invoke a selected source.", Effect: runnablev0.ToolExposure_EFFECT_MUTATION} + declared.Recovery = basev0.RunnableExecution_RECOVERY_RECEIPT return declared } @@ -75,6 +76,16 @@ func TestOperationDocumentRoundTripsEveryCoreField(t *testing.T) { require.Equal(t, coreFields.Len()+1, reflect.TypeFor[runnablespkg.Operation]().NumField()) } +// undeclarableFields are Core fields an operation may never declare: Core +// refuses the policy that sets one, so the document has nothing to carry and +// the fixture cannot populate it. Each is earned by a test proving the refusal +// reaches this command, never by being hard to populate. +var undeclarableFields = map[protoreflect.FullName]string{ + // An owner is the authority's to select for an installation (core#845); + // TestRunnablesRefusesADeclaredResourceOwner proves the refusal. + "codefly.base.v0.WorkScopeV1.resource_owner_id": "never declared", +} + // Descend into every message, including repeated scopes and slots. A future // nested field must be populated before proto.Equal can prove it survived. func requireEveryMessageFieldPopulated(t *testing.T, message protoreflect.Message) { @@ -82,6 +93,10 @@ func requireEveryMessageFieldPopulated(t *testing.T, message protoreflect.Messag fields := message.Descriptor().Fields() for i := 0; i < fields.Len(); i++ { field := fields.Get(i) + if _, undeclarable := undeclarableFields[field.FullName()]; undeclarable { + require.Falsef(t, message.Has(field), "%s is never declared; leave it out of the fixture", field.FullName()) + continue + } require.Truef(t, message.Has(field), "populate %s in the round-trip fixture", field.FullName()) if field.Message() == nil { continue @@ -102,6 +117,20 @@ func requireEveryMessageFieldPopulated(t *testing.T, message protoreflect.Messag } } +// A policy that declares a field Core never lets an operation declare is +// refused by the derivation with Core's reason, so the document's silence on +// that field is the contract rather than a loss. +func TestRunnablesRefusesADeclaredResourceOwner(t *testing.T) { + declared := slotOperation(true) + declared.InvokeScopes[0].ResourceOwnerId = "principal-u" + root, _ := saveConnectRunnableFixture(t, context.Background(), descriptorSet(t, ingestionFile(declared)), "0.1.0") + t.Chdir(root) + resetRunnablesFlags(t) + err := RunnablesCmd.RunE(RunnablesCmd, []string{"documents"}) + require.ErrorContains(t, err, `selects resource owner "principal-u"`) + require.ErrorContains(t, err, undeclarableFields["codefly.base.v0.WorkScopeV1.resource_owner_id"]) +} + func TestOperationDocumentRefusesUnknownNestedCoreFields(t *testing.T) { for _, data := range []string{ `{"tool":{"name":"invoke_source","new_authority":true}}`, @@ -147,11 +176,11 @@ func verifyResolvedBinding(t *testing.T, value string, tool *runnablev0.ToolExpo require.NoError(t, err) require.NoError(t, corerunnable.VerifyPrepared(prepared)) require.Empty(t, prepared.Policy.RequiredScopeSlots) + // Every runnable is a tool: one whose policy names none is presented + // from its operation. exposure, err := corerunnable.ToolFromPrepared(prepared) - if tool == nil { - require.ErrorIs(t, err, corerunnable.ErrNotATool) - } else { - require.NoError(t, err) + require.NoError(t, err) + if tool != nil { require.True(t, proto.Equal(tool, exposure)) } // The host receipt and the caller binding consume the SAME resolution. @@ -200,9 +229,12 @@ func TestGenerateRunnableBindingsResolvesThreeOperationFixtures(t *testing.T) { switch { case strings.HasSuffix(operation.Entry.Method, "/FixedOnly"): // The old path wrote the declared policy directly. Its canonical - // output stays byte-identical, not just semantically equivalent. + // output stays byte-identical, not just semantically equivalent, + // beside the tool presentation every runnable now carries. + require.NotNil(t, prepared.Policy.Tool) baseline := proto.CloneOf(prepared) - baseline.Policy = fixed + baseline.Policy = proto.CloneOf(fixed) + baseline.Policy.Tool = prepared.Policy.Tool before, err := corerunnable.EncodePrepared(baseline) require.NoError(t, err) require.Equal(t, string(before), values[key]) diff --git a/cmd/generate/runnable_source_operation_test.go b/cmd/generate/runnable_source_operation_test.go index 3fb999f4..56543c45 100644 --- a/cmd/generate/runnable_source_operation_test.go +++ b/cmd/generate/runnable_source_operation_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "github.com/codefly-dev/cli/pkg/environments" @@ -104,7 +105,13 @@ func TestGenerateRealSourceOperations(t *testing.T) { key := RunnableBindingKey("saas-starter", operation.Entry.Name) value := values[key] prepared := verifyResolvedBinding(t, value, nil) - require.True(t, proto.Equal(published[operation.Entry.Method].Contract, prepared.Contract), "prepared contract matches the producer's committed runnable package") + // The producer's committed package predates the wire spelling (core + // #843: JSON names, 64-bit integers as strings); module-saas-starter + // regenerates to it. Until then the two agree field for field once the + // committed one is re-spelled, which is what this holds. + require.True(t, proto.Equal(operation.Package.GetContract(), prepared.Contract), "prepared contract is the derived one") + requireSameModuloWireSpelling(t, published[operation.Entry.Method].GetContract().GetInput().GetFields(), prepared.Contract.GetInput().GetFields()) + requireSameModuloWireSpelling(t, published[operation.Entry.Method].GetContract().GetOutput().GetFields(), prepared.Contract.GetOutput().GetFields()) require.Equal(t, "connect", prepared.Operation.Endpoint) require.Len(t, prepared.Policy.InvokeScopes, 1) require.True(t, proto.Equal(sourceSelection().Invoke[0], prepared.Policy.InvokeScopes[0])) @@ -115,3 +122,42 @@ func TestGenerateRealSourceOperations(t *testing.T) { require.NoError(t, RunnablesCmd.RunE(RunnablesCmd, []string{"saas-starter"})) require.NoError(t, RunnableBindingsCmd.RunE(RunnableBindingsCmd, nil)) } + +// requireSameModuloWireSpelling holds a contract committed before the wire +// spelling to the one derived now: the same members in the same order, each +// named by its JSON name, a 64-bit integer as a STRING of format int64 or +// uint64 and nothing else, every non-required key optional, array items held +// the same way. A stopgap: it goes when module-saas-starter regenerates. +func requireSameModuloWireSpelling(t *testing.T, committed, derived []*basev0.RunnableField) { + t.Helper() + require.Len(t, derived, len(committed)) + for i, before := range committed { + requireFieldModuloWireSpelling(t, before, derived[i], true) + } +} + +func requireFieldModuloWireSpelling(t *testing.T, before, after *basev0.RunnableField, member bool) { + t.Helper() + if member { + parts := strings.Split(before.GetName(), "_") + for j := 1; j < len(parts); j++ { + if parts[j] != "" { + parts[j] = strings.ToUpper(parts[j][:1]) + parts[j][1:] + } + } + require.Equal(t, strings.Join(parts, ""), after.GetName()) + require.True(t, after.GetOptional(), "%s: proto3 JSON may omit every non-required key", after.GetName()) + } + if before.GetType() == after.GetType() { + require.Equal(t, before.GetFormat(), after.GetFormat(), "%s", after.GetName()) + } else { + require.Equal(t, basev0.RunnableField_INTEGER, before.GetType(), "%s", before.GetName()) + require.Equal(t, basev0.RunnableField_STRING, after.GetType(), "%s", before.GetName()) + require.Contains(t, []string{corerunnable.FormatInt64, corerunnable.FormatUint64}, after.GetFormat(), "%s", before.GetName()) + } + requireSameModuloWireSpelling(t, before.GetFields(), after.GetFields()) + require.Equal(t, before.GetItems() == nil, after.GetItems() == nil, "%s items", after.GetName()) + if before.GetItems() != nil { + requireFieldModuloWireSpelling(t, before.GetItems(), after.GetItems(), false) + } +} diff --git a/cmd/generate/runnables.go b/cmd/generate/runnables.go index 07ace16f..6eeb40c5 100644 --- a/cmd/generate/runnables.go +++ b/cmd/generate/runnables.go @@ -16,6 +16,7 @@ import ( runnablespkg "github.com/codefly-dev/cli/pkg/runnables" "github.com/codefly-dev/core/composition" basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + runnablev0 "github.com/codefly-dev/core/generated/go/codefly/runnable/v0" "github.com/codefly-dev/core/resources" corerunnable "github.com/codefly-dev/core/runnable" "github.com/codefly-dev/core/shared" @@ -35,19 +36,23 @@ var ( // published contracts mark as an operation. var RunnablesCmd = &cobra.Command{ Use: "runnables [module]", - Short: "Derive SERVICE-facility Runnable packages from the methods a module's contracts mark", + Short: "Derive SERVICE-facility Runnable packages, and so tools, from a module's served methods", Long: `Derive a Runnable package for every gRPC method a module's published contracts -mark with the codefly.runnable.v0.operation option. +serve behind the authority hop. Every Runnable is a tool. -A unary, idempotent method an owner service already publishes becomes a -Runnable by derivation, not by authoring: the method option says which methods -are operations and under what execution policy, and the message descriptors say -what the contract is. Nothing is written twice. +A method that declares its authority policy (authority.hop.v1.policy) is an +operation with nothing added: a policy requiring only "read" derives a +read-only tool that is safe to re-run and carries no receipt; anything else +derives a mutation resolved by its effect receipt through the SDK's generic +lookup. The codefly.runnable.v0.operation option overrides those defaults, and +codefly.runnable.v0.exclude keeps a method from every agent. The message +descriptors say what the contract is, by their proto3 JSON. Nothing is written +twice. The input is what ` + "`codefly generate contracts`" + ` already wrote — each endpoint's served procedures in the API catalog and its serialized FileDescriptorSet. -Run that first. Only served methods carrying the option are derived; their -paired Lookup must be served on the same endpoint. No flag names a method. +Run that first. Only served methods are derived; a declared Lookup must be +served on the same endpoint. No flag names a method. For each marked method this writes, under contracts/runnables: @@ -55,12 +60,13 @@ For each marked method this writes, under contracts/runnables: ///operation.json its execution policy and authority index.json one row per operation -The tree is fully owned: a method that no longer carries the option loses its -directory. A module with no marked method writes an empty index, not an error. +The tree is fully owned: a method that is no longer an operation loses its +directory. A module with no operation writes an empty index, not an error. -A streaming method, a payload outside the bounded schema profile, or an option -core refuses is named and skipped, and the command exits non-zero after -reporting every one of them. +A method derived by default that cannot be one — a stream, a policy requiring +nothing, a payload proto3 JSON cannot carry — is listed and left a plain call. +A method carrying the option that core refuses is named and skipped, and the +command exits non-zero after reporting every one of them. --check is the CI drift gate: it regenerates into a temporary directory and prints a unified diff of what differs, without writing anything. @@ -118,9 +124,7 @@ Examples: if err = writeDerivedTree(ctx, derived.files, output); err != nil { return err } - for _, entry := range derived.index.Operations { - cli.Info("runnables: %s %s %s", entry.Name, entry.Method, entry.Digest) - } + derived.report() cli.Header(1, "Derived %d runnable operation(s)", len(derived.index.Operations)) return nil }, @@ -144,6 +148,11 @@ type derivation struct { // give one release a different digest per machine. workspaceDir string refusals []string + // notices are methods a default derivation left plain calls, and why. + notices []string + // origins says, per operation name, whether the owner declared it and + // what its tool's effect is. + origins map[string]derivedOperation } // refusal reports every method that could not be derived. They are collected @@ -153,10 +162,56 @@ func (d *derivation) refusal() error { if len(d.refusals) == 0 { return nil } - return fmt.Errorf("%d method(s) carry the operation option but cannot be derived:\n %s", + return fmt.Errorf("%d method(s) cannot be derived as runnable operations:\n %s", len(d.refusals), strings.Join(d.refusals, "\n ")) } +// derivedOperation is what the report says of one operation: where it came +// from and what it does. +type derivedOperation struct { + explicit bool + effect string +} + +// report says what the run made of the module, so an owner learns which of +// its methods became tools without reading the tree: each operation with its +// origin and effect, the methods left plain calls and why, and — when any +// operation was derived by default — how to keep a method from every agent. +// +// One method published on several endpoints (gRPC and Connect) is several +// operations but one method, so the summary counts and names methods, each +// once, and a notice is printed once per method. +func (d *derivation) report() { + var defaults, mutations []string + for i := range d.index.Operations { + entry := &d.index.Operations[i] + operation := d.origins[entry.Name] + origin := "option" + if !operation.explicit { + origin = "default" + defaults = appendOnce(defaults, entry.Method) + if operation.effect == runnablev0.ToolExposure_EFFECT_MUTATION.String() { + mutations = appendOnce(mutations, entry.Method) + } + } + cli.Info("runnables: %s %s %s %s %s", entry.Name, entry.Method, origin, strings.TrimPrefix(operation.effect, "EFFECT_"), entry.Digest) + } + var notices []string + for _, notice := range d.notices { + notices = appendOnce(notices, notice) + } + for _, notice := range notices { + cli.Info("runnables: not an operation: %s", notice) + } + if len(defaults) == 0 { + return + } + cli.Info("runnables: %d method(s) derived by default from their authority policy, %d of them mutations: %s", + len(defaults), len(mutations), strings.Join(mutations, ", ")) + cli.Info("runnables: every operation is a tool, offered only to callers whose grants cover its binding; " + + "a method no agent may ever call declares `option (codefly.runnable.v0.exclude) = true;`") +} + // deriveRunnables walks every protobuf contract the module published and // derives the operations its methods declare. func deriveRunnables(ctx context.Context, workspace *resources.Workspace, module *resources.Module) (*derivation, error) { @@ -186,7 +241,8 @@ func deriveRunnablesIn(ctx context.Context, workspaceName, workspaceDir string, Module: module.Name, Operations: []runnablespkg.IndexEntry{}, }, - files: map[string][]byte{}, + files: map[string][]byte{}, + origins: map[string]derivedOperation{}, } var endpoints []composition.APIContractEndpoint @@ -311,6 +367,10 @@ func deriveMethod(module *resources.Module, endpoint *composition.APIContractEnd if errors.Is(err, corerunnable.ErrNotAnOperation) { return nil } + if errors.Is(err, corerunnable.ErrNotDerivable) { + derived.notices = append(derived.notices, fmt.Sprintf("%s: %v", fullMethod, err)) + return nil + } derived.refusals = append(derived.refusals, fmt.Sprintf("%s: %v", fullMethod, err)) return nil } @@ -333,6 +393,7 @@ func deriveMethod(module *resources.Module, endpoint *composition.APIContractEnd derived.files[filepath.ToSlash(filepath.Join(dir, runnablespkg.OperationFileName))] = operationBytes operation := pkg.GetServiceOperations()[0] + derived.origins[pkg.GetIdentity().GetName()] = derivedOperation{explicit: spec.Declared, effect: spec.Tool.GetEffect().String()} derived.index.Operations = append(derived.index.Operations, runnablespkg.IndexEntry{ Name: pkg.GetIdentity().GetName(), Version: pkg.GetIdentity().GetVersion(), @@ -427,10 +488,26 @@ func derivedVersion(release string, endpoint *composition.APIContractEndpoint) s // because one proto service may be published on both a grpc and a connect // endpoint: those are two operations reached two ways, and a name that dropped // the endpoint would give them one identity and two digests. +// +// A release name is at most maxDerivedName characters. A longer one is +// shortened deterministically — a stable prefix and a short digest of the +// whole name — so a long method name never fails generation and two long +// names never collapse into one. func derivedName(service, endpoint, method string) string { - return strings.Join([]string{service, endpoint, kebab(method)}, "-") + return corerunnable.BoundedName(strings.Join([]string{service, endpoint, kebab(method)}, "-"), maxDerivedName, "-") } +// appendOnce appends value unless the list already holds it. +func appendOnce(list []string, value string) []string { + if slices.Contains(list, value) { + return list + } + return append(list, value) +} + +// maxDerivedName is the longest release name a RunnableIdentity admits. +const maxDerivedName = 50 + // kebab spells a method name the way a resource name is spelled. An acronym // stays one word: ApplyHTTPText is apply-http-text, not apply-h-t-t-p-text. func kebab(name string) string { diff --git a/cmd/generate/runnables_default_test.go b/cmd/generate/runnables_default_test.go new file mode 100644 index 00000000..0fd8ecea --- /dev/null +++ b/cmd/generate/runnables_default_test.go @@ -0,0 +1,133 @@ +package generate + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + runnablespkg "github.com/codefly-dev/cli/pkg/runnables" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + googleproto "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +// hopPolicyProto is authority.hop.v1's policy.proto as a module's contract +// ships it beside the methods it marks. +func hopPolicyProto() *descriptorpb.FileDescriptorProto { + repeatedString := stringField("actions", 2) + repeatedString.Label = descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum() + requires := selfField("requires", 1, ".authority.hop.v1.Requirement") + requires.Label = descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum() + return &descriptorpb.FileDescriptorProto{ + Name: googleproto.String("authority/hop/v1/policy.proto"), + Package: googleproto.String("authority.hop.v1"), + Syntax: googleproto.String("proto3"), + Dependency: []string{"google/protobuf/descriptor.proto"}, + MessageType: []*descriptorpb.DescriptorProto{ + descriptorMessage("Policy", requires, boolField("any_checked_caller", 2)), + descriptorMessage("Requirement", stringField("resource_kind", 1), repeatedString), + }, + Extension: []*descriptorpb.FieldDescriptorProto{{ + Name: googleproto.String("policy"), Number: googleproto.Int32(51100), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), TypeName: googleproto.String(".authority.hop.v1.Policy"), + Extendee: googleproto.String(".google.protobuf.MethodOptions"), JsonName: googleproto.String("policy"), + }}, + } +} + +// hopPolicyMethod carries only `requires: [{resource_kind: kind, actions}]`, +// as the unknown bytes a reader without the Go type holds. +func hopPolicyMethod(name, kind string, actions ...string) *descriptorpb.MethodDescriptorProto { + var requirement []byte + requirement = protowire.AppendTag(requirement, 1, protowire.BytesType) + requirement = protowire.AppendString(requirement, kind) + for _, action := range actions { + requirement = protowire.AppendTag(requirement, 2, protowire.BytesType) + requirement = protowire.AppendString(requirement, action) + } + var policy []byte + policy = protowire.AppendTag(policy, 1, protowire.BytesType) + policy = protowire.AppendBytes(policy, requirement) + method := markedMethod(name, ".documents.ingest.v1.ApplyTextRequest", ".documents.ingest.v1.ApplyTextResponse", nil) + method.Options = &descriptorpb.MethodOptions{} + var raw []byte + raw = protowire.AppendTag(raw, 51100, protowire.BytesType) + raw = protowire.AppendBytes(raw, policy) + method.Options.ProtoReflect().SetUnknown(raw) + return method +} + +// withHopPolicy adds the policy declaration to a published descriptor set. +func withHopPolicy(t *testing.T, contract []byte) []byte { + t.Helper() + var set descriptorpb.FileDescriptorSet + require.NoError(t, googleproto.Unmarshal(contract, &set)) + set.File = append(set.File, hopPolicyProto()) + out, err := googleproto.Marshal(&set) + require.NoError(t, err) + return out +} + +// A method declaring only its authority policy is an operation, and a tool: +// a read is recomputed with no receipt; a stream is listed, not refused. +func TestGenerateRunnablesDerivesEveryMethodBehindTheHop(t *testing.T) { + ctx := context.Background() + streaming := hopPolicyMethod("StreamText", "documents", "read") + streaming.ServerStreaming = googleproto.Bool(true) + // exclude carried as the unknown bytes a contract on disk holds (51004 = true). + excluded := hopPolicyMethod("PurgeText", "documents", "write") + raw := excluded.Options.ProtoReflect().GetUnknown() + raw = protowire.AppendTag(raw, 51004, protowire.VarintType) + raw = protowire.AppendVarint(raw, 1) + excluded.Options.ProtoReflect().SetUnknown(raw) + // A method whose derived release name exceeds 50 characters. + long := hopPolicyMethod("GetDocumentTransformationReceipt", "documents", "read") + file := ingestionFile(conformingOperation(), hopPolicyMethod("CountText", "documents", "read"), + hopPolicyMethod("WriteText", "documents", "write"), streaming, excluded, long) + root, moduleDir := saveRunnableFixture(t, ctx, withHopPolicy(t, descriptorSet(t, file)), "0.1.0") + t.Chdir(root) + resetRunnablesFlags(t) + require.NoError(t, RunnablesCmd.RunE(RunnablesCmd, []string{"documents"})) + + read := func(method string) map[string]any { + data, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(runnablespkg.DerivedDir), "runtime-worker", "grpc", method, runnablespkg.OperationFileName)) + require.NoError(t, err) + var document map[string]any + require.NoError(t, json.Unmarshal(data, &document)) + return document + } + count := read("CountText") + require.Equal(t, "RECOVERY_RECOMPUTE", count["recovery"]) + require.Empty(t, count["lookup_scopes"]) + require.Equal(t, "EFFECT_READ_ONLY", count["tool"].(map[string]any)["effect"]) + require.Equal(t, "documents", count["audience"]) + + write := read("WriteText") + require.Equal(t, "RECOVERY_RECEIPT", write["recovery"]) + require.Equal(t, "EFFECT_MUTATION", write["tool"].(map[string]any)["effect"]) + require.EqualValues(t, 1, write["max_attempts"]) + // The receipt is read under the scopes that made the effect: no read is + // added on the written kind, and no lookup scopes are declared. + require.Equal(t, []any{map[string]any{"resource_kind": "documents", "actions": []any{"write"}}}, write["invoke_scopes"]) + require.Empty(t, write["lookup_scopes"]) + + // The explicit option keeps its document, beside its presentation. + apply := read("ApplyText") + require.Nil(t, apply["recovery"]) + require.Equal(t, "ApplyText", apply["tool"].(map[string]any)["name"]) + + index := readIndex(t, moduleDir) + require.Len(t, index.Operations, 4, "the stream and the excluded method are calls, not operations") + for _, entry := range index.Operations { + require.LessOrEqual(t, len(entry.Name), 50, entry.Name) + require.NotContains(t, entry.Method, "PurgeText") + if strings.HasSuffix(entry.Method, "/GetDocumentTransformationReceipt") { + require.Equal(t, derivedName("runtime-worker", "grpc", "GetDocumentTransformationReceipt"), entry.Name) + require.Regexp(t, `^runtime-worker-grpc-get-document-transf.*-[0-9a-f]{8}$`, entry.Name) + } + } +} diff --git a/cmd/generate/runnables_fixture_test.go b/cmd/generate/runnables_fixture_test.go index 1e9a1b1c..68ba726f 100644 --- a/cmd/generate/runnables_fixture_test.go +++ b/cmd/generate/runnables_fixture_test.go @@ -50,11 +50,12 @@ func int64Field(name string, number int32) *descriptorpb.FieldDescriptorProto { return field } -// uint64Field is outside the bounded profile: its range exceeds the signed -// 64-bit integer the profile carries. -func uint64Field(name string, number int32) *descriptorpb.FieldDescriptorProto { +// selfField is outside the bounded profile: a message reaching itself has no +// bounded shape, whatever proto3 JSON would spell its scalars as. +func selfField(name string, number int32, message string) *descriptorpb.FieldDescriptorProto { field := stringField(name, number) - field.Type = descriptorpb.FieldDescriptorProto_TYPE_UINT64.Enum() + field.Type = descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum() + field.TypeName = googleproto.String(message) return field } @@ -130,7 +131,7 @@ func ingestionFile(declared *runnablev0.Operation, extra ...*descriptorpb.Method ), descriptorMessage("LookupTextRequest", stringField("effect_digest", 1)), // CountRequest is deliberately out of the bounded profile. - descriptorMessage("CountRequest", uint64Field("total", 1)), + descriptorMessage("CountRequest", selfField("total", 1, ".documents.ingest.v1.CountRequest")), }, Service: []*descriptorpb.ServiceDescriptorProto{{ Name: googleproto.String("IngestionService"), @@ -139,8 +140,8 @@ func ingestionFile(declared *runnablev0.Operation, extra ...*descriptorpb.Method } } -// outOfProfileMethod takes a uint64 the bounded profile has no type for. It is -// marked, so it is refused rather than skipped. +// outOfProfileMethod takes a recursive message the bounded profile has no +// shape for. It is marked, so it is refused rather than skipped. func outOfProfileMethod() *descriptorpb.MethodDescriptorProto { return markedMethod("CountText", ".documents.ingest.v1.CountRequest", ".documents.ingest.v1.ApplyTextResponse", conformingOperation()) } diff --git a/cmd/generate/runnables_output.go b/cmd/generate/runnables_output.go index 4f68299e..a1609719 100644 --- a/cmd/generate/runnables_output.go +++ b/cmd/generate/runnables_output.go @@ -37,9 +37,20 @@ func newOperationDocument(spec *corerunnable.OperationSpec) *runnablespkg.Operat Completion: spec.Completion.String(), Tool: spec.Tool, RequiredScopeSlots: spec.RequiredScopeSlots, + Recovery: recoveryName(spec.Recovery), } } +// recoveryName spells a declared recovery, empty when the owner stated none: +// an operation derived before recovery could be stated keeps its document +// byte for byte, and reads back as the receipt it always was. +func recoveryName(recovery basev0.RunnableExecution_Recovery) string { + if recovery == basev0.RunnableExecution_RECOVERY_UNKNOWN { + return "" + } + return recovery.String() +} + func scopeDocuments(scopes []*basev0.WorkScopeV1) []runnablespkg.Scope { documents := make([]runnablespkg.Scope, 0, len(scopes)) for _, scope := range scopes { diff --git a/cmd/generate/runnables_test.go b/cmd/generate/runnables_test.go index a007bf0d..5fbf29b5 100644 --- a/cmd/generate/runnables_test.go +++ b/cmd/generate/runnables_test.go @@ -341,10 +341,10 @@ func TestGenerateRunnablesRefusalLeavesTheCommittedTreeAlone(t *testing.T) { t.Fatalf("expected the generated tree to hold 3 files, got %d", len(committed)) } - // The owner breaks ApplyText's payload with a uint64, outside the bounded - // profile. The method still carries the option. + // The owner breaks ApplyText's payload with a recursive field, outside the + // bounded profile. The method still carries the option. broken := ingestionFile(conformingOperation()) - broken.MessageType[0].Field = append(broken.MessageType[0].Field, uint64Field("total", 99)) + broken.MessageType[0].Field = append(broken.MessageType[0].Field, selfField("total", 99, ".documents.ingest.v1."+broken.MessageType[0].GetName())) publishContract(t, moduleDir, descriptorSet(t, broken)) err := RunnablesCmd.RunE(RunnablesCmd, []string{"documents"}) @@ -511,23 +511,23 @@ func TestGenerateRunnablesReproducesTheDocumentStoreOracle(t *testing.T) { Contract: &basev0.RunnableContract{ Protocol: resources.RunnableServiceProtocolV1, Input: &basev0.RunnableSchema{Fields: []*basev0.RunnableField{ - {Name: "origin", Type: basev0.RunnableField_STRING}, - {Name: "container", Type: basev0.RunnableField_STRING}, - {Name: "ref", Type: basev0.RunnableField_STRING}, - {Name: "path", Type: basev0.RunnableField_STRING}, - {Name: "revision", Type: basev0.RunnableField_STRING}, - {Name: "commit", Type: basev0.RunnableField_STRING}, - {Name: "cursor", Type: basev0.RunnableField_INTEGER}, - {Name: "content_type", Type: basev0.RunnableField_STRING}, - {Name: "text", Type: basev0.RunnableField_STRING}, + {Name: "origin", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "container", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "ref", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "path", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "revision", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "commit", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "cursor", Type: basev0.RunnableField_STRING, Format: corerunnable.FormatInt64, Optional: true}, + {Name: "contentType", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "text", Type: basev0.RunnableField_STRING, Optional: true}, }}, Output: &basev0.RunnableSchema{Fields: []*basev0.RunnableField{ - {Name: "reference", Type: basev0.RunnableField_STRING}, - {Name: "content_hash", Type: basev0.RunnableField_STRING}, - {Name: "effect_digest", Type: basev0.RunnableField_STRING}, - {Name: "minted", Type: basev0.RunnableField_BOOLEAN}, - {Name: "skipped", Type: basev0.RunnableField_BOOLEAN}, - {Name: "quarantined", Type: basev0.RunnableField_BOOLEAN}, + {Name: "reference", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "contentHash", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "effectDigest", Type: basev0.RunnableField_STRING, Optional: true}, + {Name: "minted", Type: basev0.RunnableField_BOOLEAN, Optional: true}, + {Name: "skipped", Type: basev0.RunnableField_BOOLEAN, Optional: true}, + {Name: "quarantined", Type: basev0.RunnableField_BOOLEAN, Optional: true}, }}, }, Execution: &basev0.RunnableExecution{ @@ -564,10 +564,13 @@ func TestGenerateRunnablesReproducesTheDocumentStoreOracle(t *testing.T) { // The digest is over the canonical package, so a field added to // RunnableExecution moves it for every package in the fleet. This one moved // with the completion mode (codefly-dev/core#678); it had been - // 4c26fb95a0236b94107b30f34783cd893f2be36e0d157340c247013fd93a9a1f. The + // 4c26fb95a0236b94107b30f34783cd893f2be36e0d157340c247013fd93a9a1f, then + // ee460290f91e2cad2d9e7efe6d740ba35ff5ec89b4395bbde1e8bd19bd8291a0 until the + // contract was spelled as the Connect wire carries it — JSON names, 64-bit + // integers as strings, every non-required key optional (core#843). The // equality check above is what proves the derivation still matches what // document-store publishes — this line only pins the number. - if asOracle.GetDigest() != "ee460290f91e2cad2d9e7efe6d740ba35ff5ec89b4395bbde1e8bd19bd8291a0" { + if asOracle.GetDigest() != "4b72d393b3fb25284036185968438f3c24be98780660c7f994de896358d4878c" { t.Fatalf("digest = %s, want the published oracle's", asOracle.GetDigest()) } } diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 78a2e5eb..6f9dcfd1 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -2950,8 +2950,9 @@ owner must still serve it. In workspace.codefly.yaml, environments[].runnable-scope-selections maps each binding key (MODULE__OPERATION) to Core ScopeSelection messages: slot, invoke, and lookup. Every required slot needs exact resource kinds, actions and ids; -missing selections, wildcards and overlapping kinds are refused. Tool exposure -is carried from the owner. See docs/runnable.md for the selection YAML. +missing selections, wildcards and overlapping kinds are refused. Every +operation is a tool; its presentation and effect are carried from the derived +policy. See docs/runnable.md for the selection YAML. An environment that installs an operation for no resource answers its key with an explicitly empty list, KEY: []. That operation is declined: its key is absent @@ -2995,21 +2996,25 @@ Flags: ## `codefly generate runnables` -Derive SERVICE-facility Runnable packages from the methods a module's contracts mark +Derive SERVICE-facility Runnable packages, and so tools, from a module's served methods ``` Derive a Runnable package for every gRPC method a module's published contracts -mark with the codefly.runnable.v0.operation option. +serve behind the authority hop. Every Runnable is a tool. -A unary, idempotent method an owner service already publishes becomes a -Runnable by derivation, not by authoring: the method option says which methods -are operations and under what execution policy, and the message descriptors say -what the contract is. Nothing is written twice. +A method that declares its authority policy (authority.hop.v1.policy) is an +operation with nothing added: a policy requiring only "read" derives a +read-only tool that is safe to re-run and carries no receipt; anything else +derives a mutation resolved by its effect receipt through the SDK's generic +lookup. The codefly.runnable.v0.operation option overrides those defaults, and +codefly.runnable.v0.exclude keeps a method from every agent. The message +descriptors say what the contract is, by their proto3 JSON. Nothing is written +twice. The input is what `codefly generate contracts` already wrote — each endpoint's served procedures in the API catalog and its serialized FileDescriptorSet. -Run that first. Only served methods carrying the option are derived; their -paired Lookup must be served on the same endpoint. No flag names a method. +Run that first. Only served methods are derived; a declared Lookup must be +served on the same endpoint. No flag names a method. For each marked method this writes, under contracts/runnables: @@ -3017,12 +3022,13 @@ For each marked method this writes, under contracts/runnables: ///operation.json its execution policy and authority index.json one row per operation -The tree is fully owned: a method that no longer carries the option loses its -directory. A module with no marked method writes an empty index, not an error. +The tree is fully owned: a method that is no longer an operation loses its +directory. A module with no operation writes an empty index, not an error. -A streaming method, a payload outside the bounded schema profile, or an option -core refuses is named and skipped, and the command exits non-zero after -reporting every one of them. +A method derived by default that cannot be one — a stream, a policy requiring +nothing, a payload proto3 JSON cannot carry — is listed and left a plain call. +A method carrying the option that core refuses is named and skipped, and the +command exits non-zero after reporting every one of them. --check is the CI drift gate: it regenerates into a temporary directory and prints a unified diff of what differs, without writing anything. diff --git a/docs/commands.md b/docs/commands.md index b1863972..19c1f3e7 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -3247,7 +3247,7 @@ codefly generate client --from contracts:module-saas-starter/contracts/api --lan codefly generate proto --proto ../proto --output ./generated # Generate code from local proto files, in the proto companion codefly generate contracts saas-starter # Export a module's interface endpoints as API contracts codefly generate contracts saas-starter --check # CI drift gate: fail if the on-disk catalog is stale -codefly generate runnables documents # Derive a Runnable package per method carrying the operation option +codefly generate runnables documents # Derive a Runnable package, and so a tool, per method served behind the hop codefly generate runnables documents --check # CI drift gate: fail if the derived packages are stale codefly generate runnable-bindings --env staging # Bind every derived operation to its owner endpoint as the environment resolves it ``` @@ -3609,17 +3609,37 @@ is the CI drift gate. #### generate runnables `codefly generate runnables [module]` derives a **SERVICE-facility Runnable -package** for every gRPC method a module's published contracts mark with the -`codefly.runnable.v0.operation` method option. A unary, idempotent method an -owner service already publishes becomes a Runnable by derivation, not by -authoring: the option says which methods are operations and under what -execution policy, and the message descriptors say what the contract is. +package** for every unary gRPC method a module's published contracts serve +behind the authority hop, and every Runnable is a tool. A method that declares +its authority policy (`authority.hop.v1.policy`) needs nothing else: a policy +requiring only `read` derives a read-only tool that is safe to re-run and +carries no receipt (`recovery: RECOVERY_RECOMPUTE`); anything else derives a +mutation with `RECOVERY_RECEIPT`, attempted once — at-most-once until the SDK's +receipt guard follows the derivation, an uncertain outcome surfaced as +unknown. A receipt lookup (another operation's `lookup_method`, or +`Lookup` beside ``) is never derived. The tool's name and description are the method's name and +leading doc comment. The `codefly.runnable.v0.operation` option overrides +those defaults, and `codefly.runnable.v0.exclude` keeps a method from every +agent. Every operation is a tool, and who reaches it is a matter of grants: a +caller offers a person only the tools whose binding the person's grants cover, +so a runtime-only or administrative operation derives and never reaches a +person without the grant. The message descriptors say what the contract is, +spelled as the Connect wire carries it — JSON member names, 64-bit integers as +strings (core docs/runnable.md). + +The run reports each operation with its origin (`option` or `default`) and +effect, lists the methods left plain calls and why, and — when any operation +was derived by default — counts them, names the mutations, and points at +`codefly.runnable.v0.exclude`. A release name longer than 50 characters is +shortened deterministically to a stable prefix and an eight-digit digest, so +a long method name never fails generation. The input is what `generate contracts` already wrote — each gRPC/connect endpoint's served `services[].procedures` in the API catalog and its `contract.binpb` — so run that first. A method is derived only when that endpoint -serves it and its descriptor carries the option. Its paired Lookup must also be -served there; descriptor availability alone is insufficient. Catalog procedures +serves it and its descriptor declares an authority policy or carries the option. +A declared Lookup must also be served there; descriptor availability alone is +insufficient. Catalog procedures absent from the endpoint's descriptor package are refused. No flag names a method. ``` @@ -3633,13 +3653,13 @@ contracts/runnables/ digest over, so what a composition reads and what was hashed are the same bytes. `operation.json` is kept beside it rather than inside it: policy and authority are installed with a binding, and two installations of one contract -may differ in both. Required scope slots and optional tool exposure are preserved -in `operation.json`; generation does not select resource IDs. `index.json` is +may differ in both. Required scope slots, the tool presentation and the recovery +are preserved in `operation.json`; generation does not select resource IDs. `index.json` is what a composition reads to prepare bindings — identity, digest, full method, input and output message names and endpoint coordinates per row. -The tree is fully owned: a method that no longer carries the option or is no +The tree is fully owned: a method that is no longer an operation or is no longer served on an endpoint loses that endpoint's directory. Only the three file names a generation produces are removed, and only the directories that empties, so pointing `--output` at a directory this @@ -3649,8 +3669,11 @@ module that declares no `interface:` and so exports no endpoint at all, both write an empty index rather than an error; a module that *does* declare an interface but has no catalog is told to run `generate contracts` first. -A streaming method, a payload outside the bounded schema profile, or an option -core refuses is named with its field path; the walk continues so an owner +A method derived by default that cannot be one — a stream, a policy requiring +nothing (`any_checked_caller`), a payload proto3 JSON cannot carry (`Any`, a +recursive message) — is listed as "not an operation" and stays a plain call; it +never fails the command. A method carrying the option that is a stream, has a +payload outside the profile, or that core refuses is named with its field path; the walk continues so an owner fixing a contract sees the whole list, and the command then exits non-zero **having written nothing**. A failing run leaves the tree exactly as it found it: writing the methods that did derive would delete the refused one's diff --git a/go.mod b/go.mod index 2ee4b8b8..f7477df9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/codefly-dev/authority/libraries/authority-client/go v0.0.0-20261025214748-c9cef7470d20 github.com/codefly-dev/authority/libraries/delivery/go v0.0.0-20261025214748-c9cef7470d20 github.com/codefly-dev/authority/libraries/hop/go v0.0.0-20261025214748-c9cef7470d20 - github.com/codefly-dev/core v0.19.7-0.20261026014924-50b50dd2a5da + github.com/codefly-dev/core v0.19.7-0.20261026023455-8b5ddef6d55a github.com/codefly-dev/golor v0.1.3 github.com/codefly-dev/llm v0.1.7 github.com/codefly-dev/sdk-go/workcontext v0.1.1-0.20261018052022-70dc7a546458 diff --git a/go.sum b/go.sum index d5120d36..d23c9ed8 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,18 @@ github.com/codefly-dev/core v0.19.7-0.20261025175252-88f4434c68c7 h1:sFkQTBSvAjz github.com/codefly-dev/core v0.19.7-0.20261025175252-88f4434c68c7/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= github.com/codefly-dev/core v0.19.7-0.20261025185759-e501458995d0 h1:YEFvkyokBsUWbHoaoKOi6iJtMrqNNR7HsWdUQGN50TA= github.com/codefly-dev/core v0.19.7-0.20261025185759-e501458995d0/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= +github.com/codefly-dev/core v0.19.7-0.20261026010229-45383f9ff8c8 h1:4xDq2A1P/1CAhTHy7PsHIlbbvAo1ywDF5vxtD2eQiJ8= +github.com/codefly-dev/core v0.19.7-0.20261026010229-45383f9ff8c8/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= +github.com/codefly-dev/core v0.19.7-0.20261026011658-87497b44fac1 h1:rlFnRxJkYiwKfCl2oZPoB5mP+H31BZgmw36jScG2yB8= +github.com/codefly-dev/core v0.19.7-0.20261026011658-87497b44fac1/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= +github.com/codefly-dev/core v0.19.7-0.20261026014202-483255ecca45 h1:2pKa3A044y3W/eKNX2iIuaNMUmd5Kk06w0SrGkoejFM= +github.com/codefly-dev/core v0.19.7-0.20261026014202-483255ecca45/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= +github.com/codefly-dev/core v0.19.7-0.20261026014844-b6c00122c09c h1:9oKvfr/IBPSN/YpzmPskG87O2mwBcjrjnxtdE7AQhSA= +github.com/codefly-dev/core v0.19.7-0.20261026014844-b6c00122c09c/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= +github.com/codefly-dev/core v0.19.7-0.20261026021736-0c02ef0fd533 h1:J2mxIa2i/wRekIWqhzRrTrbgp5rNPHU0FSsQGz4vKX4= +github.com/codefly-dev/core v0.19.7-0.20261026021736-0c02ef0fd533/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= +github.com/codefly-dev/core v0.19.7-0.20261026023455-8b5ddef6d55a h1:YnMwvoi6hpp1YBj+trwGavrhCBoFNKxo++Z73hr/b44= +github.com/codefly-dev/core v0.19.7-0.20261026023455-8b5ddef6d55a/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= github.com/codefly-dev/core v0.19.7-0.20261025230614-9386c6bf7521 h1:UP7WCPYMa/K/BQYGdZ95wlCtyRFG+RccBadFVw+qYPs= github.com/codefly-dev/core v0.19.7-0.20261025230614-9386c6bf7521/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= github.com/codefly-dev/core v0.19.7-0.20261026014924-50b50dd2a5da h1:g/I/o/KTL8L2xEDrgJsSO4qLR2oYWJI8PorbVMmKSbM= diff --git a/pkg/runnables/derived.go b/pkg/runnables/derived.go index 1d6120ab..2ea28171 100644 --- a/pkg/runnables/derived.go +++ b/pkg/runnables/derived.go @@ -167,6 +167,10 @@ type Operation struct { // policy vocabulary. Slots remain unresolved until the composition binds. Tool *runnablev0.ToolExposure `json:"tool,omitempty"` RequiredScopeSlots []*runnablev0.ScopeSlot `json:"required_scope_slots,omitempty"` + // Recovery is how an uncertain outcome is resolved, spelled as the + // codefly.base.v0.RunnableExecution.Recovery enum name. RECOVERY_RECOMPUTE + // is a read, which carries no receipt; empty is RECOVERY_RECEIPT. + Recovery string `json:"recovery,omitempty"` } // operationJSON preserves the existing document's scalar spellings and order,