feat(generate): a solution's routes become Runnables, and so tools, as a module's methods do (#1184)

cmd/generate/runnable_bindings.go

@@ -59,7 +59,7 @@ For each derived operation the value is one JSON document
 (` + corerunnable.PreparedSchemaV3 + `):
 
   operation        the owner coordinates, and the operation as the owner spells
-                   it: /pkg.Service/Method, or "POST /path"
+                   it: /pkg.Service/Method, or "GET /path" for a route
   call             where one call is sent: the resolved base URL, and a typed
                    route — connect (the procedure, POSTed as JSON on the owner's
                    Connect endpoint) or rest (the owner's own verb and path)
@@ -76,6 +76,9 @@ A gRPC owner is therefore called on its **Connect** endpoint: a service that
 publishes a runnable-marked method and declares no connect endpoint is refused
 here, by name, rather than at a call.
 An operation published directly on a Connect endpoint uses that endpoint.
+A solution's route is called on the http (or rest) endpoint whose OpenAPI
+document published it, with its own verb and path, and keyed by the solution's
+name exactly as a module operation is by its module's (SOLUTION__OPERATION).
 
 The address is resolved, never configured: it is the one a run or a render of
 the same environment gives the owner. Locally that is the address of the run's
@@ -661,7 +664,10 @@ func (r *endpointResolver) callTarget(ctx context.Context, module *resources.Mod
 			Address: address,
 			Route:   &runnablev0.PreparedCall_Connect{Connect: &runnablev0.ConnectProcedure{Procedure: spelling}},
 		}, composition.APIContractKindProtobuf, nil
-	case standards.REST:
+	case standards.REST, standards.HTTP:
+		// A solution's backend serves its routes on an http endpoint and
+		// publishes them as OpenAPI, which verifyPublishedContract holds it to:
+		// it is called exactly as a REST owner is.
 		address, err := r.address(ctx, identity, endpoints, published)
 		if err != nil {
 			return nil, "", err
@@ -675,8 +681,8 @@ func (r *endpointResolver) callTarget(ctx context.Context, module *resources.Mod
 			Route:   &runnablev0.PreparedCall_Rest{Rest: &runnablev0.HTTPRoute{Verb: verb, Path: path}},
 		}, composition.APIContractKindOpenAPI, nil
 	default:
-		return nil, "", fmt.Errorf("%s/%s/%s is a %s endpoint, and a derived operation is called over %s, %s or %s",
-			module.Name, serviceName, publishedName, published.GetApi(), standards.GRPC, standards.CONNECT, standards.REST)
+		return nil, "", fmt.Errorf("%s/%s/%s is a %s endpoint, and a derived operation is called over %s, %s, %s or %s",
+			module.Name, serviceName, publishedName, published.GetApi(), standards.GRPC, standards.CONNECT, standards.REST, standards.HTTP)
 	}
 }
 

cmd/generate/runnables.go

@@ -1,7 +1,9 @@
 package generate
 
 import (
+	"cmp"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -36,9 +38,10 @@ var (
 // published contracts mark as an operation.
 var RunnablesCmd = &cobra.Command{
 	Use:   "runnables [module]",
-	Short: "Derive SERVICE-facility Runnable packages, and so tools, from a module's served methods",
+	Short: "Derive SERVICE-facility Runnable packages, and so tools, from a module's served methods and a solution's routes",
 	Long: `Derive a Runnable package for every gRPC method a module's published contracts
-serve behind the authority hop. Every Runnable is a tool.
+serve behind the authority hop, and every REST route a solution's published
+OpenAPI document marks. Every Runnable is a tool.
 
 A method that declares its authority policy (authority.hop.v1.policy) is an
 operation with nothing added: a policy requiring only "read" derives a
@@ -49,8 +52,18 @@ codefly.runnable.v0.exclude keeps a method from every agent. The message
 descriptors say what the contract is, by their proto3 JSON. Nothing is written
 twice.
 
+A solution publishes its routes as one OpenAPI document (Swagger 2.0 or
+OpenAPI 3), and every route marked x-codefly-operation derives the same way:
+solution-runtime-go marks each read the backend serves, so a GET read becomes a
+read-only tool whose query parameters are its input. Its directory is named by
+the route's operationId. No flag names a route. A document that marks no route
+although its routes declare read-only policies — rendered by the runtime's
+InterfaceDocument rather than Server.InterfaceArtifact, or by a runtime from
+before the marker — names each such read as not an operation.
+
 The input is what ` + "`codefly generate contracts`" + ` already wrote — each endpoint's
-served procedures in the API catalog and its serialized FileDescriptorSet.
+served procedures in the API catalog and its serialized FileDescriptorSet, or
+its OpenAPI document.
 Run that first. Only served methods are derived; a declared Lookup must be
 served on the same endpoint. No flag names a method.
 
@@ -60,6 +73,8 @@ For each marked method this writes, under contracts/runnables:
   <service>/<endpoint>/<Method>/operation.json          its execution policy and authority
   index.json                                            one row per operation
 
+and the same, under <service>/<endpoint>/<operationId>/, for each marked route.
+
 The tree is fully owned: a method that is no longer an operation loses its
 directory. A module with no operation writes an empty index, not an error.
 
@@ -74,6 +89,7 @@ prints a unified diff of what differs, without writing anything.
 Examples:
   codefly generate runnables
   codefly generate runnables documents
+  codefly generate runnables releases          # a solution: its marked routes
   codefly generate runnables --check`,
 	Args: cobra.MaximumNArgs(1),
 	RunE: func(_ *cobra.Command, args []string) error {
@@ -170,7 +186,9 @@ func (d *derivation) refusal() error {
 // from and what it does.
 type derivedOperation struct {
 	explicit bool
-	effect   string
+	// marker is a route its document marks (x-codefly-operation).
+	marker bool
+	effect string
 }
 
 // report says what the run made of the module, so an owner learns which of
@@ -187,7 +205,10 @@ func (d *derivation) report() {
 		entry := &d.index.Operations[i]
 		operation := d.origins[entry.Name]
 		origin := "option"
-		if !operation.explicit {
+		switch {
+		case operation.marker:
+			origin = "marker"
+		case !operation.explicit:
 			origin = "default"
 			defaults = appendOnce(defaults, entry.Method)
 			if operation.effect == runnablev0.ToolExposure_EFFECT_MUTATION.String() {
@@ -259,7 +280,12 @@ func deriveRunnablesIn(ctx context.Context, workspaceName, workspaceDir string,
 	agents := map[string]*basev0.Agent{}
 	for i := range endpoints {
 		endpoint := &endpoints[i]
-		if endpoint.Kind != composition.APIContractKindProtobuf {
+		derive := deriveEndpoint
+		switch endpoint.Kind {
+		case composition.APIContractKindProtobuf:
+		case composition.APIContractKindOpenAPI:
+			derive = deriveOpenAPIEndpoint
+		default:
 			continue
 		}
 		agent, ok := agents[endpoint.Service]
@@ -269,7 +295,7 @@ func deriveRunnablesIn(ctx context.Context, workspaceName, workspaceDir string,
 			}
 			agents[endpoint.Service] = agent
 		}
-		if err = deriveEndpoint(module, endpoint, agent, derived); err != nil {
+		if err = derive(module, endpoint, agent, derived); err != nil {
 			return nil, err
 		}
 	}
@@ -347,54 +373,179 @@ func deriveEndpoint(module *resources.Module, endpoint *composition.APIContractE
 // derivation refuses is recorded and the walk continues.
 func deriveMethod(module *resources.Module, endpoint *composition.APIContractEndpoint, owner corerunnable.ServiceOwner, files *protoregistry.Files, fullMethod string, served []string, derived *derivation) error {
 	name := methodName(fullMethod)
-	relativeDir, err := derivedRelativeDir(derived.workspaceDir, module, endpoint, name)
+	location, err := derived.location(module, endpoint, name, name)
 	if err != nil {
 		return err
 	}
-	location := &resources.RunnableLocation{
-		Identity: &resources.RunnableIdentity{
-			Name:      derivedName(endpoint.Service, endpoint.Endpoint, name),
-			Module:    module.Name,
-			Workspace: derived.index.Workspace,
-			Version:   derivedVersion(derived.release, endpoint),
-		},
-		WorkspacePath:       derived.workspaceDir,
-		RelativeToWorkspace: relativeDir,
+	pkg, spec, err := corerunnable.PackageFromMethod(files, location, owner, fullMethod)
+	if !derived.admitted(fullMethod, err) {
+		return nil
+	}
+	if spec.LookupMethod != "" && !slices.Contains(served, spec.LookupMethod) {
+		derived.refusals = append(derived.refusals, fmt.Sprintf("%s: endpoint %s/%s does not serve lookup method %s", fullMethod, endpoint.Service, endpoint.Endpoint, spec.LookupMethod))
+		return nil
 	}
+	return derived.record(endpoint, name, pkg, spec, derivedOperation{explicit: spec.Declared, effect: spec.Tool.GetEffect().String()})
+}
 
-	pkg, spec, err := corerunnable.PackageFromMethod(files, location, owner, fullMethod)
+// deriveOpenAPIEndpoint derives every route one OpenAPI contract marks with
+// x-codefly-operation — a solution's backend publishes one, marking each read
+// it serves — through the same writers a method goes through, so a route and a
+// method land in one tree, one index and one binding key space. Routes are
+// walked in path then method order, so the index is a function of the document.
+func deriveOpenAPIEndpoint(module *resources.Module, endpoint *composition.APIContractEndpoint, agent *basev0.Agent, derived *derivation) error {
+	contract := filepath.Join(module.Dir(), filepath.FromSlash(endpoint.Path))
+	document, err := os.ReadFile(contract) //nolint:gosec // the path comes from the module's own contract catalog
 	if err != nil {
-		if errors.Is(err, corerunnable.ErrNotAnOperation) {
-			return nil
+		return fmt.Errorf("cannot read the contract of %s/%s: %w", endpoint.Service, endpoint.Endpoint, err)
+	}
+	parsed, err := corerunnable.ParseOpenAPIDocument(document)
+	if err != nil {
+		return fmt.Errorf("cannot read %s: %w", contract, err)
+	}
+	owner := corerunnable.ServiceOwner{Module: module.Name, Service: endpoint.Service, Endpoint: endpoint.Endpoint, Agent: agent}
+	paths := make([]string, 0, len(parsed.Paths))
+	for path := range parsed.Paths {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	marked := 0
+	for _, path := range paths {
+		item := parsed.Paths[path]
+		if item == nil {
+			continue
 		}
-		if errors.Is(err, corerunnable.ErrNotDerivable) {
-			derived.notices = append(derived.notices, fmt.Sprintf("%s: %v", fullMethod, err))
-			return nil
+		for _, route := range []struct {
+			method    string
+			operation *corerunnable.OpenAPIOperation
+		}{{"GET", item.Get}, {"POST", item.Post}, {"PUT", item.Put}} {
+			if route.operation == nil || len(route.operation.Marker) == 0 {
+				continue
+			}
+			marked++
+			if err = deriveRoute(module, endpoint, owner, document, route.method, path, route.operation.OperationID, derived); err != nil {
+				return err
+			}
 		}
-		derived.refusals = append(derived.refusals, fmt.Sprintf("%s: %v", fullMethod, err))
+	}
+	if marked == 0 {
+		derived.notices = append(derived.notices, unmarkedReads(document, endpoint)...)
+	}
+	return nil
+}
+
+// unmarkedReads names the reads of a document that marks no route at all
+// although its routes declare policies requiring only read: the signature of
+// an artifact rendered by solution-runtime-go's InterfaceDocument rather than
+// Server.InterfaceArtifact, or by a runtime from before the marker existed.
+// Either way every read is silently no tool, so each is named. A document that
+// marks some routes and not others is the owner's choice (Operation.Exclude)
+// and says nothing.
+func unmarkedReads(document []byte, endpoint *composition.APIContractEndpoint) []string {
+	type requirement struct {
+		Actions []string `json:"actions"`
+	}
+	var parsed struct {
+		Paths map[string]map[string]struct {
+			Policy *struct {
+				Requires []requirement `json:"requires"`
+			} `json:"x-codefly-authority-policy"`
+		} `json:"paths"`
+	}
+	if json.Unmarshal(document, &parsed) != nil {
 		return nil
 	}
-	if spec.LookupMethod != "" && !slices.Contains(served, spec.LookupMethod) {
-		derived.refusals = append(derived.refusals, fmt.Sprintf("%s: endpoint %s/%s does not serve lookup method %s", fullMethod, endpoint.Service, endpoint.Endpoint, spec.LookupMethod))
+	var notices []string
+	for path, methods := range parsed.Paths {
+		for method, operation := range methods {
+			if operation.Policy == nil || len(operation.Policy.Requires) == 0 ||
+				slices.ContainsFunc(operation.Policy.Requires, func(r requirement) bool {
+					return len(r.Actions) != 1 || r.Actions[0] != corerunnable.ReadOnlyScopeAction
+				}) {
+				continue
+			}
+			notices = append(notices, fmt.Sprintf("%s: %s/%s requires only %q but its document marks no route x-codefly-operation; render the artifact with solution-runtime-go's Server.InterfaceArtifact (InterfaceDocument, and a runtime before the marker, mark none)",
+				corerunnable.Route(method, path), endpoint.Service, endpoint.Endpoint, corerunnable.ReadOnlyScopeAction))
+		}
+	}
+	sort.Strings(notices)
+	return notices
+}
+
+// deriveRoute derives one marked route. Its directory is named by the route's
+// operationId — the identity the document publishes it under, as a method's
+// is its name — or, when the document gives none, by the route itself.
+func deriveRoute(module *resources.Module, endpoint *composition.APIContractEndpoint, owner corerunnable.ServiceOwner, document []byte, method, path, operationID string, derived *derivation) error {
+	spelling := corerunnable.Route(method, path)
+	words := strings.FieldsFunc(cmp.Or(operationID, spelling), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) })
+	for i, word := range words {
+		words[i] = kebab(word)
+	}
+	name := cmp.Or(operationID, strings.Join(words, "-"))
+	location, err := derived.location(module, endpoint, name, strings.Join(words, "-"))
+	if err != nil {
+		return err
+	}
+	pkg, spec, err := corerunnable.PackageFromOpenAPIOperation(document, location, owner, method, path)
+	if !derived.admitted(spelling, err) {
 		return nil
 	}
+	return derived.record(endpoint, name, pkg, spec, derivedOperation{marker: true, effect: spec.Tool.GetEffect().String()})
+}
 
+// location is where one operation's release lives: its directory, named dir,
+// and its identity, the release name spelled from words.
+func (d *derivation) location(module *resources.Module, endpoint *composition.APIContractEndpoint, dir, words string) (*resources.RunnableLocation, error) {
+	relativeDir, err := derivedRelativeDir(d.workspaceDir, module, endpoint, dir)
+	if err != nil {
+		return nil, err
+	}
+	return &resources.RunnableLocation{
+		Identity: &resources.RunnableIdentity{
+			Name:      derivedName(endpoint.Service, endpoint.Endpoint, words),
+			Module:    module.Name,
+			Workspace: d.index.Workspace,
+			Version:   derivedVersion(d.release, endpoint),
+		},
+		WorkspacePath:       d.workspaceDir,
+		RelativeToWorkspace: relativeDir,
+	}, nil
+}
+
+// admitted sorts one derivation's answer: no operation is an answer, an
+// operation the default derivation cannot make is a notice, anything else core
+// refuses is a refusal, and only a derived operation is admitted.
+func (d *derivation) admitted(subject string, err error) bool {
+	switch {
+	case err == nil:
+		return true
+	case errors.Is(err, corerunnable.ErrNotAnOperation):
+	case errors.Is(err, corerunnable.ErrNotDerivable):
+		d.notices = append(d.notices, fmt.Sprintf("%s: %v", subject, err))
+	default:
+		d.refusals = append(d.refusals, fmt.Sprintf("%s: %v", subject, err))
+	}
+	return false
+}
+
+// record writes one derived operation into the tree and the index.
+func (d *derivation) record(endpoint *composition.APIContractEndpoint, name string, pkg *basev0.RunnablePackage, spec *corerunnable.OperationSpec, origin derivedOperation) error {
+	operation := pkg.GetServiceOperations()[0]
 	packageBytes, err := runnablespkg.CanonicalJSON(pkg)
 	if err != nil {
-		return fmt.Errorf("%s: cannot encode the derived package: %w", fullMethod, err)
+		return fmt.Errorf("%s: cannot encode the derived package: %w", operation.GetOperation(), err)
 	}
 	operationBytes, err := marshalIndented(newOperationDocument(spec))
 	if err != nil {
-		return fmt.Errorf("%s: cannot encode the derived operation: %w", fullMethod, err)
+		return fmt.Errorf("%s: cannot encode the derived operation: %w", operation.GetOperation(), err)
 	}
 
 	dir := filepath.Join(endpoint.Service, endpoint.Endpoint, name)
-	derived.files[filepath.ToSlash(filepath.Join(dir, runnablespkg.PackageFileName))] = append(packageBytes, '\n')
-	derived.files[filepath.ToSlash(filepath.Join(dir, runnablespkg.OperationFileName))] = operationBytes
+	d.files[filepath.ToSlash(filepath.Join(dir, runnablespkg.PackageFileName))] = append(packageBytes, '\n')
+	d.files[filepath.ToSlash(filepath.Join(dir, runnablespkg.OperationFileName))] = operationBytes
 
-	operation := pkg.GetServiceOperations()[0]
-	derived.origins[pkg.GetIdentity().GetName()] = derivedOperation{explicit: spec.Declared, effect: spec.Tool.GetEffect().String()}
-	derived.index.Operations = append(derived.index.Operations, runnablespkg.IndexEntry{
+	d.origins[pkg.GetIdentity().GetName()] = origin
+	d.index.Operations = append(d.index.Operations, runnablespkg.IndexEntry{
 		Name:          pkg.GetIdentity().GetName(),
 		Version:       pkg.GetIdentity().GetVersion(),
 		Digest:        pkg.GetDigest(),
@@ -484,7 +635,9 @@ func derivedVersion(release string, endpoint *composition.APIContractEndpoint) s
 	return "0.0.0-contract-" + digest
 }
 
-// derivedName names the release a method derives. The endpoint is part of it
+// derivedName names the release an operation derives, its words spelled as a
+// resource name is (kebab): a method's name, or a route's words, which are
+// already. The endpoint is part of it
 // because one proto service may be published on both a grpc and a connect
 // endpoint: those are two operations reached two ways, and a name that dropped
 // the endpoint would give them one identity and two digests.
@@ -493,8 +646,8 @@ func derivedVersion(release string, endpoint *composition.APIContractEndpoint) s
 // shortened deterministically — a stable prefix and a short digest of the
 // whole name — so a long method name never fails generation and two long
 // names never collapse into one.
-func derivedName(service, endpoint, method string) string {
-	return corerunnable.BoundedName(strings.Join([]string{service, endpoint, kebab(method)}, "-"), maxDerivedName, "-")
+func derivedName(service, endpoint, words string) string {
+	return corerunnable.BoundedName(strings.Join([]string{service, endpoint, kebab(words)}, "-"), maxDerivedName, "-")
 }
 
 // appendOnce appends value unless the list already holds it.