feat(accounts,saas-ui): a bucket source takes include and exclude key patterns, applied before any fetch

module/contracts/api/accounts/authority/proto/saas/accounts/v1/datasource.proto

@@ -189,7 +189,48 @@ message UploadDatasourceConfig {
   // AWS-style access key id. The matching secret access key is the credential.
   string access_key_id = 5;
   // Upper bound on objects fetched per sync. Zero applies the connector default.
+  // Only objects that pass the filters below count against it.
   uint32 max_objects = 6;
+  // Which keys under the prefix are ingested. Each pattern is an RE2 regular
+  // expression matched against the object key with the prefix removed,
+  // unanchored and case-sensitive (anchor with ^ and $, fold case with (?i)).
+  // An empty include admits every key; an exclude match always wins. Applied
+  // when the prefix is listed, before anything is fetched, so an excluded
+  // object is never read and never counts against max_objects. An invalid
+  // expression is refused when the source is added or updated, never at sync:
+  // the call fails INVALID_ARGUMENT with a google.rpc.BadRequest violation, and
+  // a message that starts with the field's path, e.g.
+  // "upload.include_patterns[1]: …".
+  repeated string include_patterns = 7 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
+  repeated string exclude_patterns = 8 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
+  // Case-insensitive key suffixes, e.g. [".md", ".pdf"], intersected with the
+  // patterns. Empty admits every type. GitHub's file_extensions semantics.
+  repeated string file_extensions = 9 [(buf.validate.field).repeated = {
+    max_items: 32
+    items: {
+      string: {
+        min_len: 2
+        max_len: 33
+        pattern: "^\\.[A-Za-z0-9][A-Za-z0-9._-]*$"
+      }
+    }
+  }];
 }
 
 // Datasource is one connected external datasource. It is org-scoped and carries
@@ -437,6 +478,27 @@ message UpdateSourceRequest {
   // sole reason that the source had reconciled since the form was opened. What
   // an editor needs guarded is what they were looking at.
   SourceFetchScope expected = 6 [(buf.validate.field).required = true];
+  // An object-storage source's key patterns (see UploadDatasourceConfig),
+  // replaced whole like every field here. Its file_extensions is field 5; its
+  // branch and paths must be empty. Refused for a github source.
+  repeated string include_patterns = 7 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
+  repeated string exclude_patterns = 8 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
 }
 
 // SourceFetchScope is what a source fetches — the three values UpdateSource
@@ -467,6 +529,17 @@ message SourceFetchScope {
         max_len: 33
       }
     }
+  }];  repeated string include_patterns = 4 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {max_len: 256}
+    }
+  }];
+  repeated string exclude_patterns = 5 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {max_len: 256}
+    }
   }];
 }
 

module/contracts/api/accounts/connect/proto/saas/accounts/v1/datasource.proto

@@ -189,7 +189,48 @@ message UploadDatasourceConfig {
   // AWS-style access key id. The matching secret access key is the credential.
   string access_key_id = 5;
   // Upper bound on objects fetched per sync. Zero applies the connector default.
+  // Only objects that pass the filters below count against it.
   uint32 max_objects = 6;
+  // Which keys under the prefix are ingested. Each pattern is an RE2 regular
+  // expression matched against the object key with the prefix removed,
+  // unanchored and case-sensitive (anchor with ^ and $, fold case with (?i)).
+  // An empty include admits every key; an exclude match always wins. Applied
+  // when the prefix is listed, before anything is fetched, so an excluded
+  // object is never read and never counts against max_objects. An invalid
+  // expression is refused when the source is added or updated, never at sync:
+  // the call fails INVALID_ARGUMENT with a google.rpc.BadRequest violation, and
+  // a message that starts with the field's path, e.g.
+  // "upload.include_patterns[1]: …".
+  repeated string include_patterns = 7 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
+  repeated string exclude_patterns = 8 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
+  // Case-insensitive key suffixes, e.g. [".md", ".pdf"], intersected with the
+  // patterns. Empty admits every type. GitHub's file_extensions semantics.
+  repeated string file_extensions = 9 [(buf.validate.field).repeated = {
+    max_items: 32
+    items: {
+      string: {
+        min_len: 2
+        max_len: 33
+        pattern: "^\\.[A-Za-z0-9][A-Za-z0-9._-]*$"
+      }
+    }
+  }];
 }
 
 // Datasource is one connected external datasource. It is org-scoped and carries
@@ -437,6 +478,27 @@ message UpdateSourceRequest {
   // sole reason that the source had reconciled since the form was opened. What
   // an editor needs guarded is what they were looking at.
   SourceFetchScope expected = 6 [(buf.validate.field).required = true];
+  // An object-storage source's key patterns (see UploadDatasourceConfig),
+  // replaced whole like every field here. Its file_extensions is field 5; its
+  // branch and paths must be empty. Refused for a github source.
+  repeated string include_patterns = 7 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
+  repeated string exclude_patterns = 8 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {
+        min_len: 1
+        max_len: 256
+      }
+    }
+  }];
 }
 
 // SourceFetchScope is what a source fetches — the three values UpdateSource
@@ -467,6 +529,17 @@ message SourceFetchScope {
         max_len: 33
       }
     }
+  }];  repeated string include_patterns = 4 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {max_len: 256}
+    }
+  }];
+  repeated string exclude_patterns = 5 [(buf.validate.field).repeated = {
+    max_items: 16
+    items: {
+      string: {max_len: 256}
+    }
   }];
 }
 

module/contracts/api/catalog.codefly.json

@@ -10,7 +10,7 @@
       "kind": "protobuf",
       "package": "saas.accounts.v1",
       "path": "contracts/api/accounts/authority/contract.binpb",
-      "digest": "sha256:373a8f3a22b7968c36dd9faffb9be5ef64213b1de2a532f5e567007bd65caaa7",
+      "digest": "sha256:a899ecff5c0e74ea89f34a8c761e58c79b50c0a5d2f5538e8e70c252233a0dbf",
       "services": [
         {
           "name": "ModuleCapabilitiesService",
@@ -65,7 +65,7 @@
       "kind": "protobuf",
       "package": "saas.accounts.v1",
       "path": "contracts/api/accounts/connect/contract.binpb",
-      "digest": "sha256:373a8f3a22b7968c36dd9faffb9be5ef64213b1de2a532f5e567007bd65caaa7",
+      "digest": "sha256:a899ecff5c0e74ea89f34a8c761e58c79b50c0a5d2f5538e8e70c252233a0dbf",
       "services": [
         {
           "name": "APIKeyService",

module/module.package.codefly.yaml

@@ -18,12 +18,12 @@ services:
         kind: protobuf
         package: saas.accounts.v1
         path: contracts/api/accounts/authority/contract.binpb
-        digest: sha256:373a8f3a22b7968c36dd9faffb9be5ef64213b1de2a532f5e567007bd65caaa7
+        digest: sha256:a899ecff5c0e74ea89f34a8c761e58c79b50c0a5d2f5538e8e70c252233a0dbf
       - endpoint: connect
         kind: protobuf
         package: saas.accounts.v1
         path: contracts/api/accounts/connect/contract.binpb
-        digest: sha256:373a8f3a22b7968c36dd9faffb9be5ef64213b1de2a532f5e567007bd65caaa7
+        digest: sha256:a899ecff5c0e74ea89f34a8c761e58c79b50c0a5d2f5538e8e70c252233a0dbf
   - name: auth-gateway
     endpoints: [grpc, rest]
     api-contracts:

module/services/accounts/AGENTS.md

@@ -1042,6 +1042,42 @@ The transport underneath — h2c rather than server TLS, the resolved origin, an
 the key set a consumer verifies host-signed credentials against — is
 [../../INTERNAL_TRANSPORT.md](../../INTERNAL_TRANSPORT.md).
 
+## An object-storage source is a prefix and a key filter
+
+An `upload` ("Object storage") source reads one bucket under one `prefix`, and
+`UploadDatasourceConfig` narrows it further: `include_patterns` and
+`exclude_patterns` are RE2 expressions over the object key with the prefix
+removed (unanchored, case-sensitive; exclude wins; an empty include admits
+everything), and `file_extensions` intersects them with GitHub's suffix
+semantics. `uploadKeyScope` (`pkg/business/datasource_file_filter.go`, beside
+GitHub's own filter) is the one matcher. The connector applies it, not the
+listing source: it is the `InScope` of an `objectstore.Scope`
+(`pkg/datasource/objectstore/scope.go`), whose `Window` takes the store's
+listing whole and keeps the matches, counting only them against
+`max_objects`. So an excluded object is never fetched and never uses up the
+cap, the closing listing names only matches, and replacing what lists the
+prefix leaves the filter where it is.
+
+- **Refused at the call, never at sync.** `AddSource` and `UpdateSource`
+  compile every pattern and refuse an invalid one, more than 16, or one over 256
+  characters as `INVALID_ARGUMENT` with a `google.rpc.BadRequest` violation and
+  a message that leads with the field path (`upload.include_patterns[1]: …` on
+  add, `include_patterns[1]: …` on update). The kit places a refusal under its
+  field by that leading path, because the TypeScript SDK carries no
+  error-detail types.
+- **Narrowing deletes.** `UpdateSource` takes the patterns and file types for an
+  upload source (branch and paths are refused), under the same `expected` rule
+  as a GitHub edit, and queues the source's pull sync. That sync's closing
+  listing omits what the narrower filter excludes, and the store tombstones it
+  — unlike a GitHub snapshot, which only declares scope (below).
+- **Access follows the source.** A source is (bucket, prefix, filter) → one
+  collection → one grant; two sources on one bucket with different filters are
+  two collections with different readers. The catalog description says so.
+- **The filter is the host's whatever the store.** An S3-style listing takes a
+  prefix and a delimiter only, so the object-storage service needs nothing for
+  this. When the envelope connector replaces `runUploadSync`, it takes the
+  same `Scope` (#1211).
+
 ## A datasource source is refreshed by three things, and one of them covers every provider
 
 Ranked by latency, a source's content is refreshed by a push (near-live), by a

module/services/accounts/code/pkg/adapters/datasource_handler.go

@@ -107,6 +107,10 @@ func (h *datasourceConnectHandler) AddSource(
 			Prefix:      up.Prefix,
 			AccessKeyID: up.AccessKeyId,
 			MaxObjects:  int(up.MaxObjects),
+
+			IncludePatterns: up.IncludePatterns,
+			ExcludePatterns: up.ExcludePatterns,
+			FileExtensions:  up.FileExtensions,
 		}
 	}
 	source, err := h.svc.AddSource(ctx, actorID, input)
@@ -134,6 +138,9 @@ func (h *datasourceConnectHandler) UpdateSource(
 		Branch:         req.Msg.Branch,
 		Paths:          req.Msg.Paths,
 		FileExtensions: req.Msg.FileExtensions,
+
+		IncludePatterns: req.Msg.IncludePatterns,
+		ExcludePatterns: req.Msg.ExcludePatterns,
 	}
 	// The scope the caller was shown. The descriptor marks it required, so an
 	// absent one is already refused before this handler runs; it is mapped
@@ -144,6 +151,9 @@ func (h *datasourceConnectHandler) UpdateSource(
 			Branch:         expected.GetBranch(),
 			Paths:          expected.GetPaths(),
 			FileExtensions: expected.GetFileExtensions(),
+
+			IncludePatterns: expected.GetIncludePatterns(),
+			ExcludePatterns: expected.GetExcludePatterns(),
 		}
 	}
 	source, jobID, err := h.svc.UpdateSource(ctx, actorID, input)
@@ -452,6 +462,10 @@ func datasourceSourceToProto(source *business.DatasourceSource, registry *connec
 			Prefix:      source.Upload.Prefix,
 			AccessKeyId: source.Upload.AccessKeyID,
 			MaxObjects:  uint32(source.Upload.MaxObjects),
+
+			IncludePatterns: source.Upload.IncludePatterns,
+			ExcludePatterns: source.Upload.ExcludePatterns,
+			FileExtensions:  source.Upload.FileExtensions,
 		}
 	}
 	if source.LastSyncedAt != nil {

module/services/accounts/code/pkg/business/audit_registry.go

@@ -846,8 +846,9 @@ var auditEventCatalog = []AuditEventDefinition{
 	// reads this declaration for. An edit never moves the boundary, but it is
 	// the collection whose contents the next snapshot replaces, so whoever may
 	// read that collection is exactly who needs to see it.
-	mutation(EventDatasourceSourceUpdated, CategorySystem, "A connected datasource's branch or path filters were changed.",
-		str("repo"), str("branch"), str("previous_branch"), strs("paths"), strs("file_extensions"), str("boundary")),
+	mutation(EventDatasourceSourceUpdated, CategorySystem, "A connected datasource's branch, path or key filters were changed.",
+		str("repo"), str("branch"), str("previous_branch"), strs("paths"), strs("file_extensions"), str("boundary"),
+		str("bucket"), str("prefix"), strs("include_patterns"), strs("exclude_patterns")),
 	observation(EventDatasourceSyncCompleted, CategorySystem, "A datasource ingestion job completed.", sourceSyncFields...),
 	observation(EventDatasourceSyncFailed, CategorySystem, "A datasource ingestion attempt failed and may retry.", sourceSyncFields...),
 	observation(EventDatasourceChangeSetCompiled, CategorySystem, "A GitHub delivery was compiled into a change set.",

module/services/accounts/code/pkg/business/datasource_connectors.go

@@ -71,7 +71,7 @@ func (s *Service) newDatasourceConnectorRegistry() *connector.Registry {
 	}))
 	must(r.RegisterNonConformant(connector.Descriptor{
 		Key: DatasourceProviderUpload, DisplayName: "Object storage",
-		Description:     "An S3-compatible bucket; objects under a prefix are pulled on sync.",
+		Description:     "An S3-compatible bucket; the objects under a prefix that pass its key filter are pulled on sync. A source is one collection with one set of readers, so a bucket shared by several audiences is several sources.",
 		Interface:       connector.InterfaceFiles,
 		CredentialModes: []connector.CredentialMode{connector.CredentialStaticSecret},
 		Gap:             gapUploadDatasource,

module/services/accounts/code/pkg/business/datasource_deliveries.go

@@ -1032,7 +1032,8 @@ func forcedReconcileMode(mode string) bool {
 // connecting is an explicit act, and a source reconnected after being deleted
 // must not be dropped as a replay of the first one's schedule.
 func (s *Service) enqueueFirstPullSync(ctx context.Context, source *DatasourceSource) error {
-	return s.enqueuePullSync(ctx, source, datasourceSyncRequestSource, NewIDString())
+	_, err := s.enqueuePullSync(ctx, source, datasourceSyncRequestSource, NewIDString())
+	return err
 }
 
 // enqueueScheduledPullSync is the periodic reconcile for a provider the host
@@ -1043,15 +1044,16 @@ func (s *Service) enqueueFirstPullSync(ctx context.Context, source *DatasourceSo
 //
 // STOPGAP (issue #1112).
 func (s *Service) enqueueScheduledPullSync(ctx context.Context, source *DatasourceSource) error {
-	return s.enqueuePullSync(ctx, source, DatasourceScheduledSyncSource, scheduledReconcileKey(source))
+	_, err := s.enqueuePullSync(ctx, source, DatasourceScheduledSyncSource, scheduledReconcileKey(source))
+	return err
 }
 
 // enqueuePullSync enqueues one pull-provider sync request, the job the leased
 // sync worker performs through RunDatasourceSync. It is exactly the job
 // SyncDatasourceSource enqueues for these providers; only provenance and
-// idempotency key vary by caller.
-func (s *Service) enqueuePullSync(ctx context.Context, source *DatasourceSource, provenance, idempotencyKey string) error {
-	_, err := s.datasourceJobs.EnqueueJob(ctx, &jobsv1.EnqueueJobRequest{
+// idempotency key vary by caller. It returns the job's id.
+func (s *Service) enqueuePullSync(ctx context.Context, source *DatasourceSource, provenance, idempotencyKey string) (string, error) {
+	response, err := s.datasourceJobs.EnqueueJob(ctx, &jobsv1.EnqueueJobRequest{
 		Job: &jobsv1.NewJob{
 			Direction:      jobsv1.JobDirection_JOB_DIRECTION_INBOX,
 			Scope:          &jobsv1.JobScope{Value: &jobsv1.JobScope_OrganizationId{OrganizationId: source.OrgID}},
@@ -1067,7 +1069,10 @@ func (s *Service) enqueuePullSync(ctx context.Context, source *DatasourceSource,
 			Attributes:     map[string]string{attrSourceID: source.ID},
 		},
 	})
-	return err
+	if err != nil {
+		return "", err
+	}
+	return response.GetJobId(), nil
 }
 
 // pathInScope reports whether p is under one of the source's path prefixes at a