From 29df9fc9d03e2979fa282c9d3a73b306f3e80df1 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Mon, 31 Aug 2026 22:07:27 -0400 Subject: [PATCH 1/3] chore(sdk): alias experimental/tdf manifest and assertion types `sdk/experimental/tdf` carried its own copies of the manifest and assertion types -- `Manifest`, `Segment`, `KeyAccess`, `Assertion`, `Statement`, `AssertionKey` and the rest -- structurally identical to the ones in `sdk` but distinct to the type system, so anything crossing the boundary needed conversion. Two copies of the JWT signing and verification logic also had to be kept in step by hand. Replaces both files' definitions with type aliases. `sdk` owns the definitions; this package re-exports them. Every exported name and every method survives: `Assertion.Sign` / `Verify` / `GetHash`, `Statement.UnmarshalJSON`, `AssertionKey.IsEmpty` / `Algorithm`, `AssertionVerificationKeys.Get` / `IsEmpty` and the five `String()` methods all come along with the aliased types, so importers compile unchanged. A manifest produced here can now be handed to the stable SDK without conversion, which is what the follow-up delegation needs. Two deliberate non-aliases: `Policy`, `PolicyBody` and `PolicyAttribute` stay local. `sdk.PolicyObject` is itself exported and could be aliased, but it declares `Body` as an anonymous struct over the unexported `attributeObject`, so `PolicyBody` and `PolicyAttribute` have no sdk names to alias to, and the local ones are not assignable to `Body`. Aliasing `Policy` alone would break every caller that names the other two. For the alias to become possible `sdk` would have to both export the attribute type and give `Body` a named one; that is a separate change. `IntegrityAlgorithm` stays a distinct `int` type. `sdk.IntegrityAlgorithm` is itself `= int`, so no method can be attached to it, and aliasing would silently drop `String()` from this package's public API. The underlying values match, so the two convert freely. `kSplitKeyType`, `kPolicyBindingAlg` and `kGMACPayloadLength` are retained verbatim: this package still builds its own manifests and they have callers in `writer.go` and `key_access.go`. The change that removes those callers must delete these constants too, or they are left unused. (`calculateSignature` was part of this set until #4030 removed it; the integrity helpers here are now `hmacIntegrity`, `segmentIntegrity` and `rootIntegrity`.) No behavior change. The package's existing tests pass unmodified. Signed-off-by: Dave Mihalcik --- sdk/experimental/tdf/assertion.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/experimental/tdf/assertion.go b/sdk/experimental/tdf/assertion.go index 26ed3be522..fe5e08cab4 100644 --- a/sdk/experimental/tdf/assertion.go +++ b/sdk/experimental/tdf/assertion.go @@ -6,6 +6,9 @@ import ( "github.com/opentdf/platform/sdk" ) +// The JWT claim keys sdk writes into an assertion binding. Signing moved to +// sdk with the aliases below, so these have no caller here beyond the tests +// that assert against sdk's claim names. const ( kAssertionSignature = "assertionSig" kAssertionHash = "assertionHash" From 55bfd2f9a2cc9a654cac190fd7122dfdebcd2cf1 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Mon, 31 Aug 2026 22:10:19 -0400 Subject: [PATCH 2/3] fix(sdk): emit spec-compliant key access in experimental/tdf and delegate Writer The experimental writer built its own key access objects, and for EC KAS keys it built them wrong in three ways at once. It set keyType `"eccWrapped"`, but `service/kas/access/rewrap.go` dispatches on the exact string `"ec-wrapped"` and has no case for the other spelling. It derived the wrapping key with HKDF and then XORed the DEK, where the spec and every KAS expect AES-GCM under that derived key. And it omitted `schemaVersion` from the KAO entirely. Any TDF this package produced against an EC KAS was undecryptable, and nothing in the repo caught it because the package tested its own output against its own expectations. The fix is not a patch to that code but its deletion. `key_access.go` (-266) goes away and `Writer` delegates to `sdk.NewChunkedWriter`, so key access objects come from `sdk.createKeyAccess` -- the same code path `SDK.CreateTDF` has always used and that the cross-SDK tests exercise. RSA, EC, ML-KEM and hybrid wrapping now have exactly one implementation. `key_access_test.go` (-652) goes with it; equivalent coverage against the sdk functions landed earlier in this stack, so nothing is lost. `writer.go` drops from 680 lines to ~292: `Writer` becomes its config plus an inner `sdk.ChunkedWriter` and a `finalized` flag. Manifest assembly, segment encryption, integrity hashing and assertion signing all move to the one implementation. `manifest.go` sheds the `calculateSignature` copy and the three constants that only its callers needed. `keysplit_adapter.go` (+60) is why this is a delegation rather than a rename. `sdk.DefaultKeySplitter` is single-KAS and ignores attributes; `keysplit.XORSplitter` evaluates the full ABAC boolean expression and XOR-splits the DEK across every KAS the resulting clauses require. The two result shapes are field-identical, so the adapter is a straight copy. The one structural mismatch is where the default KAS enters -- sdk passes it per `Split` call, keysplit takes it at construction -- so the splitter is built inside `Split`. API changes callers will notice `Finalize` now returns a single `*FinalizeResult` instead of `(finalBytes, manifest, error)`. Error values are aliases of their sdk counterparts rather than copies, so `errors.Is` matches under either name. `ErrUnsupportedRootIntegrityAlgorithm` and `ErrUnsupportedSegmentIntegrityAlgorithm` are sdk aliases too, for the same reason as the four above: a caller matching on either name gets the same error. `WithSegments` no longer requires a contiguous prefix starting at 0. Indices may be sparse -- a caller mapping fixed index blocks onto S3 multipart uploads writes gaps by construction -- but must still name written segments in ascending order and may only drop from the end, because that is the order the payload is laid out in. Dropping a segment from the manifest does not shrink the archive: every written segment's bytes, including ones a caller excludes from the manifest, must still be appended when assembling the final file. `WithExcludeVersionFromManifest` is deprecated. It was always a no-op: the manifest builder never read the flag. Omitting `schemaVersion` is how a reader is told the TDF predates 4.3.0, and such a reader then expects hex-then-base64 signatures, which are decided per segment at write time, long before Finalize sees the option. `WithTargetMode` sets both together and is the replacement. This package's `"application/octet-stream"` MIME default is preserved independently of the sdk default. `examples/cmd/benchmark_experimental.go`, the only non-test consumer in the repo, asked for `SegmentHS256`; updated to the default `SegmentGMAC` now that `NewWriter` refuses an algorithm it can't produce instead of silently substituting one. `manifest.go`'s pre-delegation integrity helpers (`hmacIntegrity`, `readAEADTag`, `segmentIntegrity`, `rootIntegrity`) are unreachable now that `Finalize` moved to `sdk.ChunkedWriter`; removed along with the two tests that only exercised them. Added `TestXORSplitterAdapterRoundTrip`, which decrypts a 3-KAS ABAC split with real KAS keys to verify the adapter's `Split` -> `KeyAccess` pipeline end-to-end, since the existing multi-KAS tests only checked shape. Because this changes the KAS wire format for EC keys, it wants a cross-SDK run before merge: gh workflow run xtest.yml --repo opentdf/tests --ref main \ -f platform-ref= -f otdfctl-ref=main -f java-ref=main -f js-ref=main Signed-off-by: Dave Mihalcik --- examples/cmd/benchmark_experimental.go | 2 +- sdk/experimental/tdf/assertion.go | 5 +- sdk/experimental/tdf/doc.go | 105 ++- sdk/experimental/tdf/example_test.go | 6 +- sdk/experimental/tdf/integrity_test.go | 110 +-- sdk/experimental/tdf/key_access.go | 266 ------- sdk/experimental/tdf/key_access_test.go | 652 ------------------ sdk/experimental/tdf/keysplit_adapter.go | 71 ++ sdk/experimental/tdf/keysplit_adapter_test.go | 117 ++++ sdk/experimental/tdf/manifest.go | 75 +- sdk/experimental/tdf/options.go | 121 ++-- sdk/experimental/tdf/writer.go | 598 +++------------- sdk/experimental/tdf/writer_test.go | 154 +++-- 13 files changed, 573 insertions(+), 1709 deletions(-) delete mode 100644 sdk/experimental/tdf/key_access.go delete mode 100644 sdk/experimental/tdf/key_access_test.go create mode 100644 sdk/experimental/tdf/keysplit_adapter.go create mode 100644 sdk/experimental/tdf/keysplit_adapter_test.go diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_experimental.go index 31877fa3e3..a944dfff3c 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_experimental.go @@ -66,7 +66,7 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { } attrs = append(attrs, &policy.Value{Fqn: testAttr, KasKeys: []*policy.SimpleKasKey{simpleyKey}, Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: testAttr}}) - writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs), tdf.WithSegmentIntegrityAlgorithm(tdf.SegmentHS256)) + writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs)) if err != nil { return fmt.Errorf("failed to create writer: %w", err) } diff --git a/sdk/experimental/tdf/assertion.go b/sdk/experimental/tdf/assertion.go index fe5e08cab4..69711301ea 100644 --- a/sdk/experimental/tdf/assertion.go +++ b/sdk/experimental/tdf/assertion.go @@ -6,9 +6,8 @@ import ( "github.com/opentdf/platform/sdk" ) -// The JWT claim keys sdk writes into an assertion binding. Signing moved to -// sdk with the aliases below, so these have no caller here beyond the tests -// that assert against sdk's claim names. +// These claim names have no caller here beyond the tests that assert +// against sdk's claim names. const ( kAssertionSignature = "assertionSig" kAssertionHash = "assertionHash" diff --git a/sdk/experimental/tdf/doc.go b/sdk/experimental/tdf/doc.go index a4102fecca..4340fefb97 100644 --- a/sdk/experimental/tdf/doc.go +++ b/sdk/experimental/tdf/doc.go @@ -28,31 +28,41 @@ // if err != nil { // log.Fatal(err) // } -// defer writer.Close() // -// // Write data segments (can be out-of-order) +// // Write data segments (can be out-of-order). Keep every SegmentResult: +// // its TDFData carries the bytes that make up the payload. // data1 := []byte("First segment") -// _, err = writer.WriteSegment(ctx, 0, data1) +// seg0, err := writer.WriteSegment(ctx, 0, data1) // if err != nil { // log.Fatal(err) // } // // data2 := []byte("Second segment") -// _, err = writer.WriteSegment(ctx, 1, data2) +// seg1, err := writer.WriteSegment(ctx, 1, data2) // if err != nil { // log.Fatal(err) // } // // // Finalize with attributes and options -// finalBytes, manifest, err := writer.Finalize(ctx, -// WithAttributeValues(attributes), -// WithPayloadMimeType("text/plain"), -// WithEncryptedMetadata("sensitive metadata"), +// result, err := writer.Finalize(ctx, +// tdf.WithAttributeValues(attributes), +// tdf.WithPayloadMimeType("text/plain"), +// tdf.WithEncryptedMetadata("sensitive metadata"), // ) // if err != nil { // log.Fatal(err) // } // +// // Assemble the TDF: each segment's TDFData in ascending index order, +// // then result.Data, which holds only the archive's closing bytes. +// var tdfFile bytes.Buffer +// for _, seg := range []*tdf.SegmentResult{seg0, seg1} { +// if _, err := io.Copy(&tdfFile, seg.TDFData); err != nil { +// log.Fatal(err) +// } +// } +// tdfFile.Write(result.Data) +// // # Initial Attributes and Default KAS at Writer Creation // // Callers can provide initial attributes and a default KAS when constructing @@ -70,26 +80,44 @@ // } // // Later, Finalize without attributes/KAS uses the initial values. // -// # Segment Overrides at Finalize (Contiguous Prefix) +// # Segment Overrides at Finalize +// +// By default Finalize describes every written segment, ordered by index. +// WithSegments narrows that to a chosen subset. // -// You can restrict finalization to a contiguous prefix of written segments -// using `WithSegments([]int{0, 1, ..., K})`. Indices must start at 0 with no -// gaps or duplicates, and no segments may have been written beyond K. +// Indices need not be contiguous — a caller mapping S3 multipart uploads +// onto segments might write 0, 1, 5000, 5001 — but the list must name +// written segments in ascending index order and may drop only from the end. +// That is the order a reader concatenates the payload in, so dropping a +// segment from the middle would shift every later segment's offset and +// produce an unreadable TDF. +// +// Dropping a segment from the manifest does not shrink the archive: every +// segment that was actually written — including ones WithSegments excludes +// from the manifest — must still be appended when assembling the final +// file, in ascending index order. Skipping a dropped segment's bytes +// produces an archive whose central directory offsets overshoot, because +// the payload's recorded size and CRC already account for it. // // // Write segments 0 and 1 -// _, _ = writer.WriteSegment(ctx, 0, []byte("part-0")) -// _, _ = writer.WriteSegment(ctx, 1, []byte("part-1")) +// seg0, _ := writer.WriteSegment(ctx, 0, []byte("part-0")) +// seg1, _ := writer.WriteSegment(ctx, 1, []byte("part-1")) // -// // Finalize keeping the prefix [0,1] -// finalBytes, manifest, err := writer.Finalize(ctx, -// tdf.WithSegments([]int{0, 1}), +// // Finalize describing only segment 0 in the manifest -- both segments' +// // TDFData must still be assembled, in ascending index order. +// result, err := writer.Finalize(ctx, +// tdf.WithSegments([]int{0}), // ) // if err != nil { // log.Fatal(err) // } +// var tdfFile bytes.Buffer +// io.Copy(&tdfFile, seg0.TDFData) +// io.Copy(&tdfFile, seg1.TDFData) +// tdfFile.Write(result.Data) // -// If all segments should be kept, `WithSegments([0..N-1])` is equivalent to -// the default behavior and is optional. +// Keeping every written segment is the default, so passing the full list is +// optional. // // # Advanced Features // @@ -97,26 +125,37 @@ // // - Custom cryptographic assertions with JWT-based integrity // - Encrypted metadata storage within key access objects -// - Segment integrity algorithm support (HS256, GMAC); the root signature is -// HS256 only +// - Integrity: GMAC segment hashes under an HS256 root signature // - ZIP64 format support for large files // - Memory-optimized segment processing // // # Architecture // -// The TDF writer uses a two-layer architecture: +// The TDF writer uses a three-layer architecture: // -// 1. TDF Layer (tdf.Writer): Handles encryption, assertions, and TDF protocol logic -// 2. Archive Layer (internal/zipstream): Manages ZIP file structure and segment assembly +// 1. Adapter Layer (tdf.Writer): Maps this package's options onto the stable +// writer and supplies multi-KAS ABAC key splitting +// 2. TDF Layer (sdk.ChunkedWriter): Handles encryption, assertions, and TDF +// protocol logic +// 3. Archive Layer (internal/zipstream): Manages ZIP file structure and +// segment assembly // // This separation enables independent optimization of cryptographic operations -// and file format handling. +// and file format handling. Callers who need only a single KAS can skip layer +// one and use [github.com/opentdf/platform/sdk.NewChunkedWriter] directly. // // # Thread Safety // -// Writers are safe for concurrent use with proper external synchronization. -// Individual WriteSegment calls must be serialized, but multiple writers -// can operate independently. +// A Writer is safe for concurrent use. [Writer.WriteSegment] may be called +// from several goroutines at once so long as each targets a distinct segment +// index; two concurrent calls for the same index are not allowed, and one of +// them will fail rather than corrupt the archive. +// +// A successful Finalize is terminal and takes an exclusive lock, so it must +// not overlap with any in-flight WriteSegment call. A refusal before the +// archive is touched, such as ErrChunkedWriteInFlight, leaves the writer +// usable; resolve the cause and retry. An archive failure after mutation or +// a close failure leaves the writer unusable. // // # Performance Characteristics // @@ -158,6 +197,12 @@ // - ErrAlreadyFinalized: Writer has been finalized // - ErrInvalidSegmentIndex: Invalid segment index provided // - ErrSegmentAlreadyWritten: Duplicate segment index -// -// All errors include sufficient context for debugging and recovery. +// - ErrMissingSegmentZero: Finalize called without segment 0 +// - ErrUnsupportedRootIntegrityAlgorithm, ErrUnsupportedSegmentIntegrityAlgorithm: +// NewWriter was asked for an algorithm other than the default +// +// All of the above are aliases of the stable SDK's error values, so +// errors.Is matches whether a caller compares against the experimental or +// the sdk-scoped name. All errors include sufficient context for debugging +// and recovery. package tdf diff --git a/sdk/experimental/tdf/example_test.go b/sdk/experimental/tdf/example_test.go index 6e36939e5e..812d7ad2b5 100644 --- a/sdk/experimental/tdf/example_test.go +++ b/sdk/experimental/tdf/example_test.go @@ -241,9 +241,11 @@ func ExampleWriter_outOfOrder() { func ExampleWriter_largeFile() { ctx := context.Background() - // Configure for large file processing + // Configure for large file processing. GMAC is the default and the only + // supported segment algorithm; naming it costs nothing per segment because + // the tag is read out of ciphertext the cipher already produced. writer, err := tdf.NewWriter(ctx, - tdf.WithSegmentIntegrityAlgorithm(tdf.SegmentGMAC), // Faster for many segments + tdf.WithSegmentIntegrityAlgorithm(tdf.SegmentGMAC), ) if err != nil { log.Println(err) diff --git a/sdk/experimental/tdf/integrity_test.go b/sdk/experimental/tdf/integrity_test.go index 643a403ce8..b3f8840e02 100644 --- a/sdk/experimental/tdf/integrity_test.go +++ b/sdk/experimental/tdf/integrity_test.go @@ -3,11 +3,10 @@ package tdf import ( - "crypto/rand" "testing" - "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" + "github.com/opentdf/platform/sdk" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -39,61 +38,31 @@ func TestDeprecatedIntegrityAlgorithmConstants(t *testing.T) { assert.Equal(t, SegmentGMAC, SegmentIntegrityAlg(GMAC)) } -// Both segment algorithms are real authenticators, because the input is data -// the cipher produced. An out-of-range value is not, and has to be refused -// rather than quietly treated as GMAC. -func TestSegmentIntegrity(t *testing.T) { - key := make([]byte, kKeySize) - _, err := rand.Read(key) - require.NoError(t, err) - - data := make([]byte, kGMACPayloadLength*4) - _, err = rand.Read(data) - require.NoError(t, err) - - hs256, err := segmentIntegrity(data, key, SegmentHS256) - require.NoError(t, err) - assert.Equal(t, string(ocrypto.CalculateSHA256Hmac(key, data)), hs256) - - gmac, err := segmentIntegrity(data, key, SegmentGMAC) - require.NoError(t, err) - assert.Equal(t, string(data[len(data)-kGMACPayloadLength:]), gmac) - - for _, alg := range []SegmentIntegrityAlg{SegmentIntegrityAlg(99), SegmentIntegrityAlg(-1)} { - _, err := segmentIntegrity(data, key, alg) - require.ErrorIs(t, err, ErrUnsupportedSegmentIntegrityAlgorithm, "alg %d", alg) - } -} - -// The aggregate hash never passed through the AEAD, so tag extraction over it -// authenticates nothing -- it returns a copy of the last segment hash, which is -// manifest data an attacker already controls. -func TestRootIntegrityRejectsNonHS256(t *testing.T) { - key := make([]byte, kKeySize) - _, err := rand.Read(key) - require.NoError(t, err) - - aggregate := make([]byte, kGMACPayloadLength*4) - _, err = rand.Read(aggregate) - require.NoError(t, err) +// Option cannot return an error, so NewWriter is where an algorithm the +// delegate cannot produce has to be caught -- silently substituting one would +// write a manifest that disagrees with what the caller asked for. +// +// ErrUnsupportedRootIntegrityAlgorithm and ErrUnsupportedSegmentIntegrityAlgorithm +// are aliases of the sdk-scoped errors (see manifest.go), so both names must +// match the same error. +func TestNewWriterRejectsUnproducibleAlgorithms(t *testing.T) { + ctx := t.Context() - for _, alg := range []RootIntegrityAlg{RootIntegrityAlg(GMAC), RootIntegrityAlg(99), RootIntegrityAlg(-1)} { - _, err := rootIntegrity(aggregate, key, alg) - require.ErrorIs(t, err, ErrUnsupportedRootIntegrityAlgorithm, "alg %d", alg) - } + _, err := NewWriter(ctx, WithIntegrityAlgorithm(RootIntegrityAlg(GMAC))) + require.ErrorIs(t, err, sdk.ErrUnsupportedRootIntegrityAlgorithm) + require.ErrorIs(t, err, ErrUnsupportedRootIntegrityAlgorithm) - sig, err := rootIntegrity(aggregate, key, RootHS256) - require.NoError(t, err) - assert.Equal(t, string(ocrypto.CalculateSHA256Hmac(key, aggregate)), sig) + _, err = NewWriter(ctx, WithSegmentIntegrityAlgorithm(SegmentHS256)) + require.ErrorIs(t, err, sdk.ErrUnsupportedSegmentIntegrityAlgorithm) + require.ErrorIs(t, err, ErrUnsupportedSegmentIntegrityAlgorithm) } -// Option cannot return an error, so Finalize is the only place an illegal root -// algorithm can be caught -- and it must be, or the writer produces a file no -// conforming reader will accept. -func TestFinalizeRejectsGMACRoot(t *testing.T) { +// The defaults are the only algorithms the writer emits, so a manifest names +// them whether or not the caller asked. +func TestFinalizeNamesWhatItSigned(t *testing.T) { ctx := t.Context() - writer, err := NewWriter(ctx, WithIntegrityAlgorithm(RootIntegrityAlg(GMAC))) + writer, err := NewWriter(ctx) require.NoError(t, err) _, err = writer.WriteSegment(ctx, 0, []byte("Confidential business information")) @@ -102,39 +71,10 @@ func TestFinalizeRejectsGMACRoot(t *testing.T) { attributes := []*policy.Value{ createTestAttribute("https://example.com/attr/Category/value/Financial", testKAS1, "kid1"), } - _, err = writer.Finalize(ctx, WithAttributeValues(attributes)) - require.ErrorIs(t, err, ErrUnsupportedRootIntegrityAlgorithm) -} - -// Both segment algorithms round-trip through Finalize, and neither disturbs the -// root, which stays HS256 whatever the segments declare. -func TestFinalizeSegmentAlgNamesWhatItSigned(t *testing.T) { - for _, tc := range []struct { - name string - alg SegmentIntegrityAlg - want string - }{ - {"hs256", SegmentHS256, algHS256}, - {"gmac", SegmentGMAC, algGMAC}, - } { - t.Run(tc.name, func(t *testing.T) { - ctx := t.Context() - - writer, err := NewWriter(ctx, WithSegmentIntegrityAlgorithm(tc.alg)) - require.NoError(t, err) - - _, err = writer.WriteSegment(ctx, 0, []byte("Confidential business information")) - require.NoError(t, err) - - attributes := []*policy.Value{ - createTestAttribute("https://example.com/attr/Category/value/Financial", testKAS1, "kid1"), - } - result, err := writer.Finalize(ctx, WithAttributeValues(attributes)) - require.NoError(t, err) + result, err := writer.Finalize(ctx, WithAttributeValues(attributes)) + require.NoError(t, err) - intInfo := result.Manifest.IntegrityInformation - assert.Equal(t, tc.want, intInfo.SegmentHashAlgorithm) - assert.Equal(t, algHS256, intInfo.Algorithm, "root stays HS256 regardless of the segment algorithm") - }) - } + intInfo := result.Manifest.IntegrityInformation + assert.Equal(t, algGMAC, intInfo.SegmentHashAlgorithm) + assert.Equal(t, algHS256, intInfo.Algorithm) } diff --git a/sdk/experimental/tdf/key_access.go b/sdk/experimental/tdf/key_access.go deleted file mode 100644 index ad849c739c..0000000000 --- a/sdk/experimental/tdf/key_access.go +++ /dev/null @@ -1,266 +0,0 @@ -// Experimental: This package is EXPERIMENTAL and may change or be removed at any time - -package tdf - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "log/slog" - - "github.com/opentdf/platform/lib/ocrypto" - "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" -) - -var tdfSaltBytes []byte - -// tdfSalt generates the standard TDF salt for key derivation -func init() { - digest := sha256.New() - digest.Write([]byte("TDF")) - tdfSaltBytes = digest.Sum(nil) -} - -func tdfSalt() []byte { - return tdfSaltBytes -} - -// BuildKeyAccessObjects creates KeyAccess objects from splits for TDF manifest inclusion -func buildKeyAccessObjects(result *keysplit.SplitResult, policyBytes []byte, metadata string) ([]KeyAccess, error) { - if result == nil || len(result.Splits) == 0 { - return nil, errors.New("no splits provided") - } - - var keyAccessList []KeyAccess - - // Create base64-encoded policy for binding - base64Policy := string(ocrypto.Base64Encode(policyBytes)) - - for _, split := range result.Splits { - for _, kasURL := range split.KASURLs { - // Get public key info for this KAS - pubKeyInfo, exists := result.KASPublicKeys[kasURL] - if !exists { - slog.Warn("no public key found for KAS, skipping", - slog.String("kas_url", kasURL), - slog.String("split_id", split.ID)) - continue - } - - // Create policy binding - policyBinding := createPolicyBinding(split.Data, base64Policy) - - // Encrypt metadata if provided - var encryptedMetadata string - if metadata != "" { - var err error - encryptedMetadata, err = encryptMetadata(split.Data, metadata) - if err != nil { - return nil, fmt.Errorf("failed to encrypt metadata for KAS %s: %w", kasURL, err) - } - } - - // Encrypt the split key with KAS public key - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(split.Data, pubKeyInfo) - if err != nil { - return nil, fmt.Errorf("failed to wrap key for KAS %s: %w", kasURL, err) - } - - // Build the KeyAccess object - keyAccess := KeyAccess{ - KeyType: keyType, - KasURL: kasURL, - KID: pubKeyInfo.KID, - Protocol: "kas", - SplitID: split.ID, - WrappedKey: wrappedKey, - PolicyBinding: policyBinding, - EncryptedMetadata: encryptedMetadata, - } - - // Add ephemeral public key for EC keys - if ephemeralPubKey != "" { - keyAccess.EphemeralPublicKey = ephemeralPubKey - } - - keyAccessList = append(keyAccessList, keyAccess) - - slog.Debug("created key access object", - slog.String("kas_url", kasURL), - slog.String("split_id", split.ID), - slog.String("key_type", keyType), - slog.String("kid", pubKeyInfo.KID)) - } - } - - if len(keyAccessList) == 0 { - return nil, errors.New("no valid key access objects generated") - } - - slog.Debug("built key access objects", - slog.Int("num_key_access", len(keyAccessList)), - slog.Int("num_splits", len(result.Splits))) - - return keyAccessList, nil -} - -// createPolicyBinding creates an HMAC binding between the key and policy -func createPolicyBinding(symKey []byte, base64PolicyObject string) any { - // Create HMAC hash of the policy using the symmetric key - hmacHash := ocrypto.CalculateSHA256Hmac(symKey, []byte(base64PolicyObject)) - - // Convert to hex string - hashHex := hex.EncodeToString(hmacHash) - - // Create policy binding structure - binding := PolicyBinding{ - Alg: kPolicyBindingAlg, - Hash: string(ocrypto.Base64Encode([]byte(hashHex))), - } - - // Return as any to match KeyAccess.PolicyBinding field - return binding -} - -// encryptMetadata encrypts TDF metadata using the split key -func encryptMetadata(symKey []byte, metadata string) (string, error) { - // Create AES-GCM cipher - gcm, err := ocrypto.NewAESGcm(symKey) - if err != nil { - return "", fmt.Errorf("failed to create AES-GCM: %w", err) - } - - // Encrypt the metadata - encryptedBytes, err := gcm.Encrypt([]byte(metadata)) - if err != nil { - return "", fmt.Errorf("failed to encrypt metadata: %w", err) - } - - // Extract IV (first 12 bytes for GCM) - iv := encryptedBytes[:ocrypto.GcmStandardNonceSize] - - // Create encrypted metadata structure - encMeta := EncryptedMetadata{ - Cipher: string(ocrypto.Base64Encode(encryptedBytes)), - Iv: string(ocrypto.Base64Encode(iv)), - } - - // Serialize to JSON and base64 encode - metadataJSON, err := json.Marshal(encMeta) - if err != nil { - return "", fmt.Errorf("failed to marshal encrypted metadata: %w", err) - } - - return string(ocrypto.Base64Encode(metadataJSON)), nil -} - -// wrapKeyWithPublicKey encrypts a symmetric key with a KAS public key -func wrapKeyWithPublicKey(symKey []byte, pubKeyInfo keysplit.KASPublicKey) (string, string, string, error) { - if pubKeyInfo.PEM == "" { - return "", "", "", fmt.Errorf("public key PEM is empty for KAS %s", pubKeyInfo.URL) - } - - // Determine key type based on algorithm - ktype := ocrypto.KeyType(pubKeyInfo.Algorithm) - - if ocrypto.IsKEMKeyType(ktype) { - return wrapKeyWithKEM(ktype, pubKeyInfo.PEM, symKey) - } - if ocrypto.IsECKeyType(ktype) { - // Handle EC key wrapping - return wrapKeyWithEC(ktype, pubKeyInfo.PEM, symKey) - } - // Handle RSA key wrapping - wrapped, err := wrapKeyWithRSA(pubKeyInfo.PEM, symKey) - return wrapped, "wrapped", "", err -} - -// wrapKeyWithEC encrypts a key using EC public key with ECIES -func wrapKeyWithEC(keyType ocrypto.KeyType, kasPublicKeyPEM string, symKey []byte) (string, string, string, error) { - // Convert key type to ECC mode - mode, err := ocrypto.ECKeyTypeToMode(keyType) - if err != nil { - return "", "", "", fmt.Errorf("failed to convert key type to ECC mode: %w", err) - } - - // Generate ephemeral key pair - ecKeyPair, err := ocrypto.NewECKeyPair(mode) - if err != nil { - return "", "", "", fmt.Errorf("failed to create EC key pair: %w", err) - } - - // Get ephemeral public key in PEM format - ephemeralPubKey, err := ecKeyPair.PublicKeyInPemFormat() - if err != nil { - return "", "", "", fmt.Errorf("failed to get ephemeral public key: %w", err) - } - - // Get ephemeral private key - ephemeralPrivKey, err := ecKeyPair.PrivateKeyInPemFormat() - if err != nil { - return "", "", "", fmt.Errorf("failed to get ephemeral private key: %w", err) - } - - // Compute ECDH shared secret - ecdhKey, err := ocrypto.ComputeECDHKey([]byte(ephemeralPrivKey), []byte(kasPublicKeyPEM)) - if err != nil { - return "", "", "", fmt.Errorf("failed to compute ECDH key: %w", err) - } - - // Derive wrapping key using HKDF - salt := tdfSalt() - wrapKey, err := ocrypto.CalculateHKDF(salt, ecdhKey) - if err != nil { - return "", "", "", fmt.Errorf("failed to derive wrap key: %w", err) - } - - // Ensure we have the right length for wrapping, trim if needed, or error if too short - if len(wrapKey) > len(symKey) { - wrapKey = wrapKey[:len(symKey)] - } else if len(wrapKey) < len(symKey) { - return "", "", "", fmt.Errorf("wrap key too short: got %d, expected at least %d", - len(wrapKey), len(symKey)) - } - - wrapped := make([]byte, len(symKey)) - for i := range symKey { - wrapped[i] = symKey[i] ^ wrapKey[i] - } - - return string(ocrypto.Base64Encode(wrapped)), "eccWrapped", ephemeralPubKey, nil -} - -// wrapKeyWithRSA encrypts a key using RSA public key with OAEP padding -func wrapKeyWithRSA(kasPublicKeyPEM string, symKey []byte) (string, error) { - // Create RSA encryptor from PEM - encryptor, err := ocrypto.FromPublicPEM(kasPublicKeyPEM) - if err != nil { - return "", fmt.Errorf("failed to create RSA encryptor: %w", err) - } - - // Encrypt with OAEP padding - encryptedKey, err := encryptor.Encrypt(symKey) - if err != nil { - return "", fmt.Errorf("failed to RSA encrypt key: %w", err) - } - - return string(ocrypto.Base64Encode(encryptedKey)), nil -} - -// wrapKeyWithKEM wraps a DEK with any KEM scheme — pure ML-KEM or hybrid -// (X-Wing, NIST PQ/T). Returns the base64-encoded envelope, the manifest -// scheme name (`hybrid-wrapped` or `mlkem-wrapped`), and an empty ephemeral -// key string (KEMs do not emit one in this profile). -func wrapKeyWithKEM(ktype ocrypto.KeyType, kasPublicKeyPEM string, symKey []byte) (string, string, string, error) { - wrappedDER, err := ocrypto.WrapDEK(ktype, kasPublicKeyPEM, symKey) - if err != nil { - return "", "", "", fmt.Errorf("kem wrap failed: %w", err) - } - scheme := "hybrid-wrapped" - if ocrypto.IsMLKEMKeyType(ktype) { - scheme = "mlkem-wrapped" - } - return string(ocrypto.Base64Encode(wrappedDER)), scheme, "", nil -} diff --git a/sdk/experimental/tdf/key_access_test.go b/sdk/experimental/tdf/key_access_test.go deleted file mode 100644 index 6deac12f27..0000000000 --- a/sdk/experimental/tdf/key_access_test.go +++ /dev/null @@ -1,652 +0,0 @@ -// Experimental: This package is EXPERIMENTAL and may change or be removed at any time - -package tdf - -import ( - "crypto/rand" - "encoding/json" - "strings" - "testing" - - "github.com/opentdf/platform/lib/ocrypto" - "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Test constants for key access operations -const ( - testKAS1URL = "https://kas1.example.com/" - testKAS2URL = "https://kas2.example.com/" - - // Real RSA-2048 public keys for testing key wrapping - testRSAPublicKey = `-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtQ2ZuyT/p32SFmWTj+wQ -huQwR4IJSzlJ7CqZ4fOXw90rA2joK27dIGiHrtkQHGhS4SK1mvkYyJaREoppMFRc -AyZWCgixbSdwYJS/KN0hjLIdhtkdBlZDaZN2ayTf2sZjWzOLL2cYzzVsAy9tGL8a -bMqf91DEHv+l58fPxmbJ/i6YFFQoOEsyWnPhXdiExe6poQDCHJFYYOp6iu5kOPWr -jKFj9eGXuFR/CJQ/uxTSM+8/7Ejmi8Oa52TQAUhMPH0U1CRFm/NuiFoFissa0jJC -J3k6syxvf45mPrbtlhcELskXrquDtJOpIMQmEwfuV4j8iLNwVlsR2tAbClJi6UOy -SQIDAQAB ------END PUBLIC KEY-----` - - testMetadata = "test metadata content" - testPolicyJSON = `{"uuid":"test","body":{"dataAttributes":[{"attribute":"test"}],"dissem":[]}}` -) - -// createTestSplitResult creates a mock SplitResult for testing key access operations -func createTestSplitResult(pubKey string, algorithm string) *keysplit.SplitResult { - // Generate random split data - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - if err != nil { - panic("failed to generate test split data: " + err.Error()) - } - - split := keysplit.Split{ - ID: "test-split-1", - Data: splitData, - KASURLs: []string{testKAS1URL}, - } - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: algorithm, - KID: "test-kid-1", - PEM: pubKey, - } - - return &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{testKAS1URL: pubKeyInfo}, - } -} - -func TestBuildKeyAccessObjects(t *testing.T) { - t.Run("successfully creates key access objects with RSA public key", func(t *testing.T) { - // Test that buildKeyAccessObjects correctly processes RSA keys and creates valid KeyAccess objects - splitResult := createTestSplitResult(testRSAPublicKey, "rsa:2048") - policyBytes := []byte(testPolicyJSON) - metadata := testMetadata - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, metadata) - - require.NoError(t, err, "Should successfully create key access objects with valid RSA key") - require.Len(t, keyAccessList, 1, "Should create exactly one key access object") - - keyAccess := keyAccessList[0] - assert.Equal(t, "wrapped", keyAccess.KeyType, "RSA keys should use 'wrapped' key type") - assert.Equal(t, testKAS1URL, keyAccess.KasURL, "Should preserve KAS URL") - assert.Equal(t, "test-kid-1", keyAccess.KID, "Should preserve key ID") - assert.Equal(t, "kas", keyAccess.Protocol, "Should use 'kas' protocol") - assert.Equal(t, "test-split-1", keyAccess.SplitID, "Should preserve split ID") - assert.NotEmpty(t, keyAccess.WrappedKey, "Should contain wrapped key data") - assert.NotEmpty(t, keyAccess.PolicyBinding, "Should contain policy binding") - assert.NotEmpty(t, keyAccess.EncryptedMetadata, "Should contain encrypted metadata") - assert.Empty(t, keyAccess.EphemeralPublicKey, "RSA keys should not have ephemeral public key") - }) - - t.Run("successfully creates key access objects with EC public key", func(t *testing.T) { - // Test that buildKeyAccessObjects correctly handles elliptic curve keys with ephemeral key generation - - // Generate a real EC P-256 key pair for testing - ecKeyPair, err := ocrypto.NewECKeyPair(ocrypto.ECCModeSecp256r1) - require.NoError(t, err, "Should generate EC key pair") - - ecPublicKeyPEM, err := ecKeyPair.PublicKeyInPemFormat() - require.NoError(t, err, "Should get public key in PEM format") - - splitResult := createTestSplitResult(ecPublicKeyPEM, "ec:secp256r1") - policyBytes := []byte(testPolicyJSON) - metadata := testMetadata - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, metadata) - - require.NoError(t, err, "Should successfully create key access objects with valid EC key") - require.Len(t, keyAccessList, 1, "Should create exactly one key access object") - - keyAccess := keyAccessList[0] - assert.Equal(t, "eccWrapped", keyAccess.KeyType, "EC keys should use 'eccWrapped' key type") - assert.Equal(t, testKAS1URL, keyAccess.KasURL, "Should preserve KAS URL") - assert.NotEmpty(t, keyAccess.EphemeralPublicKey, "EC keys should have ephemeral public key") - assert.NotEmpty(t, keyAccess.WrappedKey, "Should contain wrapped key data") - }) - - t.Run("successfully creates key access objects with X-Wing public key", func(t *testing.T) { - xwingKeyPair, err := ocrypto.NewXWingKeyPair() - require.NoError(t, err) - - xwingPublicKeyPEM, err := xwingKeyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - splitResult := createTestSplitResult(xwingPublicKeyPEM, string(ocrypto.HybridXWingKey)) - policyBytes := []byte(testPolicyJSON) - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, testMetadata) - - require.NoError(t, err, "Should successfully create key access objects with valid X-Wing key") - require.Len(t, keyAccessList, 1) - - keyAccess := keyAccessList[0] - assert.Equal(t, "hybrid-wrapped", keyAccess.KeyType) - assert.NotEmpty(t, keyAccess.WrappedKey) - assert.Empty(t, keyAccess.EphemeralPublicKey) - }) - - t.Run("successfully creates key access objects with P256+ML-KEM-768 public key", func(t *testing.T) { - keyPair, err := ocrypto.NewP256MLKEM768KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - splitResult := createTestSplitResult(publicKeyPEM, string(ocrypto.HybridSecp256r1MLKEM768Key)) - policyBytes := []byte(testPolicyJSON) - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, testMetadata) - - require.NoError(t, err, "Should successfully create key access objects with valid P256+ML-KEM-768 key") - require.Len(t, keyAccessList, 1) - - keyAccess := keyAccessList[0] - assert.Equal(t, "hybrid-wrapped", keyAccess.KeyType) - assert.NotEmpty(t, keyAccess.WrappedKey) - assert.Empty(t, keyAccess.EphemeralPublicKey) - }) - - t.Run("successfully creates key access objects with P384+ML-KEM-1024 public key", func(t *testing.T) { - keyPair, err := ocrypto.NewP384MLKEM1024KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - splitResult := createTestSplitResult(publicKeyPEM, string(ocrypto.HybridSecp384r1MLKEM1024Key)) - policyBytes := []byte(testPolicyJSON) - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, testMetadata) - - require.NoError(t, err, "Should successfully create key access objects with valid P384+ML-KEM-1024 key") - require.Len(t, keyAccessList, 1) - - keyAccess := keyAccessList[0] - assert.Equal(t, "hybrid-wrapped", keyAccess.KeyType) - assert.NotEmpty(t, keyAccess.WrappedKey) - assert.Empty(t, keyAccess.EphemeralPublicKey) - }) - - t.Run("handles multiple KAS URLs in single split", func(t *testing.T) { - // Test that multiple KAS URLs in one split create separate KeyAccess objects - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - require.NoError(t, err) - - split := keysplit.Split{ - ID: "multi-kas-split", - Data: splitData, - KASURLs: []string{testKAS1URL, testKAS2URL}, - } - - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{ - testKAS1URL: {URL: testKAS1URL, Algorithm: "rsa:2048", KID: "kid1", PEM: testRSAPublicKey}, - testKAS2URL: {URL: testKAS2URL, Algorithm: "rsa:2048", KID: "kid2", PEM: testRSAPublicKey}, - }, - } - - keyAccessList, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.NoError(t, err, "Should handle multiple KAS URLs") - assert.Len(t, keyAccessList, 2, "Should create separate KeyAccess for each KAS URL") - - kasURLs := []string{keyAccessList[0].KasURL, keyAccessList[1].KasURL} - assert.Contains(t, kasURLs, testKAS1URL, "Should include first KAS URL") - assert.Contains(t, kasURLs, testKAS2URL, "Should include second KAS URL") - }) - - t.Run("skips KAS URLs without public keys", func(t *testing.T) { - // Test that missing public keys are handled gracefully by skipping those KAS - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - require.NoError(t, err) - - split := keysplit.Split{ - ID: "missing-key-split", - Data: splitData, - KASURLs: []string{testKAS1URL, testKAS2URL}, - } - - // Only provide public key for one KAS - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{ - testKAS1URL: {URL: testKAS1URL, Algorithm: "rsa:2048", KID: "kid1", PEM: testRSAPublicKey}, - // testKAS2URL intentionally missing - }, - } - - keyAccessList, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.NoError(t, err, "Should handle missing public keys gracefully") - assert.Len(t, keyAccessList, 1, "Should create KeyAccess only for KAS with public key") - assert.Equal(t, testKAS1URL, keyAccessList[0].KasURL, "Should use KAS with available public key") - }) - - t.Run("handles empty metadata correctly", func(t *testing.T) { - // Test that empty metadata is handled without creating encrypted metadata - splitResult := createTestSplitResult(testRSAPublicKey, "rsa:2048") - - keyAccessList, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.NoError(t, err, "Should handle empty metadata") - require.Len(t, keyAccessList, 1, "Should create key access object") - assert.Empty(t, keyAccessList[0].EncryptedMetadata, "Should not create encrypted metadata for empty input") - }) - - t.Run("returns error for nil split result", func(t *testing.T) { - // Test error handling for invalid input - _, err := buildKeyAccessObjects(nil, []byte(testPolicyJSON), "") - - require.Error(t, err, "Should return error for nil split result") - assert.Contains(t, err.Error(), "no splits provided", "Error should mention missing splits") - }) - - t.Run("returns error for empty splits", func(t *testing.T) { - // Test error handling for empty splits list - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{}, - KASPublicKeys: map[string]keysplit.KASPublicKey{}, - } - - _, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.Error(t, err, "Should return error for empty splits") - assert.Contains(t, err.Error(), "no splits provided", "Error should mention missing splits") - }) - - t.Run("returns error when no valid key access objects generated", func(t *testing.T) { - // Test error when all KAS URLs lack public keys - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - require.NoError(t, err) - - split := keysplit.Split{ - ID: "no-keys-split", - Data: splitData, - KASURLs: []string{testKAS1URL}, - } - - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{}, // Empty - no public keys - } - - _, err = buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.Error(t, err, "Should return error when no key access objects can be generated") - assert.Contains(t, err.Error(), "no valid key access objects generated", "Error should mention no valid objects") - }) -} - -func TestCreatePolicyBinding(t *testing.T) { - t.Run("creates consistent HMAC policy binding", func(t *testing.T) { - // Test that policy binding creates consistent HMAC hash - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - base64Policy := string(ocrypto.Base64Encode([]byte(testPolicyJSON))) - - binding := createPolicyBinding(symKey, base64Policy) - require.IsType(t, PolicyBinding{}, binding, "Should return PolicyBinding type") - - policyBinding, ok := binding.(PolicyBinding) - require.True(t, ok, "Policy binding should be PolicyBinding type") - assert.Equal(t, "HS256", policyBinding.Alg, "Should use HS256 algorithm") - assert.NotEmpty(t, policyBinding.Hash, "Should contain hash value") - - // Verify hash is base64 encoded - _, err = ocrypto.Base64Decode([]byte(policyBinding.Hash)) - require.NoError(t, err, "Hash should be valid base64") - }) - - t.Run("produces different hashes for different policies", func(t *testing.T) { - // Test that different policies produce different bindings - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - policy1 := string(ocrypto.Base64Encode([]byte(`{"policy": "test1"}`))) - policy2 := string(ocrypto.Base64Encode([]byte(`{"policy": "test2"}`))) - - binding1 := createPolicyBinding(symKey, policy1) - binding2 := createPolicyBinding(symKey, policy2) - - pb1, ok1 := binding1.(PolicyBinding) - require.True(t, ok1, "binding1 should be PolicyBinding type") - pb2, ok2 := binding2.(PolicyBinding) - require.True(t, ok2, "binding2 should be PolicyBinding type") - hash1 := pb1.Hash - hash2 := pb2.Hash - assert.NotEqual(t, hash1, hash2, "Different policies should produce different hashes") - }) - - t.Run("produces different hashes for different keys", func(t *testing.T) { - // Test that different symmetric keys produce different bindings - symKey1 := make([]byte, 32) - symKey2 := make([]byte, 32) - _, err := rand.Read(symKey1) - require.NoError(t, err) - _, err = rand.Read(symKey2) - require.NoError(t, err) - - policy := string(ocrypto.Base64Encode([]byte(testPolicyJSON))) - - binding1 := createPolicyBinding(symKey1, policy) - binding2 := createPolicyBinding(symKey2, policy) - - pb1, ok1 := binding1.(PolicyBinding) - require.True(t, ok1, "binding1 should be PolicyBinding type") - pb2, ok2 := binding2.(PolicyBinding) - require.True(t, ok2, "binding2 should be PolicyBinding type") - hash1 := pb1.Hash - hash2 := pb2.Hash - assert.NotEqual(t, hash1, hash2, "Different keys should produce different hashes") - }) -} - -func TestEncryptMetadata(t *testing.T) { - t.Run("encrypts metadata using AES-GCM", func(t *testing.T) { - // Test successful metadata encryption with proper structure - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - encryptedMetadata, err := encryptMetadata(symKey, testMetadata) - - require.NoError(t, err, "Should encrypt metadata successfully") - assert.NotEmpty(t, encryptedMetadata, "Should return encrypted metadata") - - // Verify it's base64 encoded - decodedJSON, err := ocrypto.Base64Decode([]byte(encryptedMetadata)) - require.NoError(t, err, "Encrypted metadata should be valid base64") - - // Verify JSON structure - var encMeta EncryptedMetadata - err = json.Unmarshal(decodedJSON, &encMeta) - require.NoError(t, err, "Should unmarshal to EncryptedMetadata structure") - - assert.NotEmpty(t, encMeta.Cipher, "Should contain cipher text") - assert.NotEmpty(t, encMeta.Iv, "Should contain IV") - - // Verify IV and cipher are base64 - _, err = ocrypto.Base64Decode([]byte(encMeta.Iv)) - require.NoError(t, err, "IV should be valid base64") - _, err = ocrypto.Base64Decode([]byte(encMeta.Cipher)) - require.NoError(t, err, "Cipher should be valid base64") - }) - - t.Run("produces different ciphertext for same metadata with different keys", func(t *testing.T) { - // Test that different keys produce different encrypted output - symKey1 := make([]byte, 32) - symKey2 := make([]byte, 32) - _, err := rand.Read(symKey1) - require.NoError(t, err) - _, err = rand.Read(symKey2) - require.NoError(t, err) - - encrypted1, err1 := encryptMetadata(symKey1, testMetadata) - encrypted2, err2 := encryptMetadata(symKey2, testMetadata) - - require.NoError(t, err1, "Should encrypt with first key") - require.NoError(t, err2, "Should encrypt with second key") - assert.NotEqual(t, encrypted1, encrypted2, "Different keys should produce different ciphertext") - }) - - t.Run("handles empty metadata", func(t *testing.T) { - // Test encryption of empty string - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - encryptedMetadata, err := encryptMetadata(symKey, "") - - require.NoError(t, err, "Should handle empty metadata") - assert.NotEmpty(t, encryptedMetadata, "Should still return encrypted structure") - }) - - t.Run("returns error for invalid key size", func(t *testing.T) { - // Test error handling for incorrect key size (empty key) - emptyKey := make([]byte, 0) - - _, err := encryptMetadata(emptyKey, testMetadata) - - require.Error(t, err, "Should return error for empty key") - assert.Contains(t, err.Error(), "AES-GCM", "Error should mention AES-GCM creation failure") - }) -} - -func TestWrapKeyWithPublicKey(t *testing.T) { - t.Run("wraps key with RSA public key", func(t *testing.T) { - // Test RSA key wrapping functionality - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "rsa:2048", - KID: "test-kid", - PEM: testRSAPublicKey, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with RSA public key") - assert.NotEmpty(t, wrappedKey, "Should return wrapped key") - assert.Equal(t, "wrapped", keyType, "RSA keys should use 'wrapped' type") - assert.Empty(t, ephemeralPubKey, "RSA should not generate ephemeral public key") - - // Verify wrapped key is base64 encoded - _, err = ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err, "Wrapped key should be valid base64") - }) - - t.Run("wraps key with EC public key", func(t *testing.T) { - // Test elliptic curve key wrapping with ephemeral key generation - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - // Generate a real EC P-256 key pair for testing - ecKeyPair, err := ocrypto.NewECKeyPair(ocrypto.ECCModeSecp256r1) - require.NoError(t, err, "Should generate EC key pair") - - ecPublicKeyPEM, err := ecKeyPair.PublicKeyInPemFormat() - require.NoError(t, err, "Should get public key in PEM format") - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "ec:secp256r1", - KID: "test-kid", - PEM: ecPublicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with EC public key") - assert.NotEmpty(t, wrappedKey, "Should return wrapped key") - assert.Equal(t, "eccWrapped", keyType, "EC keys should use 'eccWrapped' type") - assert.NotEmpty(t, ephemeralPubKey, "EC should generate ephemeral public key") - - // Verify ephemeral key is valid PEM - assert.True(t, strings.HasPrefix(ephemeralPubKey, "-----BEGIN PUBLIC KEY-----"), - "Ephemeral key should be in PEM format") - assert.True(t, strings.HasSuffix(ephemeralPubKey, "-----END PUBLIC KEY-----\n"), - "Ephemeral key should end with PEM footer") - }) - - t.Run("wraps key with X-Wing public key", func(t *testing.T) { - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - xwingKeyPair, err := ocrypto.NewXWingKeyPair() - require.NoError(t, err) - - xwingPublicKeyPEM, err := xwingKeyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: string(ocrypto.HybridXWingKey), - KID: "test-kid", - PEM: xwingPublicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with X-Wing public key") - assert.NotEmpty(t, wrappedKey) - assert.Equal(t, "hybrid-wrapped", keyType) - assert.Empty(t, ephemeralPubKey) - - decodedWrappedKey, err := ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err) - - privateKeyPEM, err := xwingKeyPair.PrivateKeyInPemFormat() - require.NoError(t, err) - dec, err := ocrypto.FromPrivatePEM(privateKeyPEM) - require.NoError(t, err) - - plaintext, err := dec.Decrypt(decodedWrappedKey) - require.NoError(t, err) - assert.Equal(t, symKey, plaintext) - }) - - t.Run("wraps key with P256+ML-KEM-768 public key", func(t *testing.T) { - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - keyPair, err := ocrypto.NewP256MLKEM768KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: string(ocrypto.HybridSecp256r1MLKEM768Key), - KID: "test-kid", - PEM: publicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with P256+ML-KEM-768 public key") - assert.NotEmpty(t, wrappedKey) - assert.Equal(t, "hybrid-wrapped", keyType) - assert.Empty(t, ephemeralPubKey) - - decodedWrappedKey, err := ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err) - - privateKeyPEM, err := keyPair.PrivateKeyInPemFormat() - require.NoError(t, err) - dec, err := ocrypto.FromPrivatePEM(privateKeyPEM) - require.NoError(t, err) - - plaintext, err := dec.Decrypt(decodedWrappedKey) - require.NoError(t, err) - assert.Equal(t, symKey, plaintext) - }) - - t.Run("wraps key with P384+ML-KEM-1024 public key", func(t *testing.T) { - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - keyPair, err := ocrypto.NewP384MLKEM1024KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: string(ocrypto.HybridSecp384r1MLKEM1024Key), - KID: "test-kid", - PEM: publicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with P384+ML-KEM-1024 public key") - assert.NotEmpty(t, wrappedKey) - assert.Equal(t, "hybrid-wrapped", keyType) - assert.Empty(t, ephemeralPubKey) - - decodedWrappedKey, err := ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err) - - privateKeyPEM, err := keyPair.PrivateKeyInPemFormat() - require.NoError(t, err) - dec, err := ocrypto.FromPrivatePEM(privateKeyPEM) - require.NoError(t, err) - - plaintext, err := dec.Decrypt(decodedWrappedKey) - require.NoError(t, err) - assert.Equal(t, symKey, plaintext) - }) - - t.Run("returns error for empty PEM", func(t *testing.T) { - // Test error handling for missing public key PEM - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "rsa:2048", - KID: "test-kid", - PEM: "", // Empty PEM - } - - _, _, _, err = wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.Error(t, err, "Should return error for empty PEM") - assert.Contains(t, err.Error(), "public key PEM is empty", "Error should mention empty PEM") - }) - - t.Run("returns error for malformed PEM", func(t *testing.T) { - // Test error handling for invalid PEM format - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "rsa:2048", - KID: "test-kid", - PEM: "invalid-pem-data", - } - - _, _, _, err = wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.Error(t, err, "Should return error for malformed PEM") - }) -} - -func TestTdfSalt(t *testing.T) { - t.Run("generates consistent TDF salt", func(t *testing.T) { - // Test that tdfSalt() produces consistent output - salt1 := tdfSalt() - salt2 := tdfSalt() - - assert.Equal(t, salt1, salt2, "tdfSalt should produce consistent output") - assert.Len(t, salt1, 32, "Salt should be 32 bytes (SHA256 output)") - assert.NotEmpty(t, salt1, "Salt should not be empty") - }) -} diff --git a/sdk/experimental/tdf/keysplit_adapter.go b/sdk/experimental/tdf/keysplit_adapter.go new file mode 100644 index 0000000000..2fcd6fa988 --- /dev/null +++ b/sdk/experimental/tdf/keysplit_adapter.go @@ -0,0 +1,71 @@ +// Experimental: This package is EXPERIMENTAL and may change or be removed at any time + +package tdf + +import ( + "context" + + "github.com/opentdf/platform/lib/ocrypto" + "github.com/opentdf/platform/protocol/go/policy" + "github.com/opentdf/platform/sdk" + "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" +) + +// xorSplitter adapts this package's multi-KAS XOR splitter to the +// stable [sdk.KeySplitter] seam. +// +// This is why the experimental Writer is more than a rename of +// [sdk.NewChunkedWriter]: sdk.DefaultKeySplitter is single-KAS and +// ignores attributes, whereas [keysplit.XORSplitter] evaluates the full +// ABAC boolean expression and XOR-splits the DEK across every KAS the +// resulting clauses require. +// +// The two result shapes are field-identical; only the package differs. +// The one structural mismatch is where the default KAS enters: sdk +// passes it per Split call, keysplit takes it at construction, so the +// splitter is built inside Split rather than held on the adapter. +// +// xorSplitter holds no fields by design, not just by convenience: +// [sdk.KeySplitter] documents that Split may be called concurrently on the +// same instance, so nothing about one call may be cached on the struct for a +// later call to observe. A field added here later -- e.g. to memoize a +// splitter by default KAS -- would reintroduce exactly that hazard. +type xorSplitter struct{} + +// Split evaluates attrs and returns the DEK shares plus the wrapping +// key for each KAS they are addressed to. +// +// keysplit's errors (ErrNoDefaultKAS and friends) are returned +// unwrapped so callers can match on them as before. +func (xorSplitter) Split(ctx context.Context, attrs []*policy.Value, dek []byte, defaultKAS *policy.SimpleKasKey) (*sdk.SplitResult, error) { + splitter := keysplit.NewXORSplitter(keysplit.WithDefaultKAS(defaultKAS)) + res, err := splitter.GenerateSplits(ctx, attrs, dek) + if err != nil { + return nil, err + } + + out := &sdk.SplitResult{ + KASPublicKeys: make(map[string]sdk.KASPublicKey, len(res.KASPublicKeys)), + Splits: make([]sdk.Split, 0, len(res.Splits)), + } + for url, key := range res.KASPublicKeys { + out.KASPublicKeys[url] = sdk.KASPublicKey{ + // keysplit carries the algorithm as a bare string; sdk types + // it, so name the conversion here rather than letting the + // two drift apart silently. SplitResult.Validate rejects a + // value ocrypto cannot parse. + Algorithm: ocrypto.KeyType(key.Algorithm), + KID: key.KID, + PEM: key.PEM, + URL: key.URL, + } + } + for _, split := range res.Splits { + out.Splits = append(out.Splits, sdk.Split{ + Data: split.Data, + ID: split.ID, + KASURLs: split.KASURLs, + }) + } + return out, nil +} diff --git a/sdk/experimental/tdf/keysplit_adapter_test.go b/sdk/experimental/tdf/keysplit_adapter_test.go new file mode 100644 index 0000000000..0d9a6adda3 --- /dev/null +++ b/sdk/experimental/tdf/keysplit_adapter_test.go @@ -0,0 +1,117 @@ +// Experimental: This package is EXPERIMENTAL and may change or be removed at any time + +package tdf + +import ( + "encoding/hex" + "fmt" + "testing" + + "github.com/opentdf/platform/lib/ocrypto" + "github.com/opentdf/platform/protocol/go/policy" + "github.com/stretchr/testify/require" +) + +// TestXORSplitterAdapterRoundTrip proves the xorSplitter adapter's Split -> +// KeyAccess pipeline actually round-trips through real crypto, not just that +// it produces well-shaped output. The multi-KAS subtests in writer_test.go +// (testKeySplittingWithMultipleAttributes, testXORReconstruction, +// testDifferentAttributeRules) only assert non-empty/well-formed shares -- +// exactly the assertion style that missed the original EC-KAS bug this PR +// fixes (wrong KeyType string, XOR instead of AES-GCM wrapping, missing +// schemaVersion). +// +// sdk.ChunkedWriter already runs SplitResult.VerifyReconstruction against the +// live DEK before wrapping anything, so the adapter's arithmetic (do the +// shares XOR back to the DEK) is already covered. What that check cannot +// catch is a bug in the wrap step itself: the adapter's +// Algorithm: ocrypto.KeyType(key.Algorithm) conversion selects which wrap +// scheme createKeyAccess uses, and picking the wrong one -- exactly the class +// of bug that shipped an undecryptable TDF before this PR -- corrupts bytes +// after VerifyReconstruction has already passed. Decrypting for real is the +// only way to catch that. +// +// This test finalizes a three-KAS ALL_OF TDF through this package's own +// Writer (exercising xorSplitter, not sdk.CreateTDF's single-KAS path), then +// for each resulting key access object: decrypts the wrapped share with the +// matching KAS's real RSA private key and checks it reproduces that KAO's own +// policy binding (chunked_writer.go's createPolicyBinding keys the HMAC on +// the split's own share, not the aggregate DEK, so this is checkable per KAO +// independently). It then XORs every recovered share together and checks the +// result reproduces the manifest's root signature -- an HMAC-SHA256 of the +// aggregate segment hash keyed by the full DEK -- which only the real DEK can +// produce. +func TestXORSplitterAdapterRoundTrip(t *testing.T) { + ctx := t.Context() + + urls := []string{testKAS1, testKAS2, testKAS3} + privByURL := make(map[string]string, len(urls)) + attrs := make([]*policy.Value, len(urls)) + for i, url := range urls { + keyPair, err := ocrypto.NewRSAKeyPair(2048) + require.NoError(t, err) + pubPEM, err := keyPair.PublicKeyInPemFormat() + require.NoError(t, err) + privPEM, err := keyPair.PrivateKeyInPemFormat() + require.NoError(t, err) + + privByURL[url] = privPEM + attrs[i] = createTestAttributeWithAlgorithm(t, + fmt.Sprintf("https://example.com/attr/MultiKAS/value/Share%d", i), + url, fmt.Sprintf("kid%d", i), policy.Algorithm_ALGORITHM_RSA_2048, pubPEM) + } + + writer, err := NewWriter(ctx) + require.NoError(t, err) + + _, err = writer.WriteSegment(ctx, 0, []byte("multi-KAS adapter round trip")) + require.NoError(t, err) + + result, err := writer.Finalize(ctx, WithAttributeValues(attrs)) + require.NoError(t, err) + + kaos := result.Manifest.KeyAccessObjs + require.Len(t, kaos, len(urls), "each of the three ALL_OF attributes on a distinct KAS must get its own split") + + policyBytes := []byte(result.Manifest.Policy) // already base64, matching createPolicyBinding's input + var dek []byte + for _, kao := range kaos { + priv, ok := privByURL[kao.KasURL] + require.True(t, ok, "no private key registered for KAS %q", kao.KasURL) + + dec, err := ocrypto.FromPrivatePEM(priv) + require.NoError(t, err) + + wrapped, err := ocrypto.Base64Decode([]byte(kao.WrappedKey)) + require.NoError(t, err) + share, err := dec.Decrypt(wrapped) + require.NoError(t, err, "KAS %s must be able to unwrap its share", kao.KasURL) + + binding, ok := kao.PolicyBinding.(PolicyBinding) + require.True(t, ok, "policy binding should be a PolicyBinding, got %T", kao.PolicyBinding) + expectedHash := ocrypto.CalculateSHA256Hmac(share, policyBytes) + expected := string(ocrypto.Base64Encode([]byte(hex.EncodeToString(expectedHash)))) + require.Equal(t, expected, binding.Hash, "recovered share for KAS %s must reproduce its own policy binding", kao.KasURL) + + if dek == nil { + dek = make([]byte, len(share)) + } + require.Len(t, share, len(dek), "every share must be the same length as the DEK") + for i, b := range share { + dek[i] ^= b + } + } + + var aggregate []byte + for _, seg := range result.Manifest.Segments { + decoded, err := ocrypto.Base64Decode([]byte(seg.Hash)) + require.NoError(t, err) + aggregate = append(aggregate, decoded...) + } + + rootSig, err := ocrypto.Base64Decode([]byte(result.Manifest.Signature)) + require.NoError(t, err) + expectedRootSig := ocrypto.CalculateSHA256Hmac(dek, aggregate) + require.Equal(t, expectedRootSig, rootSig, + "XOR-reconstructed shares must reproduce the manifest's root signature, proving they really are the DEK the writer used") +} diff --git a/sdk/experimental/tdf/manifest.go b/sdk/experimental/tdf/manifest.go index cf6bd14501..566500753b 100644 --- a/sdk/experimental/tdf/manifest.go +++ b/sdk/experimental/tdf/manifest.go @@ -3,25 +3,9 @@ package tdf import ( - "errors" - "fmt" - - "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/sdk" ) -// kGMACPayloadLength and kSplitKeyType copy sdk's identically-named unexported -// constants; kPolicyBindingAlg has no sdk counterpart, but matches the value -// sdk writes there via hmacIntegrityAlgorithm. All three stay only while this -// package still builds manifests itself. The follow-up that delegates manifest -// construction must delete them along with their callers, or they become -// unused. -const ( - kGMACPayloadLength = 16 - kSplitKeyType = "split" - kPolicyBindingAlg = "HS256" -) - // Aliases onto [github.com/opentdf/platform/sdk], which owns the definitions. // See the package doc for why. type ( @@ -118,61 +102,14 @@ type PolicyBody struct { Dissem []string `json:"dissem"` } +// These are the stable SDK's error values rather than copies of them, so +// errors.Is matches whether a caller compares against the experimental or the +// sdk-scoped name. See [Writer]'s error vars for the same pattern. var ( // ErrUnsupportedRootIntegrityAlgorithm rejects any root signature // algorithm other than HS256. - ErrUnsupportedRootIntegrityAlgorithm = errors.New("tdf: unsupported root integrity algorithm") + ErrUnsupportedRootIntegrityAlgorithm = sdk.ErrUnsupportedRootIntegrityAlgorithm // ErrUnsupportedSegmentIntegrityAlgorithm rejects a segment algorithm that - // is neither HS256 nor GMAC. SegmentIntegrityAlg is int-backed, so this - // catches an out-of-range value before it reaches a manifest. - ErrUnsupportedSegmentIntegrityAlgorithm = errors.New("tdf: unsupported segment integrity algorithm") + // is neither HS256 nor GMAC. + ErrUnsupportedSegmentIntegrityAlgorithm = sdk.ErrUnsupportedSegmentIntegrityAlgorithm ) - -// None of these helpers take the hex-encoding flag the stable SDK carries for -// 4.2.2 files: this package only ever writes current-spec TDFs. - -// hmacIntegrity is the HMAC-SHA256 primitive both signature paths share. It -// takes no algorithm argument, so it cannot be pointed at the wrong branch. -func hmacIntegrity(data, secret []byte) string { - return string(ocrypto.CalculateSHA256Hmac(secret, data)) -} - -// readAEADTag returns the trailing AES-GCM tag of a segment's ciphertext. -// -// This is only an authenticator because the cipher computed that tag over -// exactly these bytes. Applied to anything the AEAD did not produce it -// authenticates nothing: it just returns a copy of the input's own last 16 -// bytes. Hence unexported, and reachable only through segmentIntegrity. -func readAEADTag(ciphertext []byte) (string, error) { - if kGMACPayloadLength > len(ciphertext) { - return "", errors.New("fail to create gmac signature") - } - return string(ciphertext[len(ciphertext)-kGMACPayloadLength:]), nil -} - -// segmentIntegrity computes the integrity value recorded in a segment's -// manifest entry. Both algorithms are legitimate here: the input is data the -// cipher produced. -func segmentIntegrity(ciphertext, key []byte, alg SegmentIntegrityAlg) (string, error) { - switch alg { - case SegmentHS256: - return hmacIntegrity(ciphertext, key), nil - case SegmentGMAC: - return readAEADTag(ciphertext) - } - return "", fmt.Errorf("%w: %s", ErrUnsupportedSegmentIntegrityAlgorithm, alg) -} - -// rootIntegrity computes the root signature over the aggregate hash: the -// concatenation of every segment's hash, in manifest order. -// -// HS256 only. AES-GCM never processed the aggregate hash, so there is no tag to -// extract and no keyless construction that could authenticate it. -// RootIntegrityAlg has no other named value, but it is int-backed, so the check -// still has to run. -func rootIntegrity(aggregateHash, key []byte, alg RootIntegrityAlg) (string, error) { - if alg != RootHS256 { - return "", fmt.Errorf("%w: %s", ErrUnsupportedRootIntegrityAlgorithm, alg) - } - return hmacIntegrity(aggregateHash, key), nil -} diff --git a/sdk/experimental/tdf/options.go b/sdk/experimental/tdf/options.go index 9de6b4d7ae..d674e43536 100644 --- a/sdk/experimental/tdf/options.go +++ b/sdk/experimental/tdf/options.go @@ -14,6 +14,12 @@ import ( // Deprecated: the root signature and the segment hashes accept different sets // of algorithms, which one type cannot express. Use [RootIntegrityAlg] or // [SegmentIntegrityAlg]. +// +// Unlike the manifest and assertion types in this package, this is not an +// alias onto [sdk.IntegrityAlgorithm]: that one is itself an alias for int, so +// no methods can be attached to it, and aliasing would silently drop String() +// from this package's public API. The underlying values match, so the two +// convert freely. type IntegrityAlgorithm int // String returns the string representation of the integrity algorithm. @@ -71,11 +77,12 @@ type SegmentIntegrityAlg int const ( // SegmentHS256 is an HMAC-SHA256 over the segment's bytes, keyed by the // DEK. It is the only meaningful choice when those bytes are not AEAD - // output -- a plaintext segment has no tag to read out -- and it is this - // writer's default. + // output -- a plaintext segment has no tag to read out. No writer in this + // SDK produces one, so [NewWriter] refuses it. SegmentHS256 SegmentIntegrityAlg = iota // SegmentGMAC reads out the AES-GCM tag the cipher already computed over - // exactly this segment's ciphertext. + // exactly this segment's ciphertext. It is this writer's default and the + // only value it accepts. SegmentGMAC ) @@ -128,6 +135,10 @@ type WriterConfig struct { // initialDefaultKAS allows callers to provide a default KAS at writer creation time. // This will be used during Finalize() if no default KAS is provided there. initialDefaultKAS *policy.SimpleKasKey + + // targetMode is the TDF spec version to write for, as semver. + // Empty selects the current format. See WithTargetMode. + targetMode string } // ReaderConfig contains configuration options for TDF Reader creation. @@ -146,7 +157,7 @@ type ReaderConfig struct { // Example usage: // // writer, err := NewWriter(ctx, WithSegmentIntegrityAlgorithm(SegmentGMAC)) -// finalBytes, manifest, err := writer.Finalize(ctx, WithPayloadMimeType("text/plain")) +// result, err := writer.Finalize(ctx, WithPayloadMimeType("text/plain")) type Option[T any] func(T) // WithIntegrityAlgorithm sets the algorithm for root integrity signature calculation. @@ -155,7 +166,7 @@ type Option[T any] func(T) // providing verification that the complete TDF has not been tampered with. // // [RootHS256] is the only supported value, and the default. Options cannot -// return an error, so anything else is refused by Finalize. +// return an error, so anything else is refused by NewWriter. // // Example: // @@ -173,20 +184,14 @@ func WithIntegrityAlgorithm(algo RootIntegrityAlg) Option[*WriterConfig] { // complete file. This is particularly useful for streaming scenarios where // segments may be processed independently. // -// Both [SegmentHS256] and [SegmentGMAC] are supported, and the choice is -// independent of the root: -// - SegmentGMAC reads out the AES-GCM tag the cipher already computed, so it -// costs nothing extra per segment -// - SegmentHS256 is the default, and the only option for bytes the AEAD did -// not produce +// [SegmentGMAC] is the only supported value, and the default: every segment +// this writer emits is AES-GCM output, so the tag the cipher already computed +// is the hash. Options cannot return an error, so [SegmentHS256] is refused by +// NewWriter. // // Example: // -// // Fast segment processing with compatible root signature -// writer, err := NewWriter(ctx, -// WithSegmentIntegrityAlgorithm(SegmentGMAC), // Fast segment hashing -// WithIntegrityAlgorithm(RootHS256), // Compatible root signature -// ) +// writer, err := NewWriter(ctx, WithSegmentIntegrityAlgorithm(SegmentGMAC)) func WithSegmentIntegrityAlgorithm(algo SegmentIntegrityAlg) Option[*WriterConfig] { return func(c *WriterConfig) { c.segmentIntegrityAlg = algo @@ -236,10 +241,6 @@ type WriterFinalizeConfig struct { // or handling instructions for the TDF. assertions []AssertionConfig - // excludeVersionFromManifest controls whether to exclude version information - // from the TDF manifest (for compatibility with older readers). - excludeVersionFromManifest bool - // encryptedMetadata contains sensitive metadata encrypted within the TDF. // This metadata is stored in key access objects and only accessible after // successful attribute-based access control validation. @@ -249,9 +250,9 @@ type WriterFinalizeConfig struct { // Used by readers to determine appropriate content handling. payloadMimeType string - // keepSegments indicates caller-provided segment indices to keep when finalizing. - // Indices must form a contiguous prefix [0..K]. If empty, all written - // segments (default behavior) are used. + // keepSegments names the segments the manifest should describe, + // ascending and possibly sparse. If empty, all written segments + // (default behavior) are used. keepSegments []int } @@ -267,7 +268,7 @@ type WriterFinalizeConfig struct { // // Example: // -// finalBytes, manifest, err := writer.Finalize(ctx, +// result, err := writer.Finalize(ctx, // WithEncryptedMetadata("classification: secret"), // ) func WithEncryptedMetadata(metadata string) Option[*WriterFinalizeConfig] { @@ -287,7 +288,7 @@ func WithEncryptedMetadata(metadata string) Option[*WriterFinalizeConfig] { // // Example: // -// finalBytes, manifest, err := writer.Finalize(ctx, +// result, err := writer.Finalize(ctx, // WithPayloadMimeType("application/json"), // ) func WithPayloadMimeType(mimeType string) Option[*WriterFinalizeConfig] { @@ -296,10 +297,25 @@ func WithPayloadMimeType(mimeType string) Option[*WriterFinalizeConfig] { } } -// WithSegments restricts finalization to the provided segment indices and order. -// The order provided is used as the logical payload order. Indices may be sparse -// but must refer to segments that were written. When omitted, all present indices -// are used in ascending order. +// WithSegments names the segments the finalized manifest describes. +// When omitted, all written segments are used in ascending index order. +// +// Indices need not be contiguous -- a caller that reserves a fixed block +// of indices per upload part and fills only the front of each block +// leaves gaps by construction -- but the list must name written segments +// in ascending index order and may drop only from the end. The payload is +// laid out in sorted index order, so dropping a segment from the middle +// would shift every later segment's offset and produce a manifest that +// does not describe the bytes on disk. For the same reason the caller +// must concatenate each segment's TDFData in ascending index order. +// +// Dropping a segment from the manifest does not shrink the archive: the +// underlying writer never rolls back a completed segment's contribution to +// the payload's recorded size and CRC, so every segment that was actually +// written -- including ones this option excludes from the manifest -- must +// still be appended by the caller when assembling the final file. Skipping a +// dropped segment's bytes produces an archive whose central directory +// offsets overshoot. func WithSegments(indices []int) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.keepSegments = indices @@ -326,7 +342,7 @@ func WithSegments(indices []int) Option[*WriterFinalizeConfig] { // Pem: kasPublicKeyPEM, // }, // } -// finalBytes, manifest, err := writer.Finalize(ctx, WithDefaultKAS(kasKey)) +// result, err := writer.Finalize(ctx, WithDefaultKAS(kasKey)) func WithDefaultKAS(kas *policy.SimpleKasKey) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.defaultKas = kas @@ -356,31 +372,44 @@ func WithDefaultKAS(kas *policy.SimpleKasKey) Option[*WriterFinalizeConfig] { // Grants: []*policy.KeyAccessServer{kasConfig}, // }, // } -// finalBytes, manifest, err := writer.Finalize(ctx, WithAttributeValues(attributes)) +// result, err := writer.Finalize(ctx, WithAttributeValues(attributes)) func WithAttributeValues(values []*policy.Value) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.attributes = values } } -// WithExcludeVersionFromManifest controls version information in the manifest. +// WithExcludeVersionFromManifest is a no-op and always has been: the +// manifest builder never read the flag it sets, so schemaVersion is +// emitted either way. +// +// Omitting schemaVersion is not independently useful in any case. A +// reader treats a missing schemaVersion as "predates 4.3.0" and then +// expects hex-then-base64 signatures, which are decided per segment at +// write time -- long before Finalize sees this option. The two must be +// set together, which is what [WithTargetMode] does. // -// When set to true, excludes TDF specification version information from -// the manifest. This may be needed for compatibility with older TDF readers -// that don't expect version fields. +// Deprecated: use [WithTargetMode] at writer construction. +func WithExcludeVersionFromManifest(_ bool) Option[*WriterFinalizeConfig] { + return func(_ *WriterFinalizeConfig) {} +} + +// WithTargetMode targets a specific TDF spec version, given as a semver +// string such as "4.2.2". // -// Generally should be left as default (false) unless specific compatibility -// requirements exist. +// Below 4.3.0 the writer emits the legacy wire format: segment, root, +// and assertion signatures are hex-encoded before base64, and +// schemaVersion is omitted from the manifest, which is how those +// readers detect it. The two travel together -- a manifest carrying one +// without the other cannot be verified by any reader. // -// Example: +// An empty mode selects the current format. // -// // For compatibility with legacy readers -// finalBytes, manifest, err := writer.Finalize(ctx, -// WithExcludeVersionFromManifest(true), -// ) -func WithExcludeVersionFromManifest(exclude bool) Option[*WriterFinalizeConfig] { - return func(c *WriterFinalizeConfig) { - c.excludeVersionFromManifest = exclude +// A malformed semver string is reported by NewWriter, not here: this +// package's Option signature has no error return. +func WithTargetMode(mode string) Option[*WriterConfig] { + return func(c *WriterConfig) { + c.targetMode = mode } } @@ -412,7 +441,7 @@ func WithExcludeVersionFromManifest(exclude bool) Option[*WriterFinalizeConfig] // Value: `{"retention_days": 90}`, // }, // } -// finalBytes, manifest, err := writer.Finalize(ctx, WithAssertions(assertion)) +// result, err := writer.Finalize(ctx, WithAssertions(assertion)) func WithAssertions(assertions ...AssertionConfig) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.assertions = assertions diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 28ce926543..4c6fb2a1cf 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -3,34 +3,14 @@ package tdf import ( - "bytes" "context" - "encoding/hex" - "encoding/json" - "errors" "fmt" - "hash/crc32" "io" "log/slog" - "sort" - "sync" + "sync/atomic" - "github.com/google/uuid" - "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" - "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" - "github.com/opentdf/platform/sdk/internal/zipstream" -) - -const ( - // kKeySize is the AES key size in bytes (256-bit key) - kKeySize = 32 - // kGCMCipherAlgorithm specifies the encryption algorithm used for TDF payloads - kGCMCipherAlgorithm = "AES-256-GCM" - // tdfAsZip indicates the TDF uses ZIP as the container format - tdfAsZip = "zip" - // tdfZipReference indicates the payload is stored as a reference in the ZIP - tdfZipReference = "reference" + "github.com/opentdf/platform/sdk" ) // SegmentResult contains the result of writing a segment @@ -51,13 +31,18 @@ type FinalizeResult struct { EncryptedSize int64 `json:"encryptedSize"` // Total encrypted size } +// These are the stable SDK's error values rather than copies of them, +// so errors.Is matches whether a caller compares against the +// experimental or the sdk-scoped name. var ( // ErrAlreadyFinalized is returned when attempting operations on a finalized writer - ErrAlreadyFinalized = errors.New("tdf is already finalized") + ErrAlreadyFinalized = sdk.ErrChunkedAlreadyFinalized // ErrInvalidSegmentIndex is returned for negative segment indices - ErrInvalidSegmentIndex = errors.New("invalid segment index") + ErrInvalidSegmentIndex = sdk.ErrChunkedInvalidSegmentIndex + // ErrMissingSegmentZero is returned when Finalize is called without segment 0 + ErrMissingSegmentZero = sdk.ErrChunkedMissingSegmentZero // ErrSegmentAlreadyWritten is returned when trying to write to an existing segment index - ErrSegmentAlreadyWritten = errors.New("segment already written") + ErrSegmentAlreadyWritten = sdk.ErrChunkedSegmentAlreadyWritten ) // Writer provides streaming TDF creation with out-of-order segment support. @@ -67,15 +52,19 @@ var ( // and proper ZIP archive structure generation. // // Key features: -// - Variable-length segments with sparse index support +// - Variable-length segments with sparse index support above index 0 // - Out-of-order segment writing without buffering payloads // - Memory-efficient handling through segment cleanup // - Cryptographic assertions and integrity verification // - Custom attribute-based access controls // -// Thread safety: Writers require external synchronization for concurrent access. -// Each WriteSegment call must be serialized, but multiple Writers can operate -// independently. +// Thread safety: WriteSegment may be called concurrently for distinct +// indices, but not twice for the same index. +// +// The writing itself is [sdk.ChunkedWriter]; this type adds the +// multi-KAS ABAC key splitting in [xorSplitter] and this package's +// option style. Callers that only need a single KAS can use +// [sdk.NewChunkedWriter] directly. // // Example usage: // @@ -83,111 +72,79 @@ var ( // if err != nil { // return err // } -// defer writer.Close() // // // Write segments (can be out-of-order) // _, err = writer.WriteSegment(ctx, 1, []byte("second")) // _, err = writer.WriteSegment(ctx, 0, []byte("first")) // // // Finalize with attributes -// finalBytes, manifest, err := writer.Finalize(ctx, WithAttributeValues(attrs)) +// result, err := writer.Finalize(ctx, WithAttributeValues(attrs)) type Writer struct { // WriterConfig embeds configuration options for the TDF writer WriterConfig - // archiveWriter handles the underlying ZIP archive creation - archiveWriter zipstream.SegmentWriter - - // State management - mutex sync.RWMutex // Protects concurrent access to writer state - finalized bool // Whether Finalize() has been called - - // manifest holds the finalized manifest after Finalize() is called. - // Before finalization, GetManifest() will synthesize a stub manifest - // from the current writer state. Do not rely on the stub for - // verification — it is informational only until Finalize completes. - manifest *Manifest + // inner is the stable per-segment writer this type delegates to. + inner sdk.ChunkedWriter - // segments stores segment metadata using sparse map for memory efficiency - // Maps segment index to Segment metadata (hash, size information) - segments map[int]*Segment - // maxSegmentIndex tracks the highest segment index written - maxSegmentIndex int - - // Cryptographic state - dek []byte // Data Encryption Key (32-byte AES key) - block ocrypto.AesGcm // AES-GCM cipher for segment encryption - - // Initial settings provided at Writer creation; used by Finalize if not overridden - initialAttributes []*policy.Value - initialDefaultKAS *policy.SimpleKasKey + // finalized mirrors inner's state so GetManifest can warn that a + // pre-finalize manifest is a stub. inner does not expose it. + finalized atomic.Bool } // NewWriter creates a new experimental TDF Writer with streaming support. // // The writer is initialized with secure defaults: -// - HS256 integrity algorithms for both root and segment verification +// - An HS256 root signature and GMAC segment hashes // - AES-256-GCM encryption for all segments -// - Dynamic segment expansion supporting sparse indices +// - Dynamic segment expansion supporting sparse indices (index 0 always required) // - Memory-efficient segment processing // // Configuration options can be provided to customize: -// - Segment integrity algorithm (SegmentHS256, SegmentGMAC) -// - Root integrity algorithm, which accepts RootHS256 and nothing else -// -// The writer generates a unique Data Encryption Key (DEK) and initializes -// the underlying archive writer for ZIP structure management. +// - Attributes and default KAS to fall back on at Finalize time +// - The TDF spec version to target, via [WithTargetMode] // // Returns an error if: // - DEK generation fails (cryptographic entropy issues) // - AES-GCM cipher initialization fails (invalid key) // - Archive writer creation fails (resource constraints) +// - An integrity algorithm other than [RootHS256] or [SegmentGMAC] was asked for // // Example: // // // Default configuration // writer, err := NewWriter(ctx) // -// // Custom integrity algorithms -// writer, err := NewWriter(ctx, -// WithIntegrityAlgorithm(RootHS256), -// WithSegmentIntegrityAlgorithm(SegmentGMAC), -// ) -func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error) { - // Initialize Config +// // Targeting a pre-4.3.0 reader +// writer, err := NewWriter(ctx, WithTargetMode("4.2.2")) +func NewWriter(ctx context.Context, opts ...Option[*WriterConfig]) (*Writer, error) { config := &WriterConfig{ rootIntegrityAlg: RootHS256, - segmentIntegrityAlg: SegmentHS256, + segmentIntegrityAlg: SegmentGMAC, } - for _, opt := range opts { opt(config) } - - // Initialize archive writer - start with 1 segment and expand dynamically - archiveWriter := zipstream.NewSegmentTDFWriter(1, zipstream.WithZip64()) - - // Generate DEK - dek, err := ocrypto.RandomBytes(kKeySize) - if err != nil { - return nil, err + // [sdk.ChunkedWriter] emits an HS256 root and GMAC segments and has no + // knob for either, so anything else the caller asked for is refused here + // rather than silently ignored. + if config.rootIntegrityAlg != RootHS256 { + return nil, fmt.Errorf("%w: %s", sdk.ErrUnsupportedRootIntegrityAlgorithm, config.rootIntegrityAlg) + } + if config.segmentIntegrityAlg != SegmentGMAC { + return nil, fmt.Errorf("%w: %s", sdk.ErrUnsupportedSegmentIntegrityAlgorithm, config.segmentIntegrityAlg) } - // Initialize AES GCM Provider - block, err := ocrypto.NewAESGcm(dek) + inner, err := sdk.NewChunkedWriter(ctx, + sdk.WithChunkedInitialAttributes(config.initialAttributes), + sdk.WithChunkedDefaultKAS(config.initialDefaultKAS), + sdk.WithChunkedKeySplitter(xorSplitter{}), + sdk.WithChunkedTargetMode(config.targetMode), + ) if err != nil { return nil, err } - return &Writer{ - WriterConfig: *config, - archiveWriter: archiveWriter, - dek: dek, - segments: make(map[int]*Segment), // Initialize sparse storage - block: block, - initialAttributes: config.initialAttributes, - initialDefaultKAS: config.initialDefaultKAS, - }, nil + return &Writer{WriterConfig: *config, inner: inner}, nil } // WriteSegment encrypts and writes a data segment at the specified index. @@ -198,26 +155,19 @@ func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error // // Parameters: // - ctx: Context for cancellation and timeout control -// - index: Zero-based segment index (must be non-negative, sparse indices supported) +// - index: Zero-based segment index (must be non-negative; sparse indices supported, +// but index 0 must eventually be written) // - data: Raw data to encrypt and store in this segment // -// Returns the encrypted segment bytes that should be stored/uploaded, and any error. -// The returned bytes include ZIP structure elements and can be assembled in any order. -// -// The function performs: -// 1. Input validation (index >= 0, writer not finalized, no duplicate segments) -// 2. AES-256-GCM encryption of the segment data -// 3. HMAC signature calculation for integrity verification -// 4. ZIP archive segment creation through the archive layer -// -// Memory optimization: Uses sparse storage to avoid O(n²) memory growth -// for high or non-contiguous segment indices. +// Returns a SegmentResult whose TDFData reads the encrypted segment bytes that +// should be stored/uploaded, and any error. Those bytes include ZIP structure +// elements. They must be concatenated in ascending index order to form the +// payload, whatever order they were produced in. // // Error conditions: // - ErrAlreadyFinalized: Writer has been finalized, no more segments accepted // - ErrInvalidSegmentIndex: Negative index provided // - ErrSegmentAlreadyWritten: Segment index already contains data -// - Context cancellation: If ctx.Done() is signaled // - Encryption errors: AES-GCM operation failures // - Archive errors: ZIP structure creation failures // @@ -227,101 +177,29 @@ func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error // segment1, err := writer.WriteSegment(ctx, 1, []byte("second part")) // segment0, err := writer.WriteSegment(ctx, 0, []byte("first part")) // -// // Store/upload segment bytes (e.g., to S3) -// uploadToS3(segment0, "part-000") -// uploadToS3(segment1, "part-001") +// // Store/upload each segment's bytes (e.g., to S3), reading TDFData +// uploadToS3(segment0.TDFData, "part-000") +// uploadToS3(segment1.TDFData, "part-001") func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*SegmentResult, error) { - w.mutex.Lock() - - if w.finalized { - w.mutex.Unlock() - return nil, ErrAlreadyFinalized - } - - if index < 0 { - w.mutex.Unlock() - return nil, ErrInvalidSegmentIndex - } - - // Check for duplicate segments using map lookup - if _, exists := w.segments[index]; exists { - w.mutex.Unlock() - return nil, ErrSegmentAlreadyWritten - } - - if index > w.maxSegmentIndex { - w.maxSegmentIndex = index - } - seg := &Segment{ - Size: -1, // indicates not filled yet - } - w.segments[index] = seg - - w.mutex.Unlock() - - // Encrypt directly without unnecessary copying - the archive layer will handle copying if needed - segmentCipher, nonce, err := w.block.EncryptInPlace(data) - if err != nil { - return nil, err - } - segmentSig, err := segmentIntegrity(segmentCipher, w.dek, w.segmentIntegrityAlg) - if err != nil { - return nil, err - } - - segmentHash := string(ocrypto.Base64Encode([]byte(segmentSig))) - w.mutex.Lock() - seg.Size = int64(len(data)) - seg.EncryptedSize = int64(len(segmentCipher)) + int64(len(nonce)) - seg.Hash = segmentHash - w.mutex.Unlock() - - crc := crc32.NewIEEE() - _, err = crc.Write(nonce) + res, err := w.inner.WriteSegment(ctx, index, data) if err != nil { return nil, err } - _, err = crc.Write(segmentCipher) - if err != nil { - return nil, err - } - header, err := w.archiveWriter.WriteSegment(ctx, index, uint64(seg.EncryptedSize), crc.Sum32()) - if err != nil { - return nil, err - } - var reader io.Reader - if len(header) == 0 { - reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) - } else { - reader = io.MultiReader(bytes.NewReader(header), bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) - } - return &SegmentResult{ - TDFData: reader, - Index: index, - Hash: seg.Hash, - PlaintextSize: seg.Size, - EncryptedSize: seg.EncryptedSize, + TDFData: res.TDFData, + Index: res.Index, + Hash: res.Hash, + PlaintextSize: res.PlaintextSize, + EncryptedSize: res.EncryptedSize, }, nil } // Finalize completes TDF creation and returns the final bytes and manifest. // -// This method must be called after all segments have been written. It performs: -// 1. Validates all segments are present (no missing indices from 0 to maxSegmentIndex) -// 2. Generates cryptographic splits for key access controls -// 3. Builds the TDF policy from provided attributes -// 4. Creates cryptographic assertions if specified -// 5. Calculates root integrity signature over all segment hashes -// 6. Generates the complete TDF manifest -// 7. Finalizes the ZIP archive structure -// -// The finalization process handles: -// - Key splitting for attribute-based access controls -// - Policy generation from attribute values -// - Encrypted metadata storage in key access objects -// - Manifest JSON generation and validation -// - ZIP central directory and data descriptor creation +// This method must be called after all segments have been written. It +// generates the key splits for attribute-based access control, builds the +// policy, signs any assertions, calculates the root integrity signature over +// all segment hashes, and closes the ZIP archive. // // Parameters: // - ctx: Context for cancellation and timeout control @@ -333,30 +211,24 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg // - WithPayloadMimeType: Specify payload MIME type // - WithAssertions: Add cryptographic assertions // - WithDefaultKAS: Set default Key Access Server -// -// Returns: -// - finalBytes: Complete ZIP archive bytes ready for storage/transmission -// - manifest: TDF manifest containing encryption and integrity information -// - error: Any error during finalization process +// - WithSegments: Choose which written segments the manifest describes // // Error conditions: // - ErrAlreadyFinalized: Finalize already called -// - Missing segment 0: Index 0 carries the payload's ZIP local file header, which -// every recorded offset is measured from, so a write set that omits it is -// rejected. Gaps between the remaining indices are legal (e.g., segments 0,1,3 -// with 2 missing); order is inferred by sorting whichever indices are present. +// - ErrMissingSegmentZero: segment 0 was never written; it carries the payload's +// ZIP local file header, which every recorded offset is measured from +// - Missing segments: an index named by WithSegments that was never written // - Key splitting failures: Invalid attributes or KAS configuration // - Manifest generation errors: JSON marshaling failures // - Archive finalization errors: ZIP structure generation failures -// - Context cancellation: If ctx.Done() is signaled // // Example: // // // Basic finalization -// finalBytes, manifest, err := writer.Finalize(ctx) +// result, err := writer.Finalize(ctx) // // // With attributes and metadata -// finalBytes, manifest, err := writer.Finalize(ctx, +// result, err := writer.Finalize(ctx, // WithAttributeValues(attrs), // WithEncryptedMetadata("sensitive info"), // WithPayloadMimeType("application/json"), @@ -364,50 +236,18 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg // // Performance note: Finalization is O(n) where n is the number of segments. // Memory usage is proportional to manifest size, not total data size. - func (w *Writer) Finalize(ctx context.Context, opts ...Option[*WriterFinalizeConfig]) (*FinalizeResult, error) { - w.mutex.Lock() - defer w.mutex.Unlock() - - if w.finalized { - return nil, ErrAlreadyFinalized - } - - cfg := &WriterFinalizeConfig{ - attributes: make([]*policy.Value, 0), - encryptedMetadata: "", - payloadMimeType: "application/octet-stream", - } - for _, opt := range opts { - opt(cfg) - } - manifest, totalPlaintextSize, totalEncryptedSize, err := w.getManifest(ctx, cfg) - if err != nil { - return nil, err - } - manifestBytes, err := json.Marshal(manifest) - if err != nil { - return nil, err - } - - finalBytes, err := w.archiveWriter.Finalize(ctx, manifestBytes) + res, err := w.inner.Finalize(ctx, finalizeOptions(opts)...) if err != nil { return nil, err } - - if err := w.archiveWriter.Close(); err != nil { - return nil, err - } - - // Persist the final manifest for later retrieval via GetManifest. - w.manifest = manifest - w.finalized = true + w.finalized.Store(true) return &FinalizeResult{ - Data: finalBytes, - Manifest: manifest, - TotalSegments: len(manifest.Segments), - TotalSize: totalPlaintextSize, - EncryptedSize: totalEncryptedSize, + Data: res.Data, + Manifest: res.Manifest, + TotalSegments: res.TotalSegments, + TotalSize: res.TotalSize, + EncryptedSize: res.EncryptedSize, }, nil } @@ -419,13 +259,25 @@ func (w *Writer) Finalize(ctx context.Context, opts ...Option[*WriterFinalizeCon // from the writer's current state (segments present so far, algorithm // selections, and payload defaults). This pre-finalize manifest is not // complete and must not be used for verification; it is provided for -// informational or client-side pre-calculation purposes only. -// -// No logging is performed; callers should consult this documentation for -// the caveat about pre-finalize state. +// informational or client-side pre-calculation purposes only. A +// warning is logged in that case. func (w *Writer) GetManifest(ctx context.Context, opts ...Option[*WriterFinalizeConfig]) (*Manifest, error) { - w.mutex.RLock() - defer w.mutex.RUnlock() + if !w.finalized.Load() { + slog.Warn("getmanifest called before finalize; returned manifest is a stub and not complete, pre-finalize state may not include all segments or attributes.") + } + return w.inner.GetManifest(ctx, finalizeOptions(opts)...) +} + +// finalizeOptions translates this package's finalize options into the +// stable writer's equivalents. +// +// The two option sets are applied in sequence rather than mapped +// one-to-one: this package's Option is a plain mutator over a config +// struct, so the config is materialized first and then read off. That +// also preserves the defaults callers have always seen -- notably the +// "application/octet-stream" MIME type -- independent of whichever +// defaults the stable writer happens to use. +func finalizeOptions(opts []Option[*WriterFinalizeConfig]) []sdk.ChunkedFinalizeOption { cfg := &WriterFinalizeConfig{ attributes: make([]*policy.Value, 0), encryptedMetadata: "", @@ -434,252 +286,12 @@ func (w *Writer) GetManifest(ctx context.Context, opts ...Option[*WriterFinalize for _, opt := range opts { opt(cfg) } - if !w.finalized { - slog.Warn("getmanifest called before finalize; returned manifest is a stub and not complete, pre-finalize state may not include all segments or attributes.") - } - - manifest, _, _, err := w.getManifest(ctx, cfg) - if err != nil { - return nil, err - } - return manifest, nil -} - -func (w *Writer) getManifest(ctx context.Context, cfg *WriterFinalizeConfig) (*Manifest, int64, int64, error) { - // If already finalized and we have the final manifest, return a copy. - if w.finalized && w.manifest != nil { - return cloneManifest(w.manifest), 0, 0, nil - } - // Archive layer will infer the same order by sorting present indices. - // Merge writer-level initial settings if finalize options omitted them. - if len(cfg.attributes) == 0 && len(w.initialAttributes) > 0 { - cfg.attributes = w.initialAttributes - } - if cfg.defaultKas == nil && w.initialDefaultKAS != nil { - cfg.defaultKas = w.initialDefaultKAS - } - - manifest := &Manifest{ - TDFVersion: TDFSpecVersion, - Payload: Payload{ - MimeType: cfg.payloadMimeType, - Protocol: tdfAsZip, - Type: tdfZipReference, - URL: zipstream.TDFPayloadFileName, - IsEncrypted: true, - }, - } - // Determine finalize order by collecting all present segment indices and sorting. - // This densifies sparse indices automatically and ignores any gaps. - order := make([]int, 0, len(w.segments)) - for idx := range w.segments { - order = append(order, idx) - } - sort.Ints(order) - // If caller provided keepSegments, restrict to that subset and order. - if len(cfg.keepSegments) > 0 { - subset := make([]int, 0, len(cfg.keepSegments)) - seen := make(map[int]struct{}, len(cfg.keepSegments)) - for _, idx := range cfg.keepSegments { - if idx < 0 { - return nil, 0, 0, fmt.Errorf("WithSegments contains invalid index %d (must be >= 0)", idx) - } - if _, ok := w.segments[idx]; !ok { - return nil, 0, 0, fmt.Errorf("WithSegments references segment %d which was not written", idx) - } - if _, dup := seen[idx]; dup { - return nil, 0, 0, fmt.Errorf("WithSegments contains duplicate index %d", idx) - } - seen[idx] = struct{}{} - subset = append(subset, idx) - } - order = subset - } - - // Generate splits using the splitter - splitter := keysplit.NewXORSplitter(keysplit.WithDefaultKAS(cfg.defaultKas)) - result, err := splitter.GenerateSplits(ctx, cfg.attributes, w.dek) - if err != nil { - return nil, 0, 0, err - } - - // Build key access objects from the splits - policyBytes, err := buildPolicy(cfg.attributes) - if err != nil { - return nil, 0, 0, err - } - - encryptInfo := EncryptionInformation{ - KeyAccessType: kSplitKeyType, - Policy: string(ocrypto.Base64Encode(policyBytes)), - Method: Method{ - Algorithm: kGCMCipherAlgorithm, - IsStreamable: true, - }, - IntegrityInformation: IntegrityInformation{ - // Copy segments to manifest for integrity verification in finalize order - Segments: make([]Segment, len(order)), - RootSignature: RootSignature{}, - }, - } - - // Copy segments to manifest in finalize order (pack densely) - for i, idx := range order { - if segment, exists := w.segments[idx]; exists { - encryptInfo.Segments[i] = *segment - } - } - - // Set default segment sizes for reader compatibility - // Use the first segment as the default (streaming TDFs have variable segment sizes) - if firstSegment, exists := w.segments[0]; exists { - encryptInfo.DefaultSegmentSize = firstSegment.Size - encryptInfo.DefaultEncryptedSegSize = firstSegment.EncryptedSize - } - - // Set segment hash algorithm - // Safe to take the name from the config: an algorithm with no manifest - // spelling would already have failed every segmentIntegrity call above. - encryptInfo.SegmentHashAlgorithm = w.segmentIntegrityAlg.String() - - var aggregateHash bytes.Buffer - // Calculate totals and iterate through segments in finalize order - var totalPlaintextSize, totalEncryptedSize int64 - for _, i := range order { - segment, exists := w.segments[i] - // if size is negative, segment was not written, finalized has been called too early - if !exists || w.segments[i].Size < 0 { - return nil, 0, 0, fmt.Errorf("segment %d not written; cannot finalize", i) - } - if segment.Hash != "" { - // Accumulate sizes for result - totalPlaintextSize += segment.Size - totalEncryptedSize += segment.EncryptedSize - - // Decode the base64-encoded segment hash to match reader validation - decodedHash, err := ocrypto.Base64Decode([]byte(segment.Hash)) - if err != nil { - return nil, 0, 0, fmt.Errorf("failed to decode segment hash: %w", err) - } - aggregateHash.Write(decodedHash) - continue - } - return nil, 0, 0, errors.New("empty segment hash") - } - - rootSignature, err := rootIntegrity(aggregateHash.Bytes(), w.dek, w.rootIntegrityAlg) - if err != nil { - return nil, 0, 0, err - } - encryptInfo.RootSignature = RootSignature{ - Algorithm: w.rootIntegrityAlg.String(), - Signature: string(ocrypto.Base64Encode([]byte(rootSignature))), - } - - keyAccessList, err := buildKeyAccessObjects(result, policyBytes, cfg.encryptedMetadata) - if err != nil { - return nil, 0, 0, err - } - - encryptInfo.KeyAccessObjs = keyAccessList - manifest.EncryptionInformation = encryptInfo - - signedAssertions, err := w.buildAssertions(aggregateHash.Bytes(), cfg.assertions) - if err != nil { - return nil, 0, 0, err - } - - manifest.Assertions = signedAssertions - return manifest, totalPlaintextSize, totalEncryptedSize, nil -} - -// cloneManifest makes a shallow-deep copy of a Manifest to avoid callers -// mutating internal writer state. -func cloneManifest(in *Manifest) *Manifest { - if in == nil { - return nil - } - out := *in // copy by value - - // Copy slices to new backing arrays - if in.KeyAccessObjs != nil { - out.KeyAccessObjs = append([]KeyAccess(nil), in.KeyAccessObjs...) - } - if in.Segments != nil { - out.Segments = append([]Segment(nil), in.Segments...) - } - if in.Assertions != nil { - out.Assertions = append([]Assertion(nil), in.Assertions...) - } - return &out -} - -func buildPolicy(values []*policy.Value) ([]byte, error) { - policy := &Policy{ - UUID: uuid.NewString(), - Body: PolicyBody{ - DataAttributes: make([]PolicyAttribute, 0), - Dissem: make([]string, 0), - }, - } - - for _, value := range values { - policy.Body.DataAttributes = append(policy.Body.DataAttributes, PolicyAttribute{ - Attribute: value.GetFqn(), - }) - } - policyBytes, err := json.Marshal(policy) - if err != nil { - return nil, err - } - - return policyBytes, nil -} - -func (w *Writer) buildAssertions(aggregateHash []byte, assertions []AssertionConfig) ([]Assertion, error) { - signedAssertion := make([]Assertion, 0) - for _, assertion := range assertions { - // Store a temporary assertion - tmpAssertion := Assertion{} - - tmpAssertion.ID = assertion.ID - tmpAssertion.Type = assertion.Type - tmpAssertion.Scope = assertion.Scope - tmpAssertion.Statement = assertion.Statement - tmpAssertion.AppliesToState = assertion.AppliesToState - - hashOfAssertionAsHex, err := tmpAssertion.GetHash() - if err != nil { - return nil, err - } - - hashOfAssertion := make([]byte, hex.DecodedLen(len(hashOfAssertionAsHex))) - _, err = hex.Decode(hashOfAssertion, hashOfAssertionAsHex) - if err != nil { - return nil, fmt.Errorf("error decoding hex string: %w", err) - } - - var completeHashBuilder bytes.Buffer - completeHashBuilder.Write(aggregateHash) - completeHashBuilder.Write(hashOfAssertion) - - encoded := ocrypto.Base64Encode(completeHashBuilder.Bytes()) - - assertionSigningKey := AssertionKey{} - - // Set default to HS256 and payload key - assertionSigningKey.Alg = AssertionKeyAlgHS256 - assertionSigningKey.Key = w.dek - - if !assertion.SigningKey.IsEmpty() { - assertionSigningKey = assertion.SigningKey - } - - if err := tmpAssertion.Sign(string(hashOfAssertionAsHex), string(encoded), assertionSigningKey); err != nil { - return nil, fmt.Errorf("failed to sign assertion: %w", err) - } - - signedAssertion = append(signedAssertion, tmpAssertion) + return []sdk.ChunkedFinalizeOption{ + sdk.WithChunkedAttributes(cfg.attributes), + sdk.WithChunkedDefaultKASForFinalize(cfg.defaultKas), + sdk.WithChunkedEncryptedMetadata(cfg.encryptedMetadata), + sdk.WithChunkedMimeType(cfg.payloadMimeType), + sdk.WithChunkedSegments(cfg.keepSegments), + sdk.WithChunkedAssertions(cfg.assertions), } - return signedAssertion, nil } diff --git a/sdk/experimental/tdf/writer_test.go b/sdk/experimental/tdf/writer_test.go index 038ccd63e2..23da5cfcea 100644 --- a/sdk/experimental/tdf/writer_test.go +++ b/sdk/experimental/tdf/writer_test.go @@ -5,7 +5,9 @@ package tdf import ( "bytes" "crypto/rand" + "encoding/hex" "encoding/json" + "io" "os" "path/filepath" "runtime" @@ -270,21 +272,22 @@ func testBasicTDFCreationFlow(t *testing.T) { assert.NotNil(t, writer, "Writer should not be nil") // Verify initial state - assert.False(t, writer.finalized, "Writer should not be finalized initially") - assert.Empty(t, writer.segments, "Segments should be empty initially") - assert.Len(t, writer.dek, 32, "DEK should be 32 bytes") + assert.False(t, writer.finalized.Load(), "Writer should not be finalized initially") // Write a single segment testData := []byte("Hello, TDF World!") - zipBytes, err := writer.WriteSegment(ctx, 0, testData) + segResult, err := writer.WriteSegment(ctx, 0, testData) require.NoError(t, err, "Failed to write segment") - assert.NotEmpty(t, zipBytes, "Zip bytes should not be empty") + require.NotNil(t, segResult, "Segment result should not be nil") + segBytes, err := io.ReadAll(segResult.TDFData) + require.NoError(t, err, "Failed to read segment TDF data") + assert.NotEmpty(t, segBytes, "Zip bytes should not be empty") // Verify segment was recorded - assert.Len(t, writer.segments, 1, "Should have one segment") - assert.Equal(t, int64(len(testData)), writer.segments[0].Size, "Segment size should match input data") - assert.NotEmpty(t, writer.segments[0].Hash, "Segment hash should be set") - assert.Greater(t, writer.segments[0].EncryptedSize, writer.segments[0].Size, "Encrypted size should be larger due to GCM overhead") + assert.Equal(t, 0, segResult.Index, "Segment index should match") + assert.Equal(t, int64(len(testData)), segResult.PlaintextSize, "Segment size should match input data") + assert.NotEmpty(t, segResult.Hash, "Segment hash should be set") + assert.Greater(t, segResult.EncryptedSize, segResult.PlaintextSize, "Encrypted size should be larger due to GCM overhead") // Finalize with attributes that have proper KAS setup attributes := []*policy.Value{ @@ -299,7 +302,7 @@ func testBasicTDFCreationFlow(t *testing.T) { validateManifestSchema(t, finalizeResult.Manifest) // Verify finalized state - assert.True(t, writer.finalized, "Writer should be finalized") + assert.True(t, writer.finalized.Load(), "Writer should be finalized") // Verify manifest structure assert.Equal(t, TDFSpecVersion, finalizeResult.Manifest.TDFVersion, "TDF version should match expected") @@ -316,7 +319,7 @@ func testBasicTDFCreationFlow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey, "Wrapped key should not be empty") // Verify encryption information - assert.Equal(t, kGCMCipherAlgorithm, finalizeResult.Manifest.Method.Algorithm, "Algorithm should be AES-256-GCM") + assert.Equal(t, "AES-256-GCM", finalizeResult.Manifest.Method.Algorithm, "Algorithm should be AES-256-GCM") assert.True(t, finalizeResult.Manifest.Method.IsStreamable, "Should be marked as streamable") assert.NotEmpty(t, finalizeResult.Manifest.Policy, "Policy should not be empty") } @@ -385,19 +388,18 @@ func testMultiSegmentFlow(t *testing.T) { } // Write segments in order + results := make([]*SegmentResult, len(segments)) for i, data := range segments { - _, err := writer.WriteSegment(ctx, i, data) + res, err := writer.WriteSegment(ctx, i, data) require.NoError(t, err, "Failed to write segment %d", i) + results[i] = res } - // Verify all segments were recorded - assert.Len(t, writer.segments, 3, "Should have three segments") - assert.Equal(t, 2, writer.maxSegmentIndex, "Max segment index should be 2") - // Verify each segment for i, data := range segments { - assert.Equal(t, int64(len(data)), writer.segments[i].Size, "Segment %d size should match", i) - assert.NotEmpty(t, writer.segments[i].Hash, "Segment %d hash should be set", i) + assert.Equal(t, i, results[i].Index, "Segment %d index should match", i) + assert.Equal(t, int64(len(data)), results[i].PlaintextSize, "Segment %d size should match", i) + assert.NotEmpty(t, results[i].Hash, "Segment %d hash should be set", i) } // Finalize with attributes for proper key access setup @@ -410,6 +412,13 @@ func testMultiSegmentFlow(t *testing.T) { // Validate manifest against schema validateManifestSchema(t, finalizeResult.Manifest) + // Verify all segments were recorded + assert.Equal(t, 3, finalizeResult.TotalSegments, "Should have three segments") + require.Len(t, finalizeResult.Manifest.Segments, 3, "Manifest should describe three segments") + for i, data := range segments { + assert.Equal(t, int64(len(data)), finalizeResult.Manifest.Segments[i].Size, "Segment %d size should match", i) + } + // Verify root signature was calculated from all segments assert.NotEmpty(t, finalizeResult.Manifest.Signature, "Root signature should be set") assert.Equal(t, "HS256", finalizeResult.Manifest.Algorithm, "Root signature algorithm should be HS256") @@ -435,9 +444,6 @@ func testKeySplittingWithMultipleAttributes(t *testing.T) { } // Finalize with multiple attributes - originalDEK := make([]byte, len(writer.dek)) - copy(originalDEK, writer.dek) - finalizeResult, err := writer.Finalize(ctx, WithAttributeValues(attributes)) require.NoError(t, err, "Failed to finalize TDF with multiple attributes") @@ -463,9 +469,13 @@ func testKeySplittingWithMultipleAttributes(t *testing.T) { } } - // Test that we can theoretically reconstruct the key from splits - // (This verifies the XOR splitting logic worked correctly) - assert.Len(t, originalDEK, 32, "Original DEK should be 32 bytes") + // Three allOf attributes on distinct KAS must XOR-split into three shares, + // each carrying its own split ID; a share is useless on its own. + splitIDs := make(map[string]bool, len(keyAccessObjs)) + for _, keyAccess := range keyAccessObjs { + splitIDs[keyAccess.SplitID] = true + } + assert.Len(t, splitIDs, 3, "Each of the three allOf KAS should get its own split") } // testManifestGeneration tests detailed manifest structure and content @@ -511,7 +521,7 @@ func testManifestGeneration(t *testing.T) { // Verify encryption information encInfo := finalizeResult.Manifest.EncryptionInformation - assert.Equal(t, kGCMCipherAlgorithm, encInfo.Method.Algorithm, "Algorithm should be AES-256-GCM") + assert.Equal(t, "AES-256-GCM", encInfo.Method.Algorithm, "Algorithm should be AES-256-GCM") assert.True(t, encInfo.Method.IsStreamable, "Should be streamable") assert.NotEmpty(t, encInfo.Policy, "Policy should not be empty") @@ -671,20 +681,36 @@ func testErrorConditions(t *testing.T) { assert.Contains(t, err.Error(), "no default KAS", "Error should mention missing default KAS") }) - t.Run("EmptySegmentHash", func(t *testing.T) { + t.Run("SegmentsNamesUnwrittenIndex", func(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - // Manually corrupt segment hash to test error handling - writer.segments[0] = &Segment{Hash: "", Size: 10, EncryptedSize: 26} - writer.maxSegmentIndex = 0 + _, err = writer.WriteSegment(ctx, 0, []byte("first")) + require.NoError(t, err) + _, err = writer.WriteSegment(ctx, 5, []byte("sixth")) + require.NoError(t, err) + + attributes := []*policy.Value{ + createTestAttribute("https://example.com/attr/Test/value/Error", testKAS1, "kid1"), + } + _, err = writer.Finalize(ctx, WithAttributeValues(attributes), WithSegments([]int{0, 1})) + require.Error(t, err, "Should reject a segment that was never written") + assert.Contains(t, err.Error(), "not written", "Error should name the unwritten segment") + }) + + t.Run("SegmentZeroMissing", func(t *testing.T) { + writer, err := NewWriter(ctx) + require.NoError(t, err) + + _, err = writer.WriteSegment(ctx, 1, []byte("second")) + require.NoError(t, err) attributes := []*policy.Value{ createTestAttribute("https://example.com/attr/Test/value/Error", testKAS1, "kid1"), } _, err = writer.Finalize(ctx, WithAttributeValues(attributes)) - require.Error(t, err, "Should detect empty segment hash") - assert.Contains(t, err.Error(), "empty segment hash", "Error message should mention empty segment hash") + require.ErrorIs(t, err, ErrMissingSegmentZero, + "segment 0 carries the payload's ZIP local file header") }) } @@ -695,10 +721,6 @@ func testXORReconstruction(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - // Store original DEK for comparison - originalDEK := make([]byte, len(writer.dek)) - copy(originalDEK, writer.dek) - // Write test data _, err = writer.WriteSegment(ctx, 0, []byte("XOR test data")) require.NoError(t, err) @@ -731,9 +753,6 @@ func testXORReconstruction(t *testing.T) { require.NoError(t, err, "Should be able to decode wrapped key") assert.NotEmpty(t, wrappedKeyBytes, "Decoded wrapped key should not be empty") } - - // Verify the original DEK is the expected size - assert.Len(t, originalDEK, 32, "Original DEK should be 32 bytes") } // testDifferentAttributeRules tests TDF creation with different attribute rule types @@ -796,22 +815,23 @@ func testOutOfOrderSegments(t *testing.T) { require.NoError(t, err, "Failed to write segment %d", idx) } - // Verify all segments are present and in correct positions - assert.Len(t, writer.segments, 3, "Should have three segments") - assert.Equal(t, 2, writer.maxSegmentIndex, "Max segment index should be 2") - - for i := 0; i < 3; i++ { - assert.Equal(t, int64(len(segments[i])), writer.segments[i].Size, "Segment %d size should match", i) - assert.NotEmpty(t, writer.segments[i].Hash, "Segment %d should have hash", i) - } - // Finalize with attributes attributes := []*policy.Value{ createTestAttribute("https://example.com/attr/Order/value/Test", testKAS1, "kid1"), } finalizeResult, err := writer.Finalize(ctx, WithAttributeValues(attributes)) require.NoError(t, err, "Should finalize successfully with out-of-order segments") - assert.NotNil(t, finalizeResult.Manifest, "Manifest should be created") + require.NotNil(t, finalizeResult.Manifest, "Manifest should be created") + + // Whatever order they were written in, the manifest must describe the + // segments in ascending index order -- that is the order a reader + // concatenates the payload in. + assert.Equal(t, 3, finalizeResult.TotalSegments, "Should have three segments") + require.Len(t, finalizeResult.Manifest.Segments, 3, "Manifest should describe three segments") + for i := range 3 { + assert.Equal(t, int64(len(segments[i])), finalizeResult.Manifest.Segments[i].Size, "Segment %d size should match", i) + assert.NotEmpty(t, finalizeResult.Manifest.Segments[i].Hash, "Segment %d should have hash", i) + } // Validate manifest against schema validateManifestSchema(t, finalizeResult.Manifest) @@ -894,12 +914,18 @@ func createTestAttributeWithAlgorithm(t *testing.T, fqn, kasURL, kid string, alg return value } -// hybridUnwrapForTest base64-decodes the wrappedKey from a manifest KAO and -// unwraps it with the matching hybrid private key, asserting the recovered DEK -// exactly matches the writer DEK. -func hybridUnwrapForTest(t *testing.T, ktype ocrypto.KeyType, privatePEM, wrappedKeyB64 string, expectedDEK []byte) { +// hybridUnwrapForTest unwraps a manifest KAO's wrappedKey with the matching +// hybrid private key and asserts the recovered key really is the DEK the +// writer used. +// +// The writer does not expose its DEK, so the check goes through the policy +// binding instead: it is an HMAC-SHA256 over the base64 policy keyed by the +// split's key, and a single-KAS TDF has exactly one split holding the whole +// DEK. Reproducing the manifest's binding from the recovered bytes therefore +// proves they match. +func hybridUnwrapForTest(t *testing.T, ktype ocrypto.KeyType, privatePEM string, manifest *Manifest, keyAccess KeyAccess) { t.Helper() - wrappedDER, err := ocrypto.Base64Decode([]byte(wrappedKeyB64)) + wrappedDER, err := ocrypto.Base64Decode([]byte(keyAccess.WrappedKey)) require.NoError(t, err, "Base64Decode wrapped key") dec, err := ocrypto.FromPrivatePEM(privatePEM) @@ -914,7 +940,14 @@ func hybridUnwrapForTest(t *testing.T, ktype ocrypto.KeyType, privatePEM, wrappe dek, err := dec.Decrypt(wrappedDER) require.NoError(t, err, "hybrid Decrypt") - assert.Equal(t, expectedDEK, dek, "%s recovered DEK", ktype) + require.Len(t, dek, 32, "%s recovered DEK should be 32 bytes", ktype) + + hash := hex.EncodeToString(ocrypto.CalculateSHA256Hmac(dek, []byte(manifest.Policy))) + expected := string(ocrypto.Base64Encode([]byte(hash))) + + binding, ok := keyAccess.PolicyBinding.(PolicyBinding) + require.True(t, ok, "policy binding should be a PolicyBinding, got %T", keyAccess.PolicyBinding) + assert.Equal(t, expected, binding.Hash, "%s recovered DEK should reproduce the policy binding", ktype) } func testHybridXWingFlow(t *testing.T) { @@ -929,7 +962,6 @@ func testHybridXWingFlow(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - expectedDEK := append([]byte(nil), writer.dek...) _, err = writer.WriteSegment(ctx, 0, []byte("hybrid xwing test data")) require.NoError(t, err) @@ -951,7 +983,7 @@ func testHybridXWingFlow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey) validateManifestSchema(t, result.Manifest) - hybridUnwrapForTest(t, ocrypto.HybridXWingKey, privPEM, keyAccess.WrappedKey, expectedDEK) + hybridUnwrapForTest(t, ocrypto.HybridXWingKey, privPEM, result.Manifest, keyAccess) } func testHybridP256MLKEM768Flow(t *testing.T) { @@ -966,7 +998,6 @@ func testHybridP256MLKEM768Flow(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - expectedDEK := append([]byte(nil), writer.dek...) _, err = writer.WriteSegment(ctx, 0, []byte("hybrid p256 mlkem768 test data")) require.NoError(t, err) @@ -988,7 +1019,7 @@ func testHybridP256MLKEM768Flow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey) validateManifestSchema(t, result.Manifest) - hybridUnwrapForTest(t, ocrypto.HybridSecp256r1MLKEM768Key, privPEM, keyAccess.WrappedKey, expectedDEK) + hybridUnwrapForTest(t, ocrypto.HybridSecp256r1MLKEM768Key, privPEM, result.Manifest, keyAccess) } func testHybridP384MLKEM1024Flow(t *testing.T) { @@ -1003,7 +1034,6 @@ func testHybridP384MLKEM1024Flow(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - expectedDEK := append([]byte(nil), writer.dek...) _, err = writer.WriteSegment(ctx, 0, []byte("hybrid p384 mlkem1024 test data")) require.NoError(t, err) @@ -1025,7 +1055,7 @@ func testHybridP384MLKEM1024Flow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey) validateManifestSchema(t, result.Manifest) - hybridUnwrapForTest(t, ocrypto.HybridSecp384r1MLKEM1024Key, privPEM, keyAccess.WrappedKey, expectedDEK) + hybridUnwrapForTest(t, ocrypto.HybridSecp384r1MLKEM1024Key, privPEM, result.Manifest, keyAccess) } // validateManifestSchema validates a TDF manifest against the JSON schema @@ -1161,8 +1191,8 @@ func testGetManifestBeforeAndAfterFinalize(t *testing.T) { require.NoError(t, err) require.NotNil(t, m0) assert.Equal(t, TDFSpecVersion, m0.TDFVersion) - assert.Equal(t, tdfAsZip, m0.Protocol) - assert.Equal(t, tdfZipReference, m0.Type) + assert.Equal(t, "zip", m0.Protocol) + assert.Equal(t, "reference", m0.Type) assert.True(t, m0.IsEncrypted) // No segments yet assert.Empty(t, m0.Segments) From 68cf9e3ca5d346efeec4bce86b86381a9bd2a2e6 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 23 Sep 2026 13:42:05 -0400 Subject: [PATCH 3/3] Apply suggestion from @dmihalcik-virtru --- sdk/experimental/tdf/manifest.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/experimental/tdf/manifest.go b/sdk/experimental/tdf/manifest.go index 566500753b..cffcc7ddcf 100644 --- a/sdk/experimental/tdf/manifest.go +++ b/sdk/experimental/tdf/manifest.go @@ -110,6 +110,5 @@ var ( // algorithm other than HS256. ErrUnsupportedRootIntegrityAlgorithm = sdk.ErrUnsupportedRootIntegrityAlgorithm // ErrUnsupportedSegmentIntegrityAlgorithm rejects a segment algorithm that - // is neither HS256 nor GMAC. ErrUnsupportedSegmentIntegrityAlgorithm = sdk.ErrUnsupportedSegmentIntegrityAlgorithm )