Skip to content

chore(sdk): Lets CreateTDF use ChunkedWriter #3946

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dmihalcik-virtru merged 2 commits into from
Sep 24, 2026
Merged

chore(sdk): Lets CreateTDF use ChunkedWriter #3946

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9318a77
chore(sdk): rewrite CreateTDF on top of the chunked writer
dmihalcik-virtru Sep 1, 2026
0a6f0c7
Apply suggestion from @dmihalcik-virtru
dmihalcik-virtru Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 21 additions & 24 deletions sdk/chunked_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -736,19 +736,17 @@ func TestChunkedECKeyAccess(t *testing.T) {
for i := range dek {
dek[i] = byte(i)
}
splits := &SplitResult{
KASPublicKeys: map[string]KASPublicKey{
kasURL: {
Algorithm: ocrypto.EC256Key,
KID: "ec-kid",
PEM: pubPEM,
URL: kasURL,
},
},
Splits: []Split{{Data: dek, KASURLs: []string{kasURL}}},
}
shares := []splitShare{{
data: dek,
kases: []KASInfo{{
URL: kasURL,
PublicKey: pubPEM,
KID: "ec-kid",
Algorithm: string(ocrypto.EC256Key),
}},
}}

kaos, err := buildChunkedKeyAccessObjects(splits, dek, []byte(`{"uuid":"test"}`), "")
kaos, err := buildKeyAccessObjects(shares, `{"uuid":"test"}`, "")
require.NoError(t, err)
require.Len(t, kaos, 1)

Expand Down Expand Up @@ -800,22 +798,21 @@ func TestChunkedKeyAccessRejectsShareWithNoKAS(t *testing.T) {

const kasURL = "https://kas.example.com"
dek := make([]byte, kKeySize)
splits := &SplitResult{
KASPublicKeys: map[string]KASPublicKey{
kasURL: {
Algorithm: ocrypto.EC256Key,
KID: "ec-kid",
PEM: pubPEM,
shares := []splitShare{
{
id: "wrapped",
data: dek,
kases: []KASInfo{{
URL: kasURL,
},
},
Splits: []Split{
{ID: "wrapped", Data: dek, KASURLs: []string{kasURL}},
{ID: "orphaned", Data: dek},
PublicKey: pubPEM,
KID: "ec-kid",
Algorithm: string(ocrypto.EC256Key),
}},
},
{id: "orphaned", data: dek},
}

_, err = buildChunkedKeyAccessObjects(splits, dek, []byte(`{"uuid":"test"}`), "")
_, err = buildKeyAccessObjects(shares, `{"uuid":"test"}`, "")
require.Error(t, err, "a share with no KAS to unwrap it makes the DEK unrecoverable")
assert.Contains(t, err.Error(), "orphaned", "the error must name the split that cannot be recovered")
}
Expand Down
164 changes: 57 additions & 107 deletions sdk/chunked_writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"sync"
"time"

"github.com/google/uuid"
"github.com/opentdf/platform/lib/ocrypto"
"github.com/opentdf/platform/protocol/go/policy"
"github.com/opentdf/platform/sdk/internal/zipstream"
Expand Down Expand Up @@ -362,6 +361,11 @@ type chunkedWriterConfig struct {
// fixedClock for deterministic ZIP output.
clock clock

// dek is a pre-generated Data Encryption Key. When nil the writer
// draws one from rand. SDK.CreateTDF presets it so that it can
// resolve key access before emitting any payload bytes.
dek []byte

// excludeVersion omits the schemaVersion field from the manifest.
// Set together with useHex by WithChunkedTargetMode; readers use
// the field's absence as the pre-4.3.0 marker, so the two must
Expand All @@ -376,12 +380,22 @@ type chunkedWriterConfig struct {
// Finalize call does not supply its own.
initialDefaultKAS *policy.SimpleKasKey

// keyAccess resolves the manifest policy and key access objects.
// Defaults to a splitterKeyAccess over splitter.
keyAccess keyAccessResolver

// rand is the entropy source used to generate the DEK. Defaults
// to crypto/rand.Reader.
rand io.Reader

// segmentSize is the plaintext segment size advertised in the
// manifest. Zero means "report the first segment's actual size",
// which is right when every segment is the same length.
segmentSize int64

// splitter maps attribute values to DEK splits at Finalize time.
// Defaults to DefaultKeySplitter (single-KAS only).
// Defaults to DefaultKeySplitter (single-KAS only). Ignored when
// keyAccess is set.
splitter KeySplitter

// useHex hex-encodes segment, root, and assertion signatures
Expand Down Expand Up @@ -498,6 +512,10 @@ type chunkedWriter struct {
// when the caller does not override.
initialDefaultKAS *policy.SimpleKasKey

// keyAccess resolves the manifest policy and key access objects
// for the DEK.
keyAccess keyAccessResolver

// manifest holds the finalized manifest for post-Finalize
// GetManifest calls.
manifest *Manifest
Expand All @@ -508,9 +526,9 @@ type chunkedWriter struct {
// segments records per-index slots, reserved and then written.
segments map[int]*segmentSlot

// splitter converts attributes + DEK into key splits at
// Finalize time.
splitter KeySplitter
// segmentSize is the plaintext segment size to advertise in the
// manifest, or zero to infer it from the first segment.
segmentSize int64

// useHex selects the pre-4.3.0 doubly-encoded signature form.
// Read by WriteSegment, so it is fixed at construction rather
Expand Down Expand Up @@ -552,24 +570,40 @@ func NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWr
return nil, err
}
}
return newChunkedWriter(cfg)
}

dek := make([]byte, kKeySize)
if _, err := io.ReadFull(cfg.rand, dek); err != nil {
return nil, fmt.Errorf("generate DEK: %w", err)
// newChunkedWriter builds the writer from a fully-populated config.
// SDK.CreateTDF calls this directly with the unexported knobs its
// classic behavior needs — a preset DEK, key access resolved before
// the first payload byte, a fixed segment size — rather than going
// through the public option set.
func newChunkedWriter(cfg chunkedWriterConfig) (*chunkedWriter, error) {
dek := cfg.dek
if dek == nil {
dek = make([]byte, kKeySize)
if _, err := io.ReadFull(cfg.rand, dek); err != nil {
return nil, fmt.Errorf("generate DEK: %w", err)
}
}
block, err := cfg.cipherFactory(dek)
if err != nil {
return nil, fmt.Errorf("build segment cipher: %w", err)
}
keyAccess := cfg.keyAccess
if keyAccess == nil {
keyAccess = splitterKeyAccess{splitter: cfg.splitter}
}
return &chunkedWriter{
archiveWriter: cfg.archiveFactory(cfg.clock),
block: block,
dek: dek,
excludeVersion: cfg.excludeVersion,
initialAttributes: cfg.initialAttributes,
initialDefaultKAS: cfg.initialDefaultKAS,
keyAccess: keyAccess,
segments: make(map[int]*segmentSlot),
splitter: cfg.splitter,
segmentSize: cfg.segmentSize,
useHex: cfg.useHex,
}, nil
}
Expand Down Expand Up @@ -1003,44 +1037,24 @@ type chunkedTotals struct {
plaintext int64
}

// buildManifest composes the manifest from a snapshot, splits the DEK,
// wraps splits into KAOs, and computes the root signature.
// buildManifest composes the manifest from a snapshot, resolves the key
// access objects, and computes the root signature.
//
// It reads no mutable writer state and takes no lock: every other field it
// touches (dek, splitter, useHex) is fixed at construction. Keep it that way --
// touches (dek, keyAccess, useHex) is fixed at construction. Keep it that way --
// GetManifest calls it with the read lock released.
func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *chunkedFinalizeConfig, snap *chunkedSnapshot) (*Manifest, chunkedTotals, error) {
var totals chunkedTotals

// Hand the splitter copies of both slices. A splitter that zeroes or
// rewrites the DEK it is given -- scrubbing what it thinks is its own
// working buffer, say -- would desynchronize the DEK from the segment
// signatures already computed against it and from the root signature
// computed below, producing a TDF that fails verification exactly as a
// tampered one would.
//
// The attributes need the same defense for a different reason: on the
// fallback path cfg.attributes *is* w.initialAttributes, the writer's own
// retained slice, so a splitter that rewrites an element changes the policy
// buildChunkedPolicy writes just below -- and every later Finalize and
// GetManifest on this writer.
splits, err := w.splitter.Split(ctx, slices.Clone(cfg.attributes), slices.Clone(w.dek), cfg.defaultKAS)
if err != nil {
return nil, totals, err
}
policyBytes, err := buildChunkedPolicy(cfg.attributes)
if err != nil {
return nil, totals, err
}
kaos, err := buildChunkedKeyAccessObjects(splits, w.dek, policyBytes, cfg.encryptedMetadata)
base64Policy, kaos, err := w.keyAccess.resolve(ctx, w.dek, cfg)
if err != nil {
return nil, totals, err
}

encInfo := EncryptionInformation{
KeyAccessObjs: kaos,
KeyAccessType: kSplitKeyType,
Policy: string(ocrypto.Base64Encode(policyBytes)),
Policy: base64Policy,
Method: Method{
Algorithm: kGCMCipherAlgorithm,
IsStreamable: true,
Expand All @@ -1066,7 +1080,15 @@ func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *chunkedFinalizeC
aggregate.Write(decoded)
}

if len(snap.segments) > 0 {
// A caller that knows the segment size says so, because the first
// segment's actual length is only the right answer when every
// segment is full — and the last one usually is not, so a
// single-segment TDF would otherwise advertise a short default.
switch {
case w.segmentSize > 0:
encInfo.DefaultSegmentSize = w.segmentSize
encInfo.DefaultEncryptedSegSize = w.segmentSize + gcmIvSize + aesBlockSize
case len(snap.segments) > 0:
encInfo.DefaultEncryptedSegSize = snap.segments[0].EncryptedSize
encInfo.DefaultSegmentSize = snap.segments[0].Size
}
Expand Down Expand Up @@ -1165,78 +1187,6 @@ func (w *chunkedWriter) segmentOrderLocked(keep []int) ([]int, error) {
return out, nil
}

// buildChunkedKeyAccessObjects wraps each split share to each KAS
// listed by the splitter.
func buildChunkedKeyAccessObjects(splits *SplitResult, dek, policyBytes []byte, metadata string) ([]KeyAccess, error) {
// This is the one place caller-supplied split data is turned into
// manifest content, so it is where the splitter's contract is
// enforced -- for the default splitter and for anything injected
// through WithChunkedKeySplitter alike. Validate guarantees every
// invariant the loop below relies on: at least one split, every
// split naming at least one KAS, every named KAS resolving to a
// usable wrapping key, and split ids that the reader can group on.
if err := splits.Validate(); err != nil {
return nil, err
}
// Validate first, then this: Validate names the specific structural fault
// (no splits, empty share, duplicate id, unresolved KAS), all of which
// VerifyReconstruction can only report as a length or value mismatch.
if err := splits.VerifyReconstruction(dek); err != nil {
return nil, err
}
base64Policy := ocrypto.Base64Encode(policyBytes)

out := make([]KeyAccess, 0, len(splits.Splits))
for _, split := range splits.Splits {
// Policy binding and metadata are keyed on the split share, not
// on the KAS, so compute them once per split rather than once
// per KAS URL in an OR-group.
policyBinding := createPolicyBinding(split.Data, base64Policy)
var encMeta string
if metadata != "" {
m, err := encryptMetadata(split.Data, metadata)
if err != nil {
return nil, fmt.Errorf("encrypt metadata for split %s: %w", split.ID, err)
}
encMeta = m
}
for _, url := range split.KASURLs {
// Validate resolved every URL, so the lookup cannot miss.
pk := splits.KASPublicKeys[url]
kao, err := createKeyAccess(pk.toKASInfo(), split.Data, policyBinding, encMeta, split.ID)
if err != nil {
return nil, fmt.Errorf("wrap key for %s: %w", url, err)
}
out = append(out, kao)
}
}
return out, nil
}

// buildChunkedPolicy composes the TDF Policy document from attribute
// values.
func buildChunkedPolicy(values []*policy.Value) ([]byte, error) {
p := PolicyObject{UUID: uuid.NewString()}
p.Body.DataAttributes = make([]attributeObject, 0, len(values))
p.Body.Dissem = make([]string, 0)
for i, v := range values {
// GetFqn is nil-receiver safe, so a nil element or a value that never
// had its FQN populated yields "" rather than panicking. Emitting that
// writes {"attribute": ""} into the bound policy: the PDP denies it, so
// the TDF fails closed rather than open, but it fails at decrypt with
// nothing naming the cause -- and it does so after the caller has
// encrypted and uploaded the whole payload.
fqn := v.GetFqn()
if fqn == "" {
return nil, fmt.Errorf("chunked: attribute value at index %d has an empty FQN; the policy it produces names no attribute and no KAS could evaluate it", i)
}
p.Body.DataAttributes = append(p.Body.DataAttributes, attributeObject{
Attribute: fqn,
})
}
return json.Marshal(p)
}

// cloneChunkedManifest copies the manifest struct and clones its three
// slices, which is what makes the result safe to hand to a caller: it
// can append to or overwrite elements of Segments, KeyAccessObjs and
Expand Down
Loading
Loading