Skip to content

feat(sdk): graduate the chunked writer to stable API #3949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dmihalcik-virtru wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(sdk): graduate the chunked writer to stable API #3949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 40 additions & 31 deletions sdk/chunked_options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,16 @@ import (
"github.com/opentdf/platform/protocol/go/policy"
)

// Each injection-seam option below rejects nil rather than storing it.
// A nil seam is not detectable later: the config field is
// indistinguishable from "not set", so NewChunkedWriter installs no
// default and the nil is dereferenced during writing -- for the
// splitter, not until Finalize, long after the caller has encrypted
// every segment.
// Every option below that takes a pointer or an interface rejects nil
// rather than storing it. A stored nil is not detectable later: the
// config field is indistinguishable from "not set". For an injection
// seam that means NewChunkedWriter installs no default and the nil is
// dereferenced during writing -- for the splitter, not until Finalize,
// long after the caller has encrypted every segment. For the default
// KAS it is worse than a panic, because nothing fails: key access
// silently falls back to the platform base key, and the caller learns
// their data went to a KAS they never named only when a reader cannot
// unwrap it.

// The slice-valued options below clone what they are given. Each is retained
// for the lifetime of the writer or of one Finalize call, and each determines
Expand Down Expand Up @@ -72,8 +76,6 @@ func withChunkedClock(clock clock) ChunkedWriterOption {

// WithChunkedInitialAttributes sets attribute values used by Finalize
// when the Finalize call does not supply its own.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedInitialAttributes(values []*policy.Value) ChunkedWriterOption {
return func(c *chunkedWriterConfig) error {
c.initialAttributes = slices.Clone(values)
Expand All @@ -82,11 +84,13 @@ func WithChunkedInitialAttributes(values []*policy.Value) ChunkedWriterOption {
}

// WithChunkedDefaultKAS sets the default KAS used by Finalize when
// the Finalize call does not supply its own.
//
// Experimental: not part of the stable SDK API; may change or be removed.
// the Finalize call does not supply its own. The KAS must not be nil:
// omit the option to leave key access to be resolved some other way.
func WithChunkedDefaultKAS(kas *policy.SimpleKasKey) ChunkedWriterOption {
return func(c *chunkedWriterConfig) error {
if kas == nil {
return errors.New("chunked: default KAS must not be nil")
}
c.initialDefaultKAS = kas
return nil
}
Expand All @@ -96,14 +100,13 @@ func WithChunkedDefaultKAS(kas *policy.SimpleKasKey) ChunkedWriterOption {
// [ChunkedWriter]. Callers with multi-KAS attribute grants should
// inject a splitter that understands their grant model. The splitter
// must not be nil.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedKeySplitter(splitter KeySplitter) ChunkedWriterOption {
return func(c *chunkedWriterConfig) error {
if splitter == nil {
return errors.New("chunked: key splitter must not be nil")
}
c.splitter = splitter
c.splitterSet = true
return nil
}
}
Expand All @@ -120,12 +123,29 @@ func withChunkedRand(r io.Reader) ChunkedWriterOption {
}
}

// WithChunkedTDFOptions supplies the key access options — attributes, KAS
// information, preferred wrapping algorithm — that SDK.NewChunkedWriter
// resolves against the platform at Finalize. It has no effect on the
// package-level NewChunkedWriter, which has no platform to resolve against;
// use WithChunkedKeySplitter there.
func WithChunkedTDFOptions(opts ...TDFOption) ChunkedWriterOption {
return func(c *chunkedWriterConfig) error {
// Checked here because the options are not applied until Finalize,
// where a nil would panic after every segment has been handed back.
for _, opt := range opts {
if opt == nil {
return errors.New("chunked: TDF option must not be nil")
}
}
c.tdfOptions = append(c.tdfOptions, opts...)
return nil
}
}

// WithChunkedAssertions attaches signed assertions to the produced
// TDF. Each assertion is bound to the payload's aggregate hash, so
// they are signed at Finalize once every segment is in. Assertions
// without their own SigningKey are signed with HS256 over the DEK.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedAssertions(assertions []AssertionConfig) ChunkedFinalizeOption {
return func(c *chunkedFinalizeConfig) error {
c.assertions = slices.Clone(assertions)
Expand All @@ -143,8 +163,6 @@ func WithChunkedAssertions(assertions []AssertionConfig) ChunkedFinalizeOption {
// silently would loosen the policy on the data, which is the one
// mistake here that cannot be detected after the fact. Construct a
// writer without WithChunkedInitialAttributes instead.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedAttributes(values []*policy.Value) ChunkedFinalizeOption {
return func(c *chunkedFinalizeConfig) error {
c.attributes = slices.Clone(values)
Expand All @@ -153,14 +171,13 @@ func WithChunkedAttributes(values []*policy.Value) ChunkedFinalizeOption {
}

// WithChunkedDefaultKASForFinalize overrides the writer's initial
// default KAS for this Finalize call.
//
// A nil argument reads as "not specified", so the writer's initial
// default KAS still applies; there is no way to unset it for one call.
//
// Experimental: not part of the stable SDK API; may change or be removed.
// default KAS for this Finalize call. The KAS must not be nil: omit
// the option to keep whatever WithChunkedDefaultKAS set.
func WithChunkedDefaultKASForFinalize(kas *policy.SimpleKasKey) ChunkedFinalizeOption {
return func(c *chunkedFinalizeConfig) error {
if kas == nil {
return errors.New("chunked: default KAS must not be nil")
}
c.defaultKAS = kas
return nil
}
Expand All @@ -169,8 +186,6 @@ func WithChunkedDefaultKASForFinalize(kas *policy.SimpleKasKey) ChunkedFinalizeO
// WithChunkedEncryptedMetadata attaches AES-GCM-encrypted metadata to
// every KAO in the TDF. The metadata is keyed on the split share and
// only decryptable by a reader that has been granted access.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedEncryptedMetadata(metadata string) ChunkedFinalizeOption {
return func(c *chunkedFinalizeConfig) error {
c.encryptedMetadata = metadata
Expand All @@ -181,8 +196,6 @@ func WithChunkedEncryptedMetadata(metadata string) ChunkedFinalizeOption {
// WithChunkedTargetMode targets a specific TDF spec version, given as
// a semver string such as "4.2.2". An empty mode selects the most
// recently available target version (4.3.0).
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedTargetMode(mode string) ChunkedWriterOption {
return func(c *chunkedWriterConfig) error {
if mode == "" {
Expand All @@ -201,8 +214,6 @@ func WithChunkedTargetMode(mode string) ChunkedWriterOption {
}

// WithChunkedMimeType records the payload MIME type in the manifest.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedMimeType(mimeType string) ChunkedFinalizeOption {
return func(c *chunkedFinalizeConfig) error {
c.mimeType = mimeType
Expand Down Expand Up @@ -234,8 +245,6 @@ func WithChunkedMimeType(mimeType string) ChunkedFinalizeOption {
// excludes from the manifest -- must still be appended by the caller
// when assembling the final file. Skipping a dropped segment's bytes
// produces an archive whose central directory offsets overshoot.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedSegments(indices []int) ChunkedFinalizeOption {
return func(c *chunkedFinalizeConfig) error {
// Cloned because segmentOrderLocked validates the live slice before
Expand Down
193 changes: 187 additions & 6 deletions sdk/chunked_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,15 @@ func newChunkedWriterForTest(ctx context.Context, t *testing.T, opts ...ChunkedW
return w, kasBundle
}

// PublicKey serves the wrapping key, so that a caller who names this KAS by URL
// alone can have its key fetched rather than having to supply the PEM.
func (k *chunkedFakeKAS) PublicKey(_ context.Context, _ *connect.Request[kaspb.PublicKeyRequest]) (*connect.Response[kaspb.PublicKeyResponse], error) {
return connect.NewResponse(&kaspb.PublicKeyResponse{
PublicKey: k.publicPEM,
Kid: k.kid,
}), nil
}

// Rewrap unwraps every RSA-wrapped KAO under the KAS private key and
// re-wraps under the caller's session public key.
//
Expand Down Expand Up @@ -927,12 +936,14 @@ func TestChunkedTargetModeInvalid(t *testing.T) {
assert.Contains(t, err.Error(), "not-a-version")
}

// TestChunkedOptionsRejectNil checks that the injection-seam options
// refuse a nil value instead of storing it. A stored nil is
// indistinguishable from an unset field, so no default gets installed
// and the nil surfaces as a panic partway through writing -- for the
// key splitter, not until Finalize, after the caller has already
// encrypted and uploaded every segment.
// TestChunkedOptionsRejectNil checks that every option taking a
// pointer or an interface refuses a nil value instead of storing it. A
// stored nil is indistinguishable from an unset field, so no default
// gets installed and the nil surfaces as a panic partway through
// writing -- for the key splitter, not until Finalize, after the caller
// has already encrypted and uploaded every segment. The default KAS
// fails more quietly still: it does not panic at all, it just sends the
// data to the platform base key.
func TestChunkedOptionsRejectNil(t *testing.T) {
ctx := context.Background()
kasBundle := newChunkedFakeKAS(t)
Expand All @@ -945,8 +956,10 @@ func TestChunkedOptionsRejectNil(t *testing.T) {
{"archive writer factory", withChunkedArchiveWriterFactory(nil)},
{"cipher factory", withChunkedCipherFactory(nil)},
{"clock", withChunkedClock(nil)},
{"default KAS", WithChunkedDefaultKAS(nil)},
{"key splitter", WithChunkedKeySplitter(nil)},
{"rand", withChunkedRand(nil)},
{"TDF option", WithChunkedTDFOptions(WithAutoconfigure(false), nil)},
} {
t.Run(tc.name, func(t *testing.T) {
writer, err := NewChunkedWriter(ctx,
Expand All @@ -958,6 +971,18 @@ func TestChunkedOptionsRejectNil(t *testing.T) {
assert.Nil(t, writer)
})
}

t.Run("default KAS for finalize", func(t *testing.T) {
writer, err := NewChunkedWriter(ctx, WithChunkedDefaultKAS(kasBundle.simpleKey()))
require.NoError(t, err)

_, err = writer.WriteSegment(ctx, 0, []byte("hello"))
require.NoError(t, err)

_, err = writer.Finalize(ctx, WithChunkedDefaultKASForFinalize(nil))
require.Error(t, err)
assert.Contains(t, err.Error(), "must not be nil")
})
}

// TestChunkedIntegrityAlgorithmsAreFixed pins the only two algorithms
Expand Down Expand Up @@ -2631,3 +2656,159 @@ func TestChunkedGetManifestDoesNotBlockWriteSegment(t *testing.T) {
require.NoError(t, err)
assert.Len(t, later.Segments, 2)
}

// TestSDKChunkedWriterResolvesKeyAccess covers SDK.NewChunkedWriter, whose
// whole point over the package-level constructor is that key access comes from
// the platform rather than from a KeySplitter the caller had to write. The KAS
// info here carries no public key, so passing the round trip means the writer
// went out and fetched it.
func TestSDKChunkedWriterResolvesKeyAccess(t *testing.T) {
ctx := context.Background()
kasBundle := newChunkedFakeKAS(t)
defer kasBundle.server.Close()

s := newChunkedTestSDK(t)

writer, err := s.NewChunkedWriter(ctx, WithChunkedTDFOptions(
// Autoconfigure off: this SDK is pointed at a bare KAS, with no policy
// service to ask which KAS grants which attribute.
WithAutoconfigure(false),
WithKasInformation(KASInfo{URL: kasBundle.url}),
))
require.NoError(t, err)

body := writeChunkedSegments(ctx, t, writer, [][]byte{[]byte("sdk-"), []byte("chunked")})
fin, err := writer.Finalize(ctx)
require.NoError(t, err)
require.Len(t, fin.Manifest.KeyAccessObjs, 1)
assert.Equal(t, kasBundle.url, fin.Manifest.KeyAccessObjs[0].KasURL)
assert.Equal(t, kasBundle.kid, fin.Manifest.KeyAccessObjs[0].KID)

reader, err := s.LoadTDF(bytes.NewReader(append(body, fin.Data...)),
WithKasAllowlist([]string{kasBundle.url}),
)
require.NoError(t, err)
plain, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, []byte("sdk-chunked"), plain)
}

// TestSDKChunkedWriterKeepsAnExplicitSplitter checks that a caller who supplies
// their own splitter keeps it. The TDF options name a KAS that does not exist,
// so platform resolution would fail loudly rather than quietly produce the same
// answer.
func TestSDKChunkedWriterKeepsAnExplicitSplitter(t *testing.T) {
ctx := context.Background()
kasBundle := newChunkedFakeKAS(t)
defer kasBundle.server.Close()

s := newChunkedTestSDK(t)

writer, err := s.NewChunkedWriter(ctx,
WithChunkedKeySplitter(DefaultKeySplitter()),
WithChunkedDefaultKAS(kasBundle.simpleKey()),
WithChunkedTDFOptions(
WithAutoconfigure(false),
WithKasInformation(KASInfo{URL: "https://kas.invalid"}),
),
)
require.NoError(t, err)

body := writeChunkedSegments(ctx, t, writer, [][]byte{[]byte("explicit splitter")})
fin, err := writer.Finalize(ctx)
require.NoError(t, err)
require.Len(t, fin.Manifest.KeyAccessObjs, 1)
assert.Equal(t, kasBundle.url, fin.Manifest.KeyAccessObjs[0].KasURL)

reader, err := s.LoadTDF(bytes.NewReader(append(body, fin.Data...)),
WithKasAllowlist([]string{kasBundle.url}),
)
require.NoError(t, err)
plain, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, []byte("explicit splitter"), plain)
}

// TestSDKChunkedWriterRejectsGrantsUnderDefaultKAS checks that a pinned default
// KAS does not silently override an attribute that names its own KAS. Pinning
// turns autoconfigure off, so without the check the grant would be dropped and
// the key wrapped only to the default KAS.
func TestSDKChunkedWriterRejectsGrantsUnderDefaultKAS(t *testing.T) {
ctx := context.Background()
kasBundle := newChunkedFakeKAS(t)
defer kasBundle.server.Close()

s := newChunkedTestSDK(t)
granted := &policy.Value{
Fqn: "https://example.com/attr/a/value/b",
KasKeys: []*policy.SimpleKasKey{kasBundle.simpleKey()},
}

t.Run("initial attributes", func(t *testing.T) {
writer, err := s.NewChunkedWriter(ctx,
WithChunkedDefaultKAS(kasBundle.simpleKey()),
WithChunkedInitialAttributes([]*policy.Value{granted}),
)
require.NoError(t, err)
_, err = writer.WriteSegment(ctx, 0, []byte("granted"))
require.NoError(t, err)

_, err = writer.Finalize(ctx)
require.ErrorIs(t, err, ErrSplitterIgnoresGrants)
})

t.Run("TDF options", func(t *testing.T) {
writer, err := s.NewChunkedWriter(ctx,
WithChunkedDefaultKAS(kasBundle.simpleKey()),
WithChunkedTDFOptions(WithDataAttributeValues(granted)),
)
require.NoError(t, err)
_, err = writer.WriteSegment(ctx, 0, []byte("granted"))
require.NoError(t, err)

_, err = writer.Finalize(ctx)
require.ErrorIs(t, err, ErrSplitterIgnoresGrants)
})
}

// TestSDKChunkedWriterFallsBackToBaseKey pins what happens when the caller
// names no KAS at all: SDK.NewChunkedWriter leaves autoconfigure on, finds no
// attribute grants, and wraps to the platform base key. Nothing errors, so this
// is the case that silently sends data somewhere the caller did not choose --
// which is why WithChunkedDefaultKAS rejects nil rather than treating it as
// "unset", and why the constructor's doc comment spells the fallback out.
func TestSDKChunkedWriterFallsBackToBaseKey(t *testing.T) {
ctx := context.Background()
kasBundle := newChunkedFakeKAS(t)
defer kasBundle.server.Close()

s := newChunkedTestSDK(t)
s.wellknownConfiguration = newMockWellKnownService(map[string]interface{}{
baseKeyWellKnown: map[string]interface{}{
"kas_uri": kasBundle.url,
baseKeyPublicKey: map[string]interface{}{
baseKeyAlg: "rsa:2048",
"kid": kasBundle.kid,
"pem": kasBundle.publicPEM,
},
},
}, nil)

writer, err := s.NewChunkedWriter(ctx)
require.NoError(t, err)

body := writeChunkedSegments(ctx, t, writer, [][]byte{[]byte("base key")})
fin, err := writer.Finalize(ctx)
require.NoError(t, err)
require.Len(t, fin.Manifest.KeyAccessObjs, 1)
assert.Equal(t, kasBundle.url, fin.Manifest.KeyAccessObjs[0].KasURL)
assert.Equal(t, kasBundle.kid, fin.Manifest.KeyAccessObjs[0].KID)

reader, err := s.LoadTDF(bytes.NewReader(append(body, fin.Data...)),
WithKasAllowlist([]string{kasBundle.url}),
)
require.NoError(t, err)
plain, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, []byte("base key"), plain)
}
Loading
Loading