diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index b073b955..0f6b57f8 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -15,12 +15,15 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; import java.nio.channels.FileChannel; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; @@ -77,7 +80,8 @@ class Command { static class Supports implements Callable { // Static, always-on capabilities of this build. There is no "known but // unsupported" feature here: anything not in this list is simply unrecognized. - private static final List FEATURES = List.of("dpop", "dpop_nonce_challenge"); + private static final List FEATURES = List.of("dpop", "dpop_nonce_challenge", + "partial_decrypt", "http_range_source"); @CommandLine.Parameters(index = "0", arity = "0..1", description = "Feature to check (e.g., dpop). Omit to list all supported features.") private String feature; @@ -354,7 +358,10 @@ private SDK buildSDK() { @CommandLine.Command(name = "decrypt") void decrypt( - @Option(names = { "-f", "--file" }, required = true) Path tdfPath, + @Option(names = { "-f", "--file" }, required = true, description = "TDF file path, or an http(s):// URL read with range requests") String tdfFile, + @Option(names = { "-o", "--output" }, defaultValue = Option.NULL_VALUE, description = "Write plaintext here, only on success (default: stdout)") Optional output, + @Option(names = { "--offset" }, defaultValue = "0", description = "Plaintext byte offset to start decrypting from") long offset, + @Option(names = { "--length" }, defaultValue = "-1", description = "Plaintext bytes to decrypt from --offset; negative decrypts to the end") long length, @Option(names = { "--rewrap-key-type" }, defaultValue = Option.NULL_VALUE, description = "Preferred rewrap algorithm, one of ${COMPLETION-CANDIDATES}") Optional rewrapKeyType, @Option(names = { @@ -367,60 +374,89 @@ void decrypt( throws Exception { try (var sdk = buildSDK()) { var opts = new ArrayList>(); - try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { - try (var stdout = new BufferedOutputStream(System.out)) { - if (assertionVerification.isPresent()) { - var assertionVerificationInput = assertionVerification.get(); - Gson gson = buildGson(); - - Config.AssertionVerificationKeys assertionVerificationKeys; - try { - assertionVerificationKeys = gson.fromJson(assertionVerificationInput, - Config.AssertionVerificationKeys.class); - } catch (JsonSyntaxException e) { - // try it as a file path - try { - String fileJson = Files.readString(Paths.get(assertionVerificationInput)); - assertionVerificationKeys = gson.fromJson(fileJson, - Config.AssertionVerificationKeys.class); - } catch (JsonSyntaxException e2) { - throw new RuntimeException("Failed to parse assertion verification keys from file", e2); - } catch (Exception e3) { - throw new RuntimeException( - "Could not parse assertion verification keys as json string or path to file", - e3); - } - } - - for (Map.Entry entry : assertionVerificationKeys.keys - .entrySet()) { - try { - Object correctedKey = correctKeyType(entry.getValue().alg, entry.getValue().key, true); - entry.setValue(new AssertionConfig.AssertionKey(entry.getValue().alg, correctedKey)); - } catch (Exception e) { - throw new RuntimeException("Error with assertion verification key: " + e.getMessage(), - e); - } - } - opts.add(Config.withAssertionVerificationKeys(assertionVerificationKeys)); + if (assertionVerification.isPresent()) { + var assertionVerificationInput = assertionVerification.get(); + Gson gson = buildGson(); + + Config.AssertionVerificationKeys assertionVerificationKeys; + try { + assertionVerificationKeys = gson.fromJson(assertionVerificationInput, + Config.AssertionVerificationKeys.class); + } catch (JsonSyntaxException e) { + // try it as a file path + try { + String fileJson = Files.readString(Paths.get(assertionVerificationInput)); + assertionVerificationKeys = gson.fromJson(fileJson, + Config.AssertionVerificationKeys.class); + } catch (JsonSyntaxException e2) { + throw new RuntimeException("Failed to parse assertion verification keys from file", e2); + } catch (Exception e3) { + throw new RuntimeException( + "Could not parse assertion verification keys as json string or path to file", + e3); } + } - if (disableAssertionVerification) { - opts.add(Config.withDisableAssertionVerification(true)); + for (Map.Entry entry : assertionVerificationKeys.keys + .entrySet()) { + try { + Object correctedKey = correctKeyType(entry.getValue().alg, entry.getValue().key, true); + entry.setValue(new AssertionConfig.AssertionKey(entry.getValue().alg, correctedKey)); + } catch (Exception e) { + throw new RuntimeException("Error with assertion verification key: " + e.getMessage(), + e); } - rewrapKeyType.map(Config::WithSessionKeyType).ifPresent(opts::add); + } + opts.add(Config.withAssertionVerificationKeys(assertionVerificationKeys)); + } + + if (disableAssertionVerification) { + opts.add(Config.withDisableAssertionVerification(true)); + } + rewrapKeyType.map(Config::WithSessionKeyType).ifPresent(opts::add); - ignoreAllowlist.ifPresent(aBoolean -> opts.add(Config.WithIgnoreKasAllowlist(aBoolean))); - kasAllowlistStr.ifPresent(s -> opts.add(Config.WithKasAllowlist(s.split(",")))); + ignoreAllowlist.ifPresent(aBoolean -> opts.add(Config.WithIgnoreKasAllowlist(aBoolean))); + kasAllowlistStr.ifPresent(s -> opts.add(Config.WithKasAllowlist(s.split(",")))); - var readerConfig = Config.newTDFReaderConfig(opts.toArray(new Consumer[0])); + var readerConfig = Config.newTDFReaderConfig(opts.toArray(new Consumer[0])); + + if (tdfFile.startsWith("http://") || tdfFile.startsWith("https://")) { + var reader = sdk.loadTDF(URI.create(tdfFile), readerConfig); + writePlaintext(out -> reader.readPayload(out, offset, length), output); + } else { + try (var in = FileChannel.open(Path.of(tdfFile), StandardOpenOption.READ)) { var reader = sdk.loadTDF(in, readerConfig); - reader.readPayload(stdout); + writePlaintext(out -> reader.readPayload(out, offset, length), output); } } } } + // With an output file, write to a sibling temp file and move it into place only on + // success, so a failed decrypt (e.g. an offset past the end) leaves no file behind. + private interface PlaintextWriter { + void writeTo(OutputStream out) throws Exception; + } + + private static void writePlaintext(PlaintextWriter plaintext, Optional output) throws Exception { + if (output.isEmpty()) { + try (var stdout = new BufferedOutputStream(System.out)) { + plaintext.writeTo(stdout); + } + return; + } + Path target = output.get().toAbsolutePath(); + Path tmp = Files.createTempFile(target.getParent(), target.getFileName() + ".", ".tmp"); + try { + try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(tmp))) { + plaintext.writeTo(out); + } + Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } finally { + Files.deleteIfExists(tmp); + } + } + @CommandLine.Command(name = "metadata") void readMetadata( @Option(names = { "-f", "--file" }, required = true) Path tdfPath, diff --git a/cmdline/src/test/java/io/opentdf/platform/CommandTest.java b/cmdline/src/test/java/io/opentdf/platform/CommandTest.java index 8995271d..aa95a5b0 100644 --- a/cmdline/src/test/java/io/opentdf/platform/CommandTest.java +++ b/cmdline/src/test/java/io/opentdf/platform/CommandTest.java @@ -44,6 +44,12 @@ void supports_dpop_nonce_challenge_exits_0() { assertThat(code).isEqualTo(0); } + @Test + void supports_partial_decrypt_and_http_range_source_exit_0() { + assertThat(new CommandLine(new Command()).execute("supports", "partial_decrypt")).isEqualTo(0); + assertThat(new CommandLine(new Command()).execute("supports", "http_range_source")).isEqualTo(0); + } + @Test void supports_unknown_feature_exits_1() { int code = new CommandLine(new Command()).execute("supports", "unknown_feature"); @@ -95,7 +101,8 @@ void supports_noArgs_listsFeatures_exits0() { String out = captureStdout(() -> code[0] = new CommandLine(new Command()).execute("supports")); assertThat(code[0]).isEqualTo(0); - assertThat(out.lines()).containsExactlyInAnyOrder("dpop", "dpop_nonce_challenge"); + assertThat(out.lines()).containsExactlyInAnyOrder("dpop", "dpop_nonce_challenge", + "partial_decrypt", "http_range_source"); } @Test @@ -104,7 +111,8 @@ void supports_noArgs_json_listsFeatures() { String out = captureStdout(() -> code[0] = new CommandLine(new Command()).execute("supports", "--json")); assertThat(code[0]).isEqualTo(0); - assertThat(out.trim()).isEqualTo("{\"dpop\":true,\"dpop_nonce_challenge\":true}"); + assertThat(out.trim()).isEqualTo("{\"dpop\":true,\"dpop_nonce_challenge\":true," + + "\"partial_decrypt\":true,\"http_range_source\":true}"); } @Test diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java b/sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java new file mode 100644 index 00000000..1bcaefc2 --- /dev/null +++ b/sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java @@ -0,0 +1,152 @@ +package io.opentdf.platform.sdk; + +import okhttp3.OkHttpClient; +import okhttp3.Request; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.util.regex.Pattern; + +/** + * A read-only {@link SeekableByteChannel} over a remote resource, where every read is one + * HTTP range request. The size is learned by probing {@code bytes=0-0}, and every later + * response must report the same total size, so a resource that changes underneath the + * reader is rejected rather than spliced together. + *

+ * Redirects are not followed: the client handed in must have redirects disabled, and a 3xx + * is reported as an error so the caller can use the final URL. + */ +// ponytail: no read cache, so ZipReader's small header reads cost a few dozen requests per +// load; add a read-ahead window if that shows up over a real network +class HttpRangeChannel implements SeekableByteChannel { + private static final Pattern CONTENT_RANGE = Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+)"); + + private final OkHttpClient client; + private final URI url; + private final long size; + private long position; + private boolean open = true; + + HttpRangeChannel(OkHttpClient client, URI url) throws IOException { + this.client = client; + this.url = url; + this.size = fetch(ByteBuffer.allocate(1), 0); + } + + @Override + public int read(ByteBuffer dst) throws IOException { + ensureOpen(); + if (position >= size) { + return -1; + } + int len = (int) Math.min(dst.remaining(), size - position); + if (len == 0) { + return 0; + } + var window = dst.slice(); + window.limit(len); + long total = fetch(window, position); + if (total != size) { + throw new IOException("remote resource size changed from " + size + " to " + total + " bytes"); + } + dst.position(dst.position() + len); + position += len; + return len; + } + + /** + * Fills {@code dst} with the bytes starting at {@code offset} and returns the total size + * of the resource as reported by the server. + */ + private long fetch(ByteBuffer dst, long offset) throws IOException { + long end = offset + dst.remaining() - 1; + var request = new Request.Builder() + .url(url.toString()) + .header("Range", "bytes=" + offset + "-" + end) + .build(); + try (var response = client.newCall(request).execute()) { + int status = response.code(); + if (status == 200) { + throw new SDK.RangeNotSupportedException( + "status 200 for bytes " + offset + "-" + end); + } + if (status >= 300 && status < 400) { + throw new IOException("range request redirected (" + status + + "); redirects are not followed, use the final URL"); + } + if (status != 206) { + throw new IOException("range request for bytes " + offset + "-" + end + " failed: " + status); + } + String contentRange = response.header("Content-Range"); + var m = contentRange == null ? null : CONTENT_RANGE.matcher(contentRange); + if (m == null || !m.matches()) { + throw new SDK.RangeNotSupportedException("invalid Content-Range " + contentRange); + } + long gotStart = Long.parseLong(m.group(1)); + long gotEnd = Long.parseLong(m.group(2)); + if (gotStart != offset || gotEnd != end) { + throw new SDK.RangeNotSupportedException("requested bytes " + offset + "-" + end + + ", got " + gotStart + "-" + gotEnd); + } + var source = response.body().source(); + while (dst.hasRemaining()) { + if (source.read(dst) < 0) { + throw new IOException("range response for bytes " + offset + "-" + end + " ended early"); + } + } + return Long.parseLong(m.group(3)); + } + } + + @Override + public long position() throws IOException { + ensureOpen(); + return position; + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + ensureOpen(); + if (newPosition < 0) { + throw new IllegalArgumentException("negative position " + newPosition); + } + position = newPosition; + return this; + } + + @Override + public long size() throws IOException { + ensureOpen(); + return size; + } + + @Override + public int write(ByteBuffer src) { + throw new NonWritableChannelException(); + } + + @Override + public SeekableByteChannel truncate(long size) { + throw new NonWritableChannelException(); + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() { + open = false; + } + + private void ensureOpen() throws ClosedChannelException { + if (!open) { + throw new ClosedChannelException(); + } + } +} diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index f69b6896..041d74de 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.URI; import java.nio.channels.SeekableByteChannel; import java.util.ArrayList; import java.util.Collections; @@ -35,6 +36,8 @@ import java.util.Optional; import java.util.stream.Collectors; +import okhttp3.OkHttpClient; + /** * The SDK class represents a software development kit for interacting with the * opentdf platform. It @@ -57,6 +60,8 @@ public class SDK implements AutoCloseable { private final String platformUrl; private final ProtocolClient platformServicesClient; private final SrtSigner srtSigner; + // Fetches TDFs by URL. Unauthenticated: platform credentials never go to the TDF host. + private final OkHttpClient rangeClient; /** * Closes the SDK, including its associated services. @@ -66,6 +71,8 @@ public class SDK implements AutoCloseable { @Override public void close() throws Exception { services.close(); + rangeClient.dispatcher().executorService().shutdown(); + rangeClient.connectionPool().evictAll(); } /** @@ -114,6 +121,12 @@ public Optional getAuthInterceptor() { } SDK(Services services, TrustManager trustManager, Interceptor authInterceptor, ProtocolClient platformServicesClient, String platformUrl, SrtSigner srtSigner) { + this(services, trustManager, authInterceptor, platformServicesClient, platformUrl, srtSigner, + new OkHttpClient.Builder().followRedirects(false).followSslRedirects(false).build()); + } + + SDK(Services services, TrustManager trustManager, Interceptor authInterceptor, ProtocolClient platformServicesClient, String platformUrl, SrtSigner srtSigner, OkHttpClient rangeClient) { + this.rangeClient = rangeClient; this.platformUrl = platformUrl; this.services = services; this.trustManager = trustManager; @@ -140,6 +153,20 @@ public TDF.Reader loadTDF(SeekableByteChannel channel, Config.TDFReaderConfig co return tdf.loadTDF(channel, config, platformUrl); } + /** + * Loads a TDF served over HTTP(S), reading it with range requests so only the zip + * directory, the manifest, and the payload segments actually read are downloaded. + * The server must answer range requests with 206; redirects are not followed. + * + * @throws RangeNotSupportedException if the server ignores range requests + */ + public TDF.Reader loadTDF(URI url, Config.TDFReaderConfig config) throws SDKException, IOException { + if (!"http".equalsIgnoreCase(url.getScheme()) && !"https".equalsIgnoreCase(url.getScheme())) { + throw new IllegalArgumentException("unsupported URL scheme: " + url.getScheme()); + } + return loadTDF(new HttpRangeChannel(rangeClient, url), config); + } + public Manifest createTDF(InputStream payload, OutputStream outputStream, Config.TDFConfig config) throws SDKException, IOException { var tdf = new TDF(services); return tdf.createTDF(payload, outputStream, config).getManifest(); @@ -381,6 +408,16 @@ public boolean attributeValueExists(String valueFqn) { return resp.getFqnAttributeValuesMap().containsKey(valueFqn); } + /** + * The server holding a TDF did not honor an HTTP range request, so the TDF cannot be + * read in place; download it and load it from a file instead. + */ + public static class RangeNotSupportedException extends IOException { + public RangeNotSupportedException(String errorMessage) { + super("server did not honor the HTTP range request: " + errorMessage); + } + } + /** * Indicates that the TDF is malformed in some way */ diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java index dda13258..cac363d0 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java @@ -483,7 +483,14 @@ private KASClient getKASClient(SrtSigner srtSigner, AuthInterceptor interceptor, public SDK build() { var services = buildServices(); - return new SDK(services.services, services.trustManager, services.interceptor, services.protocolClient, platformEndpoint, services.srtSigner); + return new SDK(services.services, services.trustManager, services.interceptor, services.protocolClient, platformEndpoint, services.srtSigner, + // getHttpClient() carries the TLS settings but no credentials. Drop the h2c prior + // knowledge plaintext mode forces, since TDF hosts are ordinary HTTP servers. + getHttpClient().newBuilder() + .protocols(List.of(Protocol.HTTP_2, Protocol.HTTP_1_1)) + .followRedirects(false) + .followSslRedirects(false) + .build()); } private ProtocolClient getUnauthenticatedProtocolClient(String endpoint, OkHttpClient httpClient) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index fddead94..19c58b4e 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -388,6 +388,41 @@ public Manifest getManifest() { } public void readPayload(OutputStream outputStream) throws SDK.TamperException, IOException { + readPayload(outputStream, 0, -1); + } + + /** + * Writes the plaintext bytes {@code [offset, offset + length)} to {@code outputStream}. + * Only the segments covering the range are read, verified and decrypted, so over + * {@link SDK#loadTDF(java.net.URI, Config.TDFReaderConfig)} only those are downloaded. + * + * @param offset plaintext offset to start at; equal to the payload size writes nothing + * @param length number of bytes to write; negative means to the end of the payload, and + * a range running past the end is clamped to it + * @throws IllegalArgumentException if {@code offset} is negative or beyond the end of the + * payload, before anything is written + */ + public void readPayload(OutputStream outputStream, long offset, long length) + throws SDK.TamperException, IOException { + if (offset < 0) { + throw new IllegalArgumentException("offset must not be negative: " + offset); + } + + validateSegmentSizes(); + + List segments = manifest.encryptionInformation.integrityInformation.segments; + long total = 0; + for (Manifest.Segment segment : segments) { + total += plaintextSize(segment); + } + if (offset > total) { + throw new IllegalArgumentException( + "offset " + offset + " is beyond the end of the payload (" + total + " bytes)"); + } + long end = length < 0 || length > total - offset ? total : offset + length; + if (offset == end) { + return; + } MessageDigest digest = null; try { @@ -396,43 +431,71 @@ public void readPayload(OutputStream outputStream) throws SDK.TamperException, I throw new IllegalStateException("error getting instance of SHA-256", e); } - validateSegmentSizes(); - - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { - byte[] readBuf = new byte[(int) segment.encryptedSegmentSize]; - int bytesRead = tdfReader.readPayloadBytes(readBuf); + try (var payload = tdfReader.payload()) { + long plainStart = 0; + for (Manifest.Segment segment : segments) { + if (plainStart >= end) { + break; + } + long plainEnd = plainStart + plaintextSize(segment); + // <=, not <: a segment ending exactly at offset holds none of the range + if (plainEnd <= offset) { + if (payload.skip(segment.encryptedSegmentSize) != segment.encryptedSegmentSize) { + throw new IllegalStateException("unable to skip segment of " + + segment.encryptedSegmentSize + " bytes"); + } + plainStart = plainEnd; + continue; + } - if (readBuf.length != bytesRead) { - throw new IllegalStateException("unable to read bytes for segment (wanted " - + segment.encryptedSegmentSize + " but got " + bytesRead + ")"); + byte[] plain = readSegment(payload, segment, digest); + int from = (int) Math.max(0, offset - plainStart); + int to = (int) Math.min(plain.length, end - plainStart); + outputStream.write(plain, from, to - from); + plainStart = plainEnd; } + } + } - var isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); + // Derived from the ciphertext length rather than trusting the manifest's segmentSize, + // so a range lands on the bytes decryption actually produces. + private long plaintextSize(Manifest.Segment segment) { + return segment.encryptedSegmentSize + - (manifest.payload.isEncrypted ? kGcmIvSize + AesGcm.GCM_TAG_LENGTH : 0); + } - if (manifest.payload.isEncrypted) { - var sigAlg = segmentIntegrityAlgorithmFromManifest( - manifest.encryptionInformation.integrityInformation.segmentHashAlg); + private byte[] readSegment(InputStream payload, Manifest.Segment segment, MessageDigest digest) { + byte[] readBuf = new byte[(int) segment.encryptedSegmentSize]; + int bytesRead = TDFReader.readPayloadBytes(payload, readBuf); - var payloadSig = segmentIntegrity(readBuf, payloadKey, sigAlg); - if (isLegacyTdf) { - payloadSig = Hex.encodeHexString(payloadSig).getBytes(StandardCharsets.UTF_8); - } + if (readBuf.length != bytesRead) { + throw new IllegalStateException("unable to read bytes for segment (wanted " + + segment.encryptedSegmentSize + " but got " + bytesRead + ")"); + } - if (segment.hash.compareTo(Base64.getEncoder().encodeToString(payloadSig)) != 0) { - throw new SDK.SegmentSignatureMismatch("segment signature miss match"); - } + var isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); - byte[] writeBuf = aesGcm.decrypt(new AesGcm.Encrypted(readBuf)); - outputStream.write(writeBuf); + if (manifest.payload.isEncrypted) { + var sigAlg = segmentIntegrityAlgorithmFromManifest( + manifest.encryptionInformation.integrityInformation.segmentHashAlg); - } else { - String segmentSig = Hex.encodeHexString(digest.digest(readBuf)); - if (segment.hash.compareTo(segmentSig) != 0) { - throw new SDK.SegmentSignatureMismatch("segment signature miss match"); - } + var payloadSig = segmentIntegrity(readBuf, payloadKey, sigAlg); + if (isLegacyTdf) { + payloadSig = Hex.encodeHexString(payloadSig).getBytes(StandardCharsets.UTF_8); + } - outputStream.write(readBuf); + if (segment.hash.compareTo(Base64.getEncoder().encodeToString(payloadSig)) != 0) { + throw new SDK.SegmentSignatureMismatch("segment signature miss match"); } + + return aesGcm.decrypt(new AesGcm.Encrypted(readBuf)); + } else { + String segmentSig = Hex.encodeHexString(digest.digest(readBuf)); + if (segment.hash.compareTo(segmentSig) != 0) { + throw new SDK.SegmentSignatureMismatch("segment signature miss match"); + } + + return readBuf; } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 93548d67..80ad7e34 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -20,6 +20,7 @@ public class TDFReader { private final ZipReader.Entry manifestEntry; + private final ZipReader.Entry payloadEntry; private final InputStream payload; public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { @@ -46,13 +47,20 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { } manifestEntry = manifest; - payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData(); + payloadEntry = entries.get(TDF_PAYLOAD_FILE_NAME); + payload = payloadEntry.getData(); } String manifest() { var out = new ByteArrayOutputStream(); - try { - manifestEntry.getData().transferTo(out); + // Read in large chunks rather than transferTo's small ones: over HTTP every read + // is a range request. + var buf = new byte[1 << 20]; + try (var in = manifestEntry.getData()) { + int n; + while ((n = in.read(buf)) >= 0) { + out.write(buf, 0, n); + } } catch (IOException e) { throw new SDKException("error retrieving manifest from zip file", e); } @@ -60,7 +68,16 @@ String manifest() { return out.toString(StandardCharsets.UTF_8); } + /** A fresh stream over the payload, positioned at its start. */ + InputStream payload() throws IOException { + return payloadEntry.getData(); + } + int readPayloadBytes(byte[] buf) { + return readPayloadBytes(payload, buf); + } + + static int readPayloadBytes(InputStream payload, byte[] buf) { int totalRead = 0; int nread; try { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java index 576ba5de..d5e3a043 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java @@ -254,6 +254,15 @@ private boolean doneReading() { return offset >= fileSize; } + // Moves the cursor without reading: the default skip reads and discards, which + // over HTTP would download everything skipped. + @Override + public long skip(long n) { + long skipped = Math.max(0, Math.min(n, fileSize - offset)); + offset += skipped; + return skipped; + } + private void setChannelPosition() throws IOException { var nextPosition = startPosition + offset; if (zipChannel.position() != nextPosition) { diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java new file mode 100644 index 00000000..9c0ab181 --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java @@ -0,0 +1,130 @@ +package io.opentdf.platform.sdk; + +import okhttp3.OkHttpClient; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okio.Buffer; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HttpRangeChannelTest { + static final OkHttpClient CLIENT = new OkHttpClient.Builder() + .followRedirects(false).followSslRedirects(false).build(); + + private static final Pattern RANGE = Pattern.compile("bytes=(\\d+)-(\\d+)"); + + /** Serves {@code data} the way a static file server does, adding the bytes sent to {@code served}. */ + static Dispatcher rangeDispatcher(Supplier data, AtomicLong served) { + return new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + byte[] bytes = data.get(); + var m = RANGE.matcher(String.valueOf(request.getHeader("Range"))); + if (!m.matches()) { + return new MockResponse().setResponseCode(200).setBody(new Buffer().write(bytes)); + } + int start = Integer.parseInt(m.group(1)); + int end = Math.min(Integer.parseInt(m.group(2)), bytes.length - 1); + served.addAndGet(end - start + 1); + return new MockResponse().setResponseCode(206) + .setHeader("Content-Range", "bytes " + start + "-" + end + "/" + bytes.length) + .setBody(new Buffer().write(bytes, start, end - start + 1)); + } + }; + } + + private static URI url(MockWebServer server) { + return server.url("/x.tdf").uri(); + } + + @Test + void readsAndSeeks() throws Exception { + byte[] data = "0123456789".getBytes(StandardCharsets.UTF_8); + try (var server = new MockWebServer()) { + server.setDispatcher(rangeDispatcher(() -> data, new AtomicLong())); + try (var ch = new HttpRangeChannel(CLIENT, url(server))) { + assertThat(ch.size()).isEqualTo(10); + + var buf = ByteBuffer.allocate(4); + assertThat(ch.position(3).read(buf)).isEqualTo(4); + assertThat(new String(buf.array(), StandardCharsets.UTF_8)).isEqualTo("3456"); + assertThat(ch.position()).isEqualTo(7); + + // clamped at the end of the resource, then EOF + buf.clear(); + assertThat(ch.read(buf)).isEqualTo(3); + assertThat(ch.read(buf.clear())).isEqualTo(-1); + } + assertThat(server.takeRequest().getHeader("Authorization")).isNull(); + } + } + + @Test + void rangeNotSupported() throws Exception { + try (var server = new MockWebServer()) { + server.enqueue(new MockResponse().setResponseCode(200).setBody("not a partial response")); + assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server))) + .isInstanceOf(SDK.RangeNotSupportedException.class); + } + } + + @Test + void mismatchedContentRangeIsRangeNotSupported() throws Exception { + try (var server = new MockWebServer()) { + server.enqueue(new MockResponse().setResponseCode(206) + .setHeader("Content-Range", "bytes 0-9/10").setBody("0123456789")); + assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server))) + .isInstanceOf(SDK.RangeNotSupportedException.class); + } + } + + @Test + void errorStatusesReportTheirStatus() throws Exception { + for (int status : new int[] { 302, 403, 404 }) { + try (var server = new MockWebServer()) { + server.enqueue(new MockResponse().setResponseCode(status).setHeader("Location", "/elsewhere")); + assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server))) + .isInstanceOf(IOException.class) + .isNotInstanceOf(SDK.RangeNotSupportedException.class) + .hasMessageContaining(String.valueOf(status)) + .satisfies(e -> { + if (status == 302) { + assertThat(e).hasMessageContaining("redirects are not followed"); + } + }); + } + } + } + + @Test + void sizeChanged() throws Exception { + var data = new byte[][] { "0123456789".getBytes(StandardCharsets.UTF_8) }; + try (var server = new MockWebServer()) { + server.setDispatcher(rangeDispatcher(() -> data[0], new AtomicLong())); + try (var ch = new HttpRangeChannel(CLIENT, url(server))) { + data[0] = "0123456789abc".getBytes(StandardCharsets.UTF_8); + assertThatThrownBy(() -> ch.read(ByteBuffer.allocate(4))) + .hasMessageContaining("size changed from 10 to 13"); + } + } + } + + @Test + void rejectsNonHttpSchemes() { + var sdk = new SDK(new FakeServicesBuilder().build(), null, null, null, null, null); + assertThatThrownBy(() -> sdk.loadTDF(URI.create("file:///etc/passwd"), Config.newTDFReaderConfig())) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java index aa5b38e6..3d85c6d2 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java @@ -13,10 +13,12 @@ import io.opentdf.platform.policy.kasregistry.ListKeyAccessServersRequest; import io.opentdf.platform.policy.kasregistry.ListKeyAccessServersResponse; import io.opentdf.platform.sdk.TDF.Reader; +import okhttp3.mockwebserver.MockWebServer; import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nonnull; @@ -37,6 +39,7 @@ import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -733,6 +736,94 @@ public void testCreatingTDFWithMultipleSegments() throws Exception { assertThat(reader.getMetadata()).isEqualTo(metadata); } + private static final int SEG = Config.MIN_SEGMENT_SIZE; + + /** A TDF of {@code 2.5 * SEG} random bytes, so it has two full segments and a partial one. */ + private static byte[][] partialReadFixture() throws Exception { + var data = new byte[SEG * 5 / 2]; + new Random().nextBytes(data); + var tdfOutputStream = new ByteArrayOutputStream(); + var tdf = new TDF(new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + tdf.createTDF(new ByteArrayInputStream(data), tdfOutputStream, Config.newTDFConfig( + Config.withAutoconfigure(false), + Config.withKasInformation(getSingleRSAKASInfo()), + Config.withSegmentSize(SEG))); + return new byte[][] { data, tdfOutputStream.toByteArray() }; + } + + private static Reader loadFixture(byte[] tdfBytes) throws Exception { + var tdf = new TDF(new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + return tdf.loadTDF(new SeekableInMemoryByteChannel(tdfBytes), platformUrl); + } + + @ParameterizedTest + @CsvSource({ + "0, -1", // whole payload + "16334, 100", // crosses the first segment boundary + "16384, 100", // starts exactly where a segment ends + "16384, 16384", // exactly the second segment + "100, 0", // empty + "32768, -1", // last segment to the end + "40950, 1000", // runs past the end: clamped + "40960, -1", // offset == size: empty + }) + void readPayloadRange(long offset, long length) throws Exception { + var fixture = partialReadFixture(); + byte[] data = fixture[0]; + var out = new ByteArrayOutputStream(); + loadFixture(fixture[1]).readPayload(out, offset, length); + + int end = length < 0 ? data.length : (int) Math.min(data.length, offset + length); + assertThat(out.toByteArray()).containsExactly(Arrays.copyOfRange(data, (int) offset, end)); + } + + @Test + void readPayloadRangeCanBeRepeatedOnOneReader() throws Exception { + var fixture = partialReadFixture(); + var reader = loadFixture(fixture[1]); + for (int i = 0; i < 2; i++) { + var out = new ByteArrayOutputStream(); + reader.readPayload(out, SEG, 10); + assertThat(out.toByteArray()).containsExactly(Arrays.copyOfRange(fixture[0], SEG, SEG + 10)); + } + } + + @Test + void readPayloadRangeRejectsBadOffsetsBeforeWriting() throws Exception { + var fixture = partialReadFixture(); + var reader = loadFixture(fixture[1]); + var out = new ByteArrayOutputStream(); + assertThatThrownBy(() -> reader.readPayload(out, fixture[0].length + 1L, -1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("beyond the end"); + assertThatThrownBy(() -> reader.readPayload(out, -1, -1)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(out.size()).isZero(); + } + + @Test + void readPayloadRangeOverHttpFetchesOnlyCoveringSegments() throws Exception { + var fixture = partialReadFixture(); + var served = new AtomicLong(); + try (var server = new MockWebServer()) { + server.setDispatcher(HttpRangeChannelTest.rangeDispatcher(() -> fixture[1], served)); + var tdf = new TDF(new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + var reader = tdf.loadTDF(new HttpRangeChannel(HttpRangeChannelTest.CLIENT, server.url("/x.tdf").uri()), + Config.newTDFReaderConfig(), platformUrl); + + served.set(0); + var out = new ByteArrayOutputStream(); + reader.readPayload(out, 2L * SEG - 50, 100); + assertThat(out.toByteArray()).containsExactly(Arrays.copyOfRange(fixture[0], 2 * SEG - 50, 2 * SEG + 50)); + // the second (full) and third (half) segments, plus the payload entry's local header + long encryptedSegments = (SEG + 28) + (SEG / 2 + 28); + assertThat(served.get()).isBetween(encryptedSegments, encryptedSegments + 1024); + } + } + /** * A segment size small enough that the test payloads below span several segments. Matches * web-sdk's {@code DEFAULT_SEGMENT_SIZE}, though nothing here depends on the exact value.