From 5c21e441b70f21b42fcba84ca69ce92403683f7d Mon Sep 17 00:00:00 2001 From: KUAN-HAO HUANG <101171023+rich7420@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:07:06 +0800 Subject: [PATCH 1/2] HDDS-16658. Reuse source key details when opening the input stream in S3 CopyObject --- .../ozone/s3/endpoint/ObjectEndpoint.java | 2 +- .../ozone/s3/endpoint/TestObjectPut.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index cccbe387fe00..604425676ed8 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -1285,7 +1285,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume, ? stripQuotes(sourceETag) : null; try (OzoneInputStream src = runWithS3ActionString( - "GetObject", () -> getClientProtocol().getKey(volume.getName(), sourceBucket, sourceKey)); + "GetObject", sourceKeyDetails::getContent); DigestInputStream sourceDigestInputStream = new DigestInputStream(src, md5Digest)) { getMetrics().updateCopyKeyMetadataStats(startNanos); if (reusedETag != null) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 4ac538a4a7d8..f2f693954105 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -59,9 +59,11 @@ import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -98,6 +100,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; @@ -737,6 +740,25 @@ public void testCopyObjectMessageDigestResetDuringException() throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCopyObjectReusesSourceKeyDetails(boolean streaming) throws Exception { + assertSucceeds(() -> putObject(CONTENT)); + objectEndpoint.init(); + doReturn(streaming).when(objectEndpoint).isDatastreamEnabled(); + doReturn(0L).when(objectEndpoint).getDatastreamMinLength(); + ClientProtocol protocol = spy(objectEndpoint.getClientProtocol()); + doReturn(protocol).when(objectEndpoint).getClientProtocol(); + when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(BUCKET_NAME + "/" + urlEncode(KEY_NAME)); + + try (Response response = put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)) { + assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK); + } + assertKeyContent(destBucket, DEST_KEY, CONTENT); + verify(protocol).getKeyDetails(bucket.getVolumeName(), BUCKET_NAME, KEY_NAME); + verify(protocol, never()).getKey(bucket.getVolumeName(), BUCKET_NAME, KEY_NAME); + } + @Test void testCopyObjectReusesSourceETagWithoutRehashing() throws Exception { assertSucceeds(() -> putObject(CONTENT)); From c1d17bde1c354457668847321e57000cc78ee31b Mon Sep 17 00:00:00 2001 From: KUAN-HAO HUANG <101171023+rich7420@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:47:15 +0800 Subject: [PATCH 2/2] HDDS-16658. Support datastream sources in CopyObject tests --- .../apache/hadoop/ozone/client/OzoneBucketStub.java | 2 +- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 0aef8607c87c..841076030a1e 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -316,7 +316,7 @@ public void close() throws IOException { mtime, mtime, new ArrayList<>(), rConfig, objectMetadata, null, - null, false, + () -> readKey(key), false, UserGroupInformation.getCurrentUser().getShortUserName(), tags )); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index f2f693954105..e8dccf5ac1a5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -111,6 +111,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EmptySource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.NullAndEmptySource; @@ -741,18 +742,21 @@ public void testCopyObjectMessageDigestResetDuringException() throws Exception { } @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testCopyObjectReusesSourceKeyDetails(boolean streaming) throws Exception { - assertSucceeds(() -> putObject(CONTENT)); + @CsvSource({"false, false", "false, true", "true, false", "true, true"}) + void testCopyObjectReusesSourceKeyDetails(boolean streamingUpload, boolean streamingCopy) throws Exception { objectEndpoint.init(); - doReturn(streaming).when(objectEndpoint).isDatastreamEnabled(); + doReturn(streamingUpload).when(objectEndpoint).isDatastreamEnabled(); doReturn(0L).when(objectEndpoint).getDatastreamMinLength(); + assertSucceeds(() -> putObject(CONTENT)); + + doReturn(streamingCopy).when(objectEndpoint).isDatastreamEnabled(); ClientProtocol protocol = spy(objectEndpoint.getClientProtocol()); doReturn(protocol).when(objectEndpoint).getClientProtocol(); when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(BUCKET_NAME + "/" + urlEncode(KEY_NAME)); try (Response response = put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)) { assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK); + assertThat(((CopyObjectResponse) response.getEntity()).getETag()).isEqualTo("\"" + contentMd5Hex() + "\""); } assertKeyContent(destBucket, DEST_KEY, CONTENT); verify(protocol).getKeyDetails(bucket.getVolumeName(), BUCKET_NAME, KEY_NAME);