From da0cc5ef646d5697ef68f2b1aee412e0548bbcb4 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:23:13 -0700 Subject: [PATCH 1/4] HDDS-16739. S3 ListObjectVersions should return the null version of each object Versioning is not implemented, so every bucket behaves like an AWS S3 bucket that never had versioning enabled. ListObjectVersions now lists each object with VersionId "null" instead of returning NotImplemented, and "null" is accepted as the version ID of the current object in DeleteObjects, DeleteObject, PutObjectTagging and DeleteObjectTagging. Co-Authored-By: Claude Opus 5.5 --- hadoop-hdds/docs/content/interface/S3.md | 1 + .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 38 +++++ .../ozone/s3/commontypes/KeyMetadata.java | 23 +++ .../ozone/s3/endpoint/BucketEndpoint.java | 22 ++- .../BucketOperationNotImplementedHandler.java | 7 +- .../ozone/s3/endpoint/EndpointBase.java | 10 ++ .../ozone/s3/endpoint/ListVersionsResult.java | 139 ++++++++++++++++++ .../s3/endpoint/MultiDeleteResponse.java | 1 + .../ObjectOperationNotImplementedHandler.java | 13 +- .../s3/endpoint/ObjectTaggingHandler.java | 2 +- .../apache/hadoop/ozone/s3/util/S3Consts.java | 6 +- .../ozone/s3/endpoint/TestBucketList.java | 71 +++++++++ .../s3/endpoint/TestBucketNotImplemented.java | 9 +- .../s3/endpoint/TestObjectMultiDelete.java | 27 ++++ .../s3/endpoint/TestObjectNotImplemented.java | 12 ++ .../s3/endpoint/TestObjectTaggingDelete.java | 9 ++ .../s3/endpoint/TestObjectTaggingPut.java | 9 ++ 17 files changed, 382 insertions(+), 17 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java diff --git a/hadoop-hdds/docs/content/interface/S3.md b/hadoop-hdds/docs/content/interface/S3.md index 5fb5f4edbd01..6bd7b4b16a28 100644 --- a/hadoop-hdds/docs/content/interface/S3.md +++ b/hadoop-hdds/docs/content/interface/S3.md @@ -80,6 +80,7 @@ The Ozone S3 Gateway implements a substantial subset of the Amazon S3 REST API. | ✅ [DeleteObjects](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) | Deletes multiple objects in one request. | Bulk delete operation with aggregated error reporting. | | ✅ [CopyObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html) | Copies an object (intra- or inter-bucket). | Supports metadata directives (COPY/REPLACE). **Non-compliant behavior:** Self-copying for metadata updates may differ slightly from AWS S3. | | ✅ [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) | Lists objects in a bucket with filtering and pagination. | Supports prefix, delimiter, and continuation tokens. | +| ✅ [ListObjectVersions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html) | Lists the versions of the objects in a bucket. | Bucket versioning is not supported, so each object is listed with a single version whose `VersionId` is `null`, same as in an AWS S3 bucket that never had versioning enabled. `DeleteObject`, `DeleteObjects` and the object tagging operations accept `null` as the version ID and operate on the current object. | ### Object Tagging Operations diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 6e88f7076729..4e4d5334d98f 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -143,6 +143,9 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectTaggingRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.DeletedObject; +import software.amazon.awssdk.services.s3.model.EncodingType; import software.amazon.awssdk.services.s3.model.ExpirationStatus; import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse; @@ -167,6 +170,7 @@ import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsResponse; import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.ListObjectsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -177,6 +181,7 @@ import software.amazon.awssdk.services.s3.model.ObjectAttributes; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; +import software.amazon.awssdk.services.s3.model.ObjectVersion; import software.amazon.awssdk.services.s3.model.PutBucketAclRequest; import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -967,6 +972,39 @@ public void testDeleteObjectsWithVersionIdNotImplemented() { assertDoesNotThrow(() -> s3Client.headObject(b -> b.bucket(bucketName).key(keyName))); } + @Test + public void testDeleteObjectsListedByListObjectVersions() { + final String bucketName = getBucketName(); + final List keyNames = Arrays.asList("dir/key 3", "key+1", "key2"); + s3Client.createBucket(b -> b.bucket(bucketName)); + keyNames.forEach(k -> s3Client.putObject(b -> b.bucket(bucketName).key(k), RequestBody.fromString("bar"))); + + final List versions = s3Client.listObjectVersionsPaginator( + b -> b.bucket(bucketName).maxKeys(2).encodingType(EncodingType.URL)) + .versions().stream().collect(Collectors.toList()); + + // versioning is not implemented, each object only has the null version + assertThat(versions).extracting(ObjectVersion::key).containsExactlyElementsOf(keyNames); + assertThat(versions).allSatisfy(v -> { + assertThat(v.versionId()).isEqualTo("null"); + assertThat(v.isLatest()).isTrue(); + assertThat(v.size()).isEqualTo(3L); + }); + + final DeleteObjectsResponse deleted = s3Client.deleteObjects(b -> b.bucket(bucketName).delete(d -> d.objects( + versions.stream() + .map(v -> ObjectIdentifier.builder().key(v.key()).versionId(v.versionId()).build()) + .collect(Collectors.toList())))); + + assertThat(deleted.errors()).isEmpty(); + assertThat(deleted.deleted()).extracting(DeletedObject::key).containsExactlyInAnyOrderElementsOf(keyNames); + assertThat(deleted.deleted()).extracting(DeletedObject::versionId).containsOnly("null"); + final ListObjectVersionsResponse remaining = s3Client.listObjectVersions(b -> b.bucket(bucketName)); + assertThat(remaining.versions()).isEmpty(); + assertThat(remaining.deleteMarkers()).isEmpty(); + assertThat(remaining.isTruncated()).isFalse(); + } + @Test public void testPutObjectLegalHoldNotImplemented() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java index 1ebf28a9bc4c..d0cff56f31d3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java @@ -51,6 +51,13 @@ public class KeyMetadata { @XmlElement(name = "StorageClass") private String storageClass; + // VersionId and IsLatest are only set in ListObjectVersions responses + @XmlElement(name = "VersionId") + private String versionId; + + @XmlElement(name = "IsLatest") + private Boolean isLatest; + public EncodingTypeObject getKey() { return key; } @@ -98,4 +105,20 @@ public String getStorageClass() { public void setStorageClass(String storageClass) { this.storageClass = storageClass; } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + + public Boolean getIsLatest() { + return isLatest; + } + + public void setIsLatest(Boolean latest) { + this.isLatest = latest; + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 298ee53e8a77..4b2377f03e5f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -109,7 +109,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO final String delimiter = queryParams().containsKey(QueryParams.DELIMITER) ? queryParams().get(QueryParams.DELIMITER) : null; final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE); - final String marker = queryParams().get(QueryParams.MARKER); + final String marker = queryParams().get(isListObjectVersions() ? QueryParams.KEY_MARKER : QueryParams.MARKER); int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000); String prefix = queryParams().get(QueryParams.PREFIX, ""); String startAfter = queryParams().get(QueryParams.START_AFTER); @@ -252,7 +252,18 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO context.getPerf().appendCount(keyCount); context.getPerf().appendOpLatencyNanos(opLatencyNs); response.setKeyCount(keyCount); - return Response.ok(response).build(); + return Response.ok(toListResult(response)).build(); + } + + /** ListObjectVersions lists the same keys as ListObjects (V1), but paginates with key-marker instead of marker. */ + private boolean isListObjectVersions() { + return queryParams().get(QueryParams.VERSIONS) != null; + } + + private Object toListResult(ListObjectResponse response) { + return isListObjectVersions() + ? ListVersionsResult.of(response, queryParams().get(QueryParams.VERSION_ID_MARKER)) + : response; } private int validateMaxKeys(int maxKeys) throws OS3Exception { @@ -353,7 +364,8 @@ public MultiDeleteResponse multiDelete( } // Deleting a specific version is not implemented; ignoring VersionId would delete the current object instead. - if (request.getObjects() != null && request.getObjects().stream().anyMatch(o -> o.getVersionId() != null)) { + if (request.getObjects() != null + && request.getObjects().stream().anyMatch(o -> isUnsupportedVersionId(o.getVersionId()))) { throw newError(S3ErrorTable.NOT_IMPLEMENTED, bucketName); } @@ -385,7 +397,9 @@ public MultiDeleteResponse multiDelete( ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode()); if (deleted) { if (!request.isQuiet()) { - result.addDeleted(new DeletedObject(d.getKey())); + DeletedObject deletedObject = new DeletedObject(d.getKey()); + deletedObject.setVersionId(d.getVersionId()); + result.addDeleted(deletedObject); } } else { failedDeletes.add(d.getKey()); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java index 0ca3cd5b413b..bf97576daedb 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.s3.endpoint; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import java.io.InputStream; import java.util.Set; import javax.ws.rs.core.Response; @@ -47,9 +48,13 @@ class BucketOperationNotImplementedHandler extends BucketOperationHandler { QueryParams.PUBLIC_ACCESS_BLOCK, QueryParams.REPLICATION, QueryParams.REQUEST_PAYMENT, QueryParams.SESSION, QueryParams.VERSIONING, QueryParams.VERSIONS, QueryParams.WEBSITE); + // ListObjectVersions is implemented by BucketEndpoint + private static final Set GET_SUBRESOURCES = + Sets.difference(SUBRESOURCES, ImmutableSet.of(QueryParams.VERSIONS)).immutableCopy(); + @Override Response handleGetRequest(S3RequestContext context, String bucketName) { - return rejectNotImplemented(context, SUBRESOURCES, bucketName); + return rejectNotImplemented(context, GET_SUBRESOURCES, bucketName); } @Override diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 349d3765ad26..96af5f420086 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -41,6 +41,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.RESERVED_USER_METADATA_KEY_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CONFIG_HEADER; @@ -789,6 +790,15 @@ protected void validateObjectKeyUri(String keyPath) throws OS3Exception { } } + /** + * Versioning is not implemented, so each object only has the null version, i.e. the current object. + * + * @return true if {@code versionId} refers to any other version + */ + static boolean isUnsupportedVersionId(@Nullable String versionId) { + return versionId != null && !NULL_VERSION_ID.equals(versionId); + } + /** * Rejects the request with {@code NotImplemented} if it has any of the {@code subresources}. * diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java new file mode 100644 index 000000000000..309f4b7303cf --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; + +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.apache.hadoop.ozone.s3.commontypes.CommonPrefix; +import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; +import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; +import org.apache.hadoop.ozone.s3.commontypes.ObjectKeyNameAdapter; +import org.apache.hadoop.ozone.s3.util.S3Consts; + +/** + * Response of ListObjectVersions. + *

+ * Versioning is not implemented, so each object is listed with its only version, the null version. + * This is the same as what AWS S3 returns for a bucket that never had versioning enabled. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "ListVersionsResult", namespace = S3Consts.S3_XML_NAMESPACE) +public class ListVersionsResult { + + @XmlElement(name = "Name") + private String name; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "Prefix") + private EncodingTypeObject prefix; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "KeyMarker") + private EncodingTypeObject keyMarker; + + @XmlElement(name = "VersionIdMarker") + private String versionIdMarker; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "NextKeyMarker") + private EncodingTypeObject nextKeyMarker; + + @XmlElement(name = "NextVersionIdMarker") + private String nextVersionIdMarker; + + @XmlElement(name = "MaxKeys") + private int maxKeys; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "Delimiter") + private EncodingTypeObject delimiter; + + @XmlElement(name = "EncodingType") + private String encodingType; + + @XmlElement(name = "IsTruncated") + private boolean isTruncated; + + @XmlElement(name = "Version") + private List versions; + + @XmlElement(name = "CommonPrefixes") + private List commonPrefixes; + + /** + * @param listing result of listing the bucket with {@code key-marker} as the marker + * @param versionIdMarker {@code version-id-marker} of the request + */ + static ListVersionsResult of(ListObjectResponse listing, String versionIdMarker) { + final String encodingType = listing.getEncodingType(); + final ListVersionsResult result = new ListVersionsResult(); + result.name = listing.getName(); + result.prefix = listing.getPrefix(); + result.keyMarker = EncodingTypeObject.createNullable(listing.getMarker(), encodingType); + result.versionIdMarker = versionIdMarker == null ? "" : versionIdMarker; + result.maxKeys = listing.getMaxKeys(); + result.delimiter = listing.getDelimiter(); + result.encodingType = encodingType; + result.isTruncated = listing.isTruncated(); + if (listing.isTruncated()) { + result.nextKeyMarker = EncodingTypeObject.createNullable(listing.getNextMarker(), encodingType); + result.nextVersionIdMarker = NULL_VERSION_ID; + } + result.versions = listing.getContents(); + for (KeyMetadata version : result.versions) { + version.setVersionId(NULL_VERSION_ID); + version.setIsLatest(true); + } + result.commonPrefixes = listing.getCommonPrefixes(); + return result; + } + + public EncodingTypeObject getKeyMarker() { + return keyMarker; + } + + public String getVersionIdMarker() { + return versionIdMarker; + } + + public EncodingTypeObject getNextKeyMarker() { + return nextKeyMarker; + } + + public String getNextVersionIdMarker() { + return nextVersionIdMarker; + } + + public boolean isTruncated() { + return isTruncated; + } + + public List getVersions() { + return versions; + } + + public List getCommonPrefixes() { + return commonPrefixes; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java index 222a56239346..929d0d8e2d7e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java @@ -74,6 +74,7 @@ public static class DeletedObject { @XmlElement(name = "Key") private String key; + @XmlElement(name = "VersionId") private String versionId; public DeletedObject() { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java index afce9aca85fa..2302c48897ab 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java @@ -21,6 +21,7 @@ import java.io.InputStream; import java.util.Set; import javax.ws.rs.core.Response; +import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint.ObjectRequestContext; import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; @@ -42,12 +43,6 @@ class ObjectOperationNotImplementedHandler extends ObjectOperationHandler { QueryParams.ANNOTATION, QueryParams.ENCRYPTION, QueryParams.LEGAL_HOLD, QueryParams.RENAME_OBJECT, QueryParams.RETENTION, QueryParams.UPLOADS); - // DeleteObject with versionId targets a specific version; ignoring it would delete the current object instead. - private static final Set DELETE_SUBRESOURCES = ImmutableSet.builder() - .addAll(SUBRESOURCES) - .add(QueryParams.VERSION_ID) - .build(); - @Override Response handleGetRequest(ObjectRequestContext context, String keyName) { return rejectNotImplemented(context, SUBRESOURCES, keyName); @@ -60,6 +55,10 @@ Response handlePutRequest(ObjectRequestContext context, String keyName, InputStr @Override Response handleDeleteRequest(ObjectRequestContext context, String keyName) { - return rejectNotImplemented(context, DELETE_SUBRESOURCES, keyName); + // DeleteObject with versionId targets a specific version; ignoring it would delete the current object instead. + if (isUnsupportedVersionId(queryParams().get(QueryParams.VERSION_ID))) { + return rejectNotImplemented(context, S3GAction.NOT_IMPLEMENTED, keyName); + } + return rejectNotImplemented(context, SUBRESOURCES, keyName); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java index 5c04df9a90db..8f5ed82558bc 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java @@ -119,7 +119,7 @@ Response handleGetRequest(ObjectRequestContext context, String keyName) // Tagging a specific version is not implemented; ignoring versionId would modify the tags of the current object. private void rejectVersionId(String keyName) { - if (queryParams().get(S3Consts.QueryParams.VERSION_ID) != null) { + if (isUnsupportedVersionId(queryParams().get(S3Consts.QueryParams.VERSION_ID))) { throw S3ErrorTable.newError(S3ErrorTable.NOT_IMPLEMENTED, keyName); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 0cb61678d68b..c3300d37dc32 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -114,6 +114,9 @@ public final class S3Consts { /** AWS S3 maximum number of keys per DeleteObjects request. */ public static final int S3_DELETE_OBJECTS_MAX_KEYS = 1000; + /** Version ID of an object in a bucket that never had versioning enabled. */ + public static final String NULL_VERSION_ID = "null"; + // Bucket owner condition headers // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-owner-condition.html public static final String EXPECTED_BUCKET_OWNER_HEADER = "x-amz-expected-bucket-owner"; @@ -192,6 +195,8 @@ public static final class QueryParams { public static final String LIFECYCLE = "lifecycle"; public static final String ATTRIBUTES = "attributes"; public static final String VERSION_ID = "versionId"; + public static final String VERSION_ID_MARKER = "version-id-marker"; + public static final String VERSIONS = "versions"; // Subresources of S3 APIs that are not implemented. They are only recognized so that the // request can be rejected with NotImplemented instead of falling through to another operation. @@ -224,7 +229,6 @@ public static final class QueryParams { public static final String RETENTION = "retention"; public static final String SESSION = "session"; public static final String VERSIONING = "versioning"; - public static final String VERSIONS = "versions"; public static final String WEBSITE = "website"; private QueryParams() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index d16ef557bff3..6f4ad12c0a21 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT; import static org.apache.hadoop.ozone.s3.endpoint.EndpointBuilder.newBucketEndpointBuilder; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -669,6 +671,75 @@ public void continuationTokenXmlElementName() throws Exception { "response must not use the non-AWS element"); } + @Test + public void listObjectVersionsReturnsNullVersionOfEachKey() throws OS3Exception, IOException { + setup("dir1/file2", "file+1", "file3"); + + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSIONS, ""); + bucketEndpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + bucketEndpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, ENCODING_TYPE); + ListVersionsResult result = (ListVersionsResult) bucketEndpoint.get(BUCKET_NAME).getEntity(); + + assertThat(result.isTruncated()).isFalse(); + assertThat(result.getNextKeyMarker()).isNull(); + assertThat(result.getNextVersionIdMarker()).isNull(); + assertThat(result.getCommonPrefixes()).extracting(p -> p.getPrefix().getName()).containsExactly("dir1/"); + assertThat(result.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file+1", "file3"); + assertThat(result.getVersions()).allSatisfy(v -> { + assertThat(v.getVersionId()).isEqualTo(NULL_VERSION_ID); + assertThat(v.getIsLatest()).isTrue(); + assertThat(v.getOwner()).isNotNull(); + }); + + StringWriter writer = new StringWriter(); + JAXB.marshal(result, writer); + assertThat(writer.toString()) + .contains("", "") + .contains("", "file%2B1", "null", "true") + .doesNotContain("NextKeyMarker", "DeleteMarker"); + } + + @Test + public void listObjectVersionsPaginatesWithKeyMarker() throws OS3Exception, IOException { + setup("file+1", "file2", "file3"); + + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSIONS, ""); + bucketEndpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, ENCODING_TYPE); + bucketEndpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 1); + // marker is the pagination parameter of ListObjects, it must be ignored + bucketEndpoint.queryParamsForTest().set(QueryParams.MARKER, "file3"); + ListVersionsResult first = (ListVersionsResult) bucketEndpoint.get(BUCKET_NAME).getEntity(); + + assertThat(first.isTruncated()).isTrue(); + assertThat(first.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file+1"); + assertThat(first.getNextKeyMarker().getName()).isEqualTo("file+1"); + assertThat(first.getNextVersionIdMarker()).isEqualTo(NULL_VERSION_ID); + StringWriter writer = new StringWriter(); + JAXB.marshal(first, writer); + assertThat(writer.toString()) + .contains("file%2B1", "null"); + + bucketEndpoint.queryParamsForTest().unset(QueryParams.MAX_KEYS); + bucketEndpoint.queryParamsForTest().set(QueryParams.KEY_MARKER, first.getNextKeyMarker().getName()); + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSION_ID_MARKER, first.getNextVersionIdMarker()); + ListVersionsResult second = (ListVersionsResult) bucketEndpoint.get(BUCKET_NAME).getEntity(); + + assertThat(second.isTruncated()).isFalse(); + assertThat(second.getKeyMarker().getName()).isEqualTo("file+1"); + assertThat(second.getVersionIdMarker()).isEqualTo(NULL_VERSION_ID); + assertThat(second.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file2", "file3"); + } + + @Test + public void listObjectsDoesNotReturnVersionId() throws Exception { + setup("file1"); + + StringWriter writer = new StringWriter(); + JAXB.marshal(bucketEndpoint.get(BUCKET_NAME).getEntity(), writer); + + assertThat(writer.toString()).contains(" subresources() { QueryParams.METADATA_TABLE, QueryParams.METRICS, QueryParams.NOTIFICATION, QueryParams.OBJECT_LOCK, QueryParams.OWNERSHIP_CONTROLS, QueryParams.POLICY, QueryParams.POLICY_STATUS, QueryParams.PUBLIC_ACCESS_BLOCK, QueryParams.REPLICATION, QueryParams.REQUEST_PAYMENT, QueryParams.SESSION, QueryParams.VERSIONING, - QueryParams.VERSIONS, QueryParams.WEBSITE); + QueryParams.WEBSITE); } - /** Same as {@link #subresources()}, plus uploads, for which only GET (ListMultipartUploads) is implemented. */ + /** + * Same as {@link #subresources()}, plus uploads and versions, for which only GET (ListMultipartUploads and + * ListObjectVersions) is implemented. + */ private static Stream putSubresources() { - return Stream.concat(subresources(), Stream.of(QueryParams.UPLOADS)); + return Stream.concat(subresources(), Stream.of(QueryParams.UPLOADS, QueryParams.VERSIONS)); } /** Same as {@link #putSubresources()}, plus acl, for which only GET and PUT are implemented. */ diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java index a9a2a1d278d5..e9c9715f0740 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java @@ -22,6 +22,9 @@ import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; @@ -216,6 +219,30 @@ public void multiDeleteWithVersionIdIsNotImplemented() throws Exception { assertEquals(Sets.newHashSet("key1", "key2", "key3"), keysAtTheEnd); } + @Test + public void multiDeleteWithNullVersionIdDeletesCurrentObject() throws Exception { + final OzoneClient client = new OzoneClientStub(); + final OzoneBucket bucket = initTestData(client); + final BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() + .setClient(client) + .build(); + + final DeleteObject nullVersion = new DeleteObject("key2"); + nullVersion.setVersionId(NULL_VERSION_ID); + final MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1")); + mdr.getObjects().add(nullVersion); + + final MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertThat(response.getErrors()).isEmpty(); + // VersionId is only returned for the keys it was requested for + assertThat(response.getDeletedObjects()) + .extracting(DeletedObject::getKey, DeletedObject::getVersionId) + .containsExactly(tuple("key1", null), tuple("key2", NULL_VERSION_ID)); + assertThat(bucket.listKeys("")).toIterable().extracting(OzoneKey::getName).containsExactly("key3"); + } + private MultiDeleteRequest threeKeyRequest() { MultiDeleteRequest mdr = new MultiDeleteRequest(); mdr.getObjects().add(new DeleteObject("key1")); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java index f4270c7a699b..7cb49d7fcae0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java @@ -17,13 +17,17 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static java.net.HttpURLConnection.HTTP_NO_CONTENT; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertStatus; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.delete; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -110,4 +114,12 @@ public void deleteWithVersionIdIsNotImplementedAndDoesNotDeleteObject() throws I assertErrorResponse(S3ErrorTable.NOT_IMPLEMENTED, () -> delete(objectEndpoint, BUCKET_NAME, KEY_NAME)); assertKeyContent(bucket, KEY_NAME, CONTENT); } + + @Test + public void deleteWithNullVersionIdDeletesObject() throws IOException { + objectEndpoint.queryParamsForTest().set(QueryParams.VERSION_ID, NULL_VERSION_ID); + + assertStatus(HTTP_NO_CONTENT, () -> delete(objectEndpoint, BUCKET_NAME, KEY_NAME)); + assertThat(bucket.listKeys("")).isExhausted(); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java index a20b03569b1b..f3f85f1a532b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -115,6 +116,14 @@ public void testDeleteTaggingWithVersionIdIsNotImplemented() throws IOException client.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_WITH_TAG).getTags()); } + @Test + public void testDeleteTaggingWithNullVersionId() throws IOException, OS3Exception { + rest.queryParamsForTest().set(QueryParams.VERSION_ID, NULL_VERSION_ID); + + assertEquals(HTTP_NO_CONTENT, deleteTagging(rest, BUCKET_NAME, KEY_WITH_TAG).getStatus()); + assertTrue(client.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_WITH_TAG).getTags().isEmpty()); + } + @Test public void testDeleteTaggingNoKeyFound() { assertErrorResponse(NO_SUCH_KEY, () -> deleteTagging(rest, BUCKET_NAME, "nonexistent")); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java index f5777a6e9fc3..39e856a02b9b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java @@ -124,6 +124,15 @@ public void testPutObjectTaggingWithVersionIdIsNotImplemented() throws Exception assertThat(clientStub.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_NAME).getTags()).isEmpty(); } + @Test + public void testPutObjectTaggingWithNullVersionId() throws Exception { + objectEndpoint.queryParamsForTest().set(S3Consts.QueryParams.VERSION_ID, S3Consts.NULL_VERSION_ID); + + assertSucceeds(() -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, twoTags())); + assertThat(clientStub.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_NAME).getTags()) + .containsExactlyEntriesOf(TAGS); + } + @Test public void testPutObjectTaggingNoKeyFound() { assertErrorResponse(NO_SUCH_KEY, () -> putTagging(objectEndpoint, BUCKET_NAME, "nonexistent", twoTags())); From 0d1880e0bb690faf1d9a6bf419c24719bc5ab52d Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:31:35 -0700 Subject: [PATCH 2/4] HDDS-16739. Return VersionId in DeleteObjects errors, tidy tests Co-Authored-By: Claude Opus 5.5 --- .../ozone/s3/endpoint/BucketEndpoint.java | 7 +++--- .../s3/endpoint/MultiDeleteResponse.java | 11 +++++++++ .../s3/endpoint/TestObjectMultiDelete.java | 23 ++++++++++++++++--- .../s3/endpoint/TestObjectTaggingDelete.java | 5 ++-- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 4b2377f03e5f..d15888d22755 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -262,8 +262,7 @@ private boolean isListObjectVersions() { private Object toListResult(ListObjectResponse response) { return isListObjectVersions() - ? ListVersionsResult.of(response, queryParams().get(QueryParams.VERSION_ID_MARKER)) - : response; + ? ListVersionsResult.of(response, queryParams().get(QueryParams.VERSION_ID_MARKER)) : response; } private int validateMaxKeys(int maxKeys) throws OS3Exception { @@ -403,7 +402,9 @@ public MultiDeleteResponse multiDelete( } } else { failedDeletes.add(d.getKey()); - result.addError(new Error(d.getKey(), error.getCode(), error.getMessage())); + Error deleteError = new Error(d.getKey(), error.getCode(), error.getMessage()); + deleteError.setVersionId(d.getVersionId()); + result.addError(deleteError); } } getMetrics().updateDeleteKeySuccessStats(startNanos); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java index 929d0d8e2d7e..f2d2622838a1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java @@ -117,6 +117,9 @@ public static class Error { @XmlElement(name = "Message") private String message; + @XmlElement(name = "VersionId") + private String versionId; + public Error() { } @@ -149,5 +152,13 @@ public String getMessage() { public void setMessage(String message) { this.message = message; } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java index e9c9715f0740..ee0dc3a68e53 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java @@ -19,6 +19,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.singleton; +import static java.util.Collections.singletonMap; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; @@ -223,9 +224,7 @@ public void multiDeleteWithVersionIdIsNotImplemented() throws Exception { public void multiDeleteWithNullVersionIdDeletesCurrentObject() throws Exception { final OzoneClient client = new OzoneClientStub(); final OzoneBucket bucket = initTestData(client); - final BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() - .setClient(client) - .build(); + final BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder().setClient(client).build(); final DeleteObject nullVersion = new DeleteObject("key2"); nullVersion.setVersionId(NULL_VERSION_ID); @@ -243,6 +242,24 @@ public void multiDeleteWithNullVersionIdDeletesCurrentObject() throws Exception assertThat(bucket.listKeys("")).toIterable().extracting(OzoneKey::getName).containsExactly("key3"); } + @Test + public void multiDeleteReturnsVersionIdOfFailedKey() throws Exception { + OzoneBucket bucket = mock(OzoneBucket.class); + when(bucket.deleteKeys(any(), anyBoolean())) + .thenReturn(singletonMap("key1", new ErrorInfo("ACCESS_DENIED", "ACL check failed"))); + final DeleteObject nullVersion = new DeleteObject("key1"); + nullVersion.setVersionId(NULL_VERSION_ID); + final MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(nullVersion); + + final MultiDeleteResponse response = newEndpointFor(bucket, new HashMap<>()).multiDelete("b1", "", mdr); + + assertThat(response.getDeletedObjects()).isEmpty(); + assertThat(response.getErrors()) + .extracting(MultiDeleteResponse.Error::getKey, MultiDeleteResponse.Error::getVersionId) + .containsExactly(tuple("key1", NULL_VERSION_ID)); + } + private MultiDeleteRequest threeKeyRequest() { MultiDeleteRequest mdr = new MultiDeleteRequest(); mdr.getObjects().add(new DeleteObject("key1")); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java index f3f85f1a532b..3f7160886fe4 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java @@ -28,6 +28,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -120,8 +121,8 @@ public void testDeleteTaggingWithVersionIdIsNotImplemented() throws IOException public void testDeleteTaggingWithNullVersionId() throws IOException, OS3Exception { rest.queryParamsForTest().set(QueryParams.VERSION_ID, NULL_VERSION_ID); - assertEquals(HTTP_NO_CONTENT, deleteTagging(rest, BUCKET_NAME, KEY_WITH_TAG).getStatus()); - assertTrue(client.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_WITH_TAG).getTags().isEmpty()); + assertThat(deleteTagging(rest, BUCKET_NAME, KEY_WITH_TAG).getStatus()).isEqualTo(HTTP_NO_CONTENT); + assertThat(client.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_WITH_TAG).getTags()).isEmpty(); } @Test From c674dd8b58362d4582ce8b36e27367a772b57d07 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:12:47 -0700 Subject: [PATCH 3/4] HDDS-16739. Reject version-id-marker without key-marker Co-Authored-By: Claude Opus 5.5 --- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 15 +++++++++++++-- .../hadoop/ozone/s3/endpoint/TestBucketList.java | 11 +++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index d15888d22755..5cf0cd43b231 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -109,7 +109,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO final String delimiter = queryParams().containsKey(QueryParams.DELIMITER) ? queryParams().get(QueryParams.DELIMITER) : null; final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE); - final String marker = queryParams().get(isListObjectVersions() ? QueryParams.KEY_MARKER : QueryParams.MARKER); + final String marker = getListMarker(); int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000); String prefix = queryParams().get(QueryParams.PREFIX, ""); String startAfter = queryParams().get(QueryParams.START_AFTER); @@ -255,11 +255,22 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO return Response.ok(toListResult(response)).build(); } - /** ListObjectVersions lists the same keys as ListObjects (V1), but paginates with key-marker instead of marker. */ private boolean isListObjectVersions() { return queryParams().get(QueryParams.VERSIONS) != null; } + /** ListObjectVersions lists the same keys as ListObjects (V1), but paginates with key-marker instead of marker. */ + private String getListMarker() throws OS3Exception { + if (!isListObjectVersions()) { + return queryParams().get(QueryParams.MARKER); + } + final String keyMarker = queryParams().get(QueryParams.KEY_MARKER); + if (StringUtils.isEmpty(keyMarker) && StringUtils.isNotEmpty(queryParams().get(QueryParams.VERSION_ID_MARKER))) { + throw newError(S3ErrorTable.INVALID_ARGUMENT, "A version-id marker cannot be specified without a key marker"); + } + return keyMarker; + } + private Object toListResult(ListObjectResponse response) { return isListObjectVersions() ? ListVersionsResult.of(response, queryParams().get(QueryParams.VERSION_ID_MARKER)) : response; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index 6f4ad12c0a21..585042e88d57 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT; import static org.apache.hadoop.ozone.s3.endpoint.EndpointBuilder.newBucketEndpointBuilder; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.assertj.core.api.Assertions.assertThat; @@ -730,6 +731,16 @@ public void listObjectVersionsPaginatesWithKeyMarker() throws OS3Exception, IOEx assertThat(second.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file2", "file3"); } + @Test + public void listObjectVersionsRejectsVersionIdMarkerWithoutKeyMarker() throws IOException { + setup("file1"); + + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSIONS, ""); + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSION_ID_MARKER, NULL_VERSION_ID); + + assertErrorResponse(S3ErrorTable.INVALID_ARGUMENT, () -> bucketEndpoint.get(BUCKET_NAME)); + } + @Test public void listObjectsDoesNotReturnVersionId() throws Exception { setup("file1"); From 376ca6c70814dc6c79a01ee244da83904556ded7 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:51:56 -0700 Subject: [PATCH 4/4] HDDS-16739. Drop S3.md change, the doc moves to ozone-site Co-Authored-By: Claude Opus 5.5 --- hadoop-hdds/docs/content/interface/S3.md | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/docs/content/interface/S3.md b/hadoop-hdds/docs/content/interface/S3.md index 6bd7b4b16a28..5fb5f4edbd01 100644 --- a/hadoop-hdds/docs/content/interface/S3.md +++ b/hadoop-hdds/docs/content/interface/S3.md @@ -80,7 +80,6 @@ The Ozone S3 Gateway implements a substantial subset of the Amazon S3 REST API. | ✅ [DeleteObjects](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) | Deletes multiple objects in one request. | Bulk delete operation with aggregated error reporting. | | ✅ [CopyObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html) | Copies an object (intra- or inter-bucket). | Supports metadata directives (COPY/REPLACE). **Non-compliant behavior:** Self-copying for metadata updates may differ slightly from AWS S3. | | ✅ [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) | Lists objects in a bucket with filtering and pagination. | Supports prefix, delimiter, and continuation tokens. | -| ✅ [ListObjectVersions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html) | Lists the versions of the objects in a bucket. | Bucket versioning is not supported, so each object is listed with a single version whose `VersionId` is `null`, same as in an AWS S3 bucket that never had versioning enabled. `DeleteObject`, `DeleteObjects` and the object tagging operations accept `null` as the version ID and operate on the current object. | ### Object Tagging Operations