diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 6e88f7076729..4e4d5334d98f 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -143,6 +143,9 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectTaggingRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.DeletedObject; +import software.amazon.awssdk.services.s3.model.EncodingType; import software.amazon.awssdk.services.s3.model.ExpirationStatus; import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse; @@ -167,6 +170,7 @@ import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsResponse; import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.ListObjectsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -177,6 +181,7 @@ import software.amazon.awssdk.services.s3.model.ObjectAttributes; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; +import software.amazon.awssdk.services.s3.model.ObjectVersion; import software.amazon.awssdk.services.s3.model.PutBucketAclRequest; import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -967,6 +972,39 @@ public void testDeleteObjectsWithVersionIdNotImplemented() { assertDoesNotThrow(() -> s3Client.headObject(b -> b.bucket(bucketName).key(keyName))); } + @Test + public void testDeleteObjectsListedByListObjectVersions() { + final String bucketName = getBucketName(); + final List keyNames = Arrays.asList("dir/key 3", "key+1", "key2"); + s3Client.createBucket(b -> b.bucket(bucketName)); + keyNames.forEach(k -> s3Client.putObject(b -> b.bucket(bucketName).key(k), RequestBody.fromString("bar"))); + + final List versions = s3Client.listObjectVersionsPaginator( + b -> b.bucket(bucketName).maxKeys(2).encodingType(EncodingType.URL)) + .versions().stream().collect(Collectors.toList()); + + // versioning is not implemented, each object only has the null version + assertThat(versions).extracting(ObjectVersion::key).containsExactlyElementsOf(keyNames); + assertThat(versions).allSatisfy(v -> { + assertThat(v.versionId()).isEqualTo("null"); + assertThat(v.isLatest()).isTrue(); + assertThat(v.size()).isEqualTo(3L); + }); + + final DeleteObjectsResponse deleted = s3Client.deleteObjects(b -> b.bucket(bucketName).delete(d -> d.objects( + versions.stream() + .map(v -> ObjectIdentifier.builder().key(v.key()).versionId(v.versionId()).build()) + .collect(Collectors.toList())))); + + assertThat(deleted.errors()).isEmpty(); + assertThat(deleted.deleted()).extracting(DeletedObject::key).containsExactlyInAnyOrderElementsOf(keyNames); + assertThat(deleted.deleted()).extracting(DeletedObject::versionId).containsOnly("null"); + final ListObjectVersionsResponse remaining = s3Client.listObjectVersions(b -> b.bucket(bucketName)); + assertThat(remaining.versions()).isEmpty(); + assertThat(remaining.deleteMarkers()).isEmpty(); + assertThat(remaining.isTruncated()).isFalse(); + } + @Test public void testPutObjectLegalHoldNotImplemented() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java index 1ebf28a9bc4c..d0cff56f31d3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java @@ -51,6 +51,13 @@ public class KeyMetadata { @XmlElement(name = "StorageClass") private String storageClass; + // VersionId and IsLatest are only set in ListObjectVersions responses + @XmlElement(name = "VersionId") + private String versionId; + + @XmlElement(name = "IsLatest") + private Boolean isLatest; + public EncodingTypeObject getKey() { return key; } @@ -98,4 +105,20 @@ public String getStorageClass() { public void setStorageClass(String storageClass) { this.storageClass = storageClass; } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + + public Boolean getIsLatest() { + return isLatest; + } + + public void setIsLatest(Boolean latest) { + this.isLatest = latest; + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 298ee53e8a77..5cf0cd43b231 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -109,7 +109,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO final String delimiter = queryParams().containsKey(QueryParams.DELIMITER) ? queryParams().get(QueryParams.DELIMITER) : null; final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE); - final String marker = queryParams().get(QueryParams.MARKER); + final String marker = getListMarker(); int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000); String prefix = queryParams().get(QueryParams.PREFIX, ""); String startAfter = queryParams().get(QueryParams.START_AFTER); @@ -252,7 +252,28 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO context.getPerf().appendCount(keyCount); context.getPerf().appendOpLatencyNanos(opLatencyNs); response.setKeyCount(keyCount); - return Response.ok(response).build(); + return Response.ok(toListResult(response)).build(); + } + + private boolean isListObjectVersions() { + return queryParams().get(QueryParams.VERSIONS) != null; + } + + /** ListObjectVersions lists the same keys as ListObjects (V1), but paginates with key-marker instead of marker. */ + private String getListMarker() throws OS3Exception { + if (!isListObjectVersions()) { + return queryParams().get(QueryParams.MARKER); + } + final String keyMarker = queryParams().get(QueryParams.KEY_MARKER); + if (StringUtils.isEmpty(keyMarker) && StringUtils.isNotEmpty(queryParams().get(QueryParams.VERSION_ID_MARKER))) { + throw newError(S3ErrorTable.INVALID_ARGUMENT, "A version-id marker cannot be specified without a key marker"); + } + return keyMarker; + } + + private Object toListResult(ListObjectResponse response) { + return isListObjectVersions() + ? ListVersionsResult.of(response, queryParams().get(QueryParams.VERSION_ID_MARKER)) : response; } private int validateMaxKeys(int maxKeys) throws OS3Exception { @@ -353,7 +374,8 @@ public MultiDeleteResponse multiDelete( } // Deleting a specific version is not implemented; ignoring VersionId would delete the current object instead. - if (request.getObjects() != null && request.getObjects().stream().anyMatch(o -> o.getVersionId() != null)) { + if (request.getObjects() != null + && request.getObjects().stream().anyMatch(o -> isUnsupportedVersionId(o.getVersionId()))) { throw newError(S3ErrorTable.NOT_IMPLEMENTED, bucketName); } @@ -385,11 +407,15 @@ public MultiDeleteResponse multiDelete( ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode()); if (deleted) { if (!request.isQuiet()) { - result.addDeleted(new DeletedObject(d.getKey())); + DeletedObject deletedObject = new DeletedObject(d.getKey()); + deletedObject.setVersionId(d.getVersionId()); + result.addDeleted(deletedObject); } } else { failedDeletes.add(d.getKey()); - result.addError(new Error(d.getKey(), error.getCode(), error.getMessage())); + Error deleteError = new Error(d.getKey(), error.getCode(), error.getMessage()); + deleteError.setVersionId(d.getVersionId()); + result.addError(deleteError); } } getMetrics().updateDeleteKeySuccessStats(startNanos); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java index 0ca3cd5b413b..bf97576daedb 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.s3.endpoint; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import java.io.InputStream; import java.util.Set; import javax.ws.rs.core.Response; @@ -47,9 +48,13 @@ class BucketOperationNotImplementedHandler extends BucketOperationHandler { QueryParams.PUBLIC_ACCESS_BLOCK, QueryParams.REPLICATION, QueryParams.REQUEST_PAYMENT, QueryParams.SESSION, QueryParams.VERSIONING, QueryParams.VERSIONS, QueryParams.WEBSITE); + // ListObjectVersions is implemented by BucketEndpoint + private static final Set GET_SUBRESOURCES = + Sets.difference(SUBRESOURCES, ImmutableSet.of(QueryParams.VERSIONS)).immutableCopy(); + @Override Response handleGetRequest(S3RequestContext context, String bucketName) { - return rejectNotImplemented(context, SUBRESOURCES, bucketName); + return rejectNotImplemented(context, GET_SUBRESOURCES, bucketName); } @Override diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index b7b02b5fc538..6def8ec55227 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -41,6 +41,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.RESERVED_USER_METADATA_KEY_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CONFIG_HEADER; @@ -790,6 +791,15 @@ protected void validateObjectKeyUri(String keyPath) throws OS3Exception { } } + /** + * Versioning is not implemented, so each object only has the null version, i.e. the current object. + * + * @return true if {@code versionId} refers to any other version + */ + static boolean isUnsupportedVersionId(@Nullable String versionId) { + return versionId != null && !NULL_VERSION_ID.equals(versionId); + } + /** * Rejects the request with {@code NotImplemented} if it has any of the {@code subresources}. * diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java new file mode 100644 index 000000000000..309f4b7303cf --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; + +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.apache.hadoop.ozone.s3.commontypes.CommonPrefix; +import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; +import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; +import org.apache.hadoop.ozone.s3.commontypes.ObjectKeyNameAdapter; +import org.apache.hadoop.ozone.s3.util.S3Consts; + +/** + * Response of ListObjectVersions. + *

+ * Versioning is not implemented, so each object is listed with its only version, the null version. + * This is the same as what AWS S3 returns for a bucket that never had versioning enabled. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "ListVersionsResult", namespace = S3Consts.S3_XML_NAMESPACE) +public class ListVersionsResult { + + @XmlElement(name = "Name") + private String name; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "Prefix") + private EncodingTypeObject prefix; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "KeyMarker") + private EncodingTypeObject keyMarker; + + @XmlElement(name = "VersionIdMarker") + private String versionIdMarker; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "NextKeyMarker") + private EncodingTypeObject nextKeyMarker; + + @XmlElement(name = "NextVersionIdMarker") + private String nextVersionIdMarker; + + @XmlElement(name = "MaxKeys") + private int maxKeys; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "Delimiter") + private EncodingTypeObject delimiter; + + @XmlElement(name = "EncodingType") + private String encodingType; + + @XmlElement(name = "IsTruncated") + private boolean isTruncated; + + @XmlElement(name = "Version") + private List versions; + + @XmlElement(name = "CommonPrefixes") + private List commonPrefixes; + + /** + * @param listing result of listing the bucket with {@code key-marker} as the marker + * @param versionIdMarker {@code version-id-marker} of the request + */ + static ListVersionsResult of(ListObjectResponse listing, String versionIdMarker) { + final String encodingType = listing.getEncodingType(); + final ListVersionsResult result = new ListVersionsResult(); + result.name = listing.getName(); + result.prefix = listing.getPrefix(); + result.keyMarker = EncodingTypeObject.createNullable(listing.getMarker(), encodingType); + result.versionIdMarker = versionIdMarker == null ? "" : versionIdMarker; + result.maxKeys = listing.getMaxKeys(); + result.delimiter = listing.getDelimiter(); + result.encodingType = encodingType; + result.isTruncated = listing.isTruncated(); + if (listing.isTruncated()) { + result.nextKeyMarker = EncodingTypeObject.createNullable(listing.getNextMarker(), encodingType); + result.nextVersionIdMarker = NULL_VERSION_ID; + } + result.versions = listing.getContents(); + for (KeyMetadata version : result.versions) { + version.setVersionId(NULL_VERSION_ID); + version.setIsLatest(true); + } + result.commonPrefixes = listing.getCommonPrefixes(); + return result; + } + + public EncodingTypeObject getKeyMarker() { + return keyMarker; + } + + public String getVersionIdMarker() { + return versionIdMarker; + } + + public EncodingTypeObject getNextKeyMarker() { + return nextKeyMarker; + } + + public String getNextVersionIdMarker() { + return nextVersionIdMarker; + } + + public boolean isTruncated() { + return isTruncated; + } + + public List getVersions() { + return versions; + } + + public List getCommonPrefixes() { + return commonPrefixes; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java index 222a56239346..f2d2622838a1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java @@ -74,6 +74,7 @@ public static class DeletedObject { @XmlElement(name = "Key") private String key; + @XmlElement(name = "VersionId") private String versionId; public DeletedObject() { @@ -116,6 +117,9 @@ public static class Error { @XmlElement(name = "Message") private String message; + @XmlElement(name = "VersionId") + private String versionId; + public Error() { } @@ -148,5 +152,13 @@ public String getMessage() { public void setMessage(String message) { this.message = message; } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java index afce9aca85fa..2302c48897ab 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java @@ -21,6 +21,7 @@ import java.io.InputStream; import java.util.Set; import javax.ws.rs.core.Response; +import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint.ObjectRequestContext; import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; @@ -42,12 +43,6 @@ class ObjectOperationNotImplementedHandler extends ObjectOperationHandler { QueryParams.ANNOTATION, QueryParams.ENCRYPTION, QueryParams.LEGAL_HOLD, QueryParams.RENAME_OBJECT, QueryParams.RETENTION, QueryParams.UPLOADS); - // DeleteObject with versionId targets a specific version; ignoring it would delete the current object instead. - private static final Set DELETE_SUBRESOURCES = ImmutableSet.builder() - .addAll(SUBRESOURCES) - .add(QueryParams.VERSION_ID) - .build(); - @Override Response handleGetRequest(ObjectRequestContext context, String keyName) { return rejectNotImplemented(context, SUBRESOURCES, keyName); @@ -60,6 +55,10 @@ Response handlePutRequest(ObjectRequestContext context, String keyName, InputStr @Override Response handleDeleteRequest(ObjectRequestContext context, String keyName) { - return rejectNotImplemented(context, DELETE_SUBRESOURCES, keyName); + // DeleteObject with versionId targets a specific version; ignoring it would delete the current object instead. + if (isUnsupportedVersionId(queryParams().get(QueryParams.VERSION_ID))) { + return rejectNotImplemented(context, S3GAction.NOT_IMPLEMENTED, keyName); + } + return rejectNotImplemented(context, SUBRESOURCES, keyName); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java index 5c04df9a90db..8f5ed82558bc 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectTaggingHandler.java @@ -119,7 +119,7 @@ Response handleGetRequest(ObjectRequestContext context, String keyName) // Tagging a specific version is not implemented; ignoring versionId would modify the tags of the current object. private void rejectVersionId(String keyName) { - if (queryParams().get(S3Consts.QueryParams.VERSION_ID) != null) { + if (isUnsupportedVersionId(queryParams().get(S3Consts.QueryParams.VERSION_ID))) { throw S3ErrorTable.newError(S3ErrorTable.NOT_IMPLEMENTED, keyName); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 0cb61678d68b..c3300d37dc32 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -114,6 +114,9 @@ public final class S3Consts { /** AWS S3 maximum number of keys per DeleteObjects request. */ public static final int S3_DELETE_OBJECTS_MAX_KEYS = 1000; + /** Version ID of an object in a bucket that never had versioning enabled. */ + public static final String NULL_VERSION_ID = "null"; + // Bucket owner condition headers // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-owner-condition.html public static final String EXPECTED_BUCKET_OWNER_HEADER = "x-amz-expected-bucket-owner"; @@ -192,6 +195,8 @@ public static final class QueryParams { public static final String LIFECYCLE = "lifecycle"; public static final String ATTRIBUTES = "attributes"; public static final String VERSION_ID = "versionId"; + public static final String VERSION_ID_MARKER = "version-id-marker"; + public static final String VERSIONS = "versions"; // Subresources of S3 APIs that are not implemented. They are only recognized so that the // request can be rejected with NotImplemented instead of falling through to another operation. @@ -224,7 +229,6 @@ public static final class QueryParams { public static final String RETENTION = "retention"; public static final String SESSION = "session"; public static final String VERSIONING = "versioning"; - public static final String VERSIONS = "versions"; public static final String WEBSITE = "website"; private QueryParams() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index d16ef557bff3..585042e88d57 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -19,7 +19,10 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT; import static org.apache.hadoop.ozone.s3.endpoint.EndpointBuilder.newBucketEndpointBuilder; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -669,6 +672,85 @@ public void continuationTokenXmlElementName() throws Exception { "response must not use the non-AWS element"); } + @Test + public void listObjectVersionsReturnsNullVersionOfEachKey() throws OS3Exception, IOException { + setup("dir1/file2", "file+1", "file3"); + + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSIONS, ""); + bucketEndpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + bucketEndpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, ENCODING_TYPE); + ListVersionsResult result = (ListVersionsResult) bucketEndpoint.get(BUCKET_NAME).getEntity(); + + assertThat(result.isTruncated()).isFalse(); + assertThat(result.getNextKeyMarker()).isNull(); + assertThat(result.getNextVersionIdMarker()).isNull(); + assertThat(result.getCommonPrefixes()).extracting(p -> p.getPrefix().getName()).containsExactly("dir1/"); + assertThat(result.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file+1", "file3"); + assertThat(result.getVersions()).allSatisfy(v -> { + assertThat(v.getVersionId()).isEqualTo(NULL_VERSION_ID); + assertThat(v.getIsLatest()).isTrue(); + assertThat(v.getOwner()).isNotNull(); + }); + + StringWriter writer = new StringWriter(); + JAXB.marshal(result, writer); + assertThat(writer.toString()) + .contains("", "") + .contains("", "file%2B1", "null", "true") + .doesNotContain("NextKeyMarker", "DeleteMarker"); + } + + @Test + public void listObjectVersionsPaginatesWithKeyMarker() throws OS3Exception, IOException { + setup("file+1", "file2", "file3"); + + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSIONS, ""); + bucketEndpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, ENCODING_TYPE); + bucketEndpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 1); + // marker is the pagination parameter of ListObjects, it must be ignored + bucketEndpoint.queryParamsForTest().set(QueryParams.MARKER, "file3"); + ListVersionsResult first = (ListVersionsResult) bucketEndpoint.get(BUCKET_NAME).getEntity(); + + assertThat(first.isTruncated()).isTrue(); + assertThat(first.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file+1"); + assertThat(first.getNextKeyMarker().getName()).isEqualTo("file+1"); + assertThat(first.getNextVersionIdMarker()).isEqualTo(NULL_VERSION_ID); + StringWriter writer = new StringWriter(); + JAXB.marshal(first, writer); + assertThat(writer.toString()) + .contains("file%2B1", "null"); + + bucketEndpoint.queryParamsForTest().unset(QueryParams.MAX_KEYS); + bucketEndpoint.queryParamsForTest().set(QueryParams.KEY_MARKER, first.getNextKeyMarker().getName()); + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSION_ID_MARKER, first.getNextVersionIdMarker()); + ListVersionsResult second = (ListVersionsResult) bucketEndpoint.get(BUCKET_NAME).getEntity(); + + assertThat(second.isTruncated()).isFalse(); + assertThat(second.getKeyMarker().getName()).isEqualTo("file+1"); + assertThat(second.getVersionIdMarker()).isEqualTo(NULL_VERSION_ID); + assertThat(second.getVersions()).extracting(v -> v.getKey().getName()).containsExactly("file2", "file3"); + } + + @Test + public void listObjectVersionsRejectsVersionIdMarkerWithoutKeyMarker() throws IOException { + setup("file1"); + + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSIONS, ""); + bucketEndpoint.queryParamsForTest().set(QueryParams.VERSION_ID_MARKER, NULL_VERSION_ID); + + assertErrorResponse(S3ErrorTable.INVALID_ARGUMENT, () -> bucketEndpoint.get(BUCKET_NAME)); + } + + @Test + public void listObjectsDoesNotReturnVersionId() throws Exception { + setup("file1"); + + StringWriter writer = new StringWriter(); + JAXB.marshal(bucketEndpoint.get(BUCKET_NAME).getEntity(), writer); + + assertThat(writer.toString()).contains(" subresources() { QueryParams.METADATA_TABLE, QueryParams.METRICS, QueryParams.NOTIFICATION, QueryParams.OBJECT_LOCK, QueryParams.OWNERSHIP_CONTROLS, QueryParams.POLICY, QueryParams.POLICY_STATUS, QueryParams.PUBLIC_ACCESS_BLOCK, QueryParams.REPLICATION, QueryParams.REQUEST_PAYMENT, QueryParams.SESSION, QueryParams.VERSIONING, - QueryParams.VERSIONS, QueryParams.WEBSITE); + QueryParams.WEBSITE); } - /** Same as {@link #subresources()}, plus uploads, for which only GET (ListMultipartUploads) is implemented. */ + /** + * Same as {@link #subresources()}, plus uploads and versions, for which only GET (ListMultipartUploads and + * ListObjectVersions) is implemented. + */ private static Stream putSubresources() { - return Stream.concat(subresources(), Stream.of(QueryParams.UPLOADS)); + return Stream.concat(subresources(), Stream.of(QueryParams.UPLOADS, QueryParams.VERSIONS)); } /** Same as {@link #putSubresources()}, plus acl, for which only GET and PUT are implemented. */ diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java index a9a2a1d278d5..ee0dc3a68e53 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java @@ -19,9 +19,13 @@ import static java.util.Arrays.asList; import static java.util.Collections.singleton; +import static java.util.Collections.singletonMap; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; @@ -216,6 +220,46 @@ public void multiDeleteWithVersionIdIsNotImplemented() throws Exception { assertEquals(Sets.newHashSet("key1", "key2", "key3"), keysAtTheEnd); } + @Test + public void multiDeleteWithNullVersionIdDeletesCurrentObject() throws Exception { + final OzoneClient client = new OzoneClientStub(); + final OzoneBucket bucket = initTestData(client); + final BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder().setClient(client).build(); + + final DeleteObject nullVersion = new DeleteObject("key2"); + nullVersion.setVersionId(NULL_VERSION_ID); + final MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1")); + mdr.getObjects().add(nullVersion); + + final MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertThat(response.getErrors()).isEmpty(); + // VersionId is only returned for the keys it was requested for + assertThat(response.getDeletedObjects()) + .extracting(DeletedObject::getKey, DeletedObject::getVersionId) + .containsExactly(tuple("key1", null), tuple("key2", NULL_VERSION_ID)); + assertThat(bucket.listKeys("")).toIterable().extracting(OzoneKey::getName).containsExactly("key3"); + } + + @Test + public void multiDeleteReturnsVersionIdOfFailedKey() throws Exception { + OzoneBucket bucket = mock(OzoneBucket.class); + when(bucket.deleteKeys(any(), anyBoolean())) + .thenReturn(singletonMap("key1", new ErrorInfo("ACCESS_DENIED", "ACL check failed"))); + final DeleteObject nullVersion = new DeleteObject("key1"); + nullVersion.setVersionId(NULL_VERSION_ID); + final MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(nullVersion); + + final MultiDeleteResponse response = newEndpointFor(bucket, new HashMap<>()).multiDelete("b1", "", mdr); + + assertThat(response.getDeletedObjects()).isEmpty(); + assertThat(response.getErrors()) + .extracting(MultiDeleteResponse.Error::getKey, MultiDeleteResponse.Error::getVersionId) + .containsExactly(tuple("key1", NULL_VERSION_ID)); + } + private MultiDeleteRequest threeKeyRequest() { MultiDeleteRequest mdr = new MultiDeleteRequest(); mdr.getObjects().add(new DeleteObject("key1")); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java index f4270c7a699b..7cb49d7fcae0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java @@ -17,13 +17,17 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static java.net.HttpURLConnection.HTTP_NO_CONTENT; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertStatus; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.delete; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -110,4 +114,12 @@ public void deleteWithVersionIdIsNotImplementedAndDoesNotDeleteObject() throws I assertErrorResponse(S3ErrorTable.NOT_IMPLEMENTED, () -> delete(objectEndpoint, BUCKET_NAME, KEY_NAME)); assertKeyContent(bucket, KEY_NAME, CONTENT); } + + @Test + public void deleteWithNullVersionIdDeletesObject() throws IOException { + objectEndpoint.queryParamsForTest().set(QueryParams.VERSION_ID, NULL_VERSION_ID); + + assertStatus(HTTP_NO_CONTENT, () -> delete(objectEndpoint, BUCKET_NAME, KEY_NAME)); + assertThat(bucket.listKeys("")).isExhausted(); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java index a20b03569b1b..3f7160886fe4 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java @@ -25,8 +25,10 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; +import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -115,6 +117,14 @@ public void testDeleteTaggingWithVersionIdIsNotImplemented() throws IOException client.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_WITH_TAG).getTags()); } + @Test + public void testDeleteTaggingWithNullVersionId() throws IOException, OS3Exception { + rest.queryParamsForTest().set(QueryParams.VERSION_ID, NULL_VERSION_ID); + + assertThat(deleteTagging(rest, BUCKET_NAME, KEY_WITH_TAG).getStatus()).isEqualTo(HTTP_NO_CONTENT); + assertThat(client.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_WITH_TAG).getTags()).isEmpty(); + } + @Test public void testDeleteTaggingNoKeyFound() { assertErrorResponse(NO_SUCH_KEY, () -> deleteTagging(rest, BUCKET_NAME, "nonexistent")); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java index f5777a6e9fc3..39e856a02b9b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java @@ -124,6 +124,15 @@ public void testPutObjectTaggingWithVersionIdIsNotImplemented() throws Exception assertThat(clientStub.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_NAME).getTags()).isEmpty(); } + @Test + public void testPutObjectTaggingWithNullVersionId() throws Exception { + objectEndpoint.queryParamsForTest().set(S3Consts.QueryParams.VERSION_ID, S3Consts.NULL_VERSION_ID); + + assertSucceeds(() -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, twoTags())); + assertThat(clientStub.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_NAME).getTags()) + .containsExactlyEntriesOf(TAGS); + } + @Test public void testPutObjectTaggingNoKeyFound() { assertErrorResponse(NO_SUCH_KEY, () -> putTagging(objectEndpoint, BUCKET_NAME, "nonexistent", twoTags()));