HDDS-16739. S3 ListObjectVersions should return the null version of each object

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -143,6 +143,9 @@
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectTaggingRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.DeletedObject;
+import software.amazon.awssdk.services.s3.model.EncodingType;
 import software.amazon.awssdk.services.s3.model.ExpirationStatus;
 import software.amazon.awssdk.services.s3.model.GetBucketAclRequest;
 import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse;
@@ -167,6 +170,7 @@
 import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsResponse;
 import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
 import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
 import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
 import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
 import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
@@ -177,6 +181,7 @@
 import software.amazon.awssdk.services.s3.model.ObjectAttributes;
 import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
 import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
 import software.amazon.awssdk.services.s3.model.PutBucketAclRequest;
 import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -967,6 +972,39 @@ public void testDeleteObjectsWithVersionIdNotImplemented() {
     assertDoesNotThrow(() -> s3Client.headObject(b -> b.bucket(bucketName).key(keyName)));
   }
 
+  @Test
+  public void testDeleteObjectsListedByListObjectVersions() {
+    final String bucketName = getBucketName();
+    final List<String> keyNames = Arrays.asList("dir/key 3", "key+1", "key2");
+    s3Client.createBucket(b -> b.bucket(bucketName));
+    keyNames.forEach(k -> s3Client.putObject(b -> b.bucket(bucketName).key(k), RequestBody.fromString("bar")));
+
+    final List<ObjectVersion> versions = s3Client.listObjectVersionsPaginator(
+            b -> b.bucket(bucketName).maxKeys(2).encodingType(EncodingType.URL))
+        .versions().stream().collect(Collectors.toList());
+
+    // versioning is not implemented, each object only has the null version
+    assertThat(versions).extracting(ObjectVersion::key).containsExactlyElementsOf(keyNames);
+    assertThat(versions).allSatisfy(v -> {
+      assertThat(v.versionId()).isEqualTo("null");
+      assertThat(v.isLatest()).isTrue();
+      assertThat(v.size()).isEqualTo(3L);
+    });
+
+    final DeleteObjectsResponse deleted = s3Client.deleteObjects(b -> b.bucket(bucketName).delete(d -> d.objects(
+        versions.stream()
+            .map(v -> ObjectIdentifier.builder().key(v.key()).versionId(v.versionId()).build())
+            .collect(Collectors.toList()))));
+
+    assertThat(deleted.errors()).isEmpty();
+    assertThat(deleted.deleted()).extracting(DeletedObject::key).containsExactlyInAnyOrderElementsOf(keyNames);
+    assertThat(deleted.deleted()).extracting(DeletedObject::versionId).containsOnly("null");
+    final ListObjectVersionsResponse remaining = s3Client.listObjectVersions(b -> b.bucket(bucketName));
+    assertThat(remaining.versions()).isEmpty();
+    assertThat(remaining.deleteMarkers()).isEmpty();
+    assertThat(remaining.isTruncated()).isFalse();
+  }
+
   @Test
   public void testPutObjectLegalHoldNotImplemented() {
     final String bucketName = getBucketName();

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java

@@ -51,6 +51,13 @@ public class KeyMetadata {
   @XmlElement(name = "StorageClass")
   private String storageClass;
 
+  // VersionId and IsLatest are only set in ListObjectVersions responses
+  @XmlElement(name = "VersionId")
+  private String versionId;
+
+  @XmlElement(name = "IsLatest")
+  private Boolean isLatest;
+
   public EncodingTypeObject getKey() {
     return key;
   }
@@ -98,4 +105,20 @@ public String getStorageClass() {
   public void setStorageClass(String storageClass) {
     this.storageClass = storageClass;
   }
+
+  public String getVersionId() {
+    return versionId;
+  }
+
+  public void setVersionId(String versionId) {
+    this.versionId = versionId;
+  }
+
+  public Boolean getIsLatest() {
+    return isLatest;
+  }
+
+  public void setIsLatest(Boolean latest) {
+    this.isLatest = latest;
+  }
 }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -109,7 +109,7 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO
     final String delimiter = queryParams().containsKey(QueryParams.DELIMITER) ?
         queryParams().get(QueryParams.DELIMITER) : null;
     final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE);
-    final String marker = queryParams().get(QueryParams.MARKER);
+    final String marker = getListMarker();
     int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000);
     String prefix = queryParams().get(QueryParams.PREFIX, "");
     String startAfter = queryParams().get(QueryParams.START_AFTER);
@@ -252,7 +252,28 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO
     context.getPerf().appendCount(keyCount);
     context.getPerf().appendOpLatencyNanos(opLatencyNs);
     response.setKeyCount(keyCount);
-    return Response.ok(response).build();
+    return Response.ok(toListResult(response)).build();
+  }
+
+  private boolean isListObjectVersions() {
+    return queryParams().get(QueryParams.VERSIONS) != null;
+  }
+
+  /** ListObjectVersions lists the same keys as ListObjects (V1), but paginates with key-marker instead of marker. */
+  private String getListMarker() throws OS3Exception {
+    if (!isListObjectVersions()) {
+      return queryParams().get(QueryParams.MARKER);
+    }
+    final String keyMarker = queryParams().get(QueryParams.KEY_MARKER);
+    if (StringUtils.isEmpty(keyMarker) && StringUtils.isNotEmpty(queryParams().get(QueryParams.VERSION_ID_MARKER))) {
+      throw newError(S3ErrorTable.INVALID_ARGUMENT, "A version-id marker cannot be specified without a key marker");
+    }
+    return keyMarker;
+  }
+
+  private Object toListResult(ListObjectResponse response) {
+    return isListObjectVersions()
+        ? ListVersionsResult.of(response, queryParams().get(QueryParams.VERSION_ID_MARKER)) : response;
   }
 
   private int validateMaxKeys(int maxKeys) throws OS3Exception {
@@ -353,7 +374,8 @@ public MultiDeleteResponse multiDelete(
     }
 
     // Deleting a specific version is not implemented; ignoring VersionId would delete the current object instead.
-    if (request.getObjects() != null && request.getObjects().stream().anyMatch(o -> o.getVersionId() != null)) {
+    if (request.getObjects() != null
+        && request.getObjects().stream().anyMatch(o -> isUnsupportedVersionId(o.getVersionId()))) {
       throw newError(S3ErrorTable.NOT_IMPLEMENTED, bucketName);
     }
 
@@ -385,11 +407,15 @@ public MultiDeleteResponse multiDelete(
               ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode());
           if (deleted) {
             if (!request.isQuiet()) {
-              result.addDeleted(new DeletedObject(d.getKey()));
+              DeletedObject deletedObject = new DeletedObject(d.getKey());
+              deletedObject.setVersionId(d.getVersionId());
+              result.addDeleted(deletedObject);
             }
           } else {
             failedDeletes.add(d.getKey());
-            result.addError(new Error(d.getKey(), error.getCode(), error.getMessage()));
+            Error deleteError = new Error(d.getKey(), error.getCode(), error.getMessage());
+            deleteError.setVersionId(d.getVersionId());
+            result.addError(deleteError);
           }
         }
         getMetrics().updateDeleteKeySuccessStats(startNanos);

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.ozone.s3.endpoint;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 import java.io.InputStream;
 import java.util.Set;
 import javax.ws.rs.core.Response;
@@ -47,9 +48,13 @@ class BucketOperationNotImplementedHandler extends BucketOperationHandler {
       QueryParams.PUBLIC_ACCESS_BLOCK, QueryParams.REPLICATION, QueryParams.REQUEST_PAYMENT, QueryParams.SESSION,
       QueryParams.VERSIONING, QueryParams.VERSIONS, QueryParams.WEBSITE);
 
+  // ListObjectVersions is implemented by BucketEndpoint
+  private static final Set<String> GET_SUBRESOURCES =
+      Sets.difference(SUBRESOURCES, ImmutableSet.of(QueryParams.VERSIONS)).immutableCopy();
+
   @Override
   Response handleGetRequest(S3RequestContext context, String bucketName) {
-    return rejectNotImplemented(context, SUBRESOURCES, bucketName);
+    return rejectNotImplemented(context, GET_SUBRESOURCES, bucketName);
   }
 
   @Override

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -41,6 +41,7 @@
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.RESERVED_USER_METADATA_KEY_PREFIX;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CONFIG_HEADER;
@@ -790,6 +791,15 @@ protected void validateObjectKeyUri(String keyPath) throws OS3Exception {
     }
   }
 
+  /**
+   * Versioning is not implemented, so each object only has the null version, i.e. the current object.
+   *
+   * @return true if {@code versionId} refers to any other version
+   */
+  static boolean isUnsupportedVersionId(@Nullable String versionId) {
+    return versionId != null && !NULL_VERSION_ID.equals(versionId);
+  }
+
   /**
    * Rejects the request with {@code NotImplemented} if it has any of the {@code subresources}.
    *

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListVersionsResult.java

@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static org.apache.hadoop.ozone.s3.util.S3Consts.NULL_VERSION_ID;
+
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import org.apache.hadoop.ozone.s3.commontypes.CommonPrefix;
+import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject;
+import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata;
+import org.apache.hadoop.ozone.s3.commontypes.ObjectKeyNameAdapter;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+
+/**
+ * Response of ListObjectVersions.
+ * <p>
+ * Versioning is not implemented, so each object is listed with its only version, the null version.
+ * This is the same as what AWS S3 returns for a bucket that never had versioning enabled.
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "ListVersionsResult", namespace = S3Consts.S3_XML_NAMESPACE)
+public class ListVersionsResult {
+
+  @XmlElement(name = "Name")
+  private String name;
+
+  @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class)
+  @XmlElement(name = "Prefix")
+  private EncodingTypeObject prefix;
+
+  @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class)
+  @XmlElement(name = "KeyMarker")
+  private EncodingTypeObject keyMarker;
+
+  @XmlElement(name = "VersionIdMarker")
+  private String versionIdMarker;
+
+  @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class)
+  @XmlElement(name = "NextKeyMarker")
+  private EncodingTypeObject nextKeyMarker;
+
+  @XmlElement(name = "NextVersionIdMarker")
+  private String nextVersionIdMarker;
+
+  @XmlElement(name = "MaxKeys")
+  private int maxKeys;
+
+  @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class)
+  @XmlElement(name = "Delimiter")
+  private EncodingTypeObject delimiter;
+
+  @XmlElement(name = "EncodingType")
+  private String encodingType;
+
+  @XmlElement(name = "IsTruncated")
+  private boolean isTruncated;
+
+  @XmlElement(name = "Version")
+  private List<KeyMetadata> versions;
+
+  @XmlElement(name = "CommonPrefixes")
+  private List<CommonPrefix> commonPrefixes;
+
+  /**
+   * @param listing result of listing the bucket with {@code key-marker} as the marker
+   * @param versionIdMarker {@code version-id-marker} of the request
+   */
+  static ListVersionsResult of(ListObjectResponse listing, String versionIdMarker) {
+    final String encodingType = listing.getEncodingType();
+    final ListVersionsResult result = new ListVersionsResult();
+    result.name = listing.getName();
+    result.prefix = listing.getPrefix();
+    result.keyMarker = EncodingTypeObject.createNullable(listing.getMarker(), encodingType);
+    result.versionIdMarker = versionIdMarker == null ? "" : versionIdMarker;
+    result.maxKeys = listing.getMaxKeys();
+    result.delimiter = listing.getDelimiter();
+    result.encodingType = encodingType;
+    result.isTruncated = listing.isTruncated();
+    if (listing.isTruncated()) {
+      result.nextKeyMarker = EncodingTypeObject.createNullable(listing.getNextMarker(), encodingType);
+      result.nextVersionIdMarker = NULL_VERSION_ID;
+    }
+    result.versions = listing.getContents();
+    for (KeyMetadata version : result.versions) {
+      version.setVersionId(NULL_VERSION_ID);
+      version.setIsLatest(true);
+    }
+    result.commonPrefixes = listing.getCommonPrefixes();
+    return result;
+  }
+
+  public EncodingTypeObject getKeyMarker() {
+    return keyMarker;
+  }
+
+  public String getVersionIdMarker() {
+    return versionIdMarker;
+  }
+
+  public EncodingTypeObject getNextKeyMarker() {
+    return nextKeyMarker;
+  }
+
+  public String getNextVersionIdMarker() {
+    return nextVersionIdMarker;
+  }
+
+  public boolean isTruncated() {
+    return isTruncated;
+  }
+
+  public List<KeyMetadata> getVersions() {
+    return versions;
+  }
+
+  public List<CommonPrefix> getCommonPrefixes() {
+    return commonPrefixes;
+  }
+}

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteResponse.java

@@ -74,6 +74,7 @@ public static class DeletedObject {
     @XmlElement(name = "Key")
     private String key;
 
+    @XmlElement(name = "VersionId")
     private String versionId;
 
     public DeletedObject() {
@@ -116,6 +117,9 @@ public static class Error {
     @XmlElement(name = "Message")
     private String message;
 
+    @XmlElement(name = "VersionId")
+    private String versionId;
+
     public Error() {
     }
 
@@ -148,5 +152,13 @@ public String getMessage() {
     public void setMessage(String message) {
       this.message = message;
     }
+
+    public String getVersionId() {
+      return versionId;
+    }
+
+    public void setVersionId(String versionId) {
+      this.versionId = versionId;
+    }
   }
 }