From 8ddf38ebe74d01f0b769124f21c2d6e06b91e4e1 Mon Sep 17 00:00:00 2001 From: F64116045 Date: Sat, 3 Oct 2026 04:50:07 +0800 Subject: [PATCH 1/3] HDDS-16672. Support Unicode values in S3 user metadata Signed-off-by: F64116045 --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 19 ++++++ .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 16 +++++ .../ozone/s3/endpoint/EndpointBase.java | 65 ++++++++++++++++++- .../ozone/s3/endpoint/TestObjectGet.java | 38 +++++++++++ 4 files changed, 136 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 246eae250116..4ad19663377e 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -579,6 +579,25 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutObjectWithRfc2047Metadata() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String metadataValue = "café"; + final String encodedMetadataValue = "=?UTF-8?Q?caf=C3=A9?="; + byte[] content = "bar".getBytes(StandardCharsets.UTF_8); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(content.length); + metadata.addUserMetadata("meta1", metadataValue); + s3Client.createBucket(bucketName); + + s3Client.putObject(new PutObjectRequest(bucketName, keyName, + new ByteArrayInputStream(content), metadata)); + + assertEquals(encodedMetadataValue, + s3Client.getObjectMetadata(bucketName, keyName).getUserMetaDataOf("meta1")); + } + @Test public void testPutObjectWithEmptyContentType() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 5f908eccc810..227ed407871e 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -308,6 +308,22 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testPutObjectWithRfc2047Metadata() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String metadataValue = "café"; + final String encodedMetadataValue = "=?UTF-8?Q?caf=C3=A9?="; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName) + .metadata(Collections.singletonMap("meta1", metadataValue)), + RequestBody.fromString("bar")); + + HeadObjectResponse response = s3Client.headObject(b -> b.bucket(bucketName).key(keyName)); + assertEquals(encodedMetadataValue, response.metadata().get("meta1")); + } + @Test public void testPutObjectWithEmptyContentType() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 38aca6b6fe9c..987061e6b925 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -87,6 +87,10 @@ import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import net.jcip.annotations.Immutable; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.EncoderException; +import org.apache.commons.codec.net.BCodec; +import org.apache.commons.codec.net.QCodec; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -138,6 +142,9 @@ */ public abstract class EndpointBase { + private static final BCodec RFC_2047_B_CODEC = new BCodec(UTF_8); + private static final QCodec RFC_2047_Q_CODEC = createRfc2047QCodec(); + protected static final String ETAG_CUSTOM = RESERVED_USER_METADATA_KEY_PREFIX + "etag"; protected static final String CONTENT_TYPE_CUSTOM = @@ -451,7 +458,9 @@ protected Map getCustomMetadataFromHeaders( throw ex; } List values = requestHeaders.get(key); - String value = StringUtils.join(values, ","); + String value = values.stream() + .map(EndpointBase::decodeRfc2047MetadataValue) + .collect(Collectors.joining(",")); sizeInBytes += mapKey.getBytes(UTF_8).length; sizeInBytes += value.getBytes(UTF_8).length; @@ -470,6 +479,58 @@ protected Map getCustomMetadataFromHeaders( return customMetadata; } + private static String decodeRfc2047MetadataValue(String value) { + if (!value.startsWith("=?") || !value.endsWith("?=")) { + return value; + } + + try { + return RFC_2047_Q_CODEC.decode(value); + } catch (DecoderException ex) { + try { + return RFC_2047_B_CODEC.decode(value); + } catch (DecoderException ignored) { + return value; + } + } + } + + private static String encodeRfc2047MetadataValue(String value) { + if (value.chars().allMatch(EndpointBase::isSafeAsciiMetadataCharacter)) { + return value; + } + + try { + return containsUnprintableCharacters(value) + ? RFC_2047_B_CODEC.encode(value) + : RFC_2047_Q_CODEC.encode(value); + } catch (EncoderException ex) { + throw new IllegalStateException("Failed to encode S3 metadata", ex); + } + } + + private static boolean isSafeAsciiMetadataCharacter(int value) { + return value == '\t' || value >= ' ' && value <= '~'; + } + + private static QCodec createRfc2047QCodec() { + QCodec codec = new QCodec(UTF_8); + codec.setEncodeBlanks(true); + return codec; + } + + private static boolean containsUnprintableCharacters(String value) { + return value.codePoints().anyMatch(codePoint -> { + int type = Character.getType(codePoint); + return codePoint == '\ufffd' + || type == Character.CONTROL + || type == Character.FORMAT + || type == Character.PRIVATE_USE + || type == Character.SURROGATE + || type == Character.UNASSIGNED; + }); + } + /** * Move a user value under {@code headerName} (canonical or lower-case) to * {@code customKey} so it does not collide with the system value. @@ -500,7 +561,7 @@ protected void addCustomMetadataHeaders( metadataKey = REBUILT_RESERVED_KEYS.getOrDefault(metadataKey, metadataKey); responseBuilder .header(CUSTOM_METADATA_HEADER_PREFIX + metadataKey, - entry.getValue()); + encodeRfc2047MetadataValue(entry.getValue())); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index d33e506362cd..918bbd80e346 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -243,6 +243,44 @@ public void getKeyWithCustomMetadata() throws IOException, OS3Exception { response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "meta1")); } + @Test + public void getKeyWithQEncodedRfc2047CustomMetadata() + throws IOException, OS3Exception { + final String keyName = "key-with-rfc2047-meta"; + final String metaValue = "Hello Worldé"; + final String encodedMetaValue = "=?UTF-8?Q?Hello_World=C3=A9?="; + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "meta1", + encodedMetaValue); + when(headers.getRequestHeaders()).thenReturn(requestHeaders); + + assertSucceeds(() -> put(rest, BUCKET_NAME, keyName, CONTENT)); + + assertEquals(metaValue, bucket.getKey(keyName).getMetadata().get("meta1")); + Response response = get(rest, BUCKET_NAME, keyName); + assertEquals(encodedMetaValue, + response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "meta1")); + } + + @Test + public void getKeyWithBase64Rfc2047CustomMetadata() + throws IOException, OS3Exception { + final String keyName = "key-with-base64-rfc2047-meta"; + final String metaValue = "\u0000"; + final String encodedMetaValue = "=?UTF-8?B?AA==?="; + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "meta1", + encodedMetaValue); + when(headers.getRequestHeaders()).thenReturn(requestHeaders); + + assertSucceeds(() -> put(rest, BUCKET_NAME, keyName, CONTENT)); + + assertEquals(metaValue, bucket.getKey(keyName).getMetadata().get("meta1")); + Response response = get(rest, BUCKET_NAME, keyName); + assertEquals(encodedMetaValue, + response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "meta1")); + } + @Test public void getKeyWithTag() throws IOException, OS3Exception { //WHEN From bb4b59fa83a91759e95225f89df9e186cd2f689b Mon Sep 17 00:00:00 2001 From: F64116045 Date: Sat, 3 Oct 2026 16:25:40 +0800 Subject: [PATCH 2/3] HDDS-16672. Preserve unsupported RFC 2047 metadata values Signed-off-by: F64116045 --- .../ozone/s3/endpoint/EndpointBase.java | 26 ++++++++++++---- .../ozone/s3/endpoint/TestObjectGet.java | 30 +++++++++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 987061e6b925..886a5d746be9 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -69,6 +69,7 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -480,19 +481,32 @@ protected Map getCustomMetadataFromHeaders( } private static String decodeRfc2047MetadataValue(String value) { - if (!value.startsWith("=?") || !value.endsWith("?=")) { + if (!value.startsWith("=?")) { return value; } + int charsetEnd = value.indexOf('?', 2); + int encodingEnd = charsetEnd < 0 ? -1 : value.indexOf('?', charsetEnd + 1); + int encodedTextEnd = encodingEnd < 0 + ? -1 : value.indexOf("?=", encodingEnd + 1); + if (encodedTextEnd != value.length() - 2) { + return value; + } + + String encoding = value.substring(charsetEnd + 1, encodingEnd); try { - return RFC_2047_Q_CODEC.decode(value); - } catch (DecoderException ex) { - try { + if (encoding.equalsIgnoreCase("Q")) { + return RFC_2047_Q_CODEC.decode(value); + } + if (encoding.equalsIgnoreCase("B")) { + Base64.getDecoder().decode( + value.substring(encodingEnd + 1, encodedTextEnd)); return RFC_2047_B_CODEC.decode(value); - } catch (DecoderException ignored) { - return value; } + } catch (DecoderException | IllegalArgumentException ex) { + return value; } + return value; } private static String encodeRfc2047MetadataValue(String value) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 918bbd80e346..f6e952f6f7e2 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -281,6 +281,36 @@ public void getKeyWithBase64Rfc2047CustomMetadata() response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "meta1")); } + @Test + public void getKeyPreservesMultipleRfc2047EncodedWords() + throws IOException, OS3Exception { + final String keyName = "key-with-multiple-rfc2047-words"; + final String metaValue = "=?UTF-8?Q?a?= =?UTF-8?Q?b?="; + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "meta1", + metaValue); + when(headers.getRequestHeaders()).thenReturn(requestHeaders); + + assertSucceeds(() -> put(rest, BUCKET_NAME, keyName, CONTENT)); + + assertEquals(metaValue, bucket.getKey(keyName).getMetadata().get("meta1")); + } + + @Test + public void getKeyPreservesInvalidBase64Rfc2047Metadata() + throws IOException, OS3Exception { + final String keyName = "key-with-invalid-rfc2047-base64"; + final String metaValue = "=?UTF-8?B?!!!?="; + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "meta1", + metaValue); + when(headers.getRequestHeaders()).thenReturn(requestHeaders); + + assertSucceeds(() -> put(rest, BUCKET_NAME, keyName, CONTENT)); + + assertEquals(metaValue, bucket.getKey(keyName).getMetadata().get("meta1")); + } + @Test public void getKeyWithTag() throws IOException, OS3Exception { //WHEN From a7b3cccde9345a5a9422b1b442c25ac9ad56566b Mon Sep 17 00:00:00 2001 From: F64116045 Date: Sat, 3 Oct 2026 19:06:43 +0800 Subject: [PATCH 3/3] HDDS-16672. Simplify RFC 2047 response encoding --- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 16 +--------------- .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 7 ++++--- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 886a5d746be9..927a707a2cd3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -515,9 +515,7 @@ private static String encodeRfc2047MetadataValue(String value) { } try { - return containsUnprintableCharacters(value) - ? RFC_2047_B_CODEC.encode(value) - : RFC_2047_Q_CODEC.encode(value); + return RFC_2047_Q_CODEC.encode(value); } catch (EncoderException ex) { throw new IllegalStateException("Failed to encode S3 metadata", ex); } @@ -533,18 +531,6 @@ private static QCodec createRfc2047QCodec() { return codec; } - private static boolean containsUnprintableCharacters(String value) { - return value.codePoints().anyMatch(codePoint -> { - int type = Character.getType(codePoint); - return codePoint == '\ufffd' - || type == Character.CONTROL - || type == Character.FORMAT - || type == Character.PRIVATE_USE - || type == Character.SURROGATE - || type == Character.UNASSIGNED; - }); - } - /** * Move a user value under {@code headerName} (canonical or lower-case) to * {@code customKey} so it does not collide with the system value. diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index f6e952f6f7e2..f0f6f4b45df7 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -266,8 +266,9 @@ public void getKeyWithQEncodedRfc2047CustomMetadata() public void getKeyWithBase64Rfc2047CustomMetadata() throws IOException, OS3Exception { final String keyName = "key-with-base64-rfc2047-meta"; - final String metaValue = "\u0000"; - final String encodedMetaValue = "=?UTF-8?B?AA==?="; + final String metaValue = "café"; + final String encodedMetaValue = "=?UTF-8?B?Y2Fmw6k=?="; + final String responseMetaValue = "=?UTF-8?Q?caf=C3=A9?="; MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "meta1", encodedMetaValue); @@ -277,7 +278,7 @@ public void getKeyWithBase64Rfc2047CustomMetadata() assertEquals(metaValue, bucket.getKey(keyName).getMetadata().get("meta1")); Response response = get(rest, BUCKET_NAME, keyName); - assertEquals(encodedMetaValue, + assertEquals(responseMetaValue, response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "meta1")); }