diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java index b6886d9f58ac..5578f3073654 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.io.SeekableInputStream; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.util.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +45,6 @@ class AnalyticsAcceleratorUtil { private static final Cache, S3SeekableInputStreamFactory> STREAM_FACTORY_CACHE = Caffeine.newBuilder() - .maximumSize(100) .removalListener( (RemovalListener< Pair, S3SeekableInputStreamFactory>) @@ -57,9 +57,7 @@ public static SeekableInputStream newStream(S3InputFile inputFile) { S3URI uri = S3URI.of(inputFile.uri().bucket(), inputFile.uri().key()); S3SeekableInputStreamFactory factory = - STREAM_FACTORY_CACHE.get( - Pair.of(inputFile.asyncClient(), inputFile.s3FileIOProperties()), - AnalyticsAcceleratorUtil::createNewFactory); + streamFactory(inputFile.asyncClient(), inputFile.s3FileIOProperties()); try { S3SeekableInputStream seekableInputStream = factory.createStream(uri); @@ -70,6 +68,18 @@ public static SeekableInputStream newStream(S3InputFile inputFile) { } } + static S3SeekableInputStreamFactory streamFactory( + S3AsyncClient asyncClient, S3FileIOProperties s3FileIOProperties) { + return STREAM_FACTORY_CACHE.get( + Pair.of(asyncClient, s3FileIOProperties), AnalyticsAcceleratorUtil::createNewFactory); + } + + @VisibleForTesting + static Cache, S3SeekableInputStreamFactory> + streamFactoryCache() { + return STREAM_FACTORY_CACHE; + } + private static S3SeekableInputStreamFactory createNewFactory( Pair cacheKey) { ConnectorConfiguration connectorConfiguration = diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestAnalyticsAcceleratorUtil.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestAnalyticsAcceleratorUtil.java new file mode 100644 index 000000000000..0dd009e62c72 --- /dev/null +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestAnalyticsAcceleratorUtil.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.aws.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.Pair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamFactory; + +class TestAnalyticsAcceleratorUtil { + private static final int OPEN_FILE_IO_COUNT = 150; + + private final S3FileIOProperties properties = new S3FileIOProperties(); + private final List clients = Lists.newArrayList(); + + @AfterEach + void after() { + clients.forEach(client -> AnalyticsAcceleratorUtil.cleanupCache(client, properties)); + } + + @Test + void factoriesOfOpenFileIOsAreNotEvicted() { + List factories = Lists.newArrayList(); + for (int i = 0; i < OPEN_FILE_IO_COUNT; i += 1) { + S3AsyncClient client = Mockito.mock(S3AsyncClient.class); + clients.add(client); + factories.add(AnalyticsAcceleratorUtil.streamFactory(client, properties)); + } + + AnalyticsAcceleratorUtil.streamFactoryCache().cleanUp(); + + for (int i = 0; i < OPEN_FILE_IO_COUNT; i += 1) { + assertThat( + AnalyticsAcceleratorUtil.streamFactoryCache() + .getIfPresent(Pair.of(clients.get(i), properties))) + .as("Factory of open FileIO %s", i) + .isSameAs(factories.get(i)); + } + } +}