Skip to content

Commit f3e1e3f

Browse files
mrdrivingduckcodex
andcommitted
feat(fs): add OSS file system
Add a read-only OSS file system backed by OSS SDK v2, including configurable access, bundled build integration, documentation, and tests. Co-authored-by: GPT-5.6 Terra <codex@users.noreply.github.com>
1 parent 05d6497 commit f3e1e3f

20 files changed

Lines changed: 1138 additions & 17 deletions

‎CMakeLists.txt‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
6161
option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
6262
option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
6363
option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
64+
option(PAIMON_ENABLE_OSS "Whether to enable OSS SDK v2 file system" OFF)
6465
option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
6566
option(PAIMON_ENABLE_NETWORK_TESTS
6667
"Whether to enable tests that access real remote services over the network" OFF)
@@ -75,8 +76,10 @@ endif()
7576
if(PAIMON_ENABLE_REST)
7677
add_definitions(-DPAIMON_ENABLE_REST)
7778
endif()
78-
# libcurl backs the HTTP client shared by the S3 file system and the rest catalog.
79-
if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST)
79+
# libcurl supports the REST and S3 HTTP clients, the OSS SDK transport, and object store timestamps.
80+
if(PAIMON_ENABLE_OSS
81+
OR PAIMON_ENABLE_S3
82+
OR PAIMON_ENABLE_REST)
8083
find_package(CURL REQUIRED)
8184
endif()
8285
if(PAIMON_ENABLE_AVRO)
@@ -85,6 +88,9 @@ endif()
8588
if(PAIMON_ENABLE_JINDO)
8689
add_definitions(-DPAIMON_ENABLE_JINDO)
8790
endif()
91+
if(PAIMON_ENABLE_OSS)
92+
add_definitions(-DPAIMON_ENABLE_OSS)
93+
endif()
8894
if(PAIMON_ENABLE_S3)
8995
add_definitions(-DPAIMON_ENABLE_S3)
9096
endif()
@@ -452,6 +458,13 @@ if(PAIMON_BUILD_TESTS)
452458
paimon_jindo_file_system_shared)
453459
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
454460
endif()
461+
if(PAIMON_ENABLE_OSS)
462+
paimon_link_libraries_whole_archive(PAIMON_OSS_FILE_SYSTEM_STATIC_LINK_LIBS
463+
paimon_oss_file_system_static)
464+
paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
465+
paimon_oss_file_system_shared)
466+
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
467+
endif()
455468
if(PAIMON_ENABLE_S3)
456469
paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
457470
paimon_s3_file_system_static)
@@ -511,6 +524,7 @@ add_subdirectory(src/paimon/fs/local)
511524
if(PAIMON_ENABLE_JINDO)
512525
add_subdirectory(src/paimon/fs/jindo)
513526
endif()
527+
add_subdirectory(src/paimon/fs/oss)
514528
add_subdirectory(src/paimon/fs/s3)
515529
add_subdirectory(src/paimon/format/blob)
516530
add_subdirectory(src/paimon/format/orc)

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ Paimon C++ currently provides:
3535
- **Scan**: batch and stream scan for append tables and primary key tables without changelog.
3636
- **Read**: append table read, primary key table read with deletion vector, and primary key table merge-on-read.
3737
- **Arrow integration**: batch read and write interfaces based on the [Arrow Columnar In-Memory Format](https://arrow.apache.org).
38-
- **File systems**: file system abstraction with built-in local and Jindo file system support.
38+
- **File systems**: file system abstraction with built-in local, Jindo, OSS, and S3 file system support.
3939
- **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support.
4040
- **Runtime utilities**: memory pool and thread pool abstractions with default implementations.
4141
- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index

‎ci/scripts/build_paimon.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ CMAKE_ARGS=(
148148
"-DCMAKE_BUILD_TYPE=${build_type}"
149149
"-DPAIMON_BUILD_TESTS=ON"
150150
"-DPAIMON_ENABLE_JINDO=ON"
151+
"-DPAIMON_ENABLE_OSS=ON"
151152
"-DPAIMON_ENABLE_S3=ON"
152153
"-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
153154
"-DPAIMON_ENABLE_LUCENE=ON"

‎cmake_modules/DefineOptions.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,11 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
219219
""
220220
AUTO
221221
BUNDLED)
222+
define_option_string(OSS_SDK_V2_SOURCE
223+
"Dependency source for OSS SDK v2; SYSTEM is unsupported"
224+
""
225+
AUTO
226+
BUNDLED)
222227
define_option_string(fmt_SOURCE
223228
"Dependency source for fmt"
224229
""

‎cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,16 @@ else()
308308
endif()
309309
endif()
310310

311+
if(DEFINED ENV{PAIMON_OSS_SDK_V2_URL})
312+
set(OSS_SDK_V2_SOURCE_URL "$ENV{PAIMON_OSS_SDK_V2_URL}")
313+
elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_OSS_SDK_V2_PKG_NAME}")
314+
set_urls(OSS_SDK_V2_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_OSS_SDK_V2_PKG_NAME}")
315+
else()
316+
set_urls(OSS_SDK_V2_SOURCE_URL
317+
"${THIRDPARTY_MIRROR_URL}https://github.com/aliyun/alibabacloud-oss-cpp-sdk-v2/archive/refs/tags/${PAIMON_OSS_SDK_V2_BUILD_VERSION}.tar.gz"
318+
)
319+
endif()
320+
311321
if(DEFINED ENV{PAIMON_LUMINA_URL})
312322
set(LUMINA_SOURCE_URL "$ENV{PAIMON_LUMINA_URL}")
313323
elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LUMINA_PKG_NAME}")
@@ -492,6 +502,25 @@ function(paimon_enforce_patched_dependency_policy)
492502
PARENT_SCOPE)
493503
endif()
494504
endif()
505+
506+
if(PAIMON_ENABLE_OSS)
507+
paimon_set_dependency_source_default(
508+
OSS_SDK_V2 BUNDLED "OSS SDK v2 is only supported as a bundled dependency")
509+
paimon_get_dependency_source(OSS_SDK_V2 _oss_sdk_v2_source)
510+
if(_oss_sdk_v2_source STREQUAL "SYSTEM")
511+
message(FATAL_ERROR "OSS_SDK_V2_SOURCE=SYSTEM is not supported. "
512+
"Use OSS_SDK_V2_SOURCE=BUNDLED.")
513+
elseif(_oss_sdk_v2_source STREQUAL "AUTO")
514+
message(STATUS "Forcing OSS_SDK_V2_SOURCE to BUNDLED because paimon-cpp "
515+
"only supports the bundled OSS SDK v2")
516+
set(OSS_SDK_V2_SOURCE
517+
"BUNDLED"
518+
CACHE STRING "Dependency source for OSS SDK v2" FORCE)
519+
set(OSS_SDK_V2_SOURCE
520+
"BUNDLED"
521+
PARENT_SCOPE)
522+
endif()
523+
endif()
495524
endfunction()
496525

497526
function(paimon_apply_dependency_source_defaults)
@@ -597,6 +626,8 @@ function(paimon_get_dependency_compat_target DEPENDENCY_NAME OUT_VAR)
597626
set(_target tbb)
598627
elseif("${DEPENDENCY_NAME}" STREQUAL "Avro")
599628
set(_target avro)
629+
elseif("${DEPENDENCY_NAME}" STREQUAL "OSS_SDK_V2")
630+
set(_target alibabacloud_oss_v2::oss)
600631
else()
601632
set(_target "${DEPENDENCY_NAME}")
602633
endif()
@@ -671,6 +702,8 @@ macro(paimon_build_dependency DEPENDENCY_NAME)
671702
build_glog()
672703
elseif("${DEPENDENCY_NAME}" STREQUAL "Avro")
673704
build_avro()
705+
elseif("${DEPENDENCY_NAME}" STREQUAL "OSS_SDK_V2")
706+
build_oss_sdk_v2()
674707
elseif("${DEPENDENCY_NAME}" STREQUAL "GTest")
675708
build_gtest()
676709
elseif("${DEPENDENCY_NAME}" STREQUAL "benchmark")
@@ -1359,6 +1392,89 @@ macro(build_jindosdk_nextarch)
13591392
add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep)
13601393
endmacro()
13611394

1395+
macro(build_oss_sdk_v2)
1396+
message(STATUS "Building Alibaba Cloud OSS C++ SDK v2 from source")
1397+
find_package(CURL REQUIRED)
1398+
find_package(Threads REQUIRED)
1399+
1400+
set(OSS_SDK_V2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/oss_sdk_v2_ep-install")
1401+
set(OSS_SDK_V2_INCLUDE_DIR "${OSS_SDK_V2_PREFIX}/include")
1402+
set(OSS_SDK_V2_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
1403+
set(OSS_SDK_V2_LIB_DIR "${OSS_SDK_V2_PREFIX}/${OSS_SDK_V2_INSTALL_LIBDIR}")
1404+
set(OSS_SDK_V2_STATIC_LIB
1405+
"${OSS_SDK_V2_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}alibabacloud-oss-cpp-sdk-v2${CMAKE_STATIC_LIBRARY_SUFFIX}"
1406+
)
1407+
1408+
set(OSS_SDK_V2_CMAKE_ARGS
1409+
${EP_COMMON_CMAKE_ARGS}
1410+
"-DCMAKE_INSTALL_PREFIX=${OSS_SDK_V2_PREFIX}"
1411+
"-DCMAKE_INSTALL_LIBDIR=${OSS_SDK_V2_INSTALL_LIBDIR}"
1412+
-DCMAKE_PARENT_CXX_STANDARD=17
1413+
-DBUILD_SHARED_LIBS=OFF
1414+
-DBUILD_TESTS=OFF
1415+
-DBUILD_SAMPLES=OFF
1416+
-DENABLE_RTTI=OFF
1417+
-DUSE_CURL_TRANSPORT=ON
1418+
-DUSE_SYSTEM_CURL=ON
1419+
-DUSE_SYSTEM_OPENSSL=OFF
1420+
-DUSE_SYSTEM_MBEDTLS=OFF
1421+
-DUSE_SYSTEM_TINYXML2=OFF
1422+
-DUSE_STD_EXPECTED=OFF
1423+
-DENABLE_ENCRYPTION=OFF)
1424+
set(OSS_SDK_V2_CURL_INCLUDE_DIR "${CURL_INCLUDE_DIR}")
1425+
if(NOT OSS_SDK_V2_CURL_INCLUDE_DIR AND CURL_INCLUDE_DIRS)
1426+
list(GET CURL_INCLUDE_DIRS 0 OSS_SDK_V2_CURL_INCLUDE_DIR)
1427+
endif()
1428+
set(OSS_SDK_V2_CURL_LIBRARY "${CURL_LIBRARY_RELEASE}")
1429+
if(NOT OSS_SDK_V2_CURL_LIBRARY)
1430+
set(OSS_SDK_V2_CURL_LIBRARY "${CURL_LIBRARY}")
1431+
endif()
1432+
if(TARGET CURL::libcurl)
1433+
if(NOT OSS_SDK_V2_CURL_INCLUDE_DIR)
1434+
get_target_property(OSS_SDK_V2_CURL_INCLUDE_DIR CURL::libcurl
1435+
INTERFACE_INCLUDE_DIRECTORIES)
1436+
endif()
1437+
if(NOT OSS_SDK_V2_CURL_LIBRARY)
1438+
foreach(CURL_CONFIG RELEASE RELWITHDEBINFO DEBUG NOCONFIG)
1439+
get_target_property(OSS_SDK_V2_CURL_LIBRARY CURL::libcurl
1440+
"IMPORTED_LOCATION_${CURL_CONFIG}")
1441+
if(OSS_SDK_V2_CURL_LIBRARY)
1442+
break()
1443+
endif()
1444+
endforeach()
1445+
endif()
1446+
if(NOT OSS_SDK_V2_CURL_LIBRARY)
1447+
get_target_property(OSS_SDK_V2_CURL_LIBRARY CURL::libcurl IMPORTED_LOCATION)
1448+
endif()
1449+
endif()
1450+
if(OSS_SDK_V2_CURL_INCLUDE_DIR AND OSS_SDK_V2_CURL_LIBRARY)
1451+
list(APPEND
1452+
OSS_SDK_V2_CMAKE_ARGS
1453+
"-DCURL_INCLUDE_DIR=${OSS_SDK_V2_CURL_INCLUDE_DIR}"
1454+
"-DCURL_LIBRARY=${OSS_SDK_V2_CURL_LIBRARY}"
1455+
"-DCURL_LIBRARY_RELEASE=${OSS_SDK_V2_CURL_LIBRARY}")
1456+
endif()
1457+
1458+
externalproject_add(oss_sdk_v2_ep
1459+
${EP_COMMON_OPTIONS}
1460+
URL ${OSS_SDK_V2_SOURCE_URL}
1461+
URL_HASH "SHA256=${PAIMON_OSS_SDK_V2_BUILD_SHA256_CHECKSUM}"
1462+
CMAKE_ARGS ${OSS_SDK_V2_CMAKE_ARGS} ${THIRDPARTY_LOG_OPTIONS}
1463+
BUILD_BYPRODUCTS "${OSS_SDK_V2_STATIC_LIB}")
1464+
1465+
file(MAKE_DIRECTORY "${OSS_SDK_V2_INCLUDE_DIR}")
1466+
file(MAKE_DIRECTORY "${OSS_SDK_V2_LIB_DIR}")
1467+
1468+
add_library(alibabacloud_oss_v2::oss STATIC IMPORTED)
1469+
set_target_properties(alibabacloud_oss_v2::oss
1470+
PROPERTIES IMPORTED_LOCATION "${OSS_SDK_V2_STATIC_LIB}"
1471+
INTERFACE_INCLUDE_DIRECTORIES
1472+
"${OSS_SDK_V2_INCLUDE_DIR}")
1473+
target_link_libraries(alibabacloud_oss_v2::oss INTERFACE CURL::libcurl
1474+
Threads::Threads)
1475+
add_dependencies(alibabacloud_oss_v2::oss oss_sdk_v2_ep)
1476+
endmacro()
1477+
13621478
macro(build_protobuf)
13631479
message(STATUS "Building protobuf from source")
13641480
set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install")
@@ -1967,6 +2083,9 @@ if(PAIMON_ENABLE_JINDO)
19672083
build_jindosdk_c()
19682084
build_jindosdk_nextarch()
19692085
endif()
2086+
if(PAIMON_ENABLE_OSS)
2087+
resolve_dependency(OSS_SDK_V2)
2088+
endif()
19702089
if(PAIMON_ENABLE_S3)
19712090
include(BuildAwsAuth)
19722091
build_aws_auth()

‎docs/source/build_system.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
106106

107107
- ``Paimon::paimon_local_file_system_shared``
108108
- ``Paimon::paimon_jindo_file_system_shared``
109+
- ``Paimon::paimon_oss_file_system_shared``
110+
- ``Paimon::paimon_s3_file_system_shared``
109111

110112
- Index plugins:
111113

‎docs/source/building.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ boolean flags to ``cmake``.
178178
* ``-DPAIMON_ENABLE_ORC=ON``: Paimon integration with Apache ORC
179179
* ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration
180180
* ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems
181+
* ``-DPAIMON_ENABLE_OSS=ON``: Support for Alibaba Cloud OSS through OSS SDK V2
182+
* ``-DPAIMON_ENABLE_S3=ON``: Support for Amazon S3-compatible filesystems
181183
* ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. Requires
182184
Linux ``x86_64``; see :ref:`cpp-building-platforms`.
183185
* ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes

‎src/paimon/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST)
196196
list(APPEND PAIMON_COMMON_SRCS common/utils/http_client.cpp)
197197
set(PAIMON_CURL_LINK_LIBS CURL::libcurl)
198198
endif()
199-
if(PAIMON_ENABLE_S3)
199+
if(PAIMON_ENABLE_OSS OR PAIMON_ENABLE_S3)
200200
list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp)
201201
endif()
202202

@@ -937,7 +937,7 @@ if(PAIMON_BUILD_TESTS)
937937
endif()
938938

939939
set(PAIMON_OBJECT_STORE_FS_TEST_SOURCES)
940-
if(PAIMON_ENABLE_S3)
940+
if(PAIMON_ENABLE_OSS OR PAIMON_ENABLE_S3)
941941
list(APPEND PAIMON_OBJECT_STORE_FS_TEST_SOURCES
942942
common/fs/object_store_file_system_test.cpp)
943943
endif()

‎src/paimon/common/fs/object_store_file_system.cpp‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,14 @@
1919

2020
#include "paimon/common/fs/object_store_file_system.h"
2121

22+
#include <curl/curl.h>
23+
2224
#include <algorithm>
2325
#include <cassert>
26+
#include <cctype>
27+
#include <cerrno>
2428
#include <cstring>
29+
#include <ctime>
2530
#include <limits>
2631
#include <utility>
2732

@@ -30,6 +35,69 @@
3035
#include "paimon/common/utils/path_util.h"
3136

3237
namespace paimon {
38+
39+
int64_t ObjectStoreFileSystemUtils::ParseModificationTime(const std::string& value) {
40+
std::tm parsed_time{};
41+
const char* current = strptime(value.c_str(), "%Y-%m-%dT%H:%M:%S", &parsed_time);
42+
if (current != nullptr) {
43+
int32_t milliseconds = 0;
44+
int32_t fraction_digits = 0;
45+
if (*current == '.') {
46+
++current;
47+
const char* fraction_begin = current;
48+
while (std::isdigit(static_cast<unsigned char>(*current))) {
49+
if (fraction_digits < 3) {
50+
milliseconds = milliseconds * 10 + (*current - '0');
51+
}
52+
++fraction_digits;
53+
++current;
54+
}
55+
if (current == fraction_begin) {
56+
current = nullptr;
57+
}
58+
while (fraction_digits < 3) {
59+
milliseconds *= 10;
60+
++fraction_digits;
61+
}
62+
}
63+
64+
int32_t timezone_offset_seconds = 0;
65+
bool valid_timezone = false;
66+
if (current != nullptr && *current == 'Z' && current[1] == '\0') {
67+
valid_timezone = true;
68+
} else if (current != nullptr && (*current == '+' || *current == '-') &&
69+
std::isdigit(static_cast<unsigned char>(current[1])) &&
70+
std::isdigit(static_cast<unsigned char>(current[2])) && current[3] == ':' &&
71+
std::isdigit(static_cast<unsigned char>(current[4])) &&
72+
std::isdigit(static_cast<unsigned char>(current[5])) && current[6] == '\0') {
73+
int32_t hours = (current[1] - '0') * 10 + current[2] - '0';
74+
int32_t minutes = (current[4] - '0') * 10 + current[5] - '0';
75+
if (hours <= 23 && minutes <= 59) {
76+
timezone_offset_seconds = (hours * 60 + minutes) * 60;
77+
if (*current == '-') {
78+
timezone_offset_seconds = -timezone_offset_seconds;
79+
}
80+
valid_timezone = true;
81+
}
82+
}
83+
84+
if (valid_timezone) {
85+
errno = 0;
86+
time_t seconds = timegm(&parsed_time);
87+
if (seconds != static_cast<time_t>(-1) || errno != EOVERFLOW) {
88+
int64_t utc_seconds = static_cast<int64_t>(seconds) - timezone_offset_seconds;
89+
return utc_seconds * 1000 + milliseconds;
90+
}
91+
}
92+
}
93+
94+
time_t seconds = curl_getdate(value.c_str(), nullptr);
95+
if (seconds == static_cast<time_t>(-1)) {
96+
return FileStatus::kUnknownModificationTime;
97+
}
98+
return static_cast<int64_t>(seconds) * 1000;
99+
}
100+
33101
namespace {
34102

35103
constexpr int64_t kMaxReadAheadMemory = 64LL * 1024LL * 1024LL;

‎src/paimon/common/fs/object_store_file_system.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@
3131

3232
namespace paimon {
3333

34+
class ObjectStoreFileSystemUtils {
35+
public:
36+
static int64_t ParseModificationTime(const std::string& value);
37+
};
38+
3439
struct ObjectStorePath {
3540
std::string bucket;
3641
std::string key;
@@ -39,7 +44,7 @@ struct ObjectStorePath {
3944
struct ObjectMetadata {
4045
std::string key;
4146
int64_t size = 0;
42-
int64_t modification_time = 0;
47+
int64_t modification_time = FileStatus::kUnknownModificationTime;
4348
};
4449

4550
struct ListObjectsResult {

0 commit comments

Comments
 (0)