Skip to content

Commit dccf25e

Browse files
gripleafgripleaf
committed
refactor(scan): require explicit arguments and place memory pools last
1 parent 54301cb commit dccf25e

20 files changed

Lines changed: 122 additions & 97 deletions

‎docs/source/api/scan.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ branch. Create it once with a file system and pass it to each ``ScanContextBuild
5757
5858
PAIMON_ASSIGN_OR_RAISE(
5959
std::shared_ptr<paimon::TableScanResources> resources,
60-
paimon::TableScanResources::Create(table_path, file_system));
60+
paimon::TableScanResources::Create(table_path, file_system, "main", paimon::GetDefaultPool()));
6161
6262
// Repeat for each query, reusing resources.
6363
paimon::ScanContextBuilder builder(table_path);

‎include/paimon/scan_context.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,14 @@ class PAIMON_EXPORT ScanContext {
5151
const std::shared_ptr<ScanFilter>& scan_filter,
5252
const std::shared_ptr<GlobalIndexResult>& global_index_result,
5353
const std::shared_ptr<RealtimeContext>& realtime_context,
54-
const std::shared_ptr<MemoryPool>& memory_pool,
5554
const std::shared_ptr<Executor>& executor,
5655
const std::shared_ptr<FileSystem>& specific_file_system,
5756
const std::optional<std::string>& table_schema,
5857
const std::map<std::string, std::string>& options,
5958
const std::shared_ptr<Cache>& cache,
6059
const std::shared_ptr<FormatTable>& format_table,
61-
const std::shared_ptr<TableScanResources>& table_resources = nullptr);
60+
const std::shared_ptr<TableScanResources>& table_resources,
61+
const std::shared_ptr<MemoryPool>& memory_pool);
6262

6363
~ScanContext();
6464

‎include/paimon/table/source/table_scan_resources.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,7 @@ class PAIMON_EXPORT TableScanResources {
4848
/// @param memory_pool Non-null pool retained for shared metadata, independent of scan pools.
4949
static Result<std::shared_ptr<TableScanResources>> Create(
5050
const std::string& table_path, const std::shared_ptr<FileSystem>& file_system,
51-
const std::string& branch = "main",
52-
const std::shared_ptr<MemoryPool>& memory_pool = GetDefaultPool());
51+
const std::string& branch, const std::shared_ptr<MemoryPool>& memory_pool);
5352

5453
~TableScanResources();
5554

‎src/paimon/core/append/append_compact_coordinator.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ Result<std::unique_ptr<FileStoreScan>> CreateFileStoreScan(
149149
std::unique_ptr<AppendOnlyFileStoreScan> scan,
150150
AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list,
151151
manifest_file, table_schema, arrow_schema, scan_filter,
152-
core_options, executor, pool));
152+
core_options, executor, nullptr, pool));
153153
return std::unique_ptr<FileStoreScan>(std::move(scan));
154154
}
155155

‎src/paimon/core/operation/append_only_file_store_scan.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,12 @@ Result<std::unique_ptr<AppendOnlyFileStoreScan>> AppendOnlyFileStoreScan::Create
6262
const std::shared_ptr<TableSchema>& table_schema,
6363
const std::shared_ptr<arrow::Schema>& arrow_schema,
6464
const std::shared_ptr<ScanFilter>& scan_filters, const CoreOptions& core_options,
65-
const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool,
66-
const std::shared_ptr<SimpleStatsEvolutions>& evolutions) {
65+
const std::shared_ptr<Executor>& executor,
66+
const std::shared_ptr<SimpleStatsEvolutions>& evolutions,
67+
const std::shared_ptr<MemoryPool>& pool) {
6768
auto scan = std::unique_ptr<AppendOnlyFileStoreScan>(new AppendOnlyFileStoreScan(
6869
snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema,
69-
core_options, executor, pool, evolutions));
70+
core_options, executor, evolutions, pool));
7071
PAIMON_RETURN_NOT_OK(
7172
scan->SplitAndSetFilter(table_schema->PartitionKeys(), arrow_schema, scan_filters));
7273
return scan;
@@ -79,8 +80,8 @@ AppendOnlyFileStoreScan::AppendOnlyFileStoreScan(
7980
const std::shared_ptr<ManifestFile>& manifest_file,
8081
const std::shared_ptr<TableSchema>& table_schema, const std::shared_ptr<arrow::Schema>& schema,
8182
const CoreOptions& core_options, const std::shared_ptr<Executor>& executor,
82-
const std::shared_ptr<MemoryPool>& pool,
83-
const std::shared_ptr<SimpleStatsEvolutions>& evolutions)
83+
const std::shared_ptr<SimpleStatsEvolutions>& evolutions,
84+
const std::shared_ptr<MemoryPool>& pool)
8485
: FileStoreScan(snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
8586
schema, core_options, executor, pool) {
8687
evolutions_ =

‎src/paimon/core/operation/append_only_file_store_scan.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,9 @@ class AppendOnlyFileStoreScan : public FileStoreScan {
5858
const std::shared_ptr<TableSchema>& table_schema,
5959
const std::shared_ptr<arrow::Schema>& arrow_schema,
6060
const std::shared_ptr<ScanFilter>& scan_filters, const CoreOptions& core_options,
61-
const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool,
62-
const std::shared_ptr<SimpleStatsEvolutions>& evolutions = nullptr);
61+
const std::shared_ptr<Executor>& executor,
62+
const std::shared_ptr<SimpleStatsEvolutions>& evolutions,
63+
const std::shared_ptr<MemoryPool>& pool);
6364

6465
/// @note Keep this thread-safe.
6566
Result<bool> FilterByStats(const ManifestEntry& entry) const override;
@@ -77,8 +78,8 @@ class AppendOnlyFileStoreScan : public FileStoreScan {
7778
const std::shared_ptr<arrow::Schema>& schema,
7879
const CoreOptions& core_options,
7980
const std::shared_ptr<Executor>& executor,
80-
const std::shared_ptr<MemoryPool>& pool,
81-
const std::shared_ptr<SimpleStatsEvolutions>& evolutions);
81+
const std::shared_ptr<SimpleStatsEvolutions>& evolutions,
82+
const std::shared_ptr<MemoryPool>& pool);
8283

8384
private:
8485
std::shared_ptr<SimpleStatsEvolutions> evolutions_;

‎src/paimon/core/operation/append_only_file_store_scan_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ class AppendBucketPruningTest : public testing::Test {
7474
predicate, std::vector<std::map<std::string, std::string>>(), bucket);
7575
return AppendOnlyFileStoreScan::Create(nullptr, schema_manager_, nullptr, nullptr, schema,
7676
arrow_schema, filters, options,
77-
CreateDefaultExecutor(), pool_);
77+
CreateDefaultExecutor(), nullptr, pool_);
7878
}
7979

8080
void CheckBuckets(const std::shared_ptr<Predicate>& predicate,

‎src/paimon/core/operation/append_only_file_store_write.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,10 +151,11 @@ Result<std::unique_ptr<FileStoreScan>> AppendOnlyFileStoreWrite::CreateFileStore
151151
options_.GetManifestCompression(), file_store_path_factory_,
152152
options_.GetManifestTargetFileSize(), pool_, options_,
153153
partition_schema_));
154-
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreScan> scan,
155-
AppendOnlyFileStoreScan::Create(
156-
snapshot_manager_, schema_manager_, manifest_list, manifest_file,
157-
table_schema_, schema_, scan_filter, options_, executor_, pool_));
154+
PAIMON_ASSIGN_OR_RAISE(
155+
std::unique_ptr<FileStoreScan> scan,
156+
AppendOnlyFileStoreScan::Create(snapshot_manager_, schema_manager_, manifest_list,
157+
manifest_file, table_schema_, schema_, scan_filter,
158+
options_, executor_, nullptr, pool_));
158159
return scan;
159160
}
160161

‎src/paimon/core/operation/file_store_commit.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ CommitScanner::ScanSupplier CreateAppendScanSupplier(
7272
std::unique_ptr<AppendOnlyFileStoreScan> scan,
7373
AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list,
7474
manifest_file, table_schema, arrow_schema, scan_filter,
75-
options, executor, pool));
75+
options, executor, nullptr, pool));
7676
return std::unique_ptr<FileStoreScan>(std::move(scan));
7777
};
7878
}
@@ -92,7 +92,7 @@ CommitScanner::ScanSupplier CreatePkScanSupplier(
9292
std::unique_ptr<KeyValueFileStoreScan> scan,
9393
KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list,
9494
manifest_file, table_schema, arrow_schema, scan_filter,
95-
options, executor, pool));
95+
options, executor, nullptr, pool));
9696
return std::unique_ptr<FileStoreScan>(std::move(scan));
9797
};
9898
}

‎src/paimon/core/operation/key_value_file_store_scan.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,12 @@ Result<std::unique_ptr<KeyValueFileStoreScan>> KeyValueFileStoreScan::Create(
6262
const std::shared_ptr<TableSchema>& table_schema,
6363
const std::shared_ptr<arrow::Schema>& arrow_schema,
6464
const std::shared_ptr<ScanFilter>& scan_filters, const CoreOptions& core_options,
65-
const std::shared_ptr<Executor>& executor, const std::shared_ptr<MemoryPool>& pool,
66-
const std::shared_ptr<SimpleStatsEvolutions>& evolutions) {
65+
const std::shared_ptr<Executor>& executor,
66+
const std::shared_ptr<SimpleStatsEvolutions>& evolutions,
67+
const std::shared_ptr<MemoryPool>& pool) {
6768
auto scan = std::unique_ptr<KeyValueFileStoreScan>(new KeyValueFileStoreScan(
6869
snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema,
69-
core_options, executor, pool, evolutions));
70+
core_options, executor, evolutions, pool));
7071
PAIMON_RETURN_NOT_OK(
7172
scan->SplitAndSetFilter(table_schema->PartitionKeys(), arrow_schema, scan_filters));
7273
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> trimmed_pk, table_schema->TrimmedPrimaryKeys());
@@ -271,8 +272,8 @@ KeyValueFileStoreScan::KeyValueFileStoreScan(
271272
const std::shared_ptr<ManifestFile>& manifest_file,
272273
const std::shared_ptr<TableSchema>& table_schema, const std::shared_ptr<arrow::Schema>& schema,
273274
const CoreOptions& core_options, const std::shared_ptr<Executor>& executor,
274-
const std::shared_ptr<MemoryPool>& pool,
275-
const std::shared_ptr<SimpleStatsEvolutions>& evolutions)
275+
const std::shared_ptr<SimpleStatsEvolutions>& evolutions,
276+
const std::shared_ptr<MemoryPool>& pool)
276277
: FileStoreScan(snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
277278
schema, core_options, executor, pool) {
278279
evolutions_ =

0 commit comments

Comments
 (0)