diff --git a/.github/workflows/IntegrationTests.yml b/.github/workflows/IntegrationTests.yml index 420905328..b39538c37 100644 --- a/.github/workflows/IntegrationTests.yml +++ b/.github/workflows/IntegrationTests.yml @@ -45,8 +45,8 @@ jobs: env: GEN: ninja - CC: 'ccache gcc' - CXX: 'ccache g++' + CC: '/usr/lib/ccache/gcc' + CXX: '/usr/lib/ccache/g++' CCACHE_DIR: ${{ github.workspace }}/ccache VCPKG_TARGET_TRIPLET: x64-linux-release VCPKG_HOST_TRIPLET: x64-linux-release @@ -85,7 +85,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Build extension run: | @@ -98,7 +98,7 @@ jobs: key: ${{ steps.cache_key.outputs.value }} - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: 17 username: 'postgres' @@ -114,6 +114,7 @@ jobs: PGUSER: postgres PGPASSWORD: postgres POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_CONNECTION_IS_DIRECT: 1 LOCAL_EXTENSION_REPO: 'build/release/repository' run: | source ./create-postgres-tables.sh @@ -128,15 +129,15 @@ jobs: strategy: matrix: postgres: - - name: pg-14 - version: 14 + - name: pg-15 + version: 15 - name: pg-18 version: 18 env: GEN: ninja - CC: 'ccache gcc' - CXX: 'ccache g++' + CC: '/usr/lib/ccache/gcc' + CXX: '/usr/lib/ccache/g++' CCACHE_DIR: ${{ github.workspace }}/ccache VCPKG_TARGET_TRIPLET: x64-linux-release VCPKG_HOST_TRIPLET: x64-linux-release @@ -176,14 +177,14 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Build extension run: | make release - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: ${{ matrix.postgres.version }} username: 'postgres' @@ -269,8 +270,8 @@ jobs: env: CMAKE_BUILD_PARALLEL_LEVEL: 2 - CC: 'ccache gcc' - CXX: 'ccache g++' + CC: '/usr/lib/ccache/gcc' + CXX: '/usr/lib/ccache/g++' CCACHE_DIR: ${{ github.workspace }}/ccache VCPKG_TARGET_TRIPLET: x64-linux-release VCPKG_HOST_TRIPLET: x64-linux-release @@ -310,7 +311,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Build extension env: @@ -328,7 +329,7 @@ jobs: key: ${{ steps.cache_key.outputs.value }} - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: 17 username: 'postgres' @@ -345,6 +346,8 @@ jobs: PGPASSWORD: postgres PGSSLMODE: require POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_CONNECTION_IS_DIRECT: 1 + SANITIZER_BUILD: 1 LOCAL_EXTENSION_REPO: 'build/relassert/repository' run: | source ./create-postgres-tables.sh @@ -420,7 +423,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + vcpkgGitCommitId: 9e593bb18ea69cc5095e012465dcd675a822ed0d # Release 2026.07.29 - name: Prepare build tree shell: bash @@ -442,7 +445,7 @@ jobs: key: ${{ steps.cache_key.outputs.value }} - name: Setup Postgres - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@v9 with: postgres-version: '17' username: 'postgres' @@ -470,6 +473,7 @@ jobs: PGPASSWORD: postgres PGSSLMODE: require POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_CONNECTION_IS_DIRECT: 1 LOCAL_EXTENSION_REPO: 'build/release/repository' shell: bash run: | diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 560257565..41d051b88 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,7 +14,6 @@ jobs: name: Build extension binaries uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: - duckdb_version: main ci_tools_version: main extension_name: postgres_scanner exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw' @@ -27,7 +26,6 @@ jobs: uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: - duckdb_version: main ci_tools_version: main extension_name: postgres_scanner exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw' diff --git a/.gitignore b/.gitignore index 76c2cc0a0..3c2d9f46c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ cmake-build-debug .vscode .cache duckdb_unittest_tempdir +.envrc +docker-compose.yml diff --git a/.gitmodules b/.gitmodules index ac125b1d2..12eb48a74 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "duckdb"] path = duckdb - url = https://github.com/duckdb/duckdb.git + url = https://github.com/altertable-ai/duckdb [submodule "extension-ci-tools"] path = extension-ci-tools url = https://github.com/duckdb/extension-ci-tools.git diff --git a/CMakeLists.txt b/CMakeLists.txt index b30ca04a7..101e1640d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,33 +2,22 @@ cmake_minimum_required(VERSION 2.8.12...3.29) set(TARGET_NAME postgres_scanner) project(${TARGET_NAME}) -add_definitions( - -DFRONTEND=1 - -D_GNU_SOURCE=1 - -DUSE_OPENSSL=1 - -DHAVE_BIO_GET_DATA=1 - -DHAVE_BIO_METH_NEW=1) +add_definitions(-DFRONTEND=1 -D_GNU_SOURCE=1 -DUSE_OPENSSL=1 + -DHAVE_BIO_GET_DATA=1 -DHAVE_BIO_METH_NEW=1) find_package(OpenSSL REQUIRED) find_package(PostgreSQL REQUIRED) if(NOT MSVC) - add_compile_options( - -Wno-pedantic - -Wno-sign-compare - -Wno-unused-variable) + add_compile_options(-Wno-pedantic -Wno-sign-compare -Wno-unused-variable) endif() -include_directories( - include - database-connector/src/include - ${OPENSSL_INCLUDE_DIR} - ${PostgreSQL_INCLUDE_DIRS}) +include_directories(include database-connector/src/include + ${OPENSSL_INCLUDE_DIR} ${PostgreSQL_INCLUDE_DIRS}) if(WIN32) - include_directories( - vcpkg_ports/libpq/include/port/win32 - vcpkg_ports/libpq/include/port/win32_msvc) + include_directories(vcpkg_ports/libpq/include/port/win32 + vcpkg_ports/libpq/include/port/win32_msvc) endif() add_subdirectory(src) @@ -40,35 +29,27 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) build_static_extension(${TARGET_NAME} ${ALL_OBJECT_FILES}) build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${ALL_OBJECT_FILES}) -# Fix for ELF symbol conflict when loaded inside PostgreSQL backend -# Use -Bsymbolic to force the extension to prefer its own symbols over global ones -# This fixes the pg_link_canary_is_frontend() check failure where calls from within -# this extension resolve to the backend's version (returns false) instead of ours (returns true) +# Fix for ELF symbol conflict when loaded inside PostgreSQL backend Use +# -Bsymbolic to force the extension to prefer its own symbols over global ones +# This fixes the pg_link_canary_is_frontend() check failure where calls from +# within this extension resolve to the backend's version (returns false) instead +# of ours (returns true) if(NOT WIN32 AND NOT APPLE) - target_link_options(${LOADABLE_EXTENSION_NAME} PRIVATE - "-Wl,-Bsymbolic" - ) + target_link_options(${LOADABLE_EXTENSION_NAME} PRIVATE "-Wl,-Bsymbolic") endif() -target_link_libraries(${LOADABLE_EXTENSION_NAME} - OpenSSL::SSL - OpenSSL::Crypto - PostgreSQL::PostgreSQL - ${CURL_LIBRARIES}) +target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto + PostgreSQL::PostgreSQL ${CURL_LIBRARIES}) set_property(TARGET ${EXTENSION_NAME} PROPERTY C_STANDARD 99) set_property(TARGET ${LOADABLE_EXTENSION_NAME} PROPERTY C_STANDARD 99) if(WIN32) - target_link_libraries(${LOADABLE_EXTENSION_NAME} - wsock32 - ws2_32 - wldap32 - secur32 - crypt32) + target_link_libraries(${LOADABLE_EXTENSION_NAME} wsock32 ws2_32 wldap32 + secur32 crypt32) endif() install( - TARGETS ${EXTENSION_NAME} - EXPORT "${DUCKDB_EXPORT_SET}" - LIBRARY DESTINATION "${INSTALL_LIB_DIR}" - ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") + TARGETS ${EXTENSION_NAME} + EXPORT "${DUCKDB_EXPORT_SET}" + LIBRARY DESTINATION "${INSTALL_LIB_DIR}" + ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") diff --git a/README.md b/README.md index e1b7f6be9..0b2ce24da 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,19 @@ D SELECT * FROM postgres_db.uuids; For more information on how to use the connector, refer to the [Postgres documentation on the website](https://duckdb.org/docs/extensions/postgres). +### Connection Pooling + +By default, the extension opens a new connection whenever none are idle. Behind a transaction-mode pooler (e.g. PgBouncer `pool_mode=transaction`), this can exceed the pooler's own pool size under load and hang. If that applies to you, cap and wait instead: + +```sql +SET pg_pool_acquire_mode = 'wait'; +SET pg_pool_max_connections = 16; -- ~80-90% of your pooler's pool size + +ATTACH 'host=pgbouncer-host port=6432 dbname=mydb' AS postgres_db (TYPE postgres); +``` + +Note that `SET` statements are global and only apply pre-`ATTACH`. To change pooling on an already-attached database, use `postgres_configure_pool(catalog_name=...)` instead. Leave `force` mode if you connect directly to Postgres or through a session-mode pooler. + ### AWS RDS IAM Authentication The extension supports AWS RDS IAM-based authentication, which allows you to connect to RDS/Aurora PostgreSQL instances using IAM database authentication instead of static passwords. This feature automatically generates temporary authentication tokens using the AWS SDK. diff --git a/create-postgres-tables.sh b/create-postgres-tables.sh index 441605476..695e95d98 100755 --- a/create-postgres-tables.sh +++ b/create-postgres-tables.sh @@ -24,13 +24,14 @@ CALL dbgen(sf=0.01, schema='tpch'); CALL dsdgen(sf=0.01, schema='tpcds'); EXPORT DATABASE '${ABS_DIR_PREFIX}/postgresscannertmp'; " | \ -$DUCKDB_PATH +$DUCKDB_PATH -unsigned dropdb --if-exists postgresscanner createdb postgresscanner psql -d postgresscanner < ${ABS_DIR_PREFIX}/postgresscannertmp/schema.sql -psql -d postgresscanner < ${ABS_DIR_PREFIX}/postgresscannertmp/load.sql +sed -E "s/,\s*allow_quoted_nulls\s+[^,)]+//g" ${ABS_DIR_PREFIX}/postgresscannertmp/load.sql > ${ABS_DIR_PREFIX}/postgresscannertmp/load_filtered.sql +psql -d postgresscanner < ${ABS_DIR_PREFIX}/postgresscannertmp/load_filtered.sql rm -rf ${ABS_DIR_PREFIX}/postgresscannertmp psql -d postgresscanner < test/all_pg_types.sql diff --git a/database-connector b/database-connector index 7777dac88..c305670b5 160000 --- a/database-connector +++ b/database-connector @@ -1 +1 @@ -Subproject commit 7777dac887a48769684900a627ecaaf6feb3f4c5 +Subproject commit c305670b5729cc6dbda8d395eeb63b4d1487c064 diff --git a/duckdb b/duckdb index f0af2e1b4..24f6388b5 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit f0af2e1b4462143158289d5bb8028d52ee1f3342 +Subproject commit 24f6388b5e7e1a95ea126b6625f52e00315d3030 diff --git a/extension-ci-tools b/extension-ci-tools index a00c74e02..969bc7627 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit a00c74e02aefbfa3fd31a4c74c4b6bc260af0de7 +Subproject commit 969bc76276acec1c0a3922ff500d2ec3a1a3b037 diff --git a/extension_config.cmake b/extension_config.cmake index 04f1fa899..b19867de3 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -3,7 +3,6 @@ # Extension from this repo duckdb_extension_load(postgres_scanner SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} - DONT_LINK ) duckdb_extension_load(tpch) diff --git a/scripts/win_resources.patch b/scripts/win_resources.patch index ca35ead6a..f2a65b72e 100644 --- a/scripts/win_resources.patch +++ b/scripts/win_resources.patch @@ -1,8 +1,8 @@ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index 24afa3db42..d3f04aed2d 100644 +index 9dd2ea45d2..a583df2d9a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt -@@ -130,7 +130,6 @@ if(WIN32 AND NOT MINGW) +@@ -132,7 +132,6 @@ if(WIN32 AND NOT MINGW) set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -D DUCKDB_DEV_ITERATION=\"${RC_DEV_ITERATION}\"") @@ -11,14 +11,14 @@ index 24afa3db42..d3f04aed2d 100644 install( diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt -index 0d26359008..04c5c7f01f 100644 +index 7c376be703..956a2df514 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt -@@ -77,7 +77,6 @@ if(WIN32 AND NOT MINGW) +@@ -74,7 +74,6 @@ if(WIN32 AND NOT MINGW) "${CMAKE_RC_FLAGS} -D DUCKDB_DEV_ITERATION=\"${RC_DEV_ITERATION}\"") set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -D DUCKDB_COPYRIGHT_YEAR=\"${DUCKDB_COPYRIGHT_YEAR}\"") - target_sources(shell PRIVATE rc/duckdb.rc) - endif() - - set_target_properties(shell PROPERTIES OUTPUT_NAME duckdb) + # Opt into the Windows Segment Heap to avoid the default NT heap's lock + # contention under multi-threaded allocation (see #24027). CMake merges + # .manifest sources into the linker manifest via mt.exe. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 63d27c0b2..51653ccb5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,20 +5,20 @@ add_subdirectory(storage) add_library( postgres_ext_library OBJECT - postgres_attach.cpp postgres_aws.cpp postgres_binary_copy.cpp postgres_binary_file_reader.cpp postgres_binary_parser.cpp postgres_binary_reader.cpp + postgres_connection_string.cpp postgres_connection.cpp postgres_copy_from.cpp postgres_copy_to.cpp - postgres_execute.cpp postgres_extension.cpp postgres_filter_pushdown.cpp postgres_hstore.cpp postgres_parameters.cpp + postgres_pin_connection.cpp postgres_query.cpp postgres_scanner.cpp postgres_secrets.cpp diff --git a/src/dbconnector/CMakeLists.txt b/src/dbconnector/CMakeLists.txt index 09ece0687..bbb3a8599 100644 --- a/src/dbconnector/CMakeLists.txt +++ b/src/dbconnector/CMakeLists.txt @@ -3,12 +3,13 @@ set(POSTGRES_DBCONNECTOR_PATH add_library( postgres_ext_dbconnector OBJECT + ${POSTGRES_DBCONNECTOR_PATH}/src/attached/attached_catalog.cpp + ${POSTGRES_DBCONNECTOR_PATH}/src/attached/attached_table.cpp + ${POSTGRES_DBCONNECTOR_PATH}/src/functions/configure_pool.cpp ${POSTGRES_DBCONNECTOR_PATH}/src/query/query_writer.cpp ${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_pushdown.cpp ${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_util.cpp - ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/aggregate_optimizer.cpp - ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/optimizer_util.cpp - ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/order_by_and_limit_optimizer.cpp) + ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/optimizer_util.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/include/postgres_aws.hpp b/src/include/postgres_aws.hpp index cd14a2bb1..28e03d8dd 100644 --- a/src/include/postgres_aws.hpp +++ b/src/include/postgres_aws.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include "duckdb/main/attached_database.hpp" #include #include "duckdb.hpp" diff --git a/src/include/postgres_binary_copy.hpp b/src/include/postgres_binary_copy.hpp index fe873fc1a..2e8fb9d53 100644 --- a/src/include/postgres_binary_copy.hpp +++ b/src/include/postgres_binary_copy.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_utils.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_result.hpp" #include "duckdb/function/copy_function.hpp" @@ -34,7 +35,7 @@ class PostgresBinaryCopyFunction : public CopyFunction { GlobalFunctionData &gstate); static unique_ptr PostgresBinaryReadBind(ClientContext &context, CopyFromFunctionBindInput &info, - vector &expected_names, + vector &expected_names, vector &expected_types); }; diff --git a/src/include/postgres_binary_file_reader.hpp b/src/include/postgres_binary_file_reader.hpp index de73ad268..1aff03604 100644 --- a/src/include/postgres_binary_file_reader.hpp +++ b/src/include/postgres_binary_file_reader.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_binary_parser.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/file_system.hpp" namespace duckdb { @@ -16,9 +17,13 @@ namespace duckdb { class PostgresBinaryFileReader { public: static constexpr idx_t DEFAULT_BUFFER_SIZE = 32 * 1024 * 1024; + //! Size of the binary COPY file header: the signature, followed by the flags field and the header + //! extension area length (4 bytes each). Must match the header consumed by PostgresBinaryParser::CheckHeader. + static constexpr idx_t COPY_FILE_HEADER_SIZE = PostgresConversion::COPY_HEADER_LENGTH + 8; PostgresBinaryFileReader(ClientContext &context, const string &file_path, vector types, - vector postgres_types, idx_t buffer_size = DEFAULT_BUFFER_SIZE); + vector postgres_types, PostgresTypeConfig type_config, + idx_t buffer_size = DEFAULT_BUFFER_SIZE); bool ReadChunk(DataChunk &output); @@ -37,6 +42,7 @@ class PostgresBinaryFileReader { idx_t leftover; idx_t leftover_offset; bool finished; + bool header_scanned; }; } // namespace duckdb diff --git a/src/include/postgres_binary_parser.hpp b/src/include/postgres_binary_parser.hpp index 0a1e386b6..e477bd44f 100644 --- a/src/include/postgres_binary_parser.hpp +++ b/src/include/postgres_binary_parser.hpp @@ -9,6 +9,8 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/common/exception/conversion_exception.hpp" +#include "duckdb/common/operator/add.hpp" #include "duckdb/common/types/interval.hpp" #include "postgres_conversion.hpp" #include "postgres_utils.hpp" @@ -17,11 +19,18 @@ namespace duckdb { class PostgresBinaryParser { public: - PostgresBinaryParser(vector types, vector postgres_types); + PostgresBinaryParser(vector types, vector postgres_types, + PostgresTypeConfig type_config); void SetBuffer(data_ptr_t buf, idx_t len); + //! Decode rows from the buffer into output until it is full or the buffer runs out; + //! returns true if output is full. Only the chunk's count is updated, not its vectors' + //! sizes: the caller must call output.SetChildCardinality(output.size()) once it is + //! done filling the chunk. bool ReadChunk(DataChunk &output, const vector &column_ids); void CheckHeader(); + void ParseValue(const LogicalType &type, const PostgresType &postgres_type, int32_t value_len, Vector &out_vec, + idx_t output_offset); private: bool Ready() { @@ -38,6 +47,7 @@ class PostgresBinaryParser { vector types; vector postgres_types; + PostgresTypeConfig type_config; private: template @@ -58,10 +68,13 @@ class PostgresBinaryParser { return val; } + //! Defined out of line so the bounds checks that call it stay small enough to inline + [[noreturn]] static void ThrowOutOfBuffer(const char *function_name); + template inline T ReadInteger() { if (buffer_ptr + sizeof(T) > end) { - throw IOException("Postgres binary reader - out of buffer in ReadInteger"); + ThrowOutOfBuffer("ReadInteger"); } return ReadIntegerUnchecked(); } @@ -110,7 +123,16 @@ class PostgresBinaryParser { if (usec == POSTGRES_NINFINITY) { return timestamp_t::ninfinity(); } - return timestamp_t(usec + (POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS)); + int64_t timestamp_usec; + if (!TryAddOperator::Operation(static_cast(usec), + static_cast(POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS), timestamp_usec)) { + throw ConversionException("timestamp out of range"); + } + auto timestamp = timestamp_t(timestamp_usec); + if (!timestamp.IsFinite()) { + throw ConversionException("timestamp out of range"); + } + return timestamp; } inline interval_t ReadInterval() { @@ -131,7 +153,7 @@ class PostgresBinaryParser { const char *ReadString(idx_t string_length) { if (buffer_ptr + string_length > end) { - throw IOException("Postgres binary reader - out of buffer in ReadString"); + ThrowOutOfBuffer("ReadString"); } auto result = const_char_ptr_cast(buffer_ptr); buffer_ptr += string_length; @@ -139,15 +161,27 @@ class PostgresBinaryParser { } PostgresDecimalConfig ReadDecimalConfig(); + PostgresDecimal ReadDecimalAsDouble(); + + static PostgresDecimalKind NonFiniteDecimalKindFromSign(uint16_t dec_sign); + + static string NonFiniteDecimalKindToString(PostgresDecimalKind kind); template - T ReadDecimal() { + PostgresDecimal ReadDecimal() { // this is wild auto config = ReadDecimalConfig(); + + // we do not support non-finite numerics, need to return NULL or throw + if (config.sign == NUMERIC_NAN || config.sign == NUMERIC_PINF || config.sign == NUMERIC_NINF) { + PostgresDecimalKind kind = NonFiniteDecimalKindFromSign(config.sign); + return PostgresDecimal(kind); + } + auto scale_POWER = OP::GetPowerOfTen(config.scale); if (config.ndigits == 0) { - return 0; + return PostgresDecimal(static_cast(0)); } T integral_part = 0, fractional_part = 0; @@ -199,7 +233,8 @@ class PostgresBinaryParser { // finally auto base_res = OP::Finalize(config, integral_part + fractional_part); - return (config.is_negative ? -base_res : base_res); + auto val = (config.is_negative ? -base_res : base_res); + return PostgresDecimal(val); } void ReadGeometry(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset); @@ -208,6 +243,8 @@ class PostgresBinaryParser { uint32_t current_count, uint32_t dimensions[], uint32_t ndim); void ReadValue(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset); + + bool CheckDecimalKindSetNull(Vector &out_vec, idx_t output_offset, PostgresDecimalKind dec_kind); }; } // namespace duckdb diff --git a/src/include/postgres_binary_reader.hpp b/src/include/postgres_binary_reader.hpp index 26f89fbe3..c50542fe9 100644 --- a/src/include/postgres_binary_reader.hpp +++ b/src/include/postgres_binary_reader.hpp @@ -9,11 +9,14 @@ #pragma once #include "postgres_binary_parser.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_result_reader.hpp" #include "postgres_connection.hpp" namespace duckdb { +enum class PostgresBinaryReaderMode { UNINITIALIZED, COPY, PLAIN_QUERY }; + struct PostgresBinaryReader : public PostgresResultReader { explicit PostgresBinaryReader(PostgresConnection &con, const vector &column_ids, const PostgresBindData &bind_data); @@ -27,9 +30,17 @@ struct PostgresBinaryReader : public PostgresResultReader { bool FetchNextBuffer(); void FreeBuffer(); + PostgresReadResult ReadCopy(DataChunk &result); + PostgresReadResult ReadPlain(DataChunk &result); + private: + PostgresBinaryReaderMode mode = PostgresBinaryReaderMode::UNINITIALIZED; PostgresBinaryParser parser; + // copy mode data_ptr_t buffer = nullptr; + // plain mode + unique_ptr result; + idx_t row_offset = 0; }; } // namespace duckdb diff --git a/src/include/postgres_binary_writer.hpp b/src/include/postgres_binary_writer.hpp index 1d951afa7..77d2acb7e 100644 --- a/src/include/postgres_binary_writer.hpp +++ b/src/include/postgres_binary_writer.hpp @@ -9,7 +9,9 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/common/operator/subtract.hpp" #include "duckdb/common/types/interval.hpp" +#include "duckdb/common/types/geometry_crs.hpp" #include "duckdb/common/serializer/memory_stream.hpp" #include "duckdb/common/vector/list_vector.hpp" #include "duckdb/common/vector/map_vector.hpp" @@ -105,12 +107,12 @@ class PostgresBinaryWriter { } void WriteTime(dtime_t value) { - WriteInteger(value.micros); + WriteInteger(value.value); } void WriteTimeTZ(dtime_tz_t value) { WriteRawInteger(sizeof(uint64_t) + sizeof(int32_t)); - WriteRawInteger(value.time().micros); + WriteRawInteger(value.time().value); WriteRawInteger(-value.offset()); } @@ -121,7 +123,14 @@ class PostgresBinaryWriter { if (value == timestamp_t::ninfinity()) { return POSTGRES_NINFINITY; } - return uint64_t(value.value - (POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS)); + int64_t postgres_usec; + if (!TrySubtractOperator::Operation(value.value, static_cast(POSTGRES_EPOCH_TS - DUCKDB_EPOCH_TS), + postgres_usec) || + postgres_usec < POSTGRES_MIN_TIMESTAMP || postgres_usec >= POSTGRES_END_TIMESTAMP) { + throw InvalidInputException("TIMESTAMP \"%s\" is out of range for Postgres' TIMESTAMP field", + Timestamp::ToString(value)); + } + return static_cast(postgres_usec); } void WriteTimestamp(timestamp_t value) { @@ -210,6 +219,44 @@ class PostgresBinaryWriter { stream.WriteData(const_data_ptr_cast(str_data), str_size); } + //! Write GEOMETRY to PostGIS. If the type carries CRS metadata, writes EWKB + //! (WKB with SRID) so PostGIS receives the correct SRID. + void WriteGeometry(string_t value, const LogicalType &type) { + if (!GeoType::HasCRS(type)) { + WriteRawBlob(value); + return; + } + auto &crs = GeoType::GetCRS(type); + + // Extract SRID from CRS identifier (e.g., "EPSG:4326" → 4326) + auto &id = crs.GetIdentifier(); + auto colon = id.find(':'); + if (colon == string::npos) { + WriteRawBlob(value); + return; + } + int32_t srid; + try { + srid = std::stoi(id.substr(colon + 1)); + } catch (...) { + WriteRawBlob(value); + return; + } + + // Write EWKB: WKB with SRID flag set on the type field + 4-byte SRID inserted. + // [byte_order:1] [type|0x20000000:4] [srid:4] [coordinates...] + auto wkb_size = value.GetSize(); + auto wkb_data = const_data_ptr_cast(value.GetData()); + WriteRawInteger(NumericCast(wkb_size + 4)); + stream.WriteData(wkb_data, 1); // byte order + uint32_t wkb_type; + memcpy(&wkb_type, wkb_data + 1, 4); + wkb_type |= 0x20000000; + stream.WriteData(const_data_ptr_cast(reinterpret_cast(&wkb_type)), 4); // type + SRID flag + stream.WriteData(const_data_ptr_cast(reinterpret_cast(&srid)), 4); // SRID (LE, matching WKB) + stream.WriteData(wkb_data + 5, wkb_size - 5); // rest of payload + } + void WriteVarchar(string_t value) { auto str_size = value.GetSize(); auto str_data = value.GetData(); @@ -357,7 +404,7 @@ class PostgresBinaryWriter { } case LogicalTypeId::GEOMETRY: { auto data = FlatVector::GetData(col)[r]; - WriteRawBlob(data); + WriteGeometry(data, type); break; } case LogicalTypeId::ENUM: { diff --git a/src/include/postgres_connection.hpp b/src/include/postgres_connection.hpp index 8d5cc7b72..715a7bb6d 100644 --- a/src/include/postgres_connection.hpp +++ b/src/include/postgres_connection.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_utils.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_parameters.hpp" #include "postgres_result.hpp" #include "duckdb/common/shared_ptr.hpp" @@ -47,12 +48,12 @@ class PostgresConnection { public: static PostgresConnection Open(const string &dsn, const string &attach_path); void Execute(optional_ptr context, const string &query, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); unique_ptr TryQuery(optional_ptr context, const string &query, optional_ptr error_message = nullptr, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); unique_ptr Query(optional_ptr context, const string &query, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); //! Submits a set of queries to be executed in the connection. vector> ExecuteQueries(ClientContext &context, const string &queries); @@ -95,7 +96,7 @@ class PostgresConnection { private: PGresult *PQExecute(optional_ptr context, const string &query, - const PostgresParameters ¶ms = PostgresParameters()); + const PostgresParameters ¶ms = PostgresParameters(), int format = 0); shared_ptr connection; string dsn; diff --git a/src/include/postgres_connection_string.hpp b/src/include/postgres_connection_string.hpp new file mode 100644 index 000000000..d65fb9592 --- /dev/null +++ b/src/include/postgres_connection_string.hpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// postgres_binary_writer.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "duckdb.hpp" + +namespace duckdb { + +enum class PostgresConnectionStringFormat { KEY_VALUE, URI }; + +struct PostgresConnectionString { + static string Escape(const string &input); + + static string ExtractOption(const KeyValueSecret &kv_secret, const string &name); + + static bool ContainsApplicationName(const std::string &conn_str); + + //! Whether libpq reads the connection string as a URI rather than key/value pairs + static bool IsURI(const string &conn_string); + + static void AppendDefaultApplicationName(string &conn_string, PostgresConnectionStringFormat format); +}; + +} // namespace duckdb diff --git a/src/include/postgres_conversion.hpp b/src/include/postgres_conversion.hpp index 391429e50..1d1241c9c 100644 --- a/src/include/postgres_conversion.hpp +++ b/src/include/postgres_conversion.hpp @@ -29,8 +29,11 @@ #define POSTGRES_DATE_NINF 2147483648U #define POSTGRES_EPOCH_TS 211813488000000000ULL #define DUCKDB_EPOCH_TS 210866803200000000ULL -#define POSTGRES_INFINITY 9223372036854775807ULL -#define POSTGRES_NINFINITY 9223372036854775808ULL +/* Values from PostgreSQL's datatype/timestamp.h. The end is exclusive. */ +#define POSTGRES_MIN_TIMESTAMP (-211813488000000000LL) +#define POSTGRES_END_TIMESTAMP 9223371331200000000LL +#define POSTGRES_INFINITY 9223372036854775807ULL +#define POSTGRES_NINFINITY 9223372036854775808ULL #define NBASE 10000 #define DEC_DIGITS 4 /* decimal digits per NBASE digit */ @@ -96,6 +99,21 @@ struct PostgresDecimalConfig { uint16_t ndigits; int16_t weight; bool is_negative; + uint16_t sign; +}; + +enum class PostgresDecimalKind { ORDINARY, NOT_A_NUMBER, POSITIVE_INFINITY, NEGATIVE_INFINITY }; + +template +struct PostgresDecimal { + T value; + PostgresDecimalKind kind; + + PostgresDecimal(T val) : value(val), kind(PostgresDecimalKind::ORDINARY) { + } + + PostgresDecimal(PostgresDecimalKind kind_p) : value(0), kind(kind_p) { + } }; struct PostgresConversion { @@ -190,14 +208,4 @@ struct DecimalConversionHugeint { } }; -struct DecimalConversionDouble { - static double GetPowerOfTen(idx_t index) { - return pow(10, double(index)); - } - - static double Finalize(const PostgresDecimalConfig &config, double result) { - return result / GetPowerOfTen(config.scale); - } -}; - } // namespace duckdb diff --git a/src/include/postgres_filter_pushdown.hpp b/src/include/postgres_filter_pushdown.hpp index 3b1a02988..ad667aab3 100644 --- a/src/include/postgres_filter_pushdown.hpp +++ b/src/include/postgres_filter_pushdown.hpp @@ -8,12 +8,16 @@ #pragma once +#include "duckdb/planner/filter/expression_filter.hpp" +#include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/table_filter_set.hpp" namespace duckdb { class PostgresFilterPushdown { public: + static bool CanPushExpressionDown(ClientContext &, const LogicalGet &, Expression &expr); + static string TransformFilters(const vector &column_ids, optional_ptr filters, const vector &names); diff --git a/src/include/postgres_logging.hpp b/src/include/postgres_logging.hpp index 1fc9650b9..ef4c840a8 100644 --- a/src/include/postgres_logging.hpp +++ b/src/include/postgres_logging.hpp @@ -2,6 +2,7 @@ #pragma once #include "duckdb/logging/logging.hpp" +#include "duckdb/logging/log_type.hpp" #include "duckdb/common/string_util.hpp" namespace duckdb { diff --git a/src/include/postgres_oauth.hpp b/src/include/postgres_oauth.hpp index 2e36feede..6e1a62eae 100644 --- a/src/include/postgres_oauth.hpp +++ b/src/include/postgres_oauth.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" namespace duckdb { diff --git a/src/include/postgres_result.hpp b/src/include/postgres_result.hpp index 262adfef5..2ba2e1f67 100644 --- a/src/include/postgres_result.hpp +++ b/src/include/postgres_result.hpp @@ -48,10 +48,14 @@ class PostgresResult { D_ASSERT(res); return PQntuples(res); } - idx_t AffectedRows() { - auto affected = PQcmdTuples(res); - if (!affected) { - throw InternalException("Postgres scanner - AffectedRows called but none were available"); + idx_t ColumnCount() { + D_ASSERT(res); + return PQnfields(res); + } + int64_t AffectedRows() { + char *affected = PQcmdTuples(res); + if (!affected || strlen(affected) == 0) { + return -1; } return atoll(affected); } diff --git a/src/include/postgres_result_reader.hpp b/src/include/postgres_result_reader.hpp index 1915c3b53..13d4d7d40 100644 --- a/src/include/postgres_result_reader.hpp +++ b/src/include/postgres_result_reader.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/types/interval.hpp" #include "postgres_conversion.hpp" #include "postgres_utils.hpp" diff --git a/src/include/postgres_scanner.hpp b/src/include/postgres_scanner.hpp index 9e1fb6d3c..46a4b2803 100644 --- a/src/include/postgres_scanner.hpp +++ b/src/include/postgres_scanner.hpp @@ -9,8 +9,7 @@ #pragma once #include "duckdb.hpp" - -#include "dbconnector/bind_data.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_utils.hpp" #include "postgres_connection.hpp" @@ -23,15 +22,23 @@ struct PostgresLocalState; struct PostgresGlobalState; class PostgresTransaction; -struct PostgresBindData : public dbconnector::BindData { +struct PostgresBindData : public FunctionData { static constexpr const idx_t DEFAULT_PAGES_PER_TASK = 1000; public: - PostgresBindData(ClientContext &context); + PostgresBindData(ClientContext &context, const Identifier &catalog_name); - PostgresVersion version; string schema_name; string table_name; + + Identifier catalog_name; + QualifiedName qualified_table_name; + + PostgresQueryMode query_mode = PostgresQueryMode::AUTO; + + // required for get_bind_info and only used there + weak_ptr context_ptr; + string sql; PostgresParameters params; idx_t pages_approx = 0; @@ -54,45 +61,21 @@ struct PostgresBindData : public dbconnector::BindData { //! DML without RETURNING). InitGlobalState executes it and returns a single-row Success result. bool command_only = false; idx_t max_threads = 1; + PostgresTypeConfig type_config; + + string limit_clause; - dbconnector::optimizer::OrderByAndLimitBindData order_by_and_limit_bind_data; - dbconnector::optimizer::AggregateBindData aggregate_bind_data; + uint64_t pinned_connection_id = 0; public: void SetTablePages(idx_t approx_num_pages); - void SetCatalog(PostgresCatalog &catalog); - void SetTable(PostgresTableEntry &table); - optional_ptr GetCatalog() const { - return pg_catalog; - } - optional_ptr GetTable() const { - return pg_table; - } - unique_ptr Copy() const override { throw NotImplementedException(""); } bool Equals(const FunctionData &other_p) const override { return false; } - - dbconnector::optimizer::OrderByAndLimitBindData &GetOrderByAndLimitBindData() override { - return order_by_and_limit_bind_data; - } - - dbconnector::optimizer::AggregateBindData &GetAggregateBindData() override { - return aggregate_bind_data; - } - -private: - optional_ptr pg_catalog; - optional_ptr pg_table; -}; - -class PostgresAttachFunction : public TableFunction { -public: - PostgresAttachFunction(); }; class PostgresScanFunction : public TableFunction { @@ -100,12 +83,7 @@ class PostgresScanFunction : public TableFunction { PostgresScanFunction(); static void PrepareBind(PostgresVersion version, ClientContext &context, PostgresBindData &bind, - int64_t approx_num_pages); -}; - -class PostgresScanFunctionFilterPushdown : public TableFunction { -public: - PostgresScanFunctionFilterPushdown(); + int64_t approx_num_pages, optional_ptr pg_catalog); }; class PostgresClearCacheFunction : public TableFunction { @@ -126,4 +104,14 @@ class PostgresExecuteFunction : public TableFunction { PostgresExecuteFunction(); }; +class PostgresPinConnectionFunction : public ScalarFunction { +public: + PostgresPinConnectionFunction(); +}; + +class PostgresClosePinnedConnectionFunction : public ScalarFunction { +public: + PostgresClosePinnedConnectionFunction(); +}; + } // namespace duckdb diff --git a/src/include/postgres_secrets.hpp b/src/include/postgres_secrets.hpp index a4713abfe..45e4b5dc1 100644 --- a/src/include/postgres_secrets.hpp +++ b/src/include/postgres_secrets.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/main/secret/secret_manager.hpp" namespace duckdb { diff --git a/src/include/postgres_text_reader.hpp b/src/include/postgres_text_reader.hpp index cbd9491af..c68251be6 100644 --- a/src/include/postgres_text_reader.hpp +++ b/src/include/postgres_text_reader.hpp @@ -9,6 +9,7 @@ #pragma once #include "postgres_result_reader.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_connection.hpp" #include "postgres_result.hpp" diff --git a/src/include/postgres_utils.hpp b/src/include/postgres_utils.hpp index 84c07968b..45dc11e60 100644 --- a/src/include/postgres_utils.hpp +++ b/src/include/postgres_utils.hpp @@ -8,15 +8,21 @@ #pragma once +#include + #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include #include "postgres_version.hpp" #include "duckdb/main/secret/secret_manager.hpp" namespace duckdb { +class PostgresCatalog; class PostgresSchemaEntry; class PostgresTransaction; +enum class PostgresQueryMode { AUTO, COPY_BINARY, PLAIN_QUERY_BINARY, PLAIN_QUERY_TEXT }; + struct PostgresTypeData { int64_t type_modifier = 0; string type_name; @@ -24,6 +30,14 @@ struct PostgresTypeData { idx_t array_dimensions = 0; }; +struct PostgresTypeConfig { + bool array_as_varchar = false; + bool numeric_as_varchar = false; + bool numeric_nan_as_null = true; + + static PostgresTypeConfig FromContext(optional_ptr context); +}; + enum class PostgresTypeAnnotation { STANDARD, CAST_TO_VARCHAR, @@ -58,13 +72,22 @@ struct PostgresCopyState { enum class PostgresIsolationLevel { READ_COMMITTED, REPEATABLE_READ, SERIALIZABLE }; +enum class PostgresConnectCheck { THROW_ON_CONNECTION_BAD, RETURN_ON_CONNECTION_BAD }; + +enum class PostgresUseCtidScan : uint8_t { DEFAULT, ENABLED, DISABLED }; + class PostgresUtils { public: static PGconn *PGConnect(const string &dsn, const string &attach_path); + static bool UseInformationSchemaIntrospection(optional_ptr context); + static bool StalenessQueryEnabled(ClientContext &context); + static string GetCustomStalenessQuery(ClientContext &context); + static string DataTypeToTypeName(const string &data_type); static LogicalType ToPostgresType(const LogicalType &input); static LogicalType TypeToLogicalType(optional_ptr transaction, - optional_ptr schema, const PostgresTypeData &input, + optional_ptr schema, + const PostgresTypeConfig &type_config, const PostgresTypeData &input, PostgresType &postgres_type); static string TypeToString(const LogicalType &input); static string PostgresOidToName(uint32_t oid); @@ -77,12 +100,22 @@ class PostgresUtils { static PostgresVersion ExtractPostgresVersion(const string &version); - static string EscapeConnectionString(const string &input); - static string ExtractConnectionOption(const KeyValueSecret &kv_secret, const string &name); static string WriteLiteral(const string &identifier); + static string WriteLiteralsCommaSeparated(const vector &literals); static string WriteIdentifier(const string &identifier); + static PostgresQueryMode ExtractQueryMode(ClientContext &ctx); + static PostgresQueryMode ExtractQueryMode(Value &value); + static PostgresCatalog &GetCatalogByName(ClientContext &context, const std::string &name); + + static void DeduplicateColumns(vector &names); + private: + static mutex libpq_init_lock; + static bool libpq_connect_succeeded_at_least_once; + + static PGconn *PGConnectInternal(const string &dsn, const string &attach_path, PostgresConnectCheck check); + static string EscapeQuotes(const string &text, char quote); static string WriteQuoted(const string &text, char quote); }; diff --git a/src/include/postgres_version.hpp b/src/include/postgres_version.hpp index d26b0526a..aa1db5090 100644 --- a/src/include/postgres_version.hpp +++ b/src/include/postgres_version.hpp @@ -14,6 +14,10 @@ namespace duckdb { enum class PostgresInstanceType { UNKNOWN, POSTGRES, AURORA, REDSHIFT }; +//! How scans decide between the binary COPY protocol and the text protocol. +//! AUTO derives it from the server (the binary COPY protocol is not supported by Redshift). +enum class PostgresTextProtocolMode { AUTO, TEXT, BINARY }; + struct PostgresVersion { PostgresVersion() { } diff --git a/src/include/storage/postgres_catalog.hpp b/src/include/storage/postgres_catalog.hpp index 9161e46fd..e0bf84639 100644 --- a/src/include/storage/postgres_catalog.hpp +++ b/src/include/storage/postgres_catalog.hpp @@ -8,7 +8,15 @@ #pragma once +#include "dbconnector/attached.hpp" + #include +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include #include "duckdb/catalog/catalog.hpp" @@ -26,21 +34,24 @@ class PostgresSchemaEntry; class PostgresCatalog : public Catalog { public: explicit PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, string attach_path, AccessMode access_mode, - string schema_to_load, PostgresIsolationLevel isolation_level, const string &secret_name, - SecretStorageTable secret_storage_table_p); + vector schemas_to_load, PostgresIsolationLevel isolation_level, + const string &secret_name, SecretStorageTable secret_storage_table_p, + PostgresTextProtocolMode text_protocol_mode, string connect_display = string()); ~PostgresCatalog(); string attach_path; AccessMode access_mode; PostgresIsolationLevel isolation_level; + PostgresTextProtocolMode text_protocol_mode; + PostgresUseCtidScan use_ctid_scan = PostgresUseCtidScan::DEFAULT; public: void Initialize(bool load_builtin) override; string GetCatalogType() override { return "postgres"; } - string GetDefaultSchema() const override { - return default_schema.empty() ? "public" : default_schema; + optional GetDefaultSchema() const override { + return default_schema.empty() ? Identifier("public") : default_schema; } string GetConnectionString(); @@ -72,6 +83,8 @@ class PostgresCatalog : public Catalog { return version; } + bool UseTextProtocol(PostgresQueryMode query_mode, bool use_text_protocol_setting) const; + //! Label all postgres scans in the sub-tree as requiring materialization //! This is used for e.g. insert queries that have both (1) a scan from a postgres table, and (2) a sink into one static void MaterializePostgresScans(PhysicalOperator &op); @@ -95,6 +108,8 @@ class PostgresCatalog : public Catalog { return connection_pool; } + static dbconnector::attached::AttachedCatalog Lookup(ClientContext &ctx, const Identifier &name); + void ClearCache(); //! Whether or not this catalog should search a specific type with the standard priority @@ -123,8 +138,10 @@ class PostgresCatalog : public Catalog { PostgresVersion version; PostgresSchemaSet schemas; shared_ptr connection_pool; - string default_schema; + Identifier default_schema; SecretStorageTable secret_storage_table; + //! Overrides what the shell prompt displays for this catalog - see GetConnectDisplay + string connect_display; PostgresAwsRdsTokenConfig rds_token_config; std::mutex rds_token_mutex; diff --git a/src/include/storage/postgres_catalog_set.hpp b/src/include/storage/postgres_catalog_set.hpp index d454c591d..e0c5f43be 100644 --- a/src/include/storage/postgres_catalog_set.hpp +++ b/src/include/storage/postgres_catalog_set.hpp @@ -9,9 +9,12 @@ #pragma once #include "duckdb/transaction/transaction.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/mutex.hpp" #include "duckdb/common/shared_ptr.hpp" +#include "duckdb/common/thread.hpp" namespace duckdb { struct DropInfo; @@ -41,7 +44,24 @@ class PostgresCatalogSet { virtual bool HasInternalDependencies() const { return false; } + //! Empty (default) means staleness is never checked once loaded + virtual string GetStalenessQuery(ClientContext &context) const { + return string(); + } void TryLoadEntries(ClientContext &context, PostgresTransaction &transaction); + void RefreshStalenessSignature(PostgresTransaction &transaction, bool use_transaction_connection); + +private: + //! Runs LoadEntries + RefreshStalenessSignature, must be called with load_lock held + void LoadEntriesLocked(ClientContext &context, PostgresTransaction &transaction); + //! Must be called with load_lock held + void ClearEntriesLocked(); + //! Returns with entry_lock held + unique_lock LoadEntriesForRead(ClientContext &context, PostgresTransaction &transaction); + +protected: +public: + void PromoteStalenessSignature(string signature); protected: Catalog &catalog; @@ -52,6 +72,8 @@ class PostgresCatalogSet { unordered_map> entries; case_insensitive_map_t entry_map; atomic is_loaded; + atomic loading_thread; + string staleness_signature; }; class PostgresInSchemaSet : public PostgresCatalogSet { diff --git a/src/include/storage/postgres_connection_pool.hpp b/src/include/storage/postgres_connection_pool.hpp index b366423ec..982350705 100644 --- a/src/include/storage/postgres_connection_pool.hpp +++ b/src/include/storage/postgres_connection_pool.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include "duckdb/main/client_context.hpp" #include "duckdb/common/common.hpp" #include "duckdb/common/mutex.hpp" @@ -30,14 +31,7 @@ class PostgresConnectionPool : public dbconnector::pool::ConnectionPool index_result = nullptr); public: - static string GetInitializeQuery(const string &schema = string()); + static string GetInitializeQuery(const vector &schemas); optional_ptr CreateIndex(PostgresTransaction &transaction, CreateIndexInfo &info, TableCatalogEntry &table); diff --git a/src/include/storage/postgres_insert.hpp b/src/include/storage/postgres_insert.hpp index 3d20e97f5..072006390 100644 --- a/src/include/storage/postgres_insert.hpp +++ b/src/include/storage/postgres_insert.hpp @@ -9,18 +9,24 @@ #pragma once #include "duckdb/execution/physical_operator.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "duckdb/common/index_vector.hpp" +#include "storage/postgres_catalog.hpp" namespace duckdb { class PostgresInsert : public PhysicalOperator { public: //! INSERT INTO - PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, - physical_index_vector_t column_index_map); + PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, TableCatalogEntry &table, physical_index_vector_t column_index_map); //! CREATE TABLE AS - PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, - unique_ptr info); + PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info); //! The table to insert into optional_ptr table; @@ -32,6 +38,8 @@ class PostgresInsert : public PhysicalOperator { physical_index_vector_t column_index_map; //! Whether or not we can keep the copy alive during Sink calls bool keep_copy_alive = true; + //! Whether to use plain INSERTs instead of COPY + bool use_plain_inserts = false; public: // Source interface @@ -44,10 +52,30 @@ class PostgresInsert : public PhysicalOperator { public: // Sink interface - unique_ptr GetGlobalSinkState(ClientContext &context) const override; - SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; + unique_ptr GetGlobalSinkState(ClientContext &context) const override { + if (use_plain_inserts) { + return GetGlobalSinkStatePlain(context); + } else { + return GetGlobalSinkStateCopy(context); + } + } + + SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override { + if (use_plain_inserts) { + return SinkPlain(context, chunk, input); + } else { + return SinkCopy(context, chunk, input); + } + } + SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const override; + OperatorSinkFinalizeInput &input) const override { + if (use_plain_inserts) { + return FinalizePlain(pipeline, event, context, input); + } else { + return FinalizeCopy(pipeline, event, context, input); + } + } bool IsSink() const override { return true; @@ -59,6 +87,17 @@ class PostgresInsert : public PhysicalOperator { string GetName() const override; InsertionOrderPreservingMap ParamsToString() const override; + +private: + static bool UsePlainInserts(PostgresCatalog &pg_catalog, ClientContext &context); + unique_ptr GetGlobalSinkStateCopy(ClientContext &context) const; + unique_ptr GetGlobalSinkStatePlain(ClientContext &context) const; + SinkResultType SinkCopy(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const; + SinkResultType SinkPlain(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const; + SinkFinalizeType FinalizeCopy(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const; + SinkFinalizeType FinalizePlain(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const; }; } // namespace duckdb diff --git a/src/include/storage/postgres_optimizer.hpp b/src/include/storage/postgres_optimizer.hpp index 832a5d86c..c6d47513e 100644 --- a/src/include/storage/postgres_optimizer.hpp +++ b/src/include/storage/postgres_optimizer.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/main/config.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "duckdb/optimizer/optimizer_extension.hpp" namespace duckdb { diff --git a/src/include/storage/postgres_schema_entry.hpp b/src/include/storage/postgres_schema_entry.hpp index 75e23643a..bf63c7da2 100644 --- a/src/include/storage/postgres_schema_entry.hpp +++ b/src/include/storage/postgres_schema_entry.hpp @@ -9,6 +9,10 @@ #pragma once #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" #include "storage/postgres_table_set.hpp" #include "storage/postgres_index_set.hpp" #include "storage/postgres_type_set.hpp" diff --git a/src/include/storage/postgres_schema_set.hpp b/src/include/storage/postgres_schema_set.hpp index 979a545e5..9ac1a3cb1 100644 --- a/src/include/storage/postgres_schema_set.hpp +++ b/src/include/storage/postgres_schema_set.hpp @@ -9,6 +9,8 @@ #pragma once #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "storage/postgres_schema_entry.hpp" namespace duckdb { @@ -16,19 +18,21 @@ struct CreateSchemaInfo; class PostgresSchemaSet : public PostgresCatalogSet { public: - explicit PostgresSchemaSet(Catalog &catalog, string schema_to_load); + explicit PostgresSchemaSet(Catalog &catalog, vector schemas_to_load); public: optional_ptr CreateSchema(PostgresTransaction &transaction, CreateSchemaInfo &info); - static string GetInitializeQuery(const string &schema = string()); + static string GetInitializeQuery(const vector &schemas); + static string GetInitializeQueryInformationSchema(const vector &schemas); protected: void LoadEntries(ClientContext &context, PostgresTransaction &transaction) override; + void LoadEntriesInformationSchema(ClientContext &context, PostgresTransaction &transaction); protected: //! Schema to load - if empty loads all schemas (default behavior) - string schema_to_load; + vector schemas_to_load; }; } // namespace duckdb diff --git a/src/include/storage/postgres_secret_storage.hpp b/src/include/storage/postgres_secret_storage.hpp index 3504fe208..68a77e4f6 100644 --- a/src/include/storage/postgres_secret_storage.hpp +++ b/src/include/storage/postgres_secret_storage.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/main/secret/secret_storage.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/main/secret/secret_manager.hpp" #include "storage/postgres_connection_pool.hpp" diff --git a/src/include/storage/postgres_table_entry.hpp b/src/include/storage/postgres_table_entry.hpp index f21f8c3fb..abf4c5c83 100644 --- a/src/include/storage/postgres_table_entry.hpp +++ b/src/include/storage/postgres_table_entry.hpp @@ -8,8 +8,14 @@ #pragma once +#include "dbconnector/attached.hpp" + #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/common/unordered_map.hpp" #include "postgres_utils.hpp" namespace duckdb { @@ -20,7 +26,7 @@ struct PostgresTableInfo { create_info->columns.SetAllowDuplicates(true); } PostgresTableInfo(const string &schema, const string &table) { - create_info = make_uniq(Identifier(), Identifier(schema), Identifier(table)); + create_info = make_uniq(QualifiedName(Identifier(), Identifier(schema), Identifier(table))); create_info->columns.SetAllowDuplicates(true); } PostgresTableInfo(const SchemaCatalogEntry &schema, const string &table) { @@ -29,13 +35,14 @@ struct PostgresTableInfo { } const string &GetTableName() const { - return create_info->table.GetIdentifierName(); + return create_info->GetTableName().GetIdentifierName(); } unique_ptr create_info; vector postgres_types; vector postgres_names; int64_t approx_num_pages = 0; + unordered_map attnum_to_logical; }; class PostgresTableEntry : public TableCatalogEntry { @@ -44,6 +51,8 @@ class PostgresTableEntry : public TableCatalogEntry { PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, PostgresTableInfo &info); public: + const ColumnList &GetColumns() const override; + unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; TableFunction GetScanFunction(ClientContext &context, unique_ptr &bind_data) override; @@ -56,6 +65,14 @@ class PostgresTableEntry : public TableCatalogEntry { //! Get the copy format (text or binary) that should be used when writing data to this table PostgresCopyFormat GetCopyFormat(ClientContext &context); + static dbconnector::attached::AttachedTable Lookup(ClientContext &ctx, QualifiedName name); + +private: + PostgresCopyFormat GetBinaryCopyFormatIfTypesAllow(); + +protected: + ColumnList columns; + public: //! Postgres type annotations vector postgres_types; diff --git a/src/include/storage/postgres_table_set.hpp b/src/include/storage/postgres_table_set.hpp index fcdff5428..8812e16da 100644 --- a/src/include/storage/postgres_table_set.hpp +++ b/src/include/storage/postgres_table_set.hpp @@ -9,6 +9,8 @@ #pragma once #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" #include "storage/postgres_table_entry.hpp" namespace duckdb { @@ -33,12 +35,16 @@ class PostgresTableSet : public PostgresInSchemaSet { void AlterTable(ClientContext &context, PostgresTransaction &transaction, AlterTableInfo &info); static string GetInitializeQuery(const string &schema = string(), const string &table = string()); + static string GetInitializeQuery(const vector &schemas, const string &table = string()); + static string GetInitializeQueryInformationSchema(const string &schema = string(), const string &table = string()); + static string GetInitializeQueryInformationSchema(const vector &schemas, const string &table = string()); protected: void LoadEntries(ClientContext &context, PostgresTransaction &transaction) override; bool SupportReload() const override { return true; } + string GetStalenessQuery(ClientContext &context) const override; void AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameTableInfo &info); void AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameColumnInfo &info); @@ -46,11 +52,12 @@ class PostgresTableSet : public PostgresInSchemaSet { void AlterTable(ClientContext &context, PostgresTransaction &transaction, RemoveColumnInfo &info); static void AddColumn(optional_ptr transaction, optional_ptr schema, - PostgresResult &result, idx_t row, PostgresTableInfo &table_info); + const PostgresTypeConfig &type_config, PostgresResult &result, idx_t row, + PostgresTableInfo &table_info); static void AddConstraint(PostgresResult &result, idx_t row, PostgresTableInfo &table_info); static void AddColumnOrConstraint(optional_ptr transaction, - optional_ptr schema, PostgresResult &result, idx_t row, - PostgresTableInfo &table_info); + optional_ptr schema, const PostgresTypeConfig &type_config, + PostgresResult &result, idx_t row, PostgresTableInfo &table_info); void CreateEntries(PostgresTransaction &transaction, PostgresResult &result, idx_t start, idx_t end); diff --git a/src/include/storage/postgres_transaction.hpp b/src/include/storage/postgres_transaction.hpp index 9cbb5eb7e..aef6aacfe 100644 --- a/src/include/storage/postgres_transaction.hpp +++ b/src/include/storage/postgres_transaction.hpp @@ -9,11 +9,14 @@ #pragma once #include "duckdb/transaction/transaction.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "postgres_connection.hpp" #include "storage/postgres_connection_pool.hpp" namespace duckdb { class PostgresCatalog; +class PostgresCatalogSet; class PostgresSchemaEntry; class PostgresTableEntry; @@ -43,13 +46,21 @@ class PostgresTransaction : public Transaction { string GetTemporarySchema(); + PostgresTransactionState GetTransactionState() const { + return transaction_state; + } + void StageStalenessSignature(PostgresCatalogSet &catalog_set, string signature); + private: PostgresPoolConnection connection; PostgresTransactionState transaction_state; AccessMode access_mode; PostgresIsolationLevel isolation_level; string temporary_schema; + mutex referenced_entries_lock; reference_map_t> referenced_entries; + mutex pending_signatures_lock; + vector, string>> pending_signatures; private: //! Retrieves the connection **without** starting a transaction if none is active diff --git a/src/include/storage/postgres_transaction_manager.hpp b/src/include/storage/postgres_transaction_manager.hpp index 24b82b435..023831f91 100644 --- a/src/include/storage/postgres_transaction_manager.hpp +++ b/src/include/storage/postgres_transaction_manager.hpp @@ -8,27 +8,13 @@ #pragma once -#include "duckdb/transaction/transaction_manager.hpp" +#include "dbconnector/storage/transaction_manager.hpp" + #include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" -#include "duckdb/common/reference_map.hpp" namespace duckdb { -class PostgresTransactionManager : public TransactionManager { -public: - PostgresTransactionManager(AttachedDatabase &db_p, PostgresCatalog &postgres_catalog); - - Transaction &StartTransaction(ClientContext &context) override; - ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; - void RollbackTransaction(Transaction &transaction) override; - - void Checkpoint(ClientContext &context, bool force = false) override; - -private: - PostgresCatalog &postgres_catalog; - mutex transaction_lock; - reference_map_t> transactions; -}; +using PostgresTransactionManager = dbconnector::storage::TransactionManager; } // namespace duckdb diff --git a/src/include/storage/postgres_type_entry.hpp b/src/include/storage/postgres_type_entry.hpp index 6ea79d3e4..861ca5538 100644 --- a/src/include/storage/postgres_type_entry.hpp +++ b/src/include/storage/postgres_type_entry.hpp @@ -9,6 +9,8 @@ #pragma once #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_type_info.hpp" #include "postgres_utils.hpp" diff --git a/src/include/storage/postgres_type_set.hpp b/src/include/storage/postgres_type_set.hpp index 9b6321b78..5df937262 100644 --- a/src/include/storage/postgres_type_set.hpp +++ b/src/include/storage/postgres_type_set.hpp @@ -9,6 +9,7 @@ #pragma once #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" #include "storage/postgres_type_entry.hpp" namespace duckdb { @@ -25,8 +26,8 @@ class PostgresTypeSet : public PostgresInSchemaSet { public: optional_ptr CreateType(PostgresTransaction &transaction, CreateTypeInfo &info); - static string GetInitializeEnumsQuery(PostgresVersion version, const string &schema = string()); - static string GetInitializeCompositesQuery(const string &schema = string()); + static string GetInitializeEnumsQuery(PostgresVersion version, const vector &schemas); + static string GetInitializeCompositesQuery(const vector &schemas); protected: bool HasInternalDependencies() const override { diff --git a/src/include/storage/postgres_update.hpp b/src/include/storage/postgres_update.hpp index 1d567b978..44c184e69 100644 --- a/src/include/storage/postgres_update.hpp +++ b/src/include/storage/postgres_update.hpp @@ -9,6 +9,9 @@ #pragma once #include "duckdb/execution/physical_operator.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "duckdb/common/index_vector.hpp" namespace duckdb { diff --git a/src/postgres_attach.cpp b/src/postgres_attach.cpp deleted file mode 100644 index a9fc5a39f..000000000 --- a/src/postgres_attach.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "duckdb.hpp" - -#include "duckdb/parser/parsed_data/create_table_function_info.hpp" - -#include "postgres_filter_pushdown.hpp" -#include "postgres_scanner.hpp" -#include "postgres_result.hpp" -#include "postgres_utils.hpp" - -namespace duckdb { - -struct AttachFunctionData : public TableFunctionData { - bool finished = false; - string source_schema = "public"; - string sink_schema = "main"; - string suffix = ""; - bool overwrite = false; - bool filter_pushdown = false; - - string dsn = ""; -}; - -static unique_ptr AttachBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto result = make_uniq(); - result->dsn = input.inputs[0].GetValue(); - - for (auto &kv : input.named_parameters) { - if (kv.first == "source_schema") { - result->source_schema = StringValue::Get(kv.second); - } else if (kv.first == "sink_schema") { - result->sink_schema = StringValue::Get(kv.second); - } else if (kv.first == "overwrite") { - result->overwrite = BooleanValue::Get(kv.second); - } else if (kv.first == "filter_pushdown") { - result->filter_pushdown = BooleanValue::Get(kv.second); - } - } - - return_types.push_back(LogicalType::BOOLEAN); - names.emplace_back("Success"); - return std::move(result); -} - -static void AttachFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (AttachFunctionData &)*data_p.bind_data; - if (data.finished) { - return; - } - - auto conn = PostgresConnection::Open(data.dsn, data.dsn); - auto dconn = Connection(context.db->GetDatabase(context)); - auto fetch_table_query = StringUtil::Format( - R"( -SELECT relname -FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid -JOIN pg_attribute ON pg_class.oid = pg_attribute.attrelid -WHERE relkind = 'r' AND attnum > 0 AND nspname = %s -GROUP BY relname -ORDER BY relname; -)", - PostgresUtils::WriteLiteral(data.source_schema)); - auto res = conn.Query(context, fetch_table_query); - for (idx_t row = 0; row < PQntuples(res->res); row++) { - auto table_name = res->GetString(row, 0); - string query; - if (data.overwrite) { - query = "CREATE OR REPLACE VIEW "; - } else { - query = "CREATE VIEW IF NOT EXISTS "; - } - if (!data.sink_schema.empty()) { - query += PostgresUtils::WriteIdentifier(data.sink_schema) + "."; - } - query += PostgresUtils::WriteIdentifier(table_name); - query += " AS SELECT * FROM "; - if (data.filter_pushdown) { - query += "postgres_scan_pushdown"; - } else { - query += "postgres_scan"; - } - query += "("; - query += PostgresUtils::WriteLiteral(data.dsn); - query += ", "; - query += PostgresUtils::WriteLiteral(data.source_schema); - query += ", "; - query += PostgresUtils::WriteLiteral(table_name); - query += ");"; - dconn.Query(query); - } - res.reset(); - - data.finished = true; -} - -PostgresAttachFunction::PostgresAttachFunction() - : TableFunction("postgres_attach", {LogicalType::VARCHAR}, AttachFunction, AttachBind) { - named_parameters["overwrite"] = LogicalType::BOOLEAN; - named_parameters["filter_pushdown"] = LogicalType::BOOLEAN; - - named_parameters["source_schema"] = LogicalType::VARCHAR; - named_parameters["sink_schema"] = LogicalType::VARCHAR; - named_parameters["suffix"] = LogicalType::VARCHAR; -} - -} // namespace duckdb diff --git a/src/postgres_aws.cpp b/src/postgres_aws.cpp index 708c89889..9aafa040a 100644 --- a/src/postgres_aws.cpp +++ b/src/postgres_aws.cpp @@ -1,4 +1,6 @@ #include "postgres_aws.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/database.hpp" #include #include @@ -7,6 +9,7 @@ #include "dbconnector/defer.hpp" +#include "postgres_connection_string.hpp" #include "postgres_secrets.hpp" #include "postgres_utils.hpp" @@ -117,12 +120,20 @@ PostgresAwsRdsTokenConfig PostgresAws::ExtractTokenConfigFromSecret(optional_ptr "specified - generated IAM token will be used instead of the password"); } + bool has_app_name = !kv.TryGetValue("application_name").IsNull(); + // Build the base connection string (without password) for (const string &opt_name : PostgresSecrets::ConnectionOptionNames()) { if (opt_name == "password" || opt_name == "passfile") { continue; } - config.base_connection_string += PostgresUtils::ExtractConnectionOption(kv, opt_name); + config.base_connection_string += PostgresConnectionString::ExtractOption(kv, opt_name); + } + + if (!has_app_name) { + PostgresConnectionString::AppendDefaultApplicationName(config.base_connection_string, + PostgresConnectionStringFormat::KEY_VALUE); + config.base_connection_string.push_back(' '); } return config; diff --git a/src/postgres_binary_copy.cpp b/src/postgres_binary_copy.cpp index 0ceafa61e..9387cb98d 100644 --- a/src/postgres_binary_copy.cpp +++ b/src/postgres_binary_copy.cpp @@ -1,4 +1,5 @@ #include "postgres_binary_copy.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_binary_writer.hpp" #include "postgres_binary_file_reader.hpp" #include "duckdb/common/serializer/buffered_file_writer.hpp" @@ -97,7 +98,7 @@ void PostgresBinaryCopyFunction::PostgresBinaryWriteFinalize(ClientContext &cont struct PostgresBinaryReadBindData : public TableFunctionData { string file_path; - vector names; + vector names; vector types; vector postgres_types; idx_t buffer_size = PostgresBinaryFileReader::DEFAULT_BUFFER_SIZE; @@ -124,7 +125,7 @@ struct PostgresBinaryReadGlobalState : public GlobalTableFunctionState { unique_ptr PostgresBinaryCopyFunction::PostgresBinaryReadBind(ClientContext &context, CopyFromFunctionBindInput &info, - vector &expected_names, + vector &expected_names, vector &expected_types) { auto result = make_uniq(); result->file_path = info.info.file_path; @@ -140,8 +141,9 @@ static unique_ptr PostgresBinaryReadInitGlobal(ClientC TableFunctionInitInput &input) { auto &bind_data = input.bind_data->Cast(); auto result = make_uniq(); + PostgresTypeConfig type_config = PostgresTypeConfig::FromContext(context); result->reader = make_uniq(context, bind_data.file_path, bind_data.types, - bind_data.postgres_types, bind_data.buffer_size); + bind_data.postgres_types, type_config, bind_data.buffer_size); return std::move(result); } @@ -156,11 +158,11 @@ static void PostgresBinaryReadScan(ClientContext &context, TableFunctionInput &d } static unique_ptr ReadPostgresBinaryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); result->file_path = input.inputs[0].GetValue(); - if (!input.named_parameters.count("columns")) { + if (!input.named_parameters.contains("columns")) { throw BinderException("read_postgres_binary requires a 'columns' parameter, " "e.g. columns={col1: 'INTEGER', col2: 'VARCHAR'}"); } @@ -172,7 +174,7 @@ static unique_ptr ReadPostgresBinaryBind(ClientContext &context, T auto col_type_str = column_map[i].GetValue(); auto col_type = TransformStringToLogicalType(col_type_str, context); - names.push_back(col_name.GetIdentifierName()); + names.push_back(col_name); return_types.push_back(col_type); result->postgres_types.push_back(PostgresUtils::CreateEmptyPostgresType(col_type)); } @@ -180,7 +182,7 @@ static unique_ptr ReadPostgresBinaryBind(ClientContext &context, T result->names = names; result->types = return_types; - if (input.named_parameters.count("buffer_size")) { + if (input.named_parameters.contains("buffer_size")) { result->buffer_size = input.named_parameters.at("buffer_size").GetValue(); } @@ -188,7 +190,7 @@ static unique_ptr ReadPostgresBinaryBind(ClientContext &context, T } static void PostgresBinaryReadSerialize(Serializer &serializer, const optional_ptr bind_data_p, - const TableFunction &function) { + const BoundTableFunction &function) { auto &bind_data = bind_data_p->Cast(); serializer.WriteProperty(100, "file_path", bind_data.file_path); serializer.WriteProperty(101, "names", bind_data.names); @@ -196,7 +198,8 @@ static void PostgresBinaryReadSerialize(Serializer &serializer, const optional_p serializer.WriteProperty(103, "buffer_size", bind_data.buffer_size); } -static unique_ptr PostgresBinaryReadDeserialize(Deserializer &deserializer, TableFunction &function) { +static unique_ptr PostgresBinaryReadDeserialize(Deserializer &deserializer, + BoundTableFunction &function) { auto result = make_uniq(); deserializer.ReadProperty(100, "file_path", result->file_path); deserializer.ReadProperty(101, "names", result->names); @@ -220,11 +223,17 @@ PostgresBinaryCopyFunction::PostgresBinaryCopyFunction() : CopyFunction("postgre copy_from_function = PostgresReadBinaryFunction(); } +static FunctionSignature ReadPostgresBinarySignature() { + FunctionSignature signature; + signature.AddParameter("file_path", LogicalType::VARCHAR).WithTypedKwargs("options", [&](TypedKwargs &options) { + options.Add("columns", LogicalType::ANY).Add("buffer_size", LogicalType::UBIGINT); + }); + return signature; +} + PostgresReadBinaryFunction::PostgresReadBinaryFunction() - : TableFunction("read_postgres_binary", {LogicalType::VARCHAR}, PostgresBinaryReadScan, ReadPostgresBinaryBind, - PostgresBinaryReadInitGlobal) { - named_parameters["columns"] = LogicalType::ANY; - named_parameters["buffer_size"] = LogicalType::UBIGINT; + : TableFunction("read_postgres_binary", ReadPostgresBinarySignature(), PostgresBinaryReadScan, + ReadPostgresBinaryBind, PostgresBinaryReadInitGlobal) { serialize = PostgresBinaryReadSerialize; deserialize = PostgresBinaryReadDeserialize; } diff --git a/src/postgres_binary_file_reader.cpp b/src/postgres_binary_file_reader.cpp index 480bcb3db..72cbe1fc6 100644 --- a/src/postgres_binary_file_reader.cpp +++ b/src/postgres_binary_file_reader.cpp @@ -1,4 +1,6 @@ #include "postgres_binary_file_reader.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/common/file_system.hpp" namespace duckdb { @@ -12,9 +14,10 @@ static vector MakeSequentialColumnIds(idx_t count) { PostgresBinaryFileReader::PostgresBinaryFileReader(ClientContext &context, const string &file_path, vector types_p, vector postgres_types_p, - idx_t buffer_size_p) - : column_ids(MakeSequentialColumnIds(types_p.size())), parser(std::move(types_p), std::move(postgres_types_p)), - buffer_size(buffer_size_p), file_offset(0), leftover(0), leftover_offset(0), finished(false) { + PostgresTypeConfig type_config, idx_t buffer_size_p) + : column_ids(MakeSequentialColumnIds(types_p.size())), + parser(std::move(types_p), std::move(postgres_types_p), type_config), buffer_size(buffer_size_p), file_offset(0), + leftover(0), leftover_offset(0), finished(false), header_scanned(false) { auto &fs = FileSystem::GetFileSystem(context); file_handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ); file_size = file_handle->GetFileSize(); @@ -29,18 +32,18 @@ PostgresBinaryFileReader::PostgresBinaryFileReader(ClientContext &context, const } bool PostgresBinaryFileReader::ReadChunk(DataChunk &output) { + bool full = true; while (output.size() < STANDARD_VECTOR_SIZE) { if (parser.ReadChunk(output, column_ids)) { - return true; - } - if (finished) { - return false; + break; } - if (!FillBuffer()) { - return false; + if (finished || !FillBuffer()) { + full = false; + break; } } - return true; + output.SetChildCardinality(output.size()); + return full; } bool PostgresBinaryFileReader::FillBuffer() { @@ -65,15 +68,25 @@ bool PostgresBinaryFileReader::FillBuffer() { return false; } - idx_t valid = FindLastCompleteRow(read_buffer.get(), total); - if (valid == 0) { - if (file_offset >= file_size) { - valid = total; - } else { - throw IOException("Postgres binary file contains a row larger than the read buffer (%llu bytes). " - "Increase buffer_size.", - buffer_size); - } + // the first buffer starts with the COPY file header, which is not a tuple - skip it while scanning for + // row boundaries, but keep it in the buffer so that CheckHeader can still validate it + idx_t scan_offset = header_scanned ? 0 : MinValue(COPY_FILE_HEADER_SIZE, total); + header_scanned = true; + + idx_t valid = FindLastCompleteRow(read_buffer.get() + scan_offset, total - scan_offset); + if (valid > 0) { + valid += scan_offset; + } else if (file_offset >= file_size) { + valid = total; + } else if (scan_offset > 0) { + // no complete row fits next to the header. The header is a boundary of its own, so hand the parser + // just the header and carry the partial row over to the next fill, where the whole buffer is + // available for it - a row only has to fit in buffer_size, not in buffer_size minus the header. + valid = scan_offset; + } else { + throw IOException("Postgres binary file contains a row larger than the read buffer (%llu bytes). " + "Increase buffer_size.", + buffer_size); } parser.SetBuffer(read_buffer.get(), valid); diff --git a/src/postgres_binary_parser.cpp b/src/postgres_binary_parser.cpp index 900a2905c..2073cfe02 100644 --- a/src/postgres_binary_parser.cpp +++ b/src/postgres_binary_parser.cpp @@ -1,4 +1,5 @@ #include "postgres_binary_parser.hpp" +#include "duckdb/common/operator/double_cast_operator.hpp" #include "duckdb/common/types/geometry.hpp" #include "duckdb/common/vector/flat_vector.hpp" @@ -7,8 +8,9 @@ namespace duckdb { -PostgresBinaryParser::PostgresBinaryParser(vector types_p, vector postgres_types_p) - : types(std::move(types_p)), postgres_types(std::move(postgres_types_p)) { +PostgresBinaryParser::PostgresBinaryParser(vector types_p, vector postgres_types_p, + PostgresTypeConfig type_config_p) + : types(std::move(types_p)), postgres_types(std::move(postgres_types_p)), type_config(type_config_p) { } void PostgresBinaryParser::SetBuffer(data_ptr_t buf, idx_t len) { @@ -22,16 +24,19 @@ bool PostgresBinaryParser::ReadChunk(DataChunk &output, const vector & return false; } - auto tuple_count = ReadInteger(); - if (tuple_count <= 0) { - // tuple_count of -1 signifies the file trailer (i.e. footer) + auto field_count = ReadInteger(); + if (field_count <= 0) { + // a field count of -1 signifies the file trailer (i.e. footer) // clear the buffer so Ready() returns false and the caller can free it buffer_ptr = nullptr; end = nullptr; continue; } - D_ASSERT(tuple_count == column_ids.size()); + if (field_count != column_ids.size()) { + throw InvalidInputException("Postgres binary reader - expected %llu field(s) per row, got %d", + column_ids.size(), field_count); + } idx_t output_offset = output.size(); for (idx_t output_idx = 0; output_idx < output.ColumnCount(); output_idx++) { @@ -45,11 +50,16 @@ bool PostgresBinaryParser::ReadChunk(DataChunk &output, const vector & ReadValue(types[col_idx], postgres_types[col_idx], out_vec, output_offset); } } - output.SetChildCardinality(output_offset + 1); + // only update the count; this runs every row, and SetChildCardinality would visit every vector each time + output.SetCardinalityUnsafe(output_offset + 1); } return true; } +void PostgresBinaryParser::ThrowOutOfBuffer(const char *function_name) { + throw IOException("Postgres binary reader - out of buffer in %s", function_name); +} + void PostgresBinaryParser::CheckHeader() { auto magic_len = PostgresConversion::COPY_HEADER_LENGTH; auto flags_len = 8; @@ -58,7 +68,9 @@ void PostgresBinaryParser::CheckHeader() { if (!buffer_ptr) { throw IOException("buffer_ptr not set in CheckHeader"); } - if (buffer_ptr + header_len >= end) { + // a buffer holding exactly the header is valid - the reader hands one over when the first row does + // not fit alongside it + if (buffer_ptr + header_len > end) { throw IOException("Unable to read binary COPY data, invalid header"); } if (memcmp(buffer_ptr, PostgresConversion::COPY_HEADER, magic_len) != 0) { @@ -79,12 +91,50 @@ PostgresDecimalConfig PostgresBinaryParser::ReadDecimalConfig() { sign == NUMERIC_NEG)) { throw NotImplementedException("Postgres numeric NA/Inf"); } + config.sign = sign; config.is_negative = sign == NUMERIC_NEG; config.scale = ReadInteger(); return config; } +PostgresDecimal PostgresBinaryParser::ReadDecimalAsDouble() { + const auto config = ReadDecimalConfig(); + if (config.sign == NUMERIC_NAN || config.sign == NUMERIC_PINF || config.sign == NUMERIC_NINF) { + return PostgresDecimal(NonFiniteDecimalKindFromSign(config.sign)); + } + if (config.ndigits == 0) { + return PostgresDecimal(0); + } + + // A scientific exponent avoids expanding implicit zero groups or applying the display scale. + const auto exponent = (int32_t(config.weight) + 1 - int32_t(config.ndigits)) * DEC_DIGITS; + const auto exponent_text = std::to_string(exponent); + // Preserve the base-10000 digits and round only once when converting to DOUBLE. + string text; + text.reserve(idx_t(config.ndigits) * DEC_DIGITS + exponent_text.size() + 1 + config.is_negative); + if (config.is_negative) { + text.push_back('-'); + } + for (idx_t i = 0; i < config.ndigits; i++) { + const auto digit = ReadInteger(); + if (digit >= NBASE) { + throw InvalidInputException("Invalid Postgres numeric digit: %d", digit); + } + text.push_back(char('0' + digit / 1000)); + text.push_back(char('0' + (digit / 100) % 10)); + text.push_back(char('0' + (digit / 10) % 10)); + text.push_back(char('0' + digit % 10)); + } + text.push_back('e'); + text += exponent_text; + double result = 0; + if (!TryDoubleCast(text.c_str(), text.size(), result, false)) { + throw InvalidInputException("Could not convert Postgres numeric to DOUBLE"); + } + return PostgresDecimal(result); +} + void PostgresBinaryParser::ReadGeometry(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset) { idx_t element_count = 0; @@ -149,47 +199,85 @@ void PostgresBinaryParser::ReadArray(const LogicalType &type, const PostgresType ListVector::SetListSize(out_vec, child_offset + child_count); } +// Defined out of line so the length checks that call it stay small enough to inline +[[noreturn]] static void ThrowValueLengthMismatch(int32_t value_len, idx_t expected_len, const LogicalType &type) { + throw InvalidInputException("Postgres binary reader - expected %llu bytes for a value of type %s, got %d", + expected_len, type.ToString(), value_len); +} + +// Check that the length of the value Postgres sent matches the length we expect. COPY format only +// tells us the length of the value, so this is the only check we have that the data matches the +// type we expect. +static void CheckValueLength(int32_t value_len, idx_t expected_len, const LogicalType &type) { + if (value_len != expected_len) { + ThrowValueLengthMismatch(value_len, expected_len, type); + } +} + void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset) { auto value_len = ReadInteger(); + ParseValue(type, postgres_type, value_len, out_vec, output_offset); +} + +void PostgresBinaryParser::ParseValue(const LogicalType &type, const PostgresType &postgres_type, int32_t value_len, + Vector &out_vec, idx_t output_offset) { if (value_len == -1) { // NULL FlatVector::SetNull(out_vec, output_offset, true); return; } switch (type.id()) { case LogicalTypeId::SMALLINT: - D_ASSERT(value_len == sizeof(int16_t)); + CheckValueLength(value_len, sizeof(int16_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::INTEGER: - D_ASSERT(value_len == sizeof(int32_t)); + CheckValueLength(value_len, sizeof(int32_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::UINTEGER: - D_ASSERT(value_len == sizeof(uint32_t)); + CheckValueLength(value_len, sizeof(uint32_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::BIGINT: if (postgres_type.info == PostgresTypeAnnotation::CTID) { - D_ASSERT(value_len == 6); + CheckValueLength(value_len, sizeof(int32_t) + sizeof(int16_t), type); int64_t page_index = ReadInteger(); int64_t row_in_page = ReadInteger(); FlatVector::GetDataMutable(out_vec)[output_offset] = (page_index << 16LL) + row_in_page; return; } - D_ASSERT(value_len == sizeof(int64_t)); + CheckValueLength(value_len, sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadInteger(); break; case LogicalTypeId::FLOAT: - D_ASSERT(value_len == sizeof(float)); + CheckValueLength(value_len, sizeof(float), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadFloat(); break; case LogicalTypeId::DOUBLE: { if (postgres_type.info == PostgresTypeAnnotation::NUMERIC_AS_DOUBLE) { - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + auto dec = ReadDecimalAsDouble(); + double double_value = 0; + switch (dec.kind) { + case PostgresDecimalKind::ORDINARY: + double_value = dec.value; + break; + case PostgresDecimalKind::NOT_A_NUMBER: + double_value = std::numeric_limits::quiet_NaN(); + break; + case PostgresDecimalKind::POSITIVE_INFINITY: + double_value = std::numeric_limits::infinity(); + break; + case PostgresDecimalKind::NEGATIVE_INFINITY: + double_value = -std::numeric_limits::infinity(); + break; + default: + throw InvalidInputException("Unsupported decimal kind"); + } + FlatVector::GetDataMutable(out_vec)[output_offset] = double_value; break; } - D_ASSERT(value_len == sizeof(double)); + CheckValueLength(value_len, sizeof(double), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDouble(); break; } @@ -225,7 +313,7 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType break; } case LogicalTypeId::BOOLEAN: - D_ASSERT(value_len == sizeof(bool)); + CheckValueLength(value_len, sizeof(bool), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadBoolean(); break; case LogicalTypeId::DECIMAL: { @@ -233,19 +321,34 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType throw InvalidInputException("Need at least 8 bytes to read a Postgres decimal. Got %d", value_len); } switch (type.InternalType()) { - case PhysicalType::INT16: - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + case PhysicalType::INT16: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; - case PhysicalType::INT32: - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + } + case PhysicalType::INT32: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; - case PhysicalType::INT64: - FlatVector::GetDataMutable(out_vec)[output_offset] = ReadDecimal(); + } + case PhysicalType::INT64: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; - case PhysicalType::INT128: - FlatVector::GetDataMutable(out_vec)[output_offset] = - ReadDecimal(); + } + case PhysicalType::INT128: { + PostgresDecimal dec = ReadDecimal(); + if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) { + FlatVector::GetDataMutable(out_vec)[output_offset] = dec.value; + } break; + } default: throw InvalidInputException("Unsupported decimal storage type"); } @@ -253,24 +356,24 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType } case LogicalTypeId::DATE: { - D_ASSERT(value_len == sizeof(int32_t)); + CheckValueLength(value_len, sizeof(int32_t), type); auto out_ptr = FlatVector::GetDataMutable(out_vec); out_ptr[output_offset] = ReadDate(); break; } case LogicalTypeId::TIME: { - D_ASSERT(value_len == sizeof(int64_t)); + CheckValueLength(value_len, sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadTime(); break; } case LogicalTypeId::TIME_TZ: { - D_ASSERT(value_len == sizeof(int64_t) + sizeof(int32_t)); + CheckValueLength(value_len, sizeof(int64_t) + sizeof(int32_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadTimeTZ(); break; } case LogicalTypeId::TIMESTAMP_TZ: case LogicalTypeId::TIMESTAMP: { - D_ASSERT(value_len == sizeof(int64_t)); + CheckValueLength(value_len, sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadTimestamp(); break; } @@ -304,7 +407,7 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType break; } case LogicalTypeId::UUID: { - D_ASSERT(value_len == 2 * sizeof(int64_t)); + CheckValueLength(value_len, 2 * sizeof(int64_t), type); FlatVector::GetDataMutable(out_vec)[output_offset] = ReadUUID(); break; } @@ -329,7 +432,11 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType default: break; } - D_ASSERT(value_len >= 3 * sizeof(uint32_t)); + if (value_len < 3 * sizeof(uint32_t)) { + throw InvalidInputException( + "Postgres binary reader - expected at least %llu bytes for an array header of type %s, got %d", + 3 * sizeof(uint32_t), type.ToString(), value_len); + } auto array_dim = ReadInteger(); auto array_has_null = ReadInteger(); // whether or not the array has nulls - ignore auto value_oid = ReadInteger(); // value_oid - not necessary @@ -363,7 +470,7 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType case LogicalTypeId::STRUCT: { auto &child_entries = StructVector::GetEntries(out_vec); if (postgres_type.info == PostgresTypeAnnotation::GEOM_POINT) { - D_ASSERT(value_len == sizeof(double) * 2); + CheckValueLength(value_len, sizeof(double) * 2, type); FlatVector::GetDataMutable(child_entries[0])[output_offset] = ReadDouble(); FlatVector::GetDataMutable(child_entries[1])[output_offset] = ReadDouble(); break; @@ -385,4 +492,42 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType } } +PostgresDecimalKind PostgresBinaryParser::NonFiniteDecimalKindFromSign(uint16_t dec_sign) { + if (dec_sign == NUMERIC_NAN) { + return PostgresDecimalKind::NOT_A_NUMBER; + } + if (dec_sign == NUMERIC_PINF) { + return PostgresDecimalKind::POSITIVE_INFINITY; + } + if (dec_sign == NUMERIC_NINF) { + return PostgresDecimalKind::NEGATIVE_INFINITY; + } + throw InvalidInputException("Unsupported unbound NUMERIC sign: %u", static_cast(dec_sign)); +} + +string PostgresBinaryParser::NonFiniteDecimalKindToString(PostgresDecimalKind kind) { + switch (kind) { + case PostgresDecimalKind::NOT_A_NUMBER: + return "NaN"; + case PostgresDecimalKind::POSITIVE_INFINITY: + return "Infinity"; + case PostgresDecimalKind::NEGATIVE_INFINITY: + return "-Infinity"; + default: + throw InvalidInputException("Unsupported unbound NUMERIC kind"); + } +} + +bool PostgresBinaryParser::CheckDecimalKindSetNull(Vector &out_vec, idx_t output_offset, PostgresDecimalKind dec_kind) { + if (dec_kind == PostgresDecimalKind::ORDINARY) { + return false; + } + if (!type_config.numeric_nan_as_null) { + string kind_str = NonFiniteDecimalKindToString(dec_kind); + throw InvalidInputException("Unsupported NUMERIC value: %s", kind_str); + } + FlatVector::SetNull(out_vec, output_offset, true); + return true; +} + } // namespace duckdb diff --git a/src/postgres_binary_reader.cpp b/src/postgres_binary_reader.cpp index c41e5fe3d..ff6251a75 100644 --- a/src/postgres_binary_reader.cpp +++ b/src/postgres_binary_reader.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/vector/map_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" #include "duckdb/common/vector/struct_vector.hpp" @@ -9,7 +10,13 @@ namespace duckdb { PostgresBinaryReader::PostgresBinaryReader(PostgresConnection &con_p, const vector &column_ids, const PostgresBindData &bind_data) - : PostgresResultReader(con_p, column_ids, bind_data), parser(bind_data.types, bind_data.postgres_types) { + : PostgresResultReader(con_p, column_ids, bind_data), + parser(bind_data.types, bind_data.postgres_types, bind_data.type_config) { + if (bind_data.query_mode == PostgresQueryMode::PLAIN_QUERY_BINARY) { + this->mode = PostgresBinaryReaderMode::PLAIN_QUERY; + } else { + this->mode = PostgresBinaryReaderMode::COPY; + } } PostgresBinaryReader::~PostgresBinaryReader() { @@ -17,24 +24,40 @@ PostgresBinaryReader::~PostgresBinaryReader() { } void PostgresBinaryReader::BeginCopy(ClientContext &context, const string &sql) { - con.BeginCopyFrom(context, sql, PGRES_COPY_OUT); - if (!FetchNextBuffer()) { - throw IOException("Failed to fetch header for COPY \"%s\"", sql); + if (mode == PostgresBinaryReaderMode::PLAIN_QUERY) { + this->result = con.Query(context, sql, bind_data.params, 1); + this->row_offset = 0; + } else { + con.BeginCopyFrom(context, sql, PGRES_COPY_OUT); + if (!FetchNextBuffer()) { + throw IOException("Failed to fetch header for COPY \"%s\"", sql); + } + parser.CheckHeader(); } - parser.CheckHeader(); } -PostgresReadResult PostgresBinaryReader::Read(DataChunk &output) { +PostgresReadResult PostgresBinaryReader::Read(DataChunk &result) { + if (mode == PostgresBinaryReaderMode::PLAIN_QUERY) { + return ReadPlain(result); + } else { + return ReadCopy(result); + } +} + +PostgresReadResult PostgresBinaryReader::ReadCopy(DataChunk &output) { + auto result = PostgresReadResult::HAVE_MORE_TUPLES; while (output.size() < STANDARD_VECTOR_SIZE) { if (parser.ReadChunk(output, column_ids)) { - return PostgresReadResult::HAVE_MORE_TUPLES; + break; } FreeBuffer(); if (!FetchNextBuffer()) { - return PostgresReadResult::FINISHED; + result = PostgresReadResult::FINISHED; + break; } } - return PostgresReadResult::HAVE_MORE_TUPLES; + output.SetChildCardinality(output.size()); + return result; } bool PostgresBinaryReader::FetchNextBuffer() { @@ -58,21 +81,71 @@ bool PostgresBinaryReader::FetchNextBuffer() { return false; } + // take ownership of the buffer before validating it, so that a short read + // does not leak it - PQgetCopyData allocates even when it returns a length + // we cannot use + buffer = new_buffer; + // len -2 is error // we expect at least 2 bytes in each message for the tuple count if (!new_buffer || len < sizeof(int16_t)) { throw IOException("Unable to read binary COPY data from Postgres: %s", string(PQerrorMessage(con.GetConn()))); } - buffer = new_buffer; parser.SetBuffer(buffer, len); return true; } void PostgresBinaryReader::FreeBuffer() { - if (buffer) { - PQfreemem(buffer); + if (mode == PostgresBinaryReaderMode::PLAIN_QUERY) { + result.reset(); + row_offset = 0; + } else { + if (buffer) { + PQfreemem(buffer); + } + buffer = nullptr; + } +} + +PostgresReadResult PostgresBinaryReader::ReadPlain(DataChunk &output) { + if (!result) { + return PostgresReadResult::FINISHED; } - buffer = nullptr; + + idx_t output_offset = output.size(); + for (; output_offset < STANDARD_VECTOR_SIZE && row_offset < result->Count(); row_offset++, output_offset++) { + for (idx_t output_idx = 0; output_idx < output.ColumnCount(); output_idx++) { + auto col_idx = column_ids[output_idx]; + auto &out_vec = output.data[output_idx]; + + if (result->IsNull(row_offset, output_idx)) { + FlatVector::SetNull(out_vec, output_offset, true); + continue; + } + + // Obtain the raw binary data from the result cell + auto val = result->GetStringRef(row_offset, output_idx); + parser.SetBuffer(data_ptr_cast(val.GetDataWriteable()), val.GetSize()); + + if (col_idx == COLUMN_IDENTIFIER_ROW_ID) { + PostgresType ctid_type; + ctid_type.info = PostgresTypeAnnotation::CTID; + parser.ParseValue(LogicalType::BIGINT, ctid_type, NumericCast(val.GetSize()), out_vec, + output_offset); + } else { + parser.ParseValue(bind_data.types[col_idx], bind_data.postgres_types[col_idx], + NumericCast(val.GetSize()), out_vec, output_offset); + } + } + } + output.SetChildCardinality(output_offset); + + bool finished = row_offset >= result->Count(); + if (finished) { + FreeBuffer(); + return PostgresReadResult::FINISHED; + } + return PostgresReadResult::HAVE_MORE_TUPLES; } } // namespace duckdb diff --git a/src/postgres_connection.cpp b/src/postgres_connection.cpp index b5a10689f..a896df3de 100644 --- a/src/postgres_connection.cpp +++ b/src/postgres_connection.cpp @@ -1,12 +1,16 @@ +#include "postgres_connection.hpp" + #include "duckdb/storage/table_storage_info.hpp" +#include "duckdb/logging/logger.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/column_list.hpp" #include "duckdb/parser/parser.hpp" -#include "postgres_connection.hpp" -#include "postgres_logging.hpp" #include "duckdb/common/types/uuid.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/helper.hpp" +#include "postgres_logging.hpp" + namespace duckdb { static bool debug_postgres_print_queries = false; @@ -62,7 +66,7 @@ static bool ResultHasError(PGresult *result) { } PGresult *PostgresConnection::PQExecute(optional_ptr context, const string &query, - const PostgresParameters ¶ms) { + const PostgresParameters ¶ms, int format) { if (PostgresConnection::DebugPrintQueries()) { Printer::Print(query + "\n"); } @@ -73,11 +77,10 @@ PGresult *PostgresConnection::PQExecute(optional_ptr context, con PGconn *conn = GetConn(); PGresult *res = nullptr; - if (params.Empty()) { + if (params.Empty() && format == 0) { res = PQexec(GetConn(), query.c_str()); } else { // Unlike PQexec, PQexecParams allows at most one SQL command in the given string. - int format = 0; // text format res = PQexecParams(conn, query.c_str(), params.Count(), params.Types(), params.Values(), params.Lengths(), params.Formats(), format); } @@ -93,14 +96,14 @@ PGresult *PostgresConnection::PQExecute(optional_ptr context, con unique_ptr PostgresConnection::TryQuery(optional_ptr context, const string &query, optional_ptr error_message, - const PostgresParameters ¶ms) { + const PostgresParameters ¶ms, int format) { lock_guard guard(connection->connection_lock); - auto result = PQExecute(context, query.c_str(), params); + auto result = PQExecute(context, query.c_str(), params, format); if (ResultHasError(result)) { if (error_message) { - *error_message = StringUtil::Format("Failed to execute query \"" + query + - "\": " + string(PQresultErrorMessage(result))); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + *error_message = StringUtil::Format("Failed to execute query \"" + query + "\": " + string(error)); } PQclear(result); return nullptr; @@ -109,9 +112,9 @@ unique_ptr PostgresConnection::TryQuery(optional_ptr PostgresConnection::Query(optional_ptr context, const string &query, - const PostgresParameters ¶ms) { + const PostgresParameters ¶ms, int format) { string error_msg; - auto result = TryQuery(context, query, &error_msg, params); + auto result = TryQuery(context, query, &error_msg, params, format); if (!result) { throw std::runtime_error(error_msg); } @@ -119,8 +122,8 @@ unique_ptr PostgresConnection::Query(optional_ptr } void PostgresConnection::Execute(optional_ptr context, const string &query, - const PostgresParameters ¶ms) { - Query(context, query, params); + const PostgresParameters ¶ms, int format) { + Query(context, query, params, format); } vector> PostgresConnection::ExecuteQueries(ClientContext &context, const string &queries) { @@ -135,6 +138,7 @@ vector> PostgresConnection::ExecuteQueries(ClientCont throw std::runtime_error("Failed to execute query \"" + queries + "\": " + string(PQerrorMessage(GetConn()))); } vector> results; + string error_message; while (true) { auto res = PQgetResult(GetConn()); if (!res) { @@ -142,14 +146,19 @@ vector> PostgresConnection::ExecuteQueries(ClientCont } auto result = make_uniq(res); if (ResultHasError(res)) { - throw std::runtime_error("Failed to execute query \"" + queries + - "\": " + string(PQresultErrorMessage(res))); + if (error_message.empty()) { + error_message = "Failed to execute query \"" + queries + "\": " + string(PQresultErrorMessage(res)); + } + continue; } if (PQresultStatus(res) != PGRES_TUPLES_OK) { continue; } results.push_back(std::move(result)); } + if (!error_message.empty()) { + throw std::runtime_error(error_message); + } int64_t end_time = std::chrono::time_point_cast(std::chrono::steady_clock::now()) .time_since_epoch() .count(); diff --git a/src/postgres_connection_string.cpp b/src/postgres_connection_string.cpp new file mode 100644 index 000000000..31aab061f --- /dev/null +++ b/src/postgres_connection_string.cpp @@ -0,0 +1,103 @@ +#include "postgres_connection_string.hpp" + +#include +#include +#include +#include + +#include "duckdb.hpp" + +#include "dbconnector/query/query_writer.hpp" + +namespace duckdb { + +string PostgresConnectionString::Escape(const string &input) { + using namespace dbconnector::query; + auto config = QueryWriter::CreateConfig('\'', QuoteEscapeStyle::BACKSLASH); + return QueryWriter::WriteQuotedAndEscaped(config, input); +} + +string PostgresConnectionString::ExtractOption(const KeyValueSecret &kv_secret, const string &name) { + Value input_val = kv_secret.TryGetValue(Identifier(name)); + if (input_val.IsNull()) { + // not provided + return string(); + } + string result; + result += name; + result += "="; + result += Escape(input_val.ToString()); + result += " "; + return result; +} + +bool PostgresConnectionString::ContainsApplicationName(const std::string &conninfo_str) { + char *errmsg = nullptr; + + // Parse the connection string into an array of PQconninfoOption structs + PQconninfoOption *options = PQconninfoParse(conninfo_str.c_str(), &errmsg); + + if (!options) { + std::string err_str; + if (errmsg) { + err_str = std::string(errmsg); + PQfreemem(errmsg); + } + throw InvalidInputException("Failed to parse connection string: \"%s\"", err_str); + } + + bool found = false; + + for (PQconninfoOption *opt = options; opt->keyword != nullptr; ++opt) { + if (std::strcmp(opt->keyword, "application_name") == 0) { + // opt->val will be non-null if it was explicitly defined in the string + if (opt->val != nullptr && std::strlen(opt->val) > 0) { + found = true; + } + break; + } + } + + // cleanup + PQconninfoFree(options); + if (errmsg) { + PQfreemem(errmsg); + } + + return found; +} + +// libpq allows '?' in the user info, so the query starts at the first '?' after it +static std::string::size_type FindURIQueryStart(const string &uri) { + auto scheme_end = uri.find("://"); + auto start = scheme_end == std::string::npos ? 0 : scheme_end + 3; + auto user_info_end = uri.find_first_of("@/", start); + if (user_info_end != std::string::npos && uri[user_info_end] == '@') { + start = user_info_end + 1; + } + return uri.find('?', start); +} + +bool PostgresConnectionString::IsURI(const string &conn_string) { + return StringUtil::StartsWith(conn_string, "postgresql://") || StringUtil::StartsWith(conn_string, "postgres://"); +} + +void PostgresConnectionString::AppendDefaultApplicationName(string &conn_string, + PostgresConnectionStringFormat format) { + string app_name = "application_name=DuckDB"; + if (format == PostgresConnectionStringFormat::KEY_VALUE) { + if (conn_string.length() == 0 || conn_string[conn_string.length() - 1] != ' ') { + conn_string.push_back(' '); + } + } else { + auto query_start = FindURIQueryStart(conn_string); + if (query_start == std::string::npos) { + conn_string.push_back('?'); + } else if (query_start + 1 < conn_string.length() && conn_string.back() != '&') { + conn_string.push_back('&'); + } + } + conn_string += app_name; +} + +} // namespace duckdb diff --git a/src/postgres_copy_from.cpp b/src/postgres_copy_from.cpp index a39fe419d..3c8623a19 100644 --- a/src/postgres_copy_from.cpp +++ b/src/postgres_copy_from.cpp @@ -1,4 +1,5 @@ #include "postgres_connection.hpp" +#include "duckdb/main/client_context.hpp" #include "postgres_binary_reader.hpp" namespace duckdb { @@ -7,7 +8,8 @@ void PostgresConnection::BeginCopyFrom(ClientContext &context, const string &que PostgresResult pg_res(PQExecute(context, query.c_str())); auto result = pg_res.res; if (!result || PQresultStatus(result) != expected_result) { - throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(PQresultErrorMessage(result))); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(error)); } } diff --git a/src/postgres_copy_to.cpp b/src/postgres_copy_to.cpp index 7796a126a..079572b10 100644 --- a/src/postgres_copy_to.cpp +++ b/src/postgres_copy_to.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/vector/map_vector.hpp" #include "duckdb/common/vector/struct_vector.hpp" @@ -63,7 +64,8 @@ void PostgresConnection::BeginCopyTo(ClientContext &context, PostgresCopyState & PostgresResult pg_res(PQExecute(context, query.c_str())); auto result = pg_res.res; if (!result || PQresultStatus(result) != PGRES_COPY_IN) { - throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(PQresultErrorMessage(result))); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + throw std::runtime_error("Failed to prepare COPY \"" + query + "\": " + string(error)); } if (state.format == PostgresCopyFormat::BINARY) { // binary copy requires a header @@ -112,8 +114,8 @@ void PostgresConnection::FinishCopyTo(PostgresCopyState &state) { PostgresResult pg_res(PQgetResult(GetConn())); auto result = pg_res.res; if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) { - string error_msg(PQresultErrorMessage(result)); - throw std::runtime_error("Failed to copy data: " + error_msg); + auto error = result ? PQresultErrorMessage(result) : PQerrorMessage(GetConn()); + throw std::runtime_error("Failed to copy data: " + string(error)); } } diff --git a/src/postgres_execute.cpp b/src/postgres_execute.cpp deleted file mode 100644 index 46caebb1f..000000000 --- a/src/postgres_execute.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "duckdb.hpp" - -#include "duckdb/parser/parsed_data/create_table_function_info.hpp" -#include "postgres_scanner.hpp" -#include "duckdb/main/database_manager.hpp" -#include "duckdb/main/attached_database.hpp" -#include "storage/postgres_catalog.hpp" -#include "storage/postgres_transaction.hpp" - -namespace duckdb { - -struct PGExecuteBindData : public TableFunctionData { - explicit PGExecuteBindData(PostgresCatalog &pg_catalog, string query_p, bool use_transaction) - : pg_catalog(pg_catalog), query(std::move(query_p)), use_transaction(use_transaction) { - } - - bool finished = false; - PostgresCatalog &pg_catalog; - string query; - bool use_transaction = true; -}; - -static duckdb::unique_ptr PGExecuteBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - return_types.emplace_back(LogicalType::BOOLEAN); - names.emplace_back("Success"); - - // look up the database to query - auto db_name = input.inputs[0].GetValue(); - auto &db_manager = DatabaseManager::Get(context); - auto db = db_manager.GetDatabase(context, Identifier(db_name)); - if (!db) { - throw BinderException("Failed to find attached database \"%s\" referenced in postgres_query", db_name); - } - auto &catalog = db->GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - throw BinderException("Attached database \"%s\" does not refer to a Postgres database", db_name); - } - auto &pg_catalog = catalog.Cast(); - - bool use_transaction = true; - for (auto &kv : input.named_parameters) { - if (kv.first == "use_transaction") { - use_transaction = BooleanValue::Get(kv.second); - } - } - - return make_uniq(pg_catalog, input.inputs[1].GetValue(), use_transaction); -} - -static void PGExecuteFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = data_p.bind_data->CastNoConst(); - if (data.finished) { - return; - } - auto &transaction = Transaction::Get(context, data.pg_catalog).Cast(); - if (data.use_transaction) { - transaction.Query(data.query); - } else { - transaction.QueryWithoutTransaction(data.query); - } - - data.finished = true; -} - -PostgresExecuteFunction::PostgresExecuteFunction() - : TableFunction("postgres_execute", {LogicalType::VARCHAR, LogicalType::VARCHAR}, PGExecuteFunction, - PGExecuteBind) { - named_parameters["use_transaction"] = LogicalType::BOOLEAN; -} - -} // namespace duckdb diff --git a/src/postgres_extension.cpp b/src/postgres_extension.cpp index debc9e01c..956021849 100644 --- a/src/postgres_extension.cpp +++ b/src/postgres_extension.cpp @@ -1,7 +1,7 @@ #define DUCKDB_BUILD_LOADABLE_EXTENSION #include "duckdb.hpp" -#include -#include +#include "duckdb/logging/log_manager.hpp" +#include "duckdb/main/config.hpp" #include "postgres_scanner.hpp" #include "postgres_storage.hpp" @@ -83,6 +83,10 @@ static void SetPostgresConnectionLimit(ClientContext &context, SetScope scope, V } } +static void ValidateQueryMode(ClientContext &, SetScope, Value ¶meter) { + PostgresUtils::ExtractQueryMode(parameter); +} + static void DisablePool(ClientContext &context, SetScope scope, Value ¶meter) { if (scope == SetScope::LOCAL) { throw InvalidInputException("pg_connection_cache can only be set globally"); @@ -118,24 +122,9 @@ static std::string CreatePoolNote(const std::string &option) { } static void LoadInternal(ExtensionLoader &loader) { - // Prevent a race with OpenSSL init that manifests itself with - // the following message on the first connection attempt: - // "port 5432 failed: could not create SSL context: unknown option" - OPENSSL_init_crypto(0, nullptr); - OPENSSL_init_ssl(0, nullptr); - // Register the OAuth bearer token hook before any connections are made PostgresInitOAuthHook(); - PostgresScanFunction postgres_fun; - loader.RegisterFunction(postgres_fun); - - PostgresScanFunctionFilterPushdown postgres_fun_filter_pushdown; - loader.RegisterFunction(postgres_fun_filter_pushdown); - - PostgresAttachFunction attach_func; - loader.RegisterFunction(attach_func); - PostgresClearCacheFunction clear_cache_func; loader.RegisterFunction(clear_cache_func); @@ -154,6 +143,12 @@ static void LoadInternal(ExtensionLoader &loader) { PostgresConfigurePoolFunction configure_pool_function; loader.RegisterFunction(configure_pool_function); + PostgresPinConnectionFunction pin_connection_function; + loader.RegisterFunction(pin_connection_function); + + PostgresClosePinnedConnectionFunction postgres_close_pinned_connection_function; + loader.RegisterFunction(postgres_close_pinned_connection_function); + RegisterHstoreFunctions(loader); // Register the new type @@ -168,6 +163,9 @@ static void LoadInternal(ExtensionLoader &loader) { auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); StorageExtension::Register(config, "postgres_scanner", make_shared_ptr()); + config.AddExtensionOption("pg_query_mode", + "Query mode, one of: 'AUTO', 'COPY_BINARY', 'PLAIN_QUERY_BINARY', 'PLAIN_QUERY_TEXT'", + LogicalType::VARCHAR, Value("AUTO"), ValidateQueryMode); config.AddExtensionOption("pg_use_binary_copy", "Whether or not to use BINARY copy to read data", LogicalType::BOOLEAN, Value::BOOLEAN(true)); config.AddExtensionOption("pg_use_ctid_scan", "Whether or not to parallelize scanning using table ctids", @@ -185,6 +183,13 @@ static void LoadInternal(ExtensionLoader &loader) { config.AddExtensionOption( "pg_array_as_varchar", "Read Postgres arrays as varchar - enables reading mixed dimensional arrays", LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting); + config.AddExtensionOption( + "pg_numeric_as_varchar", + "Read Postgres numerics without precision and scale or precision > 38 as varchar instead of double", + LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting); + config.AddExtensionOption( + "pg_numeric_nan_as_null", "Read Postgres numeric NaN value as NULL instead of throwing an error", + LogicalType::BOOLEAN, Value::BOOLEAN(true), PostgresClearCacheFunction::ClearCacheOnSetting); config.AddExtensionOption( "pg_connection_cache", "Whether or not to use the connection pooling." @@ -193,9 +198,7 @@ static void LoadInternal(ExtensionLoader &loader) { "postgres_configure_pool(catalog_name='my_attached_postgres_db', " "max_connections=0)\" for already attached Postgres databases.", LogicalType::BOOLEAN, Value::BOOLEAN(true), DisablePool); - config.AddExtensionOption("pg_experimental_filter_pushdown", "Whether or not to use filter pushdown", - LogicalType::BOOLEAN, Value::BOOLEAN(true)); - config.AddExtensionOption("pg_order_pushdown", "Push ORDER BY and LIMIT clauses to Postgres (default: true)", + config.AddExtensionOption("pg_enable_filter_pushdown", "Whether or not to use filter pushdown", LogicalType::BOOLEAN, Value::BOOLEAN(true)); config.AddExtensionOption("pg_null_byte_replacement", "When writing NULL bytes to Postgres, replace them with the given character", @@ -210,6 +213,23 @@ static void LoadInternal(ExtensionLoader &loader) { "Whether or not to use TEXT protocol to read data. This is slower, but provides better " "compatibility with non-Postgres systems", LogicalType::BOOLEAN, Value::BOOLEAN(false)); + config.AddExtensionOption( + "pg_use_information_schema_introspection", + "Use SQL-standard information_schema views for ATTACH-time schema discovery instead of pg_catalog." + "For compatibility with pg protocol compatible databases like spanner, cockroach, redshift, ...", + LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting); + config.AddExtensionOption( + "pg_staleness_query_enabled", + "Whether or not the table catalog cache checks Postgres for external DDL changes before serving a " + "cache hit. Defaults to the opposite of pg_use_information_schema_introspection when not explicitly set " + "(off for pg protocol compatible databases that may not support the underlying query, on for Postgres).", + LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting, SetScope::GLOBAL); + config.AddExtensionOption( + "pg_staleness_query", + "Custom query used in place of the default table staleness query when pg_staleness_query_enabled " + "resolves to true. Must contain a ${SCHEMA} placeholder and return at least 3 columns " + "(identity, name, revision marker). Empty (default) uses the built-in pg_class/xmin query.", + LogicalType::VARCHAR, Value(), PostgresClearCacheFunction::ClearCacheOnSetting, SetScope::GLOBAL); config.AddExtensionOption("pg_statement_timeout_millis", "Postgres statement timeout in milliseconds to set on scan connections", LogicalType::UINTEGER, Value()); @@ -270,8 +290,7 @@ static void LoadInternal(ExtensionLoader &loader) { "The query that is used to check that the connection is healthy. Setting this option to " "an empty string disables the health check. " + CreatePoolNote("health_check_query=SELECT 42"), - LogicalType::VARCHAR, PostgresConnectionPool::DefaultHealthCheckQuery(), nullptr, - SetScope::GLOBAL); + LogicalType::VARCHAR, default_pool_config.health_check_query, nullptr, SetScope::GLOBAL); OptimizerExtension postgres_optimizer; postgres_optimizer.optimize_function = PostgresOptimizer::Optimize; diff --git a/src/postgres_filter_pushdown.cpp b/src/postgres_filter_pushdown.cpp index 907b4ad82..98ffcf4be 100644 --- a/src/postgres_filter_pushdown.cpp +++ b/src/postgres_filter_pushdown.cpp @@ -1,22 +1,40 @@ #include "postgres_filter_pushdown.hpp" -#include "duckdb/parser/keyword_helper.hpp" -#include "duckdb/function/scalar/struct_utils.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_conjunction_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/planner/expression/bound_operator_expression.hpp" -#include "duckdb/planner/filter/table_filter_functions.hpp" #include "duckdb/common/enum_util.hpp" +#include "duckdb/common/string_util.hpp" #include "dbconnector/table_scan/filter_pushdown.hpp" #include "dbconnector/table_scan/filter_util.hpp" -#include "postgres_utils.hpp" - namespace duckdb { +static string WriteDistinctFrom(ExpressionType distinct_type, const string &column_name, + const string &constant_string) { + switch (distinct_type) { + case ExpressionType::COMPARE_DISTINCT_FROM: + return StringUtil::Format("%s %s %s", column_name, "IS DISTINCT FROM", constant_string); + case ExpressionType::COMPARE_NOT_DISTINCT_FROM: + return StringUtil::Format("%s %s %s", column_name, "IS NOT DISTINCT FROM", constant_string); + default: + throw InvalidInputException("Unsupported DISTINCT FROM comparion type: %s", EnumUtil::ToString(distinct_type)); + } +} + +static dbconnector::table_scan::FilterPushdown::Config CreatePostgresConfig() { + using namespace dbconnector; + + return table_scan::FilterPushdown::CreateConfig('"', '\'', query::QuoteEscapeStyle::DOUBLE_QUOTE, "'\\x", + "'::BYTEA", "C", WriteDistinctFrom); +} + +bool PostgresFilterPushdown::CanPushExpressionDown(ClientContext &, const LogicalGet &, Expression &expr) { + using dbconnector::table_scan::FilterPushdown; + + auto config = CreatePostgresConfig(); + string filter = FilterPushdown::TransformFilterExpression(config, "dummy", expr); + return !filter.empty(); +} + string PostgresFilterPushdown::TransformFilters(const vector &column_ids, optional_ptr filters, const vector &names) { using namespace dbconnector; @@ -34,8 +52,7 @@ string PostgresFilterPushdown::TransformFilters(const vector &column_i column_name = names[column_id]; } auto &filter = entry.Filter(); - auto config = table_scan::FilterPushdown::CreateConfig('"', '\'', query::QuoteEscapeStyle::DOUBLE_QUOTE, "'\\x", - "::BYTEA"); + auto config = CreatePostgresConfig(); auto filter_text = table_scan::FilterPushdown::TransformFilter(config, column_name, filter, column_id); if (filter_text.empty()) { @@ -43,7 +60,7 @@ string PostgresFilterPushdown::TransformFilters(const vector &column_i continue; } throw NotImplementedException( - "Unsupported filter pushdown, use 'pg_experimental_filter_pushdown=FALSE' to disable pushdowns." + "Unsupported filter pushdown, use 'pg_enable_filter_pushdown=FALSE' to disable pushdowns." " Problematic filter: \"%s\"", table_scan::FilterUtil::ToString(filter)); } diff --git a/src/postgres_hstore.cpp b/src/postgres_hstore.cpp index c1fe02ada..860e74e18 100644 --- a/src/postgres_hstore.cpp +++ b/src/postgres_hstore.cpp @@ -216,10 +216,12 @@ void PostgresHstoreToJsonFun(DataChunk &args, ExpressionState &state, Vector &re void RegisterHstoreFunctions(ExtensionLoader &loader) { auto hstore_get = ScalarFunction("postgres_hstore_get", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, PostgresHstoreGetFun); + hstore_get.SetFallible(); loader.RegisterFunction(hstore_get); auto hstore_to_json = ScalarFunction("postgres_hstore_to_json", {LogicalType::VARCHAR}, LogicalType::JSON(), PostgresHstoreToJsonFun); + hstore_to_json.SetFallible(); loader.RegisterFunction(hstore_to_json); } diff --git a/src/postgres_oauth.cpp b/src/postgres_oauth.cpp index 619532aec..a9cc073aa 100644 --- a/src/postgres_oauth.cpp +++ b/src/postgres_oauth.cpp @@ -1,4 +1,5 @@ #include "postgres_oauth.hpp" +#include "duckdb/main/client_context.hpp" #include #include diff --git a/src/postgres_pin_connection.cpp b/src/postgres_pin_connection.cpp new file mode 100644 index 000000000..527f3bf62 --- /dev/null +++ b/src/postgres_pin_connection.cpp @@ -0,0 +1,64 @@ +#include "postgres_scanner.hpp" + +#include "storage/postgres_catalog.hpp" + +namespace duckdb { + +static void PostgresPinConnection(DataChunk &args, ExpressionState &state, Vector &result) { + UnifiedVectorFormat catalog_name_data; + args.data[0].ToUnifiedFormat(catalog_name_data); + if (!catalog_name_data.validity.RowIsValid(0)) { + throw InvalidInputException("Specified attached database name must be not null"); + } + + auto catalog_name_strt = UnifiedVectorFormat::GetData(catalog_name_data)[0]; + string catalog_name(catalog_name_strt.GetData(), catalog_name_strt.GetSize()); + + auto &catalog = PostgresUtils::GetCatalogByName(state.GetContext(), catalog_name); + auto &pool = catalog.GetConnectionPool(); + + auto conn = pool.ForceAcquire(); + uint64_t conn_id = pool.PinConnection(std::move(conn)); + + auto result_data = FlatVector::GetDataMutable(result); + result_data[0] = conn_id; +} + +PostgresPinConnectionFunction::PostgresPinConnectionFunction() + : ScalarFunction("postgres_pin_connection", {LogicalType::VARCHAR}, LogicalType::UBIGINT, PostgresPinConnection) { + SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + SetFallible(); + SetVolatile(); +} + +static void PostgresClosePinnedConnection(DataChunk &args, ExpressionState &state, Vector &result) { + UnifiedVectorFormat catalog_name_data; + args.data[0].ToUnifiedFormat(catalog_name_data); + if (!catalog_name_data.validity.RowIsValid(0)) { + throw InvalidInputException("Specified catalog name must be not null"); + } + auto catalog_name_strt = UnifiedVectorFormat::GetData(catalog_name_data)[0]; + string catalog_name(catalog_name_strt.GetData(), catalog_name_strt.GetSize()); + + UnifiedVectorFormat conn_id_data; + args.data[1].ToUnifiedFormat(conn_id_data); + if (!conn_id_data.validity.RowIsValid(0)) { + throw InvalidInputException("Specified connection ID must be not null"); + } + auto conn_id = UnifiedVectorFormat::GetData(conn_id_data)[0]; + + auto &catalog = PostgresUtils::GetCatalogByName(state.GetContext(), catalog_name); + { auto conn = catalog.GetConnectionPool().UnpinConnection(conn_id); } + + auto &result_validity = FlatVector::ValidityMutable(result); + result_validity.SetInvalid(0); +} + +PostgresClosePinnedConnectionFunction::PostgresClosePinnedConnectionFunction() + : ScalarFunction("postgres_close_pinned_connection", {LogicalType::VARCHAR, LogicalType::UBIGINT}, + LogicalType::BOOLEAN, PostgresClosePinnedConnection) { + SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + SetFallible(); + SetVolatile(); +} +} // namespace duckdb diff --git a/src/postgres_query.cpp b/src/postgres_query.cpp index 8096f14fd..f0b310d6c 100644 --- a/src/postgres_query.cpp +++ b/src/postgres_query.cpp @@ -1,5 +1,8 @@ #include "duckdb.hpp" +#include "dbconnector/defer.hpp" + +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "postgres_parameters.hpp" #include "postgres_scanner.hpp" @@ -10,56 +13,163 @@ namespace duckdb { -static unique_ptr PGQueryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto result = make_uniq(context); +static bool ExtractFlag(TableFunctionBindInput &input, const string &name, bool default_val) { + auto it = input.named_parameters.find(Identifier(name)); + if (it != input.named_parameters.end()) { + Value &bool_val = it->second; + if (!bool_val.IsNull()) { + return BooleanValue::Get(bool_val); + } + } + return default_val; +} - if (input.inputs[0].IsNull() || input.inputs[1].IsNull()) { - throw BinderException("Parameters to postgres_query cannot be NULL"); +static unique_ptr BindDML(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names, + PostgresCatalog &catalog, PostgresConnection &con, std::string sql, + bool use_transaction, uint64_t pinned_connection_id) { + // The statement returns no result columns: it's a command (DDL, or DML without RETURNING). + // Instead of failing, run it as a command and return a single-row Success result. We reuse + // the prepare/describe just done — no extra round-trip — and defer execution to + // InitGlobalState (execution time, not bind, so EXPLAIN does not run it). + auto result = make_uniq(context, catalog.GetName()); + result->command_only = true; + result->pinned_connection_id = pinned_connection_id; + if (ExtractFlag(input, "suppress_dml_output", false)) { + // This invocation wraps a command with no result set (DDL, or DML without RETURNING). Tell the + // binder via the return-type modifier so that when this is routed through CONNECT the outer + // statement is reported as NOTHING and displays like a native command (no spurious result table). + input.table_function.call_return_type = StatementReturnType::NOTHING; } + return_types.emplace_back(LogicalType::BIGINT); + names.emplace_back(Identifier("rowcount")); + result->dsn = con.GetDSN(); + result->types = return_types; + result->names.emplace_back(names[0].GetIdentifierName()); + result->read_only = false; + result->sql = std::move(sql); + result->use_transaction = use_transaction; + PostgresScanFunction::PrepareBind(catalog.GetPostgresVersion(), context, *result, 0, catalog); + return std::move(result); +} - // look up the database to query - auto db_name = input.inputs[0].GetValue(); - auto &db_manager = DatabaseManager::Get(context); - auto db = db_manager.GetDatabase(context, Identifier(db_name)); - if (!db) { - throw BinderException("Failed to find attached database \"%s\" referenced in postgres_query", db_name); - } - auto &catalog = db->GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - throw BinderException("Attached database \"%s\" does not refer to a Postgres database", db_name); - } - auto &pg_catalog = catalog.Cast(); - auto &transaction = Transaction::Get(context, catalog).Cast(); - auto sql = input.inputs[1].GetValue(); - // strip any trailing semicolons +static PostgresQueryMode ResolveQueryMode(ClientContext &context, TableFunctionBindInput &input, + const vector ¶ms) { + auto it = input.named_parameters.find(Identifier("query_mode")); + if (it != input.named_parameters.end()) { + Value &str_val = it->second; + if (!str_val.IsNull()) { + if (str_val.type().id() != LogicalTypeId::VARCHAR) { + throw BinderException("Invalid non-VARCHAR 'query_mode' parameter specified"); + } + PostgresQueryMode mode = PostgresUtils::ExtractQueryMode(str_val); + if (mode != PostgresQueryMode::AUTO) { + return mode; + } + } + } + PostgresQueryMode mode = PostgresUtils::ExtractQueryMode(context); + if (mode != PostgresQueryMode::AUTO) { + return mode; + } + Value text_proto_val; + if (context.TryGetCurrentSetting("pg_use_text_protocol", text_proto_val) && !text_proto_val.IsNull() && + BooleanValue::Get(text_proto_val)) { + return PostgresQueryMode::PLAIN_QUERY_TEXT; + } + if (params.size() > 0) { + return PostgresQueryMode::PLAIN_QUERY_BINARY; + } + return PostgresQueryMode::AUTO; +} + +static string StripTrailingSemicolon(string sql) { StringUtil::RTrim(sql); while (!sql.empty() && sql.back() == ';') { sql = sql.substr(0, sql.size() - 1); StringUtil::RTrim(sql); } + return sql; +} - bool use_transaction = true; - for (auto &kv : input.named_parameters) { - if (kv.first == "use_transaction") { - use_transaction = BooleanValue::Get(kv.second); - } - } - - vector param_values; +static vector ExtractParams(TableFunctionBindInput &input) { auto params_it = input.named_parameters.find("params"); if (params_it != input.named_parameters.end()) { Value &struct_val = params_it->second; if (struct_val.IsNull()) { throw BinderException("Parameters to postgres_query cannot be NULL"); } - if (struct_val.type().id() != LogicalTypeId::STRUCT) { + if (struct_val.type().id() != LogicalTypeId::STRUCT && struct_val.type().id() != LogicalTypeId::TUPLE) { throw BinderException("Query parameters must be specified in a STRUCT"); } - param_values = StructValue::GetChildren(struct_val); + return StructValue::GetChildren(struct_val); } + return vector(); +} - auto &con = use_transaction ? transaction.GetConnection() : transaction.GetConnectionWithoutTransaction(); +static uint64_t ExtractPinnedConnId(TableFunctionBindInput &input) { + uint64_t pinned_connection_id = 0; + auto conn_it = input.named_parameters.find("connection"); + if (conn_it != input.named_parameters.end()) { + Value &conn_val = conn_it->second; + if (conn_val.IsNull()) { + throw BinderException("Specified connection must be not null"); + } + pinned_connection_id = UBigIntValue::Get(conn_val); + } + return pinned_connection_id; +} + +static unique_ptr PGQueryBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + if (input.inputs[0].IsNull() || input.inputs[1].IsNull()) { + throw BinderException("Parameters to postgres_query cannot be NULL"); + } + + auto db_name = input.inputs[0].GetValue(); + auto &catalog = PostgresUtils::GetCatalogByName(context, db_name); + auto input_sql = input.inputs[1].GetValue(); + auto sql = StripTrailingSemicolon(std::move(input_sql)); + bool use_transaction = ExtractFlag(input, "use_transaction", true); + vector param_values = ExtractParams(input); + uint64_t pinned_connection_id = ExtractPinnedConnId(input); + + optional_ptr con_ptr = nullptr; + PostgresPoolConnection pinned_connection; + + if (pinned_connection_id > 0) { + pinned_connection = catalog.GetConnectionPool().UnpinConnection(pinned_connection_id); + PostgresConnection &conn = pinned_connection.GetConnection(); + con_ptr = &conn; + } else { + auto &transaction = Transaction::Get(context, catalog).Cast(); + if (use_transaction) { + con_ptr = &transaction.GetConnection(); + } else { + con_ptr = &transaction.GetConnectionWithoutTransaction(); + } + } + + auto deferred_pin = dbconnector::Defer([&pinned_connection, &catalog] { + if (!pinned_connection) { + return; + } + try { + catalog.GetConnectionPool().PinConnection(std::move(pinned_connection)); + } catch (...) { + // suppress + } + }); + + PostgresConnection &con = *con_ptr; + + if (!ExtractFlag(input, "prepare", true)) { + if (param_values.size() > 0) { + throw BinderException("query parameters cannot be used with 'prepare=FALSE'"); + } + return BindDML(context, input, return_types, names, catalog, con, std::move(sql), use_transaction, + pinned_connection_id); + } auto conn = con.GetConn(); // prepare execution of the query to figure out the result types and names @@ -80,34 +190,26 @@ static unique_ptr PGQueryBind(ClientContext &context, TableFunctio } int nfields = PQnfields(describe_prepared); if (nfields <= 0) { - // The statement returns no result columns: it's a command (DDL, or DML without RETURNING). - // Instead of failing, run it as a command and return a single-row Success result. We reuse - // the prepare/describe just done — no extra round-trip — and defer execution to - // InitGlobalState (execution time, not bind, so EXPLAIN does not run it). - result->command_only = true; - return_types.emplace_back(LogicalType::BOOLEAN); - names.emplace_back("Success"); - result->SetCatalog(pg_catalog); - result->dsn = con.GetDSN(); - result->types = return_types; - result->names = names; - result->read_only = false; - result->SetTablePages(0); - result->sql = std::move(sql); - result->use_transaction = use_transaction; - return std::move(result); + return BindDML(context, input, return_types, names, catalog, con, std::move(sql), use_transaction, + pinned_connection_id); } + auto result = make_uniq(context, catalog.GetName()); + result->pinned_connection_id = pinned_connection_id; + auto type_config = PostgresTypeConfig::FromContext(context); for (idx_t c = 0; c < nfields; c++) { PostgresType postgres_type; postgres_type.oid = PQftype(describe_prepared, c); PostgresTypeData type_data; type_data.type_name = PostgresUtils::PostgresOidToName(postgres_type.oid); type_data.type_modifier = PQfmod(describe_prepared, c); - auto converted_type = PostgresUtils::TypeToLogicalType(nullptr, nullptr, type_data, postgres_type); + auto converted_type = PostgresUtils::TypeToLogicalType(nullptr, nullptr, type_config, type_data, postgres_type); result->postgres_types.push_back(postgres_type); return_types.emplace_back(converted_type); - names.emplace_back(PQfname(describe_prepared, c)); + char *name_ptr = PQfname(describe_prepared, c); + string name = name_ptr ? name_ptr : "unnamed"; + names.emplace_back(name); } + PostgresUtils::DeduplicateColumns(names); int nparams = PQnparams(describe_prepared); if (nparams != param_values.size()) { throw BinderException("Incorrect number of parameters specified, expected: %d, actual: %zu, query: \"%s\"", @@ -120,22 +222,49 @@ static unique_ptr PGQueryBind(ClientContext &context, TableFunctio } // set up the bind data - result->SetCatalog(pg_catalog); + result->type_config = type_config; result->dsn = con.GetDSN(); result->types = return_types; - result->names = names; + for (auto &nm : names) { + result->names.emplace_back(nm.GetIdentifierName()); + } result->read_only = false; - result->SetTablePages(0); result->sql = std::move(sql); + result->query_mode = ResolveQueryMode(context, input, param_values); result->params = PostgresParameters(std::move(param_types), std::move(param_values)); result->use_transaction = use_transaction; + PostgresScanFunction::PrepareBind(catalog.GetPostgresVersion(), context, *result, 0, catalog); return std::move(result); } +static FunctionSignature PostgresQuerySignature(bool suppress_dml_output) { + FunctionSignature signature; + signature.AddParameter("database", LogicalType::VARCHAR) + .AddParameter("sql", LogicalType::VARCHAR) + .WithTypedKwargs("options", [&](TypedKwargs &options) { + options.Add("use_transaction", LogicalType::BOOLEAN).Add("params", LogicalType::ANY); + if (suppress_dml_output) { + options.Add("suppress_dml_output", LogicalType::BOOLEAN); + } + options.Add("prepare", LogicalType::BOOLEAN); + options.Add("query_mode", LogicalType::VARCHAR); + options.Add("connection", LogicalType::UBIGINT); + }); + return signature; +} + PostgresQueryFunction::PostgresQueryFunction() - : TableFunction("postgres_query", {LogicalType::VARCHAR, LogicalType::VARCHAR}, nullptr, PGQueryBind) { - named_parameters["use_transaction"] = LogicalType::BOOLEAN; - named_parameters["params"] = LogicalType::ANY; + : TableFunction("postgres_query", PostgresQuerySignature(true), nullptr, PGQueryBind) { + PostgresScanFunction scan_function; + init_global = scan_function.init_global; + init_local = scan_function.init_local; + function = scan_function.function; + projection_pushdown = true; + global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; +} + +PostgresExecuteFunction::PostgresExecuteFunction() + : TableFunction("postgres_execute", PostgresQuerySignature(false), nullptr, PGQueryBind) { PostgresScanFunction scan_function; init_global = scan_function.init_global; init_local = scan_function.init_local; diff --git a/src/postgres_scanner.cpp b/src/postgres_scanner.cpp index 1673c4b0b..34cc17218 100644 --- a/src/postgres_scanner.cpp +++ b/src/postgres_scanner.cpp @@ -1,7 +1,10 @@ #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include +#include "dbconnector/attached.hpp" + #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/helper.hpp" @@ -44,6 +47,8 @@ struct PostgresGlobalState : public GlobalTableFunctionState { explicit PostgresGlobalState(idx_t max_threads) : page_idx(0), batch_idx(0), max_threads(max_threads) { } + ~PostgresGlobalState(); + mutable mutex lock; idx_t page_idx; idx_t batch_idx; @@ -56,6 +61,8 @@ struct PostgresGlobalState : public GlobalTableFunctionState { PostgresConnection &GetConnection(); void SetConnection(PostgresConnection connection); void SetConnection(shared_ptr connection); + void SetPinnedConnection(PostgresPoolConnection pinned_connection, + dbconnector::attached::AttachedCatalog attached_catalog); bool TryOpenNewConnection(ClientContext &context, PostgresLocalState &lstate, const PostgresBindData &bind_data); idx_t MaxThreads() const override { @@ -64,52 +71,44 @@ struct PostgresGlobalState : public GlobalTableFunctionState { private: PostgresConnection connection; + PostgresPoolConnection pinned_connection; + dbconnector::attached::AttachedCatalog attached_catalog; }; -static void PostgresGetSnapshot(ClientContext &context, PostgresVersion version, const PostgresBindData &bind_data, +static void PostgresGetSnapshot(ClientContext &context, const PostgresBindData &bind_data, PostgresGlobalState &gstate) { - unique_ptr result; // by default disable snapshotting gstate.snapshot = string(); if (gstate.max_threads <= 1) { return; } - if (version.type_v == PostgresInstanceType::AURORA) { - return; - } // SET TRANSACTION SNAPSHOT requires REPEATABLE READ or SERIALIZABLE - auto pg_catalog = bind_data.GetCatalog(); - if (pg_catalog && pg_catalog->isolation_level == PostgresIsolationLevel::READ_COMMITTED) { + D_ASSERT(!bind_data.catalog_name.empty()); + auto attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + auto &pg_catalog = attached_catalog.Get(); + if (pg_catalog.isolation_level == PostgresIsolationLevel::READ_COMMITTED) { return; } // reader threads can use the same snapshot auto &con = gstate.GetConnection(); - // pg_stat_wal_receiver was introduced in PostgreSQL 9.6 - if (version < PostgresVersion(9, 6, 0)) { - result = con.TryQuery(context, "SELECT pg_is_in_recovery(), pg_export_snapshot()"); - if (result) { - auto in_recovery = result->GetBool(0, 0); - if (!in_recovery) { - gstate.snapshot = result->GetString(0, 1); - } - } - return; - } - - result = con.TryQuery( - context, "SELECT pg_is_in_recovery(), pg_export_snapshot(), (select count(*) from pg_stat_wal_receiver)"); + // snapshot export works on replicas since PostgreSQL 10 and on RDS/Aurora, so no recovery check is needed + auto result = con.TryQuery(context, "SELECT pg_export_snapshot()"); if (result) { - auto in_recovery = result->GetBool(0, 0) || result->GetInt64(0, 2) > 0; - gstate.snapshot = ""; - if (!in_recovery) { - gstate.snapshot = result->GetString(0, 1); - } - return; + gstate.snapshot = result->GetString(0, 0); } } void PostgresScanFunction::PrepareBind(PostgresVersion version, ClientContext &context, PostgresBindData &bind_data, - int64_t approx_num_pages) { + int64_t approx_num_pages, optional_ptr pg_catalog) { + // resolve the protocol before any decision that depends on it (ctid scan, max threads) + // note that bind_data.use_text_protocol holds the value of the pg_use_text_protocol setting here + if (pg_catalog) { + // for an attached database the mode set at ATTACH time decides + bind_data.use_text_protocol = pg_catalog->UseTextProtocol(bind_data.query_mode, bind_data.use_text_protocol); + } else if (version.type_v == PostgresInstanceType::REDSHIFT) { + // no attached database (postgres_scan over a dsn) - Redshift does not support the binary COPY protocol + bind_data.use_text_protocol = true; + } Value pages_per_task; if (context.TryGetCurrentSetting("pg_pages_per_task", pages_per_task)) { bind_data.pages_per_task = UBigIntValue::Get(pages_per_task); @@ -122,6 +121,9 @@ void PostgresScanFunction::PrepareBind(PostgresVersion version, ClientContext &c if (context.TryGetCurrentSetting("pg_use_ctid_scan", pg_use_ctid_scan)) { use_ctid_scan = BooleanValue::Get(pg_use_ctid_scan); } + if (pg_catalog && pg_catalog->use_ctid_scan != PostgresUseCtidScan::DEFAULT) { + use_ctid_scan = pg_catalog->use_ctid_scan == PostgresUseCtidScan::ENABLED; + } if (bind_data.use_text_protocol) { // ctid scan is only supported for binary copy use_ctid_scan = false; @@ -140,13 +142,10 @@ void PostgresScanFunction::PrepareBind(PostgresVersion version, ClientContext &c approx_num_pages = 0; } bind_data.SetTablePages(static_cast(approx_num_pages)); - bind_data.version = version; - if (version.type_v == PostgresInstanceType::REDSHIFT) { - bind_data.use_text_protocol = true; - } } -PostgresBindData::PostgresBindData(ClientContext &context) { +PostgresBindData::PostgresBindData(ClientContext &context, const Identifier &catalog_name_p) + : catalog_name(catalog_name_p), query_mode(PostgresUtils::ExtractQueryMode(context)) { Value text_protocol; if (context.TryGetCurrentSetting("pg_use_text_protocol", text_protocol)) { if (BooleanValue::Get(text_protocol)) { @@ -176,40 +175,9 @@ void PostgresGlobalState::SetConnection(shared_ptr conn this->connection = PostgresConnection(std::move(connection)); } -void PostgresBindData::SetCatalog(PostgresCatalog &catalog) { - this->pg_catalog = &catalog; -} - -void PostgresBindData::SetTable(PostgresTableEntry &table) { - this->pg_table = &table; -} - -static unique_ptr PostgresBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(context); - - bind_data->dsn = input.inputs[0].GetValue(); - bind_data->schema_name = input.inputs[1].GetValue(); - bind_data->table_name = input.inputs[2].GetValue(); - bind_data->attach_path = bind_data->dsn; - - auto con = PostgresConnection::Open(bind_data->dsn, bind_data->attach_path); - auto version = con.GetPostgresVersion(context); - // query the table schema so we can interpret the bits in the pages - auto info = PostgresTableSet::GetTableInfo(context, con, bind_data->schema_name, bind_data->table_name); - - bind_data->postgres_types = info->postgres_types; - for (auto &col : info->create_info->columns.Logical()) { - names.push_back(col.GetName().GetIdentifierName()); - return_types.push_back(col.GetType()); - } - bind_data->names = info->postgres_names; - bind_data->types = return_types; - bind_data->can_use_main_thread = true; - bind_data->requires_materialization = false; - - PostgresScanFunction::PrepareBind(version, context, *bind_data, info->approx_num_pages); - return std::move(bind_data); +static unique_ptr PostgresBind(ClientContext &, TableFunctionBindInput &, vector &, + vector &) { + throw InternalException("PostgresBind"); } static bool ContainsCastToVarchar(const PostgresType &type) { @@ -224,55 +192,64 @@ static bool ContainsCastToVarchar(const PostgresType &type) { return false; } -static void PostgresInitInternal(ClientContext &context, const PostgresBindData *bind_data_p, - PostgresLocalState &lstate, idx_t task_min, idx_t task_max) { - D_ASSERT(bind_data_p); - D_ASSERT(task_min <= task_max); - - auto bind_data = (const PostgresBindData *)bind_data_p; - +static string WriteColumnNames(const PostgresBindData &bind_data, const PostgresLocalState &lstate, + bool &col_names_required) { string col_names; + col_names_required = bind_data.names.size() != lstate.column_ids.size(); for (auto &column_id : lstate.column_ids) { if (!col_names.empty()) { col_names += ", "; } if (column_id == COLUMN_IDENTIFIER_ROW_ID) { - if (bind_data->table_name.empty() || !bind_data->emit_ctid) { + if (bind_data.table_name.empty() || !bind_data.emit_ctid) { // count(*) over postgres_query col_names += "NULL"; } else { col_names += "ctid"; } + col_names_required = true; } else { - col_names += PostgresUtils::WriteIdentifier(bind_data->names[column_id]); - if (bind_data->postgres_types[column_id].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { + col_names += PostgresUtils::WriteIdentifier(bind_data.names[column_id]); + if (bind_data.postgres_types[column_id].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { col_names += "::VARCHAR"; - } else if (bind_data->types[column_id].id() == LogicalTypeId::LIST) { - if (bind_data->postgres_types[column_id].info != PostgresTypeAnnotation::STANDARD) { + col_names_required = true; + } else if (bind_data.types[column_id].id() == LogicalTypeId::LIST) { + if (bind_data.postgres_types[column_id].info != PostgresTypeAnnotation::STANDARD) { continue; } - if (bind_data->postgres_types[column_id].children[0].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { + if (bind_data.postgres_types[column_id].children[0].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { col_names += "::VARCHAR[]"; + col_names_required = true; } } else { - if (ContainsCastToVarchar(bind_data->postgres_types[column_id])) { + if (ContainsCastToVarchar(bind_data.postgres_types[column_id])) { throw NotImplementedException("Error reading table \"%s\" - cast to varchar not implemented for " "composite column \"%s\" (type %s)", - bind_data->table_name, bind_data->names[column_id], - bind_data->types[column_id].ToString()); + bind_data.table_name, bind_data.names[column_id], + bind_data.types[column_id].ToString()); } } } } + return col_names; +} - string filter_string = - PostgresFilterPushdown::TransformFilters(lstate.column_ids, lstate.filters, bind_data->names); +static void PostgresInitInternal(ClientContext &context, const PostgresBindData *bind_data_p, + PostgresLocalState &lstate, idx_t task_min, idx_t task_max) { + D_ASSERT(bind_data_p); + D_ASSERT(task_min <= task_max); + + auto bind_data = bind_data_p->Cast(); + + bool col_names_required = false; + string col_names = WriteColumnNames(bind_data, lstate, col_names_required); + string filter_string = PostgresFilterPushdown::TransformFilters(lstate.column_ids, lstate.filters, bind_data.names); string filter; lstate.exec = false; lstate.done = false; - if (bind_data->pages_approx > 0) { + if (bind_data.pages_approx > 0) { filter = StringUtil::Format("WHERE ctid BETWEEN '(%d,0)'::tid AND '(%d,0)'::tid", task_min, task_max); } if (!filter_string.empty()) { @@ -283,28 +260,28 @@ static void PostgresInitInternal(ClientContext &context, const PostgresBindData } filter += filter_string; } - string query; - if (bind_data->table_name.empty()) { - D_ASSERT(!bind_data->sql.empty()); - query = - StringUtil::Format(R"(SELECT %s FROM (%s) AS __unnamed_subquery %s)", col_names, bind_data->sql, filter); + string query; + if (bind_data.table_name.empty()) { + D_ASSERT(!bind_data.sql.empty()); + if (col_names_required) { + query = + StringUtil::Format("SELECT %s FROM (%s) AS __unnamed_subquery %s", col_names, bind_data.sql, filter); + } else if (!filter.empty()) { + query = StringUtil::Format("%s %s", bind_data.sql, filter); + } else { + query = bind_data.sql; + } } else { - query = StringUtil::Format(R"(SELECT %s FROM %s.%s %s)", col_names, - PostgresUtils::WriteIdentifier(bind_data->schema_name), - PostgresUtils::WriteIdentifier(bind_data->table_name), filter); + query = StringUtil::Format("SELECT %s FROM %s.%s %s", col_names, + PostgresUtils::WriteIdentifier(bind_data.schema_name), + PostgresUtils::WriteIdentifier(bind_data.table_name), filter); } - if (!bind_data->order_by_and_limit_bind_data.order_by_clause.empty()) { - query += bind_data->order_by_and_limit_bind_data.order_by_clause; - query += " NULLS LAST"; + if (!bind_data.limit_clause.empty()) { + query += bind_data.limit_clause; } - if (!bind_data->order_by_and_limit_bind_data.limit_clause.empty()) { - query += bind_data->order_by_and_limit_bind_data.limit_clause; - } - if (!bind_data->use_text_protocol) { - query = StringUtil::Format(R"(COPY (%s) TO STDOUT (FORMAT "binary");)", query); - } else { - query += ";"; + if (!(bind_data.use_text_protocol || bind_data.query_mode == PostgresQueryMode::PLAIN_QUERY_BINARY)) { + query = StringUtil::Format("COPY (%s) TO STDOUT (FORMAT \"binary\");", query); } lstate.sql = std::move(query); } @@ -343,45 +320,55 @@ static unique_ptr PostgresInitGlobalState(ClientContex TableFunctionInitInput &input) { auto &bind_data = input.bind_data->Cast(); auto result = make_uniq(PostgresMaxThreads(context, input.bind_data.get())); + optional_ptr pg_catalog = nullptr; + dbconnector::attached::AttachedCatalog attached_catalog; + D_ASSERT(!bind_data.catalog_name.empty()); + attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + pg_catalog = &attached_catalog.Get(); + + optional_ptr con_ptr = nullptr; + PostgresPoolConnection pinned_connection; + + if (bind_data.pinned_connection_id > 0) { + pinned_connection = pg_catalog->GetConnectionPool().UnpinConnection(bind_data.pinned_connection_id); + PostgresConnection &conn = pinned_connection.GetConnection(); + con_ptr = &conn; + } else { + auto &transaction = Transaction::Get(context, *pg_catalog).Cast(); + if (bind_data.use_transaction) { + con_ptr = &transaction.GetConnection(); + } else { + con_ptr = &transaction.GetConnectionWithoutTransaction(); + } + } + PostgresConnection &con = *con_ptr; + + result->SetConnection(con.GetConnection()); + if (pinned_connection) { + result->SetPinnedConnection(std::move(pinned_connection), std::move(attached_catalog)); + } + if (bind_data.command_only) { // A command (no result columns). Execute it here — at execution time, so EXPLAIN does not - // fire it — and hand back a single-row Success result through the materialized-collection + // fire it — and hand back a rowcount result through the materialized-collection // fast path (PostgresScan returns gstate.collection directly, and GetLocalState short-circuits // when a collection is present, so no postgres scan connection is opened). - auto command_catalog = bind_data.GetCatalog(); - D_ASSERT(command_catalog); - auto &transaction = Transaction::Get(context, *command_catalog).Cast(); - if (bind_data.use_transaction) { - transaction.Query(bind_data.sql); - } else { - transaction.QueryWithoutTransaction(bind_data.sql); - } - vector success_types {LogicalType::BOOLEAN}; - auto materialized = make_uniq(Allocator::Get(context), success_types); - DataChunk success_chunk; - success_chunk.Initialize(Allocator::Get(context), success_types); - success_chunk.SetChildCardinality(1); - success_chunk.data[0].SetValue(0, Value::BOOLEAN(true)); + unique_ptr pg_res = con.Query(context, bind_data.sql); + vector rowcount_types {LogicalType::BIGINT}; + auto materialized = make_uniq(Allocator::Get(context), rowcount_types); + DataChunk rowcount_chunk; + rowcount_chunk.Initialize(Allocator::Get(context), rowcount_types); + rowcount_chunk.SetChildCardinality(1); + int64_t *vec_data = FlatVector::GetDataMutable(rowcount_chunk.data[0]); + vec_data[0] = pg_res->AffectedRows(); ColumnDataAppendState append_state; materialized->InitializeAppend(append_state); - materialized->Append(append_state, success_chunk); + materialized->Append(append_state, rowcount_chunk); result->collection = std::move(materialized); result->collection->InitializeScan(result->scan_state); return std::move(result); } - auto pg_catalog = bind_data.GetCatalog(); - if (pg_catalog) { - auto &transaction = Transaction::Get(context, *pg_catalog).Cast(); - auto &con = - bind_data.use_transaction ? transaction.GetConnection() : transaction.GetConnectionWithoutTransaction(); - result->SetConnection(con.GetConnection()); - } else { - auto con = PostgresConnection::Open(bind_data.dsn, bind_data.attach_path); - if (bind_data.use_transaction) { - PostgresScanConnect(context, con, string(), AccessMode::READ_ONLY, PostgresIsolationLevel::REPEATABLE_READ); - } - result->SetConnection(std::move(con)); - } + if (bind_data.requires_materialization) { // if requires_materialization is enabled we scan and materialize the table in its entirety up-front vector types; @@ -408,7 +395,7 @@ static unique_ptr PostgresInitGlobalState(ClientContex result->collection->InitializeScan(result->scan_state); } else { // we create a transaction here, and get the snapshot id to enable transaction-safe parallelism - PostgresGetSnapshot(context, bind_data.version, bind_data, *result); + PostgresGetSnapshot(context, bind_data, *result); } return std::move(result); } @@ -437,7 +424,11 @@ static bool PostgresParallelStateNext(ClientContext &context, const FunctionData bool PostgresGlobalState::TryOpenNewConnection(ClientContext &context, PostgresLocalState &lstate, const PostgresBindData &bind_data) { - auto pg_catalog = bind_data.GetCatalog(); + optional_ptr pg_catalog = nullptr; + dbconnector::attached::AttachedCatalog attached_catalog; + D_ASSERT(!bind_data.catalog_name.empty()); + attached_catalog = PostgresCatalog::Lookup(context, bind_data.catalog_name); + pg_catalog = &attached_catalog.Get(); { lock_guard parallel_lock(lock); if (!used_main_thread) { @@ -448,29 +439,27 @@ bool PostgresGlobalState::TryOpenNewConnection(ClientContext &context, PostgresL // we HAVE to open a new connection { auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); - lstate.pool_connection = pg_catalog->GetConnectionPool().ForceGetConnection(); + lstate.pool_connection = pg_catalog->GetConnectionPool().ForceAcquire(); } lstate.connection = PostgresConnection(lstate.pool_connection.GetConnection().GetConnection()); + // unlike the main-thread connection this one is not in a transaction yet, so we pin it to + // the snapshot to prevent reads at its own instant + PostgresScanConnect(context, lstate.connection, snapshot, pg_catalog->access_mode, + pg_catalog->isolation_level); } used_main_thread = true; return true; } } - if (pg_catalog) { - { - auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); - if (!pg_catalog->GetConnectionPool().TryGetConnection(lstate.pool_connection)) { - return false; - } + { + auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); + if (!pg_catalog->GetConnectionPool().TryGetConnection(lstate.pool_connection)) { + return false; } - lstate.connection = PostgresConnection(lstate.pool_connection.GetConnection().GetConnection()); - PostgresScanConnect(context, lstate.connection, snapshot, pg_catalog->access_mode, pg_catalog->isolation_level); - } else { - lstate.connection = PostgresConnection::Open(bind_data.dsn, bind_data.attach_path); - PostgresScanConnect(context, lstate.connection, snapshot, AccessMode::READ_ONLY, - PostgresIsolationLevel::REPEATABLE_READ); } + lstate.connection = PostgresConnection(lstate.pool_connection.GetConnection().GetConnection()); + PostgresScanConnect(context, lstate.connection, snapshot, pg_catalog->access_mode, pg_catalog->isolation_level); return true; } @@ -500,6 +489,27 @@ static unique_ptr GetLocalState(ClientContext &context, return std::move(local_state); } +void PostgresGlobalState::SetPinnedConnection(PostgresPoolConnection pinned_connection, + dbconnector::attached::AttachedCatalog attached_catalog) { + if (this->pinned_connection) { + throw InternalException("Invalid global state: pinned connection alread initialized"); + } + this->pinned_connection = std::move(pinned_connection); + this->attached_catalog = std::move(attached_catalog); +} + +PostgresGlobalState::~PostgresGlobalState() { + connection.Close(); + if (pinned_connection) { + try { + PostgresCatalog &catalog = attached_catalog.Get(); + catalog.GetConnectionPool().PinConnection(std::move(pinned_connection)); + } catch (...) { + // suppress + } + } +} + static unique_ptr PostgresInitLocalState(ExecutionContext &context, TableFunctionInitInput &input, GlobalTableFunctionState *global_state) { @@ -542,7 +552,10 @@ static void PostgresScan(ClientContext &context, TableFunctionInput &data, DataC auto &gstate = data.global_state->Cast(); if (gstate.collection) { - gstate.collection->Scan(gstate.scan_state, output); + bool scan_happened = gstate.collection->Scan(gstate.scan_state, output); + if (!scan_happened) { + output.SetChildCardinality(0); + } return; } auto &local_state = data.local_state->Cast(); @@ -601,38 +614,30 @@ double PostgresScanProgress(ClientContext &context, const FunctionData *bind_dat } static void PostgresScanSerialize(Serializer &serializer, const optional_ptr bind_data_p, - const TableFunction &function) { + const BoundTableFunction &function) { throw NotImplementedException("PostgresScanSerialize"); } -static unique_ptr PostgresScanDeserialize(Deserializer &deserializer, TableFunction &function) { +static unique_ptr PostgresScanDeserialize(Deserializer &deserializer, BoundTableFunction &function) { throw NotImplementedException("PostgresScanDeserialize"); } -static BindInfo PostgresGetBindInfo(const optional_ptr bind_data_p) { - auto &bind_data = bind_data_p->Cast(); - auto table = bind_data.GetTable(); - BindInfo info(ScanType::EXTERNAL); - info.table = table.get(); - return info; +static optional_ptr PostgresGetTableEntry(optional_ptr bind_data_p) { + auto &bdata = bind_data_p->Cast(); + shared_ptr ctx = bdata.context_ptr.lock(); + if (!ctx) { // cannot fail in known scenarios + return nullptr; + } + auto attached_table = PostgresTableEntry::Lookup(*ctx, bdata.qualified_table_name); + return attached_table.Get(); } PostgresScanFunction::PostgresScanFunction() - : TableFunction("postgres_scan", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, PostgresScan, - PostgresBind, PostgresInitGlobalState, PostgresInitLocalState) { - to_string = PostgresScanToString; - serialize = PostgresScanSerialize; - deserialize = PostgresScanDeserialize; - get_partition_data = PostgresGetPartitionData; - cardinality = PostgresScanCardinality; - table_scan_progress = PostgresScanProgress; - get_bind_info = PostgresGetBindInfo; - projection_pushdown = true; - global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; -} - -PostgresScanFunctionFilterPushdown::PostgresScanFunctionFilterPushdown() - : TableFunction("postgres_scan_pushdown", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + : TableFunction("postgres_scan", + FunctionSignature() + .AddPositionalOnly("dsn", LogicalType::VARCHAR) + .AddPositionalOnly("schema_name", LogicalType::VARCHAR) + .AddPositionalOnly("table_name", LogicalType::VARCHAR), PostgresScan, PostgresBind, PostgresInitGlobalState, PostgresInitLocalState) { to_string = PostgresScanToString; serialize = PostgresScanSerialize; @@ -640,9 +645,8 @@ PostgresScanFunctionFilterPushdown::PostgresScanFunctionFilterPushdown() get_partition_data = PostgresGetPartitionData; cardinality = PostgresScanCardinality; table_scan_progress = PostgresScanProgress; - get_bind_info = PostgresGetBindInfo; + get_table_entry = PostgresGetTableEntry; projection_pushdown = true; - filter_pushdown = true; global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; } diff --git a/src/postgres_secrets.cpp b/src/postgres_secrets.cpp index 8d3dddc55..02f268c16 100644 --- a/src/postgres_secrets.cpp +++ b/src/postgres_secrets.cpp @@ -1,4 +1,5 @@ #include "postgres_secrets.hpp" +#include "duckdb/main/client_context.hpp" #include #include diff --git a/src/postgres_storage.cpp b/src/postgres_storage.cpp index 23a0cf1c6..a7d8779cf 100644 --- a/src/postgres_storage.cpp +++ b/src/postgres_storage.cpp @@ -1,13 +1,115 @@ #include "duckdb.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/main/config.hpp" #include "duckdb/main/settings.hpp" #include "postgres_storage.hpp" #include "storage/postgres_catalog.hpp" #include "duckdb/parser/parsed_data/attach_info.hpp" #include "storage/postgres_transaction_manager.hpp" +#include "utf8proc_wrapper.hpp" namespace duckdb { +static vector ExtractSchemas(Value &value) { + if (value.IsNull()) { + throw BinderException("Value for \"SCHEMA\" option must not be null"); + } + switch (value.type().id()) { + case LogicalTypeId::VARCHAR: { + vector res; + const string &name = StringValue::Get(value); + if (name.empty()) { + throw BinderException("Value \"SCHEMA\" option must be not empty"); + } + res.push_back(name); + return res; + } + case LogicalTypeId::LIST: { + if (ListType::GetChildType(value.type()).id() != LogicalTypeId::VARCHAR) { + throw BinderException( + "Value for \"SCHEMA\" option must be either \"VARCHAR\" or \"VARCHAR[]\", was: \"%s\"", + value.type().ToString()); + } + vector res; + for (const Value &en : ListValue::GetChildren(value)) { + if (en.IsNull()) { + throw BinderException("Values for \"SCHEMA\" option must not be null"); + } + const string &name = StringValue::Get(en); + if (name.empty()) { + throw BinderException("Values \"SCHEMA\" option must be not empty"); + } + res.push_back(name); + } + return res; + } + default: + throw BinderException("Value for `SCHEMA` option must be either \"VARCHAR\" or \"VARCHAR[]\", was: \"%s\"", + value.type().ToString()); + } +} + +//! The connect display is rendered verbatim into the shell prompt, so reject anything that could take over the +//! terminal or trip up the code that renders it +static string ExtractConnectDisplay(Value &value) { + if (value.IsNull()) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must not be null"); + } + // generous upper bound - this is only here to keep pathological input from making the prompt unusable, + // every realistic endpoint is far below it + const idx_t max_width = 1024; + auto display = value.ToString(); + // this has to happen before anything below inspects the string - RenderWidth walks UTF-8 sequences and + // relies on them being well formed + if (!Utf8Proc::IsValid(display.c_str(), display.size())) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must be valid UTF-8"); + } + for (idx_t i = 0; i < display.size(); i++) { + auto c = static_cast(display[i]); + // C0 control characters and DEL + bool is_control = c < 0x20 || c == 0x7f; + // C1 control characters - these are encoded as 0xC2 0x80 through 0xC2 0x9F in UTF-8 + if (c == 0xc2 && i + 1 < display.size()) { + auto next = static_cast(display[i + 1]); + is_control = is_control || (next >= 0x80 && next <= 0x9f); + } + if (is_control) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must not contain control characters - the " + "value is rendered directly into the shell prompt"); + } + } + auto width = Utf8Proc::RenderWidth(display); + if (width > max_width) { + throw BinderException("Value for \"CONNECT_DISPLAY\" option must be at most %d columns wide, was %d", max_width, + idx_t(width)); + } + return display; +} + +static PostgresTextProtocolMode ExtractTextProtocolMode(Value &value) { + if (value.IsNull()) { + throw BinderException("Value for \"USE_TEXT_PROTOCOL\" option must not be null"); + } + if (value.type().id() == LogicalTypeId::BOOLEAN) { + return BooleanValue::Get(value) ? PostgresTextProtocolMode::TEXT : PostgresTextProtocolMode::BINARY; + } + auto param = value.ToString(); + auto lparam = StringUtil::Lower(param); + if (lparam == "auto") { + return PostgresTextProtocolMode::AUTO; + } + if (lparam == "true") { + return PostgresTextProtocolMode::TEXT; + } + if (lparam == "false") { + return PostgresTextProtocolMode::BINARY; + } + throw BinderException("Invalid value \"%s\" for use_text_protocol, expected TRUE, FALSE or AUTO", param); +} + static unique_ptr PostgresAttach(optional_ptr storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AttachOptions &attach_options) { @@ -18,16 +120,19 @@ static unique_ptr PostgresAttach(optional_ptr sto string attach_path = info.path; string secret_name; - string schema_to_load; + vector schemas_to_load; PostgresIsolationLevel isolation_level = PostgresIsolationLevel::REPEATABLE_READ; + PostgresTextProtocolMode text_protocol_mode = PostgresTextProtocolMode::AUTO; string secret_storage_table_name; bool secret_storage_table_specified_explicitly = false; + string connect_display; + PostgresUseCtidScan use_ctid_scan = PostgresUseCtidScan::DEFAULT; for (auto &entry : attach_options.options) { auto lower_name = StringUtil::Lower(entry.first); if (lower_name == "secret") { secret_name = entry.second.ToString(); } else if (lower_name == "schema") { - schema_to_load = entry.second.ToString(); + schemas_to_load = ExtractSchemas(entry.second); } else if (lower_name == "isolation_level") { auto param = entry.second.ToString(); auto lparam = StringUtil::Lower(param); @@ -38,22 +143,32 @@ static unique_ptr PostgresAttach(optional_ptr sto } else if (lparam == "serializable") { isolation_level = PostgresIsolationLevel::SERIALIZABLE; } else { - throw InvalidInputException("Invalid value \"%s\" for isolation_level, expected READ COMMITTED, " - "REPEATABLE READ or SERIALIZABLE", - param); + throw BinderException("Invalid value \"%s\" for isolation_level, expected READ COMMITTED, " + "REPEATABLE READ or SERIALIZABLE", + param); } + } else if (lower_name == "use_text_protocol") { + text_protocol_mode = ExtractTextProtocolMode(entry.second); } else if (lower_name == "secret_storage_table") { secret_storage_table_name = entry.second.ToString(); secret_storage_table_specified_explicitly = true; + } else if (lower_name == "connect_display") { + connect_display = ExtractConnectDisplay(entry.second); + } else if (lower_name == "use_ctid_scan") { + use_ctid_scan = BooleanValue::Get(entry.second.DefaultCastAs(LogicalType::BOOLEAN)) + ? PostgresUseCtidScan::ENABLED + : PostgresUseCtidScan::DISABLED; } else { throw BinderException("Unrecognized option for Postgres attach: %s", entry.first); } } SecretStorageTable secret_storage_table(std::move(secret_storage_table_name), secret_storage_table_specified_explicitly); - return make_uniq(context, db, std::move(attach_path), attach_options.access_mode, - std::move(schema_to_load), isolation_level, secret_name, - std::move(secret_storage_table)); + auto catalog = make_uniq( + context, db, std::move(attach_path), attach_options.access_mode, std::move(schemas_to_load), isolation_level, + secret_name, std::move(secret_storage_table), text_protocol_mode, std::move(connect_display)); + catalog->use_ctid_scan = use_ctid_scan; + return catalog; } static unique_ptr PostgresCreateTransactionManager(optional_ptr storage_info, diff --git a/src/postgres_text_reader.cpp b/src/postgres_text_reader.cpp index f1c70640f..daf8c4f32 100644 --- a/src/postgres_text_reader.cpp +++ b/src/postgres_text_reader.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/common/vector/map_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" #include "duckdb/common/vector/struct_vector.hpp" @@ -392,8 +393,8 @@ PostgresReadResult PostgresTextReader::Read(DataChunk &output) { scan_chunk.Initialize(context, types); } scan_chunk.Reset(); - for (; scan_chunk.size() < STANDARD_VECTOR_SIZE && row_offset < result->Count(); row_offset++) { - idx_t output_offset = scan_chunk.size(); + idx_t output_offset = 0; + for (; output_offset < STANDARD_VECTOR_SIZE && row_offset < result->Count(); row_offset++, output_offset++) { for (idx_t output_idx = 0; output_idx < output.ColumnCount(); output_idx++) { auto col_idx = column_ids[output_idx]; auto &out_vec = scan_chunk.data[output_idx]; @@ -405,8 +406,8 @@ PostgresReadResult PostgresTextReader::Read(DataChunk &output) { col_data[output_offset] = StringVector::AddStringOrBlob(out_vec, result->GetStringRef(row_offset, output_idx)); } - scan_chunk.SetChildCardinality(scan_chunk.size() + 1); } + scan_chunk.SetChildCardinality(output_offset); for (idx_t c = 0; c < output.ColumnCount(); c++) { auto col_idx = column_ids[c]; if (col_idx == COLUMN_IDENTIFIER_ROW_ID) { diff --git a/src/postgres_utils.cpp b/src/postgres_utils.cpp index e60bccd4a..dd1f98ebd 100644 --- a/src/postgres_utils.cpp +++ b/src/postgres_utils.cpp @@ -1,31 +1,92 @@ #include "postgres_utils.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "dbconnector/query/query_writer.hpp" +#include "storage/postgres_catalog.hpp" #include "storage/postgres_schema_entry.hpp" #include "storage/postgres_transaction.hpp" +#include "postgres_scanner.hpp" #include "postgres_type_oids.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/geometry_crs.hpp" namespace duckdb { +mutex PostgresUtils::libpq_init_lock; +bool PostgresUtils::libpq_connect_succeeded_at_least_once = false; + static void PGNoticeProcessor(void *arg, const char *message) { } PGconn *PostgresUtils::PGConnect(const string &dsn, const string &attach_path) { + bool lock_needed = true; + { + lock_guard guard(libpq_init_lock); + if (libpq_connect_succeeded_at_least_once) { + lock_needed = false; + } + } + + if (lock_needed) { + lock_guard guard(libpq_init_lock); + idx_t max_attempts = 2; + for (idx_t i = 0; i < max_attempts; i++) { // observed sporadic OpenSSL failures on the first connection + PostgresConnectCheck check = i < max_attempts - 1 ? PostgresConnectCheck::RETURN_ON_CONNECTION_BAD + : PostgresConnectCheck::THROW_ON_CONNECTION_BAD; + PGconn *conn = PGConnectInternal(dsn, attach_path, check); + if (PQstatus(conn) == CONNECTION_BAD) { + PQfinish(conn); + continue; + } + libpq_connect_succeeded_at_least_once = true; + return conn; + } + } + + return PGConnectInternal(dsn, attach_path, PostgresConnectCheck::THROW_ON_CONNECTION_BAD); +} + +PGconn *PostgresUtils::PGConnectInternal(const string &dsn, const string &attach_path, + PostgresConnectCheck check_behaviour) { PGconn *conn = PQconnectdb(dsn.c_str()); // both PQStatus and PQerrorMessage check for nullptr if (PQstatus(conn) == CONNECTION_BAD) { - char *msg_cstr = PQerrorMessage(conn); - string msg = msg_cstr != nullptr ? string(msg_cstr) : string(); - PQfinish(conn); - throw IOException("Unable TODO:REMOVEME to connect to Postgres at \"%s\": %s", attach_path, msg); + if (check_behaviour == PostgresConnectCheck::THROW_ON_CONNECTION_BAD) { + char *msg_cstr = PQerrorMessage(conn); + string msg = msg_cstr != nullptr ? string(msg_cstr) : string(); + PQfinish(conn); + throw IOException("Unable to connect to Postgres at \"%s\": %s", attach_path, msg); + } else { + return conn; + } } PQsetNoticeProcessor(conn, PGNoticeProcessor, nullptr); return conn; } string PostgresUtils::TypeToString(const LogicalType &input) { + // Handle GEOMETRY('EPSG:N') first: the CRS-bearing GEOMETRY type can have + // an alias of "GEOMETRY", which would otherwise short-circuit the alias + // branch below and emit an untyped `geometry` column (typmod -1). When + // CRS is present, emit a typmod-bearing PostGIS column type so the SRID + // survives a CREATE TABLE round trip; otherwise fall through. + if (input.id() == LogicalTypeId::GEOMETRY && GeoType::HasCRS(input)) { + auto &crs = GeoType::GetCRS(input); + auto &id = crs.GetIdentifier(); + auto colon = id.find(':'); + if (colon != string::npos) { + try { + int32_t srid = std::stoi(id.substr(colon + 1)); + if (srid > 0) { + return "geometry(Geometry, " + std::to_string(srid) + ")"; + } + } catch (...) { + // fall through to plain GEOMETRY + } + } + } if (input.HasAlias()) { return input.GetAlias(); } @@ -78,6 +139,69 @@ LogicalType PostgresUtils::RemoveAlias(const LogicalType &type) { } } +bool PostgresUtils::UseInformationSchemaIntrospection(optional_ptr context) { + if (!context) { + return false; + } + Value value; + if (context->TryGetCurrentSetting("pg_use_information_schema_introspection", value)) { + return BooleanValue::Get(value); + } + return false; +} + +bool PostgresUtils::StalenessQueryEnabled(ClientContext &context) { + Value value; + if (context.TryGetCurrentSetting("pg_staleness_query_enabled", value) && !value.IsNull()) { + return BooleanValue::Get(value); + } + return !UseInformationSchemaIntrospection(&context); +} + +string PostgresUtils::GetCustomStalenessQuery(ClientContext &context) { + Value value; + if (context.TryGetCurrentSetting("pg_staleness_query", value) && !value.IsNull()) { + return StringValue::Get(value); + } + return string(); +} + +string PostgresUtils::DataTypeToTypeName(const string &data_type) { + // Map SQL-standard information_schema.columns.data_type values to the + // pg_type.typname + auto type_name = StringUtil::Lower(data_type); + if (type_name == "boolean") { + return "bool"; + } else if (type_name == "smallint") { + return "int2"; + } else if (type_name == "integer") { + return "int4"; + } else if (type_name == "bigint") { + return "int8"; + } else if (type_name == "real") { + return "float4"; + } else if (type_name == "double precision") { + return "float8"; + } else if (type_name == "character varying") { + return "varchar"; + } else if (type_name == "character") { + return "bpchar"; + } else if (type_name == "\"char\"") { + return "char"; + } else if (type_name == "time without time zone") { + return "time"; + } else if (type_name == "time with time zone") { + return "timetz"; + } else if (type_name == "timestamp without time zone") { + return "timestamp"; + } else if (type_name == "timestamp with time zone") { + return "timestamptz"; + } else if (type_name == "bit varying") { + return "varbit"; + } + return data_type; +} + uint32_t PostgresUtils::TypeNameToPostgresOid(const string &type_name) { if (type_name == "bool") { return BOOLOID; @@ -117,25 +241,35 @@ uint32_t PostgresUtils::TypeNameToPostgresOid(const string &type_name) { return 0; } +PostgresTypeConfig PostgresTypeConfig::FromContext(optional_ptr context) { + PostgresTypeConfig result; + if (!context) { + return result; + } + Value setting; + if (context->TryGetCurrentSetting("pg_array_as_varchar", setting)) { + result.array_as_varchar = BooleanValue::Get(setting); + } + if (context->TryGetCurrentSetting("pg_numeric_as_varchar", setting)) { + result.numeric_as_varchar = BooleanValue::Get(setting); + } + if (context->TryGetCurrentSetting("pg_numeric_nan_as_null", setting)) { + result.numeric_nan_as_null = BooleanValue::Get(setting); + } + return result; +} + LogicalType PostgresUtils::TypeToLogicalType(optional_ptr transaction, optional_ptr schema, - const PostgresTypeData &type_info, PostgresType &postgres_type) { + const PostgresTypeConfig &type_config, const PostgresTypeData &type_info, + PostgresType &postgres_type) { auto &pgtypename = type_info.type_name; // postgres array types start with an _ if (StringUtil::StartsWith(pgtypename, "_")) { - if (transaction) { - auto context = transaction->GetContext(); - if (!context) { - throw InternalException("Context is destroyed!?"); - } - Value array_as_varchar; - if (context->TryGetCurrentSetting("pg_array_as_varchar", array_as_varchar)) { - if (BooleanValue::Get(array_as_varchar)) { - postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; - return LogicalType::VARCHAR; - } - } + if (type_config.array_as_varchar) { + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; } // get the array dimension information idx_t dimensions = type_info.array_dimensions; @@ -148,7 +282,8 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t child_type_info.type_modifier = type_info.type_modifier; child_type_info.type_schema = type_info.type_schema; PostgresType child_pg_type; - auto child_type = PostgresUtils::TypeToLogicalType(transaction, schema, child_type_info, child_pg_type); + auto child_type = + PostgresUtils::TypeToLogicalType(transaction, schema, type_config, child_type_info, child_pg_type); // populate the child OID from the actual Postgres type name if (child_pg_type.oid == 0) { child_pg_type.oid = TypeNameToPostgresOid(child_type_info.type_name); @@ -183,7 +318,10 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t auto width = ((type_info.type_modifier - sizeof(int32_t)) >> 16) & 0xffff; auto scale = (((type_info.type_modifier - sizeof(int32_t)) & 0x7ff) ^ 1024) - 1024; if (type_info.type_modifier == -1 || width < 0 || scale < 0 || width > 38) { - // fallback to double + if (type_config.numeric_as_varchar) { + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; + } postgres_type.info = PostgresTypeAnnotation::NUMERIC_AS_DOUBLE; return LogicalType::DOUBLE; } @@ -197,6 +335,20 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t postgres_type.info = PostgresTypeAnnotation::JSONB; return LogicalType::VARCHAR; } else if (pgtypename == "geometry") { + // PostGIS encodes the column-level SRID in the type modifier of + // `geometry(TYPE, SRID)` columns. The bit layout is: + // bits 0 : has-M + // bits 1 : has-Z + // bits 2-7 : geometry type code (POINT=1, LINESTRING=2, ...) + // bits 8-29 : SRID + // Untyped `geometry` columns carry typmod -1 and we fall back to + // plain GEOMETRY (no CRS), matching the previous behavior. + if (type_info.type_modifier > 0) { + int32_t srid = static_cast((static_cast(type_info.type_modifier) & 0x0FFFFF00u) >> 8); + if (srid > 0) { + return LogicalType::GEOMETRY("EPSG:" + std::to_string(srid)); + } + } return LogicalType::GEOMETRY(); } else if (pgtypename == "date") { return LogicalType::DATE; @@ -238,34 +390,44 @@ LogicalType PostgresUtils::TypeToLogicalType(optional_ptr t } else if (pgtypename == "circle") { postgres_type.info = PostgresTypeAnnotation::GEOM_CIRCLE; return LogicalType::LIST(LogicalType::DOUBLE); + } + + if (!transaction || !schema || type_info.type_schema.empty()) { + // unsupported, or no type schema to resolve custom types in + // (information_schema introspection) - fallback to varchar + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; + } + + auto context = transaction->GetContext(); + if (!context) { + throw InternalException("Context is destroyed!?"); + } + + Identifier type_schema = Identifier(type_info.type_schema); + optional_ptr lookup_schema; + if (type_schema == schema->name) { + lookup_schema = schema.get(); } else { - if (!transaction) { - // unsupported so fallback to varchar - postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; - return LogicalType::VARCHAR; - } - auto context = transaction->GetContext(); - if (!context) { - throw InternalException("Context is destroyed!?"); - } - optional_ptr lookup_schema = - Identifier(type_info.type_schema) != schema->name - ? schema->ParentCatalog().GetSchema(*context, Identifier(type_info.type_schema), - OnEntryNotFound::THROW_EXCEPTION) - : schema.get(); + Catalog &parent_catalog = schema->ParentCatalog(); + lookup_schema = parent_catalog.GetSchema(*context, type_schema, OnEntryNotFound::RETURN_NULL); + } + + if (lookup_schema) { auto entry = lookup_schema->GetEntry(CatalogTransaction(lookup_schema->ParentCatalog(), *context), CatalogType::TYPE_ENTRY, Identifier(pgtypename)); - if (!entry) { - // unsupported so fallback to varchar - postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; - return LogicalType::VARCHAR; + if (entry) { + // custom type (e.g. composite or enum) + auto &type_entry = entry->Cast(); + auto result_type = RemoveAlias(type_entry.user_type); + postgres_type = type_entry.postgres_type; + return result_type; } - // custom type (e.g. composite or enum) - auto &type_entry = entry->Cast(); - auto result_type = RemoveAlias(type_entry.user_type); - postgres_type = type_entry.postgres_type; - return result_type; } + + // either schema or type is not available so fallback to varchar + postgres_type.info = PostgresTypeAnnotation::CAST_TO_VARCHAR; + return LogicalType::VARCHAR; } LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { @@ -297,9 +459,7 @@ LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { auto &type = StructType::GetChildType(input, c); new_types.push_back(make_pair(name, ToPostgresType(type))); } - auto result = LogicalType::STRUCT(std::move(new_types)); - result.SetAlias(input.GetAlias()); - return result; + return LogicalType::STRUCT(std::move(new_types)).WithAlias(input.GetAlias()); } case LogicalTypeId::TIMESTAMP_SEC: case LogicalTypeId::TIMESTAMP_MS: @@ -316,7 +476,11 @@ LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { case LogicalTypeId::HUGEINT: return LogicalType::DOUBLE; case LogicalTypeId::GEOMETRY: - return LogicalType::GEOMETRY(); + // Preserve CRS metadata so downstream TypeToString can emit a typmod- + // bearing PostGIS column type and the SRID survives a CREATE TABLE + // AS round trip. Returning a CRS-less GEOMETRY here strips the CRS + // before it ever reaches the schema-emission path. + return input; default: return LogicalType::VARCHAR; } @@ -549,36 +713,92 @@ string PostgresUtils::QuotePostgresIdentifier(const string &text) { return PostgresUtils::WriteIdentifier(text); } -string PostgresUtils::EscapeConnectionString(const string &input) { - using namespace dbconnector::query; - auto config = QueryWriter::CreateConfig('\'', QuoteEscapeStyle::BACKSLASH); - return QueryWriter::WriteQuotedAndEscaped(config, input); -} - -string PostgresUtils::ExtractConnectionOption(const KeyValueSecret &kv_secret, const string &name) { - Value input_val = kv_secret.TryGetValue(Identifier(name)); - if (input_val.IsNull()) { - // not provided - return string(); - } - string result; - result += name; - result += "="; - result += EscapeConnectionString(input_val.ToString()); - result += " "; - return result; -} - string PostgresUtils::WriteLiteral(const string &literal) { using namespace dbconnector::query; auto config = QueryWriter::CreateConfig('\'', QuoteEscapeStyle::DOUBLE_QUOTE); return QueryWriter::WriteQuotedAndEscaped(config, literal); } +string PostgresUtils::WriteLiteralsCommaSeparated(const vector &literals) { + string res; + for (idx_t i = 0; i < literals.size(); i++) { + const string &lit = literals[i]; + if (i > 0) { + res.append(", "); + } + res.append(WriteLiteral(lit)); + } + return res; +} + string PostgresUtils::WriteIdentifier(const string &identifier) { using namespace dbconnector::query; auto config = QueryWriter::CreateConfig('"', QuoteEscapeStyle::DOUBLE_QUOTE); return QueryWriter::WriteQuotedAndEscaped(config, identifier); } +PostgresQueryMode PostgresUtils::ExtractQueryMode(ClientContext &ctx) { + Value val; + if (ctx.TryGetCurrentSetting("pg_query_mode", val)) { + return ExtractQueryMode(val); + } + return PostgresQueryMode::AUTO; +} + +PostgresQueryMode PostgresUtils::ExtractQueryMode(Value &value) { + if (value.IsNull()) { + throw InvalidInputException("Invalid NULL query mode specified"); + } + string str_raw = value.ToString(); + string str = StringUtil::Upper(str_raw); + if (str == "AUTO") { + return PostgresQueryMode::AUTO; + } else if (str == "COPY_BINARY") { + return PostgresQueryMode::COPY_BINARY; + + } else if (str == "PLAIN_QUERY_BINARY") { + return PostgresQueryMode::PLAIN_QUERY_BINARY; + + } else if (str == "PLAIN_QUERY_TEXT") { + return PostgresQueryMode::PLAIN_QUERY_TEXT; + + } else { + throw InvalidInputException("Invalid query mode specified, must be on of: 'AUTO', 'COPY_BINARY', " + "'PLAIN_QUERY_BINARY', 'PLAIN_QUERY_TEXT'"); + } +} + +PostgresCatalog &PostgresUtils::GetCatalogByName(ClientContext &context, const std::string &name) { + auto &db_manager = DatabaseManager::Get(context); + auto db = db_manager.GetDatabase(context, Identifier(name)); + if (!db) { + throw BinderException("Failed to find attached database \"%s\"", name); + } + auto &catalog = db->GetCatalog(); + if (catalog.GetCatalogType() != "postgres") { + throw BinderException("Attached database \"%s\" does not refer to a Postgres database", name); + } + return catalog.Cast(); +} + +void PostgresUtils::DeduplicateColumns(vector &names) { + identifier_map_t name_map; + for (auto &column_name : names) { + if (name_map.find(column_name) == name_map.end()) { + // Name does not exist yet + name_map[column_name]++; + } else { + // Name already exists, we add _x where x is the repetition number + Identifier new_column_name(column_name + "_" + std::to_string(name_map[column_name])); + while (name_map.find(new_column_name) != name_map.end()) { + // This name is already here due to a previous definition + name_map[column_name]++; + new_column_name = Identifier(column_name + "_" + std::to_string(name_map[column_name])); + } + column_name = new_column_name; + name_map[new_column_name]++; + } + } +} + } // namespace duckdb diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 24c903a85..1af878ac7 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -4,7 +4,6 @@ add_library( postgres_catalog_set.cpp postgres_connection_pool.cpp postgres_clear_cache.cpp - postgres_configure_pool.cpp postgres_delete.cpp postgres_index.cpp postgres_index_entry.cpp @@ -18,7 +17,6 @@ add_library( postgres_table_entry.cpp postgres_table_set.cpp postgres_transaction.cpp - postgres_transaction_manager.cpp postgres_type_entry.cpp postgres_type_set.cpp postgres_update.cpp) diff --git a/src/storage/postgres_catalog.cpp b/src/storage/postgres_catalog.cpp index 64c6305db..d52aab974 100644 --- a/src/storage/postgres_catalog.cpp +++ b/src/storage/postgres_catalog.cpp @@ -1,32 +1,49 @@ #include "storage/postgres_catalog.hpp" -#include "storage/postgres_schema_entry.hpp" -#include "storage/postgres_transaction.hpp" -#include "postgres_oauth.hpp" -#include "postgres_connection.hpp" -#include "postgres_secrets.hpp" -#include "storage/postgres_secret_storage.hpp" -#include "postgres_utils.hpp" + +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "duckdb/storage/database_size.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/comparison_expression.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/expression/function_expression.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" #include "duckdb/main/attached_database.hpp" #include "duckdb/main/secret/secret_manager.hpp" +#include "postgres_connection_string.hpp" +#include "postgres_connection.hpp" +#include "postgres_oauth.hpp" +#include "postgres_secrets.hpp" +#include "postgres_utils.hpp" +#include "storage/postgres_schema_entry.hpp" +#include "storage/postgres_transaction.hpp" +#include "storage/postgres_secret_storage.hpp" + namespace duckdb { unique_ptr PostgresCatalog::RemoteExecute(ClientContext &context, const string &sql) { vector> args; - args.push_back(make_uniq(Value(GetName()))); - args.push_back(make_uniq(Value(sql))); + args.push_back(ConstantExpression::FromValue(Value(GetName()))); + args.push_back(ConstantExpression::FromValue(Value(sql))); + args.push_back(make_uniq(ExpressionType::COMPARE_EQUAL, + make_uniq("suppress_dml_output"), + ConstantExpression::FromValue(Value::BOOLEAN(true)))); auto func_ref = make_uniq(); func_ref->function = make_uniq("postgres_query", std::move(args)); return func_ref; } string PostgresCatalog::GetConnectDisplay() { + if (!connect_display.empty()) { + // an explicit display string was provided in the ATTACH - this is used by extensions that attach + // Postgres compatible databases (such as AWS RDS or Redshift) through this extension, so that the + // prompt can show how the database was attached instead of the Postgres connection we ended up with + return connect_display; + } // `postgres://[:]` — standard postgres URI shorthand. dbname is already shown // via current_database in the prompt, so it's omitted here. string host, port; @@ -68,11 +85,15 @@ unique_ptr GetSecret(ClientContext &context, const string &secret_n } PostgresCatalog::PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, string attach_path_p, - AccessMode access_mode, string schema_to_load, PostgresIsolationLevel isolation_level, - const string &secret_name, SecretStorageTable secret_storage_table_p) + AccessMode access_mode, vector schemas_to_load, + PostgresIsolationLevel isolation_level, const string &secret_name, + SecretStorageTable secret_storage_table_p, PostgresTextProtocolMode text_protocol_mode, + string connect_display_p) : Catalog(db_p), attach_path(std::move(attach_path_p)), access_mode(access_mode), isolation_level(isolation_level), - schemas(*this, schema_to_load), connection_pool(make_shared_ptr(*this, ctx)), - default_schema(schema_to_load), secret_storage_table(std::move(secret_storage_table_p)) { + text_protocol_mode(text_protocol_mode), schemas(*this, schemas_to_load), + connection_pool(make_shared_ptr(*this, ctx)), + default_schema(schemas_to_load.size() > 0 ? Identifier(schemas_to_load[0]) : Identifier()), + secret_storage_table(std::move(secret_storage_table_p)), connect_display(std::move(connect_display_p)) { auto secret_entry = GetSecretEntry(ctx, secret_name); this->rds_token_config = PostgresAws::ExtractTokenConfigFromSecret(secret_entry); if (!rds_token_config.Enabled()) { @@ -86,7 +107,7 @@ PostgresCatalog::PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, str PostgresPoolConnection connection; { auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(ctx); - connection = connection_pool->GetConnection(); + connection = connection_pool->Acquire(); } this->version = connection.GetConnection().GetPostgresVersion(ctx); } @@ -109,6 +130,9 @@ unique_ptr PostgresCatalog::GetSecretEntry(ClientContext &ctx, cons string PostgresCatalog::CreateConnectionString(optional_ptr secret_entry, const string &attach_path) { string connection_string = attach_path; + bool user_specified_app_name = PostgresConnectionString::ContainsApplicationName(connection_string); + bool secret_has_app_name = false; + string secret_connection_info; if (secret_entry) { // secret found - read data const auto &kv_secret = dynamic_cast(*secret_entry->secret); @@ -128,15 +152,34 @@ string PostgresCatalog::CreateConnectionString(optional_ptr secret_ throw BinderException("ATTACH path must be empty when 'URI' option is specified, was: \"%s\"", attach_path); } - return uri_val.ToString(); + string uri = uri_val.ToString(); + bool uri_has_app_name = PostgresConnectionString::ContainsApplicationName(uri); + if (!uri_has_app_name) { + PostgresConnectionString::AppendDefaultApplicationName(uri, PostgresConnectionStringFormat::URI); + } + return uri; } - string new_connection_info; + secret_has_app_name = !kv_secret.TryGetValue("application_name").IsNull(); + for (const string &opt_name : PostgresSecrets::ConnectionOptionNames()) { - new_connection_info += PostgresUtils::ExtractConnectionOption(kv_secret, opt_name); + secret_connection_info += PostgresConnectionString::ExtractOption(kv_secret, opt_name); + } + } + + if (!(user_specified_app_name || secret_has_app_name)) { + if (PostgresConnectionString::IsURI(connection_string)) { + // libpq cannot read key/value pairs in front of a URI, so the name goes into the URI + PostgresConnectionString::AppendDefaultApplicationName(connection_string, + PostgresConnectionStringFormat::URI); + } else { + PostgresConnectionString::AppendDefaultApplicationName(secret_connection_info, + PostgresConnectionStringFormat::KEY_VALUE); + secret_connection_info.push_back(' '); } - connection_string = new_connection_info + connection_string; } + connection_string = secret_connection_info + connection_string; + return connection_string; } @@ -161,7 +204,7 @@ string PostgresCatalog::GetConnectionString() { this->rds_token = PostgresAws::GenerateRdsAuthToken(GetAttached(), rds_token_config); this->rds_token_last_refreshed = now; this->connection_string = rds_token_config.base_connection_string + - "password=" + PostgresUtils::EscapeConnectionString(rds_token) + " " + attach_path; + "password=" + PostgresConnectionString::Escape(rds_token) + " " + attach_path; } return std::string(connection_string.data(), connection_string.length()); } @@ -175,13 +218,14 @@ void PostgresCatalog::Initialize(bool load_builtin) { optional_ptr PostgresCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { auto &postgres_transaction = PostgresTransaction::Get(transaction.GetContext(), *this); - auto entry = schemas.GetEntry(transaction.GetContext(), postgres_transaction, info.schema.GetIdentifierName()); + auto entry = schemas.GetEntry(transaction.GetContext(), postgres_transaction, + info.GetQualifiedName().Schema().GetIdentifierName()); if (entry) { switch (info.on_conflict) { case OnCreateConflict::REPLACE_ON_CONFLICT: { DropInfo try_drop; try_drop.type = CatalogType::SCHEMA_ENTRY; - try_drop.name = info.schema; + try_drop.SetName(info.GetQualifiedName().Schema()); try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; try_drop.cascade = false; schemas.DropEntry(postgres_transaction, try_drop); @@ -191,7 +235,8 @@ optional_ptr PostgresCatalog::CreateSchema(CatalogTransaction tran return entry; case OnCreateConflict::ERROR_ON_CONFLICT: default: - throw BinderException("Failed to create schema \"%s\": schema already exists", info.schema); + throw BinderException("Failed to create schema \"%s\": schema already exists", + info.GetQualifiedName().Schema()); } } return schemas.CreateSchema(postgres_transaction, info); @@ -256,7 +301,7 @@ void PostgresCatalog::RegisterSecretStorage() { } // Look up whether the table exists in DB - auto connection = connection_pool->GetConnection(); + auto connection = connection_pool->Acquire(); bool secret_storage_table_exists = secret_storage_table.Exists(connection.GetConnection()); string attached_database_name = GetAttached().GetName().GetIdentifierName(); @@ -306,4 +351,38 @@ void PostgresCatalog::RegisterSecretStorage() { } } +dbconnector::attached::AttachedCatalog PostgresCatalog::Lookup(ClientContext &ctx, const Identifier &name) { + using namespace dbconnector::attached; + + AttachedCatalog attached_catalog = AttachedCatalog::Lookup(ctx, "postgres", name); + if (!attached_catalog) { + throw InvalidInputException("Attached PostgreSQL database not found in the specified client session, name: %s", + name); + } + return attached_catalog; +} +//! Resolve whether scans against this database must use the text protocol instead of the binary COPY +//! protocol. This is the single place where that decision is made for an attached database, so every +//! scan path (table scans, postgres_query/CONNECT, ...) agrees. +bool PostgresCatalog::UseTextProtocol(PostgresQueryMode query_mode, bool use_text_protocol_setting) const { + switch (query_mode) { + case PostgresQueryMode::COPY_BINARY: + case PostgresQueryMode::PLAIN_QUERY_BINARY: + return false; + case PostgresQueryMode::PLAIN_QUERY_TEXT: + return true; + default: { + switch (text_protocol_mode) { + case PostgresTextProtocolMode::TEXT: + return true; + case PostgresTextProtocolMode::BINARY: + return false; + default: + // Redshift does not support the binary COPY protocol + return use_text_protocol_setting || version.type_v == PostgresInstanceType::REDSHIFT; + } + } + } +} + } // namespace duckdb diff --git a/src/storage/postgres_catalog_set.cpp b/src/storage/postgres_catalog_set.cpp index 2243adb2a..98e7e8808 100644 --- a/src/storage/postgres_catalog_set.cpp +++ b/src/storage/postgres_catalog_set.cpp @@ -1,19 +1,22 @@ #include "storage/postgres_catalog_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "storage/postgres_schema_entry.hpp" namespace duckdb { -PostgresCatalogSet::PostgresCatalogSet(Catalog &catalog, bool is_loaded_p) : catalog(catalog), is_loaded(is_loaded_p) { +PostgresCatalogSet::PostgresCatalogSet(Catalog &catalog, bool is_loaded_p) + : catalog(catalog), is_loaded(is_loaded_p), loading_thread(thread_id()) { } optional_ptr PostgresCatalogSet::GetEntry(ClientContext &context, PostgresTransaction &transaction, const string &name) { - TryLoadEntries(context, transaction); { - lock_guard l(entry_lock); + auto l = LoadEntriesForRead(context, transaction); auto entry = entries.find(name); if (entry != entries.end()) { // entry found @@ -42,18 +45,115 @@ optional_ptr PostgresCatalogSet::GetEntry(ClientContext &context, return nullptr; } +static string ComputeStalenessSignature(PostgresResult &result) { + if (result.ColumnCount() < 3) { + throw InvalidInputException( + "pg_staleness_query must return at least 3 columns (identity, name, revision marker), got %llu", + result.ColumnCount()); + } + string signature; + auto rows = result.Count(); + for (idx_t row = 0; row < rows; row++) { + signature += result.GetString(row, 0); + signature += '\0'; + signature += result.GetString(row, 1); + signature += '\0'; + signature += result.GetString(row, 2); + signature += '\x01'; + } + return signature; +} + +//! Runs the staleness query on a short-lived pooled connection, not the caller's own. +//! Avoids pinning the caller's connection (and, under PgBouncer transaction-mode pooling, +//! a backend slot) for the round-trip. +static unique_ptr RunStalenessQuery(Catalog &catalog, ClientContext &context, + const string &staleness_query) { + auto connection = catalog.Cast().GetConnectionPool().Acquire(); + return connection.GetConnection().Query(context, staleness_query); +} + +void PostgresCatalogSet::RefreshStalenessSignature(PostgresTransaction &transaction, bool use_transaction_connection) { + auto &context = *transaction.GetContext(); + auto staleness_query = GetStalenessQuery(context); + if (staleness_query.empty()) { + return; + } + auto result = use_transaction_connection ? transaction.Query(staleness_query) + : RunStalenessQuery(catalog, context, staleness_query); + auto signature = ComputeStalenessSignature(*result); + if (transaction.GetTransactionState() == PostgresTransactionState::TRANSACTION_STARTED) { + // transaction still open - only valid on commit, since xmin reverts on rollback + transaction.StageStalenessSignature(*this, std::move(signature)); + return; + } + staleness_signature = std::move(signature); +} + +void PostgresCatalogSet::PromoteStalenessSignature(string signature) { + // called from Commit(), possibly a different thread than the one that staged it + lock_guard lock(load_lock); + staleness_signature = std::move(signature); +} + +void PostgresCatalogSet::LoadEntriesLocked(ClientContext &context, PostgresTransaction &transaction) { + loading_thread = ThreadUtil::GetThreadId(); + try { + LoadEntries(context, transaction); + } catch (...) { + loading_thread = thread_id(); + throw; + } + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/false); + is_loaded = true; + loading_thread = thread_id(); +} + +unique_lock PostgresCatalogSet::LoadEntriesForRead(ClientContext &context, PostgresTransaction &transaction) { + while (true) { + TryLoadEntries(context, transaction); + unique_lock l(entry_lock); + // the loading thread may read partially loaded entries + if (is_loaded || loading_thread == ThreadUtil::GetThreadId()) { + return l; + } + } +} + void PostgresCatalogSet::TryLoadEntries(ClientContext &context, PostgresTransaction &transaction) { if (HasInternalDependencies()) { - if (is_loaded) { + if (is_loaded || loading_thread == ThreadUtil::GetThreadId()) { return; } } + if (is_loaded) { + auto staleness_query = GetStalenessQuery(context); + if (staleness_query.empty()) { + return; + } + // runs outside load_lock - avoids blocking other threads on this thread's network round-trip + auto result = RunStalenessQuery(catalog, context, staleness_query); + auto signature = ComputeStalenessSignature(*result); + + lock_guard lock(load_lock); + if (is_loaded) { + if (signature == staleness_signature) { + return; + } + ClearEntriesLocked(); + } + // else: someone else cleared/reloaded while we were checking - fall through to the + // reload below, still holding load_lock. + LoadEntriesLocked(context, transaction); + return; + } + lock_guard lock(load_lock); if (is_loaded) { + // someone else already loaded while we were waiting for the lock. return; } - is_loaded = true; - LoadEntries(context, transaction); + LoadEntriesLocked(context, transaction); } optional_ptr PostgresCatalogSet::ReloadEntry(PostgresTransaction &transaction, const string &name) { @@ -66,25 +166,28 @@ void PostgresCatalogSet::DropEntry(PostgresTransaction &transaction, DropInfo &i if (info.if_not_found == OnEntryNotFound::RETURN_NULL) { drop_query += " IF EXISTS "; } - if (!info.schema.empty()) { - drop_query += PostgresUtils::WriteIdentifier(info.schema.GetIdentifierName()) + "."; + if (!info.GetQualifiedName().Schema().empty() && info.type != CatalogType::SCHEMA_ENTRY) { + drop_query += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Schema().GetIdentifierName()) + "."; } - drop_query += PostgresUtils::WriteIdentifier(info.name.GetIdentifierName()); + drop_query += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Name().GetIdentifierName()); if (info.cascade) { - drop_query += "CASCADE"; + drop_query += " CASCADE"; } transaction.Query(drop_query); // erase the entry from the catalog set - lock_guard l(entry_lock); - entries.erase(info.name.GetIdentifierName()); + { + lock_guard l(entry_lock); + entries.erase(info.GetQualifiedName().Name().GetIdentifierName()); + } + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/true); } void PostgresCatalogSet::Scan(ClientContext &context, PostgresTransaction &transaction, const std::function &callback) { - TryLoadEntries(context, transaction); - lock_guard l(entry_lock); + auto l = LoadEntriesForRead(context, transaction); for (auto &entry : entries) { + transaction.ReferenceEntry(entry.second); callback(*entry.second); } } @@ -102,6 +205,11 @@ optional_ptr PostgresCatalogSet::CreateEntry(PostgresTransaction & } void PostgresCatalogSet::ClearEntries() { + lock_guard load_guard(load_lock); + ClearEntriesLocked(); +} + +void PostgresCatalogSet::ClearEntriesLocked() { lock_guard entry_guard(entry_lock); entry_map.clear(); entries.clear(); diff --git a/src/storage/postgres_clear_cache.cpp b/src/storage/postgres_clear_cache.cpp index 058cbad30..b064d9770 100644 --- a/src/storage/postgres_clear_cache.cpp +++ b/src/storage/postgres_clear_cache.cpp @@ -1,4 +1,5 @@ #include "duckdb.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "postgres_scanner.hpp" @@ -13,7 +14,7 @@ struct ClearCacheFunctionData : public TableFunctionData { }; static unique_ptr ClearCacheBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); return_types.push_back(LogicalType::BOOLEAN); names.emplace_back("Success"); @@ -46,6 +47,6 @@ void PostgresClearCacheFunction::ClearCacheOnSetting(ClientContext &context, Set } PostgresClearCacheFunction::PostgresClearCacheFunction() - : TableFunction("pg_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { + : TableFunction("postgres_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/postgres_configure_pool.cpp b/src/storage/postgres_configure_pool.cpp deleted file mode 100644 index 2032b2346..000000000 --- a/src/storage/postgres_configure_pool.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#include "storage/postgres_connection_pool.hpp" - -#include - -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/database_manager.hpp" - -#include "storage/postgres_catalog.hpp" - -namespace duckdb { - -namespace { - -enum class ExecState { UNINITIALIZED, EXHAUSTED }; - -struct ConfigurePoolBindData : public TableFunctionData { - std::pair catalog_name; - std::pair acquire_mode; - std::pair max_connections; - std::pair wait_timeout_millis; - std::pair enable_thread_local_cache; - std::pair max_lifetime_millis; - std::pair idle_timeout_millis; - std::pair enable_reaper_thread; - std::pair health_check_query; - - static Value Lookup(const named_parameter_map_t &map, const std::string &key) { - auto it = map.find(Identifier(key)); - if (it == map.end()) { - return Value(); - } - return it->second; - } - - static std::pair LookupString(const named_parameter_map_t &map, const std::string &key) { - Value val = Lookup(map, key); - if (val.IsNull()) { - return std::make_pair("", true); - } - std::string str = StringValue::Get(val); - return std::make_pair(std::move(str), false); - } - - static std::pair LookupUBigInt(const named_parameter_map_t &map, const std::string &key) { - Value val = Lookup(map, key); - if (val.IsNull()) { - return std::make_pair(0, true); - } - uint64_t num = UBigIntValue::Get(val); - return std::make_pair(num, false); - } - - static std::pair LookupBool(const named_parameter_map_t &map, const std::string &key) { - Value val = Lookup(map, key); - if (val.IsNull()) { - return std::make_pair(false, true); - } - bool flag = BooleanValue::Get(val); - return std::make_pair(flag, false); - } - - static std::pair LookupAcquireMode(const named_parameter_map_t &map, - const std::string &key) { - std::pair st_pair = LookupString(map, key); - if (st_pair.second) { - return std::make_pair(dbconnector::pool::AcquireMode::FORCE, true); - } - try { - dbconnector::pool::AcquireMode mode = dbconnector::pool::AcquireModeHelpers::FromString(st_pair.first); - return std::make_pair(mode, false); - } catch (const std::exception &e) { - throw BinderException(e.what()); - } - } - - ConfigurePoolBindData(const named_parameter_map_t &map) - : catalog_name(LookupString(map, "catalog_name")), acquire_mode(LookupAcquireMode(map, "acquire_mode")), - max_connections(LookupUBigInt(map, "max_connections")), - wait_timeout_millis(LookupUBigInt(map, "wait_timeout_millis")), - enable_thread_local_cache(LookupBool(map, "enable_thread_local_cache")), - max_lifetime_millis(LookupUBigInt(map, "max_lifetime_millis")), - idle_timeout_millis(LookupUBigInt(map, "idle_timeout_millis")), - enable_reaper_thread(LookupBool(map, "enable_reaper_thread")), - health_check_query(LookupString(map, "health_check_query")) { - if (catalog_name.second && - !(acquire_mode.second && max_connections.second && wait_timeout_millis.second && - enable_thread_local_cache.second && max_lifetime_millis.second && idle_timeout_millis.second && - enable_reaper_thread.second && health_check_query.second)) { - throw BinderException("'catalog_name' argument must be specified to change any option value on the " - "connection pool of this catalog"); - } - } -}; - -struct GlobalState : public GlobalTableFunctionState {}; - -struct LocalState : public LocalTableFunctionState { - ExecState exec_state = ExecState::UNINITIALIZED; -}; - -} // namespace - -static void AddColumn(vector &return_types, vector &names, const std::string &col_name, - LogicalType col_type) { - names.emplace_back(col_name); - return_types.emplace_back(col_type); -} - -static unique_ptr ConfigurePoolBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - AddColumn(return_types, names, "catalog_name", LogicalType::VARCHAR); - AddColumn(return_types, names, "acquire_mode", LogicalType::VARCHAR); - AddColumn(return_types, names, "available_connections", LogicalType::UBIGINT); - AddColumn(return_types, names, "max_connections", LogicalType::UBIGINT); - AddColumn(return_types, names, "wait_timeout_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "cache_hits", LogicalType::UBIGINT); - AddColumn(return_types, names, "cache_misses", LogicalType::UBIGINT); - AddColumn(return_types, names, "try_failures", LogicalType::UBIGINT); - AddColumn(return_types, names, "thread_local_cache_enabled", LogicalType::BOOLEAN); - AddColumn(return_types, names, "thread_local_cache_hits", LogicalType::UBIGINT); - AddColumn(return_types, names, "thread_local_cache_misses", LogicalType::UBIGINT); - AddColumn(return_types, names, "max_lifetime_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "idle_timeout_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "reaper_thread_running", LogicalType::BOOLEAN); - AddColumn(return_types, names, "reaper_thread_period_millis", LogicalType::UBIGINT); - AddColumn(return_types, names, "health_check_query", LogicalType::VARCHAR); - - return make_uniq(input.named_parameters); -} - -static unique_ptr ConfigurePoolInitGlobalState(ClientContext &, TableFunctionInitInput &) { - return make_uniq(); -} - -static unique_ptr ConfigurePoolInitLocalState(ExecutionContext &, TableFunctionInitInput &, - GlobalTableFunctionState *) { - return make_uniq(); -} - -static void ConfigurePoolFunction(ClientContext &context, TableFunctionInput &input, DataChunk &output) { - auto &bdata = input.bind_data->Cast(); - auto &lstate = input.local_state->Cast(); - - if (lstate.exec_state == ExecState::EXHAUSTED) { - output.SetChildCardinality(0); - return; - } - - // collect pools - std::vector cat_names; - std::vector> pools; - auto databases = DatabaseManager::Get(context).GetDatabases(context); - for (auto &db_ref : databases) { - auto &db = *db_ref; - auto &catalog = db.GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - continue; - } - if (!bdata.catalog_name.second && catalog.GetName() != Identifier(bdata.catalog_name.first)) { - continue; - } - cat_names.push_back(catalog.GetName().GetIdentifierName()); - shared_ptr pool = catalog.Cast().GetConnectionPoolPtr(); - pools.emplace_back(std::move(pool)); - } - - if (!bdata.catalog_name.second && pools.size() == 0) { - throw InvalidInputException("Catalog not found, name: '%s'", bdata.catalog_name.first); - } - - // configure the single pool if specified - if (!bdata.catalog_name.second && pools.size() > 0) { - auto &pool = pools.at(0); - if (!bdata.acquire_mode.second) { - pool->SetAcquireMode(bdata.acquire_mode.first); - } - if (!bdata.max_connections.second) { - pool->SetMaxConnections(bdata.max_connections.first); - } - if (!bdata.wait_timeout_millis.second) { - pool->SetWaitTimeoutMillis(bdata.wait_timeout_millis.first); - } - if (!bdata.enable_thread_local_cache.second) { - pool->SetThreadLocalCacheEnabled(bdata.enable_thread_local_cache.first); - } - if (!bdata.max_lifetime_millis.second) { - pool->SetMaxLifetimeMillis(bdata.max_lifetime_millis.first); - } - if (!bdata.idle_timeout_millis.second) { - pool->SetIdleTimeoutMillis(bdata.idle_timeout_millis.first); - } - if (!bdata.enable_reaper_thread.second) { - if (bdata.enable_reaper_thread.first) { - pool->EnsureReaperRunning(); - } else { - pool->ShutdownReaper(); - } - } - if (!bdata.health_check_query.second) { - pool->SetHealthCheckQuery(bdata.health_check_query.first); - } - } - - // set results - idx_t row_idx = 0; - for (auto &pool : pools) { - idx_t col_idx = 0; - output.data[col_idx++].SetValue(row_idx, Value(cat_names.at(row_idx))); - output.data[col_idx++].SetValue(row_idx, - Value(dbconnector::pool::AcquireModeHelpers::ToString(pool->GetAcquireMode()))); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetAvailableConnections())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetMaxConnections())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetWaitTimeoutMillis())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetCacheHits())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetCacheMisses())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetTryFailures())); - output.data[col_idx++].SetValue(row_idx, Value::BOOLEAN(pool->IsThreadLocalCacheEnabled())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetThreadLocalCacheHits())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetThreadLocalCacheMisses())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetMaxLifetimeMillis())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetIdleTimeoutMillis())); - output.data[col_idx++].SetValue(row_idx, Value::BOOLEAN(pool->IsReaperRunning())); - output.data[col_idx++].SetValue(row_idx, Value::UBIGINT(pool->GetReaperPeriodMillis())); - output.data[col_idx++].SetValue(row_idx, Value(pool->GetHealthCheckQuery())); - row_idx++; - } - - output.SetChildCardinality(row_idx); - lstate.exec_state = ExecState::EXHAUSTED; -} - -PostgresConfigurePoolFunction::PostgresConfigurePoolFunction() - : TableFunction("postgres_configure_pool", std::vector(), ConfigurePoolFunction, ConfigurePoolBind, - ConfigurePoolInitGlobalState, ConfigurePoolInitLocalState) { - named_parameters["catalog_name"] = LogicalType::VARCHAR; - named_parameters["acquire_mode"] = LogicalType::VARCHAR; - named_parameters["max_connections"] = LogicalType::UBIGINT; - named_parameters["wait_timeout_millis"] = LogicalType::UBIGINT; - named_parameters["enable_thread_local_cache"] = LogicalType::BOOLEAN; - named_parameters["max_lifetime_millis"] = LogicalType::UBIGINT; - named_parameters["idle_timeout_millis"] = LogicalType::UBIGINT; - named_parameters["enable_reaper_thread"] = LogicalType::BOOLEAN; - named_parameters["health_check_query"] = LogicalType::VARCHAR; -} - -} // namespace duckdb diff --git a/src/storage/postgres_connection_pool.cpp b/src/storage/postgres_connection_pool.cpp index 03518ec84..ea6ca9ee9 100644 --- a/src/storage/postgres_connection_pool.cpp +++ b/src/storage/postgres_connection_pool.cpp @@ -1,31 +1,22 @@ #include "storage/postgres_connection_pool.hpp" +#include #include +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/function/table_function.hpp" + +#include "dbconnector/functions/configure_pool.hpp" + #include "storage/postgres_catalog.hpp" namespace duckdb { -static dbconnector::pool::ConnectionPoolConfig CreateConfig(ClientContext &context); - -static std::string GetHealthCheckQueryFromConfig(ClientContext &context) { - Value val; - if (context.TryGetCurrentSetting("pg_pool_health_check_query", val)) { - if (val.IsNull()) { - return std::string(); - } - return StringValue::Get(val); - } - return PostgresConnectionPool::DefaultHealthCheckQuery(); -} +using dbconnector::functions::ConfigurePool; PostgresConnectionPool::PostgresConnectionPool(PostgresCatalog &postgres_catalog, ClientContext &context) - : dbconnector::pool::ConnectionPool(CreateConfig(context)), postgres_catalog(postgres_catalog), - health_check_query(GetHealthCheckQueryFromConfig(context)) { -} - -PostgresPoolConnection PostgresConnectionPool::ForceGetConnection() { - return ForceAcquire(); + : dbconnector::pool::ConnectionPool(CreateConfig(context)), postgres_catalog(postgres_catalog) { } bool PostgresConnectionPool::TryGetConnection(PostgresPoolConnection &connection) { @@ -37,10 +28,6 @@ bool PostgresConnectionPool::TryGetConnection(PostgresPoolConnection &connection return true; } -PostgresPoolConnection PostgresConnectionPool::GetConnection() { - return Acquire(); -} - std::unique_ptr PostgresConnectionPool::CreateNewConnection() { auto conn = PostgresConnection::Open(postgres_catalog.GetConnectionString(), postgres_catalog.attach_path); return make_uniq(std::move(conn)); @@ -59,21 +46,7 @@ void PostgresConnectionPool::ResetConnection(PostgresConnection &conn) { conn.Reset(query); } -std::string PostgresConnectionPool::GetHealthCheckQuery() { - std::lock_guard guard(config_mutex); - return std::string(health_check_query.data(), health_check_query.length()); -} - -void PostgresConnectionPool::SetHealthCheckQuery(const std::string &query) { - std::lock_guard guard(config_mutex); - this->health_check_query = std::string(query.data(), query.length()); -} - -std::string PostgresConnectionPool::DefaultHealthCheckQuery() { - return "SELECT 1"; -} - -static dbconnector::pool::ConnectionPoolConfig CreateConfig(ClientContext &ctx) { +dbconnector::pool::ConnectionPoolConfig PostgresConnectionPool::CreateConfig(ClientContext &ctx) { dbconnector::pool::ConnectionPoolConfig config; { @@ -118,6 +91,12 @@ static dbconnector::pool::ConnectionPoolConfig CreateConfig(ClientContext &ctx) config.start_reaper_thread = BooleanValue::Get(val); } } + { + Value val; + if (ctx.TryGetCurrentSetting("pg_pool_health_check_query", val) && !val.IsNull()) { + config.health_check_query = StringValue::Get(val); + } + } return config; } @@ -141,4 +120,22 @@ void PostgresConnectionPool::ValidatePoolAcquireMode(ClientContext &context, Set } } +static shared_ptr GetConnnectionPoolFromCatalog(Catalog &catalog) { + if (catalog.GetCatalogType() != "postgres") { + return nullptr; + } + return catalog.Cast().GetConnectionPoolPtr(); +} + +PostgresConfigurePoolFunction::PostgresConfigurePoolFunction() + : TableFunction("postgres_configure_pool", std::vector(), + ConfigurePool::Function, ConfigurePool::Bind, + ConfigurePool::InitGlobalState, ConfigurePool::InitLocalState) { + GetSignature().WithTypedKwargs("options", [](TypedKwargs &options) { + for (auto &en : ConfigurePool::NamedParameters()) { + options.Add(en.first, en.second); + } + }); +} + } // namespace duckdb diff --git a/src/storage/postgres_delete.cpp b/src/storage/postgres_delete.cpp index 97b829fff..e47bab1bb 100644 --- a/src/storage/postgres_delete.cpp +++ b/src/storage/postgres_delete.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_delete.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_table_entry.hpp" #include "duckdb/planner/operator/logical_delete.hpp" diff --git a/src/storage/postgres_index.cpp b/src/storage/postgres_index.cpp index cf26c37bd..acacc7c36 100644 --- a/src/storage/postgres_index.cpp +++ b/src/storage/postgres_index.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_catalog.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_index.hpp" #include "duckdb/parser/statement/create_statement.hpp" #include "duckdb/planner/operator/logical_extension_operator.hpp" @@ -25,19 +29,21 @@ SourceResultType PostgresCreateIndex::GetDataInternal(ExecutionContext &context, auto &catalog = table.catalog; auto &schema = table.schema; auto transaction = catalog.GetCatalogTransaction(context.client); - auto existing = schema.GetEntry(transaction, CatalogType::INDEX_ENTRY, info->index_name); + auto existing = schema.GetEntry(transaction, CatalogType::INDEX_ENTRY, info->GetIndexName()); if (existing) { switch (info->on_conflict) { case OnCreateConflict::IGNORE_ON_CONFLICT: return SourceResultType::FINISHED; case OnCreateConflict::ERROR_ON_CONFLICT: - throw BinderException("Index with name \"%s\" already exists in schema \"%s\"", info->index_name, + throw BinderException("Index with name \"%s\" already exists in schema \"%s\"", info->GetIndexName(), table.schema.name); case OnCreateConflict::REPLACE_ON_CONFLICT: { DropInfo drop_info; drop_info.type = CatalogType::INDEX_ENTRY; - drop_info.schema = info->schema; - drop_info.name = info->index_name; + drop_info.SetQualifiedName(QualifiedName(drop_info.GetQualifiedName().Catalog(), + info->GetQualifiedName().Schema(), + drop_info.GetQualifiedName().Name())); + drop_info.SetName(info->GetIndexName()); schema.DropEntry(context.client, drop_info); break; } diff --git a/src/storage/postgres_index_entry.cpp b/src/storage/postgres_index_entry.cpp index 0f42ef32d..1c33b6108 100644 --- a/src/storage/postgres_index_entry.cpp +++ b/src/storage/postgres_index_entry.cpp @@ -1,4 +1,5 @@ #include "storage/postgres_index_entry.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" namespace duckdb { diff --git a/src/storage/postgres_index_set.cpp b/src/storage/postgres_index_set.cpp index e4f4c6e78..00cc881ba 100644 --- a/src/storage/postgres_index_set.cpp +++ b/src/storage/postgres_index_set.cpp @@ -1,4 +1,7 @@ #include "storage/postgres_index_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" @@ -15,7 +18,7 @@ PostgresIndexSet::PostgresIndexSet(PostgresSchemaEntry &schema, unique_ptr &schemas) { string base_query = R"( SELECT pg_namespace.oid, tablename, indexname FROM pg_indexes @@ -24,8 +27,8 @@ JOIN pg_namespace ON (schemaname=nspname) ORDER BY pg_namespace.oid; )"; string condition; - if (!schema.empty()) { - condition += "WHERE pg_namespace.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "WHERE pg_namespace.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } @@ -39,9 +42,11 @@ void PostgresIndexSet::LoadEntries(ClientContext &context, PostgresTransaction & auto table_name = result.GetString(row, 1); auto index_name = result.GetString(row, 2); CreateIndexInfo info; - info.schema = schema.name; + info.constraint_type = IndexConstraintType::NONE; + info.SetQualifiedName( + QualifiedName(info.GetQualifiedName().Catalog(), schema.name, info.GetQualifiedName().Name())); info.table = Identifier(table_name); - info.index_name = Identifier(index_name); + info.SetIndexName(Identifier(index_name)); auto index_entry = make_shared_ptr(catalog, schema, info, table_name); CreateEntry(transaction, std::move(index_entry)); } @@ -65,7 +70,7 @@ string PGGetCreateIndexSQL(CreateIndexInfo &info, TableCatalogEntry &tbl) { sql += " UNIQUE"; } sql += " INDEX "; - sql += PostgresUtils::QuotePostgresIdentifier(info.index_name.GetIdentifierName()); + sql += PostgresUtils::QuotePostgresIdentifier(info.GetIndexName().GetIdentifierName()); sql += " ON "; sql += PostgresUtils::QuotePostgresIdentifier(tbl.schema.name.GetIdentifierName()) + "."; sql += PostgresUtils::QuotePostgresIdentifier(tbl.name.GetIdentifierName()); diff --git a/src/storage/postgres_insert.cpp b/src/storage/postgres_insert.cpp index 467b995b9..a37389357 100644 --- a/src/storage/postgres_insert.cpp +++ b/src/storage/postgres_insert.cpp @@ -1,4 +1,9 @@ #include "storage/postgres_insert.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_catalog.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/planner/operator/logical_insert.hpp" @@ -14,24 +19,25 @@ namespace duckdb { -PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, +PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, TableCatalogEntry &table, physical_index_vector_t column_index_map_p) : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(&table), schema(nullptr), - column_index_map(std::move(column_index_map_p)) { + column_index_map(std::move(column_index_map_p)), use_plain_inserts(UsePlainInserts(pg_catalog, context)) { } -PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, - unique_ptr info) +PostgresInsert::PostgresInsert(PhysicalPlan &physical_plan, PostgresCatalog &pg_catalog, ClientContext &context, + LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info) : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(nullptr), schema(&schema), - info(std::move(info)) { + info(std::move(info)), use_plain_inserts(UsePlainInserts(pg_catalog, context)) { } //===--------------------------------------------------------------------===// // States //===--------------------------------------------------------------------===// -class PostgresInsertGlobalState : public GlobalSinkState { +class PostgresInsertCopyGlobalState : public GlobalSinkState { public: - explicit PostgresInsertGlobalState(ClientContext &context, PostgresTableEntry &table, PostgresCopyFormat format) + explicit PostgresInsertCopyGlobalState(ClientContext &context, PostgresTableEntry &table, PostgresCopyFormat format) : table(table), insert_count(0), format(format) { } @@ -52,7 +58,22 @@ class PostgresInsertGlobalState : public GlobalSinkState { } }; -vector GetInsertColumns(const PostgresInsert &insert, PostgresTableEntry &entry) { +class PostgresInsertPlainGlobalState : public GlobalSinkState { +public: + explicit PostgresInsertPlainGlobalState(ClientContext &context, PostgresTableEntry &table, + const vector &varchar_types) + : table(table), insert_count(0) { + varchar_chunk.Initialize(context, varchar_types); + } + + PostgresTableEntry &table; + DataChunk varchar_chunk; + idx_t insert_count; + string base_insert_query; + string insert_values; +}; + +static vector GetInsertColumns(const PostgresInsert &insert, PostgresTableEntry &entry) { vector column_names; auto &columns = entry.GetColumns(); idx_t column_count; @@ -78,7 +99,7 @@ vector GetInsertColumns(const PostgresInsert &insert, PostgresTableEntry return column_names; } -unique_ptr PostgresInsert::GetGlobalSinkState(ClientContext &context) const { +unique_ptr PostgresInsert::GetGlobalSinkStateCopy(ClientContext &context) const { optional_ptr insert_table; if (!table) { auto &schema_ref = *schema.get_mutable(); @@ -91,7 +112,7 @@ unique_ptr PostgresInsert::GetGlobalSinkState(ClientContext &co auto &connection = transaction.GetConnection(); auto insert_columns = GetInsertColumns(*this, *insert_table); auto format = insert_table->GetCopyFormat(context); - auto result = make_uniq(context, *insert_table, format); + auto result = make_uniq(context, *insert_table, format); auto &insert_column_names = result->insert_column_names; if (!insert_columns.empty()) { for (auto &str : insert_columns) { @@ -107,11 +128,53 @@ unique_ptr PostgresInsert::GetGlobalSinkState(ClientContext &co return std::move(result); } +static string GetBaseInsertQuery(const PostgresTableEntry &table, const vector &column_names) { + string query; + query += "INSERT INTO "; + query += PostgresUtils::WriteIdentifier(table.schema.name.GetIdentifierName()); + query += "."; + query += PostgresUtils::WriteIdentifier(table.name.GetIdentifierName()); + query += " "; + if (!column_names.empty()) { + query += "("; + for (idx_t c = 0; c < column_names.size(); c++) { + if (c > 0) { + query += ", "; + } + query += PostgresUtils::WriteIdentifier(column_names[c]); + } + query += ")"; + } + query += " VALUES "; + return query; +} + +unique_ptr PostgresInsert::GetGlobalSinkStatePlain(ClientContext &context) const { + PostgresTableEntry *insert_table; + if (!table) { + auto &schema_ref = *schema.get_mutable(); + insert_table = + &schema_ref.CreateTable(schema_ref.GetCatalogTransaction(context), *info)->Cast(); + } else { + insert_table = &table.get_mutable()->Cast(); + } + auto insert_columns = GetInsertColumns(*this, *insert_table); + vector insert_types; + idx_t insert_column_count = + insert_columns.empty() ? insert_table->GetColumns().LogicalColumnCount() : insert_columns.size(); + for (idx_t c = 0; c < insert_column_count; c++) { + insert_types.push_back(LogicalType::VARCHAR); + } + auto result = make_uniq(context, *insert_table, insert_types); + result->base_insert_query = GetBaseInsertQuery(*insert_table, insert_columns); + return std::move(result); +} + //===--------------------------------------------------------------------===// // Sink //===--------------------------------------------------------------------===// -SinkResultType PostgresInsert::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { - auto &gstate = input.global_state.Cast(); +SinkResultType PostgresInsert::SinkCopy(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { + auto &gstate = input.global_state.Cast(); auto &transaction = PostgresTransaction::Get(context.client, gstate.table.catalog); auto &connection = transaction.GetConnection(); if (!gstate.copy_is_active) { @@ -130,12 +193,132 @@ SinkResultType PostgresInsert::Sink(ExecutionContext &context, DataChunk &chunk, return SinkResultType::NEED_MORE_INPUT; } +static void PostgresCastBlob(const Vector &input, Vector &result, idx_t count) { + static constexpr const char *HEX_TABLE = "0123456789ABCDEF"; + auto input_data = FlatVector::GetData(input); + auto result_data = FlatVector::GetDataMutable(result); + for (idx_t r = 0; r < count; r++) { + if (FlatVector::IsNull(input, r)) { + FlatVector::SetNull(result, r, true); + continue; + } + auto blob_data = const_data_ptr_cast(input_data[r].GetData()); + auto blob_size = input_data[r].GetSize(); + if (blob_size == 0) { + result_data[r] = StringVector::AddString(result, "''"); + continue; + } + string result_blob = "'\\x"; + for (idx_t b = 0; b < blob_size; b++) { + auto blob_entry = blob_data[b]; + auto byte_a = blob_entry >> 4; + auto byte_b = blob_entry & 0x0F; + result_blob += string(1, HEX_TABLE[byte_a]); + result_blob += string(1, HEX_TABLE[byte_b]); + } + result_blob += "'::BYTEA"; + result_data[r] = StringVector::AddString(result, result_blob); + } +} + +SinkResultType PostgresInsert::SinkPlain(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { + static constexpr const idx_t INSERT_FLUSH_SIZE = 8000; + + auto &gstate = input.global_state.Cast(); + auto &transaction = PostgresTransaction::Get(context.client, gstate.table.catalog); + auto &con = transaction.GetConnection(); + // cast to varchar + D_ASSERT(chunk.ColumnCount() == gstate.varchar_chunk.ColumnCount()); + chunk.Flatten(); + gstate.varchar_chunk.Reset(); + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + switch (chunk.data[c].GetType().id()) { + case LogicalTypeId::BLOB: + PostgresCastBlob(chunk.data[c], gstate.varchar_chunk.data[c], chunk.size()); + break; + case LogicalTypeId::TIMESTAMP_TZ: { + Vector timestamp_vector(LogicalType::TIMESTAMP); + timestamp_vector.Reinterpret(chunk.data[c]); + VectorOperations::Cast(context.client, timestamp_vector, gstate.varchar_chunk.data[c], chunk.size()); + break; + } + default: + VectorOperations::Cast(context.client, chunk.data[c], gstate.varchar_chunk.data[c], chunk.size()); + break; + } + } + gstate.varchar_chunk.SetChildCardinality(chunk.size()); + // for each column type check if we need to add quotes or not + vector add_quotes; + for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + bool add_quotes_for_type; + switch (chunk.data[c].GetType().id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::BLOB: + add_quotes_for_type = false; + break; + case LogicalTypeId::ARRAY: + case LogicalTypeId::LIST: + case LogicalTypeId::STRUCT: + throw InvalidInputException("Insertion of '%s' columns is not supported with compatibility mode enabled " + "('pg_use_text_protocol' option)", + chunk.data[c].GetType().ToString()); + default: + add_quotes_for_type = true; + break; + } + add_quotes.push_back(add_quotes_for_type); + } + + // generate INSERT INTO statements + for (idx_t r = 0; r < chunk.size(); r++) { + if (!gstate.insert_values.empty()) { + gstate.insert_values += ", "; + } + gstate.insert_values += "("; + for (idx_t c = 0; c < gstate.varchar_chunk.ColumnCount(); c++) { + if (c > 0) { + gstate.insert_values += ", "; + } + if (FlatVector::IsNull(gstate.varchar_chunk.data[c], r)) { + gstate.insert_values += "NULL"; + } else { + auto data = FlatVector::GetDataMutable(gstate.varchar_chunk.data[c]); + if (add_quotes[c]) { + gstate.insert_values += PostgresUtils::WriteLiteral(data[r].GetString()); + } else { + gstate.insert_values += data[r].GetString(); + } + } + } + gstate.insert_values += ")"; + if (gstate.insert_values.size() >= INSERT_FLUSH_SIZE) { + // perform the actual insert + con.Execute(context.client, gstate.base_insert_query + gstate.insert_values); + // reset the to-be-inserted values + gstate.insert_values = string(); + } + } + gstate.insert_count += chunk.size(); + return SinkResultType::NEED_MORE_INPUT; +} + //===--------------------------------------------------------------------===// // Finalize //===--------------------------------------------------------------------===// -SinkFinalizeType PostgresInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - auto &gstate = input.global_state.Cast(); +SinkFinalizeType PostgresInsert::FinalizeCopy(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); auto &transaction = PostgresTransaction::Get(context, gstate.table.catalog); auto &connection = transaction.GetConnection(); gstate.FinishCopyTo(connection); @@ -147,14 +330,33 @@ SinkFinalizeType PostgresInsert::Finalize(Pipeline &pipeline, Event &event, Clie return SinkFinalizeType::READY; } +SinkFinalizeType PostgresInsert::FinalizePlain(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); + if (!gstate.insert_values.empty()) { + // perform the final insert + auto &transaction = PostgresTransaction::Get(context, gstate.table.catalog); + auto &con = transaction.GetConnection(); + con.Execute(context, gstate.base_insert_query + gstate.insert_values); + } + return SinkFinalizeType::READY; +} + //===--------------------------------------------------------------------===// // GetData //===--------------------------------------------------------------------===// SourceResultType PostgresInsert::GetDataInternal(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const { - auto &insert_gstate = sink_state->Cast(); + idx_t insert_count = 0; + if (use_plain_inserts) { + auto &insert_gstate = sink_state->Cast(); + insert_count = insert_gstate.insert_count; + } else { + auto &insert_gstate = sink_state->Cast(); + insert_count = insert_gstate.insert_count; + } chunk.SetChildCardinality(1); - chunk.data[0].SetValue(0, Value::BIGINT(insert_gstate.insert_count)); + chunk.data[0].SetValue(0, Value::BIGINT(insert_count)); return SourceResultType::FINISHED; } @@ -168,10 +370,20 @@ string PostgresInsert::GetName() const { InsertionOrderPreservingMap PostgresInsert::ParamsToString() const { InsertionOrderPreservingMap result; - result["Table Name"] = table ? table->name.GetIdentifierName() : info->Base().table.GetIdentifierName(); + result["Table Name"] = table ? table->name.GetIdentifierName() : info->Base().GetTableName().GetIdentifierName(); return result; } +bool PostgresInsert::UsePlainInserts(PostgresCatalog &pg_catalog, ClientContext &context) { + bool use_text_proto_user_option = false; + Value value; + if (context.TryGetCurrentSetting("pg_use_text_protocol", value) && !value.IsNull()) { + use_text_proto_user_option = BooleanValue::Get(value); + } + PostgresQueryMode query_mode = PostgresUtils::ExtractQueryMode(context); + return pg_catalog.UseTextProtocol(query_mode, use_text_proto_user_option); +} + //===--------------------------------------------------------------------===// // Plan //===--------------------------------------------------------------------===// @@ -215,13 +427,13 @@ PhysicalOperator &AddCastToPostgresTypes(ClientContext &context, PhysicalPlanGen } bool PostgresCatalog::IsPostgresScan(const string &name) { - return name == "postgres_scan" || name == "postgres_scan_pushdown" || name == "postgres_query"; + return name == "postgres_scan" || name == "postgres_query"; } void PostgresCatalog::MaterializePostgresScans(PhysicalOperator &op) { if (op.type == PhysicalOperatorType::TABLE_SCAN) { auto &table_scan = op.Cast(); - if (PostgresCatalog::IsPostgresScan(table_scan.function.name.GetIdentifierName())) { + if (PostgresCatalog::IsPostgresScan(table_scan.function.GetName().GetIdentifierName())) { auto &bind_data = table_scan.bind_data->Cast(); bind_data.requires_materialization = true; bind_data.max_threads = 1; @@ -247,7 +459,7 @@ PhysicalOperator &PostgresCatalog::PlanInsert(ClientContext &context, PhysicalPl MaterializePostgresScans(*plan); auto &inner_plan = AddCastToPostgresTypes(context, planner, *plan); - auto &insert = planner.Make(op, op.table, op.column_index_map); + auto &insert = planner.Make(*this, context, op, op.table, op.column_index_map); insert.children.push_back(inner_plan); return insert; } @@ -257,7 +469,7 @@ PhysicalOperator &PostgresCatalog::PlanCreateTableAs(ClientContext &context, Phy auto &inner_plan = AddCastToPostgresTypes(context, planner, plan); MaterializePostgresScans(inner_plan); - auto &insert = planner.Make(op, op.schema, std::move(op.info)); + auto &insert = planner.Make(*this, context, op, op.schema, std::move(op.info)); insert.children.push_back(inner_plan); return insert; } diff --git a/src/storage/postgres_merge_into.cpp b/src/storage/postgres_merge_into.cpp index 7bcab80df..16d1421b3 100644 --- a/src/storage/postgres_merge_into.cpp +++ b/src/storage/postgres_merge_into.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_catalog.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" #include "duckdb/execution/operator/persistent/physical_merge_into.hpp" #include "duckdb/planner/operator/logical_merge_into.hpp" #include "storage/postgres_update.hpp" @@ -98,6 +100,7 @@ unique_ptr PostgresPlanMergeIntoAction(PostgresCatalog &catal return result; } +// TODO: consider using generic PlanMergeInto from the base class PhysicalOperator &PostgresCatalog::PlanMergeInto(ClientContext &context, PhysicalPlanGenerator &planner, LogicalMergeInto &op, PhysicalOperator &plan) { if (op.return_chunk) { @@ -106,18 +109,27 @@ PhysicalOperator &PostgresCatalog::PlanMergeInto(ClientContext &context, Physica map>> actions; // plan the merge into clauses + // disable parallelism when we have multiple operations + idx_t append_count = 0; for (auto &entry : op.actions) { vector> planned_actions; for (auto &action : entry.second) { + if (action->action_type == MergeActionType::MERGE_INSERT || + action->action_type == MergeActionType::MERGE_UPDATE || + action->action_type == MergeActionType::MERGE_DELETE) { + append_count++; + } planned_actions.push_back(PostgresPlanMergeIntoAction(*this, context, op, planner, *action, plan)); } actions.emplace(entry.first, std::move(planned_actions)); } - auto &result = planner.Make(op.types, std::move(actions), op.row_id_start, op.source_marker, - false, op.return_chunk); - result.children.push_back(plan); - return result; + bool parallel = false; // append_count <= 1 && !op.return_chunk; + // multiple operators appending to the same table cannot run concurrently - run their actions one after the other + bool serialize_actions = append_count > 1; + + return planner.Make(op.types, plan, std::move(actions), op.row_id_start, op.source_marker, + parallel, op.return_chunk, serialize_actions); } } // namespace duckdb diff --git a/src/storage/postgres_optimizer.cpp b/src/storage/postgres_optimizer.cpp index a035516c4..afbb21978 100644 --- a/src/storage/postgres_optimizer.cpp +++ b/src/storage/postgres_optimizer.cpp @@ -1,9 +1,12 @@ #include "storage/postgres_optimizer.hpp" +#include + +#include "duckdb/planner/logical_operator.hpp" + #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_limit.hpp" -#include "dbconnector/optimizer/order_by_and_limit_optimizer.hpp" #include "dbconnector/optimizer/optimizer_util.hpp" #include "postgres_scanner.hpp" @@ -15,24 +18,19 @@ namespace duckdb { struct PostgresOperators { - reference_map_t>> scans; + std::map>> scans; }; static void GatherPostgresScans(LogicalOperator &op, PostgresOperators &result) { if (op.type == LogicalOperatorType::LOGICAL_GET) { auto &get = op.Cast(); auto &table_scan = get.function; - if (!PostgresCatalog::IsPostgresScan(table_scan.name.GetIdentifierName())) { + if (!PostgresCatalog::IsPostgresScan(table_scan.GetName().GetIdentifierName())) { // not a postgres scan - skip return; } auto &bind_data = get.bind_data->Cast(); - auto catalog = bind_data.GetCatalog(); - if (!catalog) { - // "postgres_scan" functions are fully independent - we can always stream them - return; - } - result.scans[*catalog].push_back(get); + result.scans[bind_data.catalog_name.GetIdentifierName()].push_back(get); } // recurse into children for (auto &child : op.children) { @@ -40,34 +38,77 @@ static void GatherPostgresScans(LogicalOperator &op, PostgresOperators &result) } } -static void DisableParallelLimit(LogicalOperator &op) { - LogicalGet *get = nullptr; - dbconnector::BindData *bind_data = nullptr; - if (dbconnector::optimizer::OptimizerUtil::FindExtensionGet("postgres_scan", op, get, bind_data)) { - auto &pg_bind_data = bind_data->Cast(); - if (!pg_bind_data.order_by_and_limit_bind_data.limit_clause.empty()) { +static void OptimizePostgresScanLimitPushdown(unique_ptr &op) { + if (op->type == LogicalOperatorType::LOGICAL_LIMIT) { + auto &limit = op->Cast(); + reference child = *op->children[0]; + + while (child.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + child = *child.get().children[0]; + } + + if (child.get().type != LogicalOperatorType::LOGICAL_GET) { + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + + auto &get = child.get().Cast(); + if (!PostgresCatalog::IsPostgresScan(get.function.GetName().GetIdentifierName())) { + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + + switch (limit.limit_val.Type()) { + case LimitNodeType::CONSTANT_VALUE: + case LimitNodeType::UNSET: + break; + default: + // not a constant or unset limit + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + switch (limit.offset_val.Type()) { + case LimitNodeType::CONSTANT_VALUE: + case LimitNodeType::UNSET: + break; + default: + // not a constant or unset offset + OptimizePostgresScanLimitPushdown(op->children[0]); + return; + } + + auto &bind_data = get.bind_data->Cast(); + + string generated_limit_clause = ""; + if (limit.limit_val.Type() != LimitNodeType::UNSET) { + generated_limit_clause += " LIMIT " + to_string(limit.limit_val.GetConstantValue()); + } + if (limit.offset_val.Type() != LimitNodeType::UNSET) { + generated_limit_clause += " OFFSET " + to_string(limit.offset_val.GetConstantValue()); + } + + if (!generated_limit_clause.empty()) { + bind_data.limit_clause = generated_limit_clause; // When LIMIT is pushed down to Postgres, we must ensure single-task execution // to avoid each task (whether parallel or sequential) applying the LIMIT independently. // Setting pages_approx = 0 disables CTID-based task splitting, ensuring a single query. - pg_bind_data.pages_approx = 0; - pg_bind_data.max_threads = 1; + bind_data.pages_approx = 0; + bind_data.max_threads = 1; + + op = std::move(op->children[0]); + return; } } - for (auto &child : op.children) { - DisableParallelLimit(*child); + for (auto &child : op->children) { + OptimizePostgresScanLimitPushdown(child); } } void PostgresOptimizer::Optimize(OptimizerExtensionInput &input, unique_ptr &plan) { using namespace dbconnector; // look at query plan and check if we can find LIMIT/OFFSET to pushdown - // OptimizePostgresScanLimitPushdown(plan); - - auto order_config = optimizer::OrderByAndLimitOptimizer::CreateConfig( - input.context, "pg_order_pushdown", '"', query::QuoteEscapeStyle::DOUBLE_QUOTE, "postgres_scan"); - optimizer::OrderByAndLimitOptimizer::Optimize(order_config, input, plan); - DisableParallelLimit(*plan); + OptimizePostgresScanLimitPushdown(plan); // look at the query plan and check if we can enable streaming query scans PostgresOperators operators; @@ -77,7 +118,6 @@ void PostgresOptimizer::Optimize(OptimizerExtensionInput &input, unique_ptr 1; for (auto &scan : entry.second) { auto &bind_data = scan.get().bind_data->Cast(); diff --git a/src/storage/postgres_schema_entry.cpp b/src/storage/postgres_schema_entry.cpp index f320e6f57..9bea7b54f 100644 --- a/src/storage/postgres_schema_entry.cpp +++ b/src/storage/postgres_schema_entry.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_schema_entry.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "storage/postgres_table_entry.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" @@ -42,7 +46,7 @@ PostgresTransaction &GetPostgresTransaction(CatalogTransaction transaction) { void PostgresSchemaEntry::TryDropEntry(ClientContext &context, CatalogType catalog_type, const string &name) { DropInfo info; info.type = catalog_type; - info.name = Identifier(name); + info.SetName(Identifier(name)); info.cascade = false; info.if_not_found = OnEntryNotFound::RETURN_NULL; DropEntry(context, info); @@ -52,7 +56,7 @@ optional_ptr PostgresSchemaEntry::CreateTable(CatalogTransaction t BoundCreateTableInfo &info) { auto &postgres_transaction = GetPostgresTransaction(transaction); auto &base_info = info.Base(); - string table_name = base_info.table.GetIdentifierName(); + string table_name = base_info.GetTableName().GetIdentifierName(); if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { // CREATE OR REPLACE - drop any existing entries first (if any) TryDropEntry(transaction.GetContext(), CatalogType::TABLE_ENTRY, table_name); @@ -75,7 +79,7 @@ string PGGetCreateViewSQL(PostgresSchemaEntry &schema, CreateViewInfo &info) { string sql; sql = "CREATE VIEW "; sql += PostgresUtils::QuotePostgresIdentifier(schema.name.GetIdentifierName()) + "."; - sql += PostgresUtils::QuotePostgresIdentifier(info.view_name.GetIdentifierName()); + sql += PostgresUtils::QuotePostgresIdentifier(info.GetViewName().GetIdentifierName()); sql += " "; if (!info.aliases.empty()) { sql += "("; @@ -99,26 +103,26 @@ optional_ptr PostgresSchemaEntry::CreateView(CatalogTransaction tr } if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { - auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.view_name); + auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.GetViewName()); if (current_entry) { if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { return current_entry; } // CREATE OR REPLACE - drop any existing entries first (if any) - TryDropEntry(transaction.GetContext(), CatalogType::VIEW_ENTRY, info.view_name.GetIdentifierName()); + TryDropEntry(transaction.GetContext(), CatalogType::VIEW_ENTRY, info.GetViewName().GetIdentifierName()); } } auto &postgres_transaction = GetPostgresTransaction(transaction); postgres_transaction.Query(PGGetCreateViewSQL(*this, info)); - return tables.ReloadEntry(postgres_transaction, info.view_name.GetIdentifierName()); + return tables.ReloadEntry(postgres_transaction, info.GetViewName().GetIdentifierName()); } optional_ptr PostgresSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { auto &postgres_transaction = GetPostgresTransaction(transaction); - auto type_name = info.name; + auto type_name = info.GetTypeName(); if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { // CREATE OR REPLACE - drop any existing entries first (if any) - TryDropEntry(transaction.GetContext(), CatalogType::TYPE_ENTRY, info.name.GetIdentifierName()); + TryDropEntry(transaction.GetContext(), CatalogType::TYPE_ENTRY, info.GetTypeName().GetIdentifierName()); } return types.CreateType(postgres_transaction, info); } @@ -182,7 +186,7 @@ void PostgresSchemaEntry::Scan(CatalogType type, const std::function schemas_to_load_p) + : PostgresCatalogSet(catalog, false), schemas_to_load(std::move(schemas_to_load_p)) { } vector> SliceResult(PostgresResult &schemas, unique_ptr to_slice_ptr) { @@ -37,7 +39,30 @@ vector> SliceResult(PostgresResult &schemas, uni return result; } -string PostgresSchemaSet::GetInitializeQuery(const string &schema) { +// Variant of SliceResult +// uses field name instead of an integer oid. +// Used by the information_schema introspection path where namespace oids are'nt available. +static vector> SliceResultByName(PostgresResult &schemas, + unique_ptr to_slice_ptr) { + auto shared_result = shared_ptr(to_slice_ptr.release()); + auto &to_slice = *shared_result; + + vector> result; + idx_t current_offset = 0; + for (idx_t schema_idx = 0; schema_idx < schemas.Count(); schema_idx++) { + auto name = schemas.GetString(schema_idx, 0); + idx_t start = current_offset; + for (; current_offset < to_slice.Count(); current_offset++) { + if (to_slice.GetString(current_offset, 0) != name) { + break; + } + } + result.push_back(make_uniq(shared_result, start, current_offset)); + } + return result; +} + +string PostgresSchemaSet::GetInitializeQuery(const vector &schemas) { string base_query = R"( SELECT oid, nspname FROM pg_namespace @@ -45,20 +70,71 @@ FROM pg_namespace ORDER BY oid; )"; string condition; - if (!schema.empty()) { - condition += "WHERE pg_namespace.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "WHERE pg_namespace.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; + } + return StringUtil::Replace(base_query, "${CONDITION}", condition); +} + +string PostgresSchemaSet::GetInitializeQueryInformationSchema(const vector &schemas) { + string base_query = R"( +SELECT schema_name +FROM information_schema.schemata +WHERE schema_name NOT IN ('information_schema', 'pg_catalog', 'pg_toast') +${CONDITION} +ORDER BY schema_name; +)"; + string condition; + if (schemas.size() > 0) { + condition += "AND schema_name IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } +void PostgresSchemaSet::LoadEntriesInformationSchema(ClientContext &context, PostgresTransaction &transaction) { + string schema_query = GetInitializeQueryInformationSchema(schemas_to_load); + string tables_query = PostgresTableSet::GetInitializeQueryInformationSchema(schemas_to_load); + // Reading enums, composite types and indexes differs across DBs that emulate PG + // choice to either return those empty or have vendor specific queries for each. + // empty for now + string enum_types_query = "SELECT NULL, NULL, NULL, NULL LIMIT 0;\n"; + string composite_types_query = "SELECT NULL, NULL, NULL, NULL, NULL LIMIT 0;\n"; + string index_query = "SELECT NULL, NULL, NULL LIMIT 0;\n"; + + auto full_query = schema_query + tables_query + enum_types_query + composite_types_query + index_query; + + auto results = transaction.ExecuteQueries(context, full_query); + auto result = std::move(results[0]); + results.erase(results.begin()); + auto rows = result->Count(); + + auto tables = SliceResultByName(*result, std::move(results[0])); + auto enums = SliceResultByName(*result, std::move(results[1])); + auto composite_types = SliceResultByName(*result, std::move(results[2])); + auto indexes = SliceResultByName(*result, std::move(results[3])); + for (idx_t row = 0; row < rows; row++) { + auto schema_name = result->GetString(row, 0); + CreateSchemaInfo info; + info.SetSchema(Identifier(schema_name)); + info.internal = PostgresSchemaEntry::SchemaIsInternal(schema_name); + auto schema = make_shared_ptr(catalog, info, std::move(tables[row]), std::move(enums[row]), + std::move(composite_types[row]), std::move(indexes[row])); + CreateEntry(transaction, std::move(schema)); + } +} + void PostgresSchemaSet::LoadEntries(ClientContext &context, PostgresTransaction &transaction) { + if (PostgresUtils::UseInformationSchemaIntrospection(context)) { + LoadEntriesInformationSchema(context, transaction); + return; + } auto &pg_catalog = catalog.Cast(); auto pg_version = pg_catalog.GetPostgresVersion(); - string schema_query = PostgresSchemaSet::GetInitializeQuery(schema_to_load); - string tables_query = PostgresTableSet::GetInitializeQuery(schema_to_load); - string enum_types_query = PostgresTypeSet::GetInitializeEnumsQuery(pg_version, schema_to_load); - string composite_types_query = PostgresTypeSet::GetInitializeCompositesQuery(schema_to_load); - string index_query = PostgresIndexSet::GetInitializeQuery(schema_to_load); + string schema_query = PostgresSchemaSet::GetInitializeQuery(schemas_to_load); + string tables_query = PostgresTableSet::GetInitializeQuery(schemas_to_load); + string enum_types_query = PostgresTypeSet::GetInitializeEnumsQuery(pg_version, schemas_to_load); + string composite_types_query = PostgresTypeSet::GetInitializeCompositesQuery(schemas_to_load); + string index_query = PostgresIndexSet::GetInitializeQuery(schemas_to_load); auto full_query = schema_query + tables_query + enum_types_query + composite_types_query + index_query; @@ -75,7 +151,8 @@ void PostgresSchemaSet::LoadEntries(ClientContext &context, PostgresTransaction auto oid = result->GetInt64(row, 0); auto schema_name = result->GetString(row, 1); CreateSchemaInfo info; - info.schema = Identifier(schema_name); + info.SetQualifiedName( + QualifiedName(info.GetQualifiedName().Catalog(), Identifier(schema_name), info.GetQualifiedName().Name())); info.internal = PostgresSchemaEntry::SchemaIsInternal(schema_name); auto schema = make_shared_ptr(catalog, info, std::move(tables[row]), std::move(enums[row]), std::move(composite_types[row]), std::move(indexes[row])); @@ -88,10 +165,10 @@ optional_ptr PostgresSchemaSet::CreateSchema(PostgresTransaction & if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { create_sql += " IF NOT EXISTS"; } - create_sql += PostgresUtils::WriteIdentifier(info.schema.GetIdentifierName()); + create_sql += PostgresUtils::WriteIdentifier(info.GetQualifiedName().Schema().GetIdentifierName()); transaction.Query(create_sql); auto info_copy = info.Copy(); - info.internal = PostgresSchemaEntry::SchemaIsInternal(info_copy->schema.GetIdentifierName()); + info.internal = PostgresSchemaEntry::SchemaIsInternal(info_copy->GetQualifiedName().Schema().GetIdentifierName()); auto schema_entry = make_shared_ptr(catalog, info_copy->Cast()); return CreateEntry(transaction, std::move(schema_entry)); } diff --git a/src/storage/postgres_table_entry.cpp b/src/storage/postgres_table_entry.cpp index 8479e8bb9..7f5235f55 100644 --- a/src/storage/postgres_table_entry.cpp +++ b/src/storage/postgres_table_entry.cpp @@ -1,15 +1,20 @@ #include "storage/postgres_catalog.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "storage/postgres_table_entry.hpp" #include "storage/postgres_transaction.hpp" #include "duckdb/storage/statistics/base_statistics.hpp" #include "duckdb/storage/table_storage_info.hpp" #include "duckdb/parser/constraints/unique_constraint.hpp" +#include "postgres_filter_pushdown.hpp" #include "postgres_scanner.hpp" namespace duckdb { PostgresTableEntry::PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) - : TableCatalogEntry(catalog, schema, info) { + : TableCatalogEntry(catalog, schema, info), columns(std::move(info.columns)) { for (idx_t c = 0; c < columns.LogicalColumnCount(); c++) { auto &col = columns.GetColumnMutable(LogicalIndex(c)); if (col.GetType().HasAlias()) { @@ -22,12 +27,16 @@ PostgresTableEntry::PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &sch } PostgresTableEntry::PostgresTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, PostgresTableInfo &info) - : TableCatalogEntry(catalog, schema, *info.create_info), postgres_types(std::move(info.postgres_types)), - postgres_names(std::move(info.postgres_names)) { + : TableCatalogEntry(catalog, schema, *info.create_info), columns(std::move(info.create_info->columns)), + postgres_types(std::move(info.postgres_types)), postgres_names(std::move(info.postgres_names)) { D_ASSERT(postgres_types.size() == columns.LogicalColumnCount()); approx_num_pages.store(info.approx_num_pages, std::memory_order_release); } +const ColumnList &PostgresTableEntry::GetColumns() const { + return columns; +} + unique_ptr PostgresTableEntry::GetStatistics(ClientContext &context, column_t column_id) { return nullptr; } @@ -40,29 +49,33 @@ TableFunction PostgresTableEntry::GetScanFunction(ClientContext &context, unique auto &pg_catalog = catalog.Cast(); auto &transaction = Transaction::Get(context, catalog).Cast(); - auto result = make_uniq(context); + auto result = make_uniq(context, pg_catalog.GetName()); result->schema_name = schema.name.GetIdentifierName(); result->table_name = name.GetIdentifierName(); result->dsn = transaction.GetDSN(); result->attach_path = pg_catalog.attach_path; - result->SetCatalog(pg_catalog); - result->SetTable(*this); + result->qualified_table_name = QualifiedName(ParentCatalog().GetName(), ParentSchema().name, name); + result->context_ptr = transaction.context; for (auto &col : columns.Logical()) { result->types.push_back(col.GetType()); } result->names = postgres_names; result->postgres_types = postgres_types; result->read_only = transaction.IsReadOnly(); + result->type_config = PostgresTypeConfig::FromContext(context); PostgresScanFunction::PrepareBind(pg_catalog.GetPostgresVersion(), context, *result, - approx_num_pages.load(std::memory_order_acquire)); + approx_num_pages.load(std::memory_order_acquire), pg_catalog); bind_data = std::move(result); auto function = PostgresScanFunction(); Value filter_pushdown; - if (context.TryGetCurrentSetting("pg_experimental_filter_pushdown", filter_pushdown)) { + if (context.TryGetCurrentSetting("pg_enable_filter_pushdown", filter_pushdown)) { function.filter_pushdown = BooleanValue::Get(filter_pushdown); } + if (function.filter_pushdown) { + function.pushdown_expression = PostgresFilterPushdown::CanPushExpressionDown; + } return function; } @@ -133,13 +146,32 @@ static bool CopyRequiresText(const LogicalType &type, const PostgresType &pg_typ } PostgresCopyFormat PostgresTableEntry::GetCopyFormat(ClientContext &context) { - Value use_binary_copy; - if (context.TryGetCurrentSetting("pg_use_binary_copy", use_binary_copy)) { - if (!BooleanValue::Get(use_binary_copy)) { - return PostgresCopyFormat::TEXT; + PostgresQueryMode query_mode = PostgresUtils::ExtractQueryMode(context); + switch (query_mode) { + case PostgresQueryMode::COPY_BINARY: + case PostgresQueryMode::PLAIN_QUERY_BINARY: + return GetBinaryCopyFormatIfTypesAllow(); + case PostgresQueryMode::PLAIN_QUERY_TEXT: + return PostgresCopyFormat::TEXT; + default: { + Value use_binary_copy; + if (context.TryGetCurrentSetting("pg_use_binary_copy", use_binary_copy)) { + if (!BooleanValue::Get(use_binary_copy)) { + return PostgresCopyFormat::TEXT; + } } + return GetBinaryCopyFormatIfTypesAllow(); + } + } +} + +PostgresCopyFormat PostgresTableEntry::GetBinaryCopyFormatIfTypesAllow() { + idx_t types_count = postgres_types.size(); + idx_t columns_count = columns.LogicalColumnCount(); + if (types_count != columns_count) { + throw IOException("Schema discrepancy detected, types count: %llu, columns count: %llu", types_count, + columns_count); } - D_ASSERT(postgres_types.size() == columns.LogicalColumnCount()); for (idx_t c = 0; c < postgres_types.size(); c++) { if (CopyRequiresText(columns.GetColumn(LogicalIndex(c)).GetType(), postgres_types[c])) { return PostgresCopyFormat::TEXT; @@ -148,4 +180,15 @@ PostgresCopyFormat PostgresTableEntry::GetCopyFormat(ClientContext &context) { return PostgresCopyFormat::BINARY; } +dbconnector::attached::AttachedTable PostgresTableEntry::Lookup(ClientContext &ctx, QualifiedName name) { + using namespace dbconnector::attached; + + AttachedTable table = AttachedTable::Lookup(ctx, "postgres", name); + if (!table) { + throw InvalidInputException("Attached PostgreSQL table, name: %s is not found in the specified client session", + name.ToString()); + } + return table; +} + } // namespace duckdb diff --git a/src/storage/postgres_table_set.cpp b/src/storage/postgres_table_set.cpp index 663e6d523..7ef008b6b 100644 --- a/src/storage/postgres_table_set.cpp +++ b/src/storage/postgres_table_set.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_table_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" #include "duckdb/parser/constraints/not_null_constraint.hpp" @@ -23,6 +25,12 @@ PostgresTableSet::PostgresTableSet(PostgresSchemaEntry &schema, unique_ptr vec; + vec.push_back(schema); + return GetInitializeQuery(vec, table); +} + +string PostgresTableSet::GetInitializeQuery(const vector &schemas, const string &table) { string base_query = R"( SELECT pg_namespace.oid AS namespace_id, relname, relpages, attname, pg_type.typname type_name, atttypmod type_modifier, pg_attribute.attndims ndim, @@ -51,8 +59,8 @@ WHERE relkind IN ('r', 'v', 'm', 'f', 'p') AND contype IN ('p', 'u') ${CONDITION ORDER BY namespace_id, relname, attnum, constraint_id; )"; string condition; - if (!schema.empty()) { - condition += "AND pg_namespace.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "AND pg_namespace.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } if (!table.empty()) { condition += "AND relname=" + PostgresUtils::WriteLiteral(table); @@ -60,13 +68,40 @@ ORDER BY namespace_id, relname, attnum, constraint_id; return StringUtil::Replace(base_query, "${CONDITION}", condition); } +string PostgresTableSet::GetInitializeQueryInformationSchema(const string &schema, const string &table) { + vector vec; + vec.push_back(schema); + return GetInitializeQueryInformationSchema(vec, table); +} + +string PostgresTableSet::GetInitializeQueryInformationSchema(const vector &schemas, const string &table) { + string base_query = R"( +SELECT table_schema AS namespace_id, table_name AS relname, 0 AS relpages, column_name AS attname, + data_type AS type_name, -1 AS type_modifier, 0 AS ndim, ordinal_position AS attnum, + CASE WHEN is_nullable = 'NO' THEN 't' ELSE 'f' END AS notnull, + NULL AS constraint_id, NULL AS constraint_type, NULL AS constraint_key, + NULL AS type_schema, NULL AS column_comment, NULL AS table_comment +FROM information_schema.columns +WHERE table_schema NOT IN ('information_schema', 'pg_catalog', 'pg_toast') ${CONDITION} +ORDER BY table_schema, table_name, ordinal_position; +)"; + string condition; + if (schemas.size() > 0) { + condition += "AND table_schema IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; + } + if (!table.empty()) { + condition += " AND table_name=" + PostgresUtils::WriteLiteral(table); + } + return StringUtil::Replace(base_query, "${CONDITION}", condition); +} + void PostgresTableSet::AddColumn(optional_ptr transaction, - optional_ptr schema, PostgresResult &result, idx_t row, - PostgresTableInfo &table_info) { + optional_ptr schema, const PostgresTypeConfig &type_config, + PostgresResult &result, idx_t row, PostgresTableInfo &table_info) { PostgresTypeData type_info; idx_t column_index = 3; auto column_name = result.GetString(row, column_index); - type_info.type_name = result.GetString(row, column_index + 1); + type_info.type_name = PostgresUtils::DataTypeToTypeName(result.GetString(row, column_index + 1)); type_info.type_modifier = result.GetInt64(row, column_index + 2); type_info.array_dimensions = result.GetInt64(row, column_index + 3); bool is_not_null = result.GetBool(row, column_index + 5); @@ -78,7 +113,7 @@ void PostgresTableSet::AddColumn(optional_ptr transaction, string default_value; PostgresType postgres_type; - auto column_type = PostgresUtils::TypeToLogicalType(transaction, schema, type_info, postgres_type); + auto column_type = PostgresUtils::TypeToLogicalType(transaction, schema, type_config, type_info, postgres_type); table_info.postgres_types.push_back(std::move(postgres_type)); table_info.postgres_names.push_back(column_name); ColumnDefinition column(Identifier(std::move(column_name)), std::move(column_type)); @@ -86,13 +121,17 @@ void PostgresTableSet::AddColumn(optional_ptr transaction, column.SetComment(Value(column_comment)); } if (!default_value.empty()) { - auto expressions = Parser::ParseExpressionList(default_value); + auto expressions = Parser::GetBuiltinParser().ParseExpressionList(default_value); if (expressions.empty()) { throw InternalException("Expression list is empty"); } column.SetDefaultValue(std::move(expressions[0])); } auto &create_info = *table_info.create_info; + if (!result.IsNull(row, 7)) { + auto attnum = result.GetInt64(row, 7); + table_info.attnum_to_logical[attnum] = create_info.columns.LogicalColumnCount(); + } if (is_not_null) { create_info.constraints.push_back( make_uniq(LogicalIndex(create_info.columns.PhysicalColumnCount()))); @@ -114,24 +153,26 @@ void PostgresTableSet::AddConstraint(PostgresResult &result, idx_t row, Postgres auto splits = StringUtil::Split(constraint_key.substr(1, constraint_key.size() - 2), ","); vector columns; for (auto &split : splits) { - auto index = std::stoull(split); - if (index <= 0 || index > create_info.columns.LogicalColumnCount()) { + auto attnum = std::stoll(split); + auto entry = table_info.attnum_to_logical.find(attnum); + if (entry == table_info.attnum_to_logical.end()) { return; } - columns.push_back(create_info.columns.GetColumn(LogicalIndex(index - 1)).Name()); + columns.push_back(create_info.columns.GetColumn(LogicalIndex(entry->second)).Name()); } create_info.constraints.push_back(make_uniq(std::move(columns), constraint_type == "p")); } void PostgresTableSet::AddColumnOrConstraint(optional_ptr transaction, - optional_ptr schema, PostgresResult &result, - idx_t row, PostgresTableInfo &table_info) { + optional_ptr schema, + const PostgresTypeConfig &type_config, PostgresResult &result, idx_t row, + PostgresTableInfo &table_info) { if (result.IsNull(row, 3)) { // constraint AddConstraint(result, row, table_info); } else { - AddColumn(transaction, schema, result, row, table_info); + AddColumn(transaction, schema, type_config, result, row, table_info); } } @@ -139,6 +180,7 @@ void PostgresTableSet::CreateEntries(PostgresTransaction &transaction, PostgresR vector> tables; unique_ptr info; + auto type_config = PostgresTypeConfig::FromContext(transaction.GetContext()); for (idx_t row = start; row < end; row++) { auto table_name = result.GetString(row, 1); if (!info || info->GetTableName() != table_name) { @@ -152,7 +194,7 @@ void PostgresTableSet::CreateEntries(PostgresTransaction &transaction, PostgresR info->create_info->comment = Value(result.GetString(row, 14)); } } - AddColumnOrConstraint(&transaction, &schema, result, row, *info); + AddColumnOrConstraint(&transaction, &schema, type_config, result, row, *info); } if (info) { tables.push_back(std::move(info)); @@ -168,7 +210,9 @@ void PostgresTableSet::LoadEntries(ClientContext &context, PostgresTransaction & CreateEntries(transaction, table_result->GetResult(), table_result->start, table_result->end); table_result.reset(); } else { - auto query = GetInitializeQuery(schema.name.GetIdentifierName()); + auto query = PostgresUtils::UseInformationSchemaIntrospection(context) + ? GetInitializeQueryInformationSchema(schema.name.GetIdentifierName()) + : GetInitializeQuery(schema.name.GetIdentifierName()); auto result = transaction.Query(query); auto rows = result->Count(); @@ -177,17 +221,42 @@ void PostgresTableSet::LoadEntries(ClientContext &context, PostgresTransaction & } } +string PostgresTableSet::GetStalenessQuery(ClientContext &context) const { + if (!PostgresUtils::StalenessQueryEnabled(context)) { + return string(); + } + auto custom_query = PostgresUtils::GetCustomStalenessQuery(context); + if (!custom_query.empty()) { + if (custom_query.find("${SCHEMA}") == string::npos) { + throw InvalidInputException("pg_staleness_query must contain a ${SCHEMA} placeholder"); + } + return StringUtil::Replace(custom_query, "${SCHEMA}", + PostgresUtils::WriteLiteral(schema.name.GetIdentifierName())); + } + string base_query = R"( +SELECT pg_class.oid, relname, pg_class.xmin +FROM pg_class +JOIN pg_namespace ON relnamespace = pg_namespace.oid +WHERE relkind IN ('r', 'v', 'm', 'f', 'p') AND pg_namespace.nspname = ${SCHEMA} +ORDER BY pg_class.oid; +)"; + return StringUtil::Replace(base_query, "${SCHEMA}", PostgresUtils::WriteLiteral(schema.name.GetIdentifierName())); +} + unique_ptr PostgresTableSet::GetTableInfo(PostgresTransaction &transaction, PostgresSchemaEntry &schema, const string &table_name) { - auto query = PostgresTableSet::GetInitializeQuery(schema.name.GetIdentifierName(), table_name); + auto query = PostgresUtils::UseInformationSchemaIntrospection(transaction.GetContext()) + ? GetInitializeQueryInformationSchema(schema.name.GetIdentifierName(), table_name) + : GetInitializeQuery(schema.name.GetIdentifierName(), table_name); auto result = transaction.Query(query); auto rows = result->Count(); if (rows == 0) { return nullptr; } auto table_info = make_uniq(schema, table_name); + auto type_config = PostgresTypeConfig::FromContext(transaction.GetContext()); for (idx_t row = 0; row < rows; row++) { - AddColumnOrConstraint(&transaction, &schema, *result, row, *table_info); + AddColumnOrConstraint(&transaction, &schema, type_config, *result, row, *table_info); } table_info->approx_num_pages = result->IsNull(0, 2) ? 0 : result->GetInt64(0, 2); // Read table-level comment from 14 @@ -199,15 +268,18 @@ unique_ptr PostgresTableSet::GetTableInfo(PostgresTransaction unique_ptr PostgresTableSet::GetTableInfo(ClientContext &context, PostgresConnection &connection, const string &schema_name, const string &table_name) { - auto query = PostgresTableSet::GetInitializeQuery(schema_name, table_name); + auto query = PostgresUtils::UseInformationSchemaIntrospection(context) + ? GetInitializeQueryInformationSchema(schema_name, table_name) + : GetInitializeQuery(schema_name, table_name); auto result = connection.Query(context, query); auto rows = result->Count(); if (rows == 0) { throw InvalidInputException("Table %s does not contain any columns.", table_name); } auto table_info = make_uniq(schema_name, table_name); + auto type_config = PostgresTypeConfig::FromContext(context); for (idx_t row = 0; row < rows; row++) { - AddColumnOrConstraint(nullptr, nullptr, *result, row, *table_info); + AddColumnOrConstraint(nullptr, nullptr, type_config, *result, row, *table_info); } table_info->approx_num_pages = result->IsNull(0, 2) ? 0 : result->GetInt64(0, 2); // Read table-level comment @@ -223,7 +295,9 @@ optional_ptr PostgresTableSet::ReloadEntry(PostgresTransaction &tr return nullptr; } auto table_entry = make_shared_ptr(catalog, schema, *table_info); - return CreateEntry(transaction, std::move(table_entry)); + auto result = CreateEntry(transaction, std::move(table_entry)); + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/false); + return result; } // FIXME - this is almost entirely copied from TableCatalogEntry::ColumnsToSQL - should be unified @@ -331,11 +405,11 @@ string GetPostgresCreateTable(CreateTableInfo &info) { if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { ss << "IF NOT EXISTS "; } - if (!info.schema.empty()) { - ss << PostgresUtils::WriteIdentifier(info.schema.GetIdentifierName()); + if (!info.GetQualifiedName().Schema().empty()) { + ss << PostgresUtils::WriteIdentifier(info.GetQualifiedName().Schema().GetIdentifierName()); ss << "."; } - ss << PostgresUtils::WriteIdentifier(info.table.GetIdentifierName()); + ss << PostgresUtils::WriteIdentifier(info.GetTableName().GetIdentifierName()); ss << PostgresColumnsToSQL(info.columns, info.constraints); ss << ";"; return ss.str(); @@ -345,7 +419,9 @@ optional_ptr PostgresTableSet::CreateTable(PostgresTransaction &tr auto create_sql = GetPostgresCreateTable(info.Base()); transaction.Query(create_sql); auto tbl_entry = make_shared_ptr(catalog, schema, info.Base()); - return CreateEntry(transaction, std::move(tbl_entry)); + auto result = CreateEntry(transaction, std::move(tbl_entry)); + RefreshStalenessSignature(transaction, /*use_transaction_connection=*/true); + return result; } string PostgresTableSet::GetAlterTablePrefix(const string &name, optional_ptr entry) { @@ -375,15 +451,15 @@ string PostgresTableSet::GetAlterTablePrefix(ClientContext &context, PostgresTra } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameTableInfo &info) { - string sql = GetAlterTablePrefix(context, transaction, info.name.GetIdentifierName()); + string sql = GetAlterTablePrefix(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); sql += " RENAME TO "; sql += PostgresUtils::WriteIdentifier(info.new_table_name.GetIdentifierName()); transaction.Query(sql); } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, RenameColumnInfo &info) { - auto entry = GetEntry(context, transaction, info.name.GetIdentifierName()); - string sql = GetAlterTablePrefix(info.name.GetIdentifierName(), entry); + auto entry = GetEntry(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); + string sql = GetAlterTablePrefix(info.GetQualifiedName().Name().GetIdentifierName(), entry); sql += " RENAME COLUMN "; string column_name = GetAlterTableColumnName(info.old_name.GetIdentifierName(), entry); sql += PostgresUtils::WriteIdentifier(column_name); @@ -394,7 +470,7 @@ void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &t } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, AddColumnInfo &info) { - string sql = GetAlterTablePrefix(context, transaction, info.name.GetIdentifierName()); + string sql = GetAlterTablePrefix(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); sql += " ADD COLUMN "; if (info.if_column_not_exists) { sql += "IF NOT EXISTS "; @@ -410,7 +486,7 @@ void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &t "only constant DEFAULT expressions are supported"); } const ConstantExpression &default_const_expr = default_expr.Cast(); - std::string sql_str = default_const_expr.GetValue().ToSQLString(); + std::string sql_str = default_const_expr.GetLiteral().ToValue().ToSQLString(); sql += " DEFAULT "; sql += sql_str; } @@ -419,8 +495,8 @@ void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &t } void PostgresTableSet::AlterTable(ClientContext &context, PostgresTransaction &transaction, RemoveColumnInfo &info) { - auto entry = GetEntry(context, transaction, info.name.GetIdentifierName()); - string sql = GetAlterTablePrefix(info.name.GetIdentifierName(), entry); + auto entry = GetEntry(context, transaction, info.GetQualifiedName().Name().GetIdentifierName()); + string sql = GetAlterTablePrefix(info.GetQualifiedName().Name().GetIdentifierName(), entry); sql += " DROP COLUMN "; if (info.if_column_exists) { sql += "IF EXISTS "; diff --git a/src/storage/postgres_transaction.cpp b/src/storage/postgres_transaction.cpp index 26c990bcf..087c549e6 100644 --- a/src/storage/postgres_transaction.cpp +++ b/src/storage/postgres_transaction.cpp @@ -1,5 +1,8 @@ #include "storage/postgres_transaction.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog.hpp" #include "storage/postgres_catalog.hpp" +#include "storage/postgres_catalog_set.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" @@ -13,7 +16,7 @@ PostgresTransaction::PostgresTransaction(PostgresCatalog &postgres_catalog, Tran : Transaction(manager, context), access_mode(postgres_catalog.access_mode), isolation_level(postgres_catalog.isolation_level) { auto oauth_token_holder = SetThreadLocalOAuthTokenFromSessionOption(context); - connection = postgres_catalog.GetConnectionPool().GetConnection(); + connection = postgres_catalog.GetConnectionPool().Acquire(); } PostgresTransaction::~PostgresTransaction() = default; @@ -30,12 +33,22 @@ void PostgresTransaction::Commit() { transaction_state = PostgresTransactionState::TRANSACTION_FINISHED; GetConnectionRaw().Execute(GetContext(), "COMMIT"); } + vector, string>> to_promote; + { + lock_guard l(pending_signatures_lock); + to_promote = std::move(pending_signatures); + } + for (auto &entry : to_promote) { + entry.first.get().PromoteStalenessSignature(std::move(entry.second)); + } } void PostgresTransaction::Rollback() { if (transaction_state == PostgresTransactionState::TRANSACTION_STARTED) { transaction_state = PostgresTransactionState::TRANSACTION_FINISHED; GetConnectionRaw().Execute(GetContext(), "ROLLBACK"); } + lock_guard l(pending_signatures_lock); + pending_signatures.clear(); } string PostgresTransaction::GetBeginTransactionQuery() { @@ -126,10 +139,16 @@ vector> PostgresTransaction::ExecuteQueries(ClientCon optional_ptr PostgresTransaction::ReferenceEntry(shared_ptr &entry) { auto &ref = *entry; + lock_guard l(referenced_entries_lock); referenced_entries.emplace(ref, entry); return ref; } +void PostgresTransaction::StageStalenessSignature(PostgresCatalogSet &catalog_set, string signature) { + lock_guard l(pending_signatures_lock); + pending_signatures.emplace_back(catalog_set, std::move(signature)); +} + string PostgresTransaction::GetTemporarySchema() { if (temporary_schema.empty()) { auto result = Query("SELECT nspname FROM pg_namespace WHERE oid = pg_my_temp_schema();"); diff --git a/src/storage/postgres_transaction_manager.cpp b/src/storage/postgres_transaction_manager.cpp deleted file mode 100644 index b1347ba1e..000000000 --- a/src/storage/postgres_transaction_manager.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "storage/postgres_transaction_manager.hpp" -#include "duckdb/main/attached_database.hpp" - -namespace duckdb { - -PostgresTransactionManager::PostgresTransactionManager(AttachedDatabase &db_p, PostgresCatalog &postgres_catalog) - : TransactionManager(db_p), postgres_catalog(postgres_catalog) { -} - -Transaction &PostgresTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(postgres_catalog, *this, context); - transaction->Start(); - auto &result = *transaction; - lock_guard l(transaction_lock); - transactions[result] = std::move(transaction); - return result; -} - -ErrorData PostgresTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &postgres_transaction = transaction.Cast(); - postgres_transaction.Commit(); - lock_guard l(transaction_lock); - transactions.erase(transaction); - return ErrorData(); -} - -void PostgresTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &postgres_transaction = transaction.Cast(); - postgres_transaction.Rollback(); - lock_guard l(transaction_lock); - transactions.erase(transaction); -} - -void PostgresTransactionManager::Checkpoint(ClientContext &context, bool force) { - auto &transaction = PostgresTransaction::Get(context, db.GetCatalog()); - auto &db = transaction.GetConnection(); - db.Execute(context, "CHECKPOINT"); -} - -} // namespace duckdb diff --git a/src/storage/postgres_type_entry.cpp b/src/storage/postgres_type_entry.cpp index 5d9d621c4..ccc02821c 100644 --- a/src/storage/postgres_type_entry.cpp +++ b/src/storage/postgres_type_entry.cpp @@ -1,4 +1,6 @@ #include "storage/postgres_type_entry.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" namespace duckdb { diff --git a/src/storage/postgres_type_set.cpp b/src/storage/postgres_type_set.cpp index cee52ac28..f956aa86f 100644 --- a/src/storage/postgres_type_set.cpp +++ b/src/storage/postgres_type_set.cpp @@ -1,4 +1,5 @@ #include "storage/postgres_type_set.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/parser/parsed_data/create_type_info.hpp" #include "duckdb/parser/expression/constant_expression.hpp" @@ -23,7 +24,7 @@ PostgresTypeSet::PostgresTypeSet(PostgresSchemaEntry &schema, unique_ptr &schemas) { if (version.major_v < 8 || (version.major_v == 8 && version.minor_v < 3)) { // pg_enum support has been present since v8.3 - https://www.postgresql.org/docs/8.3/catalog-pg-enum.html // for older postgres versions we don't support enums instead @@ -41,8 +42,8 @@ JOIN pg_namespace AS n ON (typnamespace=n.oid) ORDER BY n.oid, enumtypid, enumsortorder; )"; string condition; - if (!schema.empty()) { - condition += "WHERE n.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "WHERE n.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } @@ -52,15 +53,14 @@ void PostgresTypeSet::CreateEnum(PostgresTransaction &transaction, PostgresResul PostgresType postgres_type; CreateTypeInfo info; postgres_type.oid = result.GetInt64(start_row, 1); - info.name = Identifier(result.GetString(start_row, 2)); + info.SetTypeName(Identifier(result.GetString(start_row, 2))); // construct the enum idx_t enum_count = end_row - start_row; Vector duckdb_levels(LogicalType::VARCHAR, enum_count); for (idx_t enum_idx = 0; enum_idx < enum_count; enum_idx++) { duckdb_levels.SetValue(enum_idx, result.GetString(start_row + enum_idx, 3)); } - info.type = LogicalType::ENUM(duckdb_levels, enum_count); - info.type.SetAlias(info.name.GetIdentifierName()); + info.type = LogicalType::ENUM(duckdb_levels, enum_count).WithAlias(info.GetTypeName().GetIdentifierName()); auto type_entry = make_shared_ptr(catalog, schema, info, postgres_type); CreateEntry(transaction, std::move(type_entry)); } @@ -85,7 +85,7 @@ void PostgresTypeSet::InitializeEnums(PostgresTransaction &transaction, Postgres } } -string PostgresTypeSet::GetInitializeCompositesQuery(const string &schema) { +string PostgresTypeSet::GetInitializeCompositesQuery(const vector &schemas) { string base_query = R"( SELECT n.oid, t.typrelid AS id, t.typname as type, pg_attribute.attname, sub_type.typname, sub_type_ns.nspname AS sub_type_schema @@ -103,8 +103,8 @@ AND NOT pg_attribute.attisdropped ORDER BY n.oid, t.oid, attrelid, attnum; )"; string condition; - if (!schema.empty()) { - condition += "AND n.nspname=" + PostgresUtils::WriteLiteral(schema); + if (schemas.size() > 0) { + condition += "AND n.nspname IN (" + PostgresUtils::WriteLiteralsCommaSeparated(schemas) + ")"; } return StringUtil::Replace(base_query, "${CONDITION}", condition); } @@ -114,21 +114,22 @@ void PostgresTypeSet::CreateCompositeType(PostgresTransaction &transaction, Post PostgresType postgres_type; CreateTypeInfo info; postgres_type.oid = result.GetInt64(start_row, 1); - info.name = Identifier(result.GetString(start_row, 2)); + info.SetTypeName(Identifier(result.GetString(start_row, 2))); child_list_t child_types; + auto type_config = PostgresTypeConfig::FromContext(transaction.GetContext()); for (idx_t row = start_row; row < end_row; row++) { auto type_name = result.GetString(row, 3); PostgresTypeData type_data; type_data.type_name = result.GetString(row, 4); type_data.type_schema = result.GetString(row, 5); PostgresType child_type; - child_types.push_back(make_pair( - Identifier(type_name), PostgresUtils::TypeToLogicalType(&transaction, &schema, type_data, child_type))); + child_types.push_back( + make_pair(Identifier(type_name), + PostgresUtils::TypeToLogicalType(&transaction, &schema, type_config, type_data, child_type))); postgres_type.children.push_back(std::move(child_type)); } - info.type = LogicalType::STRUCT(std::move(child_types)); - info.type.SetAlias(info.name.GetIdentifierName()); + info.type = LogicalType::STRUCT(std::move(child_types)).WithAlias(info.GetTypeName().GetIdentifierName()); auto type_entry = make_shared_ptr(catalog, schema, info, postgres_type); CreateEntry(transaction, std::move(type_entry)); } @@ -165,7 +166,7 @@ void PostgresTypeSet::LoadEntries(ClientContext &context, PostgresTransaction &t string GetCreateTypeSQL(CreateTypeInfo &info) { string sql = "CREATE TYPE "; - sql += PostgresUtils::WriteIdentifier(info.name.GetIdentifierName()); + sql += PostgresUtils::WriteIdentifier(info.GetTypeName().GetIdentifierName()); sql += " AS "; switch (info.type.id()) { case LogicalTypeId::ENUM: { @@ -206,7 +207,7 @@ optional_ptr PostgresTypeSet::CreateType(PostgresTransaction &tran auto create_sql = GetCreateTypeSQL(info); conn.Execute(transaction.GetContext(), create_sql); - info.type.SetAlias(info.name.GetIdentifierName()); + info.type = info.type.WithAlias(info.GetTypeName().GetIdentifierName()); auto pg_type = PostgresUtils::CreateEmptyPostgresType(info.type); auto type_entry = make_shared_ptr(catalog, schema, info, pg_type); return CreateEntry(transaction, std::move(type_entry)); diff --git a/src/storage/postgres_update.cpp b/src/storage/postgres_update.cpp index 49c47e00c..95ab2e81f 100644 --- a/src/storage/postgres_update.cpp +++ b/src/storage/postgres_update.cpp @@ -1,4 +1,8 @@ #include "storage/postgres_update.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" #include "storage/postgres_table_entry.hpp" #include "duckdb/planner/operator/logical_update.hpp" diff --git a/test/sql/concurrent/concurrent_attach.test b/test/sql/concurrent/concurrent_attach.test new file mode 100644 index 000000000..93ebabea3 --- /dev/null +++ b/test/sql/concurrent/concurrent_attach.test @@ -0,0 +1,31 @@ +# name: test/sql/concurrent/concurrent_attach.test +# description: Test ATTACH called concurrently from multiple threads +# group: [concurrent] + +require mysql_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s + +statement ok +SET threads = 4 + +concurrentloop i 0 16 + +statement ok +ATTACH 'dbname=postgresscanner' AS s{i} (TYPE POSTGRES) + +query I +SELECT count(*) FROM s{i}.pg_numtypes +---- +4 + +statement ok +DETACH s{i} + +endloop diff --git a/test/sql/concurrent/concurrent_clear_cache.test b/test/sql/concurrent/concurrent_clear_cache.test new file mode 100644 index 000000000..5580fb892 --- /dev/null +++ b/test/sql/concurrent/concurrent_clear_cache.test @@ -0,0 +1,36 @@ +# name: test/sql/concurrent/concurrent_clear_cache.test +# description: Test concurrent drop of the table being quried +# group: [concurrent] + +require mysql_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.concurrent_drop_test AS +SELECT unnest(range(1<<12)) col1 + +loop i 0 16 + +concurrentloop j 0 16 + +statement ok +CALL postgres_clear_cache() + +query I +SELECT divide(count(*), 1) FROM s.concurrent_drop_test +---- +4096 + +endloop + +endloop + +statement ok +DROP TABLE s.concurrent_drop_test + +statement ok +DETACH s diff --git a/test/sql/storage/attach_concurrent_queries.test b/test/sql/concurrent/concurrent_queries.test similarity index 90% rename from test/sql/storage/attach_concurrent_queries.test rename to test/sql/concurrent/concurrent_queries.test index 852602f0f..890fad701 100644 --- a/test/sql/storage/attach_concurrent_queries.test +++ b/test/sql/concurrent/concurrent_queries.test @@ -1,6 +1,6 @@ -# name: test/sql/storage/attach_concurrent_queries.test +# name: test/sql/concurrent/concurrent_queries.test # description: Run queries concurrently over different connections -# group: [storage] +# group: [concurrent] require postgres_scanner diff --git a/test/sql/concurrent/concurrent_results.test b/test/sql/concurrent/concurrent_results.test new file mode 100644 index 000000000..1c881a5da --- /dev/null +++ b/test/sql/concurrent/concurrent_results.test @@ -0,0 +1,32 @@ +# name: test/sql/concurrent/concurrent_results.test +# description: Test concurrent multiple results reading from multple scans +# group: [concurrent] + +require mysql_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_a AS +SELECT '2020-01-01' insert_date, unnest(range(1<<14)) amount + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_b AS +SELECT '2021-01-01' insert_date, unnest(range(1<<14)) amount + +statement ok +SET threads = 4 + +concurrentloop i 0 16 + +query II +SELECT date(t.insert_date) AS col1, divide(sum(t.amount), 1) AS col2 +FROM s.aggr_union_test_a t +GROUP BY 1 +---- +2020-01-01 134209536 + +endloop diff --git a/test/sql/storage/attach_connection_pool.test_slow b/test/sql/connection_pool/connection_pool.test_slow similarity index 96% rename from test/sql/storage/attach_connection_pool.test_slow rename to test/sql/connection_pool/connection_pool.test_slow index 56f0d7f77..99f680322 100644 --- a/test/sql/storage/attach_connection_pool.test_slow +++ b/test/sql/connection_pool/connection_pool.test_slow @@ -1,6 +1,6 @@ -# name: test/sql/storage/attach_connection_pool.test_slow +# name: test/sql/connection_pool/connection_pool.test_slow # description: Test that the connection pool correctly limits the number of connections -# group: [storage] +# group: [connection_pool] require postgres_scanner diff --git a/test/sql/storage/attach_connection_pool_configure.test b/test/sql/connection_pool/connection_pool_configure.test similarity index 66% rename from test/sql/storage/attach_connection_pool_configure.test rename to test/sql/connection_pool/connection_pool_configure.test index 84c46e5ec..aebe2157b 100644 --- a/test/sql/storage/attach_connection_pool_configure.test +++ b/test/sql/connection_pool/connection_pool_configure.test @@ -1,17 +1,11 @@ -# name: test/sql/storage/attach_connection_pool_configure.test +# name: test/sql/connection_pool/connection_pool_configure.test # description: Test connection pool live configuration -# group: [storage] +# group: [connection_pool] require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -ATTACH 'dbname=postgresscanner' AS loader (TYPE POSTGRES); - -statement ok -DETACH loader; - statement ok SET pg_pool_max_connections = 8 @@ -24,13 +18,13 @@ ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES); statement ok RESET GLOBAL pg_pool_max_connections -query IIIIIIIIII -SELECT catalog_name, max_connections, /*cache_hits,*/ cache_misses, try_failures, thread_local_cache_enabled, max_lifetime_millis, idle_timeout_millis, reaper_thread_running, reaper_thread_period_millis, health_check_query +query IIIIIIIIIIII +SELECT catalog_name, max_connections, available_connections, total_connections, /*cache_hits,*/ cache_misses, try_failures, thread_local_cache_enabled, max_lifetime_millis, idle_timeout_millis, reaper_thread_running, reaper_thread_period_millis, health_check_query FROM postgres_configure_pool() ORDER BY catalog_name ---- -s 8 1 0 FALSE 0 60000 1 30000 SELECT 1 -s1 8 1 0 FALSE 0 60000 1 30000 SELECT 1 +s 8 1 1 1 0 FALSE 0 60000 1 30000 SELECT 1 +s1 8 1 1 1 0 FALSE 0 60000 1 30000 SELECT 1 query II SELECT catalog_name, acquire_mode @@ -157,3 +151,57 @@ DETACH s1 statement ok DETACH s + +# health check query + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + +query II +SELECT catalog_name, health_check_query +FROM postgres_configure_pool(catalog_name='s') +---- +s SELECT 1 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +query II +SELECT catalog_name, health_check_query +FROM postgres_configure_pool(catalog_name='s', health_check_query='SELECT_FAIL') +---- +s SELECT_FAIL + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +DETACH s diff --git a/test/sql/storage/attach_connection_pool_options.test b/test/sql/connection_pool/connection_pool_options.test similarity index 78% rename from test/sql/storage/attach_connection_pool_options.test rename to test/sql/connection_pool/connection_pool_options.test index 4c1841be4..988789ba0 100644 --- a/test/sql/storage/attach_connection_pool_options.test +++ b/test/sql/connection_pool/connection_pool_options.test @@ -1,6 +1,6 @@ -# name: test/sql/storage/attach_connection_pool_options.test +# name: test/sql/connection_pool/connection_pool_options.test # description: Test connection pool options -# group: [storage] +# group: [connection_pool] require postgres_scanner @@ -214,3 +214,56 @@ RESET GLOBAL pg_connection_cache statement ok DETACH s + +# health check query + +query I +SELECT current_setting('pg_pool_health_check_query') +---- +SELECT 1 + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +statement ok +DETACH s + +statement ok +SET GLOBAL pg_pool_health_check_query = 'SELECT_FAIL' + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES); + +query I +SELECT current_setting('pg_pool_health_check_query') +---- +SELECT_FAIL + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +query II +SELECT catalog_name, available_connections +FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +RESET GLOBAL pg_pool_health_check_query + +statement ok +DETACH s diff --git a/test/sql/misc/postgres_binary.test b/test/sql/misc/postgres_binary.test index d855f19a8..1adf735f7 100644 --- a/test/sql/misc/postgres_binary.test +++ b/test/sql/misc/postgres_binary.test @@ -6,12 +6,15 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) +# DuckDB supports timestamps below PostgreSQL's minimum. +statement error +COPY (SELECT TIMESTAMP '290309-12-22 (BC)') TO '{TEST_DIR}/pg_binary.bin' (FORMAT postgres_binary); +---- +out of range for Postgres + # straightforward integer copy statement ok COPY (SELECT i::INT AS i FROM range(100) t(i)) TO '{TEST_DIR}/pg_binary.bin' (FORMAT postgres_binary); @@ -33,7 +36,8 @@ CREATE TABLE all_types_tbl AS SELECT bool, smallint, int, bigint, float, double, dec_4_1, dec_9_4, dec_18_6, dec38_10, case when date < '1992-01-01' then '4712-01-01 (BC)' else '5874896-01-01' end as date, -- postgres has more constrained date ranges time, - timestamp, interval, uuid, blob, + case when timestamp < TIMESTAMP '4713-01-01 (BC)' then TIMESTAMP '4713-01-01 (BC)' else timestamp end as timestamp, -- postgres has a more constrained lower timestamp bound + interval, uuid, blob, replace(varchar, chr(0), '') as varchar, -- postgres does not support null bytes in varchar columns blob, int_array, varchar_array FROM test_all_types() diff --git a/test/sql/misc/postgres_binary_read_basic.test b/test/sql/misc/postgres_binary_read_basic.test index 34d659bd6..9bceef1b9 100644 --- a/test/sql/misc/postgres_binary_read_basic.test +++ b/test/sql/misc/postgres_binary_read_basic.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- BOOLEAN --- statement ok diff --git a/test/sql/misc/postgres_binary_read_buffer.test b/test/sql/misc/postgres_binary_read_buffer.test new file mode 100644 index 000000000..100d3bb75 --- /dev/null +++ b/test/sql/misc/postgres_binary_read_buffer.test @@ -0,0 +1,83 @@ +# name: test/sql/misc/postgres_binary_read_buffer.test +# description: Test reading postgres binary files that span multiple read buffers +# group: [misc] + +require postgres_scanner + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s + +# Each row here is 10 bytes (2 byte field count + 4 byte length + 4 byte payload) and the file +# header is 19 bytes, so these buffers hold only a handful of rows at a time. +statement ok +COPY (SELECT i::INTEGER AS a FROM range(1000) t(i)) TO '{TEST_DIR}/many_rows.bin' (FORMAT postgres_binary); + +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=64); +---- +1000 0 999 + +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=32); +---- +1000 0 999 + +# a buffer with no room for a row next to the header: the header is a boundary of its own, so the +# first row is carried over to the next fill rather than rejected +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=24); +---- +1000 0 999 + +# a buffer of exactly the header size +query III +SELECT count(*), min(a), max(a) FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=19); +---- +1000 0 999 + +# the values must round-trip exactly, not just in aggregate +query I +SELECT count(*) FROM ( + SELECT a FROM read_postgres_binary('{TEST_DIR}/many_rows.bin', columns={a: 'INTEGER'}, buffer_size=64) + EXCEPT + SELECT i::INTEGER FROM range(1000) t(i) +); +---- +0 + +# variable width columns and NULLs across buffer boundaries +statement ok +COPY (SELECT i::INTEGER AS a, ('str_' || i)::VARCHAR AS b, CASE WHEN i % 3 = 0 THEN NULL ELSE i::BIGINT END AS c FROM range(500) t(i)) TO '{TEST_DIR}/mixed_rows.bin' (FORMAT postgres_binary); + +query IIII +SELECT count(*), count(c), min(b), max(a) FROM read_postgres_binary('{TEST_DIR}/mixed_rows.bin', columns={a: 'INTEGER', b: 'VARCHAR', c: 'BIGINT'}, buffer_size=128); +---- +500 333 str_0 499 + +# rows of 56 bytes (2 byte field count + 4 byte length + 50 byte payload): larger than +# buffer_size - 19 but no larger than buffer_size, so they only fit once the header is out of the way +statement ok +COPY (SELECT repeat('x', 50) AS a FROM range(20) t(i)) TO '{TEST_DIR}/wide_rows.bin' (FORMAT postgres_binary); + +query III +SELECT count(*), min(length(a)), max(length(a)) FROM read_postgres_binary('{TEST_DIR}/wide_rows.bin', columns={a: 'VARCHAR'}, buffer_size=64); +---- +20 50 50 + +# the same rows in a buffer of exactly one row +query III +SELECT count(*), min(length(a)), max(length(a)) FROM read_postgres_binary('{TEST_DIR}/wide_rows.bin', columns={a: 'VARCHAR'}, buffer_size=56); +---- +20 50 50 + +# a row genuinely larger than the whole buffer is still rejected +statement ok +COPY (SELECT repeat('y', 100) AS a FROM range(5) t(i)) TO '{TEST_DIR}/huge_rows.bin' (FORMAT postgres_binary); + +statement error +SELECT count(*) FROM read_postgres_binary('{TEST_DIR}/huge_rows.bin', columns={a: 'VARCHAR'}, buffer_size=64); +---- +row larger than the read buffer diff --git a/test/sql/misc/postgres_binary_read_complex.test b/test/sql/misc/postgres_binary_read_complex.test index 65de94d70..b98121556 100644 --- a/test/sql/misc/postgres_binary_read_complex.test +++ b/test/sql/misc/postgres_binary_read_complex.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- INTEGER ARRAY --- statement ok diff --git a/test/sql/misc/postgres_binary_read_copy_from.test b/test/sql/misc/postgres_binary_read_copy_from.test index f4ec70eec..a230eb70e 100644 --- a/test/sql/misc/postgres_binary_read_copy_from.test +++ b/test/sql/misc/postgres_binary_read_copy_from.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- COPY FROM basic --- statement ok diff --git a/test/sql/misc/postgres_binary_read_errors.test b/test/sql/misc/postgres_binary_read_errors.test index f3ed80d7f..464c977a5 100644 --- a/test/sql/misc/postgres_binary_read_errors.test +++ b/test/sql/misc/postgres_binary_read_errors.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- Non-existent file --- statement error diff --git a/test/sql/misc/postgres_binary_read_schema_mismatch.test b/test/sql/misc/postgres_binary_read_schema_mismatch.test new file mode 100644 index 000000000..84b438770 --- /dev/null +++ b/test/sql/misc/postgres_binary_read_schema_mismatch.test @@ -0,0 +1,62 @@ +# name: test/sql/misc/postgres_binary_read_schema_mismatch.test +# description: Reading a postgres binary file with a type list that does not match the file is an error, not wrong data +# group: [misc] + +require postgres_scanner + +statement ok +LOAD postgres_scanner + +# --- field width mismatch: a BIGINT column read as INTEGER --- +statement ok +COPY (SELECT i::INTEGER AS id, 180388626432::BIGINT AS amount FROM range(1, 6) t(i)) TO '{TEST_DIR}/width_mismatch.bin' (FORMAT postgres_binary); + +query II +SELECT * FROM read_postgres_binary('{TEST_DIR}/width_mismatch.bin', columns={id: 'INTEGER', amount: 'BIGINT'}); +---- +1 180388626432 +2 180388626432 +3 180388626432 +4 180388626432 +5 180388626432 + +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/width_mismatch.bin', columns={id: 'INTEGER', amount: 'INTEGER'}); +---- +expected 4 bytes for a value of type INTEGER, got 8 + +# --- variable-length source: a DECIMAL column read as BIGINT --- +statement ok +COPY (SELECT 123.45::DECIMAL(10,2) AS amount) TO '{TEST_DIR}/decimal_as_bigint.bin' (FORMAT postgres_binary); + +query I +SELECT * FROM read_postgres_binary('{TEST_DIR}/decimal_as_bigint.bin', columns={amount: 'DECIMAL(10,2)'}); +---- +123.45 + +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/decimal_as_bigint.bin', columns={amount: 'BIGINT'}); +---- +expected 8 bytes for a value of type BIGINT, got 12 + +# --- field count mismatch: a two column file read as one column --- +statement ok +COPY (SELECT i::INTEGER AS a, (i * 10)::INTEGER AS b FROM range(1, 4) t(i)) TO '{TEST_DIR}/count_mismatch.bin' (FORMAT postgres_binary); + +query II +SELECT * FROM read_postgres_binary('{TEST_DIR}/count_mismatch.bin', columns={a: 'INTEGER', b: 'INTEGER'}); +---- +1 10 +2 20 +3 30 + +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/count_mismatch.bin', columns={a: 'INTEGER'}); +---- +expected 1 field(s) per row, got 2 + +# --- array header too short: a scalar column read as a list --- +statement error +SELECT * FROM read_postgres_binary('{TEST_DIR}/count_mismatch.bin', columns={a: 'INTEGER[]', b: 'INTEGER'}); +---- +expected at least 12 bytes for an array header of type INTEGER[], got 4 diff --git a/test/sql/misc/postgres_binary_read_temporal.test b/test/sql/misc/postgres_binary_read_temporal.test index 853041a51..ebba99ad9 100644 --- a/test/sql/misc/postgres_binary_read_temporal.test +++ b/test/sql/misc/postgres_binary_read_temporal.test @@ -5,7 +5,10 @@ require postgres_scanner statement ok -LOAD postgres_scanner; +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DETACH s # --- DATE --- statement ok diff --git a/test/sql/scanner/application_name.test b/test/sql/scanner/application_name.test new file mode 100644 index 000000000..9593a0f11 --- /dev/null +++ b/test/sql/scanner/application_name.test @@ -0,0 +1,296 @@ +# name: test/sql/scanner/application_name.test +# description: Check default and custom application_name +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +require-env POSTGRES_CONNECTION_IS_DIRECT + +# default key-value + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# name in conn_string + +statement ok +ATTACH 'dbname=postgresscanner application_name=custom1' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom1'' +', +connection=getvariable('conn')) +---- +custom1 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +# name in secret + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + APPLICATION_NAME custom2 +) + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom2'' +', +connection=getvariable('conn')) +---- +custom2 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# default URI + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + URI 'postgresql://127.0.0.1:/postgresscanner' +) + +statement ok +ATTACH '' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# default URI 2 + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + URI 'postgresql://127.0.0.1:/postgresscanner?sslmode=allow' +) + +statement ok +ATTACH '' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# custom URI + +statement ok +CREATE TEMPORARY SECRET pgsec1 ( + TYPE postgres, + URI 'postgresql://127.0.0.1:/postgresscanner?sslmode=allow&application_name=custom2' +) + +statement ok +ATTACH '' AS s (TYPE POSTGRES, SECRET pgsec1) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom2'' +', +connection=getvariable('conn')) +---- +custom2 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +statement ok +DROP SECRET pgsec1 + +# a connection error fails the URI cases instead of skipping the test + +set ignore_error_messages + +# default in URI ATTACH path + +foreach uri postgresql://127.0.0.1:/postgresscanner postgres://postgres@127.0.0.1:/postgresscanner postgresql://127.0.0.1:/postgresscanner? postgresql://127.0.0.1:/postgresscanner?sslmode=allow postgresql://127.0.0.1:/postgresscanner?sslmode=allow& postgresql://127.0.0.1:/postgresscanner?fallback_application_name=x? + +statement ok +ATTACH '{uri}' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +', +connection=getvariable('conn')) +---- +DuckDB active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s + +endloop + +# name in URI ATTACH path + +statement ok +ATTACH 'postgres://127.0.0.1:/postgresscanner?sslmode=allow&application_name=custom3' AS s (TYPE POSTGRES) + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +query II +FROM postgres_query('s', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom3'' +', +connection=getvariable('conn')) +---- +custom3 active + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s diff --git a/test/sql/scanner/arrays.test b/test/sql/scanner/arrays.test index 657a481c4..a7840573a 100644 --- a/test/sql/scanner/arrays.test +++ b/test/sql/scanner/arrays.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query IIIIII select * exclude (float_col, double_col) from pg_numarraytypes; @@ -45,4 +40,6 @@ NULL NULL NULL NULL NULL statement ok USE memory -endloop +statement ok +DETACH s + diff --git a/test/sql/scanner/bug63.test b/test/sql/scanner/bug63.test index 28d275e4e..a0e2bbeae 100644 --- a/test/sql/scanner/bug63.test +++ b/test/sql/scanner/bug63.test @@ -7,9 +7,21 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner', source_schema='some_schema'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +SET schema 'some_schema' query I -SELECT * FROM some_table; +SELECT * FROM some_table ---- two + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/bug69.test b/test/sql/scanner/bug69.test index 40104ba74..cf7969a70 100644 --- a/test/sql/scanner/bug69.test +++ b/test/sql/scanner/bug69.test @@ -6,19 +6,14 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query I -SELECT * FROM fail; +SELECT * FROM fail ---- 42.8875000 42.0000000 @@ -33,4 +28,5 @@ SELECT * FROM fail; statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug71.test b/test/sql/scanner/bug71.test index a2d612915..8f1959b9d 100644 --- a/test/sql/scanner/bug71.test +++ b/test/sql/scanner/bug71.test @@ -6,23 +6,19 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query III -SELECT * FROM my_table; +SELECT * FROM my_table ---- 42 something something else statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug74.test b/test/sql/scanner/bug74.test index 26385162f..88ff49189 100644 --- a/test/sql/scanner/bug74.test +++ b/test/sql/scanner/bug74.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query I SELECT * from oids @@ -26,4 +21,5 @@ SELECT * from oids statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug76.test b/test/sql/scanner/bug76.test index 3f053d621..fb36cbe26 100644 --- a/test/sql/scanner/bug76.test +++ b/test/sql/scanner/bug76.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query II SELECT * from daterange; @@ -25,4 +20,5 @@ SELECT * from daterange; statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/bug77.test b/test/sql/scanner/bug77.test index 146765f61..4b0698af9 100644 --- a/test/sql/scanner/bug77.test +++ b/test/sql/scanner/bug77.test @@ -6,17 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE - -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query III SELECT id, my_ints,e FROM duckdb_arr_test; @@ -28,4 +22,5 @@ SELECT id, my_ints,e FROM duckdb_arr_test; statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/count_star.test b/test/sql/scanner/count_star.test index 91f93f688..83b888fd7 100644 --- a/test/sql/scanner/count_star.test +++ b/test/sql/scanner/count_star.test @@ -7,9 +7,18 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query I -select count(*) from cars; +select count(*) from cars ---- -4 \ No newline at end of file +4 + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/ctid_scan.test b/test/sql/scanner/ctid_scan.test new file mode 100644 index 000000000..55dd74c70 --- /dev/null +++ b/test/sql/scanner/ctid_scan.test @@ -0,0 +1,43 @@ +# name: test/sql/scanner/ctid_scan.test +# description: Test parallel ctid scan with different readers +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +CREATE OR REPLACE TABLE tab1 AS FROM range(10000) + +statement ok +SET pg_pages_per_task=1 + +foreach mode COPY_BINARY PLAIN_QUERY_BINARY AUTO + +statement ok +SET pg_query_mode = '{mode}' + +query I +SELECT divide(COUNT(*),1) FROM tab1 +---- +10000 + +endloop + +statement ok +use memory + +statement ok +DETACH s + +statement ok +RESET pg_pages_per_task + +statement ok +RESET pg_query_mode diff --git a/test/sql/scanner/daterange_array.test b/test/sql/scanner/daterange_array.test index c0620bc27..631d3e447 100644 --- a/test/sql/scanner/daterange_array.test +++ b/test/sql/scanner/daterange_array.test @@ -6,23 +6,19 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query II -SELECT * from daterange_array; +SELECT * from daterange_array ---- 1108 ['["2010-01-01 14:30:00","2010-01-01 15:30:00")'] statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/scanner/decimals.test_slow b/test/sql/scanner/decimals.test_slow index cab435845..40a815d52 100644 --- a/test/sql/scanner/decimals.test_slow +++ b/test/sql/scanner/decimals.test_slow @@ -7,7 +7,10 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query I SELECT * FROM dec_0_1 @@ -132488,3 +132491,9 @@ SELECT * FROM dec_18_18 100000000000000000.000000000000000001 -100000000000000000.000000000000000001 0.0 + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/interval.test b/test/sql/scanner/interval.test index a05437a7c..d6bab6931 100644 --- a/test/sql/scanner/interval.test +++ b/test/sql/scanner/interval.test @@ -7,7 +7,10 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query I select * from intervals @@ -20,3 +23,9 @@ select * from intervals 00:00:00.42 -42 days NULL + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/missing_table.test b/test/sql/scanner/missing_table.test deleted file mode 100644 index 7c044864a..000000000 --- a/test/sql/scanner/missing_table.test +++ /dev/null @@ -1,11 +0,0 @@ -# name: test/sql/scanner/missing_table.test -# description: Missing table -# group: [scanner] - -require postgres_scanner - -require-env POSTGRES_TEST_DATABASE_AVAILABLE - -statement error -select * from postgres_scan('dbname=postgresscanner', 'public', 'does_not_exist') ----- \ No newline at end of file diff --git a/test/sql/scanner/nulls.test b/test/sql/scanner/nulls.test index c951df86e..b108b8b6a 100644 --- a/test/sql/scanner/nulls.test +++ b/test/sql/scanner/nulls.test @@ -7,7 +7,10 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner'); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s query IIIIIIIIII select * from nulltest; @@ -15,4 +18,10 @@ select * from nulltest; 1 2 3 4 5 6 7 8 9 10 1 NULL 3 4 NULL 6 7 8 NULL 10 NULL NULL 3 4 5 6 7 NULL NULL NULL -NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL \ No newline at end of file +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/numeric_double_rounding.test b/test/sql/scanner/numeric_double_rounding.test new file mode 100644 index 000000000..8a77a1207 --- /dev/null +++ b/test/sql/scanner/numeric_double_rounding.test @@ -0,0 +1,88 @@ +# name: test/sql/scanner/numeric_double_rounding.test +# description: Binary NUMERIC conversion rounds once and agrees with PostgreSQL float8 +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +foreach query_mode COPY_BINARY PLAIN_QUERY_BINARY PLAIN_QUERY_TEXT + +statement ok +SET pg_query_mode = '{query_mode}' + +# Materialize the decoded values so the comparison cannot be pushed into PostgreSQL. +# Include the examples from issue #604 and exact halfway-rounding cases. +statement ok +CREATE OR REPLACE TEMP TABLE numeric_values AS SELECT * FROM postgres_query('s', $$ + SELECT n, n::float8 AS expected FROM (VALUES + (8629433::numeric / 1000), + ((275.38::numeric + 232.58 + 785.33) / 3), + (1e32::numeric), + (-8629433::numeric / 1000), + (-1e32::numeric), + (0.1::numeric), + (0.00000000000123456789::numeric), + (12345678901234567890.1234567890123456789::numeric), + (1.00000000000000011102230246251565404236316680908203125::numeric), + (1.00000000000000033306690738754696212708950042724609375::numeric), + (-1.00000000000000011102230246251565404236316680908203125::numeric), + (1e308::numeric), + (1.7976931348623157e308::numeric), + (-1.7976931348623157e308::numeric), + (1e-300::numeric), + (2.2250738585072014e-308::numeric), + (4.9406564584124654e-324::numeric), + (-4.9406564584124654e-324::numeric), + (1e-323::numeric), + (0::numeric), + (NULL::numeric), + ('NaN'::numeric), + ('Infinity'::numeric), + ('-Infinity'::numeric) + ) AS values_to_check(n) +$$) + +query I +SELECT count(*) FROM numeric_values WHERE n IS DISTINCT FROM expected +---- +0 + +# The display scale must not cause a finite value to overflow during decoding. +statement ok +CREATE OR REPLACE TEMP TABLE high_scale_values AS SELECT * FROM postgres_query('s', $$ + SELECT n, n::float8 AS expected FROM (VALUES + (1.5::numeric(1000,500)), + (-1.5::numeric(1000,500)), + (0.125::numeric(1000,500)), + (1e300::numeric(1000,500)), + (0::numeric(1000,500)) + ) AS values_to_check(n) +$$) + +query I +SELECT count(*) FROM high_scale_values WHERE n IS DISTINCT FROM expected +---- +0 + +# NUMERIC array elements use the same binary conversion. +statement ok +CREATE OR REPLACE TEMP TABLE numeric_arrays AS SELECT * FROM postgres_query('s', $$ + SELECT n, n::float8[] AS expected FROM ( + SELECT ARRAY[8629.433::numeric, 1e32::numeric, -1e32::numeric, + 2.2250738585072014e-308::numeric, NULL::numeric] AS n + ) AS values_to_check +$$) + +query I +SELECT n IS NOT DISTINCT FROM expected FROM numeric_arrays +---- +true + +endloop + +statement ok +RESET pg_query_mode diff --git a/test/sql/scanner/numerics_nan_inf.test b/test/sql/scanner/numerics_nan_inf.test new file mode 100644 index 000000000..a06c97cd2 --- /dev/null +++ b/test/sql/scanner/numerics_nan_inf.test @@ -0,0 +1,360 @@ +# name: test/sql/scanner/numerics_nan_inf.test +# description: Test for out-of-bound values for numeric types +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# FLOAT + +query I +FROM postgres_query('s', 'SELECT ''NaN''::REAL') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Inf''::REAL') +---- +Inf + +query I +FROM postgres_query('s', 'SELECT ''-Inf''::REAL') +---- +-Inf + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::REAL') +) +---- +FLOAT + +# DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''NaN''::DOUBLE PRECISION') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Inf''::DOUBLE PRECISION') +---- +Inf + +query I +FROM postgres_query('s', 'SELECT ''-Inf''::DOUBLE PRECISION') +---- +-Inf + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::DOUBLE PRECISION') +) +---- +DOUBLE + +# NUMERIC + +# int16_t + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(4,3)') +) +---- +DECIMAL(4,3) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(4,3)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(4,3)') +---- +A field with precision 4, scale 3 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(4,3)') +---- +A field with precision 4, scale 3 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(4,3)') +---- +Unsupported NUMERIC value: NaN + +statement ok +RESET pg_numeric_nan_as_null + +# int32_t + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(9,3)') +) +---- +DECIMAL(9,3) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(9,3)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(9,3)') +---- +A field with precision 9, scale 3 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(9,3)') +---- +A field with precision 9, scale 3 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(9,3)') +---- +Unsupported NUMERIC value: NaN + +statement ok +RESET pg_numeric_nan_as_null + +# int64_t + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(18,3)') +) +---- +DECIMAL(18,3) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(18,3)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(18,3)') +---- +A field with precision 18, scale 3 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(18,3)') +---- +A field with precision 18, scale 3 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(18,3)') +---- +Unsupported NUMERIC value: NaN + +statement ok +RESET pg_numeric_nan_as_null + +# huge_int + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(38,19)') +) +---- +DECIMAL(38,19) + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(38,19)') +---- +NULL + +statement error +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC(38,19)') +---- +A field with precision 38, scale 19 cannot hold an infinite value. + +statement error +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC(38,19)') +---- +A field with precision 38, scale 19 cannot hold an infinite value. + +statement ok +SET pg_numeric_nan_as_null = FALSE + +statement error +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC(38,19)') +---- +Unsupported NUMERIC value: NaN + +# plain scan + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS out_of_bounds_numeric_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE out_of_bounds_numeric_test(col1 INT, col2 NUMERIC(38,19))') + +statement ok +CALL postgres_execute('s', 'INSERT INTO out_of_bounds_numeric_test values(42, ''NaN'')') + +statement error +SELECT divide(col1, 1), col2 FROM s.out_of_bounds_numeric_test +---- +Unsupported NUMERIC value: NaN + +statement ok +CALL postgres_execute('s', 'DROP TABLE out_of_bounds_numeric_test') + +statement ok +RESET pg_numeric_nan_as_null + +# NUMERIC as DOUBLE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +SET pg_numeric_nan_as_null = FALSE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +) +---- +DOUBLE + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +RESET pg_numeric_nan_as_null + +# NUMERIC as VARCHAR + +statement ok +SET pg_numeric_as_varchar = TRUE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +VARCHAR + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +SET pg_numeric_nan_as_null = FALSE + +query I +SELECT column_type FROM ( + DESCRIBE FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +) +---- +VARCHAR + +query I +FROM postgres_query('s', 'SELECT ''NaN''::NUMERIC') +---- +NaN + +query I +FROM postgres_query('s', 'SELECT ''Infinity''::NUMERIC') +---- +Infinity + +query I +FROM postgres_query('s', 'SELECT ''-Infinity''::NUMERIC') +---- +-Infinity + +statement ok +RESET pg_numeric_nan_as_null + +statement ok +RESET pg_numeric_as_varchar + +statement ok +DETACH s diff --git a/test/sql/scanner/postgres_query.test b/test/sql/scanner/postgres_query.test index 7d589ba89..56a8c53ce 100644 --- a/test/sql/scanner/postgres_query.test +++ b/test/sql/scanner/postgres_query.test @@ -10,7 +10,7 @@ statement ok ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) query III -select * from postgres_query('s1', 'SELECT * FROM cars'); +FROM postgres_query('s1', 'SELECT * FROM cars'); ---- ferari testarosa red aston martin db2 blue @@ -19,12 +19,12 @@ ford T black # filters query III -select * from postgres_query('s1', 'SELECT * FROM cars WHERE color=''red'''); +FROM postgres_query('s1', 'SELECT * FROM cars WHERE color=''red'''); ---- ferari testarosa red query III -select * from postgres_query('s1', 'SELECT * FROM cars') WHERE color='red'; +FROM postgres_query('s1', 'SELECT * FROM cars') WHERE color='red'; ---- ferari testarosa red @@ -77,24 +77,24 @@ DATE[] TIME[] TIME WITH TIME ZONE[] TIMESTAMP[] TIMESTAMP WITH TIME ZONE[] # semicolons query III -select * from postgres_query('s1', 'SELECT * FROM cars;') WHERE color='red'; +FROM postgres_query('s1', 'SELECT * FROM cars;') WHERE color='red'; ---- ferari testarosa red query III -select * from postgres_query('s1', 'SELECT * FROM cars; ; ') WHERE color='red'; +FROM postgres_query('s1', 'SELECT * FROM cars; ; ') WHERE color='red'; ---- ferari testarosa red # incorrect usage statement error -select * from postgres_query('s1', 'CREATE TABLE my_table(i INT)'); +FROM postgres_query('s1', 'CREATE TABLE my_table(i INT)'); ---- Invalid Error: Failed to execute query # database not found statement error -select * from postgres_query('xx', 'SELECT * FROM cars'); +FROM postgres_query('xx', 'SELECT * FROM cars'); ---- Binder Error: Failed to find attached database @@ -102,12 +102,12 @@ mode skip # empty statements statement error -select * from postgres_query('s1', '') WHERE color='red'; +FROM postgres_query('s1', '') WHERE color='red'; ---- Invalid Error: Failed to execute query statement error -select * from postgres_query('s1', ';;') WHERE color='red'; +FROM postgres_query('s1', ';;') WHERE color='red'; ---- Invalid Error: Failed to execute query @@ -115,12 +115,12 @@ mode unskip # NULL parameters statement error -select * from postgres_query(NULL, 'SELECT 42'); +FROM postgres_query(NULL, 'SELECT 42'); ---- cannot be NULL statement error -select * from postgres_query('s1', NULL); +FROM postgres_query('s1', NULL); ---- cannot be NULL @@ -129,7 +129,7 @@ statement ok ATTACH ':memory:' AS ddb statement error -select * from postgres_query('ddb', 'SELECT * FROM cars'); +FROM postgres_query('ddb', 'SELECT * FROM cars'); ---- does not refer to a Postgres database @@ -144,3 +144,217 @@ statement error select count(*) from postgres_query('s1', 'SELECT * FROM nonexistent_table'); ---- does not exist + +# DML + +query I +FROM postgres_query('s1', 'DROP TABLE IF EXISTS query_dml_test') +---- +-1 + +query I +FROM postgres_query('s1', 'CREATE TABLE query_dml_test(col1 INT)') +---- +-1 + +query I +FROM postgres_query('s1', 'INSERT INTO query_dml_test values(41), (42)') +---- +2 + +query I +FROM postgres_query('s1', 'UPDATE query_dml_test SET col1 = 43') +---- +2 + +query I +FROM postgres_query('s1', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +---- +0 + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_query('s1', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +) +---- +rowcount BIGINT + +statement ok +CALL postgres_query('s1', 'DROP TABLE query_dml_test') + +# suppress DML output + +statement ok +CALL postgres_query('s1', 'DROP TABLE IF EXISTS dml_output_test') + +statement ok +CALL postgres_query('s1', 'CREATE TABLE dml_output_test (col1 INT)') + +statement ok +CALL postgres_query('s1', 'INSERT INTO dml_output_test values(41),(42)') + +# unittester ignores StatementReturnType::NOTHING +query I +CALL postgres_query('s1', 'UPDATE dml_output_test SET col1 = 43', suppress_dml_output=TRUE) +---- +2 + +statement ok +CALL postgres_query('s1', 'DROP TABLE dml_output_test') + +# non-preparable statement + +statement error +CALL postgres_query('s1', 'PREPARE p1 AS SELECT 42; EXECUTE p1;') +---- +Failed to prepare + +statement error +CALL postgres_query('s1', 'PREPARE p1 AS SELECT 42; EXECUTE p1;', prepare=FALSE, params=row(42)) +---- +query parameters cannot be used with 'prepare=FALSE' + +# use the same connection + +statement ok +BEGIN TRANSACTION + +statement ok +CALL postgres_query('s1', 'PREPARE p1 AS SELECT 42; EXECUTE p1;', prepare=FALSE) + +query I +FROM postgres_query('s1', 'EXECUTE p1', prepare=FALSE) +---- +1 + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_query('s1', 'EXECUTE p1', prepare=FALSE) +) +---- +rowcount BIGINT + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_execute('s1', 'EXECUTE p1', prepare=FALSE) +) +---- +rowcount BIGINT + +statement ok +CALL postgres_query('s1', 'DEALLOCATE p1', prepare=FALSE) + +statement error +FROM postgres_query('s1', 'EXECUTE p1', prepare=FALSE) +---- +does not exist + +statement ok +ROLLBACK + +# wrapping rules + +statement ok +CALL enable_logging('PostgresQueryLog') + +# no wrap copy + +query II +FROM postgres_query('s1', 'SELECT brand, model FROM cars', query_mode='COPY_BINARY'); +---- +ferari testarosa +aston martin db2 +bentley mulsanne +ford T + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'COPY (SELECT brand, model FROM cars) TO STDOUT (FORMAT "binary");' +---- +1 + +# no wrap binary + +query II +FROM postgres_query('s1', 'SELECT brand, model FROM cars', query_mode='PLAIN_QUERY_BINARY'); +---- +ferari testarosa +aston martin db2 +bentley mulsanne +ford T + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT brand, model FROM cars' +---- +1 + +# no wrap text + +query II +FROM postgres_query('s1', 'SELECT model, brand FROM cars', query_mode='PLAIN_QUERY_TEXT'); +---- +testarosa ferari +db2 aston martin +mulsanne bentley +T ford + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT model, brand FROM cars' +---- +1 + +# wrap color_t + +query II +FROM postgres_query('s1', 'SELECT brand, color FROM cars', query_mode='PLAIN_QUERY_BINARY'); +---- +ferari red +aston martin blue +bentley gray +ford black + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT "brand", "color"::VARCHAR FROM (SELECT brand, color FROM cars) AS __unnamed_subquery ' +---- +1 + +# wrap projection + +query I +select count(*) from postgres_query('s1', 'SELECT * FROM cars WHERE color=''red''', query_mode='PLAIN_QUERY_BINARY'); +---- +1 + +query I +SELECT COUNT(*) +FROM duckdb_logs_parsed('PostgresQueryLog') +WHERE query = 'SELECT "brand" FROM (SELECT * FROM cars WHERE color=''red'') AS __unnamed_subquery ' +---- +1 + +statement ok +CALL disable_logging() + +# duplicate column names + +query III +FROM postgres_query('s1', 'SELECT 41, 42, 43') +---- +41 42 43 + +query I +SELECT column_name FROM (DESCRIBE FROM postgres_query('s1', 'SELECT 41, 42, 43')) +---- +?column? +?column?_1 +?column?_2 + +statement ok +DETACH s1 diff --git a/test/sql/scanner/postgres_query_error.test b/test/sql/scanner/postgres_query_error.test index e752fe384..7f6b186af 100644 --- a/test/sql/scanner/postgres_query_error.test +++ b/test/sql/scanner/postgres_query_error.test @@ -13,7 +13,7 @@ statement ok BEGIN statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query III select * from postgres_query('s1', 'SELECT * FROM cars'); diff --git a/test/sql/scanner/postgres_query_params.test b/test/sql/scanner/postgres_query_params.test index e83d041e9..4672f9e59 100644 --- a/test/sql/scanner/postgres_query_params.test +++ b/test/sql/scanner/postgres_query_params.test @@ -6,10 +6,6 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -# COPY does not support parameters: ERROR: there is no parameter $1 -statement ok -SET pg_use_text_protocol=true - statement ok ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) @@ -108,3 +104,39 @@ query I FROM postgres_query('s1', 'SELECT $1::DOUBLE PRECISION', params=row(42.123::DOUBLE)) ---- 42.123 + +statement error +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo'), + query_mode='COPY_BINARY') +---- +Failed to prepare COPY + +statement ok +SET pg_query_mode = 'COPY_BINARY' + +statement error +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo')) +---- +Failed to prepare COPY + +query II +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo'), + query_mode='PLAIN_QUERY_BINARY') +---- +42 foo + +query II +FROM postgres_query('s1', 'SELECT $1::INTEGER, $2::TEXT', + params=row(42, 'foo'), + query_mode='PLAIN_QUERY_TEXT') +---- +42 foo + +statement ok +RESET pg_query_mode + +statement ok +DETACH s1 \ No newline at end of file diff --git a/test/sql/scanner/query_pin_connection.test b/test/sql/scanner/query_pin_connection.test new file mode 100644 index 000000000..196417567 --- /dev/null +++ b/test/sql/scanner/query_pin_connection.test @@ -0,0 +1,96 @@ +# name: test/sql/scanner/query_pin_connection.test +# description: Test connection pinning with postgres_query +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement error +SELECT postgres_pin_connection() +---- +No function matches the given name and argument types + +statement error +SELECT postgres_pin_connection(NULL) +---- +Specified attached database name must be not null + +statement error +SELECT postgres_pin_connection('fail') +---- +Failed to find attached database + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +statement error +SELECT postgres_close_pinned_connection('fail', 0) +---- +Failed to find attached database + +statement error +SELECT postgres_close_pinned_connection('s', 0) +---- +not found in the specified attached database + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement error +SELECT postgres_close_pinned_connection('s', getvariable('conn')) +---- +not found in the specified attached database + +statement ok +RESET VARIABLE conn + +statement ok +SET VARIABLE conn = postgres_pin_connection('s') + +statement ok +CALL postgres_execute('s', 'CREATE TEMPORARY TABLE temp1(col1 INT)', connection=getvariable('conn')) + +statement ok +CALL postgres_execute('s', 'INSERT INTO temp1 values(42)', connection=getvariable('conn')) + +query I +FROM postgres_query('s', 'SELECT * FROM temp1', connection=getvariable('conn')) +---- +42 + +statement error +FROM postgres_query('s', 'SELECT 42', connection=getvariable('conn')) +UNION ALL +FROM postgres_query('s', 'SELECT 43', connection=getvariable('conn')) +---- +not found in the specified attached database + +statement ok +SET VARIABLE conn2 = postgres_pin_connection('s') + +query I +FROM postgres_query('s', 'SELECT 42', connection=getvariable('conn')) +UNION ALL +FROM postgres_query('s', 'SELECT 43', connection=getvariable('conn2')) +---- +42 +43 + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn2')) + +statement ok +SELECT postgres_close_pinned_connection('s', getvariable('conn')) + +statement ok +RESET VARIABLE conn1 + +statement ok +RESET VARIABLE conn + +statement ok +DETACH s \ No newline at end of file diff --git a/test/sql/scanner/remote_connect.test b/test/sql/scanner/remote_connect.test new file mode 100644 index 000000000..3aa2fe156 --- /dev/null +++ b/test/sql/scanner/remote_connect.test @@ -0,0 +1,57 @@ +# name: test/sql/scanner/remote_connect.test +# description: Test CONNECT statement +# group: [scanner] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +query I +SELECT current_database() +---- +memory + +statement ok +CONNECT 'dbname=postgresscanner' (TYPE POSTGRES) + +query I +SELECT current_database() +---- +postgresscanner + +statement ok +DROP TABLE IF EXISTS connect_test + +statement ok +CREATE TABLE connect_test(col1 INT) + +statement ok +INSERT INTO connect_test values (41), (42) + +statement error +FROM connect_test +---- +Failed to prepare query + +query I +SELECT col1::regclass FROM connect_test ORDER BY col1 +---- +41 +42 + +# unittester ignores StatementReturnType::NOTHING +query I +UPDATE connect_test SET col1 = 43 +---- +2 + +statement ok +DROP TABLE connect_test + +statement ok +DISCONNECT + +query I +SELECT current_database() +---- +memory diff --git a/test/sql/scanner/timestamp_range.test b/test/sql/scanner/timestamp_range.test new file mode 100644 index 000000000..4e05f0170 --- /dev/null +++ b/test/sql/scanner/timestamp_range.test @@ -0,0 +1,49 @@ +# name: test/sql/scanner/timestamp_range.test +# description: Reject Postgres timestamps outside DuckDB's supported range +# group: [scanner] + +require postgres_scanner + +require parquet + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS timestamp_range_test') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE timestamp_range_test (id INTEGER PRIMARY KEY, ts TIMESTAMP, tstz TIMESTAMPTZ)') + +# postgres_execute sends this directly to PostgreSQL, whose timestamp range is wider than DuckDB's. +statement ok +CALL postgres_execute('s', 'INSERT INTO timestamp_range_test VALUES + (1, ''294276-12-31 23:59:59'', ''294276-12-31 23:59:59+00''), + (2, ''10000-01-01 00:00:00'', ''10000-01-01 00:00:00+00''), + (3, ''2026-01-15 12:00:00'', ''2026-01-15 12:00:00+00''), + (4, ''4713-01-01 BC'', ''4713-01-01 BC'')') + +# TODO: investigate me +#4 4713-01-01 (BC) 00:00:00 4713-01-01 (BC) 00:25:21+00 +query III +SELECT id, ts::VARCHAR, tstz::VARCHAR FROM s.timestamp_range_test WHERE id > 1 AND id < 4 ORDER BY id +---- +2 10000-01-01 00:00:00 10000-01-01 00:00:00+00 +3 2026-01-15 12:00:00 2026-01-15 12:00:00+00 + +statement error +SELECT ts FROM s.timestamp_range_test WHERE id = 1 +---- +timestamp out of range + +statement error +SELECT tstz FROM s.timestamp_range_test WHERE id = 1 +---- +timestamp out of range + +statement error +COPY (SELECT ts, tstz FROM s.timestamp_range_test WHERE id = 1) TO '{TEST_DIR}/timestamp_range.parquet' (FORMAT PARQUET) +---- +timestamp out of range diff --git a/test/sql/scanner/tpch.test_slow b/test/sql/scanner/tpch.test_slow index 865c164d6..3ae9b233e 100644 --- a/test/sql/scanner/tpch.test_slow +++ b/test/sql/scanner/tpch.test_slow @@ -9,7 +9,13 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -CALL postgres_attach('dbname=postgresscanner', source_schema='tpch', filter_pushdown=true); +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +SET schema 'tpch' loop i 1 9 @@ -28,3 +34,9 @@ PRAGMA tpch({i}) :duckdb/extension/tpch/dbgen/answers/sf0.01/q{i}.csv endloop + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/scanner/types.test b/test/sql/scanner/types.test index df8aac10d..c25bce164 100644 --- a/test/sql/scanner/types.test +++ b/test/sql/scanner/types.test @@ -6,16 +6,11 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE -statement ok -CALL postgres_attach('dbname=postgresscanner'); - statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -USE s; - -loop i 0 2 +USE s query IIIIIIII select * from pg_numtypes; @@ -50,4 +45,5 @@ ford T black statement ok USE memory -endloop +statement ok +DETACH s diff --git a/test/sql/storage/attach_comments.test b/test/sql/storage/attach_comments.test index 8f4723766..f720a3c71 100644 --- a/test/sql/storage/attach_comments.test +++ b/test/sql/storage/attach_comments.test @@ -30,7 +30,7 @@ CALL postgres_execute('s', $$COMMENT ON COLUMN comment_test.name IS 'this is a c # Clear DuckDB's metadata cache so fresh comments are loaded statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); # Verify the table comment appears in duckdb_tables() query I diff --git a/test/sql/storage/attach_connect_display.test b/test/sql/storage/attach_connect_display.test new file mode 100644 index 000000000..3b44b9a57 --- /dev/null +++ b/test/sql/storage/attach_connect_display.test @@ -0,0 +1,162 @@ +# name: test/sql/storage/attach_connect_display.test +# description: Test validation of the CONNECT_DISPLAY attach option +# group: [storage] + +require postgres_scanner + +# CONNECT_DISPLAY is validated while binding the ATTACH, before any connection is made, so none of this +# needs a Postgres to connect to. Values that pass validation get as far as the connection attempt, which +# is why the accepted cases below expect "Unable to connect to Postgres at". + +# NULL is rejected by DuckDB itself, before the option reaches the Postgres extension +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY NULL) +---- +NULL is not supported as a valid option for ATTACH option + +# the value ends up in the shell prompt verbatim, so escape sequences must not survive +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'rds://x' || chr(27) || '[31m') +---- +must not contain control characters + +# newline, carriage return and tab would all break up the prompt +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'line1' || chr(10) || 'line2') +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(13)) +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(9) || 'b') +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(127)) +---- +must not contain control characters + +# C1 control characters - 0xC2 0x9B is CSI, which some terminals honour. These are multi-byte in UTF-8, +# so a check that only looks at bytes below 0x20 would let them through. +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY chr(194) || chr(155)) +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY chr(194) || chr(128)) +---- +must not contain control characters + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('x', 1025)) +---- +must be at most 1024 columns wide, was 1025 + +# the limit is on rendered width, so wide characters count double +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('日', 513)) +---- +must be at most 1024 columns wide, was 1026 + +# unrecognized options are still rejected +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY_NAME 'rds://my-instance') +---- +Unrecognized option for Postgres attach: connect_display_name + +# --- values that pass validation --- + +# a realistic endpoint is nowhere near the width limit +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'redshift://my-cluster.abc123xyz789.eu-west-1.redshift.amazonaws.com:5439/analytics') +---- +Unable to connect to Postgres at + +# exactly at the width limit, in both narrow and wide characters +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('x', 1024)) +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY repeat('日', 512)) +---- +Unable to connect to Postgres at + +# printable non-ASCII is fine - only control characters are rejected +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'rds://café-日本-🚀') +---- +Unable to connect to Postgres at + +# an empty display string is accepted and falls back to the derived postgres://host:port +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY '') +---- +Unable to connect to Postgres at + +# the option name is matched case insensitively +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, connect_display 'rds://x') +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CoNnEcT_DiSpLaY 'rds://x') +---- +Unable to connect to Postgres at + +# whitespace and zero width characters are not control characters, so they are accepted +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY ' ') +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a' || chr(8203) || 'b') +---- +Unable to connect to Postgres at + +# there is no test for invalid UTF-8 because it cannot be reached from SQL - DuckDB VARCHAR is always +# valid UTF-8, and casting a BLOB to VARCHAR yields the escaped text form rather than the raw bytes. +# The UTF-8 check in ExtractConnectDisplay guards the C++ path, where a Value is built directly. +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY CAST('\xC3\x28'::BLOB AS VARCHAR)) +---- +Unable to connect to Postgres at + +# non-VARCHAR values are accepted and converted with ToString(), matching how SECRET is handled +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 42) +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY ['a','b']) +---- +Unable to connect to Postgres at + +# --- the option must not disturb the rest of the attach --- + +# omitting it entirely still binds - it is an optional constructor argument +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES) +---- +Unable to connect to Postgres at + +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'rds://x', SCHEMA 'public', READ_ONLY) +---- +Unable to connect to Postgres at + +# repeating the option is resolved by DuckDB before the extension sees it +statement error +ATTACH 'dbname=nope' AS d (TYPE POSTGRES, CONNECT_DISPLAY 'a', CONNECT_DISPLAY 'b') +---- +Parser Error: Unexpected duplicate option "connect_display" diff --git a/test/sql/storage/attach_describe.test b/test/sql/storage/attach_describe.test index 2c478b0f4..94603b635 100644 --- a/test/sql/storage/attach_describe.test +++ b/test/sql/storage/attach_describe.test @@ -48,7 +48,7 @@ hello, world mode skip statement ok -CALL pg_clear_cache() +CALL postgres_clear_cache() query IIIIII DESCRIBE s1.test diff --git a/test/sql/storage/attach_duckdb_indexes_constraint_type.test b/test/sql/storage/attach_duckdb_indexes_constraint_type.test new file mode 100644 index 000000000..43ed05244 --- /dev/null +++ b/test/sql/storage/attach_duckdb_indexes_constraint_type.test @@ -0,0 +1,35 @@ +# name: test/sql/storage/attach_duckdb_indexes_constraint_type.test +# description: duckdb_indexes() must not crash scanning Postgres indexes (constraint_type uninitialized read) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.constraint_type_tbl + +statement ok +CREATE TABLE s.constraint_type_tbl(i INTEGER) + +statement ok +CREATE INDEX constraint_type_idx ON s.constraint_type_tbl(i) + +query I +SELECT count(*) FROM duckdb_indexes() WHERE database_name = 's' AND index_name = 'constraint_type_idx' +---- +1 + +query II +SELECT is_unique, is_primary FROM duckdb_indexes() WHERE database_name = 's' AND index_name = 'constraint_type_idx' +---- +false false + +statement ok +DROP INDEX s.constraint_type_idx + +statement ok +DROP TABLE s.constraint_type_tbl diff --git a/test/sql/storage/attach_existing_multidimensional_array.test b/test/sql/storage/attach_existing_multidimensional_array.test index 5fc34f775..cc2f9451f 100644 --- a/test/sql/storage/attach_existing_multidimensional_array.test +++ b/test/sql/storage/attach_existing_multidimensional_array.test @@ -47,10 +47,17 @@ statement ok SET pg_array_as_varchar=true statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query I SELECT * FROM mixed_arrays ---- {{{1,2,3}},{{4,5,6}},{{7,8,9}}} {1,2,3} + +# pg_array_as_varchar also applies to postgres_query +query I +SELECT * FROM postgres_query('s', 'SELECT * FROM mixed_arrays') +---- +{{{1,2,3}},{{4,5,6}},{{7,8,9}}} +{1,2,3} diff --git a/test/sql/storage/attach_filter_pushdown.test b/test/sql/storage/attach_filter_pushdown.test index 4d01031ba..e603dcf60 100644 --- a/test/sql/storage/attach_filter_pushdown.test +++ b/test/sql/storage/attach_filter_pushdown.test @@ -1,5 +1,5 @@ # name: test/sql/storage/attach_filter_pushdown.test -# description: Test experimental filter pushdown +# description: Test filter pushdown for expressions when remote execute is not supported # group: [storage] require postgres_scanner @@ -7,58 +7,358 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok -ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok -CREATE OR REPLACE TABLE s1.filter_pushdown(i INTEGER) +CREATE OR REPLACE TABLE s.filter_pushdown_test(col1 INTEGER, col2 VARCHAR) statement ok -INSERT INTO s1.filter_pushdown FROM range(1000000) +INSERT INTO s.filter_pushdown_test values (41, 'foo'), (42, 'bar'), (44, NULL) -statement ok -SET pg_experimental_filter_pushdown=true +# ExpressionClass::BOUND_CONJUNCTION + +# ExpressionType::CONJUNCTION_AND + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 AND col2 = 'foo' +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 AND col2 = 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col1 = 41.*col2 = 'foo'.* + +# ExpressionType::CONJUNCTION_OR + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE (col1 = 41 AND col2 = 'foo') OR col2 = 'bar' +---- +41 +42 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE (col1 = 41 AND col2 = 'foo') OR col2 = 'bar' +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 = 'foo'\) OR \(col2 = 'bar'\)\).* + +# ExpressionClass::BOUND_OPERATOR + +# ExpressionType::OPERATOR_IS_NULL + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NULL +---- +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NULL +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS NULL.* + +# ExpressionType::OPERATOR_IS_NOT_NULL + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT NULL +---- +41 +42 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT NULL +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS NOT NULL.* + +# ExpressionType::COMPARE_IN: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 IN (41, 44) +---- +41 +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 IN (41, 44) +---- +physical_plan :.*Postgres Scan.*Filters: optional: \(col1 IN \(41, 44\)\).* + +# indirect reference + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE len(col2) IN (3, 4) +---- +41 +42 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE len(col2) IN (3, 4) +---- +physical_plan :.*Postgres Scan.*Filters:.* + +# ExpressionClass::BOUND_FUNCTION + +# ExpressionType::COMPARE_EQUAL: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 = 41 +---- +physical_plan :.*Postgres Scan.*Filters: col1 = 41.* + +# ExpressionType::COMPARE_NOTEQUAL: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 != 41 +---- +42 +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 != 41 +---- +physical_plan :.*Postgres Scan.*Filters: col1 != 41.* + +# ExpressionType::COMPARE_LESSTHAN: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 < 42 +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 < 42 +---- +physical_plan :.*Postgres Scan.*Filters: col1 < 42.* + +# ExpressionType::COMPARE_GREATERTHAN: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 > 42 +---- +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 > 42 +---- +physical_plan :.*Postgres Scan.*Filters: col1 > 42.* + +# ExpressionType::COMPARE_LESSTHANOREQUALTO: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 <= 41 +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 <= 41 +---- +physical_plan :.*Postgres Scan.*Filters: col1 <= 41.* + +# ExpressionType::COMPARE_GREATERTHANOREQUALTO: + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 >= 44 +---- +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col1 >= 44 +---- +physical_plan :.*Postgres Scan.*Filters: col1 >= 44.* + +# ExpressionType::COMPARE_DISTINCT_FROM: query I -SELECT * FROM s1.filter_pushdown WHERE i=52525 +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS DISTINCT FROM 'foo' ---- -52525 +42 +44 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS DISTINCT FROM 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS DISTINCT FROM 'foo'.* + +# ExpressionType::COMPARE_NOT_DISTINCT_FROM: query I -SELECT * FROM s1.filter_pushdown WHERE i IN (32, 52525, 777777) ORDER BY ALL +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT DISTINCT FROM 'foo' ---- -32 -52525 -777777 +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 IS NOT DISTINCT FROM 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col2 IS NOT DISTINCT FROM 'foo'.* + +# VARCHAR comparison logic + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'foo' +---- +41 + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'foo' +---- +physical_plan :.*Postgres Scan.*Filters: col2 = 'foo'.* + +query I +SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'FOO' +---- + +query II +EXPLAIN SELECT divide(col1, 1) FROM s.filter_pushdown_test WHERE col2 = 'FOO' +---- +physical_plan :.*Postgres Scan.*Filters: col2 = 'FOO'.* statement ok -CREATE OR REPLACE TABLE s1.filter_pushdown_types(v VARCHAR, d DATE); +DROP TABLE s.filter_pushdown_test + +# VARCHAR statement ok -INSERT INTO s1.filter_pushdown_types VALUES ('hello world', DATE '1992-01-01'), (NULL, NULL), ('I''m here', DATE '2020-01-01'); +CREATE OR REPLACE TABLE s.filter_pushdown_string(i STRING) + +statement ok +INSERT INTO s.filter_pushdown_string SELECT CAST(range AS STRING) FROM range(10000); + +query II +SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i = '5252' +---- +5252 4 + +query II +EXPLAIN SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i = '5252' +---- +physical_plan :.*Postgres Scan.*Filters: i = '5252'.* + +query II +SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i > '10' ORDER BY i LIMIT 10 +---- +100 3 +1000 4 +1001 4 +1002 4 +1003 4 +1004 4 +1005 4 +1006 4 +1007 4 +1008 4 + +query II +EXPLAIN SELECT i, divide(len(i), 1) FROM s.filter_pushdown_string WHERE i > '10' ORDER BY i LIMIT 10 +---- +physical_plan :.*Postgres Scan.*Filters:.*i > '10'.* + +statement ok +DROP TABLE s.filter_pushdown_string + +# timestamp pushdown + +statement ok +CREATE OR REPLACE TABLE s.datetime_tbl(id INTEGER, d DATE, ts TIMESTAMP) + +statement ok +INSERT INTO s.datetime_tbl VALUES (1, '2020-02-03', '2029-02-14 08:47:23'),(2, '1000-01-01', '1970-01-01 00:00:01.000000'), (3, '9999-12-31', '2038-01-18 22:14:07.499999'), (4, NULL, NULL) + +query I +SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE d >= DATE '2000-01-01' +---- +2 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE d >= DATE '2000-01-01' +---- +physical_plan :.*Postgres Scan.*Filters: d >= '2000-01-01'::DATE.* query I -SELECT v FROM s1.filter_pushdown_types WHERE v IN ('hello world', 'I''m here') ORDER BY ALL +SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' +---- +2 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' ---- -I'm here -hello world +physical_plan :.*Postgres Scan.*Filters:.*ts >= '2000-01-01 00:00:00'::TIMESTAMP.* query I -SELECT v FROM s1.filter_pushdown_types WHERE v='I''m here' +SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' +---- +2 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.datetime_tbl WHERE ts >= TIMESTAMP '2000-01-01' ---- -I'm here +physical_plan :.*Postgres Scan.*Filters:.*ts >= '2000-01-01 00:00:00'::TIMESTAMP.* + +statement ok +DROP TABLE s.datetime_tbl + +# boolean pushdown + +statement ok +CREATE OR REPLACE TABLE s.booleans_tbl(b BOOLEAN); + +statement ok +INSERT INTO s.booleans_tbl VALUES (false), (true), (NULL); query I -SELECT d FROM s1.filter_pushdown_types WHERE d=DATE '1992-01-01' +SELECT divide(b::INT, 1) FROM s.booleans_tbl WHERE b = true ---- -1992-01-01 +1 + +query II +EXPLAIN SELECT divide(b::INT, 1) FROM s.booleans_tbl WHERE b = true +---- +physical_plan :.*Postgres Scan.*Filters: b = true.* + +statement ok +DROP TABLE s.booleans_tbl + +# string pushdown + +statement ok +CREATE OR REPLACE TABLE s.text_tbl(v VARCHAR(4), c CHAR(4), t TEXT); + +statement ok +INSERT INTO s.text_tbl VALUES ('ab ', 'ab ', 'thisisalongstring'), ('', '', ''), ('🦆', '🦆', '🦆🦆🦆🦆'), (NULL, NULL, NULL) query I -SELECT * FROM s1.composites_of_composites WHERE b.k < 8 +SELECT divide(COUNT(*), 1) FROM s.text_tbl WHERE v= '🦆' ---- -{'a': {'i': 1, 'j': 2}, 'k': 3} -{'a': {'i': 4, 'j': 5}, 'k': 6} +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.text_tbl WHERE v= '🦆' +---- +physical_plan :.*Postgres Scan.*Filters: v = '🦆'.* + +statement ok +DROP TABLE s.text_tbl + +# blob pushdown + +statement ok +CREATE OR REPLACE TABLE s.blob_tbl(bl BLOB); + +statement ok +INSERT INTO s.blob_tbl VALUES (''), ('\x80'), (NULL) query I -SELECT * FROM s1.composites_of_composites WHERE b.a.j = 5 +SELECT divide(COUNT(*), 1) FROM s.blob_tbl WHERE bl= BLOB '\x80' ---- -{'a': {'i': 4, 'j': 5}, 'k': 6} +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.blob_tbl WHERE bl= BLOB '\x80' +---- +physical_plan :.*Postgres Scan.*Filters: bl = '\\x80'::BLOB.* + +statement ok +DROP TABLE s.blob_tbl diff --git a/test/sql/storage/attach_filter_pushdown_nonfinite.test b/test/sql/storage/attach_filter_pushdown_nonfinite.test new file mode 100644 index 000000000..3cb2b8e0d --- /dev/null +++ b/test/sql/storage/attach_filter_pushdown_nonfinite.test @@ -0,0 +1,286 @@ +# name: test/sql/storage/attach_filter_pushdown_nonfinite.test +# description: Test filter pushdown with non-finite input for expressions when remote execute is not supported +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.filter_pushdown_test_infinite(col1 FLOAT, col2 DOUBLE, col3 DATE, col4 TIMESTAMP) + +statement ok +INSERT INTO s.filter_pushdown_test_infinite values (41.0, 42.0, '2020-12-31', '2020-12-31 23:59:59') + +# ExpressionType::COMPARE_IN: + +# FLOAT + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT, 41.0::FLOAT) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT, 41.0::FLOAT) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col1 IN \('inf'::FLOAT, 'nan'::FLOAT, 41.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 IN ('inf'::FLOAT, 'nan'::FLOAT) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col1 IN \('inf'::FLOAT, 'nan'::FLOAT\)\).* + +# DOUBLE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE, 42.0::DOUBLE) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE, 42.0::DOUBLE) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col2 IN \(1e1000, 'nan'::DOUBLE, 42.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 IN ('inf'::DOUBLE, 'nan'::DOUBLE) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col2 IN \(1e1000, 'nan'::DOUBLE\)\).* + +# DATE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE, '2020-12-31'::DATE) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE, '2020-12-31'::DATE) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col3 IN \('infinity'::DATE, '2020-12-31'::DATE\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 IN ('inf'::DATE) +---- +physical_plan :.*Postgres Scan.*Filters:.*col3 = 'infinity'::DATE.* + +# TIMESTAMP + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP, '2020-12-31 23:59:59'::TIMESTAMP) +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP, '2020-12-31 23:59:59'::TIMESTAMP) +---- +physical_plan :.*Postgres Scan.*Filters:.*\(col4 IN \('infinity'::TIMESTAMP, '2020-12-31 23:59:59'::TIMESTAMP\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP) +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 IN ('inf'::TIMESTAMP) +---- +physical_plan :.*Postgres Scan.*Filters:.*col4 = 'infinity'::TIMESTAMP.* + +# ExpressionType::COMPARE_EQUAL + +# FLOAT + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT OR col1 = 41.0::FLOAT +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT OR col1 = 41.0::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 = 'inf'::FLOAT\) OR \(col1 = 'nan'::FLOAT\) OR \(col1 = 41.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 = 'inf'::FLOAT OR col1 = 'nan'::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 = 'inf'::FLOAT\) OR \(col1 = 'nan'::FLOAT\)\).* + +# DOUBLE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE OR col2 = 42.0::DOUBLE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE OR col2 = 42.0::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 = 1e1000\) OR \(col2 = 'nan'::DOUBLE\) OR \(col2 = 42.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 = 'inf'::DOUBLE OR col2 = 'nan'::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 = 1e1000\) OR \(col2 = 'nan'::DOUBLE\)\).* + +# DATE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE OR col3 = '2020-12-31'::DATE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE OR col3 = '2020-12-31'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col3 = 'infinity'::DATE\) OR \(col3 = '2020-12-31'::DATE\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 = 'inf'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*col3 = 'infinity'::DATE.* + +# TIMESTAMP + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP OR col4 = '2020-12-31 23:59:59'::TIMESTAMP +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP OR col4 = '2020-12-31 23:59:59'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col4 = 'infinity'::TIMESTAMP\) OR \(col4 = '2020-12-31 23:59:59'::TIMESTAMP\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 = 'inf'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*col4 = 'infinity'::TIMESTAMP.* + +# ExpressionType::COMPARE_GREATERTHAN + +# FLOAT + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT OR col1 > 40.0::FLOAT +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT OR col1 > 40.0::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 > 'inf'::FLOAT\) OR \(col1 > 'nan'::FLOAT\) OR \(col1 > 40.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col1 > 'inf'::FLOAT OR col1 > 'nan'::FLOAT +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col1 > 'inf'::FLOAT\) OR \(col1 > 'nan'::FLOAT\)\).* + +# DOUBLE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE OR col2 > 40.0::DOUBLE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE OR col2 > 40.0::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 > 1e1000\) OR \(col2 > 'nan'::DOUBLE\) OR \(col2 > 40.0\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col2 > 'inf'::DOUBLE OR col2 > 'nan'::DOUBLE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col2 > 1e1000\) OR \(col2 > 'nan'::DOUBLE\)\).* + +# DATE + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE OR col3 > '2020-12-30'::DATE +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE OR col3 > '2020-12-30'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col3 > 'infinity'::DATE\) OR \(col3 > '2020-12-30'::DATE\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col3 > 'inf'::DATE +---- +physical_plan :.*Postgres Scan.*Filters:.*col3 > 'infinity'::DATE.* + +# TIMESTAMP + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP OR col4 > '2020-12-31 23:59:58'::TIMESTAMP +---- +1 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP OR col4 > '2020-12-31 23:59:58'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*\(\(col4 > 'infinity'::TIMESTAMP\) OR \(col4 > '2020-12-31 23:59:58'::TIMESTAMP\)\).* + +query I +SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP +---- +0 + +query II +EXPLAIN SELECT divide(COUNT(*), 1) FROM s.filter_pushdown_test_infinite WHERE col4 > 'inf'::TIMESTAMP +---- +physical_plan :.*Postgres Scan.*Filters:.*col4 > 'infinity'::TIMESTAMP.* diff --git a/test/sql/storage/attach_filter_pushdown_or_enum.test b/test/sql/storage/attach_filter_pushdown_or_enum.test new file mode 100644 index 000000000..76c1f7a3b --- /dev/null +++ b/test/sql/storage/attach_filter_pushdown_or_enum.test @@ -0,0 +1,49 @@ +# name: test/sql/storage/attach_filter_pushdown_or_enum.test +# description: Test filter pushdown of an OR filter with a child that cannot be pushed down +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.filter_pushdown_or_enum + +statement ok +DROP TYPE IF EXISTS s.filter_pushdown_mood + +statement ok +CREATE TYPE s.filter_pushdown_mood AS ENUM ('sad', 'happy') + +statement ok +CREATE TABLE s.filter_pushdown_or_enum(id INTEGER, m s.filter_pushdown_mood) + +statement ok +INSERT INTO s.filter_pushdown_or_enum VALUES (1, 'sad'), (2, 'happy'), (3, NULL) + +# ENUM comparisons are not pushed down, so neither is the OR +query I +SELECT id FROM s.filter_pushdown_or_enum WHERE m = 'happy' OR m IS NULL ORDER BY id +---- +2 +3 + +query I +SELECT id FROM s.filter_pushdown_or_enum WHERE (m = 'happy' AND m IS NOT NULL) OR m IS NULL ORDER BY id +---- +2 +3 + +query I +SELECT id FROM s.filter_pushdown_or_enum WHERE id = 1 OR id = 3 ORDER BY id +---- +1 +3 + +query II +EXPLAIN SELECT id FROM s.filter_pushdown_or_enum WHERE id = 1 OR id = 3 +---- +physical_plan :.*Postgres Scan.*Filters:.*id = 1.*id = 3.* diff --git a/test/sql/storage/attach_filter_pushdown_timestamptz.test b/test/sql/storage/attach_filter_pushdown_timestamptz.test new file mode 100644 index 000000000..eeed7ed97 --- /dev/null +++ b/test/sql/storage/attach_filter_pushdown_timestamptz.test @@ -0,0 +1,36 @@ +# name: test/sql/storage/attach_filter_pushdown_timestamptz.test +# description: Test filter pushdown of TIMESTAMPTZ constants when the session time zone of the server is not UTC +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner options=-ctimezone=America/New_York' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.filter_pushdown_timestamptz(id INTEGER, ts TIMESTAMPTZ) + +statement ok +INSERT INTO s.filter_pushdown_timestamptz VALUES (1, '2024-01-01 12:00:00+00'), (2, '2024-01-01 13:00:00+00') + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts < TIMESTAMPTZ '2024-01-01 11:30:00+00' +---- + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts > TIMESTAMPTZ '2024-01-01 12:30:00+00' +---- +2 + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts = TIMESTAMPTZ '2024-01-01 12:00:00+00' +---- +1 + +query I +SELECT id FROM s.filter_pushdown_timestamptz WHERE ts < 'infinity'::TIMESTAMPTZ ORDER BY id +---- +1 +2 diff --git a/test/sql/storage/attach_giant_numeric.test b/test/sql/storage/attach_giant_numeric.test index 1f85debfc..4317ebfdc 100644 --- a/test/sql/storage/attach_giant_numeric.test +++ b/test/sql/storage/attach_giant_numeric.test @@ -14,13 +14,13 @@ query I SELECT * FROM s.pg_giant_numeric ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 query I SELECT * FROM s.pg_giant_numeric_array ---- -[0.0, 1.0000000000000002e+32, -1.234567891234568e+26] +[0.0, 1e+32, -1.2345678912345679e+26] [] # insert into a NUMERIC field @@ -35,8 +35,8 @@ query I SELECT * FROM s.pg_numeric_empty ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 statement ok DELETE FROM s.pg_numeric_array_empty @@ -49,5 +49,5 @@ query I SELECT UNNEST(n) FROM s.pg_numeric_array_empty ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 diff --git a/test/sql/storage/attach_isolation_level.test b/test/sql/storage/attach_isolation_level.test index b660fbde1..5db6051c4 100644 --- a/test/sql/storage/attach_isolation_level.test +++ b/test/sql/storage/attach_isolation_level.test @@ -14,8 +14,8 @@ query I SELECT * FROM s.pg_giant_numeric ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 statement ok DETACH s @@ -28,8 +28,8 @@ query I SELECT * FROM s.pg_giant_numeric ---- 0.0 -1.0000000000000002e+32 --1.234567891234568e+26 +1e+32 +-1.2345678912345679e+26 statement ok DETACH s diff --git a/test/sql/storage/attach_multi_join_large.test_slow b/test/sql/storage/attach_multi_join_large.test_slow index 8b0525c85..762898028 100644 --- a/test/sql/storage/attach_multi_join_large.test_slow +++ b/test/sql/storage/attach_multi_join_large.test_slow @@ -30,14 +30,3 @@ left join test3_large c on a.id = c.id; ---- 1000000 - -# same query but with postgres_scan -query I -select count(*) -from postgres_scan('dbname=postgresscanner', 'public', 'test1_large') a -left join postgres_scan('dbname=postgresscanner', 'public', 'test2_large') b - on a.id = b.id -left join postgres_scan('dbname=postgresscanner', 'public', 'test3_large') c - on a.id = c.id; ----- -1000000 diff --git a/test/sql/storage/attach_numeric_as_varchar.test b/test/sql/storage/attach_numeric_as_varchar.test new file mode 100644 index 000000000..235adba9d --- /dev/null +++ b/test/sql/storage/attach_numeric_as_varchar.test @@ -0,0 +1,116 @@ +# name: test/sql/storage/attach_numeric_as_varchar.test +# description: Test reading unbound numerics as varchar with pg_numeric_as_varchar +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS unbound_numerics') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE unbound_numerics(n NUMERIC)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO unbound_numerics VALUES (12345678901234567890123456789012345678901234567890), (0.12345678901234567890123456789012345678901234567890), (NULL)') + +# by default unbound numerics are read as double +query I +SELECT typeof(n) FROM s.unbound_numerics LIMIT 1 +---- +DOUBLE + +statement ok +SET pg_numeric_as_varchar=true + +statement ok +CALL postgres_clear_cache(); + +# with pg_numeric_as_varchar the values are read as varchar without precision loss +query I +SELECT typeof(n) FROM s.unbound_numerics LIMIT 1 +---- +VARCHAR + +query I +SELECT n FROM s.unbound_numerics +---- +12345678901234567890123456789012345678901234567890 +0.12345678901234567890123456789012345678901234567890 +NULL + +# the setting also applies to postgres_query +query I +SELECT * FROM postgres_query('s', 'SELECT n FROM unbound_numerics WHERE n > 1') +---- +12345678901234567890123456789012345678901234567890 + +# unbound numerics inside arrays +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS unbound_numeric_arrays') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE unbound_numeric_arrays(n NUMERIC[])') + +statement ok +CALL postgres_execute('s', 'INSERT INTO unbound_numeric_arrays VALUES (ARRAY[12345678901234567890123456789012345678901234567890::NUMERIC, 2])') + +statement ok +CALL postgres_clear_cache(); + +query I +SELECT n FROM s.unbound_numeric_arrays +---- +[12345678901234567890123456789012345678901234567890, 2] + +# bounded numerics with a precision above DuckDB's decimal maximum of 38 take the same fallback +statement ok +CALL postgres_execute('s', 'DROP TABLE IF EXISTS oversized_numerics') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE oversized_numerics(n NUMERIC(50,10))') + +statement ok +CALL postgres_execute('s', 'INSERT INTO oversized_numerics VALUES (1234567890123456789012345678901234567890.0123456789)') + +statement ok +CALL postgres_clear_cache(); + +query I +SELECT typeof(n) FROM s.oversized_numerics +---- +VARCHAR + +query I +SELECT n FROM s.oversized_numerics +---- +1234567890123456789012345678901234567890.0123456789 + +statement ok +SET pg_numeric_as_varchar=false + +statement ok +CALL postgres_clear_cache(); + +query I +SELECT typeof(n) FROM s.unbound_numerics LIMIT 1 +---- +DOUBLE + +query I +SELECT typeof(n) FROM s.oversized_numerics +---- +DOUBLE + +statement ok +CALL postgres_execute('s', 'DROP TABLE unbound_numerics') + +statement ok +CALL postgres_execute('s', 'DROP TABLE oversized_numerics') + +statement ok +CALL postgres_execute('s', 'DROP TABLE unbound_numeric_arrays') diff --git a/test/sql/storage/attach_oauth_token.test b/test/sql/storage/attach_oauth_token.test index c1c4ca6a0..3d6e198b5 100644 --- a/test/sql/storage/attach_oauth_token.test +++ b/test/sql/storage/attach_oauth_token.test @@ -10,10 +10,6 @@ require-env PGOAUTHTOKEN require-env PG_OAUTH_DSN -# Explicitly load the extension so settings are registered -statement ok -LOAD postgres_scanner; - # Test 1: Connect using the pg_oauth_token DuckDB setting statement ok SET pg_oauth_token = '${PGOAUTHTOKEN}'; diff --git a/test/sql/storage/attach_on_conflict_dropped_column.test b/test/sql/storage/attach_on_conflict_dropped_column.test new file mode 100644 index 000000000..d97b99fd8 --- /dev/null +++ b/test/sql/storage/attach_on_conflict_dropped_column.test @@ -0,0 +1,107 @@ +# name: test/sql/storage/attach_on_conflict_dropped_column.test +# description: ON CONFLICT must resolve UNIQUE/PRIMARY KEY constraints correctly after DROP COLUMN (issue #454) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) + +# Postgres does not renumber attnums after DROP COLUMN, so a column dropped +# physically before the key columns leaves a gap between the constraint's +# attnums and the remaining columns. UNIQUE(a, b, c) has attnums {3, 4, 5}, and +# dropping 'e' (attnum 2) must not break resolution of the conflict target. +statement ok +CREATE OR REPLACE TABLE s1.tbl(d INTEGER, e INTEGER, a INTEGER, b INTEGER, c INTEGER, UNIQUE(a, b, c)); + +statement ok +ALTER TABLE s1.tbl DROP COLUMN e; + +statement ok +INSERT INTO s1.tbl(a, b, c, d) VALUES (1, 2, 3, 4); + +# DO NOTHING leaves the existing row untouched +statement ok +INSERT INTO s1.tbl(a, b, c, d) VALUES (1, 2, 3, 99) ON CONFLICT (a, b, c) DO NOTHING; + +query IIII +SELECT d, a, b, c FROM s1.tbl +---- +4 1 2 3 + +# DO UPDATE targets the conflicting row +statement ok +INSERT INTO s1.tbl(a, b, c, d) VALUES (1, 2, 3, 7) ON CONFLICT (a, b, c) DO UPDATE SET d = excluded.d; + +query IIII +SELECT d, a, b, c FROM s1.tbl +---- +7 1 2 3 + +# The same attnum gap must not silently shift the key onto the wrong in-range +# columns: the real UNIQUE key is (a, b) (attnums {3, 4}), so (b, c) is not a +# valid conflict target while (a, b) is. +statement ok +CREATE OR REPLACE TABLE s1.tbl2(d INTEGER, e INTEGER, a INTEGER, b INTEGER, c INTEGER, f INTEGER, UNIQUE(a, b)); + +statement ok +ALTER TABLE s1.tbl2 DROP COLUMN e; + +statement error +INSERT INTO s1.tbl2(a, b, c) VALUES (1, 2, 3) ON CONFLICT (b, c) DO NOTHING; +---- +The specified columns as conflict target are not referenced by a UNIQUE/PRIMARY KEY CONSTRAINT + +statement ok +INSERT INTO s1.tbl2(a, b, c) VALUES (1, 2, 3) ON CONFLICT (a, b) DO NOTHING; + +# PRIMARY KEY decodes conkey the same way; verify the gap case there too. +statement ok +CREATE OR REPLACE TABLE s1.tbl3(d INTEGER, e INTEGER, a INTEGER, b INTEGER, PRIMARY KEY(a, b)); + +statement ok +ALTER TABLE s1.tbl3 DROP COLUMN e; + +statement ok +INSERT INTO s1.tbl3(a, b, d) VALUES (1, 2, 10); + +statement ok +INSERT INTO s1.tbl3(a, b, d) VALUES (1, 2, 20) ON CONFLICT (a, b) DO UPDATE SET d = excluded.d; + +query III +SELECT a, b, d FROM s1.tbl3 +---- +1 2 20 + +# Dropping a column that comes physically *after* the key columns leaves no gap +# in the key's attnums, but must still resolve correctly. +statement ok +CREATE OR REPLACE TABLE s1.tbl4(a INTEGER, b INTEGER, e INTEGER, c INTEGER, UNIQUE(a, b)); + +statement ok +ALTER TABLE s1.tbl4 DROP COLUMN e; + +statement ok +INSERT INTO s1.tbl4(a, b, c) VALUES (1, 2, 3); + +statement ok +INSERT INTO s1.tbl4(a, b, c) VALUES (1, 2, 99) ON CONFLICT (a, b) DO NOTHING; + +query III +SELECT a, b, c FROM s1.tbl4 +---- +1 2 3 + +statement ok +DROP TABLE s1.tbl + +statement ok +DROP TABLE s1.tbl2 + +statement ok +DROP TABLE s1.tbl3 + +statement ok +DROP TABLE s1.tbl4 diff --git a/test/sql/storage/attach_order_pushdown.test b/test/sql/storage/attach_order_pushdown.test index ed0aef328..7545c7a6b 100644 --- a/test/sql/storage/attach_order_pushdown.test +++ b/test/sql/storage/attach_order_pushdown.test @@ -16,7 +16,7 @@ statement ok CALL postgres_execute('s1', 'CREATE TABLE order_test (id INT NOT NULL PRIMARY KEY, val INT, name TEXT)'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); statement ok INSERT INTO s1.order_test VALUES @@ -146,13 +146,11 @@ SELECT id FROM s1.order_test ORDER BY name; # WHERE + ORDER BY on non-output column + LIMIT query I -SELECT name FROM s1.order_test WHERE val IS NOT NULL ORDER BY val LIMIT 5; +SELECT name FROM s1.order_test WHERE val IS NOT NULL ORDER BY val LIMIT 3; ---- heidi alice eve -bob -judy # Output multiple columns, sort by non-output column query II diff --git a/test/sql/storage/attach_postgres_execute.test b/test/sql/storage/attach_postgres_execute.test index 32b25cb88..c38a29b87 100644 --- a/test/sql/storage/attach_postgres_execute.test +++ b/test/sql/storage/attach_postgres_execute.test @@ -27,3 +27,40 @@ statement error CALL postgres_execute('s', 'CREATE TABLE public.my_unlogged_table(i INTEGER)') ---- already exists + +# row count + +query I +FROM postgres_execute('s', 'DROP TABLE IF EXISTS query_dml_test') +---- +-1 + +query I +FROM postgres_execute('s', 'CREATE TABLE query_dml_test(col1 INT)') +---- +-1 + +query I +FROM postgres_execute('s', 'INSERT INTO query_dml_test values(41), (42)') +---- +2 + +query I +FROM postgres_execute('s', 'UPDATE query_dml_test SET col1 = 43') +---- +2 + +query I +FROM postgres_execute('s', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +---- +0 + +query II +SELECT column_name, column_type FROM ( + DESCRIBE FROM postgres_execute('s', 'UPDATE query_dml_test SET col1 = 44 WHERE 1 < 0') +) +---- +rowcount BIGINT + +statement ok +CALL postgres_execute('s', 'DROP TABLE query_dml_test') diff --git a/test/sql/storage/attach_rds_iam.test b/test/sql/storage/attach_rds_iam.test index e969ee98c..4228127e5 100644 --- a/test/sql/storage/attach_rds_iam.test +++ b/test/sql/storage/attach_rds_iam.test @@ -43,6 +43,55 @@ FROM postgres_query('rds_attached', 'SELECT current_user') ---- postgres +query II +FROM postgres_query('rds_attached', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +') +---- +DuckDB active + +statement ok +DROP SECRET rds_secret + +statement ok +DETACH rds_attached + +# custom app name + +statement ok +CREATE SECRET rds_secret ( + TYPE postgres, + HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com', + PORT '5432', + USER 'postgres', + DATABASE 'postgres', + SSLMODE require, + AWS_RDS_SECRET rds_template_secret, + APPLICATION_NAME custom1 +); + +statement ok +ATTACH '' AS rds_attached (TYPE POSTGRES, SECRET rds_secret) + +query II +FROM postgres_query('rds_attached', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''DuckDB'' +') +---- + +query II +FROM postgres_query('rds_attached', +' +SELECT application_name, state FROM pg_stat_activity +WHERE application_name = ''custom1'' +') +---- +custom1 active + statement ok DROP SECRET rds_secret diff --git a/test/sql/storage/attach_rollback_throws.test b/test/sql/storage/attach_rollback_throws.test new file mode 100644 index 000000000..d81d6ae38 --- /dev/null +++ b/test/sql/storage/attach_rollback_throws.test @@ -0,0 +1,90 @@ +# name: test/sql/storage/attach_rollback_throws.test +# description: Test that a throw on a server ROLLBACK or COMMIT does not leak a pool connection +# group: [storage] + +# see: https://github.com/duckdb/duckdb-postgres/issues/545 + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# ROLLBACK throws + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# a connection is opened on ATTACH and kept in the pool +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +# kill Postgres backend of our connection, ROLLBACK is attempted and fails +statement error +CALL postgres_execute('s', 'SELECT pg_terminate_backend(pg_backend_pid())') +---- +terminating connection due to administrator command + +# connection must have been discarded, as it cannot pass the health check on returning it to the pool +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +DETACH s + +# COMMIT throws + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) + +statement ok +SET VARIABLE s1_pid = (SELECT pid FROM postgres_query('s', 'SELECT pg_backend_pid() AS pid')) + +statement ok +USE s + +statement ok +BEGIN TRANSACTION + +query I +FROM postgres_query('s', 'SELECT 42') +---- +42 + +statement ok +SET pg_use_text_protocol=TRUE + +statement ok +CALL postgres_query('s1', 'SELECT pg_terminate_backend($1)', params=row(getvariable(s1_pid))) + +statement ok +RESET pg_use_text_protocol + +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 1 + +statement error +COMMIT +---- +closed + +query II +SELECT catalog_name, total_connections FROM postgres_configure_pool(catalog_name='s') +---- +s 0 + +statement ok +DETACH s1 + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/storage/attach_schema_param_list.test b/test/sql/storage/attach_schema_param_list.test new file mode 100644 index 000000000..85815302c --- /dev/null +++ b/test/sql/storage/attach_schema_param_list.test @@ -0,0 +1,106 @@ +# name: test/sql/storage/attach_schema_param_list.test +# description: Test attaching multiple specific schemas +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# prepare schemas + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS duckdb_test_sch1 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS duckdb_test_sch2 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS duckdb_test_sch3 CASCADE') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA duckdb_test_sch1') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA duckdb_test_sch2') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA duckdb_test_sch3') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE duckdb_test_sch1.tab1(col1 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO duckdb_test_sch1.tab1 VALUES(42)') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE duckdb_test_sch2.tab2(col2 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO duckdb_test_sch2.tab2 VALUES(43)') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE duckdb_test_sch3.tab3(col3 INT)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO duckdb_test_sch3.tab3 VALUES(44)') + +statement ok +DETACH s + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA NULL) +---- +Binder Error + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA 42) +---- +Binder Error + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA [41, 42]) +---- +Binder Error + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA ['foo', NULL]) +---- +Binder Error + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA ['duckdb_test_sch1', 'duckdb_test_sch2']) + +query I +FROM s.duckdb_test_sch1.tab1 +---- +42 + +query I +FROM s.duckdb_test_sch2.tab2 +---- +43 + +statement error +FROM s.duckdb_test_sch3.tab3 +---- +Catalog Error: Table with name "duckdb_test_sch3.tab3" does not exist because schema "duckdb_test_sch3" does not exist. + +# cleanup + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA duckdb_test_sch1 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA duckdb_test_sch2 CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA duckdb_test_sch3 CASCADE') + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/storage/attach_secret.test b/test/sql/storage/attach_secret.test index 0d479a982..4dd83bbc7 100644 --- a/test/sql/storage/attach_secret.test +++ b/test/sql/storage/attach_secret.test @@ -152,7 +152,7 @@ CREATE OR REPLACE SECRET postgres_db ( ); statement error -ATTACH 'bar' AS secret_attach (TYPE POSTGRES, SECRET postgres_db) +ATTACH 'sslmode=require' AS secret_attach (TYPE POSTGRES, SECRET postgres_db) ---- 'URI' option diff --git a/test/sql/storage/attach_text_protocol.test b/test/sql/storage/attach_text_protocol.test new file mode 100644 index 000000000..ab0a573a9 --- /dev/null +++ b/test/sql/storage/attach_text_protocol.test @@ -0,0 +1,110 @@ +# name: test/sql/storage/attach_text_protocol.test +# description: Test the use_text_protocol option of ATTACH +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.text_protocol + +statement ok +CREATE TABLE s.text_protocol(i INTEGER, s VARCHAR) + +statement ok +INSERT INTO s.text_protocol VALUES (42, 'hello'), (NULL, NULL) + +statement ok +DETACH s + +# the protocol is resolved at ATTACH time and applies to every scan of the database: +# both regular table scans and postgres_query (which is what CONNECT runs) + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL true) + +query II +SELECT * FROM s.text_protocol ORDER BY i +---- +42 hello +NULL NULL + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL false) + +query II +SELECT * FROM s.text_protocol ORDER BY i +---- +42 hello +NULL NULL + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL 'auto') + +query II +SELECT * FROM s.text_protocol ORDER BY i +---- +42 hello +NULL NULL + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +# an explicit option at ATTACH time overrides the pg_use_text_protocol setting + +statement ok +SET pg_use_text_protocol=true + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL false) + +query II +SELECT * FROM postgres_query('s', 'SELECT * FROM text_protocol ORDER BY i') +---- +42 hello +NULL NULL + +statement ok +DETACH s + +statement ok +RESET pg_use_text_protocol + +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_TEXT_PROTOCOL 'maybe') +---- +Invalid value "maybe" for use_text_protocol, expected TRUE, FALSE or AUTO + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +DROP TABLE IF EXISTS s.text_protocol diff --git a/test/sql/storage/attach_timeout_error.test b/test/sql/storage/attach_timeout_error.test index 156f78fc6..033f65485 100644 --- a/test/sql/storage/attach_timeout_error.test +++ b/test/sql/storage/attach_timeout_error.test @@ -13,3 +13,24 @@ statement error SELECT * FROM postgres_query(s, 'select count(*) from tpch.lineitem, tpch.lineitem t2, tpch.lineitem t3'); ---- canceling statement due to statement timeout + +# The server closes the connection while the local transaction is doing +# unrelated work, then the failure is surfaced by its rollback. +statement ok +ATTACH 'dbname=postgresscanner port=5432 sslmode=allow options=''-c idle_in_transaction_session_timeout=100''' AS idle_timeout (TYPE POSTGRES, READ_ONLY); + +statement ok +BEGIN; + +query I +CALL postgres_query('idle_timeout', 'SELECT 1'); +---- +1 + +statement ok +SELECT sum(i) FROM range(1000000000) t(i); + +statement error +ROLLBACK; +---- +server closed the connection unexpectedly diff --git a/test/sql/storage/attach_types.test b/test/sql/storage/attach_types.test index b495ffaab..dde2a0849 100644 --- a/test/sql/storage/attach_types.test +++ b/test/sql/storage/attach_types.test @@ -9,6 +9,11 @@ require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) +foreach mode COPY_BINARY PLAIN_QUERY_BINARY AUTO + +statement ok +SET pg_query_mode = '{mode}' + statement ok DROP TABLE IF EXISTS s.types @@ -32,8 +37,8 @@ NULL NULL NULL NULL # test all types statement ok -CREATE TABLE all_types_tbl AS SELECT * -EXCLUDE (float, double, ubigint, hugeint, uhugeint, nested_int_array, struct, struct_of_arrays, array_of_structs, map, "union", fixed_int_array, fixed_varchar_array, fixed_nested_int_array, fixed_nested_varchar_array, fixed_struct_array, struct_of_fixed_array, fixed_array_of_int_list, list_of_fixed_int_array, bignum, geometry, timestamp_tz_ns, empty_struct) +CREATE OR REPLACE TABLE all_types_tbl AS SELECT * +EXCLUDE (float, double, ubigint, hugeint, uhugeint, nested_int_array, struct, struct_of_arrays, array_of_structs, map, "union", fixed_int_array, fixed_varchar_array, fixed_nested_int_array, fixed_nested_varchar_array, fixed_struct_array, struct_of_fixed_array, fixed_array_of_int_list, list_of_fixed_int_array, bignum, geometry, timestamp_tz_ns, empty_struct, tuple) REPLACE( CASE WHEN int IS NOT NULL THEN '2000-01-01' ELSE NULL END AS date, CASE WHEN int IS NOT NULL THEN '2000-01-01 01:02:03' ELSE NULL END AS timestamp, @@ -68,7 +73,10 @@ true 127 32767 2147483647 9223372036854775807 255 65535 4294967295 2000-01-01 24 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL # filter pushdown -foreach column_name bool tinyint smallint int bigint utinyint usmallint uint date time timestamp timestamp_s timestamp_ms timestamp_ns time_tz timestamp_tz dec_4_1 dec_9_4 dec_18_6 dec38_10 uuid interval varchar blob bit small_enum medium_enum large_enum int_array double_array date_array timestamp_array timestamptz_array varchar_array +# TODO: re-add varchar_array. Since duckdb#23539 (nested comparison semantics) an equality +# comparison on a list containing NULL evaluates to NULL, so the value cannot be round-tripped +# through `WHERE col = ` anymore. The test_all_types varchar_array holds a NULL element. +foreach column_name bool tinyint smallint int bigint utinyint usmallint uint date time timestamp timestamp_s timestamp_ms timestamp_ns time_tz timestamp_tz dec_4_1 dec_9_4 dec_18_6 dec38_10 uuid interval varchar blob bit small_enum medium_enum large_enum int_array double_array date_array timestamp_array timestamptz_array statement ok SET VARIABLE minimum_value=(SELECT MIN({column_name}) min_val FROM s.all_types); @@ -80,6 +88,8 @@ true endloop +endloop + # text protocol statement ok SET pg_use_text_protocol=true; diff --git a/test/sql/storage/attach_types_blob.test b/test/sql/storage/attach_types_blob.test index faad34505..3a602bb63 100644 --- a/test/sql/storage/attach_types_blob.test +++ b/test/sql/storage/attach_types_blob.test @@ -9,6 +9,11 @@ require-env POSTGRES_TEST_DATABASE_AVAILABLE statement ok ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE s.blobs(b BLOB); @@ -27,3 +32,8 @@ SELECT * FROM s.blobs \x00\x00\x00 NULL thisisalongstring\x00\x00withnullbytes + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_char.test b/test/sql/storage/attach_types_char.test index 9160b6b9d..e198532b9 100644 --- a/test/sql/storage/attach_types_char.test +++ b/test/sql/storage/attach_types_char.test @@ -22,6 +22,11 @@ hello 5 (empty) 0 NULL NULL +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement error INSERT INTO chars VALUES ('this string is too long') ---- @@ -39,6 +44,9 @@ INSERT INTO chars VALUES ('hello'), ('world'), ('maxlength1'), ('hello '), ( statement ok COMMIT +statement ok +SET pg_use_text_protocol = FALSE + query II SELECT c, LENGTH(c) FROM chars ---- @@ -71,7 +79,15 @@ hello 10 5 NULL NULL +statement ok +SET pg_use_text_protocol = {plain} + statement error INSERT INTO varchars_fixed_len VALUES ('this string is too long') ---- value too long + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_date.test b/test/sql/storage/attach_types_date.test index 629940539..99464f12e 100644 --- a/test/sql/storage/attach_types_date.test +++ b/test/sql/storage/attach_types_date.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE dates(d DATE); @@ -59,6 +64,13 @@ INSERT INTO dates VALUES (DATE '5881580-07-10'); ---- out of range +endloop + +statement ok +RESET pg_use_text_protocol + +# TODO: BC with text proto + query I SELECT * FROM dates ---- diff --git a/test/sql/storage/attach_types_decimal.test b/test/sql/storage/attach_types_decimal.test index 1d8844f10..0b20f90f3 100644 --- a/test/sql/storage/attach_types_decimal.test +++ b/test/sql/storage/attach_types_decimal.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + # no scale (integers) statement ok CREATE OR REPLACE TABLE decimals(d DECIMAL(4,0)); @@ -183,3 +188,8 @@ SELECT * FROM decimals 1234567891234567891.1234567891234567891 9999999999999999999.9999999999999999999 -9999999999999999999.9999999999999999999 + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_enum.test b/test/sql/storage/attach_types_enum.test index fdae4a705..c352aa8f4 100644 --- a/test/sql/storage/attach_types_enum.test +++ b/test/sql/storage/attach_types_enum.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok DROP TABLE IF EXISTS enums @@ -52,6 +57,11 @@ sad happy NULL +endloop + +statement ok +RESET pg_use_text_protocol + # reconnect statement ok ATTACH ':memory:' AS x @@ -68,6 +78,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + query I SELECT UNNEST(enum_range(NULL::MOOD)) ---- @@ -98,3 +113,8 @@ statement error CREATE OR REPLACE TABLE enums(e ENUM('sad', 'happy')); ---- Enums in Postgres must be named + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_interval.test b/test/sql/storage/attach_types_interval.test index e9fd77229..ce37bc6ec 100644 --- a/test/sql/storage/attach_types_interval.test +++ b/test/sql/storage/attach_types_interval.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE attach_intervals(i INTERVAL); @@ -38,3 +43,8 @@ NULL 00:30:15 2 months 2 days 00:15:00 -2 days + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_json.test b/test/sql/storage/attach_types_json.test index 9652ae4cc..384cd9400 100644 --- a/test/sql/storage/attach_types_json.test +++ b/test/sql/storage/attach_types_json.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + query I SELECT * FROM pg_json ---- @@ -27,3 +32,8 @@ query I SELECT * FROM pg_json ---- {"a": 42, "b": "string"} + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_macaddr.test b/test/sql/storage/attach_types_macaddr.test index 9b5d99772..37e20ae4c 100644 --- a/test/sql/storage/attach_types_macaddr.test +++ b/test/sql/storage/attach_types_macaddr.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + # macaddress only statement ok DELETE FROM pg_macaddr @@ -28,6 +33,11 @@ SELECT * FROM pg_macaddr 08:00:2b:01:02:03 NULL +endloop + +statement ok +RESET pg_use_text_protocol + # complex types, including list of macaddr statement ok DELETE FROM pg_complex_types_mix diff --git a/test/sql/storage/attach_types_multidimensional_array.test b/test/sql/storage/attach_types_multidimensional_array.test index 79c4f0c6a..3c42a9c3b 100644 --- a/test/sql/storage/attach_types_multidimensional_array.test +++ b/test/sql/storage/attach_types_multidimensional_array.test @@ -93,7 +93,7 @@ NULL # clear cache, then query again # this forces us to load the metadata from postgres itself statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query I SELECT * FROM two_dimensional_array diff --git a/test/sql/storage/attach_types_table_row.test b/test/sql/storage/attach_types_table_row.test index 0a5cf8766..aa3f958d5 100644 --- a/test/sql/storage/attach_types_table_row.test +++ b/test/sql/storage/attach_types_table_row.test @@ -29,7 +29,7 @@ statement ok CALL postgres_execute('s', 'CREATE VIEW row_types.widget_row AS SELECT w FROM row_types.widget AS w'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types.widget_row @@ -44,7 +44,7 @@ statement ok CALL postgres_execute('s', 'INSERT INTO row_types.wrapper VALUES (1, ROW(2, ''nested''))'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query III SELECT id, payload.id, payload.label FROM row_types.wrapper @@ -56,7 +56,7 @@ statement ok SET pg_use_text_protocol=true; statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types.widget_row @@ -99,7 +99,7 @@ statement ok CALL postgres_execute('s', 'INSERT INTO row_types_b.wrapper VALUES (1, ROW(2, ''nested''))'); statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types_b.widget_view @@ -115,7 +115,7 @@ statement ok SET pg_use_text_protocol=true; statement ok -CALL pg_clear_cache(); +CALL postgres_clear_cache(); query II SELECT w.id, w.label FROM row_types_b.widget_view diff --git a/test/sql/storage/attach_types_time.test b/test/sql/storage/attach_types_time.test index 470d16259..11670954b 100644 --- a/test/sql/storage/attach_types_time.test +++ b/test/sql/storage/attach_types_time.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE times(t TIME); @@ -53,3 +58,8 @@ SELECT * FROM timestz NULL 23:59:59+00 23:59:59+06 + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_timestamp.test b/test/sql/storage/attach_types_timestamp.test index a6b2d13ed..01c46daf6 100644 --- a/test/sql/storage/attach_types_timestamp.test +++ b/test/sql/storage/attach_types_timestamp.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE timestamps(t TIMESTAMP); @@ -49,6 +54,11 @@ insert into timestamps values (TIMESTAMP 'infinity'); statement ok insert into timestamps values (TIMESTAMP '-infinity'); +# TODO: BC with text proto + +statement ok +SET pg_use_text_protocol = FALSE + query I SELECT * FROM timestamps ---- @@ -60,6 +70,9 @@ NULL infinity -infinity +statement ok +SET pg_use_text_protocol = {plain} + # timestamptz statement ok CREATE OR REPLACE TABLE timestamptz(t TIMESTAMPTZ); @@ -137,3 +150,8 @@ SELECT * FROM timestamp_tbl 2000-01-01 00:00:00 NULL 1980-01-01 23:59:59.123456 + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_uuid.test b/test/sql/storage/attach_types_uuid.test index b92094e9a..a53e887be 100644 --- a/test/sql/storage/attach_types_uuid.test +++ b/test/sql/storage/attach_types_uuid.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE uuids(u UUID); @@ -34,3 +39,8 @@ SELECT * FROM uuids NULL 00000000-0000-0000-0000-000000000001 ffffffff-ffff-ffff-ffff-ffffffffffff + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_varchar.test b/test/sql/storage/attach_types_varchar.test index c6ba6ee5c..d568b88a6 100644 --- a/test/sql/storage/attach_types_varchar.test +++ b/test/sql/storage/attach_types_varchar.test @@ -12,6 +12,11 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement ok USE s; +foreach plain FALSE TRUE + +statement ok +SET pg_use_text_protocol = {plain} + statement ok CREATE OR REPLACE TABLE varchars(v VARCHAR); @@ -34,3 +39,8 @@ SELECT * FROM varchars this is a long string 🦆🦆🦆🦆🦆🦆🦆🦆 NULL + +endloop + +statement ok +RESET pg_use_text_protocol diff --git a/test/sql/storage/attach_types_varchar_fallback.test b/test/sql/storage/attach_types_varchar_fallback.test new file mode 100644 index 000000000..67329bee1 --- /dev/null +++ b/test/sql/storage/attach_types_varchar_fallback.test @@ -0,0 +1,78 @@ +# name: test/sql/storage/attach_types_varchar_fallback.test +# description: Test for using types from different schema that must fall back to VARCHAR +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# prepare schemas + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS data_s CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA IF EXISTS types_s CASCADE') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA types_s') + +statement ok +CALL postgres_execute('s', 'CREATE SCHEMA data_s') + +statement ok +CALL postgres_execute('s', 'CREATE TYPE types_s.mood AS ENUM (''happy'',''sad'')') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE data_s.users (id int, name text)') + +statement ok +CALL postgres_execute('s', 'CREATE TABLE data_s.moods (id int, m types_s.mood)') + +statement ok +CALL postgres_execute('s', 'INSERT INTO data_s.users VALUES (1, ''a'')') + +statement ok +CALL postgres_execute('s', 'INSERT INTO data_s.moods VALUES (1, ''happy'')') + +statement ok +DETACH s + +# attach only the data schema + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, SCHEMA data_s, READ_ONLY) + +statement ok +USE s + +query I +SELECT count(*) FROM s.data_s.users +---- +1 + +# cleanup + +statement ok +USE memory + +statement ok +DETACH s + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA data_s CASCADE') + +statement ok +CALL postgres_execute('s', 'DROP SCHEMA types_s CASCADE') + +statement ok +USE memory + +statement ok +DETACH s diff --git a/test/sql/storage/attach_use_ctid_scan.test b/test/sql/storage/attach_use_ctid_scan.test new file mode 100644 index 000000000..86d1c7125 --- /dev/null +++ b/test/sql/storage/attach_use_ctid_scan.test @@ -0,0 +1,73 @@ +# name: test/sql/storage/attach_use_ctid_scan.test +# description: Test the per-attach use_ctid_scan option overriding the global pg_use_ctid_scan setting +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +PRAGMA enable_verification + +# per-attach option disables ctid scan even though the global setting defaults to true +statement ok +SET pg_use_ctid_scan=true; + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN false) + +query I +SELECT * FROM s.pg_giant_numeric +---- +0.0 +1.0000000000000002e+32 +-1.234567891234568e+26 + +statement ok +DETACH s + +# per-attach option enables ctid scan even though the global setting is false (override works both ways) +statement ok +SET pg_use_ctid_scan=false; + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN true) + +query I +SELECT * FROM s.pg_giant_numeric +---- +0.0 +1.0000000000000002e+32 +-1.234567891234568e+26 + +statement ok +DETACH s + +# when omitted, behavior follows the global setting (here: false) and still produces correct results +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +query I +SELECT * FROM s.pg_giant_numeric +---- +0.0 +1.0000000000000002e+32 +-1.234567891234568e+26 + +statement ok +DETACH s + +statement ok +SET pg_use_ctid_scan=true; + +# boolean-typed values are accepted +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN 1) + +statement ok +DETACH s + +# a non-boolean value raises a clear error +statement error +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES, USE_CTID_SCAN 'not_a_bool') +---- diff --git a/test/sql/storage/attach_views.test b/test/sql/storage/attach_views.test index c5279c050..35e68d554 100644 --- a/test/sql/storage/attach_views.test +++ b/test/sql/storage/attach_views.test @@ -45,7 +45,7 @@ DELETE FROM v1 statement error INSERT INTO v1 VALUES (1, 1); ---- -table v1 has 1 columns but 2 values were supplied +has 1 columns but 2 values were supplied statement ok CREATE VIEW IF NOT EXISTS v1 AS SELECT 84; diff --git a/test/sql/storage/catalog_cache_external_ddl_visibility.test b/test/sql/storage/catalog_cache_external_ddl_visibility.test new file mode 100644 index 000000000..cc277fb2b --- /dev/null +++ b/test/sql/storage/catalog_cache_external_ddl_visibility.test @@ -0,0 +1,58 @@ +# name: test/sql/storage/catalog_cache_external_ddl_visibility.test +# description: DDL from a separate ClientContext must become visible without an explicit postgres_clear_cache() call +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# con1: prime an empty-schema cache +statement ok con1 +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +USE s + +statement ok con1 +DROP TABLE IF EXISTS ext_ddl_visibility_tbl + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 's' AND table_name = 'ext_ddl_visibility_tbl' +---- +0 + +# con2: a genuinely separate ClientContext, attaching the same physical database +statement ok con2 +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +statement ok con2 +USE s2 + +statement ok con2 +CREATE TABLE ext_ddl_visibility_tbl AS SELECT 1 AS x + +# con1: re-query without any postgres_clear_cache() call from either connection +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 's' AND table_name = 'ext_ddl_visibility_tbl' +---- +1 + +query I con1 +SELECT x FROM ext_ddl_visibility_tbl +---- +1 + +statement ok con1 +DROP TABLE ext_ddl_visibility_tbl + +statement ok con2 +USE memory + +statement ok con2 +DETACH s2 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_cache_identity_swap_detected.test b/test/sql/storage/catalog_cache_identity_swap_detected.test new file mode 100644 index 000000000..6ad856c84 --- /dev/null +++ b/test/sql/storage/catalog_cache_identity_swap_detected.test @@ -0,0 +1,64 @@ +# name: test/sql/storage/catalog_cache_identity_swap_detected.test +# description: drop+create in the same schema that leaves row count unchanged must still be detected as stale (adversarial case that rules out a pure cardinality or xmin-aggregation staleness signal) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +USE s + +statement ok con1 +DROP TABLE IF EXISTS identity_swap_foo + +statement ok con1 +DROP TABLE IF EXISTS identity_swap_bar + +statement ok con1 +CREATE TABLE identity_swap_foo AS SELECT 1 AS x + +# prime the cache with foo present +query I con1 +SELECT table_name FROM duckdb_tables() WHERE table_name LIKE 'identity_swap_%' ORDER BY table_name +---- +identity_swap_foo + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +statement ok con2 +USE s2 + +# net pg_class row count for this schema is unchanged: one dropped, one created +statement ok con2 +DROP TABLE identity_swap_foo + +statement ok con2 +CREATE TABLE identity_swap_bar AS SELECT 2 AS y + +# con1 must see bar and must NOT still see foo, despite unchanged cardinality +# (filtered to database_name = 's' - con2's own s2 attach of the same physical +# DB would otherwise also surface identity_swap_bar a second time) +query I con1 +SELECT table_name FROM duckdb_tables() WHERE database_name = 's' AND table_name LIKE 'identity_swap_%' ORDER BY table_name +---- +identity_swap_bar + +statement ok con1 +DROP TABLE identity_swap_bar + +statement ok con2 +USE memory + +statement ok con2 +DETACH s2 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_cache_staged_signature_cross_connection.test b/test/sql/storage/catalog_cache_staged_signature_cross_connection.test new file mode 100644 index 000000000..374ecc856 --- /dev/null +++ b/test/sql/storage/catalog_cache_staged_signature_cross_connection.test @@ -0,0 +1,89 @@ +# name: test/sql/storage/catalog_cache_staged_signature_cross_connection.test +# description: a pending staleness signature staged by one connection's open explicit transaction must not be promoted or discarded by a concurrent, unrelated autocommit write from a different connection +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +USE s + +statement ok con1 +DROP TABLE IF EXISTS staged_sig_test + +statement ok con1 +CREATE TABLE staged_sig_test(i INTEGER) + +statement ok con1 +INSERT INTO s.staged_sig_test VALUES (42) + +# prime con1's cache +query I con1 +SELECT column_name FROM duckdb_columns() WHERE table_name = 'staged_sig_test' ORDER BY column_name +---- +i + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS s2 (TYPE POSTGRES) + +statement ok con2 +USE s2 + +# con1 opens an explicit transaction, alters the table, and forces a reload that stages a +# pending signature (not yet committed to con1's PostgresTableSet::staleness_signature) +statement ok con1 +BEGIN + +statement ok con1 +ALTER TABLE s.staged_sig_test ADD COLUMN j INTEGER + +query I con1 +SELECT j FROM s.staged_sig_test +---- +NULL + +# concurrently, con2 (autocommit, unrelated connection) writes a *different* table in the +# same schema and commits - this must not promote or clobber con1's still-pending signature +statement ok con2 +DROP TABLE IF EXISTS staged_sig_unrelated + +statement ok con2 +CREATE TABLE staged_sig_unrelated(x INTEGER) + +# con1 rolls back its own transaction - its pending signature (with column j) must be discarded, +# not promoted, regardless of what con2 did concurrently +statement ok con1 +ROLLBACK + +statement error con1 +SELECT j FROM s.staged_sig_test +---- +not found + +# con1's cache must still correctly see con2's committed, unrelated change on the next access +query I con1 +SELECT table_name FROM duckdb_tables() WHERE database_name = 's' AND table_name = 'staged_sig_unrelated' +---- +staged_sig_unrelated + +statement ok con1 +DROP TABLE staged_sig_test + +statement ok con1 +DROP TABLE staged_sig_unrelated + +statement ok con2 +USE memory + +statement ok con2 +DETACH s2 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_cache_staleness_wire_compat_gating.test b/test/sql/storage/catalog_cache_staleness_wire_compat_gating.test new file mode 100644 index 000000000..f1520dd7f --- /dev/null +++ b/test/sql/storage/catalog_cache_staleness_wire_compat_gating.test @@ -0,0 +1,259 @@ +# name: test/sql/storage/catalog_cache_staleness_wire_compat_gating.test +# description: pg_staleness_query_enabled gates automatic staleness detection, default tied to pg_use_information_schema_introspection +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# --------------------------------------------------------------------------- +# Default-on for real Postgres (unchanged behavior) +# --------------------------------------------------------------------------- +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc1 (TYPE POSTGRES) + +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +DROP TABLE IF EXISTS wc1.public.staleness_gating_tbl1 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc1' AND table_name = 'staleness_gating_tbl1' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc1b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc1b.public.staleness_gating_tbl1 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc1' AND table_name = 'staleness_gating_tbl1' +---- +1 + +statement ok con1 +DROP TABLE wc1.public.staleness_gating_tbl1 + +statement ok con2 +DETACH wc1b + +statement ok con1 +DETACH wc1 + +# --------------------------------------------------------------------------- +# Default-off when compat mode is signaled +# --------------------------------------------------------------------------- +statement ok +SET GLOBAL pg_staleness_query_enabled = NULL + +statement ok con1 +SET pg_use_information_schema_introspection=true + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc2 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc2.public.staleness_gating_tbl2 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc2' AND table_name = 'staleness_gating_tbl2' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc2b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc2b.public.staleness_gating_tbl2 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc2' AND table_name = 'staleness_gating_tbl2' +---- +0 + +statement ok con2 +DROP TABLE wc2b.public.staleness_gating_tbl2 + +statement ok con2 +DETACH wc2b + +statement ok con1 +RESET pg_use_information_schema_introspection + +statement ok con1 +DETACH wc2 + +# --------------------------------------------------------------------------- +# Explicit override wins, on-direction +# --------------------------------------------------------------------------- +statement ok con1 +SET pg_use_information_schema_introspection=true + +statement ok con1 +SET GLOBAL pg_staleness_query_enabled=true + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc3 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc3.public.staleness_gating_tbl3 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc3' AND table_name = 'staleness_gating_tbl3' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc3b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc3b.public.staleness_gating_tbl3 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc3' AND table_name = 'staleness_gating_tbl3' +---- +1 + +statement ok con1 +DROP TABLE wc3.public.staleness_gating_tbl3 + +statement ok con2 +DETACH wc3b + +statement ok con1 +RESET pg_use_information_schema_introspection + +statement ok con1 +RESET GLOBAL pg_staleness_query_enabled + +statement ok con1 +DETACH wc3 + +# --------------------------------------------------------------------------- +# Explicit override wins, off-direction +# --------------------------------------------------------------------------- +statement ok con1 +SET GLOBAL pg_staleness_query_enabled=false + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc4 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc4.public.staleness_gating_tbl4 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc4' AND table_name = 'staleness_gating_tbl4' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc4b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc4b.public.staleness_gating_tbl4 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc4' AND table_name = 'staleness_gating_tbl4' +---- +0 + +statement ok con2 +DROP TABLE wc4b.public.staleness_gating_tbl4 + +statement ok con2 +DETACH wc4b + +statement ok con1 +RESET GLOBAL pg_staleness_query_enabled + +statement ok con1 +DETACH wc4 + +# --------------------------------------------------------------------------- +# Custom query used in place of the default +# --------------------------------------------------------------------------- +statement ok +SET GLOBAL pg_staleness_query_enabled = TRUE + +statement ok con1 +SET GLOBAL pg_staleness_query='SELECT pg_class.oid, relname, pg_class.xmin FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid WHERE relkind IN (''r'',''v'',''m'',''f'',''p'') AND pg_namespace.nspname = ${SCHEMA} ORDER BY pg_class.oid' + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc5 (TYPE POSTGRES) + +statement ok con1 +DROP TABLE IF EXISTS wc5.public.staleness_gating_tbl5 + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc5' AND table_name = 'staleness_gating_tbl5' +---- +0 + +statement ok con2 +ATTACH 'dbname=postgresscanner' AS wc5b (TYPE POSTGRES) + +statement ok con2 +CREATE TABLE wc5b.public.staleness_gating_tbl5 AS SELECT 1 AS x + +query I con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc5' AND table_name = 'staleness_gating_tbl5' +---- +1 + +statement ok con1 +DROP TABLE wc5.public.staleness_gating_tbl5 + +statement ok con2 +DETACH wc5b + +statement ok con1 +RESET GLOBAL pg_staleness_query + +statement ok con1 +DETACH wc5 + +# --------------------------------------------------------------------------- +# Missing ${SCHEMA} placeholder is rejected before running +# --------------------------------------------------------------------------- +statement ok con1 +SET GLOBAL pg_staleness_query='SELECT pg_class.oid, relname, pg_class.xmin FROM pg_class ORDER BY pg_class.oid' + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc6 (TYPE POSTGRES) + +statement error con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc6' +---- +:.*pg_staleness_query must contain a \$\{SCHEMA\} placeholder.* + +statement ok con1 +RESET GLOBAL pg_staleness_query + +statement ok con1 +DETACH wc6 + +# --------------------------------------------------------------------------- +# Wrong column count is rejected, malformed query fails loud +# --------------------------------------------------------------------------- +statement ok con1 +SET GLOBAL pg_staleness_query='SELECT pg_class.oid FROM pg_class JOIN pg_namespace ON relnamespace = pg_namespace.oid WHERE pg_namespace.nspname = ${SCHEMA}' + +statement ok con1 +ATTACH 'dbname=postgresscanner' AS wc7 (TYPE POSTGRES) + +statement error con1 +SELECT count(*) FROM duckdb_tables() WHERE database_name = 'wc7' +---- +:.*pg_staleness_query must return at least 3 columns.* + +statement ok con1 +RESET GLOBAL pg_staleness_query + +statement ok con1 +DETACH wc7 + +statement ok +RESET GLOBAL pg_staleness_query_enabled diff --git a/test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow b/test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow new file mode 100644 index 000000000..1c519efea --- /dev/null +++ b/test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow @@ -0,0 +1,49 @@ +# name: test/sql/storage/catalog_scan_concurrent_clear_stress.test_slow +# description: Concurrent postgres_clear_cache() vs duckdb_tables()/duckdb_indexes() UAF stress repro (run under ASAN) +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +require-env SANITIZER_BUILD + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +DROP TABLE IF EXISTS scan_stress_tbl + +statement ok +CREATE TABLE scan_stress_tbl (i INTEGER) + +statement ok +INSERT INTO scan_stress_tbl VALUES (1) + +concurrentloop i 0 200 + +statement maybe +CALL postgres_clear_cache(); +---- + +statement maybe +SELECT * FROM duckdb_tables(); +---- + +statement maybe +SELECT * FROM duckdb_indexes(); +---- + +statement maybe +SELECT * FROM scan_stress_tbl; +---- + +endloop + +query I +SELECT * FROM scan_stress_tbl +---- +1 diff --git a/test/sql/storage/catalog_scan_entry_survives_clear.test b/test/sql/storage/catalog_scan_entry_survives_clear.test new file mode 100644 index 000000000..12168af7e --- /dev/null +++ b/test/sql/storage/catalog_scan_entry_survives_clear.test @@ -0,0 +1,35 @@ +# name: test/sql/storage/catalog_scan_entry_survives_clear.test +# description: PostgresCatalogSet::Scan entries must survive a concurrent postgres_clear_cache() +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +USE s + +statement ok +DROP TABLE IF EXISTS scan_survives_clear_tbl + +statement ok +CREATE TABLE scan_survives_clear_tbl AS SELECT 1 AS x + +query I +SELECT count(*) >= 0 FROM duckdb_tables() +---- +1 + +statement ok +CALL postgres_clear_cache() + +query I +SELECT table_name FROM duckdb_tables() WHERE table_name = 'scan_survives_clear_tbl' +---- +scan_survives_clear_tbl + +statement ok +DROP TABLE scan_survives_clear_tbl diff --git a/test/sql/storage/detach_prepared.test b/test/sql/storage/detach_prepared.test new file mode 100644 index 000000000..ca487b64a --- /dev/null +++ b/test/sql/storage/detach_prepared.test @@ -0,0 +1,126 @@ +# name: test/sql/storage/detach_prepared.test +# description: Test the behaviour of DETACH with PREPARE/EXECUTE +# group: [storage] + +require mysql_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.detach_prepare_test AS FROM range(41, 43) r + +statement ok +PREPARE p1 AS SELECT divide(r, 1) FROM s.detach_prepare_test + +query I +EXECUTE p1 +---- +41 +42 + +query I +EXECUTE p1 +---- +41 +42 + +statement ok +DETACH s + +statement error +EXECUTE p1 +---- +Catalog Error: Table with name "s.detach_prepare_test" does not exist because schema "s" does not exist. + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +query I +EXECUTE p1 +---- +41 +42 + +statement ok +DROP TABLE s.detach_prepare_test + +statement error +EXECUTE p1 +---- +Catalog Error: Table with name detach_prepare_test does not exist! + +statement ok +DEALLOCATE p1 + +statement error +EXECUTE p1 +---- +Binder Error: Prepared statement "p1" does not exist + +statement ok +DETACh s + +# prepared postgres_query, no cached tables + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CALL postgres_query('s', 'CREATE TABLE detach_prepare_test(r INT)') + +statement ok +CALL postgres_query('s', 'INSERT INTO detach_prepare_test VALUES (41),(42)') + +statement ok +PREPARE p2 AS FROM postgres_query('s', 'SELECT * FROM detach_prepare_test') + +query I +EXECUTE p2 +---- +41 +42 + +query I +EXECUTE p2 +---- +41 +42 + +statement ok +DETACH s + +statement error +EXECUTE p2 +---- +Invalid Input Error: Attached PostgreSQL database not found in the specified client session, name: "s" + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +query I +EXECUTE p2 +---- +41 +42 + +statement ok +CALL postgres_query('s', 'DROP TABLE detach_prepare_test') + +statement error +EXECUTE p2 +---- +ERROR: relation "detach_prepare_test" does not exist + +statement ok +DEALLOCATE p2 + +statement error +EXECUTE p2 +---- +Binder Error: Prepared statement "p2" does not exist + +statement ok +DETACh s diff --git a/test/sql/storage/filter_pushdown_blob_in.test b/test/sql/storage/filter_pushdown_blob_in.test new file mode 100644 index 000000000..9ba0734bd --- /dev/null +++ b/test/sql/storage/filter_pushdown_blob_in.test @@ -0,0 +1,61 @@ +# name: test/sql/storage/filter_pushdown_blob_in.test +# description: Test pushing down IN filters over blob columns +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +SET pg_enable_filter_pushdown=true + +statement ok +CREATE OR REPLACE TABLE s.blob_in(id INTEGER, b BLOB); + +statement ok +INSERT INTO s.blob_in VALUES (1, '\x01\x02'), (2, '\x03\x04'), (3, 'dpfkg'), (4, 'other'), (5, NULL); + +# blob constants in an IN list must be written as bytea literals, not bare hex digits +query I +SELECT id FROM s.blob_in WHERE b IN ('dpfkg'::BLOB, 'other'::BLOB) ORDER BY id +---- +3 +4 + +query I +SELECT id FROM s.blob_in WHERE b IN ('\x01\x02'::BLOB, '\x03\x04'::BLOB) ORDER BY id +---- +1 +2 + +# a longer list, which the binder is more likely to keep as a COMPARE_IN expression +query I +SELECT id FROM s.blob_in WHERE b IN ('\x01\x02'::BLOB, '\x03\x04'::BLOB, 'dpfkg'::BLOB, 'other'::BLOB, '\xff'::BLOB, '\xfe'::BLOB) ORDER BY id +---- +1 +2 +3 +4 + +query I +SELECT count(*) FROM s.blob_in WHERE b NOT IN ('dpfkg'::BLOB, 'other'::BLOB) +---- +2 + +# blobs containing quote and backslash bytes must survive the round trip +statement ok +INSERT INTO s.blob_in VALUES (6, '\x27\x5c'); + +query I +SELECT id FROM s.blob_in WHERE b IN ('\x27\x5c'::BLOB, '\x00'::BLOB) ORDER BY id +---- +6 + +statement ok +DROP TABLE s.blob_in + +statement ok +DETACH s diff --git a/test/sql/storage/information_schema_introspection.test b/test/sql/storage/information_schema_introspection.test new file mode 100644 index 000000000..b68df5da7 --- /dev/null +++ b/test/sql/storage/information_schema_introspection.test @@ -0,0 +1,90 @@ +# name: test/sql/storage/information_schema_introspection.test +# description: Test pg_use_information_schema_introspection — schema discovery via SQL-standard information_schema instead of pg_catalog +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +# baseline ATTACH (pg_catalog path) — collect the table set for comparison +statement ok +ATTACH 'dbname=postgresscanner' AS pgc (TYPE POSTGRES) + +# default is false; default path is unchanged +query I +SELECT current_setting('pg_use_information_schema_introspection') +---- +false + +# snapshot schema as obtained the pg way +statement ok +CREATE TEMP TABLE pgc_tables AS SELECT schema_name, table_name FROM duckdb_tables() WHERE database_name='pgc' AND schema_name NOT IN ('pg_catalog', 'information_schema') + +statement ok +DETACH pgc + +statement ok +SET pg_use_information_schema_introspection=true + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +# the two paths discover the same tables +query I +SELECT COUNT(*) FROM ( + SELECT schema_name, table_name FROM duckdb_tables() WHERE database_name='s' + EXCEPT + SELECT schema_name, table_name FROM pgc_tables +) +---- +0 + +# query it the other way around to ensure exact match +query I +SELECT COUNT(*) FROM ( + SELECT schema_name, table_name FROM pgc_tables + EXCEPT + SELECT schema_name, table_name FROM duckdb_tables() WHERE database_name='s' +) +---- +0 + +# round-trip a table with the data_type forms the new path has to map +statement ok +DROP TABLE IF EXISTS s.public.is_types + +statement ok +CREATE TABLE s.public.is_types(i BIGINT, name VARCHAR, ts TIMESTAMPTZ, flag BOOLEAN) + +statement ok +INSERT INTO s.public.is_types VALUES (42, 'hello', TIMESTAMPTZ '2024-01-01 00:00:00+00', true) + +statement ok +INSERT INTO s.public.is_types VALUES (NULL, NULL, NULL, NULL) + +# force a fresh introspection of the new table +statement ok +CALL postgres_clear_cache() + +query IIII +SELECT i, name, ts, flag FROM s.public.is_types ORDER BY i NULLS LAST +---- +42 hello 2024-01-01 00:00:00+00 true +NULL NULL NULL NULL + +# information_schema reports verbose data_type names; verify the mapping landed +# on the right DuckDB types (bigint→BIGINT, character varying→VARCHAR, +# timestamp with time zone→TIMESTAMPTZ (which duckdb then reports back as timestamp with timezone), boolean→BOOLEAN) +query IIII +SELECT typeof(i), typeof(name), typeof(ts), typeof(flag) FROM s.public.is_types LIMIT 1 +---- +BIGINT VARCHAR TIMESTAMP WITH TIME ZONE BOOLEAN + +statement ok +DROP TABLE s.public.is_types + +statement ok +DETACH s + +statement ok +RESET pg_use_information_schema_introspection diff --git a/test/sql/storage/limit_parallel_bug.test b/test/sql/storage/limit_parallel_bug.test_slow similarity index 96% rename from test/sql/storage/limit_parallel_bug.test rename to test/sql/storage/limit_parallel_bug.test_slow index dc325a9e7..76570f50d 100644 --- a/test/sql/storage/limit_parallel_bug.test +++ b/test/sql/storage/limit_parallel_bug.test_slow @@ -1,4 +1,4 @@ -# name: test/sql/storage/limit_parallel_bug.test +# name: test/sql/storage/limit_parallel_bug.test_slow # description: Test LIMIT bug with parallelism (issues #380, #395) # group: [storage] diff --git a/test/sql/storage/multi_scan_aggregate.test b/test/sql/storage/multi_scan_aggregate.test new file mode 100644 index 000000000..9ae996a94 --- /dev/null +++ b/test/sql/storage/multi_scan_aggregate.test @@ -0,0 +1,55 @@ +# name: test/sql/storage/multi_scan_aggregate.test +# description: Test multiple simultaneous scans that are aggregated on DuckDB side +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_a AS +SELECT '2020-01-01' insert_date, unnest(range(1<<14)) amount + +statement ok +CREATE OR REPLACE TABLE s.aggr_union_test_b AS +SELECT '2021-01-01' insert_date, unnest(range(1<<14)) amount + +# Note, 'divide()' is necessary to prevent remote-execute pushdown + +query II +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_a t +GROUP BY 1 +---- +2020-01-01 134209536 + +query II +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_b t +GROUP BY 1 +---- +2021-01-01 134209536 + +query II +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_a t +GROUP BY 1 +UNION ALL +SELECT date(t.insert_date) AS d, divide(sum(t.amount), 1) AS s +FROM s.aggr_union_test_b t +GROUP BY 1 +---- +2020-01-01 134209536 +2021-01-01 134209536 + +statement ok +DROP TABLE s.aggr_union_test_a + +statement ok +DROP TABLE s.aggr_union_test_b + +statement ok +DETACH s; diff --git a/test/sql/storage/postgres_execute_transaction.test b/test/sql/storage/postgres_execute_transaction.test index 8ce709d7a..ed8b1c7e3 100644 --- a/test/sql/storage/postgres_execute_transaction.test +++ b/test/sql/storage/postgres_execute_transaction.test @@ -39,7 +39,7 @@ CALL postgres_query('s', 'SELECT 42') 42 statement ok -CALL postgres_execute('s', 'INSERT INTO postgres_execute_attempt VALUES (42); INSERT INTO postgres_execute_attempt VALUES (84)') +CALL postgres_execute('s', 'INSERT INTO postgres_execute_attempt VALUES (42), (84)') statement ok COMMIT @@ -57,7 +57,11 @@ statement ok DESCRIBE query I -SELECT query FROM duckdb_logs_parsed('PostgresQueryLog'); +SELECT count(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query LIKE 'BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ%' ---- -BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ -COMMIT +1 + +query I +SELECT count(*) FROM duckdb_logs_parsed('PostgresQueryLog') WHERE query = 'COMMIT' +---- +1 diff --git a/test/sql/storage/postgres_execute_use_transaction.test b/test/sql/storage/postgres_execute_use_transaction.test index 5257dd3a6..2c083720b 100644 --- a/test/sql/storage/postgres_execute_use_transaction.test +++ b/test/sql/storage/postgres_execute_use_transaction.test @@ -12,20 +12,17 @@ ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES) statement error CALL postgres_execute('s', 'VACUUM') ---- -Invalid Error: Failed to execute query "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; -VACUUM": ERROR: VACUUM cannot run inside a transaction block +Invalid Error: Failed to execute query "VACUUM": ERROR: VACUUM cannot run inside a transaction block statement error CALL postgres_execute('s', 'VACUUM', use_transaction=true) ---- -Invalid Error: Failed to execute query "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; -VACUUM": ERROR: VACUUM cannot run inside a transaction block +Invalid Error: Failed to execute query "VACUUM": ERROR: VACUUM cannot run inside a transaction block statement error CALL postgres_execute('s', 'VACUUM', use_transaction=true) ---- -Invalid Error: Failed to execute query "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; -VACUUM": ERROR: VACUUM cannot run inside a transaction block +Invalid Error: Failed to execute query "VACUUM": ERROR: VACUUM cannot run inside a transaction block statement ok CALL postgres_execute('s', 'VACUUM', use_transaction=false) diff --git a/vcpkg.json b/vcpkg.json index fb16ba635..d64381d70 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,5 @@ { "dependencies": [ - "openssl", "libpq" ], "vcpkg-configuration": { diff --git a/vcpkg_ports/libpq/Makefile b/vcpkg_ports/libpq/Makefile deleted file mode 100644 index d743b5335..000000000 --- a/vcpkg_ports/libpq/Makefile +++ /dev/null @@ -1,57 +0,0 @@ -subdir = . -top_builddir = . -include src/Makefile.global - -ifeq ($(LIBPQ_LIBRARY_TYPE), static) -LIBPQ_INSTALL_LIBS = install-stlib -endif - -ifeq ($(LIBPQ_LIBRARY_TYPE), shared) -ifeq ($(LIBPQ_USING_MINGW), yes) -# The import library name is the same as the static library name -EXTRA_TARGET = install-lib-static -endif -endif - -.PHONY: all -all: - $(MAKE) -C src/include - $(MAKE) -C src/common - $(MAKE) -C src/port - $(MAKE) -C src/interfaces/libpq all-$(LIBPQ_LIBRARY_TYPE)-lib - $(MAKE) -C src/interfaces/ecpg/include - $(MAKE) -C src/interfaces/ecpg/pgtypeslib - $(MAKE) -C src/interfaces/ecpg/ecpglib - $(MAKE) -C src/interfaces/ecpg/compatlib -ifeq ($(LIBPQ_LIBRARY_TYPE), static) - $(AR) $(AROPT) src/interfaces/libpq/libpq.a src/common/encnames_shlib.o -endif -ifeq ($(LIBPQ_ENABLE_TOOLS), yes) - $(MAKE) -C src/interfaces/ecpg/preproc - $(MAKE) -C src/fe_utils - $(MAKE) -C src/bin -else - $(MAKE) -C src/bin/pg_config -endif - -.PHONY: install-stlib -install-stlib: - $(MAKE) -C src/common install -o all - mv -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' '$(DESTDIR)$(libdir)/libpgcommon.a' - $(MAKE) -C src/port install -o all - rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' - -.PHONY: install -install: $(LIBPQ_INSTALL_LIBS) - $(MAKE) -C src/include install - $(MAKE) -C src/interfaces/libpq $(EXTRA_TARGET) install-lib-$(LIBPQ_LIBRARY_TYPE) install-lib-pc install -o all -o install-lib - $(MAKE) -C src/interfaces/ecpg/include install - $(MAKE) -C src/interfaces/ecpg/pgtypeslib install - $(MAKE) -C src/interfaces/ecpg/ecpglib install - $(MAKE) -C src/interfaces/ecpg/compatlib install -ifeq ($(LIBPQ_ENABLE_TOOLS), yes) - $(MAKE) -C src/interfaces/ecpg/preproc install - $(MAKE) -C src/bin install -else - $(MAKE) -C src/bin/pg_config install -endif diff --git a/vcpkg_ports/libpq/build-msvc.cmake b/vcpkg_ports/libpq/build-msvc.cmake deleted file mode 100644 index 36dd79391..000000000 --- a/vcpkg_ports/libpq/build-msvc.cmake +++ /dev/null @@ -1,221 +0,0 @@ -function(build_msvc source_path) - # Strip meson build to only compile client libraries and tools. - # The full PostgreSQL meson build includes the server backend, timezone data, - # PL languages, contrib modules, and tests - none of which we need. - - # src/meson.build: keep only bin (client tools) and interfaces (ecpg) - file(WRITE "${source_path}/src/meson.build" [=[ -subdir('bin') -subdir('interfaces') -]=]) - - # src/bin/meson.build: only client tools (mirrors unix/no-server-tools.patch) - file(WRITE "${source_path}/src/bin/meson.build" [=[ -subdir('pg_amcheck') -subdir('pg_basebackup') -subdir('pg_config') -subdir('pg_dump') -subdir('pg_verifybackup') -subdir('pgbench') -subdir('pgevent') -subdir('psql') -subdir('scripts') -]=]) - - # Truncate meson.build before the test/pseudo-target sections that reference - # variables from skipped subdirs (pg_regress, regress_module, docs, etc.). - # The subdirs themselves are already commented out by windows/meson-vcpkg.patch, - # but the test infrastructure section still references variables from them. - file(READ "${source_path}/meson.build" meson_content) - string(FIND "${meson_content}" "# all targets that require building code" truncate_pos) - if(NOT truncate_pos EQUAL -1) - string(SUBSTRING "${meson_content}" 0 ${truncate_pos} meson_content) - file(WRITE "${source_path}/meson.build" "${meson_content}") - endif() - - # For static builds, remove __declspec(dllimport) from installed headers - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - vcpkg_replace_string("${source_path}/src/include/port/win32.h" - "__declspec (dllimport)" "") - endif() - - # The meson build compiles the static pgcommon with -DUSE_PRIVATE_ENCODING_FUNCS, - # which renames pg_char_to_encoding -> pg_char_to_encoding_private. But libpq - # (compiled as FRONTEND without that flag) calls the non-private names, causing - # unresolved symbols for consumers. The autoconf Makefile works around this by - # installing pgcommon_shlib (non-private names) as pgcommon. Since we don't build - # the server backend, there's no symbol conflict risk, so just remove the flag. - vcpkg_replace_string("${source_path}/src/common/meson.build" - "'c_args': ['-DUSE_PRIVATE_ENCODING_FUNCS']," - "# 'c_args': ['-DUSE_PRIVATE_ENCODING_FUNCS'], # vcpkg: use non-private names") - - # Map vcpkg features to meson options - vcpkg_list(SET MESON_OPTIONS) - - # Disable auto-detection so we don't pick up random system libraries - list(APPEND MESON_OPTIONS -Dauto_features=disabled) - - # Features that map directly to meson option names - foreach(option IN ITEMS icu lz4 zlib zstd) - if(option IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -D${option}=enabled) - endif() - endforeach() - - # Features with different meson option names - if("openssl" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dssl=openssl) - else() - list(APPEND MESON_OPTIONS -Dssl=none) - endif() - if("xml" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dlibxml=enabled) - endif() - if("xslt" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dlibxslt=enabled) - endif() - if("nls" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dnls=enabled) - # Static intl depends on iconv, but PostgreSQL's meson build uses - # cc.find_library('intl') which doesn't resolve transitive deps. - # Patch to also link iconv when intl is found. - vcpkg_replace_string("${source_path}/meson.build" - "i18n = import('i18n')" - "iconv_dep = cc.find_library('iconv', required: false, dirs: test_lib_d)\n if iconv_dep.found()\n libintl = declare_dependency(dependencies: [libintl, iconv_dep])\n endif\n i18n = import('i18n')") - endif() - # plpython requires matching debug/release Python libraries. - # vcpkg's Python is release-only, so only enable for release builds. - vcpkg_list(SET MESON_OPTIONS_RELEASE) - vcpkg_list(SET MESON_OPTIONS_DEBUG) - if("python" IN_LIST FEATURES) - if(VCPKG_CROSSCOMPILING) - # plpython can't be configured when cross-compiling because meson - # needs to run the Python interpreter at configure time, but the - # target binary won't execute on the build host. - message(STATUS "Disabling plpython for cross-compilation build") - list(APPEND MESON_OPTIONS -Dplpython=disabled) - else() - # Use the vcpkg python3 interpreter (which has matching dev libraries) - # instead of the standalone tools Python (which has no SDK). - string(REPLACE [[\]] [[/]] VCPKG_PYTHON3_PATH "${CURRENT_INSTALLED_DIR}/tools/python3/python.exe") - list(APPEND MESON_OPTIONS_RELEASE -Dplpython=enabled "-DPYTHON=${VCPKG_PYTHON3_PATH}") - list(APPEND MESON_OPTIONS_DEBUG -Dplpython=disabled) - endif() - endif() - if("tcl" IN_LIST FEATURES) - list(APPEND MESON_OPTIONS -Dpltcl=enabled) - # The vcpkg tcl port doesn't generate pkg-config files on Windows, - # so meson's dependency('tcl90') fails. Its fallback cc.find_library() - # also fails because meson's link-test probe doesn't work reliably - # with vcpkg's directory layout. Generate .pc files so meson can - # find Tcl through its preferred pkg-config path. - # Tcl naming: tcl90 (dynamic), tcl90s (static), +g suffix for debug. - set(_tcl_pc_name "tcl90") - # Determine the release library name - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - set(_tcl_rel_libname "tcl90s") - else() - set(_tcl_rel_libname "tcl90") - endif() - # Generate release .pc file - set(_tcl_pc_dir "${CURRENT_INSTALLED_DIR}/lib/pkgconfig") - file(MAKE_DIRECTORY "${_tcl_pc_dir}") - file(WRITE "${_tcl_pc_dir}/${_tcl_pc_name}.pc" -"prefix=${CURRENT_INSTALLED_DIR}\nlibdir=\${prefix}/lib\nincludedir=\${prefix}/include\n\nName: ${_tcl_pc_name}\nDescription: Tcl scripting language\nVersion: 9.0\nLibs: -L\${libdir} -l${_tcl_rel_libname}\nCflags: -I\${includedir}\n") - # Determine the debug library name - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - set(_tcl_dbg_libname "tcl90sg") - else() - set(_tcl_dbg_libname "tcl90g") - endif() - # Generate debug .pc file - if(NOT VCPKG_BUILD_TYPE) - set(_tcl_pc_dbg_dir "${CURRENT_INSTALLED_DIR}/debug/lib/pkgconfig") - file(MAKE_DIRECTORY "${_tcl_pc_dbg_dir}") - file(WRITE "${_tcl_pc_dbg_dir}/${_tcl_pc_name}.pc" -"prefix=${CURRENT_INSTALLED_DIR}/debug\nlibdir=\${prefix}/lib\nincludedir=${CURRENT_INSTALLED_DIR}/include\n\nName: ${_tcl_pc_name}\nDescription: Tcl scripting language\nVersion: 9.0\nLibs: -L\${libdir} -l${_tcl_dbg_libname}\nCflags: -I\${includedir}\n") - endif() - list(APPEND MESON_OPTIONS "-Dtcl_version=${_tcl_pc_name}") - endif() - - # Provide paths to required tools - vcpkg_list(SET ADDITIONAL_BINARIES) - string(REPLACE [[\]] [[/]] BISON_PATH "${BISON}") - string(REPLACE [[\]] [[/]] FLEX_PATH "${FLEX}") - string(REPLACE [[\]] [[/]] PERL_PATH "${PERL}") - list(APPEND ADDITIONAL_BINARIES - "bison = ['${BISON_PATH}']" - "flex = ['${FLEX_PATH}']" - "perl = ['${PERL_PATH}']" - ) - - # Extra include/lib dirs for vcpkg dependencies (only if they exist; - # with no optional deps installed, CURRENT_INSTALLED_DIR may lack these). - # Debug and release use different lib dirs (debug libs have different - # suffixes, e.g. tcl90g.lib vs tcl90.lib). - if(EXISTS "${CURRENT_INSTALLED_DIR}/include") - list(APPEND MESON_OPTIONS "-Dextra_include_dirs=['${CURRENT_INSTALLED_DIR}/include']") - endif() - if(EXISTS "${CURRENT_INSTALLED_DIR}/lib") - list(APPEND MESON_OPTIONS_RELEASE "-Dextra_lib_dirs=['${CURRENT_INSTALLED_DIR}/lib']") - endif() - if(EXISTS "${CURRENT_INSTALLED_DIR}/debug/lib") - list(APPEND MESON_OPTIONS_DEBUG "-Dextra_lib_dirs=['${CURRENT_INSTALLED_DIR}/debug/lib']") - endif() - - vcpkg_configure_meson( - SOURCE_PATH "${source_path}" - OPTIONS - ${MESON_OPTIONS} - OPTIONS_RELEASE - ${MESON_OPTIONS_RELEASE} - OPTIONS_DEBUG - ${MESON_OPTIONS_DEBUG} - LANGUAGES C - ADDITIONAL_BINARIES - ${ADDITIONAL_BINARIES} - ) - vcpkg_install_meson() - - # Remove _shlib variants of pgcommon/pgport that consumers don't need - file(GLOB _shlib_libs - "${CURRENT_PACKAGES_DIR}/lib/*pgcommon_shlib*" - "${CURRENT_PACKAGES_DIR}/lib/*pgport_shlib*" - "${CURRENT_PACKAGES_DIR}/debug/lib/*pgcommon_shlib*" - "${CURRENT_PACKAGES_DIR}/debug/lib/*pgport_shlib*" - ) - if(_shlib_libs) - file(REMOVE ${_shlib_libs}) - endif() - - # Remove server-related installed files we don't need - file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/lib/postgresql" - "${CURRENT_PACKAGES_DIR}/debug/lib/postgresql" - "${CURRENT_PACKAGES_DIR}/share/postgresql" - ) - - if(HAS_TOOLS) - set(TOOL_NAMES - clusterdb createdb createuser dropdb dropuser ecpg pgbench - pg_amcheck pg_basebackup pg_config pg_createsubscriber - pg_dump pg_dumpall pg_isready pg_receivewal pg_recvlogical - pg_restore pg_verifybackup psql reindexdb vacuumdb - ) - vcpkg_copy_tools(TOOL_NAMES ${TOOL_NAMES} AUTO_CLEAN) - else() - # Remove all executables (keep DLLs in bin/ for dynamic linkage) - file(GLOB exe_files - "${CURRENT_PACKAGES_DIR}/bin/*.exe" - "${CURRENT_PACKAGES_DIR}/debug/bin/*.exe" - ) - if(exe_files) - file(REMOVE ${exe_files}) - endif() - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/tools") - endif() - - if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") - endif() -endfunction() diff --git a/vcpkg_ports/libpq/libintl.diff b/vcpkg_ports/libpq/libintl.diff new file mode 100644 index 000000000..a5d116731 --- /dev/null +++ b/vcpkg_ports/libpq/libintl.diff @@ -0,0 +1,17 @@ +diff --git a/meson.build b/meson.build +index dd7ca47..8e0964a 100644 +--- a/meson.build ++++ b/meson.build +@@ -3023,6 +3023,12 @@ if not nlsopt.disabled() + endif + + if libintl.found() ++ if build_system == 'windows' ++ iconv_dep = cc.find_library('iconv', required: false, dirs: test_lib_d) ++ if iconv_dep.found() ++ libintl = declare_dependency(dependencies: [libintl, iconv_dep]) ++ endif ++ endif + i18n = import('i18n') + cdata.set('ENABLE_NLS', 1) + endif diff --git a/vcpkg_ports/libpq/libpq-and-client-tools.diff b/vcpkg_ports/libpq/libpq-and-client-tools.diff new file mode 100644 index 000000000..2f0cddb90 --- /dev/null +++ b/vcpkg_ports/libpq/libpq-and-client-tools.diff @@ -0,0 +1,127 @@ +diff --git a/config/meson.build b/config/meson.build +index ab19c38..b6203b5 100644 +--- a/config/meson.build ++++ b/config/meson.build +@@ -1,3 +1,4 @@ ++subdir_done() + install_data( + 'install-sh', 'missing', + install_dir: dir_pgxs / 'config' +diff --git a/meson.build b/meson.build +index 4016033..dd7ca47 100644 +--- a/meson.build ++++ b/meson.build +@@ -3381,6 +3381,10 @@ test_install_libs = [] + + subdir('src') + ++contrib_data_dir = dir_data_extension ++ ++if false # skip for vcpkg port ++ + subdir('contrib') + + subdir('src/test') +@@ -3924,6 +3928,7 @@ if not meson.is_subproject() + endif + endif + ++endif # skip for vcpkg port libpq + + + ############################################################### +diff --git a/meson_options.txt b/meson_options.txt +index 06bf562..5505bfd 100644 +--- a/meson_options.txt ++++ b/meson_options.txt +@@ -1,3 +1,6 @@ ++option('tools', type: 'feature', value: 'disabled', ++ description: 'Enable tool executables') ++ + # Copyright (c) 2022-2025, PostgreSQL Global Development Group + + # Data layout influencing options +diff --git a/src/backend/meson.build b/src/backend/meson.build +index 2b0db21..75d8f68 100644 +--- a/src/backend/meson.build ++++ b/src/backend/meson.build +@@ -40,6 +40,8 @@ subdir('po', if_found: libintl) + backend_link_args = [] + backend_link_depends = [] + ++subdir_done() ++if false # skip for vcpkg port + + # On windows when compiling with msvc we need to make postgres export all its + # symbols so that extension libraries can use them. For that we need to scan +@@ -140,6 +142,8 @@ postgres = executable('postgres', + + backend_targets += postgres + ++endif # skip for vcpkg port ++ + pg_mod_c_args = cflags_mod + pg_mod_cpp_args = cxxflags_mod + pg_mod_link_args = ldflags_sl + ldflags_mod +@@ -187,6 +191,7 @@ pg_test_mod_args = pg_mod_args + { + } + + ++subdir_done() + + # Shared modules that, on some system, link against the server binary. Only + # enter these after we defined the server build. +diff --git a/src/bin/meson.build b/src/bin/meson.build +index b33cb6c..b08941f 100644 +--- a/src/bin/meson.build ++++ b/src/bin/meson.build +@@ -1,24 +1,13 @@ + # Copyright (c) 2022-2025, PostgreSQL Global Development Group + +-subdir('initdb') ++# vcpkg: Reduced to documented "Client applications" (and scripts) ++# Cf. https://www.postgresql.org/docs/current/reference-client.html + subdir('pg_amcheck') +-subdir('pg_archivecleanup') + subdir('pg_basebackup') +-subdir('pg_checksums') + subdir('pg_combinebackup') + subdir('pg_config') +-subdir('pg_controldata') +-subdir('pg_ctl') + subdir('pg_dump') +-subdir('pg_resetwal') +-subdir('pg_rewind') +-subdir('pg_test_fsync') +-subdir('pg_test_timing') +-subdir('pg_upgrade') + subdir('pg_verifybackup') +-subdir('pg_waldump') +-subdir('pg_walsummary') + subdir('pgbench') +-subdir('pgevent') + subdir('psql') + subdir('scripts') +diff --git a/src/meson.build b/src/meson.build +index f098f9f..28eb09c 100644 +--- a/src/meson.build ++++ b/src/meson.build +@@ -3,12 +3,18 @@ + # libraries that other subsystems might depend upon first, in their respective + # dependency order + ++subdir('interfaces') ++if not get_option('tools').disabled() ++ + subdir('timezone') + + subdir('backend') + + subdir('bin') + ++endif ++subdir_done() ++ + subdir('pl') + + subdir('interfaces') diff --git a/vcpkg_ports/libpq/libpq.props.in b/vcpkg_ports/libpq/libpq.props.in deleted file mode 100644 index 223e528fe..000000000 --- a/vcpkg_ports/libpq/libpq.props.in +++ /dev/null @@ -1,25 +0,0 @@ - - - - - %(AdditionalOptions) @VCPKG_COMBINED_C_FLAGS_DEBUG@ - - - %(AdditionalOptions) @VCPKG_COMBINED_SHARED_LINKER_FLAGS_DEBUG@ - - - %(AdditionalOptions) @VCPKG_COMBINED_STATIC_LINKER_FLAGS_DEBUG@ - - - - - %(AdditionalOptions) @VCPKG_COMBINED_C_FLAGS_RELEASE@ - - - %(AdditionalOptions) @VCPKG_COMBINED_SHARED_LINKER_FLAGS_RELEASE@ - - - %(AdditionalOptions) @VCPKG_COMBINED_STATIC_LINKER_FLAGS_RELEASE@ - - - diff --git a/vcpkg_ports/libpq/library-linkage.diff b/vcpkg_ports/libpq/library-linkage.diff new file mode 100644 index 000000000..6c3c260d3 --- /dev/null +++ b/vcpkg_ports/libpq/library-linkage.diff @@ -0,0 +1,241 @@ +diff --git a/meson.build b/meson.build +index 6e1c867..4016033 100644 +--- a/meson.build ++++ b/meson.build +@@ -3293,6 +3293,12 @@ frontend_shlib_code = declare_dependency( + dependencies: [shlib_code, os_deps, libintl], + ) + ++if get_option('default_library') == 'shared' ++ frontend_lib_code = frontend_shlib_code ++else ++ frontend_lib_code = frontend_stlib_code ++endif ++ + # For frontend code that doesn't use fe_utils - this mainly exists for libpq's + # tests, which are defined before fe_utils is defined, as fe_utils depends on + # libpq. +@@ -3326,7 +3332,7 @@ subdir('src/interfaces/libpq-oauth') + # for frontend binaries + frontend_code = declare_dependency( + include_directories: [postgres_inc], +- link_with: [fe_utils, common_static, pgport_static], ++ link_with: [fe_utils, common_default, pgport_default], + sources: generated_headers, + dependencies: [os_deps, libintl], + ) +diff --git a/src/bin/initdb/meson.build b/src/bin/initdb/meson.build +index 06958e3..a25abd1 100644 +--- a/src/bin/initdb/meson.build ++++ b/src/bin/initdb/meson.build +@@ -20,7 +20,7 @@ initdb = executable('initdb', + # from libpq, else we have risks of version skew if we run with a libpq + # shared library from a different PG version. Define + # USE_PRIVATE_ENCODING_FUNCS to ensure that that happens. +- c_args: ['-DUSE_PRIVATE_ENCODING_FUNCS'], ++ # But vcpkg cannot build the libs with this definition. + dependencies: [frontend_code, libpq, icu, icu_i18n], + kwargs: default_bin_args, + ) +diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build +index bbc4c57..9ea8597 100644 +--- a/src/bin/pg_combinebackup/meson.build ++++ b/src/bin/pg_combinebackup/meson.build +@@ -17,7 +17,7 @@ endif + + pg_combinebackup = executable('pg_combinebackup', + pg_combinebackup_sources, +- dependencies: [frontend_code], ++ dependencies: [frontend_code, libpq], + kwargs: default_bin_args, + ) + bin_targets += pg_combinebackup +diff --git a/src/common/meson.build b/src/common/meson.build +index 1540ba6..e29d594 100644 +--- a/src/common/meson.build ++++ b/src/common/meson.build +@@ -114,6 +114,8 @@ common_sources_frontend_shlib += files( + ) + + common_sources_excluded_shlib = files( ++) ++common_sources_frontend_shlib += files( + 'fe_memutils.c', + 'logging.c', + ) +@@ -144,7 +146,7 @@ pgcommon_variants = { + 'dependencies': [frontend_common_code], + # Files in libpgcommon.a should use/export the "xxx_private" versions + # of pg_char_to_encoding() and friends. +- 'c_args': ['-DUSE_PRIVATE_ENCODING_FUNCS'], ++ # But vcpkg's static library linkage must not add the private suffix + }, + '_shlib': default_lib_args + { + 'pic': true, +@@ -159,6 +161,19 @@ pgcommon_variants = { + + foreach name, opts : pgcommon_variants + ++ if name == '_srv' ++ pgcommon += {name: disabler()} ++ continue ++ endif ++ if name == '' and get_option('default_library') == 'shared' ++ pgcommon += {name: disabler()} ++ continue ++ endif ++ if name == '_shlib' and get_option('default_library') == 'static' ++ pgcommon += {name: disabler()} ++ continue ++ endif ++ + # Build internal static libraries for sets of files that need to be built + # with different cflags + cflag_libs = [] +@@ -195,6 +210,7 @@ foreach name, opts : pgcommon_variants + } + ) + pgcommon += {name: lib} ++ common_default = lib + endforeach + + common_srv = pgcommon['_srv'] +diff --git a/src/interfaces/ecpg/compatlib/meson.build b/src/interfaces/ecpg/compatlib/meson.build +index 56e0a21..e3c8675 100644 +--- a/src/interfaces/ecpg/compatlib/meson.build ++++ b/src/interfaces/ecpg/compatlib/meson.build +@@ -41,11 +41,17 @@ ecpg_compat_so = shared_library('libecpg_compat', + ) + ecpg_targets += ecpg_compat_so + ++if get_option('default_library') == 'shared' ++ ecpg_compat_libraries = ecpg_compat_so ++else ++ ecpg_compat_libraries = ecpg_compat_st ++endif ++ + pkgconfig.generate( + name: 'libecpg_compat', + description: 'PostgreSQL libecpg_compat library', + url: pg_url, +- libraries: ecpg_compat_so, +- libraries_private: [frontend_stlib_code, thread_dep], ++ libraries: ecpg_compat_libraries, ++ libraries_private: [frontend_lib_code, thread_dep], + requires_private: ['libecpg', 'libpgtypes'], + ) +diff --git a/src/interfaces/ecpg/ecpglib/meson.build b/src/interfaces/ecpg/ecpglib/meson.build +index 8f478c6..fbae057 100644 +--- a/src/interfaces/ecpg/ecpglib/meson.build ++++ b/src/interfaces/ecpg/ecpglib/meson.build +@@ -52,12 +52,18 @@ ecpglib_so = shared_library('libecpg', + ) + ecpg_targets += ecpglib_so + ++if get_option('default_library') == 'shared' ++ ecpglib_libraries = ecpglib_so ++else ++ ecpglib_libraries = ecpglib_st ++endif ++ + pkgconfig.generate( + name: 'libecpg', + description: 'PostgreSQL libecpg library', + url: pg_url, +- libraries: ecpglib_so, +- libraries_private: [frontend_stlib_code, thread_dep], ++ libraries: ecpglib_libraries, ++ libraries_private: [frontend_lib_code, thread_dep], + requires_private: ['libpgtypes', 'libpq'], + ) + +diff --git a/src/interfaces/ecpg/pgtypeslib/meson.build b/src/interfaces/ecpg/pgtypeslib/meson.build +index 02301ec..e695b98 100644 +--- a/src/interfaces/ecpg/pgtypeslib/meson.build ++++ b/src/interfaces/ecpg/pgtypeslib/meson.build +@@ -46,10 +46,16 @@ ecpg_pgtypes_so = shared_library('libpgtypes', + ) + ecpg_targets += ecpg_pgtypes_so + ++if get_option('default_library') == 'shared' ++ ecpg_pgtypes_libraries = ecpg_pgtypes_so ++else ++ ecpg_pgtypes_libraries = ecpg_pgtypes_st ++endif ++ + pkgconfig.generate( + name: 'libpgtypes', + description: 'PostgreSQL libpgtypes library', + url: pg_url, +- libraries: ecpg_pgtypes_so, +- libraries_private: [frontend_stlib_code], ++ libraries: ecpg_pgtypes_libraries, ++ libraries_private: [frontend_lib_code], + ) +diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build +index a74e885..7354286 100644 +--- a/src/interfaces/libpq/meson.build ++++ b/src/interfaces/libpq/meson.build +@@ -60,7 +60,7 @@ libpq_so_c_args = ['-DUSE_DYNAMIC_OAUTH'] + libpq_st = static_library('libpq', + libpq_sources, + include_directories: [libpq_inc], +- c_args: libpq_c_args, ++ c_args: libpq_c_args + ['-DFRONTEND'], + c_pch: pch_postgres_fe_h, + dependencies: [frontend_stlib_code, libpq_deps], + kwargs: default_lib_args, +@@ -80,13 +80,19 @@ libpq_so = shared_library('libpq', + kwargs: default_lib_args, + ) + ++if get_option('default_library') == 'shared' ++ libpq_link_with = [libpq_so] ++else ++ libpq_link_with = [libpq_st] ++endif ++ + libpq = declare_dependency( +- link_with: [libpq_so], ++ link_with: libpq_link_with, + include_directories: [include_directories('.')] + ) + + private_deps = [ +- frontend_stlib_code, ++ frontend_lib_code, + libpq_deps, + ] + +diff --git a/src/port/meson.build b/src/port/meson.build +index fc7b059..3b8cf40 100644 +--- a/src/port/meson.build ++++ b/src/port/meson.build +@@ -167,6 +167,19 @@ pgport_variants = { + + foreach name, opts : pgport_variants + ++ if name == '_srv' ++ pgport += {name: disabler()} ++ continue ++ endif ++ if name == '' and get_option('default_library') == 'shared' ++ pgport += {name: disabler()} ++ continue ++ endif ++ if name == '_shlib' and get_option('default_library') == 'static' ++ pgport += {name: disabler()} ++ continue ++ endif ++ + # Build internal static libraries for sets of files that need to be built + # with different cflags + cflag_libs = [] +@@ -196,6 +209,7 @@ foreach name, opts : pgport_variants + } + ) + pgport += {name: lib} ++ pgport_default = lib + endforeach + + pgport_srv = pgport['_srv'] diff --git a/vcpkg_ports/libpq/portfile.cmake b/vcpkg_ports/libpq/portfile.cmake index b40d25866..97758ed1f 100644 --- a/vcpkg_ports/libpq/portfile.cmake +++ b/vcpkg_ports/libpq/portfile.cmake @@ -2,148 +2,111 @@ vcpkg_download_distfile(ARCHIVE URLS "https://ftp.postgresql.org/pub/source/v${VERSION}/postgresql-${VERSION}.tar.bz2" "https://www.mirrorservice.org/sites/ftp.postgresql.org/source/v${VERSION}/postgresql-${VERSION}.tar.bz2" FILENAME "postgresql-${VERSION}.tar.bz2" - SHA512 fdbe6d726f46738cf14acab96e5c05f7d65aefe78563281b416bb14a27c7c42e4df921e26b32816a5030ddbe506b95767e2c74a35afc589916504df38d1cb11c + SHA512 1a7606b5b2460a4fc817d491e1e03262df17ece6869fa3d2b80647baffcfb790fbcfa019c3b10fab9f9a14218651b6a358864e40a5d8c5ddc796db2d6e5b469c ) vcpkg_extract_source_archive( SOURCE_PATH ARCHIVE "${ARCHIVE}" PATCHES - unix/installdirs.patch - unix/fix-configure.patch - unix/single-linkage.patch - unix/no-server-tools.patch - unix/mingw-install.patch - unix/mingw-static-importlib-fix.patch - unix/python.patch + library-linkage.diff + libpq-and-client-tools.diff + libintl.diff + zic.diff windows/macro-def.patch windows/spin_delay.patch - windows/tcl-9.0-alpha.patch - windows/meson-vcpkg.patch windows/getopt.patch + windows/clang-cl-c99-test.patch + windows/clang-cl-msvc-abi.patch ) file(GLOB _py3_include_path "${CURRENT_HOST_INSTALLED_DIR}/include/python3*") string(REGEX MATCH "python3\\.([0-9]+)" _python_version_tmp "${_py3_include_path}") set(PYTHON_VERSION_MINOR "${CMAKE_MATCH_1}") -if("client" IN_LIST FEATURES) - set(HAS_TOOLS TRUE) -else() - set(HAS_TOOLS FALSE) -endif() - vcpkg_cmake_get_vars(cmake_vars_file) include("${cmake_vars_file}") -set(required_programs BISON FLEX) -if(VCPKG_DETECTED_MSVC OR NOT VCPKG_HOST_IS_WINDOWS) - list(APPEND required_programs PERL) -endif() -foreach(program_name IN LISTS required_programs) - vcpkg_find_acquire_program(${program_name}) - get_filename_component(program_dir ${${program_name}} DIRECTORY) - vcpkg_add_to_path(PREPEND "${program_dir}") +vcpkg_find_acquire_program(BISON) +vcpkg_find_acquire_program(FLEX) +vcpkg_find_acquire_program(PERL) + +vcpkg_list(SET MESON_OPTIONS_RELEASE) +vcpkg_list(SET MESON_OPTIONS) +foreach(option IN ITEMS icu lz4 zlib zstd) + if(option IN_LIST FEATURES) + list(APPEND MESON_OPTIONS -D${option}=enabled) + endif() endforeach() -if(VCPKG_DETECTED_MSVC) - include("${CMAKE_CURRENT_LIST_DIR}/build-msvc.cmake") - build_msvc("${SOURCE_PATH}") +if("openssl" IN_LIST FEATURES) + list(APPEND MESON_OPTIONS -Dssl=openssl) else() - file(COPY "${CMAKE_CURRENT_LIST_DIR}/Makefile" DESTINATION "${SOURCE_PATH}") + list(APPEND MESON_OPTIONS -Dssl=none) +endif() - vcpkg_list(SET BUILD_OPTS) - foreach(option IN ITEMS bonjour icu lz4 readline zlib zstd) - if(option IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-${option}) - else() - list(APPEND BUILD_OPTS --without-${option}) - endif() - endforeach() - if("openssl" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-openssl) - else() - list(APPEND BUILD_OPTS --without-openssl) - endif() - if("xml" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-libxml) - else() - list(APPEND BUILD_OPTS --without-libxml) - endif() - if("xslt" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-libxslt) - else() - list(APPEND BUILD_OPTS --without-libxslt) - endif() - if("nls" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --enable-nls) - else() - list(APPEND BUILD_OPTS --disable-nls) - endif() - if("nls" IN_LIST FEATURES) - set(ENV{MSGFMT} "${CURRENT_HOST_INSTALLED_DIR}/tools/gettext/bin/msgfmt${VCPKG_HOST_EXECUTABLE_SUFFIX}") - endif() - if("python" IN_LIST FEATURES) - list(APPEND BUILD_OPTS --with-python=3.${PYTHON_VERSION_MINOR}) - vcpkg_find_acquire_program(PYTHON3) - list(APPEND BUILD_OPTS "PYTHON=${PYTHON3}") - endif() - if(VCPKG_TARGET_IS_ANDROID AND (NOT VCPKG_CMAKE_SYSTEM_VERSION OR VCPKG_CMAKE_SYSTEM_VERSION LESS "26")) - list(APPEND BUILD_OPTS ac_cv_header_langinfo_h=no) - endif() - if(VCPKG_DETECTED_CMAKE_OSX_SYSROOT) - list(APPEND BUILD_OPTS "PG_SYSROOT=${VCPKG_DETECTED_CMAKE_OSX_SYSROOT}") - endif() - if(VCPKG_TARGET_IS_OSX) - set(ENV{LDFLAGS} "$ENV{LDFLAGS} -headerpad_max_install_names") - endif() - vcpkg_configure_make( - SOURCE_PATH "${SOURCE_PATH}" - COPY_SOURCE - AUTOCONFIG - ADDITIONAL_MSYS_PACKAGES autoconf-archive - DIRECT_PACKAGES - "https://mirror.msys2.org/msys/x86_64/tzcode-2025b-1-x86_64.pkg.tar.zst" - 824779e3ac4857bb21cbdc92fa881fa24bf89dfa8bc2f9ca816e9a9837a6d963805e8e0991499c43337a134552215fdee50010e643ddc8bd699170433a4c83de - OPTIONS - ${BUILD_OPTS} - OPTIONS_DEBUG - --enable-debug - ) +if("nls" IN_LIST FEATURES) + list(APPEND MESON_OPTIONS -Dnls=enabled) +endif() - if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") - set(ENV{LIBPQ_LIBRARY_TYPE} shared) - else() - set(ENV{LIBPQ_LIBRARY_TYPE} static) - endif() - if(VCPKG_TARGET_IS_MINGW) - set(ENV{LIBPQ_USING_MINGW} yes) - endif() - if(HAS_TOOLS) - set(ENV{LIBPQ_ENABLE_TOOLS} yes) +if("client" IN_LIST FEATURES) + list(APPEND MESON_OPTIONS_RELEASE -Dtools=enabled) + if(VCPKG_CROSSCOMPILING) + list(APPEND MESON_OPTIONS "-DZIC=${CURRENT_HOST_INSTALLED_DIR}/manual-tools/${PORT}/zic${VCPKG_HOST_EXECUTABLE_SUFFIX}") endif() +endif() - # https://stackoverflow.com/a/76688525 - set(ENV{MAKELEVEL} 0) +vcpkg_configure_meson( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + -Dauto_features=disabled + ${MESON_OPTIONS} + # cannot use ADDITIONAL_BINARIES for "native" programs + "-DBISON=['${BISON}']" + "-DFLEX=['${FLEX}']" + "-DPERL=${PERL}" + OPTIONS_RELEASE + ${MESON_OPTIONS_RELEASE} +) +vcpkg_install_meson() +vcpkg_fixup_pkgconfig() - vcpkg_install_make() +if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW AND VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + file(GLOB pc_files "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/*.pc" "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/*.pc") + foreach(file IN LISTS pc_files) + vcpkg_replace_string("${file}" " -l(pq|pg|ecpg)" " -llib\\1" REGEX) + endforeach() +endif() - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/postgresql/server/pg_config.h" "#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/pg_config.h" "#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS") +vcpkg_copy_tools(TOOL_NAMES ecpg AUTO_CLEAN) +if("client" IN_LIST FEATURES) + if(NOT VCPKG_CROSSCOMPILING) + vcpkg_copy_tools(TOOL_NAMES zic AUTO_CLEAN DESTINATION "${CURRENT_PACKAGES_DIR}/manual-tools/${PORT}") + endif() + vcpkg_copy_tools( + TOOL_NAMES + clusterdb createdb createuser + dropdb dropuser + pg_amcheck + pg_basebackup pgbench + pg_combinebackup pg_config pg_createsubscriber + pg_dump pg_dumpall + pg_isready + pg_receivewal pg_recvlogical pg_restore + pg_verifybackup + psql + reindexdb + vacuumdb + AUTO_CLEAN + ) endif() -vcpkg_fixup_pkgconfig() -configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" "${CURRENT_PACKAGES_DIR}/share/postgresql/vcpkg-cmake-wrapper.cmake" @ONLY) file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/doc" "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/debug/symbols" - "${CURRENT_PACKAGES_DIR}/debug/tools" - "${CURRENT_PACKAGES_DIR}/symbols" - "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug" ) +configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" "${CURRENT_PACKAGES_DIR}/share/postgresql/vcpkg-cmake-wrapper.cmake" @ONLY) file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYRIGHT") diff --git a/vcpkg_ports/libpq/unix/fix-configure.patch b/vcpkg_ports/libpq/unix/fix-configure.patch deleted file mode 100644 index c7cc663a2..000000000 --- a/vcpkg_ports/libpq/unix/fix-configure.patch +++ /dev/null @@ -1,60 +0,0 @@ -diff --git a/configure.ac b/configure.ac ---- a/configure.ac -+++ b/configure.ac -@@ -20,6 +20,7 @@ AC_INIT([PostgreSQL], [18.3], [pgsql-bugs@lists.postgresql.org], [], [https://ww --m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. --Untested combinations of 'autoconf' and PostgreSQL versions are not --recommended. You can remove the check from 'configure.ac' but it is then --your responsibility whether the result works or not.])]) -+cross_compiling=yes # Avoid conftest loading shared objects -+m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_warn([unsupported],[Autoconf version 2.69 is required. -+Untested combinations of 'autoconf' and PostgreSQL versions are not -+recommended. You can remove the check from 'configure.ac' but it is then -+your responsibility whether the result works or not.])]) - AC_COPYRIGHT([Copyright (c) 1996-2025, PostgreSQL Global Development Group]) - AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) -@@ -1347,3 +1348,4 @@ AC_SEARCH_LIBS(pthread_barrier_wait, pthread) - if test "$with_readline" = yes; then -- PGAC_CHECK_READLINE -+ PKG_CHECK_MODULES([READLINE], [readline], [HAVE_LIBREADLINE=1], [pgac_cv_check_readline=no]) -+ LIBS="$READLINE_LIBS $LIBS" - if test x"$pgac_cv_check_readline" = x"no"; then -@@ -1357,3 +1359,3 @@ fi - if test "$with_zlib" = yes; then -- AC_CHECK_LIB(z, inflate, [], -+ PKG_CHECK_MODULES([ZLIB], [zlib], [LIBS="$ZLIB_LIBS $LIBS"], - [AC_MSG_ERROR([zlib library not found -@@ -1394,5 +1396,9 @@ fi - AC_DEFINE(OPENSSL_API_COMPAT, [0x10101000L], - [Define to the OpenSSL API version in use. This avoids deprecation warnings from newer OpenSSL versions.]) -+ PKG_CHECK_MODULES([CRYPTO_new_ex_data], [libcrypto], [LIBS="$CRYPTO_new_ex_data_LIBS $LIBS"], [AC_MSG_ERROR([library 'crypto' is required for OpenSSL])]) -+ PKG_CHECK_MODULES([SSL_new], [libssl], [LIBS="$SSL_new_LIBS $LIBS"], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])]) -+ if false ; then - AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, [], [AC_MSG_ERROR([library 'crypto' is required for OpenSSL])]) - AC_CHECK_LIB(ssl, SSL_new, [], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])]) -+ fi - # Functions introduced in OpenSSL 1.1.1. -@@ -1415,3 +1421,4 @@ fi - if test "$with_libxml" = yes ; then -- AC_CHECK_LIB(xml2, xmlSaveToBuffer, [], [AC_MSG_ERROR([library 'xml2' (version >= 2.6.23) is required for XML support])]) -+ PKG_CHECK_MODULES([LIBXML2], [libxml-2.0 >= 2.6.23], [AC_DEFINE(HAVE_LIBXML2,1,[Define to 1 if with xml2])], [AC_MSG_ERROR([library 'xml2' (version >= 2.6.23) is required for XML support])]) -+ LIBS="$LIBXML2_LIBS $LIBS" - fi -@@ -1419,3 +1426,4 @@ fi - if test "$with_libxslt" = yes ; then -- AC_CHECK_LIB(xslt, xsltCleanupGlobals, [], [AC_MSG_ERROR([library 'xslt' is required for XSLT support])]) -+ PKG_CHECK_MODULES([LIBXSLT], [libxslt], [AC_DEFINE(HAVE_LIBXSLT,1,[Define to 1 if with xslt])], [AC_MSG_ERROR([library 'xslt' is required for XSLT support])]) -+ LIBS="$LIBXSLT_LIBS $LIBS" - fi -@@ -1430,3 +1438,4 @@ fi - if test "$with_lz4" = yes ; then -- AC_CHECK_LIB(lz4, LZ4_compress_default, [], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])]) -+ PKG_CHECK_MODULES([LZ4], [liblz4], [AC_DEFINE(HAVE_LIBLZ4,1,[Define to 1 if with lz4])], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])]) -+ LIBS="$LZ4_LIBS $LIBS" - fi -@@ -1434,3 +1443,4 @@ fi - if test "$with_zstd" = yes ; then -- AC_CHECK_LIB(zstd, ZSTD_compress, [], [AC_MSG_ERROR([library 'zstd' is required for ZSTD support])]) -+ PKG_CHECK_MODULES([ZSTD], [libzstd], [AC_DEFINE(HAVE_LIBZSTD,1,[Define to 1 if with zstd])], [AC_MSG_ERROR([library 'zstd' is required for ZSTD support])]) -+ LIBS="$ZSTD_LIBS $LIBS" - fi diff --git a/vcpkg_ports/libpq/unix/installdirs.patch b/vcpkg_ports/libpq/unix/installdirs.patch deleted file mode 100644 index bbba0fe07..000000000 --- a/vcpkg_ports/libpq/unix/installdirs.patch +++ /dev/null @@ -1,52 +0,0 @@ -diff --git a/src/Makefile.global.in b/src/Makefile.global.in ---- a/src/Makefile.global.in -+++ b/src/Makefile.global.in -@@ -113,42 +113,42 @@ bindir := @bindir@ - --datadir := @datadir@ -+datadir := @datadir@/postgresql - ifeq "$(findstring pgsql, $(datadir))" "" - ifeq "$(findstring postgres, $(datadir))" "" - override datadir := $(datadir)/postgresql - endif - endif - --sysconfdir := @sysconfdir@ -+sysconfdir := @sysconfdir@/postgresql - ifeq "$(findstring pgsql, $(sysconfdir))" "" - ifeq "$(findstring postgres, $(sysconfdir))" "" - override sysconfdir := $(sysconfdir)/postgresql - endif - endif - - libdir := @libdir@ - --pkglibdir = $(libdir) -+pkglibdir = $(libdir)/postgresql - ifeq "$(findstring pgsql, $(pkglibdir))" "" - ifeq "$(findstring postgres, $(pkglibdir))" "" - override pkglibdir := $(pkglibdir)/postgresql - endif - endif - - includedir := @includedir@ - --pkgincludedir = $(includedir) -+pkgincludedir = $(includedir)/postgresql - ifeq "$(findstring pgsql, $(pkgincludedir))" "" - ifeq "$(findstring postgres, $(pkgincludedir))" "" - override pkgincludedir := $(pkgincludedir)/postgresql - endif - endif - - mandir := @mandir@ - --docdir := @docdir@ -+docdir := @docdir@/postgresql - ifeq "$(findstring pgsql, $(docdir))" "" - ifeq "$(findstring postgres, $(docdir))" "" - override docdir := $(docdir)/postgresql - endif - endif - - htmldir := @htmldir@ diff --git a/vcpkg_ports/libpq/unix/mingw-install.patch b/vcpkg_ports/libpq/unix/mingw-install.patch deleted file mode 100644 index ffeea023c..000000000 --- a/vcpkg_ports/libpq/unix/mingw-install.patch +++ /dev/null @@ -1,36 +0,0 @@ -diff --git a/src/Makefile.shlib b/src/Makefile.shlib -index 16255d7..51e9ed2 100644 ---- a/src/Makefile.shlib -+++ b/src/Makefile.shlib -@@ -85,7 +85,11 @@ else - # Naming convention for dynamically loadable modules - shlib = $(NAME)$(DLSUFFIX) - endif -+ifeq ($(PORTNAME)-$(LIBPQ_LIBRARY_TYPE), win32-shared) -+stlib = lib$(NAME).dll.a -+else - stlib = lib$(NAME).a -+endif - - ifndef soname - # additional flags for backend modules -@@ -468,6 +472,9 @@ endif - else # no soname - $(INSTALL_SHLIB) $< '$(DESTDIR)$(pkglibdir)/$(shlib)' - endif -+ifeq ($(PORTNAME)-$(LIBPQ_LIBRARY_TYPE), win32-shared) -+ mv '$(DESTDIR)$(libdir)/$(shlib)' '$(DESTDIR)$(libdir)/../bin/$(shlib)' -+endif - - - installdirs-lib: -@@ -476,6 +483,9 @@ ifdef soname - else - $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' - endif -+ifeq ($(PORTNAME)-$(LIBPQ_LIBRARY_TYPE), win32-shared) -+ $(MKDIR_P) '$(DESTDIR)$(libdir)/../bin' -+endif - - - ## diff --git a/vcpkg_ports/libpq/unix/mingw-static-importlib-fix.patch b/vcpkg_ports/libpq/unix/mingw-static-importlib-fix.patch deleted file mode 100644 index 44d465710..000000000 --- a/vcpkg_ports/libpq/unix/mingw-static-importlib-fix.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff --git a/src/Makefile.shlib b/src/Makefile.shlib -index f94d59d..77aa6bf 100644 ---- a/src/Makefile.shlib -+++ b/src/Makefile.shlib -@@ -216,8 +216,10 @@ ifeq ($(PORTNAME), win32) - ifdef SO_MAJOR_VERSION - shlib = lib$(NAME)$(DLSUFFIX) - endif -+ifneq ($(LIBPQ_LIBRARY_TYPE), static) - haslibarule = yes - endif -+endif - - - # If the shared library doesn't have an export file, mark all symbols not -@@ -323,8 +325,10 @@ else - # Win32 case - - # See notes in src/backend/parser/Makefile about the following two rules -+ifneq ($(LIBPQ_LIBRARY_TYPE), static) - $(stlib): $(shlib) - touch $@ -+endif - - # XXX A backend that loads a module linked with libgcc_s_dw2-1.dll will exit - # uncleanly, hence -static-libgcc. (Last verified with MinGW-w64 compilers - -diff --git a/src/makefiles/Makefile.win32 b/src/makefiles/Makefile.win32: -index dc1aafa..1ab97a8 100644 ---- a/src/makefiles/Makefile.win32 -+++ b/src/makefiles/Makefile.win32 -@@ -22,21 +22,32 @@ endif - endif - endif - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring src/common,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif -+ - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring src/port,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif -+ - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring timezone,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif -+ - -+ifeq (,$(filter static,$(LIBPQ_LIBRARY_TYPE))) - ifneq (,$(findstring ecpg/ecpglib,$(subdir))) - override CPPFLAGS+= -DBUILDING_DLL - endif -+endif - - # required by Python headers - ifneq (,$(findstring src/pl/plpython,$(subdir))) diff --git a/vcpkg_ports/libpq/unix/no-server-tools.patch b/vcpkg_ports/libpq/unix/no-server-tools.patch deleted file mode 100644 index 1677edb75..000000000 --- a/vcpkg_ports/libpq/unix/no-server-tools.patch +++ /dev/null @@ -1,39 +0,0 @@ -diff --git a/src/bin/Makefile b/src/bin/Makefile -index 7f9dde9..bc6d835 100644 ---- a/src/bin/Makefile -+++ b/src/bin/Makefile -@@ -13,31 +13,20 @@ subdir = src/bin - top_builddir = ../.. - include $(top_builddir)/src/Makefile.global - -+.NOTPARALLEL: -+# incl. https://www.postgresql.org/docs/current/reference-client.html -+# excl. https://www.postgresql.org/docs/current/reference-server.html - SUBDIRS = \ -- initdb \ - pg_amcheck \ -- pg_archivecleanup \ - pg_basebackup \ -- pg_checksums \ -- pg_combinebackup \ - pg_config \ -- pg_controldata \ -- pg_ctl \ - pg_dump \ -- pg_resetwal \ -- pg_rewind \ -- pg_test_fsync \ -- pg_test_timing \ -- pg_upgrade \ - pg_verifybackup \ -- pg_waldump \ -- pg_walsummary \ - pgbench \ - psql \ - scripts - - ifeq ($(PORTNAME), win32) --SUBDIRS += pgevent - else - ALWAYS_SUBDIRS += pgevent - endif diff --git a/vcpkg_ports/libpq/unix/python.patch b/vcpkg_ports/libpq/unix/python.patch deleted file mode 100644 index 7d8f58d4a..000000000 --- a/vcpkg_ports/libpq/unix/python.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/configure.ac b/configure.ac -index 7f97248992..33b6c84fc4 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -891,7 +891,9 @@ AC_SUBST(with_perl) - # Optionally build Python modules (PL/Python) - # - AC_MSG_CHECKING([whether to build Python modules]) --PGAC_ARG_BOOL(with, python, no, [build Python modules (PL/Python)]) -+PGAC_ARG_OPTARG(with, python, [PYTHON_VERSION], [build Python modules (PL/Python)], -+ [], -+ [python_version=$withval]) - AC_MSG_RESULT([$with_python]) - AC_SUBST(with_python) - -@@ -1214,7 +1216,18 @@ fi - - if test "$with_python" = yes; then - PGAC_PATH_PYTHON -- PGAC_CHECK_PYTHON_EMBED_SETUP -+ python_majorversion=3 -+ PKG_CHECK_MODULES(PYTHON_EMBED, python-${python_version}-embed) -+ python_includespec="${PYTHON_EMBED_CFLAGS}" -+ python_libdir=[$(echo " ${PYTHON_EMBED_LIBS}" | sed -e 's/\( -L[^ ]*\).*/\1/' -e 's/^.* -L//')] -+ python_libspec="${PYTHON_EMBED_LIBS}" -+ python_additional_libs="" -+ AC_SUBST(python_majorversion) -+ AC_SUBST(python_version) -+ AC_SUBST(python_includespec) -+ AC_SUBST(python_libdir) -+ AC_SUBST(python_libspec) -+ AC_SUBST(python_additional_libs) - fi - - if test x"$cross_compiling" = x"yes" && test -z "$with_system_tzdata"; then diff --git a/vcpkg_ports/libpq/unix/single-linkage.patch b/vcpkg_ports/libpq/unix/single-linkage.patch deleted file mode 100644 index 1c334a58a..000000000 --- a/vcpkg_ports/libpq/unix/single-linkage.patch +++ /dev/null @@ -1,56 +0,0 @@ -diff --git a/src/Makefile.shlib b/src/Makefile.shlib -index 551023c..16255d7 100644 ---- a/src/Makefile.shlib -+++ b/src/Makefile.shlib -@@ -260,10 +260,14 @@ endif - - .PHONY: all-lib all-static-lib all-shared-lib - -+ifndef LIBPQ_LIBRARY_TYPE - all-lib: all-shared-lib - ifdef soname - # no static library when building a dynamically loadable module - all-lib: all-static-lib -+endif -+else -+all-lib: all-$(LIBPQ_LIBRARY_TYPE)-lib - all-lib: lib$(NAME).pc - endif - -@@ -417,9 +421,13 @@ endif # PORTNAME == cygwin || PORTNAME == win32 - ## - - .PHONY: install-lib install-lib-static install-lib-shared installdirs-lib -+ifndef LIBPQ_LIBRARY_TYPE - install-lib: install-lib-shared - ifdef soname - install-lib: install-lib-static -+endif -+else -+install-lib: install-lib-$(LIBPQ_LIBRARY_TYPE) - install-lib: install-lib-pc - endif - -diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile -index 8abdb09..185461e 100644 ---- a/src/interfaces/libpq/Makefile -+++ b/src/interfaces/libpq/Makefile -@@ -115,6 +115,7 @@ backend_src = $(top_srcdir)/src/backend - # Also skip the test on platforms where libpq infrastructure may be provided - # by statically-linked libraries, as we can't expect them to honor this - # coding rule. -+ifeq ($(LIBPQ_LIBRARY_TYPE), shared) - libpq-refs-stamp: $(shlib) - ifneq ($(enable_coverage), yes) - ifeq (,$(filter solaris,$(PORTNAME))) -@@ -124,6 +125,10 @@ ifeq (,$(filter solaris,$(PORTNAME))) - endif - endif - touch $@ -+else -+.PHONY: libpq-refs-stamp -+libpq-refs-stamp: -+endif - - # Make dependencies on pg_config_paths.h visible in all builds. - fe-connect.o: fe-connect.c $(top_builddir)/src/port/pg_config_paths.h diff --git a/vcpkg_ports/libpq/usage b/vcpkg_ports/libpq/usage index 8dfcd2754..1f36ee18b 100644 --- a/vcpkg_ports/libpq/usage +++ b/vcpkg_ports/libpq/usage @@ -1,4 +1,4 @@ -The package libpq provides CMake integration: +libpq provides CMake integration: - find_package(PostgreSQL REQUIRED) - target_link_libraries(main PRIVATE PostgreSQL::PostgreSQL) + find_package(PostgreSQL REQUIRED) + target_link_libraries(main PRIVATE PostgreSQL::PostgreSQL) diff --git a/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake b/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake index fa7d3b56e..fdb714590 100644 --- a/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake +++ b/vcpkg_ports/libpq/vcpkg-cmake-wrapper.cmake @@ -44,9 +44,9 @@ if(PostgreSQL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") endforeach() if(WIN32) if(TARGET PostgreSQL::PostgreSQL) - set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "Secur32.lib") + set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "secur32.lib") endif() - list(APPEND PostgreSQL_LIBRARIES Secur32.lib) + list(APPEND PostgreSQL_LIBRARIES secur32.lib) endif() cmake_policy(PUSH) cmake_policy(SET CMP0057 NEW) @@ -56,13 +56,13 @@ if(PostgreSQL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") if(TARGET PostgreSQL::PostgreSQL) set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "OpenSSL::SSL") endif() - list(APPEND PostgreSQL_LIBRARIES OpenSSL::SSL) + list(APPEND PostgreSQL_LIBRARIES ${OPENSSL_LIBRARIES}) endif() if(WIN32) if(TARGET PostgreSQL::PostgreSQL) - set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "Wldap32.lib") + set_property(TARGET PostgreSQL::PostgreSQL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "wldap32.lib") endif() - list(APPEND PostgreSQL_LIBRARIES Wldap32.lib) + list(APPEND PostgreSQL_LIBRARIES wldap32.lib) endif() unset(Z_VCPKG_PORT_FEATURES) cmake_policy(POP) diff --git a/vcpkg_ports/libpq/vcpkg-libs.props.in b/vcpkg_ports/libpq/vcpkg-libs.props.in deleted file mode 100644 index cb7e5f92c..000000000 --- a/vcpkg_ports/libpq/vcpkg-libs.props.in +++ /dev/null @@ -1,31 +0,0 @@ - - - - @CURRENT_INSTALLED_DIR@/debug/lib/icuind.lib;@CURRENT_INSTALLED_DIR@/debug/lib/icuucd.lib;@CURRENT_INSTALLED_DIR@/debug/lib/icudtd.lib; - @CURRENT_INSTALLED_DIR@/debug/lib/lz4d.lib - @CURRENT_INSTALLED_DIR@/debug/lib/intl.lib;@CURRENT_INSTALLED_DIR@/debug/lib/iconv.lib;@CURRENT_INSTALLED_DIR@/debug/lib/charset.lib - @CURRENT_INSTALLED_DIR@/debug/lib/libssl.lib;@CURRENT_INSTALLED_DIR@/debug/lib/libcrypto.lib;crypt32.lib;ws2_32.lib;secur32.lib - @CURRENT_INSTALLED_DIR@/debug/lib/python3@PYTHON_VERSION_MINOR@_d.lib - @CURRENT_INSTALLED_DIR@/debug/lib/tcl90g.lib - @CURRENT_INSTALLED_DIR@/debug/lib/tcl90sg.lib - @CURRENT_INSTALLED_DIR@/debug/lib/tcl90sgx.lib - @LIBXML2_LIBS_DEBUG@ - @LIBXSLT_LIBS_DEBUG@ - @CURRENT_INSTALLED_DIR@/debug/lib/zlibd.lib - @CURRENT_INSTALLED_DIR@/debug/lib/zstd.lib - - - @CURRENT_INSTALLED_DIR@/lib/icuin.lib;@CURRENT_INSTALLED_DIR@/lib/icuuc.lib;@CURRENT_INSTALLED_DIR@/lib/icudt.lib; - @CURRENT_INSTALLED_DIR@/lib/lz4.lib - @CURRENT_INSTALLED_DIR@/lib/intl.lib;@CURRENT_INSTALLED_DIR@/lib/iconv.lib;@CURRENT_INSTALLED_DIR@/lib/charset.lib - @CURRENT_INSTALLED_DIR@/lib/libssl.lib;@CURRENT_INSTALLED_DIR@/lib/libcrypto.lib;crypt32.lib;ws2_32.lib;secur32.lib - @CURRENT_INSTALLED_DIR@/lib/python3@PYTHON_VERSION_MINOR@.lib - @CURRENT_INSTALLED_DIR@/lib/tcl90.lib - @CURRENT_INSTALLED_DIR@/lib/tcl90s.lib - @CURRENT_INSTALLED_DIR@/lib/tcl90sx.lib - @LIBXML2_LIBS_RELEASE@ - @LIBXSLT_LIBS_RELEASE@ - @CURRENT_INSTALLED_DIR@/lib/zlib.lib - @CURRENT_INSTALLED_DIR@/lib/zstd.lib - - diff --git a/vcpkg_ports/libpq/vcpkg.json b/vcpkg_ports/libpq/vcpkg.json index 782747bda..2dd780c6a 100644 --- a/vcpkg_ports/libpq/vcpkg.json +++ b/vcpkg_ports/libpq/vcpkg.json @@ -1,27 +1,18 @@ { "name": "libpq", - "version": "18.3", + "version": "18.4", "description": "The official database access API of postgresql", "homepage": "https://www.postgresql.org/", "license": "PostgreSQL", "supports": "!uwp & !emscripten", "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "bonjour" - ], - "platform": "osx" - }, { "name": "vcpkg-cmake-get-vars", "host": true }, { "name": "vcpkg-tool-meson", - "host": true, - "platform": "windows & !mingw" + "host": true } ], "default-features": [ @@ -38,55 +29,35 @@ "features": [ "client", "icu", - "xml", - "xslt", + "lz4", + "nls", + "openssl", + "zlib", "zstd" ] }, - { - "name": "libpq", - "features": [ - "bonjour" - ], - "platform": "osx" - }, - { - "name": "libpq", - "features": [ - "nls" - ], - "platform": "!osx" - }, { "name": "libpq", "features": [ "readline" ], "platform": "!windows" - }, - { - "name": "libpq", - "features": [ - "python" - ], - "platform": "!android & !mingw" - }, + } + ] + }, + "client": { + "description": "Build all client tools and libraries", + "dependencies": [ { "name": "libpq", + "host": true, + "default-features": false, "features": [ - "tcl" - ], - "platform": "windows & !mingw & !arm" + "client" + ] } ] }, - "bonjour": { - "description": "Build with Bonjour support", - "supports": "osx" - }, - "client": { - "description": "Build all client tools and libraries" - }, "icu": { "description": "Build with support for the ICU library", "dependencies": [ @@ -119,20 +90,6 @@ "openssl" ] }, - "python": { - "$supports": "!(windows & (static | mingw))", - "description": "build the PL/Python server programming language", - "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "client" - ] - }, - "python3" - ] - }, "readline": { "description": "Use readline", "supports": "!windows, mingw", @@ -140,43 +97,6 @@ "readline" ] }, - "tcl": { - "$supports": "!(windows & (static | mingw))", - "description": "build the PL/Tcl procedural language", - "supports": "windows & !mingw", - "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "client" - ] - }, - "tcl" - ] - }, - "xml": { - "description": "Build with libxml", - "dependencies": [ - { - "name": "libxml2", - "default-features": false - } - ] - }, - "xslt": { - "description": "Build with libxslt", - "dependencies": [ - { - "name": "libpq", - "default-features": false, - "features": [ - "xml" - ] - }, - "libxslt" - ] - }, "zlib": { "description": "Use zlib", "dependencies": [ diff --git a/vcpkg_ports/libpq/windows/clang-cl-c99-test.patch b/vcpkg_ports/libpq/windows/clang-cl-c99-test.patch new file mode 100644 index 000000000..1a200cbde --- /dev/null +++ b/vcpkg_ports/libpq/windows/clang-cl-c99-test.patch @@ -0,0 +1,15 @@ +diff --git a/meson.build b/meson.build +index 5b9ea5f..a552243 100644 +--- a/meson.build ++++ b/meson.build +@@ -1711,8 +1711,10 @@ + # unnecessarily, because we optionally rely on newer features. + c99_test = ''' + #include ++#ifndef _MSC_VER + #include + #include ++#endif + #include + + struct named_init_test { diff --git a/vcpkg_ports/libpq/windows/clang-cl-msvc-abi.patch b/vcpkg_ports/libpq/windows/clang-cl-msvc-abi.patch new file mode 100644 index 000000000..db890396d --- /dev/null +++ b/vcpkg_ports/libpq/windows/clang-cl-msvc-abi.patch @@ -0,0 +1,316 @@ +diff --git a/meson.build b/meson.build +index 2c94236..e42075d 100644 +--- a/meson.build ++++ b/meson.build +@@ -40,6 +40,7 @@ build_system = build_machine.system() + host_cpu = host_machine.cpu_family() + + cc = meson.get_compiler('c') ++msvc_abi = cc.get_argument_syntax() == 'msvc' + + not_found_dep = dependency('', required: false) + thread_dep = dependency('threads') +@@ -275,14 +276,14 @@ elif host_system == 'windows' + exesuffix = '.exe' + dlsuffix = '.dll' + library_path_var = '' +- if cc.get_id() != 'msvc' ++ if not msvc_abi + # define before including for getting localtime_r() etc. on MinGW + cppflags += '-D_POSIX_C_SOURCE' + endif + + export_file_format = 'win' + export_file_suffix = 'def' +- if cc.get_id() == 'msvc' ++ if msvc_abi + export_fmt = '/DEF:@0@' + mod_link_with_name = '@0@.lib' + else +@@ -296,7 +297,7 @@ elif host_system == 'windows' + sema_kind = 'win32' + + cdata.set('WIN32_STACK_RLIMIT', 4194304) +- if cc.get_id() == 'msvc' ++ if msvc_abi + ldflags += '/INCREMENTAL:NO' + ldflags += '/STACK:@0@'.format(cdata.get('WIN32_STACK_RLIMIT')) + # ldflags += '/nxcompat' # generated by msbuild, should have it for ninja? +@@ -314,7 +315,7 @@ elif host_system == 'windows' + libpq_deps += secur32_dep + + postgres_inc_d += 'src/include/port/win32' +- if cc.get_id() == 'msvc' ++ if msvc_abi + postgres_inc_d += 'src/include/port/win32_msvc' + endif + +@@ -1216,7 +1217,7 @@ if not perlopt.disabled() + if host_system == 'windows' + perl_ccflags += ['-DPLPERL_HAVE_UID_GID'] + +- if cc.get_id() == 'msvc' ++ if msvc_abi + # prevent binary mismatch between MSVC built plperl and Strawberry or + # msys ucrt perl libraries + perl_v = run_command(perl, '-V', check: false).stdout() +@@ -1429,7 +1430,7 @@ Use -Dreadline=disabled to disable readline support.'''.format(readline_dep)) + + # On windows with mingw readline requires auto-import to successfully + # link, as the headers don't use declspec(dllimport) +- if host_system == 'windows' and cc.get_id() != 'msvc' ++ if host_system == 'windows' and not msvc_abi + readline = declare_dependency(dependencies: readline, + link_args: '-Wl,--enable-auto-import') + endif +@@ -2018,50 +2019,78 @@ if cc.links(''' + endif + + # Check for __get_cpuid() and __cpuid(). +-if cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); +- } +- ''', name: '__get_cpuid', +- args: test_c_args) +- cdata.set('HAVE__GET_CPUID', 1) +-elif cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __cpuid(exx, 1); +- } +- ''', name: '__cpuid', +- args: test_c_args) +- cdata.set('HAVE__CPUID', 1) ++if msvc_abi ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuid(exx, 1); ++ } ++ ''', name: '__cpuid', ++ args: test_c_args) ++ cdata.set('HAVE__CPUID', 1) ++ endif ++else ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ unsigned int exx[4] = {0, 0, 0, 0}; ++ __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); ++ } ++ ''', name: '__get_cpuid', ++ args: test_c_args) ++ cdata.set('HAVE__GET_CPUID', 1) ++ elif cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuid(exx, 1); ++ } ++ ''', name: '__cpuid', ++ args: test_c_args) ++ cdata.set('HAVE__CPUID', 1) ++ endif + endif + + + # Check for __get_cpuid_count() and __cpuidex() in a similar fashion. +-if cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); +- } +- ''', name: '__get_cpuid_count', +- args: test_c_args) +- cdata.set('HAVE__GET_CPUID_COUNT', 1) +-elif cc.links(''' +- #include +- int main(int arg, char **argv) +- { +- unsigned int exx[4] = {0, 0, 0, 0}; +- __cpuidex(exx, 7, 0); +- } +- ''', name: '__cpuidex', +- args: test_c_args) +- cdata.set('HAVE__CPUIDEX', 1) ++if msvc_abi ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuidex(exx, 7, 0); ++ } ++ ''', name: '__cpuidex', ++ args: test_c_args) ++ cdata.set('HAVE__CPUIDEX', 1) ++ endif ++else ++ if cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ unsigned int exx[4] = {0, 0, 0, 0}; ++ __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); ++ } ++ ''', name: '__get_cpuid_count', ++ args: test_c_args) ++ cdata.set('HAVE__GET_CPUID_COUNT', 1) ++ elif cc.links(''' ++ #include ++ int main(int arg, char **argv) ++ { ++ int exx[4] = {0, 0, 0, 0}; ++ __cpuidex(exx, 7, 0); ++ } ++ ''', name: '__cpuidex', ++ args: test_c_args) ++ cdata.set('HAVE__CPUIDEX', 1) ++ endif + endif + + +@@ -2183,7 +2212,7 @@ foreach w : negative_warning_flags + endforeach + + +-if cc.get_id() == 'msvc' ++if msvc_abi + cflags_warn += [ + '/wd4018', # signed/unsigned mismatch + '/wd4244', # conversion from 'type1' to 'type2', possible loss of data +@@ -2442,7 +2471,7 @@ have_optimized_crc = false + cflags_crc = [] + if host_cpu == 'x86' or host_cpu == 'x86_64' + +- if cc.get_id() == 'msvc' ++ if msvc_abi + cdata.set('USE_SSE42_CRC32C', false) + cdata.set('USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 1) + have_optimized_crc = true +@@ -2591,7 +2620,7 @@ endif + # Other CPU specific stuff + ############################################################### + +-if host_cpu == 'x86_64' ++if host_cpu == 'x86_64' and not msvc_abi + + if cc.compiles(''' + void main(void) +diff --git a/src/backend/meson.build b/src/backend/meson.build +index 2b0db21..aad9a3b 100644 +--- a/src/backend/meson.build ++++ b/src/backend/meson.build +@@ -70,7 +70,7 @@ postgres_lib = static_library('postgres_lib', + kwargs: internal_lib_args, + ) + +-if cc.get_id() == 'msvc' ++if msvc_abi + postgres_def = custom_target('postgres.def', + command: [perl, files('../tools/msvc_gendef.pl'), + '--arch', host_cpu, +diff --git a/src/bin/pgevent/meson.build b/src/bin/pgevent/meson.build +index 6ec04e9..24b2bd7 100644 +--- a/src/bin/pgevent/meson.build ++++ b/src/bin/pgevent/meson.build +@@ -17,7 +17,7 @@ pgevent_sources += windows.compile_resources('pgmsgevent.rc') + # XXX: We may want to revisit the addition of this flag and improve the + # way symbols are exported. + pgevent_link_args = [] +-if cc.get_id() == 'msvc' ++if msvc_abi + pgevent_link_args += '/ignore:4104' + endif + +diff --git a/src/port/meson.build b/src/port/meson.build +index fc7b059..57a6a06 100644 +--- a/src/port/meson.build ++++ b/src/port/meson.build +@@ -54,7 +54,7 @@ elif host_system == 'cygwin' + ) + endif + +-if cc.get_id() == 'msvc' ++if msvc_abi + pgport_sources += files( + 'dirent.c', + 'win32gettimeofday.c', +@@ -136,7 +136,7 @@ endforeach + + + if (host_system == 'windows' or host_system == 'cygwin') and \ +- (cc.get_id() != 'msvc' or cc.version().version_compare('<14.0')) ++ (not msvc_abi or cc.version().version_compare('<14.0')) + + # Cygwin and (apparently, based on test results) Mingw both + # have a broken strtof(), so substitute its implementation. +diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c +index 61c7388..941e1b5 100644 +--- a/src/port/pg_bitutils.c ++++ b/src/port/pg_bitutils.c +@@ -143,7 +143,7 @@ pg_popcount_available(void) + #if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUID) +- __cpuid(exx, 1); ++ __cpuid((int *) exx, 1); + #else + #error cpuid instruction not available + #endif +diff --git a/src/port/pg_crc32c_sse42_choose.c b/src/port/pg_crc32c_sse42_choose.c +index 74d2421..f18ea81 100644 +--- a/src/port/pg_crc32c_sse42_choose.c ++++ b/src/port/pg_crc32c_sse42_choose.c +@@ -73,7 +73,7 @@ pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) + #if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUID) +- __cpuid(exx, 1); ++ __cpuid((int *) exx, 1); + #else + #error cpuid instruction not available + #endif +@@ -92,7 +92,7 @@ pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) + #if defined(HAVE__GET_CPUID_COUNT) + __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUIDEX) +- __cpuidex(exx, 7, 0); ++ __cpuidex((int *) exx, 7, 0); + #endif + + #ifdef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK +diff --git a/src/port/pg_popcount_avx512.c b/src/port/pg_popcount_avx512.c +index 896bc74..9a0aa75 100644 +--- a/src/port/pg_popcount_avx512.c ++++ b/src/port/pg_popcount_avx512.c +@@ -44,7 +44,7 @@ xsave_available(void) + #if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUID) +- __cpuid(exx, 1); ++ __cpuid((int *) exx, 1); + #else + #error cpuid instruction not available + #endif +@@ -82,7 +82,7 @@ avx512_popcnt_available(void) + #if defined(HAVE__GET_CPUID_COUNT) + __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); + #elif defined(HAVE__CPUIDEX) +- __cpuidex(exx, 7, 0); ++ __cpuidex((int *) exx, 7, 0); + #else + #error cpuid instruction not available + #endif diff --git a/vcpkg_ports/libpq/windows/getopt.patch b/vcpkg_ports/libpq/windows/getopt.patch index 073b32adb..c6e0126a3 100644 --- a/vcpkg_ports/libpq/windows/getopt.patch +++ b/vcpkg_ports/libpq/windows/getopt.patch @@ -1,21 +1,14 @@ diff --git a/meson.build b/meson.build +index 34456b3..052e8c8 100644 --- a/meson.build +++ b/meson.build -@@ -2624,7 +2624,7 @@ header_checks = [ - 'copyfile.h', - 'crtdefs.h', - 'execinfo.h', -- 'getopt.h', -+ # 'getopt.h', # vcpkg: skip to avoid getopt-win32 header pollution - 'ifaddrs.h', - 'mbarrier.h', - 'strings.h', -@@ -2714,7 +2714,7 @@ endforeach +@@ -2650,6 +2650,9 @@ foreach header : header_checks + description: 'Define to 1 if you have the <@0@> header file.'.format(header)) + endforeach - if cc.has_type('struct option', - args: test_c_args, include_directories: postgres_inc, -- prefix: '@0@'.format(cdata.get('HAVE_GETOPT_H')) == '1' ? '#include ' : '') -+ prefix: '@0@'.format(cdata.get('HAVE_GETOPT_H', 0)) == '1' ? '#include ' : '') - cdata.set('HAVE_STRUCT_OPTION', 1) - endif ++if cc.get_id() == 'msvc' ++ cdata.set('HAVE_GETOPT_H', false) ++endif + decl_checks = [ + ['F_FULLFSYNC', 'fcntl.h'], diff --git a/vcpkg_ports/libpq/windows/meson-vcpkg.patch b/vcpkg_ports/libpq/windows/meson-vcpkg.patch deleted file mode 100644 index 84543bcd0..000000000 --- a/vcpkg_ports/libpq/windows/meson-vcpkg.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/meson.build b/meson.build ---- a/meson.build -+++ b/meson.build -@@ -83,7 +83,7 @@ endif - ############################################################### - - postgres_inc_d = ['src/include'] --postgres_inc_d += get_option('extra_include_dirs') -+# postgres_inc_d += get_option('extra_include_dirs') # vcpkg: moved below - - postgres_lib_d = get_option('extra_lib_dirs') - -@@ -316,6 +316,7 @@ elif host_system == 'windows' - postgres_inc_d += 'src/include/port/win32' - if cc.get_id() == 'msvc' - postgres_inc_d += 'src/include/port/win32_msvc' -+ postgres_inc_d += get_option('extra_include_dirs') # vcpkg: moved here for correct priority - endif - - windows = import('windows') -@@ -3218,6 +3219,7 @@ endif - generated_headers = [] - # headers that the backend build depends on - generated_backend_headers = [] -+generated_backend_sources = [] - # configure_files() output, needs a way of converting to file names - configure_files = [] - -@@ -3372,12 +3374,12 @@ test_install_libs = [] - - subdir('src') - --subdir('contrib') -+# subdir('contrib') # vcpkg: skip - --subdir('src/test') --subdir('src/interfaces/ecpg/test') -+# subdir('src/test') # vcpkg: skip -+# subdir('src/interfaces/ecpg/test') # vcpkg: skip - --subdir('doc/src/sgml') -+# subdir('doc/src/sgml') # vcpkg: skip - - generated_sources_ac += {'': ['GNUmakefile']} - diff --git a/vcpkg_ports/libpq/windows/tcl-9.0-alpha.patch b/vcpkg_ports/libpq/windows/tcl-9.0-alpha.patch deleted file mode 100644 index 154efcb8f..000000000 --- a/vcpkg_ports/libpq/windows/tcl-9.0-alpha.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -Naur postgresql-16.9/src/pl/tcl/pltcl.c postgresql-16.9-vcpkg/src/pl/tcl/pltcl.c ---- postgresql-16.9/src/pl/tcl/pltcl.c 2025-05-05 22:30:08.000000000 +0200 -+++ postgresql-16.9-vcpkg/src/pl/tcl/pltcl.c 2025-05-23 22:53:43.485828671 +0200 -@@ -56,7 +56,7 @@ - #define CONST86 - #endif - --#if !HAVE_TCL_VERSION(8,7) -+#if !HAVE_TCL_VERSION(8,7) || (TCL_MAJOR_VERSION == 9 && TCL_MINOR_VERSION == 0 && TCL_RELEASE_LEVEL == TCL_ALPHA_RELEASE) - typedef int Tcl_Size; - #endif - diff --git a/vcpkg_ports/libpq/zic.diff b/vcpkg_ports/libpq/zic.diff new file mode 100644 index 000000000..737182f2d --- /dev/null +++ b/vcpkg_ports/libpq/zic.diff @@ -0,0 +1,22 @@ +diff --git a/src/timezone/meson.build b/src/timezone/meson.build +index ad9f6f6..bb06082 100644 +--- a/src/timezone/meson.build ++++ b/src/timezone/meson.build +@@ -27,7 +27,7 @@ if get_option('system_tzdata') == '' + # FIXME: For cross builds, it would need a native built libpgport/pgcommon to + # build our zic. But for that we'd need to run a good chunk of the configure + # tests both natively and cross. Unclear if it's worth it. +- if meson.is_cross_build() ++ if meson.is_cross_build() or get_option('ZIC') != 'zic' + zic = find_program(get_option('ZIC'), native: true, required: true) + else + if host_system == 'windows' +@@ -38,7 +38,7 @@ if get_option('system_tzdata') == '' + + zic = executable('zic', zic_sources, + dependencies: [frontend_code], +- kwargs: default_bin_args + {'install': false} ++ kwargs: default_bin_args + {'install': true} + ) + endif + diff --git a/vcpkg_ports/vcpkg-tool-meson/portfile.cmake b/vcpkg_ports/vcpkg-tool-meson/portfile.cmake new file mode 100644 index 000000000..0e7ac53c6 --- /dev/null +++ b/vcpkg_ports/vcpkg-tool-meson/portfile.cmake @@ -0,0 +1,7 @@ +include("${VCPKG_ROOT_DIR}/ports/${PORT}/portfile.cmake") + +vcpkg_replace_string( + "${CURRENT_PACKAGES_DIR}/share/${PORT}/vcpkg_install_meson.cmake" + "COMMAND \"\${NINJA}\" install -v" + "COMMAND \"\${NINJA}\" -v install" +) diff --git a/vcpkg_ports/vcpkg-tool-meson/vcpkg.json b/vcpkg_ports/vcpkg-tool-meson/vcpkg.json new file mode 100644 index 000000000..5c010bde2 --- /dev/null +++ b/vcpkg_ports/vcpkg-tool-meson/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "vcpkg-tool-meson", + "version": "1.9.0", + "port-version": 10, + "description": "Meson build system", + "homepage": "https://github.com/mesonbuild/meson", + "license": "Apache-2.0", + "supports": "native", + "dependencies": [ + "vcpkg-cmake-get-vars" + ] +}