From 6e1318e87a9bad7f709e5c1e1ec01e05a2f26f17 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 19 Oct 2026 17:51:50 +0700 Subject: [PATCH 1/2] test(linux): run the million-row case on SQLite in its own binary --- linux/ROADMAP.md | 2 +- .../tests/integration/conformance_engine.rs | 1 + .../tests/integration/conformance_engine.rs | 1 + .../tests/integration/conformance_engine.rs | 2 + .../tests/integration/conformance_engine.rs | 1 + .../tests/integration/conformance_engine.rs | 133 +--------------- .../drivers/sqlite/tests/integration/main.rs | 2 + .../sqlite/tests/integration/metadata.rs | 3 +- .../tests/integration/open_semantics.rs | 3 +- .../sqlite/tests/integration/sqlite_engine.rs | 149 ++++++++++++++++++ .../sqlite/tests/integration/support.rs | 11 +- .../sqlite/tests/integration/value_corpus.rs | 2 +- .../drivers/sqlite/tests/large_table.rs | 12 ++ .../conformance/large_table_suite_macro.rs | 42 +++-- linux/docs/adding-drivers.md | 2 +- linux/docs/testing.md | 4 +- 16 files changed, 212 insertions(+), 158 deletions(-) create mode 100644 linux/crates/drivers/sqlite/tests/integration/sqlite_engine.rs create mode 100644 linux/crates/drivers/sqlite/tests/large_table.rs diff --git a/linux/ROADMAP.md b/linux/ROADMAP.md index 724cc96079..13b174c756 100644 --- a/linux/ROADMAP.md +++ b/linux/ROADMAP.md @@ -181,7 +181,7 @@ What has to hold before the first release, for a user on GNOME 50 with Ubuntu 26 | 1 | Installs `tablepro` from the PPA or the COPR project, and gets updates from the system's updater | CI builds both packages offline the way Launchpad and COPR do, and starts them in fresh containers | The release setup, the first tag, and an install then an upgrade from the PPA and COPR | | 2 | Connects to their everyday PostgreSQL, MySQL, MariaDB, SQL Server, ClickHouse or SQLite database | The supported releases and why others are out, in the README's [supported servers](README.md#supported-servers). CI runs every driver's Docker suite on each tested release, MySQL 8.0 and SQL Server 2017 and 2019 among them | Nothing | | 3 | Browses tables of real-world data (dates, decimals, JSON, UUIDs, NULLs) and sees correct values | Per-engine decoding with the type fallback and unsupported-type conformance cases ([ADR 0014](docs/decisions/0014-lossless-values-and-dialects.md)). A value corpus on every engine, MariaDB too: dates and times at their extremes and in other zones, decimals, JSON, UUIDs, Unicode and binary values, read back through a table tab, Export All and an editor run | Nothing | -| 4 | Runs typical queries against tables of a million rows or more without running out of memory or freezing | The 10,000-row editor limit with Export All, and driver tests over 2,000,000 rows. Pages seek past the edge row's key, or walk PostgreSQL heap block windows for a table with no key; OFFSET only under a user sort, a restored deep offset or a partitioned keyless table. Docker tests page 60k and 25k-row tables both ways and check the plan. A 1,000,000-row table is paged to its end and exported on PostgreSQL, MySQL, MariaDB, SQL Server and ClickHouse, with bounds on time and on resident peak growth | The same test on SQLite, in a process of its own: its suites run in the unit suite, where the peak would count every test's | +| 4 | Runs typical queries against tables of a million rows or more without running out of memory or freezing | The 10,000-row editor limit with Export All, and driver tests over 2,000,000 rows. Pages seek past the edge row's key, or walk PostgreSQL heap block windows for a table with no key; OFFSET only under a user sort, a restored deep offset or a partitioned keyless table. Docker tests page 60k and 25k-row tables both ways and check the plan. A 1,000,000-row table is paged to its end and exported on PostgreSQL, MySQL, MariaDB, SQL Server, ClickHouse and SQLite, with bounds on time and on resident peak growth. SQLite's runs in the unit suite as a test binary of its own, so the peak is its alone | Nothing | | 5 | Edits data in place, and the changes commit correctly even when the connection or the app misbehaves | Write, row guard and commit resolution conformance on every engine, with the fault fakes and the grid save Docker test. `kill_during_save` kills TablePro halfway through a SQLite save: the file keeps none of it, and the next launch restores the session with its history, settings and saved tabs | A save into a SQLite file another program holds locked fails as `DriverError::Busy`, whose message says the server has no room for another connection | | 6 | Trusts the app with credentials: TLS, SSH tunnels, no password in plain text | Verify full by default, the TLS conformance suite, the Secret Service, and statement logging silenced | An outside security review | | 7 | Recovers when the network drops or the database restarts | Health monitoring, backoff and replay rules with their tests, a stopped container in the liveness tests, and a session test that restarts PostgreSQL: the read in flight runs again, the save in flight does not and leaves no row | Nothing | diff --git a/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs b/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs index cdb7df8f4d..4365912ff4 100644 --- a/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs +++ b/linux/crates/drivers/clickhouse/tests/integration/conformance_engine.rs @@ -195,5 +195,6 @@ mod suite { tablepro_test_support::large_table_conformance_suite! { engine: ClickHouseEngine, + docker: true, } } diff --git a/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs b/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs index abe64b4483..b2fe8eae57 100644 --- a/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs +++ b/linux/crates/drivers/mssql/tests/integration/conformance_engine.rs @@ -192,5 +192,6 @@ mod suite { tablepro_test_support::large_table_conformance_suite! { engine: MssqlEngine, + docker: true, } } diff --git a/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs b/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs index 4cfb79143b..801467fcd5 100644 --- a/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs +++ b/linux/crates/drivers/mysql/tests/integration/conformance_engine.rs @@ -256,6 +256,7 @@ mod mysql_suite { tablepro_test_support::large_table_conformance_suite! { engine: MySqlEngine, + docker: true, } } @@ -279,5 +280,6 @@ mod mariadb_suite { tablepro_test_support::large_table_conformance_suite! { engine: MySqlEngine, + docker: true, } } diff --git a/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs b/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs index 7212771396..e7f324e2c0 100644 --- a/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs +++ b/linux/crates/drivers/postgres/tests/integration/conformance_engine.rs @@ -207,5 +207,6 @@ mod suite { tablepro_test_support::large_table_conformance_suite! { engine: PgEngine, + docker: true, } } diff --git a/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs b/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs index b26d695a7d..416a6c8dd9 100644 --- a/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs +++ b/linux/crates/drivers/sqlite/tests/integration/conformance_engine.rs @@ -1,135 +1,6 @@ -//! The connection conformance suite on SQLite. +//! The connection and DDL conformance suites on SQLite. -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; - -use async_trait::async_trait; -use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; -use sqlx::{Connection as _, Executor as _}; -use tablepro_core::{ - ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, FileEndpoint, LivenessPolicy, TableRef, - WorkspaceSettings, -}; -use tablepro_driver_sqlite::SqliteDriver; -use tablepro_test_support::conformance::{ - ConformanceDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, PooledConformanceEngine, -}; -use tempfile::TempDir; - -use crate::support::create_database; - -/// A database file of the case's own, which exists before the driver -/// opens it: the suite never relies on the driver creating one. -pub struct SqliteEngine { - _directory: TempDir, - path: PathBuf, - endpoint: FileEndpoint, -} - -fn quote(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - -#[async_trait] -impl ConformanceEngine for SqliteEngine { - fn driver() -> Arc { - Arc::new(SqliteDriver) - } - - async fn start() -> Result { - let directory = tempfile::tempdir().map_err(ConformanceFailure::setup)?; - let path = directory.path().join("conformance.sqlite3"); - create_database(&path).await.map_err(ConformanceFailure::setup)?; - let endpoint = FileEndpoint::open_existing(path.clone()).map_err(ConformanceFailure::setup)?; - Ok(Self { - _directory: directory, - path, - endpoint, - }) - } - - fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { - ConnectOptions { - target: ConnectTarget::File(self.endpoint.clone()), - workspace_settings: settings, - liveness, - environment: ConnectEnvironment::new("conformance"), - } - } - - fn scratch_table(&self, name: &str) -> TableRef { - TableRef::unqualified(name) - } - - fn create_scratch_table_sql(&self, table: &TableRef) -> String { - format!( - "CREATE TABLE {} (id integer primary key, label text not null)", - quote(&table.name) - ) - } - - /// SQLite has no sleep, but a recursive CTE without a stop never - /// produces its count. - fn unbounded_statement_sql(&self, marker: &str) -> String { - format!("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r) SELECT count(*), '{marker}' FROM r") - } - - /// Hold an exclusive lock on a connection of the engine's own for - /// `hold`, and return a read that waits for it in the busy handler. - async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { - let options = SqliteConnectOptions::new().filename(&self.path); - let mut holder = SqliteConnection::connect_with(&options) - .await - .map_err(ConformanceFailure::setup)?; - holder - .execute("BEGIN EXCLUSIVE") - .await - .map_err(ConformanceFailure::setup)?; - tokio::spawn(async move { - tokio::time::sleep(hold).await; - let _ = holder.execute("ROLLBACK").await; - let _ = holder.close().await; - }); - Ok(format!("SELECT count(*), '{marker}' FROM sqlite_schema")) - } - - fn row_source_sql(&self, rows: u64) -> String { - format!("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < {rows}) SELECT n FROM r") - } -} - -impl PooledConformanceEngine for SqliteEngine { - const WORKSPACE_POOL_MAX_SIZE: usize = tablepro_driver_sqlite::WORKSPACE_POOL_MAX_SIZE; -} - -impl EditorTransactionEngine for SqliteEngine { - fn begin_sql(&self) -> String { - "BEGIN".to_owned() - } - - fn commit_sql(&self) -> String { - "COMMIT".to_owned() - } -} - -impl ConformanceDdlEngine for SqliteEngine { - fn integer_type(&self) -> &'static str { - "INTEGER" - } - - fn text_type(&self) -> &'static str { - "TEXT" - } - - fn timestamp_type(&self) -> &'static str { - "TEXT" - } - - fn current_timestamp_expression(&self) -> &'static str { - "CURRENT_TIMESTAMP" - } -} +use crate::sqlite_engine::SqliteEngine; tablepro_test_support::ddl_conformance_suite! { engine: SqliteEngine, diff --git a/linux/crates/drivers/sqlite/tests/integration/main.rs b/linux/crates/drivers/sqlite/tests/integration/main.rs index 1aa23b796c..05ad354b38 100644 --- a/linux/crates/drivers/sqlite/tests/integration/main.rs +++ b/linux/crates/drivers/sqlite/tests/integration/main.rs @@ -23,6 +23,8 @@ mod open_semantics; #[cfg(test)] mod sessions; #[cfg(test)] +mod sqlite_engine; +#[cfg(test)] mod support; mod value_corpus; #[cfg(test)] diff --git a/linux/crates/drivers/sqlite/tests/integration/metadata.rs b/linux/crates/drivers/sqlite/tests/integration/metadata.rs index d93d47d2fe..7d58fde7d6 100644 --- a/linux/crates/drivers/sqlite/tests/integration/metadata.rs +++ b/linux/crates/drivers/sqlite/tests/integration/metadata.rs @@ -6,7 +6,8 @@ use tablepro_core::meta::{ }; use tablepro_core::{Connection, RowLimit, Value}; -use crate::support::{TestFile, apply, call, create_database, query, statement}; +use crate::sqlite_engine::create_database; +use crate::support::{TestFile, apply, call, query, statement}; fn table(name: &str) -> TableRef { TableRef::unqualified(name) diff --git a/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs b/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs index a4c4116481..5a61080652 100644 --- a/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs +++ b/linux/crates/drivers/sqlite/tests/integration/open_semantics.rs @@ -7,7 +7,8 @@ use std::path::PathBuf; use tablepro_core::{DriverError, ErrorCategory, FileEndpoint, Value, WorkspaceSettings}; use tokio_util::sync::CancellationToken; -use crate::support::{TestFile, apply, connect, create_database, query, writable}; +use crate::sqlite_engine::create_database; +use crate::support::{TestFile, apply, connect, query, writable}; #[tokio::test] async fn open_existing_missing_path_file_not_found_no_file_created() { diff --git a/linux/crates/drivers/sqlite/tests/integration/sqlite_engine.rs b/linux/crates/drivers/sqlite/tests/integration/sqlite_engine.rs new file mode 100644 index 0000000000..4a4fe9f0b5 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/sqlite_engine.rs @@ -0,0 +1,149 @@ +//! The conformance engine adapter on SQLite. The `large_table` test +//! target shares it, so it uses nothing else from this target. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; +use sqlx::{Connection as _, Executor as _}; +use tablepro_core::{ + ConnectEnvironment, ConnectOptions, ConnectTarget, DatabaseDriver, FileEndpoint, LivenessPolicy, TableRef, + WorkspaceSettings, +}; +use tablepro_driver_sqlite::SqliteDriver; +use tablepro_test_support::conformance::{ + ConformanceDdlEngine, ConformanceEngine, ConformanceFailure, EditorTransactionEngine, LargeTableEngine, + PooledConformanceEngine, +}; +use tempfile::TempDir; + +/// Make an empty database the way another program would, so the +/// driver only ever opens a file that exists. +pub async fn create_database(path: &Path) -> Result<(), sqlx::Error> { + let options = SqliteConnectOptions::new().filename(path).create_if_missing(true); + SqliteConnection::connect_with(&options).await?.close().await +} + +/// A database file of the case's own, which exists before the driver +/// opens it: the suite never relies on the driver creating one. +pub struct SqliteEngine { + _directory: TempDir, + path: PathBuf, + endpoint: FileEndpoint, +} + +fn quote(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +#[async_trait] +impl ConformanceEngine for SqliteEngine { + fn driver() -> Arc { + Arc::new(SqliteDriver) + } + + async fn start() -> Result { + let directory = tempfile::tempdir().map_err(ConformanceFailure::setup)?; + let path = directory.path().join("conformance.sqlite3"); + create_database(&path).await.map_err(ConformanceFailure::setup)?; + let endpoint = FileEndpoint::open_existing(path.clone()).map_err(ConformanceFailure::setup)?; + Ok(Self { + _directory: directory, + path, + endpoint, + }) + } + + fn connect_options(&self, settings: WorkspaceSettings, liveness: LivenessPolicy) -> ConnectOptions { + ConnectOptions { + target: ConnectTarget::File(self.endpoint.clone()), + workspace_settings: settings, + liveness, + environment: ConnectEnvironment::new("conformance"), + } + } + + fn scratch_table(&self, name: &str) -> TableRef { + TableRef::unqualified(name) + } + + fn create_scratch_table_sql(&self, table: &TableRef) -> String { + format!( + "CREATE TABLE {} (id integer primary key, label text not null)", + quote(&table.name) + ) + } + + /// SQLite has no sleep, but a recursive CTE without a stop never + /// produces its count. + fn unbounded_statement_sql(&self, marker: &str) -> String { + format!("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r) SELECT count(*), '{marker}' FROM r") + } + + /// Hold an exclusive lock on a connection of the engine's own for + /// `hold`, and return a read that waits for it in the busy handler. + async fn blocking_statement_sql(&self, marker: &str, hold: Duration) -> Result { + let options = SqliteConnectOptions::new().filename(&self.path); + let mut holder = SqliteConnection::connect_with(&options) + .await + .map_err(ConformanceFailure::setup)?; + holder + .execute("BEGIN EXCLUSIVE") + .await + .map_err(ConformanceFailure::setup)?; + tokio::spawn(async move { + tokio::time::sleep(hold).await; + let _ = holder.execute("ROLLBACK").await; + let _ = holder.close().await; + }); + Ok(format!("SELECT count(*), '{marker}' FROM sqlite_schema")) + } + + fn row_source_sql(&self, rows: u64) -> String { + format!("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < {rows}) SELECT n FROM r") + } +} + +impl PooledConformanceEngine for SqliteEngine { + const WORKSPACE_POOL_MAX_SIZE: usize = tablepro_driver_sqlite::WORKSPACE_POOL_MAX_SIZE; +} + +impl EditorTransactionEngine for SqliteEngine { + fn begin_sql(&self) -> String { + "BEGIN".to_owned() + } + + fn commit_sql(&self) -> String { + "COMMIT".to_owned() + } +} + +impl ConformanceDdlEngine for SqliteEngine { + fn integer_type(&self) -> &'static str { + "INTEGER" + } + + fn text_type(&self) -> &'static str { + "TEXT" + } + + fn timestamp_type(&self) -> &'static str { + "TEXT" + } + + fn current_timestamp_expression(&self) -> &'static str { + "CURRENT_TIMESTAMP" + } +} + +impl LargeTableEngine for SqliteEngine { + fn fill_scratch_table_sql(&self, table: &TableRef, rows: u64, label_width: usize) -> Vec { + vec![format!( + "WITH RECURSIVE numbers(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM numbers WHERE n < {rows}) \ + INSERT INTO {} (id, label) SELECT n, printf('%0{label_width}d', n) FROM numbers", + quote(&table.name) + )] + } +} diff --git a/linux/crates/drivers/sqlite/tests/integration/support.rs b/linux/crates/drivers/sqlite/tests/integration/support.rs index 85669ed933..ccc33401d3 100644 --- a/linux/crates/drivers/sqlite/tests/integration/support.rs +++ b/linux/crates/drivers/sqlite/tests/integration/support.rs @@ -6,8 +6,6 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use async_trait::async_trait; -use sqlx::Connection as _; -use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; use tablepro_core::meta::TableRef; use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; use tablepro_core::{ @@ -21,6 +19,8 @@ use tempfile::TempDir; use tokio::time::Instant; use tokio_util::sync::CancellationToken; +use crate::sqlite_engine::create_database; + /// A database file in a directory of its own, removed with it. pub struct TestFile { directory: TempDir, @@ -48,13 +48,6 @@ impl TestFile { } } -/// Make an empty database the way another program would, so the -/// driver only ever opens a file that exists. -pub async fn create_database(path: &Path) -> Result<(), sqlx::Error> { - let options = SqliteConnectOptions::new().filename(path).create_if_missing(true); - SqliteConnection::connect_with(&options).await?.close().await -} - pub fn writable() -> WorkspaceSettings { WorkspaceSettings::workspace(false) } diff --git a/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs b/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs index 8e68e8f70e..642736c93a 100644 --- a/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs +++ b/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs @@ -4,7 +4,7 @@ use tablepro_test_support::conformance::{CorpusColumn, ValueCorpusEngine}; -use crate::conformance_engine::SqliteEngine; +use crate::sqlite_engine::SqliteEngine; impl ValueCorpusEngine for SqliteEngine { fn value_corpus(&self) -> Vec { diff --git a/linux/crates/drivers/sqlite/tests/large_table.rs b/linux/crates/drivers/sqlite/tests/large_table.rs new file mode 100644 index 0000000000..e6e51b7e62 --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/large_table.rs @@ -0,0 +1,12 @@ +//! The large-table case on SQLite, a test binary of its own because it +//! bounds the process's resident peak. `cargo test` runs the tests of +//! one binary in one process, and the integration target's would count +//! toward the peak. Add no other test here. + +#[path = "integration/sqlite_engine.rs"] +mod sqlite_engine; + +tablepro_test_support::large_table_conformance_suite! { + engine: sqlite_engine::SqliteEngine, + docker: false, +} diff --git a/linux/crates/test-support/src/conformance/large_table_suite_macro.rs b/linux/crates/test-support/src/conformance/large_table_suite_macro.rs index 7066eafe5a..8102eaaf5b 100644 --- a/linux/crates/test-support/src/conformance/large_table_suite_macro.rs +++ b/linux/crates/test-support/src/conformance/large_table_suite_macro.rs @@ -1,23 +1,41 @@ /// Emit the large-table case for one engine adapter, which implements /// `LargeTableEngine`. It fills a table of its own, so it runs within -/// `LARGE_TABLE_CASE_LIMIT` rather than `CASE_LIMIT`, and only under -/// Docker: its memory bound reads the whole process's resident peak, -/// which tests running beside it in the unit suite's one process would -/// raise too. +/// `LARGE_TABLE_CASE_LIMIT` rather than `CASE_LIMIT`. `docker: true` +/// marks it `#[ignore = "requires docker"]`. +/// +/// Its memory bound reads the whole process's resident peak, so the case +/// needs a process to itself. nextest gives every test one. `cargo test` +/// runs the tests of one target in one process, so with `docker: false` +/// the case goes in a test target that holds nothing else. #[macro_export] macro_rules! large_table_conformance_suite { - (engine: $engine:ty $(,)?) => { + (engine: $engine:ty, docker: true $(,)?) => { #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires docker"] async fn a_million_row_table_pages_to_its_end_and_exports() -> ::core::result::Result<(), $crate::conformance::ConformanceFailure> { - $crate::conformance::run_case_within( - "a_million_row_table_pages_to_its_end_and_exports", - $crate::conformance::cases::large_table::LARGE_TABLE_CASE_LIMIT, - <$engine as $crate::conformance::ConformanceEngine>::start(), - $crate::conformance::cases::large_table::a_million_row_table_pages_to_its_end_and_exports::<$engine>, - ) - .await + $crate::__large_table_case!($engine) } }; + (engine: $engine:ty, docker: false $(,)?) => { + #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_million_row_table_pages_to_its_end_and_exports() + -> ::core::result::Result<(), $crate::conformance::ConformanceFailure> { + $crate::__large_table_case!($engine) + } + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __large_table_case { + ($engine:ty) => { + $crate::conformance::run_case_within( + "a_million_row_table_pages_to_its_end_and_exports", + $crate::conformance::cases::large_table::LARGE_TABLE_CASE_LIMIT, + <$engine as $crate::conformance::ConformanceEngine>::start(), + $crate::conformance::cases::large_table::a_million_row_table_pages_to_its_end_and_exports::<$engine>, + ) + .await + }; } diff --git a/linux/docs/adding-drivers.md b/linux/docs/adding-drivers.md index 30500b945e..9dcc43b481 100644 --- a/linux/docs/adding-drivers.md +++ b/linux/docs/adding-drivers.md @@ -357,7 +357,7 @@ The layout every driver follows: - One integration target with one module per topic, as `crates/drivers/postgres/tests/integration/main.rs`: `support.rs` (start a fixture, connect, run), `conformance_engine.rs`, `tls.rs` for a network engine, and the engine's own behaviour (`metadata.rs`, `write_batch.rs`, `cancellation.rs` and so on). A network engine's test starts its own container and is `#[ignore = "requires docker"]`. - A container fixture in `crates/test-fixtures` for anything the default image cannot do: TLS from the generated PKI, a second principal that logs in with a client certificate. -The conformance suites are the contract's own tests. The driver's `conformance_engine.rs` invokes the connection and DDL suites, plus the large-table suite for a network engine, its `value_corpus.rs` the value corpus, and a network engine's `tls.rs` the TLS suite. [testing.md](testing.md#conformance) has the adapter traits, the flags and the cases, with the [DDL](testing.md#ddl-conformance), [large-table](testing.md#large-table-conformance), [value corpus](testing.md#value-corpus) and [TLS](testing.md#tls-conformance) suites after them. +The conformance suites are the contract's own tests. The driver's `conformance_engine.rs` invokes the connection and DDL suites, plus the large-table suite for a network engine, its `value_corpus.rs` the value corpus, and a network engine's `tls.rs` the TLS suite. A file engine runs the large-table suite from a test target of its own, as SQLite's `tests/large_table.rs` does. [testing.md](testing.md#conformance) has the adapter traits, the flags and the cases, with the [DDL](testing.md#ddl-conformance), [large-table](testing.md#large-table-conformance), [value corpus](testing.md#value-corpus) and [TLS](testing.md#tls-conformance) suites after them. Behaviour the suites do not cover gets a test of its own against the real engine: the engine's stop, its error mapping, what a pooler does to it, each write outcome the ledger can produce. diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 033ae75294..1aa3a667bf 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -176,7 +176,9 @@ The PostgreSQL, MySQL, SQL Server, ClickHouse and SQLite targets run it, and the - It exports the table through the statement `build_export` writes, streamed by `stream_read` with no deadline into a CSV file in 64 KiB chunks, as the export dialog's Entire Table does. Every line of the file is checked. - The walk, each page and the export have time bounds, and where pages seek, the last pages may not take much longer than the first. Neither the walk nor the export may raise the process's resident peak (`VmHWM`, reset through `/proc/self/clear_refs`) past a memory bound. The bounds are constants in `crates/test-support/src/conformance/cases/large_table.rs`. -The PostgreSQL, MySQL, MariaDB, SQL Server and ClickHouse targets run it. The `ci` nextest profile prints its times and memory for a passing run, so the job log keeps what the bounds are set from. SQLite has no case: its suites run in the unit suite, where tests share one process, so the peak would count theirs too. +The peak is the whole process's, so the case needs a process to itself. The PostgreSQL, MySQL, MariaDB, SQL Server and ClickHouse targets pass `docker: true` and run under cargo-nextest, which gives each test a process of its own. The `ci` nextest profile prints its times and memory for a passing run, so the job log keeps what the bounds are set from. + +SQLite passes `docker: false`, so the case runs in the unit suite, where `cargo test` runs the tests of one binary in one process. It is a test target of its own, `crates/drivers/sqlite/tests/large_table.rs`, which holds nothing else and includes the adapter from `tests/integration/sqlite_engine.rs`. A passing run there prints nothing; `cargo test -p tablepro-driver-sqlite --test large_table -- --nocapture` shows the numbers. ### Value corpus From 2473ec280c97d84516d9ff97b4eea9ebece67f8e Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 19 Oct 2026 18:11:31 +0700 Subject: [PATCH 2/2] docs(linux): word the large-table docs for a file engine --- linux/ROADMAP.md | 2 +- .../crates/test-support/src/conformance/large_table_engine.rs | 4 ++-- linux/crates/test-support/src/conformance/resident_peak.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/linux/ROADMAP.md b/linux/ROADMAP.md index 13b174c756..67eed988ae 100644 --- a/linux/ROADMAP.md +++ b/linux/ROADMAP.md @@ -181,7 +181,7 @@ What has to hold before the first release, for a user on GNOME 50 with Ubuntu 26 | 1 | Installs `tablepro` from the PPA or the COPR project, and gets updates from the system's updater | CI builds both packages offline the way Launchpad and COPR do, and starts them in fresh containers | The release setup, the first tag, and an install then an upgrade from the PPA and COPR | | 2 | Connects to their everyday PostgreSQL, MySQL, MariaDB, SQL Server, ClickHouse or SQLite database | The supported releases and why others are out, in the README's [supported servers](README.md#supported-servers). CI runs every driver's Docker suite on each tested release, MySQL 8.0 and SQL Server 2017 and 2019 among them | Nothing | | 3 | Browses tables of real-world data (dates, decimals, JSON, UUIDs, NULLs) and sees correct values | Per-engine decoding with the type fallback and unsupported-type conformance cases ([ADR 0014](docs/decisions/0014-lossless-values-and-dialects.md)). A value corpus on every engine, MariaDB too: dates and times at their extremes and in other zones, decimals, JSON, UUIDs, Unicode and binary values, read back through a table tab, Export All and an editor run | Nothing | -| 4 | Runs typical queries against tables of a million rows or more without running out of memory or freezing | The 10,000-row editor limit with Export All, and driver tests over 2,000,000 rows. Pages seek past the edge row's key, or walk PostgreSQL heap block windows for a table with no key; OFFSET only under a user sort, a restored deep offset or a partitioned keyless table. Docker tests page 60k and 25k-row tables both ways and check the plan. A 1,000,000-row table is paged to its end and exported on PostgreSQL, MySQL, MariaDB, SQL Server, ClickHouse and SQLite, with bounds on time and on resident peak growth. SQLite's runs in the unit suite as a test binary of its own, so the peak is its alone | Nothing | +| 4 | Runs typical queries against tables of a million rows or more without running out of memory or freezing | The 10,000-row editor limit with Export All, and driver tests over 2,000,000 rows. Pages seek past the edge row's key, or walk PostgreSQL heap block windows for a table with no key; OFFSET only under a user sort, a restored deep offset or a partitioned keyless table. Docker tests page 60k and 25k-row tables both ways and check the plan. A 1,000,000-row table is paged to its end and exported on PostgreSQL, MySQL, MariaDB, SQL Server, ClickHouse and SQLite, with bounds on time and on resident peak growth. SQLite's case runs in the unit suite in a test binary of its own, so the peak is its alone | Nothing | | 5 | Edits data in place, and the changes commit correctly even when the connection or the app misbehaves | Write, row guard and commit resolution conformance on every engine, with the fault fakes and the grid save Docker test. `kill_during_save` kills TablePro halfway through a SQLite save: the file keeps none of it, and the next launch restores the session with its history, settings and saved tabs | A save into a SQLite file another program holds locked fails as `DriverError::Busy`, whose message says the server has no room for another connection | | 6 | Trusts the app with credentials: TLS, SSH tunnels, no password in plain text | Verify full by default, the TLS conformance suite, the Secret Service, and statement logging silenced | An outside security review | | 7 | Recovers when the network drops or the database restarts | Health monitoring, backoff and replay rules with their tests, a stopped container in the liveness tests, and a session test that restarts PostgreSQL: the read in flight runs again, the save in flight does not and leaves no row | Nothing | diff --git a/linux/crates/test-support/src/conformance/large_table_engine.rs b/linux/crates/test-support/src/conformance/large_table_engine.rs index 764125ca60..759ceb0d1e 100644 --- a/linux/crates/test-support/src/conformance/large_table_engine.rs +++ b/linux/crates/test-support/src/conformance/large_table_engine.rs @@ -2,8 +2,8 @@ use tablepro_core::TableRef; use crate::conformance::ConformanceEngine; -/// An engine whose server can fill a table with a million rows in -/// seconds, for the case that pages through one and exports it. +/// An engine that can fill a table with a million rows in seconds, for +/// the case that pages through one and exports it. pub trait LargeTableEngine: ConformanceEngine { /// Statements that fill the empty scratch `table` with `rows` rows: /// `id` from 1 to `rows`, and `label` the id left-padded with zeros diff --git a/linux/crates/test-support/src/conformance/resident_peak.rs b/linux/crates/test-support/src/conformance/resident_peak.rs index b4e7a5c983..1b2c528fdc 100644 --- a/linux/crates/test-support/src/conformance/resident_peak.rs +++ b/linux/crates/test-support/src/conformance/resident_peak.rs @@ -6,8 +6,8 @@ use crate::conformance::ConformanceFailure; /// the peak the kernel keeps for the process (`VmHWM`). /// /// The peak is the whole process's, so it means something only while no -/// other test runs in the process: nextest gives each test a process of -/// its own. +/// other test runs in the process: under nextest, which gives each test a +/// process of its own, or in a test binary that holds only the case. #[derive(Debug, Clone, Copy)] pub(crate) struct ResidentPeak { baseline: u64,