diff --git a/linux/crates/app/src/ui/export/export_producer.rs b/linux/crates/app/src/ui/export/export_producer.rs index df5881d999..727117035f 100644 --- a/linux/crates/app/src/ui/export/export_producer.rs +++ b/linux/crates/app/src/ui/export/export_producer.rs @@ -1,8 +1,8 @@ use std::sync::Arc; -use tablepro_core::browse::build_export; +use tablepro_core::browse::{DeclaredPrecisionSink, build_export}; use tablepro_core::column::ResultColumn; -use tablepro_core::{ReadStatement, ResultEvent, SinkFlow, StreamSummary, Truncation}; +use tablepro_core::{ReadStatement, ResultEvent, ResultSink, SinkFlow, StreamSummary, Truncation}; use tablepro_session::runtime::{TaskResult, Tasks}; use tablepro_session::{OperationError, OperationKind, Session}; use tokio_util::sync::CancellationToken; @@ -67,13 +67,14 @@ pub(crate) fn start_query_producer( let parts = parts.clone(); let query = Arc::clone(&query); async move { - let read = match &*query { - ExportQuery::Table(export) => { - ReadStatement::Generated(build_export(&*lease.dialect(), &export.request())?) - } - ExportQuery::Editor { sql } => ReadStatement::User(sql.clone()), + let (read, sink): (_, Box) = match &*query { + ExportQuery::Table(export) => ( + ReadStatement::Generated(build_export(&*lease.dialect(), &export.request())?), + Box::new(DeclaredPrecisionSink::new(parts.sink(), Arc::clone(&export.columns))), + ), + ExportQuery::Editor { sql } => (ReadStatement::User(sql.clone()), parts.sink()), }; - Ok(lease.stream_read(&read, parts.sink(), options).await?) + Ok(lease.stream_read(&read, sink, options).await?) } }) .await diff --git a/linux/crates/core/src/browse/declared_precision.rs b/linux/crates/core/src/browse/declared_precision.rs new file mode 100644 index 0000000000..0e448f0549 --- /dev/null +++ b/linux/crates/core/src/browse/declared_precision.rs @@ -0,0 +1,274 @@ +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::column::{ColumnInfo, ColumnKind, ResultColumn}; +use crate::query_result::QueryResult; +use crate::result_event::ResultEvent; +use crate::result_sink::{ResultSink, SinkFlow}; + +/// The page, with the fractional digits and bit lengths its columns' +/// catalogue entries declare where the result's own metadata names +/// none. +/// +/// MySQL's result metadata, as sqlx exposes it, carries neither a +/// DATETIME(6)'s digits nor a BIT(10)'s length, and the grid writes a +/// value at what its column declares. Only a column of the same kind is +/// refined: one the page projected as another type keeps the result's. +pub(super) fn with_declared_precision(result: QueryResult, catalog: &[ColumnInfo]) -> QueryResult { + match declared_columns(&result.columns, catalog) { + Some(columns) => QueryResult { columns, ..result }, + None => result, + } +} + +/// A table export's sink. Its columns take what the catalogue declares, +/// as a page's do, so the file holds what the grid showed. +pub struct DeclaredPrecisionSink { + inner: Box, + catalog: Arc<[ColumnInfo]>, +} + +impl DeclaredPrecisionSink { + pub fn new(inner: Box, catalog: Arc<[ColumnInfo]>) -> Self { + Self { inner, catalog } + } +} + +#[async_trait] +impl ResultSink for DeclaredPrecisionSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + let event = match event { + ResultEvent::ResultSetStarted { columns } => ResultEvent::ResultSetStarted { + columns: declared_columns(&columns, &self.catalog).unwrap_or(columns), + }, + ResultEvent::ColumnsResolved { set_index, columns } => ResultEvent::ColumnsResolved { + set_index, + columns: declared_columns(&columns, &self.catalog).unwrap_or(columns), + }, + other => other, + }; + self.inner.accept(event).await + } +} + +/// `None` when no column takes anything from the catalogue. +fn declared_columns(columns: &[ResultColumn], catalog: &[ColumnInfo]) -> Option> { + let refined: Vec<(usize, ColumnKind)> = columns + .iter() + .zip(catalog) + .enumerate() + .filter(|(_, (column, info))| column.name == info.name) + .filter_map(|(index, (column, info))| { + refined(&column.column_type.kind, &info.column_type.kind).map(|kind| (index, kind)) + }) + .collect(); + if refined.is_empty() { + return None; + } + let mut columns = columns.to_vec(); + for (index, kind) in refined { + if let Some(column) = columns.get_mut(index) { + column.column_type.kind = kind; + } + } + Some(columns.into()) +} + +fn refined(result: &ColumnKind, declared: &ColumnKind) -> Option { + match (result, declared) { + ( + ColumnKind::Timestamp { + fractional_digits: None, + zone, + }, + ColumnKind::Timestamp { + fractional_digits: Some(digits), + .. + }, + ) => Some(ColumnKind::Timestamp { + fractional_digits: Some(*digits), + zone: zone.clone(), + }), + ( + ColumnKind::Time { + fractional_digits: None, + with_offset, + range, + }, + ColumnKind::Time { + fractional_digits: Some(digits), + .. + }, + ) => Some(ColumnKind::Time { + fractional_digits: Some(*digits), + with_offset: *with_offset, + range: *range, + }), + ( + ColumnKind::BitString { length: None, varying }, + ColumnKind::BitString { + length: Some(length), .. + }, + ) => Some(ColumnKind::BitString { + length: Some(*length), + varying: *varying, + }), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + use crate::column::{ColumnDefault, ColumnType, ReadForm, TemporalZone, TextLength, TimeRange}; + use crate::value::Value; + + fn typed(kind: ColumnKind) -> ColumnType { + ColumnType { + name: "t".to_owned(), + kind, + catalog: None, + dynamic_storage: false, + read_form: ReadForm::Native, + } + } + + fn info(name: &str, kind: ColumnKind) -> ColumnInfo { + ColumnInfo { + name: name.to_owned(), + column_type: typed(kind), + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + } + } + + fn stamp(fractional_digits: Option, zone: TemporalZone) -> ColumnKind { + ColumnKind::Timestamp { + fractional_digits, + zone, + } + } + + fn bits(length: Option) -> ColumnKind { + ColumnKind::BitString { length, varying: false } + } + + fn result(columns: &[(&str, ColumnKind)]) -> QueryResult { + let columns: Vec = columns + .iter() + .map(|(name, kind)| ResultColumn::new(*name, typed(kind.clone()))) + .collect(); + QueryResult::new(columns, vec![vec![Value::Null; 2]]) + } + + fn kinds(result: &QueryResult) -> Vec { + result + .columns + .iter() + .map(|column| column.column_type.kind.clone()) + .collect() + } + + #[test] + fn a_result_without_digits_or_a_length_takes_the_catalogues() { + let read = result(&[("at", stamp(None, TemporalZone::Offset)), ("flag", bits(None))]); + let catalog = [ + info("at", stamp(Some(3), TemporalZone::Naive)), + info("flag", bits(Some(1))), + ]; + + let refined = with_declared_precision(read, &catalog); + + assert_eq!(kinds(&refined), [stamp(Some(3), TemporalZone::Offset), bits(Some(1))]); + assert_eq!(refined.rows, [vec![Value::Null; 2]]); + } + + #[test] + fn a_time_takes_the_digits_and_keeps_its_range() { + let time = |fractional_digits| ColumnKind::Time { + fractional_digits, + with_offset: false, + range: TimeRange::Duration, + }; + + let refined = with_declared_precision(result(&[("t", time(None))]), &[info("t", time(Some(6)))]); + + assert_eq!(kinds(&refined), [time(Some(6))]); + } + + #[test] + fn what_the_result_declares_or_projected_as_another_type_stays() { + let text = ColumnKind::Text { + length: TextLength::Unbounded, + }; + let enumeration = ColumnKind::Enum { + labels: vec!["a".to_owned()], + multiple: false, + }; + let read = result(&[("at", stamp(Some(6), TemporalZone::Naive)), ("mood", text.clone())]); + let catalog = [ + info("at", stamp(Some(3), TemporalZone::Naive)), + info("mood", enumeration), + ]; + + let refined = with_declared_precision(read, &catalog); + + assert_eq!(kinds(&refined), [stamp(Some(6), TemporalZone::Naive), text]); + } + + #[test] + fn a_column_under_another_name_is_not_the_catalogues() { + let read = result(&[("other", bits(None))]); + + let refined = with_declared_precision(read, &[info("flag", bits(Some(1)))]); + + assert_eq!(kinds(&refined), [bits(None)]); + } + + #[derive(Clone, Default)] + struct Seen(Arc>>); + + #[async_trait] + impl ResultSink for Seen { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.lock().expect("the events").push(event); + SinkFlow::Continue + } + } + + #[tokio::test] + async fn an_export_takes_the_catalogues_digits_for_every_result_set() { + let seen = Seen::default(); + let mut sink = DeclaredPrecisionSink::new(Box::new(seen.clone()), vec![info("flag", bits(Some(1)))].into()); + let read: Arc<[ResultColumn]> = vec![ResultColumn::new("flag", typed(bits(None)))].into(); + + sink.accept(ResultEvent::ResultSetStarted { columns: read.clone() }) + .await; + sink.accept(ResultEvent::ColumnsResolved { + set_index: 0, + columns: read, + }) + .await; + sink.accept(ResultEvent::Rows(vec![vec![Value::Null]])).await; + + let events = seen.0.lock().expect("the events"); + let columns: Vec = events + .iter() + .filter_map(|event| match event { + ResultEvent::ResultSetStarted { columns } | ResultEvent::ColumnsResolved { columns, .. } => { + Some(columns[0].column_type.kind.clone()) + } + _ => None, + }) + .collect(); + assert_eq!(columns, [bits(Some(1)), bits(Some(1))]); + assert!(matches!(events.last(), Some(ResultEvent::Rows(_)))); + } +} diff --git a/linux/crates/core/src/browse/mod.rs b/linux/crates/core/src/browse/mod.rs index f0d84166cf..56f49b134c 100644 --- a/linux/crates/core/src/browse/mod.rs +++ b/linux/crates/core/src/browse/mod.rs @@ -6,6 +6,7 @@ mod browse_sql_error; mod build_count; mod build_export; mod build_page; +mod declared_precision; mod export_request; mod page_layout; mod page_request; @@ -24,6 +25,7 @@ pub use browse_sql_error::BrowseSqlError; pub use build_count::build_count; pub use build_export::build_export; pub use build_page::build_page; +pub use declared_precision::DeclaredPrecisionSink; pub use export_request::ExportRequest; pub use page_layout::PageLayout; pub use page_request::PageRequest; diff --git a/linux/crates/core/src/browse/read_page.rs b/linux/crates/core/src/browse/read_page.rs index ab3e652219..ddb672a1b1 100644 --- a/linux/crates/core/src/browse/read_page.rs +++ b/linux/crates/core/src/browse/read_page.rs @@ -6,6 +6,7 @@ use crate::query_result::QueryResult; use crate::row_limit::RowLimit; use crate::value::Value; +use super::declared_precision::with_declared_precision; use super::split_page::key_of; use super::{BrowseSqlError, PageLayout, PageRequest, PageSpec, PagingMode, RowKeyLayout, build_page, paging_mode}; @@ -164,7 +165,7 @@ where .query_bound(&spec.statement, row_limit, options.clone()) .await?; check_shape(&result, &spec.layout)?; - Ok((result, spec)) + Ok((with_declared_precision(result, request.columns), spec)) } /// Fill a page from block windows, each read in ctid order, until it diff --git a/linux/crates/core/src/export/value_text.rs b/linux/crates/core/src/export/value_text.rs index 412ceb8677..7457fc6a22 100644 --- a/linux/crates/core/src/export/value_text.rs +++ b/linux/crates/core/src/export/value_text.rs @@ -1,7 +1,7 @@ use chrono::{Datelike, Timelike}; use crate::column::{ColumnKind, ColumnType}; -use crate::value::{SqlDecimal, Value}; +use crate::value::{BitString, SqlDecimal, Value}; /// Full text of a value for export, at the precision the column /// declares. @@ -42,11 +42,33 @@ pub(super) fn kind_text(value: &Value, kind: &ColumnKind) -> Option { ) }), Value::Decimal(decimal) => Some(decimal_text(decimal, kind.decimal_scale())), + Value::Bits(bits) => Some(bits_text(bits, kind)), Value::Array(values) => Some(array_text(values, Some(element_kind(kind)))), other => value_to_text(other), } } +/// A fixed bit string at the length its column declares. MySQL sends +/// a BIT(10) in the two bytes that hold it, so the six bits ahead of the +/// value are padding; bits that are set are never dropped. +fn bits_text(bits: &BitString, kind: &ColumnKind) -> String { + let text = bits.to_string(); + let ColumnKind::BitString { + length: Some(length), + varying: false, + } = kind + else { + return text; + }; + let padding = text + .len() + .saturating_sub(usize::try_from(*length).unwrap_or(usize::MAX)); + match text.get(..padding) { + Some(head) if head.bytes().all(|bit| bit == b'0') => text[padding..].to_owned(), + _ => text, + } +} + /// The kind each item of an array holds. A PostgreSQL `int4[][]` is /// declared once for every dimension, so a kind that is not itself an /// array passes down unchanged. @@ -558,4 +580,27 @@ mod tests { "without a column, a row of the array was written as a string" ); } + + #[test] + fn a_fixed_bit_string_shows_the_length_its_column_declares() { + let bit = |length| typed("bit", ColumnKind::BitString { length, varying: false }); + let sent_in_two_bytes = Value::Bits(BitString::from_u64(0b10_1010_1010, 16)); + let flag = Value::Bits(BitString::from_u64(1, 8)); + + assert_eq!( + value_text(&sent_in_two_bytes, &bit(Some(10))).as_deref(), + Some("1010101010") + ); + assert_eq!(value_text(&flag, &bit(Some(1))).as_deref(), Some("1")); + assert_eq!( + value_text(&flag, &bit(None)).as_deref(), + Some("00000001"), + "with no length declared, every bit the value holds shows" + ); + assert_eq!( + value_text(&Value::Bits(BitString::from_u64(0b1000_0001, 8)), &bit(Some(1))).as_deref(), + Some("10000001"), + "a set bit past the declared length was dropped" + ); + } } diff --git a/linux/crates/drivers/clickhouse/tests/integration/main.rs b/linux/crates/drivers/clickhouse/tests/integration/main.rs index c44c0fa5f6..35de2061c5 100644 --- a/linux/crates/drivers/clickhouse/tests/integration/main.rs +++ b/linux/crates/drivers/clickhouse/tests/integration/main.rs @@ -14,4 +14,5 @@ mod metadata; mod sessions; mod support; mod tls; +mod value_corpus; mod writes; diff --git a/linux/crates/drivers/clickhouse/tests/integration/value_corpus.rs b/linux/crates/drivers/clickhouse/tests/integration/value_corpus.rs new file mode 100644 index 0000000000..65b387a1bc --- /dev/null +++ b/linux/crates/drivers/clickhouse/tests/integration/value_corpus.rs @@ -0,0 +1,89 @@ +//! ClickHouse's value corpus: the types a real schema holds, at their +//! extremes, and the text the grid shows for each. + +use tablepro_test_support::conformance::{CorpusColumn, ValueCorpusEngine}; + +use crate::conformance_engine::ClickHouseEngine; + +impl ValueCorpusEngine for ClickHouseEngine { + fn value_corpus(&self) -> Vec { + vec![ + // A DateTime prints the wall time of its column's zone. + CorpusColumn::new("COLUMN v Nullable(DateTime('Asia/Kolkata'))") + .shows("'2024-03-10 02:30:00'", "2024-03-10 02:30:00") + .shows("'1970-01-01 05:30:00'", "1970-01-01 05:30:00") + .null(), + // Nanoseconds in an Int64 end in 2262. + CorpusColumn::new("COLUMN v DateTime64(9, 'UTC')") + .shows("'1900-01-01 00:00:00.000000001'", "1900-01-01 00:00:00.000000001") + .shows("'2262-04-11 23:47:16.854775807'", "2262-04-11 23:47:16.854775807") + .shows("'2024-02-29 12:00:00'", "2024-02-29 12:00:00.000000000"), + CorpusColumn::new("COLUMN v DateTime64(3)") + .shows("'2024-01-01 00:00:00.5'", "2024-01-01 00:00:00.500") + .shows("'2299-12-31 23:59:59.999'", "2299-12-31 23:59:59.999"), + CorpusColumn::new("COLUMN v Date") + .shows("'1970-01-01'", "1970-01-01") + .shows("'2149-06-06'", "2149-06-06"), + CorpusColumn::new("COLUMN v Date32") + .shows("'1900-01-01'", "1900-01-01") + .shows("'2299-12-31'", "2299-12-31"), + CorpusColumn::new("COLUMN v Decimal(76, 38)") + .shows( + "toDecimal256('-12345678901234567890123456789012345678.12345678901234567890123456789012345678', 38)", + "-12345678901234567890123456789012345678.12345678901234567890123456789012345678", + ) + .shows("0", "0.00000000000000000000000000000000000000"), + CorpusColumn::new("COLUMN v Nullable(Decimal(9, 2))") + .shows("-1.5", "-1.50") + .null(), + CorpusColumn::new("COLUMN v Int128").shows( + "-170141183460469231731687303715884105728", + "-170141183460469231731687303715884105728", + ), + CorpusColumn::new("COLUMN v UInt256").shows( + "'115792089237316195423570985008687907853269984665640564039457584007913129639935'", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ), + CorpusColumn::new("COLUMN v UInt64").shows("18446744073709551615", "18446744073709551615"), + CorpusColumn::new("COLUMN v Float64") + .shows("0.1", "0.1") + .shows("nan", "NaN") + .shows("-inf", "-Infinity"), + CorpusColumn::new("COLUMN v Float32").shows("1.5", "1.5"), + CorpusColumn::new("COLUMN v Bool").shows("true", "true").shows("false", "false"), + CorpusColumn::new("COLUMN v Nullable(UUID)") + .shows( + "'123e4567-e89b-12d3-a456-426614174000'", + "123e4567-e89b-12d3-a456-426614174000", + ) + .null(), + CorpusColumn::new("COLUMN v IPv4").shows("'192.168.0.1'", "192.168.0.1"), + CorpusColumn::new("COLUMN v IPv6").shows("'2001:db8::1'", "2001:db8::1"), + CorpusColumn::new("COLUMN v Enum8('a' = 1, 'b c' = 2)").shows("'b c'", "b c"), + CorpusColumn::new("COLUMN v Nullable(String)") + .shows("'naïve café'", "naïve café") + .shows("'日本語のテキスト'", "日本語のテキスト") + .shows("'🦀 emoji 👩‍💻'", "🦀 emoji 👩‍💻") + .shows("''", "") + .shows("'it\\'s'", "it's") + .shows("'line\\nbreak'", "line\nbreak") + .null(), + CorpusColumn::new("COLUMN v LowCardinality(String)").shows("'low'", "low"), + CorpusColumn::new("COLUMN v FixedString(4)").shows("'ab'", "ab\0\0"), + // A String holds bytes; ones that are not UTF-8 stay bytes. + CorpusColumn::new("COLUMN v String").shows("unhex('FF00FE')", "0xff00fe"), + CorpusColumn::new("COLUMN v Array(Nullable(Int32))").shows("[1, NULL, 3]", r#"["1",null,"3"]"#), + CorpusColumn::new("COLUMN v Map(String, UInt8)").shows("map('a', 1)", r#"{"a":1}"#), + CorpusColumn::new("COLUMN v Tuple(String, Int8)").shows("('x', -1)", r#"["x",-1]"#), + ] + } +} + +mod suite { + use crate::conformance_engine::ClickHouseEngine; + + tablepro_test_support::value_corpus_suite! { + engine: ClickHouseEngine, + docker: true, + } +} diff --git a/linux/crates/drivers/mssql/tests/integration/main.rs b/linux/crates/drivers/mssql/tests/integration/main.rs index 52031244b0..0d924af9a7 100644 --- a/linux/crates/drivers/mssql/tests/integration/main.rs +++ b/linux/crates/drivers/mssql/tests/integration/main.rs @@ -18,4 +18,5 @@ mod read_only; mod sessions; mod support; mod tls; +mod value_corpus; mod write_batch; diff --git a/linux/crates/drivers/mssql/tests/integration/value_corpus.rs b/linux/crates/drivers/mssql/tests/integration/value_corpus.rs new file mode 100644 index 0000000000..94ec342229 --- /dev/null +++ b/linux/crates/drivers/mssql/tests/integration/value_corpus.rs @@ -0,0 +1,124 @@ +//! SQL Server's value corpus: the types a real schema holds, at their +//! extremes, and the text the grid shows for each. +//! +//! tiberius describes a result column by its wire type alone, with no +//! scale, so an editor run shows the fraction a time value has. A +//! browse page takes the scale from the catalogue. + +use tablepro_test_support::conformance::{CorpusColumn, ValueCorpusEngine}; + +use crate::conformance_engine::MssqlEngine; + +impl ValueCorpusEngine for MssqlEngine { + fn value_corpus(&self) -> Vec { + vec![ + CorpusColumn::new("v datetime2(7)") + .shows_apart( + "'0001-01-01 00:00:00'", + "0001-01-01 00:00:00.0000000", + "0001-01-01 00:00:00", + ) + .shows("'9999-12-31 23:59:59.9999999'", "9999-12-31 23:59:59.9999999") + .shows_apart( + "'2024-02-29 12:00:00.5'", + "2024-02-29 12:00:00.5000000", + "2024-02-29 12:00:00.5", + ) + .null(), + // The offset a value was written with is the one it shows. The + // server keeps the instant in UTC, which has to stay in range. + CorpusColumn::new("v datetimeoffset(3)") + .shows_apart( + "'2024-03-10 02:30:00 -08:00'", + "2024-03-10 02:30:00.000-08:00", + "2024-03-10 02:30:00-08:00", + ) + .shows_apart( + "'0001-01-01 00:00:00 -14:00'", + "0001-01-01 00:00:00.000-14:00", + "0001-01-01 00:00:00-14:00", + ) + .shows("'9999-12-31 23:59:59.999 +14:00'", "9999-12-31 23:59:59.999+14:00"), + CorpusColumn::new("v datetime") + .shows_apart( + "'1753-01-01 00:00:00'", + "1753-01-01 00:00:00.000", + "1753-01-01 00:00:00", + ) + .shows("'9999-12-31 23:59:59.997'", "9999-12-31 23:59:59.997") + .shows("'2024-01-01 12:00:00.123'", "2024-01-01 12:00:00.123"), + CorpusColumn::new("v smalldatetime").shows("'2079-06-06 23:59:00'", "2079-06-06 23:59:00"), + CorpusColumn::new("v date") + .shows("'0001-01-01'", "0001-01-01") + .shows("'9999-12-31'", "9999-12-31"), + CorpusColumn::new("v time(7)") + .shows("'23:59:59.9999999'", "23:59:59.9999999") + .shows_apart("'00:00:00'", "00:00:00.0000000", "00:00:00"), + CorpusColumn::new("v decimal(38, 38)").shows( + "0.99999999999999999999999999999999999999", + "0.99999999999999999999999999999999999999", + ), + CorpusColumn::new("v decimal(38, 0)") + .shows( + "99999999999999999999999999999999999999", + "99999999999999999999999999999999999999", + ) + .shows( + "-99999999999999999999999999999999999999", + "-99999999999999999999999999999999999999", + ), + CorpusColumn::new("v decimal(10, 4)") + .shows("-1.5", "-1.5000") + .shows("0", "0.0000") + .null(), + // A table tab reads MONEY cast to a decimal. tiberius hands an + // editor run a float, exact only to about 5.5 * 10^11, past + // which the cell says it cannot read the value. + CorpusColumn::new("v money") + .shows("-12.34", "-12.3400") + .shows("123456789.1234", "123456789.1234") + .shows_apart("922337203685477.5807", "922337203685477.5807", "") + .shows_apart("-922337203685477.5808", "-922337203685477.5808", ""), + CorpusColumn::new("v smallmoney").shows("-214748.3648", "-214748.3648"), + // SQL Server keeps the first three groups little-endian, so a + // reader that takes the bytes in order scrambles them. + CorpusColumn::new("v uniqueidentifier") + .shows( + "'6F9619FF-8B86-D011-B42D-00C04FC964FF'", + "6f9619ff-8b86-d011-b42d-00c04fc964ff", + ) + .null(), + CorpusColumn::new("v nvarchar(100)") + .shows("N'naïve café'", "naïve café") + .shows("N'日本語のテキスト'", "日本語のテキスト") + .shows("N'🦀 emoji 👩‍💻'", "🦀 emoji 👩‍💻") + .shows("N''", "") + .shows("N'it''s'", "it's") + .null(), + CorpusColumn::new("v varchar(50) COLLATE Latin1_General_CI_AS").shows("'café'", "café"), + CorpusColumn::new("v nchar(5)").shows("N'ab'", "ab "), + CorpusColumn::new("v nvarchar(max)").shows("N'line' + NCHAR(10) + N'break'", "line\nbreak"), + CorpusColumn::new("v varbinary(max)") + .shows("0xDEADBEEF", "0xdeadbeef") + .shows("0x", "0x"), + CorpusColumn::new("v binary(4)").shows("0x01", "0x01000000"), + CorpusColumn::new("v bit").shows("1", "true").shows("0", "false").null(), + CorpusColumn::new("v tinyint").shows("255", "255"), + CorpusColumn::new("v bigint").shows("-9223372036854775808", "-9223372036854775808"), + CorpusColumn::new("v real").shows("1.5", "1.5").shows("-0.1", "-0.1"), + CorpusColumn::new("v float") + .shows("0.1", "0.1") + .shows("-2.5e-5", "-0.000025"), + CorpusColumn::new("v xml").shows("'1'", "1"), + ] + } +} + +mod suite { + use crate::conformance_engine::MssqlEngine; + + tablepro_test_support::value_corpus_suite! { + engine: MssqlEngine, + docker: true, + } +} diff --git a/linux/crates/drivers/mysql/tests/integration/main.rs b/linux/crates/drivers/mysql/tests/integration/main.rs index d4b9732f3a..e94b3b6a40 100644 --- a/linux/crates/drivers/mysql/tests/integration/main.rs +++ b/linux/crates/drivers/mysql/tests/integration/main.rs @@ -19,4 +19,5 @@ mod row_limit; mod server_time_zone; mod support; mod tls; +mod value_corpus; mod write_batch; diff --git a/linux/crates/drivers/mysql/tests/integration/value_corpus.rs b/linux/crates/drivers/mysql/tests/integration/value_corpus.rs new file mode 100644 index 0000000000..d99389fa7a --- /dev/null +++ b/linux/crates/drivers/mysql/tests/integration/value_corpus.rs @@ -0,0 +1,149 @@ +//! The value corpus on MySQL and MariaDB: the types a real schema holds, +//! at their extremes, and the text the grid shows for each. +//! +//! sqlx gives a result column no fractional digits and no BIT length, +//! so an editor run shows the fraction a value has and a BIT in the +//! whole bytes it arrived in. A browse page takes both from the +//! catalogue. + +use tablepro_test_fixtures::MysqlFlavour; +use tablepro_test_support::conformance::{CorpusColumn, ValueCorpusEngine}; + +use crate::conformance_engine::{MySqlEngine, Server}; + +impl ValueCorpusEngine for MySqlEngine { + fn value_corpus(&self) -> Vec { + let flavoured = match S::FLAVOUR { + // MySQL keeps JSON in a binary form of its own, so it prints + // the document again, keys sorted and numbers normalised, as + // the mysql client shows it. + MysqlFlavour::Mysql => vec![ + CorpusColumn::new("v JSON") + .shows( + r#"'{"b": 1, "a": [1, 2.50, "x"], "ü": null}'"#, + r#"{"a": [1, 2.5, "x"], "b": 1, "ü": null}"#, + ) + .shows("'[]'", "[]") + .null(), + CorpusColumn::new("v BINARY(16)").shows( + "UUID_TO_BIN('123e4567-e89b-12d3-a456-426614174000')", + "0x123e4567e89b12d3a456426614174000", + ), + ], + // MariaDB's JSON is LONGTEXT with a check, kept as written. + MysqlFlavour::Mariadb => vec![ + CorpusColumn::new("v JSON") + .shows(r#"'{"b": 1, "a": [1, 2.50, "x"]}'"#, r#"{"b": 1, "a": [1, 2.50, "x"]}"#) + .null(), + CorpusColumn::new("v UUID") + .shows( + "'123e4567-e89b-12d3-a456-426614174000'", + "123e4567-e89b-12d3-a456-426614174000", + ) + .null(), + CorpusColumn::new("v INET6").shows("'::ffff:192.0.2.1'", "::ffff:192.0.2.1"), + ], + }; + [common(), flavoured].concat() + } +} + +fn common() -> Vec { + vec![ + CorpusColumn::new("v DATETIME(6)") + .shows_apart( + "'1000-01-01 00:00:00'", + "1000-01-01 00:00:00.000000", + "1000-01-01 00:00:00", + ) + .shows("'9999-12-31 23:59:59.999999'", "9999-12-31 23:59:59.999999") + .shows_apart( + "'2024-02-29 12:00:00.5'", + "2024-02-29 12:00:00.500000", + "2024-02-29 12:00:00.5", + ) + .null(), + CorpusColumn::new("v DATETIME").shows("'2024-03-10 02:30:00'", "2024-03-10 02:30:00"), + // Browse pins its session to UTC and reads an instant; the + // editor reads the wall time in the server's zone, UTC here. + CorpusColumn::new("v TIMESTAMP(3) NULL") + .shows_apart( + "'1970-01-01 00:00:01'", + "1970-01-01 00:00:01.000+00:00", + "1970-01-01 00:00:01", + ) + .shows_apart( + "'2038-01-19 03:14:07.999'", + "2038-01-19 03:14:07.999+00:00", + "2038-01-19 03:14:07.999", + ) + .null(), + CorpusColumn::new("v DATE") + .shows("'1000-01-01'", "1000-01-01") + .shows("'9999-12-31'", "9999-12-31") + .shows("'2024-02-29'", "2024-02-29"), + CorpusColumn::new("v TIME(6)") + .shows_apart("'-838:59:59'", "-838:59:59.000000", "-838:59:59") + .shows_apart("'838:59:59'", "838:59:59.000000", "838:59:59") + .shows("'00:00:00.000001'", "00:00:00.000001"), + CorpusColumn::new("v YEAR").shows("1901", "1901").shows("2155", "2155"), + CorpusColumn::new("v DECIMAL(65, 30)") + .shows( + "12345678901234567890123456789012345.123456789012345678901234567890", + "12345678901234567890123456789012345.123456789012345678901234567890", + ) + .shows("-0.000000000000000000000000000001", "-0.000000000000000000000000000001") + .shows("0", "0.000000000000000000000000000000"), + CorpusColumn::new("v DECIMAL(10, 2)") + .shows("-12345678.9", "-12345678.90") + .shows("0.5", "0.50") + .null(), + CorpusColumn::new("v BIGINT UNSIGNED").shows("18446744073709551615", "18446744073709551615"), + CorpusColumn::new("v BIGINT").shows("-9223372036854775808", "-9223372036854775808"), + CorpusColumn::new("v TINYINT UNSIGNED").shows("255", "255"), + CorpusColumn::new("v FLOAT").shows("1.5", "1.5").shows("-0.25", "-0.25"), + CorpusColumn::new("v DOUBLE") + .shows("0.1", "0.1") + .shows("-2.5e-5", "-0.000025"), + CorpusColumn::new("v VARCHAR(100)") + .shows("'naïve café'", "naïve café") + .shows("'日本語のテキスト'", "日本語のテキスト") + .shows("'🦀 emoji 👩‍💻'", "🦀 emoji 👩‍💻") + .shows("''", "") + .shows("'it''s'", "it's") + .shows("'back\\\\slash'", "back\\slash") + .null(), + CorpusColumn::new("v VARCHAR(20) CHARACTER SET latin1").shows("'café'", "café"), + CorpusColumn::new("v CHAR(5)").shows("'ab'", "ab"), + CorpusColumn::new("v TEXT").shows("'line\\nbreak'", "line\nbreak"), + CorpusColumn::new("v VARBINARY(8)") + .shows("X'DEADBEEF'", "0xdeadbeef") + .shows("X''", "0x"), + CorpusColumn::new("v BINARY(4)").shows("X'01'", "0x01000000"), + CorpusColumn::new("v BLOB").shows("X'00FF'", "0x00ff"), + CorpusColumn::new("v BIT(10)").shows_apart("b'1010101010'", "1010101010", "0000001010101010"), + CorpusColumn::new("v BIT(1)") + .shows_apart("b'1'", "1", "00000001") + .shows_apart("b'0'", "0", "00000000"), + CorpusColumn::new("v ENUM('a', 'b c')").shows("'b c'", "b c").null(), + CorpusColumn::new("v SET('x', 'y', 'z')").shows("'x,z'", "x,z"), + ] +} + +mod mysql_suite { + use crate::conformance_engine::{MySqlEngine, MysqlServer}; + + tablepro_test_support::value_corpus_suite! { + engine: MySqlEngine, + docker: true, + } +} + +mod mariadb_suite { + use crate::conformance_engine::{MariadbServer, MySqlEngine}; + + tablepro_test_support::value_corpus_suite! { + engine: MySqlEngine, + docker: true, + } +} diff --git a/linux/crates/drivers/postgres/src/codec/decode.rs b/linux/crates/drivers/postgres/src/codec/decode.rs index 0e8c959431..2e66aedacd 100644 --- a/linux/crates/drivers/postgres/src/codec/decode.rs +++ b/linux/crates/drivers/postgres/src/codec/decode.rs @@ -11,6 +11,7 @@ use tablepro_core::value::{ use super::WireError; use super::array_decode::decode_array; +use super::far_temporal; use super::interval_wire::interval_from_sql; use super::network_text::{inet_text, macaddr_text, macaddr8_text}; use super::numeric_wire::numeric_from_sql; @@ -120,9 +121,9 @@ fn decode(ty: &Type, raw: &[u8]) -> Result { Some((&JSONB_VERSION, document)) => json(document)?, _ => return Err(WireError::Malformed.into()), }, - oid::DATE => Value::Date(date(ty, raw)?), - oid::TIMESTAMP => Value::Timestamp(timestamp(ty, raw)?), - oid::TIMESTAMPTZ => Value::TimestampTz(timestamptz(ty, raw)?), + oid::DATE => date(ty, raw)?, + oid::TIMESTAMP => timestamp(ty, raw)?, + oid::TIMESTAMPTZ => timestamptz(ty, raw)?, oid::TIME => Value::Time(time_from_sql(raw)?), oid::TIMETZ => Value::TimeTz(timetz_from_sql(raw)?), oid::INTERVAL => Value::Interval(interval_from_sql(raw)?), @@ -188,34 +189,48 @@ fn bits(raw: &[u8]) -> Result { } /// postgres-types reads the infinities and refuses a date chrono cannot -/// hold. A well-formed value it refuses is therefore out of range. -fn date(ty: &Type, raw: &[u8]) -> Result, WireError> { +/// hold, one past 262143 AD. The server's run to 5874897 AD, so those +/// keep the server's text. +fn date(ty: &Type, raw: &[u8]) -> Result { expect_len(raw, 4)?; - match Date::::from_sql(ty, raw).map_err(|_| WireError::OutOfRange)? { - Date::Value(date) => Ok(Temporal::Finite(date)), - Date::PosInfinity => Ok(Temporal::Infinity), - Date::NegInfinity => Ok(Temporal::NegInfinity), - } + Ok(match Date::::from_sql(ty, raw) { + Ok(Date::Value(date)) => Value::Date(Temporal::Finite(date)), + Ok(Date::PosInfinity) => Value::Date(Temporal::Infinity), + Ok(Date::NegInfinity) => Value::Date(Temporal::NegInfinity), + Err(_) => other( + ty, + far_temporal::date_text(wire::date_from_sql(raw).map_err(malformed)?).ok_or(WireError::OutOfRange)?, + ), + }) } -fn timestamp(ty: &Type, raw: &[u8]) -> Result, WireError> { +/// A timestamp past chrono's last year, as `date` reads a date. +fn timestamp(ty: &Type, raw: &[u8]) -> Result { expect_len(raw, 8)?; - match Timestamp::::from_sql(ty, raw).map_err(|_| WireError::OutOfRange)? { - Timestamp::Value(stamp) => Ok(Temporal::Finite(stamp)), - Timestamp::PosInfinity => Ok(Temporal::Infinity), - Timestamp::NegInfinity => Ok(Temporal::NegInfinity), - } + Ok(match Timestamp::::from_sql(ty, raw) { + Ok(Timestamp::Value(stamp)) => Value::Timestamp(Temporal::Finite(stamp)), + Ok(Timestamp::PosInfinity) => Value::Timestamp(Temporal::Infinity), + Ok(Timestamp::NegInfinity) => Value::Timestamp(Temporal::NegInfinity), + Err(_) => other(ty, far_timestamp(raw)?), + }) } /// A TIMESTAMPTZ is an instant; the server sends it in UTC, so the offset /// it carries here is UTC's. -fn timestamptz(ty: &Type, raw: &[u8]) -> Result, WireError> { +fn timestamptz(ty: &Type, raw: &[u8]) -> Result { expect_len(raw, 8)?; - match Timestamp::>::from_sql(ty, raw).map_err(|_| WireError::OutOfRange)? { - Timestamp::Value(stamp) => Ok(Temporal::Finite(OffsetTimestamp::from_datetime(stamp.fixed_offset()))), - Timestamp::PosInfinity => Ok(Temporal::Infinity), - Timestamp::NegInfinity => Ok(Temporal::NegInfinity), - } + Ok(match Timestamp::>::from_sql(ty, raw) { + Ok(Timestamp::Value(stamp)) => { + Value::TimestampTz(Temporal::Finite(OffsetTimestamp::from_datetime(stamp.fixed_offset()))) + } + Ok(Timestamp::PosInfinity) => Value::TimestampTz(Temporal::Infinity), + Ok(Timestamp::NegInfinity) => Value::TimestampTz(Temporal::NegInfinity), + Err(_) => other(ty, format!("{}+00:00", far_timestamp(raw)?)), + }) +} + +fn far_timestamp(raw: &[u8]) -> Result { + far_temporal::timestamp_text(wire::timestamp_from_sql(raw).map_err(malformed)?).ok_or(WireError::OutOfRange) } fn expect_len(raw: &[u8], len: usize) -> Result<(), WireError> { @@ -356,6 +371,32 @@ mod tests { ); } + #[test] + fn dates_and_timestamps_past_chrono_keep_the_servers_text() { + let last_date = encoded(2_145_031_948i32, &Type::INT4); + let last_stamp = encoded(9_223_371_331_199_999_999i64, &Type::INT8); + + let text = |value: Value| match value { + Value::Other(other) => (other.type_name, other.text), + other => panic!("not kept as text: {other:?}"), + }; + assert_eq!( + text(decode_binary(&Type::DATE, &last_date)), + ("date".to_owned(), "5874897-12-31".to_owned()) + ); + assert_eq!( + text(decode_binary(&Type::TIMESTAMP, &last_stamp)), + ("timestamp".to_owned(), "294276-12-31 23:59:59.999999".to_owned()) + ); + assert_eq!( + text(decode_binary(&Type::TIMESTAMPTZ, &last_stamp)), + ( + "timestamptz".to_owned(), + "294276-12-31 23:59:59.999999+00:00".to_owned() + ) + ); + } + #[test] fn timestamp_overflow_is_out_of_range() { let far = encoded(i64::MAX - 1, &Type::INT8); diff --git a/linux/crates/drivers/postgres/src/codec/far_temporal.rs b/linux/crates/drivers/postgres/src/codec/far_temporal.rs new file mode 100644 index 0000000000..d22db54163 --- /dev/null +++ b/linux/crates/drivers/postgres/src/codec/far_temporal.rs @@ -0,0 +1,92 @@ +//! Dates and timestamps past 262143 AD, the last year chrono holds. +//! PostgreSQL keeps dates to 5874897 AD and timestamps to 294276 AD, so +//! a value out there is written from its days and microseconds here, as +//! the grid writes one chrono holds. + +/// From 1970-01-01, where the civil-date arithmetic counts, to +/// 2000-01-01, where PostgreSQL counts. +const DAYS_1970_TO_2000: i64 = 10_957; +const MICROS_PER_DAY: i64 = 86_400_000_000; +const MICROS_PER_SECOND: i64 = 1_000_000; +/// 5874898-01-01 and 294277-01-01, the first date and timestamp the +/// server refuses. Anything at or past them is not a value it sent. +const END_DATE: i32 = 2_145_031_949; +const END_TIMESTAMP: i64 = 9_223_371_331_200_000_000; + +/// A DATE's binary form: days since 2000-01-01. +pub(crate) fn date_text(days_since_2000: i32) -> Option { + if days_since_2000 >= END_DATE { + return None; + } + let (year, month, day) = civil_date(i64::from(days_since_2000) + DAYS_1970_TO_2000); + Some(format!("{year:04}-{month:02}-{day:02}")) +} + +/// A TIMESTAMP's binary form: microseconds since 2000-01-01 00:00:00. +pub(crate) fn timestamp_text(micros_since_2000: i64) -> Option { + if micros_since_2000 >= END_TIMESTAMP { + return None; + } + let days = micros_since_2000.div_euclid(MICROS_PER_DAY); + let micros = micros_since_2000.rem_euclid(MICROS_PER_DAY); + let (year, month, day) = civil_date(days + DAYS_1970_TO_2000); + let seconds = micros / MICROS_PER_SECOND; + let mut text = format!( + "{year:04}-{month:02}-{day:02} {:02}:{:02}:{:02}", + seconds / 3600, + seconds / 60 % 60, + seconds % 60 + ); + let fraction = format!("{:06}", micros % MICROS_PER_SECOND); + let fraction = fraction.trim_end_matches('0'); + if !fraction.is_empty() { + text.push('.'); + text.push_str(fraction); + } + Some(text) +} + +/// The proleptic Gregorian year, month and day `days` after 1970-01-01, +/// by Howard Hinnant's `civil_from_days`, which has no range limit of +/// its own. +fn civil_date(days: i64) -> (i64, i64, i64) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_last_date_and_timestamp_postgresql_holds() { + assert_eq!(date_text(END_DATE - 1).as_deref(), Some("5874897-12-31")); + assert_eq!( + timestamp_text(END_TIMESTAMP - 1).as_deref(), + Some("294276-12-31 23:59:59.999999") + ); + assert_eq!(date_text(END_DATE), None); + assert_eq!(timestamp_text(END_TIMESTAMP), None); + } + + #[test] + fn days_around_the_epoch_match_the_calendar() { + assert_eq!(date_text(0).as_deref(), Some("2000-01-01")); + assert_eq!(date_text(-1).as_deref(), Some("1999-12-31")); + assert_eq!(date_text(59).as_deref(), Some("2000-02-29")); + assert_eq!(timestamp_text(-1).as_deref(), Some("1999-12-31 23:59:59.999999")); + assert_eq!(timestamp_text(1_500_000).as_deref(), Some("2000-01-01 00:00:01.5")); + } +} diff --git a/linux/crates/drivers/postgres/src/codec/mod.rs b/linux/crates/drivers/postgres/src/codec/mod.rs index 7214d2f499..091b0fe874 100644 --- a/linux/crates/drivers/postgres/src/codec/mod.rs +++ b/linux/crates/drivers/postgres/src/codec/mod.rs @@ -2,6 +2,7 @@ mod array_decode; mod decode; +mod far_temporal; mod interval_wire; mod network_text; mod numeric_wire; diff --git a/linux/crates/drivers/postgres/tests/integration/main.rs b/linux/crates/drivers/postgres/tests/integration/main.rs index 0ea20bbb19..a878fb62f2 100644 --- a/linux/crates/drivers/postgres/tests/integration/main.rs +++ b/linux/crates/drivers/postgres/tests/integration/main.rs @@ -22,4 +22,5 @@ mod read_only; mod slow_route; mod support; mod tls; +mod value_corpus; mod write_batch; diff --git a/linux/crates/drivers/postgres/tests/integration/value_corpus.rs b/linux/crates/drivers/postgres/tests/integration/value_corpus.rs new file mode 100644 index 0000000000..41db732fd5 --- /dev/null +++ b/linux/crates/drivers/postgres/tests/integration/value_corpus.rs @@ -0,0 +1,205 @@ +//! PostgreSQL's value corpus: the types a real schema holds, at their +//! extremes, and the text the grid shows for each. +//! +//! A browse page reads values in binary and the grid writes them out; an +//! editor run uses the simple query protocol, so it shows the server's +//! own text, as psql does. Where the two spell a value differently, the +//! value names both. + +use tablepro_test_support::conformance::{CorpusColumn, ValueCorpusEngine}; + +use crate::conformance_engine::PgEngine; + +impl ValueCorpusEngine for PgEngine { + fn value_corpus(&self) -> Vec { + [temporal(), numbers(), documents(), other_types()].concat() + } +} + +fn temporal() -> Vec { + vec![ + // An instant reads back in UTC whatever offset it was written + // with; the server's zone is UTC too. + CorpusColumn::new("v timestamptz") + .shows_apart( + "'2024-03-10 02:30:00-08'", + "2024-03-10 10:30:00+00:00", + "2024-03-10 10:30:00+00", + ) + .shows_apart( + "'2024-06-15 12:00:00.5+05:45'", + "2024-06-15 06:15:00.5+00:00", + "2024-06-15 06:15:00.5+00", + ) + .shows_apart( + "'1969-12-31 23:59:59.999999+00'", + "1969-12-31 23:59:59.999999+00:00", + "1969-12-31 23:59:59.999999+00", + ) + .shows_apart( + "'0001-01-01 00:00:00+00'", + "0001-01-01 00:00:00+00:00", + "0001-01-01 00:00:00+00", + ) + .shows_apart( + "'4713-01-01 00:00:00+00 BC'", + "4713-01-01 00:00:00+00:00 BC", + "4713-01-01 00:00:00+00 BC", + ) + .shows("'infinity'", "infinity") + .shows("'-infinity'", "-infinity") + .null(), + CorpusColumn::new("v timestamp(3)") + .shows("'2024-02-29 23:59:59.999'", "2024-02-29 23:59:59.999") + .shows_apart( + "'2000-01-01 00:00:00'", + "2000-01-01 00:00:00.000", + "2000-01-01 00:00:00", + ) + .shows("'1900-01-01 00:00:00.001'", "1900-01-01 00:00:00.001"), + CorpusColumn::new("v timestamp") + .shows("'294276-12-31 23:59:59.999999'", "294276-12-31 23:59:59.999999") + .shows("'1999-12-31 23:59:59.5'", "1999-12-31 23:59:59.5"), + CorpusColumn::new("v date") + .shows("'2024-02-29'", "2024-02-29") + .shows("'0001-01-01'", "0001-01-01") + .shows("'4713-01-01 BC'", "4713-01-01 BC") + .shows("'5874897-12-31'", "5874897-12-31") + .shows("'infinity'", "infinity") + .null(), + CorpusColumn::new("v time(6)") + .shows_apart("'00:00:00'", "00:00:00.000000", "00:00:00") + .shows("'23:59:59.999999'", "23:59:59.999999") + .shows_apart("'24:00:00'", "24:00:00.000000", "24:00:00"), + CorpusColumn::new("v timetz") + .shows("'12:34:56+05:30'", "12:34:56+05:30") + .shows("'00:00:00.25-15:59'", "00:00:00.25-15:59"), + CorpusColumn::new("v interval") + .shows_apart( + "'1 year 2 mons 3 days 04:05:06.789'", + "P1Y2M3DT4H5M6.789S", + "1 year 2 mons 3 days 04:05:06.789", + ) + .shows_apart("'-1 day +02:00:00'", "P-1DT2H", "-1 days +02:00:00") + .shows_apart("'0'", "PT0S", "00:00:00"), + ] +} + +fn numbers() -> Vec { + vec![ + CorpusColumn::new("v numeric(38, 10)") + .shows("12345678901234567890.1234567890", "12345678901234567890.1234567890") + .shows("-0.0000000001", "-0.0000000001") + .shows("1.5", "1.5000000000") + .shows("0", "0.0000000000") + .null(), + CorpusColumn::new("v numeric") + .shows( + "-123456789012345678901234567890.123456789", + "-123456789012345678901234567890.123456789", + ) + .shows("0.000000000000000000000000000001", "0.000000000000000000000000000001") + .shows("2.50", "2.50") + .shows("'NaN'", "NaN") + .shows("'Infinity'", "Infinity") + .shows("'-Infinity'", "-Infinity"), + CorpusColumn::new("v money") + .shows_apart("-12.34", "-12.34", "-$12.34") + .shows_apart( + "92233720368547758.07", + "92233720368547758.07", + "$92,233,720,368,547,758.07", + ), + CorpusColumn::new("v bigint") + .shows("-9223372036854775808", "-9223372036854775808") + .shows("9223372036854775807", "9223372036854775807"), + CorpusColumn::new("v smallint").shows("-32768", "-32768").null(), + CorpusColumn::new("v real") + .shows("1.5", "1.5") + .shows("-0.1", "-0.1") + .shows("'NaN'", "NaN") + .shows("'-Infinity'", "-Infinity"), + CorpusColumn::new("v double precision") + .shows("0.1", "0.1") + .shows_apart("-2.5e-5", "-0.000025", "-2.5e-05") + .shows("'Infinity'", "Infinity"), + CorpusColumn::new("v boolean") + .shows_apart("true", "true", "t") + .shows_apart("false", "false", "f") + .null(), + ] +} + +fn documents() -> Vec { + vec![ + CorpusColumn::new("v jsonb") + .shows( + r#"'{"b": 1, "a": [1, 2.50, "x"], "ü": null}'"#, + r#"{"a": [1, 2.50, "x"], "b": 1, "ü": null}"#, + ) + .shows("'[]'", "[]") + .shows(r#"'"just text"'"#, r#""just text""#), + CorpusColumn::new("v json").shows(r#"'{"b":1, "a":2}'"#, r#"{"b":1, "a":2}"#), + CorpusColumn::new("v uuid") + .shows( + "'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", + ) + .shows( + "'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A12'", + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12", + ) + .null(), + CorpusColumn::new("v text") + .shows("'naïve café'", "naïve café") + .shows("'日本語のテキスト'", "日本語のテキスト") + .shows("'🦀 emoji 👩‍💻'", "🦀 emoji 👩‍💻") + .shows("'עברית RTL'", "עברית RTL") + .shows("'e\u{301}'", "e\u{301}") + .shows("''", "") + .shows("E'tab\\there\\nnew line'", "tab\there\nnew line") + .shows("'it''s'", "it's") + .null(), + CorpusColumn::new("v char(5)").shows("'ab'", "ab "), + CorpusColumn::new("v varchar(10)").shows("'ünïcödé'", "ünïcödé"), + CorpusColumn::new("v bytea") + .shows_apart("'\\xdeadbeef'", "0xdeadbeef", "\\xdeadbeef") + .shows_apart("''", "0x", "\\x") + .null(), + CorpusColumn::new("v integer[]").shows_apart("'{1,NULL,3}'", r#"["1",null,"3"]"#, "{1,NULL,3}"), + CorpusColumn::new("v text[]").shows_apart( + r#"'{"a b","c,d",NULL,""}'"#, + r#"["a b","c,d",null,""]"#, + r#"{"a b","c,d",NULL,""}"#, + ), + ] +} + +fn other_types() -> Vec { + vec![ + CorpusColumn::new("v inet") + .shows("'192.168.0.1/24'", "192.168.0.1/24") + .shows("'::1'", "::1"), + CorpusColumn::new("v cidr").shows("'10.0.0.0/8'", "10.0.0.0/8"), + CorpusColumn::new("v macaddr").shows("'08:00:2B:01:02:03'", "08:00:2b:01:02:03"), + CorpusColumn::new("v corpus_mood") + .after("CREATE TYPE corpus_mood AS ENUM ('sad', 'ok', 'happy')") + .shows("'happy'", "happy"), + CorpusColumn::new("v int4range") + .shows("'[1,10)'", "[1,10)") + .shows("'empty'", "empty"), + CorpusColumn::new("v bit(4)").shows("B'1010'", "1010"), + CorpusColumn::new("v varbit(8)").shows("B'101'", "101"), + CorpusColumn::new("v point").shows("'(1.5,-2)'", "(1.5,-2)"), + CorpusColumn::new("v xml").shows("'1'", "1"), + ] +} + +mod suite { + use crate::conformance_engine::PgEngine; + + tablepro_test_support::value_corpus_suite! { + engine: PgEngine, + docker: true, + } +} diff --git a/linux/crates/drivers/sqlite/tests/integration/main.rs b/linux/crates/drivers/sqlite/tests/integration/main.rs index e023f9d73a..1aa23b796c 100644 --- a/linux/crates/drivers/sqlite/tests/integration/main.rs +++ b/linux/crates/drivers/sqlite/tests/integration/main.rs @@ -24,5 +24,6 @@ mod open_semantics; mod sessions; #[cfg(test)] mod support; +mod value_corpus; #[cfg(test)] mod write_batch_foreign_keys; diff --git a/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs b/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs new file mode 100644 index 0000000000..8e68e8f70e --- /dev/null +++ b/linux/crates/drivers/sqlite/tests/integration/value_corpus.rs @@ -0,0 +1,52 @@ +//! SQLite's value corpus. A value reads as the storage class SQLite +//! keeps it in, whatever the column declares, so the declared types +//! here are the ones real schemas use. + +use tablepro_test_support::conformance::{CorpusColumn, ValueCorpusEngine}; + +use crate::conformance_engine::SqliteEngine; + +impl ValueCorpusEngine for SqliteEngine { + fn value_corpus(&self) -> Vec { + vec![ + CorpusColumn::new("v INTEGER") + .shows("9223372036854775807", "9223372036854775807") + .shows("-9223372036854775808", "-9223372036854775808") + .null(), + CorpusColumn::new("v REAL") + .shows("0.1", "0.1") + .shows("-2.5e-5", "-0.000025") + .shows("1", "1"), + CorpusColumn::new("v TEXT") + .shows("'naïve café'", "naïve café") + .shows("'日本語のテキスト'", "日本語のテキスト") + .shows("'🦀 emoji 👩‍💻'", "🦀 emoji 👩‍💻") + .shows("''", "") + .shows("'it''s'", "it's") + .shows("'line' || char(10) || 'break'", "line\nbreak") + .null(), + CorpusColumn::new("v BLOB") + .shows("X'DEADBEEF'", "0xdeadbeef") + .shows("X''", "0x"), + CorpusColumn::new("v NUMERIC") + .shows("'1.50'", "1.5") + .shows("'12'", "12"), + CorpusColumn::new("v DECIMAL(10, 2)").shows("-12.5", "-12.5"), + CorpusColumn::new("v DATETIME") + .shows("'2024-02-29 12:00:00.123'", "2024-02-29 12:00:00.123") + .shows("'2024-03-10T02:30:00-08:00'", "2024-03-10T02:30:00-08:00"), + CorpusColumn::new("v BOOLEAN").shows("1", "1").shows("0", "0"), + CorpusColumn::new("v JSON").shows(r#"'{"b": 1, "a": [1, 2.50]}'"#, r#"{"b": 1, "a": [1, 2.50]}"#), + CorpusColumn::new("v") + .shows("42", "42") + .shows("'text'", "text") + .shows("X'01'", "0x01") + .shows("1.5", "1.5"), + ] + } +} + +tablepro_test_support::value_corpus_suite! { + engine: SqliteEngine, + docker: false, +} diff --git a/linux/crates/test-support/src/conformance/cases/mod.rs b/linux/crates/test-support/src/conformance/cases/mod.rs index 143e1ac1fb..a277de2a49 100644 --- a/linux/crates/test-support/src/conformance/cases/mod.rs +++ b/linux/crates/test-support/src/conformance/cases/mod.rs @@ -18,4 +18,5 @@ pub mod streaming; pub mod suite_flags; pub mod tls; pub mod type_fallback; +pub mod value_corpus; pub mod writes; diff --git a/linux/crates/test-support/src/conformance/cases/value_corpus.rs b/linux/crates/test-support/src/conformance/cases/value_corpus.rs new file mode 100644 index 0000000000..28e11208b5 --- /dev/null +++ b/linux/crates/test-support/src/conformance/cases/value_corpus.rs @@ -0,0 +1,408 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use tablepro_core::browse::{ + BrowseSqlError, DeclaredPrecisionSink, ExportRequest, PageRequest, build_export, read_page, +}; +use tablepro_core::column::{ColumnInfo, ColumnType, ResultColumn}; +use tablepro_core::export::value_text; +use tablepro_core::filter::FilterSet; +use tablepro_core::meta::PagePosition; +use tablepro_core::value::Value; +use tablepro_core::{Connection, DriverError, EditorSession, EditorSessionOptions, ReadStatement, TableRef, WriteKind}; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{ + CALL_DEADLINE, Collected, apply, connect, create_scratch_table, expect, run_completed, run_options, within, + writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure, CorpusColumn, ValueCorpusEngine}; + +const CASE: &str = "value_corpus_reads_as_the_grid_shows"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Path { + Browse, + Export, + Editor, +} + +/// The rows a page holds, by `id`, and the type the grid draws `v` with. +struct Read { + column_type: ColumnType, + values: BTreeMap, +} + +/// Case 17: every value of the engine's corpus reads back, in a table +/// tab's page, a table export and an editor run, as the text its entry +/// names. A wrong text is a decoder bug, and every one is listed at once. +pub async fn value_corpus_reads_as_the_grid_shows(engine: E) -> Result<(), ConformanceFailure> { + let corpus = engine.value_corpus(); + expect(CASE, !corpus.is_empty(), || "the engine lists no corpus".to_owned())?; + let connection = connect(CASE, &engine, writable()).await?; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let mut wrong = Vec::new(); + for (index, column) in corpus.iter().enumerate() { + // A column the server refuses is listed with the wrong values, + // so one run reports the whole corpus. + let table = match load(&engine, connection.as_ref(), index, column).await { + Ok(table) => table, + Err(failure) => { + wrong.push(failure.to_string()); + continue; + } + }; + let browsed = browse(connection.as_ref(), &table).await?; + compare(Path::Browse, column, &browsed, &mut wrong); + let exported = export(connection.as_ref(), &table).await?; + compare(Path::Export, column, &exported, &mut wrong); + let edited = edit(session.as_ref(), connection.as_ref(), &table).await?; + compare(Path::Editor, column, &edited, &mut wrong); + } + session.close().await; + connection.close().await?; + expect(CASE, wrong.is_empty(), || { + format!( + "{} values did not read back as written:\n{}", + wrong.len(), + wrong.join("\n") + ) + }) +} + +async fn load( + engine: &E, + connection: &dyn Connection, + index: usize, + column: &CorpusColumn, +) -> Result { + let table = create_scratch_table(engine, connection, &format!("corpus_{index}")).await?; + let dialect = connection.dialect(); + let table_sql = dialect.qualify(&table); + let mut schema = column.setup_sql.clone(); + schema.push(format!("ALTER TABLE {table_sql} ADD {}", column.declaration)); + apply(connection, &table, WriteKind::SchemaChanges, schema) + .await + .map_err(|error| ConformanceFailure::setup(format!("`{}`: {error}", column.declaration)))?; + let names = [ + dialect.quote_identifier("id"), + dialect.quote_identifier("label"), + dialect.quote_identifier("v"), + ] + .join(", "); + let inserts = column + .values + .iter() + .enumerate() + .map(|(row, value)| { + format!( + "INSERT INTO {table_sql} ({names}) VALUES ({row}, 'corpus', {})", + value.sql + ) + }) + .collect(); + apply(connection, &table, WriteKind::RowChanges, inserts) + .await + .map_err(|error| ConformanceFailure::setup(format!("`{}` values: {error}", column.declaration)))?; + Ok(table) +} + +/// The first page of the table, read by `read_page` as the browse tab +/// reads one. +async fn browse(connection: &dyn Connection, table: &TableRef) -> Result { + let columns = connection.fetch_columns(table, within(CALL_DEADLINE)).await?; + let identity = connection.row_identity(table, within(CALL_DEADLINE)).await?; + let filter = FilterSet::default(); + let request = PageRequest { + table, + columns: &columns, + filter: &filter, + sort: &[], + identity: &identity, + position: &PagePosition::First, + limit: 1_000, + }; + let page = read_page::(connection, &request, &within(CALL_DEADLINE)) + .await + .map_err(|failure| match failure { + PageFailure::Sql(error) => ConformanceFailure::setup(error), + PageFailure::Driver(error) => ConformanceFailure::Driver(error), + })?; + let visible = page.layout.visible.clone(); + let rows = page + .result + .rows + .into_iter() + .map(|mut row| { + row.truncate(visible.end); + row + }) + .collect(); + by_id(page.result.columns.get(visible).unwrap_or_default(), rows) +} + +/// Every row of the table, read as Export All reads a table tab. +async fn export(connection: &dyn Connection, table: &TableRef) -> Result { + let columns: Arc<[ColumnInfo]> = connection.fetch_columns(table, within(CALL_DEADLINE)).await?.into(); + let identity = connection.row_identity(table, within(CALL_DEADLINE)).await?; + let statement = build_export( + &*connection.dialect(), + &ExportRequest { + table, + columns: &columns, + filter: &FilterSet::default(), + sort: &[], + identity: &identity, + }, + ) + .map_err(ConformanceFailure::setup)?; + let collected = Collected::default(); + let sink = DeclaredPrecisionSink::new(collected.sink(), Arc::clone(&columns)); + connection + .stream_read( + &ReadStatement::Generated(statement), + Box::new(sink), + within(CALL_DEADLINE), + ) + .await?; + let columns = collected + .columns() + .ok_or_else(|| ConformanceFailure::setup("the export started no result set"))?; + by_id(&columns, collected.rows()) +} + +enum PageFailure { + Sql(BrowseSqlError), + Driver(DriverError), +} + +impl From for PageFailure { + fn from(error: BrowseSqlError) -> Self { + Self::Sql(error) + } +} + +impl From for PageFailure { + fn from(error: DriverError) -> Self { + Self::Driver(error) + } +} + +async fn edit( + session: &dyn EditorSession, + connection: &dyn Connection, + table: &TableRef, +) -> Result { + let dialect = connection.dialect(); + let id = dialect.quote_identifier("id"); + let sql = format!( + "SELECT {id}, {} FROM {}", + dialect.quote_identifier("v"), + dialect.qualify(table) + ); + let collected = run_completed(CASE, session, &sql, run_options()).await?; + let columns = collected + .columns() + .ok_or_else(|| ConformanceFailure::setup(format!("`{sql}` started no result set")))?; + by_id(&columns, collected.rows()) +} + +fn by_id(columns: &[ResultColumn], rows: Vec>) -> Result { + let position = |name: &str| { + columns + .iter() + .position(|column| column.name == name) + .ok_or_else(|| ConformanceFailure::setup(format!("no `{name}` column in {columns:?}"))) + }; + let (id, v) = (position("id")?, position("v")?); + let mut values = BTreeMap::new(); + for mut row in rows { + // An editor run on PostgreSQL reads every value as the server's text. + let key = match row.get(id) { + Some(Value::Int(key)) => Some(*key), + Some(Value::UInt(key)) => i64::try_from(*key).ok(), + Some(Value::Text(key)) => key.parse().ok(), + _ => None, + }; + let Some(key) = key else { + return Err(ConformanceFailure::setup(format!("an id read as {:?}", row.get(id)))); + }; + if v < row.len() { + values.insert(key, row.swap_remove(v)); + } + } + Ok(Read { + column_type: columns[v].column_type.clone(), + values, + }) +} + +fn compare(path: Path, column: &CorpusColumn, read: &Read, wrong: &mut Vec) { + let path_name = match path { + Path::Browse => "browse", + Path::Export => "export", + Path::Editor => "editor", + }; + for (row, value) in column.values.iter().enumerate() { + let declaration = &column.declaration; + let sql = &value.sql; + let expected = match (path, &value.edited) { + (Path::Editor, Some(edited)) => Some(edited.as_str()), + _ => value.shown.as_deref(), + }; + let Some(read_value) = i64::try_from(row).ok().and_then(|row| read.values.get(&row)) else { + wrong.push(format!("{path_name}: `{declaration}` lost the row holding {sql}")); + continue; + }; + let shown = match read_value { + Value::Null => None, + // What the grid shows for a value the driver could not read. + Value::Undecodable(undecoded) => Some(format!("", undecoded.type_name)), + other => Some(value_text(other, &read.column_type).unwrap_or_else(|| format!("<{other:?}>"))), + }; + if shown.as_deref() != expected { + wrong.push(format!( + "{path_name}: `{declaration}` holding {sql} shows {} as {}, expected {}", + quoted(shown.as_deref()), + read_value.variant_name(), + quoted(expected), + )); + } + } +} + +fn quoted(text: Option<&str>) -> String { + match text { + Some(text) => format!("{text:?}"), + None => "NULL".to_owned(), + } +} + +#[cfg(test)] +mod tests { + use tablepro_core::column::ColumnKind; + use tablepro_core::value::{SqlDecimal, UndecodableReason, UndecodedValue}; + + use super::*; + + fn decimal(precision: u32, scale: u32) -> ColumnType { + let mut column_type = ColumnType::unknown(); + column_type.kind = ColumnKind::Decimal { + precision: Some(precision), + scale: Some(scale), + }; + column_type + } + + fn read(column_type: ColumnType, values: Vec) -> Read { + Read { + column_type, + values: (0..).zip(values).collect(), + } + } + + fn column() -> CorpusColumn { + CorpusColumn::new("v numeric(10, 2)") + .shows("1.5", "1.50") + .shows_apart("2", "2.00", "2") + .null() + } + + #[test] + fn a_value_shows_at_the_scale_of_the_column_it_came_back_in() { + let mut wrong = Vec::new(); + let values = vec![ + Value::Decimal("1.5".parse::().unwrap()), + Value::Decimal("2".parse::().unwrap()), + Value::Null, + ]; + + compare(Path::Browse, &column(), &read(decimal(10, 2), values), &mut wrong); + + assert_eq!(wrong, Vec::::new()); + } + + #[test] + fn every_wrong_or_missing_value_is_listed() { + let mut wrong = Vec::new(); + let values = vec![Value::Text("1.5".to_owned()), Value::Int(2)]; + + compare( + Path::Editor, + &column(), + &read(ColumnType::unknown(), values), + &mut wrong, + ); + + assert_eq!( + wrong, + [ + r#"editor: `v numeric(10, 2)` holding 1.5 shows "1.5" as Text, expected "1.50""#, + "editor: `v numeric(10, 2)` lost the row holding NULL", + ] + ); + } + + #[test] + fn a_null_where_a_value_belongs_is_wrong() { + let mut wrong = Vec::new(); + let values = vec![Value::Null, Value::Int(2), Value::Null]; + + compare( + Path::Editor, + &column(), + &read(ColumnType::unknown(), values), + &mut wrong, + ); + + assert_eq!( + wrong, + [r#"editor: `v numeric(10, 2)` holding 1.5 shows NULL as Null, expected "1.50""#] + ); + } + + #[test] + fn a_value_the_driver_could_not_read_shows_as_the_grid_shows_it() { + let mut wrong = Vec::new(); + let undecoded = Value::Undecodable(Box::new(UndecodedValue { + type_name: "money".to_owned(), + reason: UndecodableReason::OutOfRange, + })); + let column = CorpusColumn::new("v money").shows_apart("1", "1", ""); + + compare( + Path::Editor, + &column, + &read(ColumnType::unknown(), vec![undecoded]), + &mut wrong, + ); + + assert_eq!(wrong, Vec::::new()); + } + + #[test] + fn rows_are_keyed_by_their_id_whatever_order_they_came_in() { + let columns = [ + ResultColumn::new("v", ColumnType::unknown()), + ResultColumn::new("id", ColumnType::unknown()), + ]; + let rows = vec![ + vec![Value::Text("c".to_owned()), Value::Text("2".to_owned())], + vec![Value::Text("b".to_owned()), Value::UInt(1)], + vec![Value::Text("a".to_owned()), Value::Int(0)], + ]; + + let read = by_id(&columns, rows).unwrap(); + + assert_eq!( + read.values.into_iter().collect::>(), + [ + (0, Value::Text("a".to_owned())), + (1, Value::Text("b".to_owned())), + (2, Value::Text("c".to_owned())) + ] + ); + } +} diff --git a/linux/crates/test-support/src/conformance/corpus_column.rs b/linux/crates/test-support/src/conformance/corpus_column.rs new file mode 100644 index 0000000000..7cc06a3edb --- /dev/null +++ b/linux/crates/test-support/src/conformance/corpus_column.rs @@ -0,0 +1,70 @@ +/// One column of an engine's value corpus: a type, and values of it the +/// way a real table holds them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CorpusColumn { + /// Statements run first, such as creating the enum the column uses. + pub setup_sql: Vec, + /// The clause `ALTER TABLE ... ADD` takes, naming the column `v`, as + /// in `UnsupportedTypeCase`. + pub declaration: String, + pub values: Vec, +} + +/// A value written as SQL, and the text the grid shows for it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CorpusValue { + pub sql: String, + /// `None` for NULL. + pub shown: Option, + /// What an editor run shows where it differs from a browse page, + /// such as a MySQL TIMESTAMP, which browse reads as an instant and + /// the editor in the server's zone. + pub edited: Option, +} + +impl CorpusColumn { + pub fn new(declaration: impl Into) -> Self { + Self { + setup_sql: Vec::new(), + declaration: declaration.into(), + values: Vec::new(), + } + } + + pub fn after(mut self, setup_sql: impl Into) -> Self { + self.setup_sql.push(setup_sql.into()); + self + } + + pub fn shows(mut self, sql: impl Into, shown: impl Into) -> Self { + self.values.push(CorpusValue { + sql: sql.into(), + shown: Some(shown.into()), + edited: None, + }); + self + } + + pub fn shows_apart( + mut self, + sql: impl Into, + browsed: impl Into, + edited: impl Into, + ) -> Self { + self.values.push(CorpusValue { + sql: sql.into(), + shown: Some(browsed.into()), + edited: Some(edited.into()), + }); + self + } + + pub fn null(mut self) -> Self { + self.values.push(CorpusValue { + sql: "NULL".to_owned(), + shown: None, + edited: None, + }); + self + } +} diff --git a/linux/crates/test-support/src/conformance/mod.rs b/linux/crates/test-support/src/conformance/mod.rs index c12b08dca5..aafe2a54ab 100644 --- a/linux/crates/test-support/src/conformance/mod.rs +++ b/linux/crates/test-support/src/conformance/mod.rs @@ -1,8 +1,9 @@ //! The suites every driver runs against a real server. //! //! A driver's integration target implements an engine adapter and -//! invokes `connection_conformance_suite!`, `ddl_conformance_suite!` and -//! `tls_conformance_suite!` with flags for what the engine supports. A +//! invokes `connection_conformance_suite!`, `ddl_conformance_suite!`, +//! `value_corpus_suite!` and `tls_conformance_suite!` with flags for +//! what the engine supports. A //! flag that is off emits no test. The flags with a capability behind them (`endpoint`, //! `editor_transactions` and the TLS `client_identity`) are checked //! against the driver's capability table, so those cases cannot be @@ -18,6 +19,7 @@ mod conformance_engine; mod conformance_failure; mod connection_suite_flags; mod connection_suite_macro; +mod corpus_column; mod ddl_conformance_engine; mod ddl_suite_macro; mod editor_transaction_engine; @@ -36,12 +38,15 @@ mod tls_conformance_engine; mod tls_suite_macro; mod typed_conformance_engine; mod unsupported_type_case; +mod value_corpus_engine; +mod value_corpus_suite_macro; pub use bounded_ddl_engine::BoundedDdlEngine; pub use client_identity_engine::ClientIdentityEngine; pub use conformance_engine::ConformanceEngine; pub use conformance_failure::ConformanceFailure; pub use connection_suite_flags::ConnectionSuiteFlags; +pub use corpus_column::{CorpusColumn, CorpusValue}; pub use ddl_conformance_engine::ConformanceDdlEngine; pub use editor_transaction_engine::EditorTransactionEngine; pub use large_table_engine::LargeTableEngine; @@ -53,3 +58,4 @@ pub use test_liveness::test_liveness; pub use tls_conformance_engine::TlsConformanceEngine; pub use typed_conformance_engine::TypedConformanceEngine; pub use unsupported_type_case::UnsupportedTypeCase; +pub use value_corpus_engine::ValueCorpusEngine; diff --git a/linux/crates/test-support/src/conformance/support.rs b/linux/crates/test-support/src/conformance/support.rs index f3238ba783..43e640b8a1 100644 --- a/linux/crates/test-support/src/conformance/support.rs +++ b/linux/crates/test-support/src/conformance/support.rs @@ -5,6 +5,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use async_trait::async_trait; +use tablepro_core::column::ResultColumn; use tablepro_core::dialect::SqlDialect; use tablepro_core::statement::{GuardedStatement, Statement, StatementBuilder}; use tablepro_core::value::Value; @@ -148,12 +149,12 @@ pub(crate) async fn count(connection: &dyn Connection, table: &TableRef) -> Resu Ok(connection.count_rows(&statement(&sql), within(CALL_DEADLINE)).await?) } -/// The events one editor run sent, kept after the sink is gone. +/// The events one run sent, kept after the sink is gone. #[derive(Debug, Clone, Default)] pub(crate) struct Collected(Arc>>); impl Collected { - fn sink(&self) -> Box { + pub(crate) fn sink(&self) -> Box { Box::new(CollectingSink(self.clone())) } @@ -168,6 +169,25 @@ impl Collected { .collect() } + /// The columns of the first result set, as `ColumnsResolved` left + /// them. + pub(crate) fn columns(&self) -> Option> { + let mut columns = None; + for event in lock(&self.0).iter() { + match event { + ResultEvent::ResultSetStarted { columns: started } if columns.is_none() => { + columns = Some(started.clone()); + } + ResultEvent::ColumnsResolved { + set_index: 0, + columns: resolved, + } => columns = Some(resolved.clone()), + _ => {} + } + } + columns + } + pub(crate) fn truncations(&self) -> Vec { lock(&self.0) .iter() diff --git a/linux/crates/test-support/src/conformance/value_corpus_engine.rs b/linux/crates/test-support/src/conformance/value_corpus_engine.rs new file mode 100644 index 0000000000..22aef61708 --- /dev/null +++ b/linux/crates/test-support/src/conformance/value_corpus_engine.rs @@ -0,0 +1,6 @@ +use crate::conformance::{ConformanceEngine, CorpusColumn}; + +/// An engine with a corpus of real values for the value corpus case. +pub trait ValueCorpusEngine: ConformanceEngine { + fn value_corpus(&self) -> Vec; +} diff --git a/linux/crates/test-support/src/conformance/value_corpus_suite_macro.rs b/linux/crates/test-support/src/conformance/value_corpus_suite_macro.rs new file mode 100644 index 0000000000..369b7ad050 --- /dev/null +++ b/linux/crates/test-support/src/conformance/value_corpus_suite_macro.rs @@ -0,0 +1,18 @@ +/// Emit the value corpus case for one engine adapter, which implements +/// `ValueCorpusEngine`. `docker: true` marks it +/// `#[ignore = "requires docker"]`. +/// +/// The case writes each corpus value as SQL, reads it back through a +/// browse page, a table export and an editor run, and checks the text +/// the grid shows for it on each path. +#[macro_export] +macro_rules! value_corpus_suite { + (engine: $engine:ty, docker: $docker:tt $(,)?) => { + $crate::__conformance_case! { + $docker, + $engine as ConformanceEngine, + value_corpus_reads_as_the_grid_shows, + $crate::conformance::cases::value_corpus::value_corpus_reads_as_the_grid_shows::<$engine> + } + }; +} diff --git a/linux/docs/adding-drivers.md b/linux/docs/adding-drivers.md index 5bf76821d8..30500b945e 100644 --- a/linux/docs/adding-drivers.md +++ b/linux/docs/adding-drivers.md @@ -357,7 +357,7 @@ The layout every driver follows: - One integration target with one module per topic, as `crates/drivers/postgres/tests/integration/main.rs`: `support.rs` (start a fixture, connect, run), `conformance_engine.rs`, `tls.rs` for a network engine, and the engine's own behaviour (`metadata.rs`, `write_batch.rs`, `cancellation.rs` and so on). A network engine's test starts its own container and is `#[ignore = "requires docker"]`. - A container fixture in `crates/test-fixtures` for anything the default image cannot do: TLS from the generated PKI, a second principal that logs in with a client certificate. -The conformance suites are the contract's own tests. The driver's `conformance_engine.rs` invokes the connection and DDL suites, plus the large-table suite for a network engine, and a network engine's `tls.rs` the TLS suite. [testing.md](testing.md#conformance) has the adapter traits, the flags and the cases, with the [DDL](testing.md#ddl-conformance), [large-table](testing.md#large-table-conformance) and [TLS](testing.md#tls-conformance) suites after them. +The conformance suites are the contract's own tests. The driver's `conformance_engine.rs` invokes the connection and DDL suites, plus the large-table suite for a network engine, its `value_corpus.rs` the value corpus, and a network engine's `tls.rs` the TLS suite. [testing.md](testing.md#conformance) has the adapter traits, the flags and the cases, with the [DDL](testing.md#ddl-conformance), [large-table](testing.md#large-table-conformance), [value corpus](testing.md#value-corpus) and [TLS](testing.md#tls-conformance) suites after them. Behaviour the suites do not cover gets a test of its own against the real engine: the engine's stop, its error mapping, what a pooler does to it, each write outcome the ledger can produce. @@ -375,7 +375,7 @@ A new package with Docker tests also needs its nextest group and CI matrix entry - [ ] Errors map to `DriverError` by the engine's code ([error-handling.md](error-handling.md)). - [ ] The client library's log target is in `SILENCED_LIBRARIES` in `crates/app/src/logging.rs`, so no statement text reaches the journal ([error-handling.md](error-handling.md#logging)). - [ ] Registered in `build_registry`; added to the workspace and to `crates/app/Cargo.toml`. -- [ ] The connection, DDL, large-table and TLS conformance suites run; the CI matrix and nextest groups include the package. +- [ ] The connection, DDL, large-table, value corpus and TLS conformance suites run; the CI matrix and nextest groups include the package. - [ ] `build-aux/update-third-party-licenses.sh` ran and its output is committed; `cargo deny check` passes. - [ ] A page in `docs/drivers/` covers timeouts, stops, the server codes the driver maps, read-only, writes, trust and the engine's limits. - [ ] `meson devenv -C _build cargo clippy --locked --workspace --all-targets -- -D warnings` passes, and again with `--no-default-features`. diff --git a/linux/docs/drivers/mysql.md b/linux/docs/drivers/mysql.md index a9cb5016a8..8d68c81b68 100644 --- a/linux/docs/drivers/mysql.md +++ b/linux/docs/drivers/mysql.md @@ -122,4 +122,5 @@ Whether a write can be undone depends on the table's storage engine, not on the - The row limit is `SET SESSION sql_select_limit = `, sent when the limit changes. The server stops a plain `SELECT` there, and the set ends with `Truncation::ServerStopped` and no total. A `SELECT` with a `LIMIT` of its own, or one inside a routine, is read to its end: the rows past the limit are counted, never decoded, and the set ends with `Truncation::ReadPastLimit` and its total. A set that ends exactly one row past the limit reads as `ServerStopped` either way. - A stop sends `KILL QUERY` on the session's own control connection and gives the statement the cancel grace to end. A statement that has not ended by then ends the session, because its answer is still on the way. - When the grid stops reading, the run sends the same `KILL QUERY` and ends as cancelled. The batch already sent is read to its end, unshown, and no later batch is sent. +- sqlx describes a result column by its type alone, with no fractional digits and no BIT length. A run shows a DATETIME(6) holding `12:00:00.5` as `12:00:00.5` and a BIT(1) in the whole byte it arrived in, `00000001`. A table tab and its export take both from the catalogue and show `12:00:00.500000` and `1`. - The transaction state after each run comes from `@@in_transaction` on MariaDB and from `performance_schema.events_transactions_current` on MySQL. On MySQL before 8.0.16, with `performance_schema` off, or for a login that cannot read it, the state is `Unknown`, and the tab's close guard warns instead of guessing. diff --git a/linux/docs/drivers/postgresql.md b/linux/docs/drivers/postgresql.md index 47c3038abd..b62636d40c 100644 --- a/linux/docs/drivers/postgresql.md +++ b/linux/docs/drivers/postgresql.md @@ -38,6 +38,7 @@ COMMIT - A connection whose flight did not end with an acknowledged COMMIT or ROLLBACK is taken out of the pool, because deadpool's recycle check (`RecyclingMethod::Verified`) passes inside a half-finished transaction. - A read that returns no rows still has its columns: the statement runs once more as a subquery right-joined to one row on a false condition, in a read-only transaction of its own. - A column of a type tokio-postgres does not know (an enum, `citext`, a domain) would make it prepare a type lookup under a name. Pages project such columns as text, so a read never does. +- chrono ends at 262143 AD, and the server keeps dates to 5874897 AD and timestamps to 294276 AD. A value past chrono's end reads as the text the server would print for it, so the cell shows it rather than saying it cannot. `probe` takes a pool slot without waiting; when every slot is busy it opens a connection of its own and runs `SELECT 1`. `stream_read` opens a connection of its own, runs `START TRANSACTION READ ONLY` with no statement timeout, and refuses user SQL that would end that transaction (transaction control, `CALL`, `DO`) before sending anything. diff --git a/linux/docs/drivers/sql-server.md b/linux/docs/drivers/sql-server.md index e42bc319f8..b4e4100ece 100644 --- a/linux/docs/drivers/sql-server.md +++ b/linux/docs/drivers/sql-server.md @@ -104,3 +104,4 @@ SQL Server has no session statement timeout. A workspace call's deadline is the - Rows past the row limit are counted and dropped, never decoded. - When the grid stops reading, the driver sends an attention instead of reading millions of unwanted rows. - The transaction state comes from `@@TRANCOUNT` and `XACT_STATE()`, so a transaction an error has doomed reads as `Failed`. +- tiberius describes a result column by its wire type, with no scale, so a `datetime2(7)` value shows the fraction it has rather than seven digits; a table tab and its export take the scale from the catalogue. tiberius also reads MONEY into a float, exact up to about 5.5 * 10^11, and a larger amount shows as unreadable in a run. A table tab reads MONEY cast to a decimal, which holds every value. diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 23befcc177..8b373304da 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -178,6 +178,21 @@ The PostgreSQL, MySQL, SQL Server, ClickHouse and SQLite targets run it, and the The PostgreSQL, MySQL, MariaDB, SQL Server and ClickHouse targets run it. The `ci` nextest profile prints its times and memory for a passing run, so the job log keeps what the bounds are set from. SQLite has no case: its suites run in the unit suite, where tests share one process, so the peak would count theirs too. +### Value corpus + +`value_corpus_suite!` checks that real values read back as the grid shows them. Each driver's `crates/drivers//tests/integration/value_corpus.rs` implements `ValueCorpusEngine` and invokes it: + +```rust +tablepro_test_support::value_corpus_suite! { + engine: PgEngine, + docker: true, +} +``` + +The corpus is a list of `CorpusColumn`s: a column clause, values of it written as SQL, and the text each one shows, or NULL. It covers the engine's dates and times at their extremes and in other zones, high-precision and negative decimals at their scale, JSON, UUIDs, Unicode text, binary values and the engine's other common types. `value_corpus_reads_as_the_grid_shows` adds each column to a scratch table, inserts its values, and reads them back three times: through `read_page`, as a table tab reads a page, through `build_export` and `stream_read` behind a `DeclaredPrecisionSink`, as Export All reads one, and through an editor run of a plain SELECT. The export has to show what the page shows. Each value's text is `value_text` on the type of the column it came back in, the text the grid draws, `` for a value the driver could not read, and every value that differs is listed in one failure, with a column the server refused. Where the two paths differ on purpose, the value says what each shows (`CorpusColumn::shows_apart`): a MySQL TIMESTAMP is an instant in a table tab and the server's wall time in the editor, and a PostgreSQL run shows the server's own text, as psql does. A value that shows the wrong text is a decoder bug: fix the decoder, never the corpus. + +Every target runs it, the MySQL target on MariaDB as well, and SQLite's runs in the unit suite. + ### TLS conformance Every network driver's `crates/drivers//tests/integration/tls.rs` also runs `tls_conformance_suite!` against a TLS-only server built from `TestPki`: