diff --git a/src/impls/reverse_translator_impls/function.rs b/src/impls/reverse_translator_impls/function.rs index afe105001..fdba01103 100644 --- a/src/impls/reverse_translator_impls/function.rs +++ b/src/impls/reverse_translator_impls/function.rs @@ -1739,9 +1739,10 @@ mod tests { // COALESCE is a name both engines share, so the forward direction // passes it through. ("ifnull", "the shared name COALESCE needs no forward rename"), - // json_build_object forwards with its keys cast to text, since - // PostgreSQL coerces a key and SQLite refuses one that is not text. + // json_build_object converts its keys and values by type. ("json_object", "FunctionTranslation::JsonBuildObject"), + // json_build_array converts its arguments by type. + ("json_array", "FunctionTranslation::JsonBuildArray"), // json_array_length forwards under a json_type guard, since SQLite // answers 0 for a non-array where PostgreSQL raises. ("json_array_length", "FunctionTranslation::JsonArrayLength"), diff --git a/src/impls/shared_helpers.rs b/src/impls/shared_helpers.rs index 32695314b..91d0ac19c 100644 --- a/src/impls/shared_helpers.rs +++ b/src/impls/shared_helpers.rs @@ -522,6 +522,47 @@ pub(crate) fn query_output_matches( Ok(false) } +/// The declaration of a scalar query's one output column, read from the query +/// on its own. +/// +/// # Errors +/// +/// Refuses a query that does not answer exactly one column, which PostgreSQL +/// refuses as a scalar subquery too, and an output with no inspectable +/// definition. +pub(crate) fn scalar_query_declared( + query: &Query, + schema: &ParserDB, + options: &crate::options::TranslationContext<'_>, + read_column: impl Fn(&sqlparser::ast::ColumnDef) -> Option, + read_expression: impl Fn( + &Expr, + &ParserDB, + &crate::options::TranslationContext<'_>, + ) -> Result, Error>, +) -> Result, Error> { + let graph = + ColumnScope::from_query(query, schema, sql_traits::structs::IdentifierCase::AsWritten)?; + let mut outputs = graph.scope_for_query(query).into_iter().flat_map(|root| root.outputs()); + let (Some(QueryOutput::Column { definition, .. }), None) = (outputs.next(), outputs.next()) + else { + return Err(Error::forward_refusal(format!( + "({query}) has to answer exactly one column to be read as a value, which PostgreSQL \ + requires of a scalar subquery with `subquery must return only one column`." + ))); + }; + match evaluate_definition(&definition, schema, options, &read_column, &read_expression)? { + DefinitionValue::Known(value) => Ok(value), + DefinitionValue::Opaque => { + Err(Error::UnresolvedColumnReference { + reference: format!("({query})"), + reason: "the relation exposes the column without an inspectable definition" + .to_owned(), + }) + } + } +} + #[cfg_attr(feature = "std", recursive::recursive)] fn definition_matches( definition: &ColumnDefinition<'_, '_, '_, ParserDB>, diff --git a/src/impls/translator_impls.rs b/src/impls/translator_impls.rs index 12c4087a8..3932700fe 100644 --- a/src/impls/translator_impls.rs +++ b/src/impls/translator_impls.rs @@ -31,6 +31,7 @@ mod helpers; pub(crate) use helpers::Forward; mod index_column; pub(crate) mod insert; +mod json_value; mod nulls_not_distinct; mod order_by_expr; pub mod plpgsql; diff --git a/src/impls/translator_impls/data_type.rs b/src/impls/translator_impls/data_type.rs index 8fe41de76..f94fcb7c6 100644 --- a/src/impls/translator_impls/data_type.rs +++ b/src/impls/translator_impls/data_type.rs @@ -400,6 +400,15 @@ pub(crate) fn is_serial_type(data_type: &DataType) -> bool { }) } +/// True when a type that reaches the parser as a custom name is stored as a +/// BLOB. +pub(crate) fn translates_to_blob(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Custom(name, ..) if matches!(translate_custom_type(name), Ok(DataType::Blob(_))) + ) +} + /// Maps PostgreSQL and extension types that reach the parser as a custom name. fn translate_custom_type( name: &sqlparser::ast::ObjectName, diff --git a/src/impls/translator_impls/function.rs b/src/impls/translator_impls/function.rs index 0d69e2776..76c817c24 100644 --- a/src/impls/translator_impls/function.rs +++ b/src/impls/translator_impls/function.rs @@ -21,6 +21,7 @@ use sqlparser::ast::{ use super::{ array::{self, ArrayFunction}, helpers::{Forward, translate_window_type}, + json_value::{json_argument, json_key, json_value}, postgis, }; use crate::{ @@ -40,8 +41,7 @@ use crate::{ session_variable, shared_helpers::{ GENERATE_SERIES_UNSUPPORTED_MESSAGE, declared_in_scope, declared_type_matches, - function_argument_exprs, referenced_column_name, rescale_minor_units, scale_of, - translate_function_arguments, + function_argument_exprs, rescale_minor_units, scale_of, translate_function_arguments, }, temporal_arithmetic::{ epoch_of_temporal_difference, subsecond_timestamp_from_epoch, trim_trailing_zeros, @@ -175,9 +175,9 @@ enum FunctionTranslation { /// `chr`, whose code point 0 PostgreSQL refuses while SQLite's `char` /// answers a one-byte NUL string that `length` then reads as empty. Chr, - /// `json_build_object`, whose keys PostgreSQL coerces to text where - /// SQLite's `json_object` answers `labels must be TEXT` when the query - /// runs. + /// `json_build_array`, whose arguments are converted by type. + JsonBuildArray, + /// `json_build_object`, whose keys and values are converted by type. JsonBuildObject, /// `json_array_length`, which PostgreSQL raises over a non-array where /// SQLite answers 0. @@ -212,10 +212,8 @@ pub(crate) const FORWARD_RENAMES: &[(&str, &str)] = &[ ("strpos", "INSTR"), // chr is NOT a rename: PostgreSQL refuses code point 0 where SQLite's // char makes a one-byte NUL string. See `FunctionTranslation::Chr`. - ("json_build_array", "json_array"), - // json_build_object is NOT a rename: PostgreSQL coerces a key to text - // where SQLite answers `json_object() labels must be TEXT`. See - // `FunctionTranslation::JsonBuildObject`. + // The JSON constructors are NOT renames: each argument is converted by its + // type first. See `json_value`. ("btrim", "trim"), // jsonb_array_length is NOT a rename: SQLite answers 0 for a non-array // where PostgreSQL raises. See `FunctionTranslation::JsonArrayLength`. @@ -713,8 +711,7 @@ fn translate_catalog_function( .to_string(), ), "to_char" => FunctionTranslation::ToChar, - // json_build_array(v, ...) -> json_array(v, ...) (handle remaining jsonb_build_*) - "jsonb_build_array" => FunctionTranslation::Rename("json_array".to_string()), + "json_build_array" | "jsonb_build_array" => FunctionTranslation::JsonBuildArray, "chr" => FunctionTranslation::Chr, "json_build_object" | "jsonb_build_object" => FunctionTranslation::JsonBuildObject, "json_array_length" | "jsonb_array_length" => FunctionTranslation::JsonArrayLength, @@ -1237,16 +1234,10 @@ fn declares_function_by_option(name: &str, options: &Pg2SqliteOptions) -> bool { /// True when `expr` already carries JSON, so `to_json` has nothing to convert. /// -/// Recognised syntactically, which covers what this translator itself emits and -/// what a migration writes inline: an array literal, which becomes JSON text -/// under the JSON array representation, and a call to a function that returns -/// JSON. -/// -/// It cannot see through a bare column reference, so an array or `json` COLUMN -/// takes the `json_quote` path and is quoted into a string, which is wrong and -/// needs the column's declared type to fix. Tracked as R89. +/// Recognised syntactically, so a column's document is found by `json_value` +/// from its type. #[cfg_attr(feature = "std", recursive::recursive)] -fn is_already_json(expr: &Expr) -> bool { +pub(crate) fn is_already_json(expr: &Expr) -> bool { const JSON_VALUED: [&str; 10] = [ "json", "jsonb", @@ -1311,13 +1302,14 @@ fn postgres_json_type_name(sqlite_type: Expr) -> Expr { } } -/// Swaps the first positional argument of an already translated argument list, -/// leaving `ORDER BY`, `DISTINCT`, and the rest of the clauses in place. -fn replace_first_argument(args: &mut FunctionArguments, replacement: Expr) { +/// Swaps the positional argument at `index` of an already translated argument +/// list, leaving `ORDER BY`, `DISTINCT`, and the rest of the clauses in place. +fn replace_argument(args: &mut FunctionArguments, index: usize, replacement: Expr) { if let FunctionArguments::List(list) = args - && let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(first))) = list.args.first_mut() + && let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(argument))) = + list.args.get_mut(index) { - *first = replacement; + *argument = replacement; } } @@ -2255,11 +2247,18 @@ impl crate::traits::translator::TranslatorWithContext for Function { translate_window_type(func.over.as_ref(), schema, options, emit)?, )) } + FunctionTranslation::JsonBuildArray => { + let arguments = function_argument_exprs(&func.args) + .into_iter() + .map(|argument| json_value(argument, "json_build_array", schema, options, emit)) + .collect::, _>>()?; + Ok(simple_function_expr( + "json_array", + arguments, + translate_window_type(func.over.as_ref(), schema, options, emit)?, + )) + } FunctionTranslation::JsonBuildObject => { - // PostgreSQL answers {"1" : 2} for json_build_object(1, 2), - // coercing the key, where SQLite answers `json_object() - // labels must be TEXT` at run time. Every key position is - // cast; the values are left alone. let exprs = function_argument_exprs(&func.args); if !exprs.len().is_multiple_of(2) { return Err(crate::errors::Error::forward_refusal( @@ -2269,18 +2268,9 @@ impl crate::traits::translator::TranslatorWithContext for Function { )); } let mut arguments = Vec::with_capacity(exprs.len()); - for (index, argument) in exprs.iter().enumerate() { - let translated = argument.translate_with_warnings(schema, options, emit)?; - arguments.push(if index.is_multiple_of(2) { - Expr::Cast { - expr: Box::new(translated), - data_type: DataType::Text, - format: None, - kind: CastKind::Cast, - } - } else { - translated - }); + for [key, value] in exprs.as_chunks::<2>().0 { + arguments.push(json_key(key, "json_build_object", schema, options, emit)?); + arguments.push(json_value(value, "json_build_object", schema, options, emit)?); } Ok(simple_function_expr( "json_object", @@ -2335,68 +2325,7 @@ impl crate::traits::translator::TranslatorWithContext for Function { type, as {argument}::text." ))); } - - // A boolean converts to the JSON words, which PostgreSQL - // writes as `true` and `false` where the translated integer - // gave 1 and 0. - if crate::impls::translator_impls::expr::is_boolean_expression( - argument, schema, options, - )? { - let rendered = crate::impls::translator_impls::expr::render_boolean_as_text( - argument, schema, options, emit, - )?; - return Ok(simple_function_expr("json", vec![rendered], None)); - } - - let translated = argument.translate_with_warnings(schema, options, emit)?; - - // An argument that is already JSON needs reading, not - // converting: quoting it would turn the document into a - // string. For a bare column the declared type is the only - // thing separating a document from its own text, so a column - // the schema cannot settle is refused rather than guessed, - // since either guess is wrong for the other's type. - if is_already_json(argument) { - return Ok(simple_function_expr("json", vec![translated], None)); - } - if referenced_column_name(argument).is_some() { - match declared_in_scope( - argument, - schema, - options, - |column| Some(is_json_document_type(&column.data_type, options)), - |expression, schema, options| { - Ok(Some(carries_json(expression, schema, options)?)) - }, - )? { - Some(true) => { - return Ok(simple_function_expr("json", vec![translated], None)); - } - Some(false) => {} - None => { - return Err(crate::errors::Error::forward_refusal(format!( - "to_json({argument}) names a column the translation schema cannot \ - resolve to one declared type, and a json, jsonb, or array column \ - must be read as a document where any other is quoted into a \ - string. Include the column's table in the translation batch, or \ - qualify the reference so the type is unambiguous." - ))); - } - } - } - - // `json_quote` renders SQL NULL as the JSON text `null` where - // PostgreSQL yields SQL NULL, and it produces that bare `null` - // for no other input, so NULLIF restores it while evaluating - // the argument once. - Ok(simple_function_expr( - "NULLIF", - vec![ - simple_function_expr("json_quote", vec![translated], None), - string_literal("null"), - ], - None, - )) + Ok(json_argument(argument, "to_json", schema, options, emit)?.into_to_json()) } FunctionTranslation::Extremum { greatest } => { let exprs = function_argument_exprs(&func.args); @@ -2585,20 +2514,10 @@ impl crate::traits::translator::TranslatorWithContext for Function { "json_agg takes exactly one argument".to_string(), )); }; - let element = argument.translate_with_warnings(schema, options, emit)?; - // A JSON column is TEXT here, so json_group_array would quote - // the document into a string. Reading it back with json() is - // only safe for a column declared JSON: json('hello') is - // `malformed JSON`. - let element = if carries_json(argument, schema, options)? { - simple_function_expr("json", vec![element], None) - } else { - element - }; - + let element = json_value(argument, "json_agg", schema, options, emit)?; let mut args = translate_function_arguments::(&func.args, schema, options, emit)?; - replace_first_argument(&mut args, element); + replace_argument(&mut args, 0, element); let aggregate = Expr::Function(Function { name: ObjectName::from(vec![Ident::new("json_group_array")]), parameters: translate_function_arguments::( @@ -2619,10 +2538,19 @@ impl crate::traits::translator::TranslatorWithContext for Function { Ok(crate::impls::idioms::nullif_empty_json_array(aggregate)) } FunctionTranslation::JsonObjectAgg => { - // Translate the arguments (key and value) and build - // json_group_object. - let args = + let exprs = function_argument_exprs(&func.args); + let [key, value] = exprs.as_slice() else { + return Err(crate::errors::Error::forward_refusal( + "json_object_agg takes exactly two arguments, a key and a value" + .to_string(), + )); + }; + let key = json_key(key, "json_object_agg", schema, options, emit)?; + let value = json_value(value, "json_object_agg", schema, options, emit)?; + let mut args = translate_function_arguments::(&func.args, schema, options, emit)?; + replace_argument(&mut args, 0, key); + replace_argument(&mut args, 1, value); let aggregate = Expr::Function(Function { name: ObjectName::from(vec![Ident::new("json_group_object")]), parameters: translate_function_arguments::( diff --git a/src/impls/translator_impls/json_value.rs b/src/impls/translator_impls/json_value.rs new file mode 100644 index 000000000..a35a30e2d --- /dev/null +++ b/src/impls/translator_impls/json_value.rs @@ -0,0 +1,378 @@ +//! The JSON value PostgreSQL makes of an argument, which `to_json` answers +//! and every JSON constructor applies to each value it takes. +//! +//! SQLite's JSON functions encode the stored value as it is, and refuse a +//! BLOB, so a type stored as something other than PostgreSQL's value is +//! converted first. + +#[cfg(not(feature = "std"))] +use alloc::{ + boxed::Box, + format, + string::{String, ToString}, + vec, +}; + +use sql_traits::structs::ParserDB; +use sqlparser::ast::{BinaryOperator, CastKind, DataType, Expr, TimezoneInfo}; + +use crate::{ + errors::Error, + impls::{ + bind_once::bind_once, + expr_helpers::case_when, + function_helpers::{number_literal, simple_function_expr, string_literal}, + object_name::last_ident, + shared_helpers::{ + declared_in_scope, minor_unit_scale, numeric_scale, referenced_column_name, + render_minor_units_as_text, scalar_query_declared, scope_declines_column, + }, + temporal_arithmetic::{TemporalKind, temporal_kind, trim_trailing_zeros}, + translator_impls::{ + array::is_json_array_representation, + data_type::translates_to_blob, + expr::{is_boolean_expression, render_boolean_as_text}, + function::is_already_json, + uuid::is_blob_uuid_representation, + }, + }, + options::TranslationContext, + prelude::Pg2SqliteOptions, + traits::translator::TranslatorWithContext, + warnings::WarningSink, +}; + +/// An argument converted to what a JSON constructor takes. +pub(crate) enum JsonArgument { + /// A stored text, integer or real, which SQLite encodes as PostgreSQL does. + Native(Expr), + /// Text PostgreSQL writes as a JSON string. + Text(Expr), + /// The JSON spelling of a scalar, such as `true` or `19.90`. + Literal(Expr), + /// A JSON document held as its text. + Document(Expr), +} + +impl JsonArgument { + /// The argument as an array element or an object member's value. + pub(crate) fn into_value(self) -> Expr { + match self { + Self::Native(value) | Self::Text(value) => value, + Self::Literal(json) | Self::Document(json) => { + simple_function_expr("json", vec![json], None) + } + } + } + + /// The argument as `to_json`'s answer, which is SQL NULL where `json_quote` + /// answers `null`. + pub(crate) fn into_to_json(self) -> Expr { + match self { + Self::Native(value) | Self::Text(value) => { + simple_function_expr( + "NULLIF", + vec![ + simple_function_expr("json_quote", vec![value], None), + string_literal("null"), + ], + None, + ) + } + Self::Literal(json) | Self::Document(json) => { + simple_function_expr("json", vec![json], None) + } + } + } +} + +/// `argument` converted the way PostgreSQL converts a value into JSON, with +/// `label` naming the constructor. +/// +/// # Errors +/// +/// Refuses a column the schema cannot settle to one type, a BLOB with no text +/// form here, and a conversion that would read a volatile operand twice. +pub(crate) fn json_argument( + argument: &Expr, + label: &str, + schema: &ParserDB, + options: &TranslationContext<'_>, + emit: WarningSink<'_>, +) -> Result { + let Some(conversion) = conversion_of(argument, schema, options)? else { + return Err(Error::forward_refusal(format!( + "{label}({argument}) reads a value the translation schema cannot settle to one \ + declared type, and PostgreSQL converts a value into JSON by its type. Include the \ + column's table in the translation batch, or qualify the reference so the type is \ + unambiguous." + ))); + }; + let stored = match conversion { + Conversion::Boolean => { + return Ok(JsonArgument::Literal(render_boolean_as_text( + argument, schema, options, emit, + )?)); + } + Conversion::Unrenderable(data_type) => { + return Err(Error::forward_refusal(format!( + "{label}({argument}) needs the JSON form of a {data_type} value, which PostgreSQL \ + takes from the type's text output. It is stored here as a BLOB with no text \ + form in SQL, and SQLite's JSON functions refuse a BLOB." + ))); + } + Conversion::Stored(stored) => stored, + }; + let value = argument.translate_with_warnings(schema, options, emit)?; + let rendered = |value: Expr, render: &dyn Fn(&Expr) -> Expr| { + bind_once(label, [argument], [value], options, |[value]| render(value)) + }; + Ok(match stored { + Stored::Native => JsonArgument::Native(value), + Stored::Document => JsonArgument::Document(value), + Stored::MinorUnits(scale) => { + JsonArgument::Literal(rendered(value, &|value| { + render_minor_units_as_text(value, scale) + })?) + } + Stored::Bytea => JsonArgument::Text(rendered(value, &bytea_text)?), + Stored::UuidBlob => JsonArgument::Text(rendered(value, &uuid_text)?), + Stored::Timestamp => JsonArgument::Text(rendered(value, &iso_timestamp)?), + Stored::TimeWithTimeZone => JsonArgument::Text(rendered(value, &time_with_time_zone_text)?), + }) +} + +/// `argument` as an array element or object member's value, converted as +/// PostgreSQL converts it. +/// +/// # Errors +/// +/// As [`json_argument`]. +pub(crate) fn json_value( + argument: &Expr, + label: &str, + schema: &ParserDB, + options: &TranslationContext<'_>, + emit: WarningSink<'_>, +) -> Result { + Ok(json_argument(argument, label, schema, options, emit)?.into_value()) +} + +/// `argument` as an object key, which PostgreSQL takes from the value's JSON +/// text. +/// +/// # Errors +/// +/// As [`json_argument`], and a JSON document, which PostgreSQL refuses as a +/// key. +pub(crate) fn json_key( + argument: &Expr, + label: &str, + schema: &ParserDB, + options: &TranslationContext<'_>, + emit: WarningSink<'_>, +) -> Result { + Ok(match json_argument(argument, label, schema, options, emit)? { + // SQLite's `json_object` takes only a TEXT label. + JsonArgument::Native(value) => { + Expr::Cast { + expr: Box::new(value), + data_type: DataType::Text, + format: None, + kind: CastKind::Cast, + } + } + JsonArgument::Text(text) | JsonArgument::Literal(text) => text, + JsonArgument::Document(_) => { + return Err(Error::forward_refusal(format!( + "{label} takes {argument} as an object key, and it is a JSON document. PostgreSQL \ + answers `key value must be scalar, not array, composite, or json`." + ))); + } + }) +} + +/// How an argument becomes PostgreSQL's JSON value. +#[derive(Clone, PartialEq, Eq)] +enum Conversion { + /// 1 or 0, which PostgreSQL writes as `true` or `false`. + Boolean, + /// A BLOB with no text form here, named by its type. + Unrenderable(String), + /// A form rendered from the stored value. + Stored(Stored), +} + +/// The stored form of a value the conversion reads. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Stored { + /// A value SQLite encodes as PostgreSQL does. + Native, + /// `json`, `jsonb`, or an array under the JSON representation. + Document, + /// A NUMERIC held as minor units at this scale. + MinorUnits(u32), + /// Raw bytes, which PostgreSQL writes as their hex text output. + Bytea, + /// A UUID held as its 16 bytes. + UuidBlob, + /// `YYYY-MM-DD HH:MM:SS`, which PostgreSQL writes in ISO 8601 with a `T`. + Timestamp, + /// `HH:MM:SS±HH:MM`, whose offset PostgreSQL writes as `±HH` on the hour. + TimeWithTimeZone, +} + +/// The conversion `expr` needs, or `None` for a column or subquery the scope +/// answers more than one type for. +#[cfg_attr(feature = "std", recursive::recursive)] +fn conversion_of( + expr: &Expr, + schema: &ParserDB, + options: &TranslationContext<'_>, +) -> Result, Error> { + if let Some(column_name) = referenced_column_name(expr) { + if scope_declines_column(expr, column_name, options) { + return Ok(Some(Conversion::Stored(Stored::Native))); + } + return declared_in_scope( + expr, + schema, + options, + |column| Some(conversion_of_type(&column.data_type, options)), + conversion_of, + ); + } + Ok(Some(match expr { + Expr::Nested(inner) => return conversion_of(inner, schema, options), + Expr::Subquery(query) => { + return scalar_query_declared( + query, + schema, + options, + |column| Some(conversion_of_type(&column.data_type, options)), + conversion_of, + ); + } + _ if is_already_json(expr) => Conversion::Stored(Stored::Document), + _ if is_boolean_expression(expr, schema, options)? => Conversion::Boolean, + Expr::Cast { data_type, .. } => conversion_of_type(data_type, options), + Expr::TypedString(typed) => conversion_of_type(&typed.data_type, options), + Expr::Function(function) + if last_ident(&function.name) + .is_some_and(|name| name.value.eq_ignore_ascii_case("decode")) => + { + Conversion::Stored(Stored::Bytea) + } + _ => { + if let Some(scale) = numeric_scale(expr, schema, options)?.filter(|scale| *scale > 0) { + Conversion::Stored(Stored::MinorUnits(scale)) + } else if temporal_kind(expr, schema, options)? == Some(TemporalKind::Timestamp) { + Conversion::Stored(Stored::Timestamp) + } else { + Conversion::Stored(Stored::Native) + } + } + })) +} + +fn conversion_of_type(data_type: &DataType, options: &Pg2SqliteOptions) -> Conversion { + match data_type { + DataType::JSON | DataType::JSONB => Conversion::Stored(Stored::Document), + DataType::Array(_) if is_json_array_representation(options) => { + Conversion::Stored(Stored::Document) + } + DataType::Boolean | DataType::Bool => Conversion::Boolean, + DataType::Bytea | DataType::Blob(_) | DataType::Binary(_) | DataType::Varbinary(_) => { + Conversion::Stored(Stored::Bytea) + } + DataType::Uuid if is_blob_uuid_representation(options) => { + Conversion::Stored(Stored::UuidBlob) + } + DataType::Timestamp(_, _) | DataType::Datetime(_) => Conversion::Stored(Stored::Timestamp), + DataType::Time(_, TimezoneInfo::Tz | TimezoneInfo::WithTimeZone) => { + Conversion::Stored(Stored::TimeWithTimeZone) + } + DataType::Custom(..) if translates_to_blob(data_type) => { + Conversion::Unrenderable(data_type.to_string()) + } + _ => { + Conversion::Stored( + minor_unit_scale(data_type).map_or(Stored::Native, Stored::MinorUnits), + ) + } + } +} + +fn concat(left: Expr, right: Expr) -> Expr { + Expr::BinaryOp { + left: Box::new(left), + op: BinaryOperator::StringConcat, + right: Box::new(right), + } +} + +fn like(value: &Expr, pattern: &str) -> Expr { + Expr::Like { + negated: false, + any: false, + expr: Box::new(value.clone()), + pattern: Box::new(string_literal(pattern)), + escape_char: None, + } +} + +fn substr(value: Expr, start: &str, length: Expr) -> Expr { + simple_function_expr("substr", vec![value, number_literal(start), length], None) +} + +/// `rendered` where `value` is set, since `hex(NULL)` answers the empty string. +fn unless_null(value: &Expr, rendered: Expr) -> Expr { + case_when(Expr::IsNotNull(Box::new(value.clone())), rendered, None) +} + +/// `'\x' || lower(hex(b))`, PostgreSQL's default `bytea_output`. +fn bytea_text(value: &Expr) -> Expr { + let hex = simple_function_expr("hex", vec![value.clone()], None); + unless_null( + value, + concat(string_literal("\\x"), simple_function_expr("lower", vec![hex], None)), + ) +} + +/// The hyphenated lowercase text of a 16-byte UUID. +fn uuid_text(value: &Expr) -> Expr { + let hex = simple_function_expr("hex", vec![value.clone()], None); + let group = |start, length| substr(hex.clone(), start, number_literal(length)); + let hyphenated = [group("9", "4"), group("13", "4"), group("17", "4"), group("21", "12")] + .into_iter() + .fold(group("1", "8"), |text, next| concat(concat(text, string_literal("-")), next)); + unless_null(value, simple_function_expr("lower", vec![hyphenated], None)) +} + +/// ISO 8601 text of a timestamp, with a canonical `timestamptz`'s `.ffffff` +/// trimmed as PostgreSQL trims it. +fn iso_timestamp(value: &Expr) -> Expr { + let with_t = |text: Expr| { + simple_function_expr("replace", vec![text, string_literal(" "), string_literal("T")], None) + }; + let instant = concat( + with_t(trim_trailing_zeros(substr(value.clone(), "1", number_literal("26")))), + string_literal("+00:00"), + ); + case_when(like(value, "%.______+00:00"), instant, Some(with_t(value.clone()))) +} + +/// A `timetz`'s text with an offset on the hour written as `±HH`. +fn time_with_time_zone_text(value: &Expr) -> Expr { + let on_the_hour = Expr::BinaryOp { + left: Box::new(like(value, "%+__:00")), + op: BinaryOperator::Or, + right: Box::new(like(value, "%-__:00")), + }; + let without_minutes = Expr::BinaryOp { + left: Box::new(simple_function_expr("length", vec![value.clone()], None)), + op: BinaryOperator::Minus, + right: Box::new(number_literal("3")), + }; + case_when(on_the_hour, substr(value.clone(), "1", without_minutes), Some(value.clone())) +} diff --git a/tests/gauntlet/parity_json_values.rs b/tests/gauntlet/parity_json_values.rs new file mode 100644 index 000000000..435e2c3ab --- /dev/null +++ b/tests/gauntlet/parity_json_values.rs @@ -0,0 +1,218 @@ +//! Parity for the JSON value every JSON constructor makes of its arguments. +//! +//! PostgreSQL converts each argument by its type: a `bytea` becomes its hex +//! text, a `uuid` its hyphenated text, a boolean `true` or `false`, a NUMERIC +//! its decimal, a timestamp ISO 8601 text and a `json` value the document +//! itself. pg2sqlite stores several of these as something else, so every probe +//! below runs the same constructor over the same row on both engines. +//! +//! Both answers are read back through PostgreSQL's `jsonb`, since `json` and +//! `jsonb` lay out whitespace differently and SQLite lays it out a third way. +//! `jsonb` keeps a number's scale, so `19.90` against `19.9` still differs. + +use diesel::{ + pg::PgConnection, + prelude::*, + sql_types::{Nullable, Text}, + sqlite::SqliteConnection, +}; +use pg2sqlite::prelude::{ArrayRepresentation, Pg2Sqlite, Pg2SqliteOptions, UuidRepresentation}; + +use crate::{helpers::establish_connection, postgres_harness}; + +const SCHEMA: &str = " +CREATE TABLE json_vals ( + id INTEGER PRIMARY KEY, + b BYTEA, + u UUID, + f BOOLEAN, + n NUMERIC(10,2), + j JSONB, + ts TIMESTAMP, + tz TIMESTAMPTZ, + tt TIMETZ, + d DATE, + s TEXT, + i INTEGER, + tags INTEGER[] +); +"; + +const SEEDS: &str = r#" +INSERT INTO json_vals VALUES (1, '\x0102', '00010203-0405-0607-0809-0a0b0c0d0e0f', true, 19.90, '{"a":1}', '2024-01-02 03:04:05', '2024-01-02 03:04:05.5+00', '03:04:05+02', '2024-01-02', 'x"y', 7, ARRAY[1,2]); +INSERT INTO json_vals VALUES (2, '\x', 'ffffffff-ffff-ffff-ffff-ffffffffffff', false, -0.05, '[1]', '2024-01-02 03:04:00', '2024-01-02 03:04:00+00', '03:04:05.25-05:30', '2024-12-31', '', 0, ARRAY[]::INTEGER[]); +INSERT INTO json_vals VALUES (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +"#; + +/// Every column, as the argument list of a constructor taking values. +const VALUES: &str = "b, u, f, n, j, ts, tz, tt, d, s, i, tags"; + +/// Every column as a member of an object, under its own name. +const MEMBERS: &str = "'b', b, 'u', u, 'f', f, 'n', n, 'j', j, 'ts', ts, 'tz', tz, 'tt', tt, \ + 'd', d, 's', s, 'i', i, 'tags', tags"; + +/// Every scalar column in a key position. The key PostgreSQL makes is the +/// value's JSON text, so `true` and `2024-01-02T03:04:05` rather than `t` and +/// `2024-01-02 03:04:05`. +const KEYED: &str = "b, 1, u, 2, f, 3, n, 4, ts, 5, tz, 6, tt, 7, d, 8, s, 9, i, 10"; + +const COLUMNS: [&str; 12] = ["b", "u", "f", "n", "j", "ts", "tz", "tt", "d", "s", "i", "tags"]; + +/// Values whose type comes from how they are written rather than from a +/// column, such as a call, casts, typed literals and arithmetic over columns. +const WRITTEN: &str = "decode('0102', 'hex'), '\\x0a'::bytea, \ + '00010203-0405-0607-0809-0a0b0c0d0e0f'::uuid, true, \ + TIMESTAMP '2024-01-02 03:04:05', '2024-01-02 03:04:05.25+00'::timestamptz, \ + '03:04:05+02'::timetz, n * 2, ts + INTERVAL '1 day', (j)"; + +fn options() -> Pg2SqliteOptions { + Pg2SqliteOptions::default() + .with_uuid_representation(UuidRepresentation::Blob) + .with_array_representation(ArrayRepresentation::Json) +} + +/// The rows a probe reads. +#[derive(Clone, Copy)] +enum Rows { + /// The one row with this id. + One(u8), + /// Every row, for an aggregate. + All, + /// The rows whose columns are all set, for an aggregate keyed by one. + NonNull, +} + +impl Rows { + fn filter(self) -> String { + match self { + Self::One(id) => format!("WHERE id = {id}"), + Self::All => String::new(), + Self::NonNull => "WHERE id < 3".to_owned(), + } + } +} + +#[derive(QueryableByName)] +struct Val { + #[diesel(sql_type = Nullable)] + val: Option, +} + +fn setup_pg() -> PgConnection { + let mut conn = postgres_harness::fresh_database(); + postgres_harness::apply(&mut conn, &format!("SET TimeZone = 'UTC';{SCHEMA}{SEEDS}")) + .expect("apply schema and seeds to PostgreSQL"); + conn +} + +// The schema and seeds are translator output, a runtime string with no typed +// form, and the seeds carry bytea, uuid and array literals no shared +// insertable covers on both backends. +fn setup_sqlite() -> SqliteConnection { + let statements = Pg2Sqlite::default() + .sql(&format!("{SCHEMA}{SEEDS}")) + .expect("parse schema and seeds") + .translate_to_sql(&options()) + .expect("translate schema and seeds") + .into_vec(); + let mut conn = establish_connection(); + for statement in &statements { + diesel::sql_query(statement.as_str()) + .execute(&mut conn) + .unwrap_or_else(|error| panic!("SQLite setup: {error}\n{statement}")); + } + conn +} + +/// PostgreSQL's answer to `select`, read through `jsonb`. +fn postgres_answer(pg: &mut PgConnection, expression: &str, filter: &str) -> Option { + // The probe is a constructor over arbitrary columns, which the DSL has no + // function for. + diesel::sql_query(format!("SELECT ({expression})::jsonb::text AS val FROM json_vals {filter}")) + .get_result::(pg) + .unwrap_or_else(|error| panic!("PostgreSQL refused {expression}: {error}")) + .val +} + +/// pg2sqlite's answer to the same probe, read through PostgreSQL's `jsonb`. +fn sqlite_answer( + pg: &mut PgConnection, + sqlite: &mut SqliteConnection, + expression: &str, + filter: &str, +) -> Result, String> { + let probe = format!("{SCHEMA}SELECT {expression} AS val FROM json_vals {filter};"); + let translated = Pg2Sqlite::default() + .sql(&probe) + .expect("parse probe") + .translate_to_sql(&options()) + .map_err(|error| format!("refused: {error}"))? + .into_vec() + .pop() + .expect("the probe emits a statement"); + // The translated probe is translator output, a runtime string. + let answer = diesel::sql_query(translated.as_str()) + .get_result::(sqlite) + .map_err(|error| format!("{error} running {translated}"))? + .val; + // A cast is not something the DSL spells over a bound value. + Ok(diesel::sql_query("SELECT CAST($1 AS jsonb)::text AS val") + .bind::, _>(answer.as_deref()) + .get_result::(pg) + .map_err(|error| format!("{answer:?} is not JSON ({error}), from {translated}"))? + .val) +} + +#[test] +fn json_constructors_convert_every_argument_as_postgresql_does() { + let mut pg = setup_pg(); + let mut sqlite = setup_sqlite(); + + let mut probes: Vec<(String, Rows)> = Vec::new(); + for id in 1..=3 { + for constructor in ["json_build_array", "jsonb_build_array"] { + probes.push((format!("{constructor}({VALUES})"), Rows::One(id))); + } + for constructor in ["json_build_object", "jsonb_build_object"] { + probes.push((format!("{constructor}({MEMBERS})"), Rows::One(id))); + } + for column in COLUMNS { + for constructor in ["to_json", "to_jsonb"] { + probes.push((format!("{constructor}({column})"), Rows::One(id))); + } + } + } + // A NULL key is refused by PostgreSQL, so the third row is left out. + for id in 1..=2 { + for constructor in ["json_build_object", "jsonb_build_object"] { + probes.push((format!("{constructor}({KEYED})"), Rows::One(id))); + } + } + for column in COLUMNS { + for aggregate in ["json_agg", "jsonb_agg"] { + probes.push((format!("{aggregate}({column} ORDER BY id)"), Rows::All)); + } + probes.push((format!("json_object_agg(id, {column} ORDER BY id)"), Rows::All)); + } + for column in ["b", "u", "f", "n", "ts", "tz", "tt", "d"] { + probes.push((format!("json_object_agg({column}, id ORDER BY id)"), Rows::NonNull)); + } + for id in 1..=3 { + probes.push((format!("json_build_array({WRITTEN})"), Rows::One(id))); + } + probes.push(("json_agg(n * 2 ORDER BY id)".to_owned(), Rows::All)); + + let mismatches: Vec = probes + .iter() + .filter_map(|(expression, rows)| { + let filter = rows.filter(); + let filter = filter.as_str(); + let expected = postgres_answer(&mut pg, expression, filter); + let actual = sqlite_answer(&mut pg, &mut sqlite, expression, filter); + (actual.as_ref() != Ok(&expected)).then(|| { + format!("{expression} {filter}\n postgres: {expected:?}\n sqlite: {actual:?}") + }) + }) + .collect(); + assert!(mismatches.is_empty(), "{} mismatches:\n{}", mismatches.len(), mismatches.join("\n")); +} diff --git a/tests/test_direction_parity.rs b/tests/test_direction_parity.rs index 68e1f4125..8becbf773 100644 --- a/tests/test_direction_parity.rs +++ b/tests/test_direction_parity.rs @@ -122,11 +122,11 @@ fn localtime_forward_emission_round_trips() { // passes for a different (correct) reason. A second test exercises the reverse // recognizer directly against the contracted NULLIF shape. -/// Forward-translate `json_object_agg(k, v)` and reverse the emission. +/// Forward-translate `json_object_agg(s, n)` and reverse the emission. /// The result must restore `json_object_agg` with no `NULLIF` wrapper. #[test] fn json_object_agg_forward_emission_round_trips_without_nullif() { - let sqlite_sql = forward("SELECT json_object_agg(k, v) FROM t") + let sqlite_sql = forward("SELECT json_object_agg(s, n) FROM t") .expect("forward json_object_agg must succeed"); let pg_sql = reverse(&sqlite_sql) .expect("the crate's own json_object_agg emission must reverse successfully"); diff --git a/tests/test_json_build_silent_passthrough.rs b/tests/test_json_build_silent_passthrough.rs index d8f0c71b6..6138f45ca 100644 --- a/tests/test_json_build_silent_passthrough.rs +++ b/tests/test_json_build_silent_passthrough.rs @@ -25,15 +25,18 @@ use diesel::{Connection, RunQueryDsl, SqliteConnection}; use pg2sqlite::prelude::{Pg2Sqlite, Pg2SqliteOptions}; -/// Helper: translate a PG `SELECT ... FROM t` statement, then execute it -/// against an in-memory SQLite database (pre-populated with a minimal -/// `t (a, b, c)` table) and assert it both translates and runs. +/// Helper: translate a PG `SELECT ... FROM t` statement after a minimal +/// `t (a, b, c)` table and one row, then execute it against an in-memory +/// SQLite database and assert it both translates and runs. /// /// Returns the translated SQL text on success so the assertion can also /// inspect the rewrite shape. fn assert_translates_and_runs(pg_sql: &str) -> String { let stmts = Pg2Sqlite::default() - .sql(pg_sql) + .sql(&format!( + "CREATE TABLE t (a TEXT, b TEXT, c TEXT); \ + INSERT INTO t (a, b, c) VALUES ('x', 'y', 'z'); {pg_sql}" + )) .and_then(|t| { t.translate(&Pg2SqliteOptions::default()).map(pg2sqlite::script::Script::into_vec) }) @@ -42,12 +45,6 @@ fn assert_translates_and_runs(pg_sql: &str) -> String { let translated_sql = stmts.iter().map(|s| s.to_string()).collect::>().join("; "); let mut conn = SqliteConnection::establish(":memory:").expect("open in-memory SQLite"); - diesel::sql_query("CREATE TABLE t (a TEXT, b TEXT, c TEXT)") - .execute(&mut conn) - .expect("seed schema"); - diesel::sql_query("INSERT INTO t (a, b, c) VALUES ('x', 'y', 'z')") - .execute(&mut conn) - .expect("seed row"); for stmt in &stmts { diesel::sql_query(stmt.to_string()).execute(&mut conn).unwrap_or_else(|e| { diff --git a/tests/test_postgres_gauntlet.rs b/tests/test_postgres_gauntlet.rs index 05dea9317..ae42fa4cd 100644 --- a/tests/test_postgres_gauntlet.rs +++ b/tests/test_postgres_gauntlet.rs @@ -53,3 +53,6 @@ mod parity_query; #[path = "gauntlet/parity_index_names.rs"] mod parity_index_names; + +#[path = "gauntlet/parity_json_values.rs"] +mod parity_json_values; diff --git a/tests/test_to_json.rs b/tests/test_to_json.rs index 014c817c2..84e169c25 100644 --- a/tests/test_to_json.rs +++ b/tests/test_to_json.rs @@ -124,3 +124,68 @@ fn to_json_over_an_unresolvable_column_is_refused() { ); assert!(error.contains("ghost"), "the refusal must name the column: {error}"); } + +/// A vector is stored as packed floats, which has no text form in SQL, so the +/// statement is refused rather than failing on SQLite's BLOB refusal. +#[test] +fn a_json_constructor_over_a_vector_is_refused() { + let error = translate_err( + "CREATE TABLE e (id INT PRIMARY KEY, v vector(3)); + SELECT json_build_array(v) FROM e;", + ); + assert!(error.contains("vector"), "the refusal must name the type: {error}"); +} + +/// PostgreSQL refuses a document as an object key. +#[test] +fn a_json_document_as_an_object_key_is_refused() { + let error = translate_err( + "CREATE TABLE t (id INT PRIMARY KEY, doc JSONB); + SELECT json_build_object(doc, 1) FROM t;", + ); + assert!(error.contains("key"), "the refusal must say the key is the problem: {error}"); +} + +/// A column a union answers as a document on one side and as text on the +/// other has no one conversion, so it is refused rather than guessed. +#[test] +fn a_column_a_union_types_two_ways_is_refused() { + let error = translate_err( + "CREATE TABLE t (id INT PRIMARY KEY, doc JSONB, s TEXT); + SELECT json_build_array(x) FROM (SELECT doc AS x FROM t UNION ALL SELECT s FROM t) q;", + ); + assert!(error.contains("one declared type"), "the refusal must name the ambiguity: {error}"); +} + +/// Rendering a bytea reads it twice, and a scalar subquery may answer +/// differently each time, so it is refused rather than read twice. +#[test] +fn a_bytea_read_by_a_scalar_subquery_is_refused() { + let error = translate_err( + "CREATE TABLE t (id INT PRIMARY KEY, b BYTEA); + SELECT json_build_array((SELECT b FROM t LIMIT 1)) FROM t;", + ); + assert!(error.contains("json_build_array"), "the refusal must name the call: {error}"); +} + +/// PostgreSQL refuses a scalar subquery with two columns, and the conversion +/// has no one type to read from it either. +#[test] +fn a_scalar_subquery_with_two_columns_is_refused() { + let error = translate_err( + "CREATE TABLE t (id INT PRIMARY KEY, s TEXT); + SELECT json_build_array((SELECT s, id FROM t LIMIT 1)) FROM t;", + ); + assert!(error.contains("exactly one column"), "the refusal must name the shape: {error}"); +} + +/// A subquery over `VALUES` declares no type, so the conversion is refused +/// rather than guessed, as it is for a column of such a relation. +#[test] +fn a_scalar_subquery_with_no_declared_type_is_refused() { + let error = translate_err( + "CREATE TABLE t (id INT PRIMARY KEY); + SELECT json_build_array((SELECT column1 FROM (VALUES (1)) v)) FROM t;", + ); + assert!(error.contains("inspectable definition"), "the refusal must name the cause: {error}"); +}