diff --git a/fuzz/fuzz_seeds/postgres_non_decimal_integer_literals.sql b/fuzz/fuzz_seeds/postgres_non_decimal_integer_literals.sql new file mode 100644 index 0000000000..99117be846 --- /dev/null +++ b/fuzz/fuzz_seeds/postgres_non_decimal_integer_literals.sql @@ -0,0 +1 @@ +SELECT 0x1F, 0X_1F, 0o17, 0b1_01, X'1F' diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 8d92f0b57d..20fe597d54 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -608,6 +608,13 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports integer literals in hexadecimal, + /// octal and binary, e.g. `0x1F`, `0o17` and `0b101`, which then parse as + /// numbers instead of `0x` hexadecimal strings. + fn supports_non_decimal_integer_literals(&self) -> bool { + false + } + /// Returns true if the dialects supports specifying null treatment /// as part of a window function's parameter list as opposed /// to after the parameter list. diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 1455e9f06f..eda91db5dc 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -490,6 +490,11 @@ impl Dialect for PostgreSqlDialect { true } + /// See: + fn supports_non_decimal_integer_literals(&self) -> bool { + true + } + /// See: fn supports_array_typedef_with_brackets(&self) -> bool { true diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1bc0b652ec..62014641b0 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5135,6 +5135,30 @@ impl<'a> Parser<'a> { }) } + #[cfg(not(feature = "bigdecimal"))] + fn parse_number_literal(s: String, _loc: Location) -> Result { + Ok(s) + } + + #[cfg(feature = "bigdecimal")] + fn parse_number_literal( + s: String, + loc: Location, + ) -> Result { + let radix = match s.get(..2) { + Some("0x" | "0X") => 16, + Some("0o" | "0O") => 8, + Some("0b" | "0B") => 2, + _ => return Self::parse(s, loc), + }; + let digits = s[2..].strip_prefix('_').unwrap_or(&s[2..]); + bigdecimal::num_bigint::BigInt::parse_bytes(digits.as_bytes(), radix) + .map(bigdecimal::BigDecimal::from) + .ok_or_else(|| { + ParserError::ParserError(format!("Could not parse '{s}' as an integer{loc}")) + }) + } + /// Parse a comma-separated list of 1+ SelectItem pub fn parse_projection(&mut self) -> Result, ParserError> { // BigQuery and Snowflake allow trailing commas, but only in project lists @@ -12987,10 +13011,10 @@ impl<'a> Parser<'a> { }, ), }, - // The call to n.parse() returns a bigdecimal when the - // bigdecimal feature is enabled, and is otherwise a no-op - // (i.e., it returns the input string). - Token::Number(n, l) => ok_value(Value::Number(Self::parse(n, span.start)?, l)), + // A BigDecimal with the bigdecimal feature, the literal as written otherwise. + Token::Number(n, l) => { + ok_value(Value::Number(Self::parse_number_literal(n, span.start)?, l)) + } Token::SingleQuotedString(ref s) => ok_value(Value::SingleQuotedString( self.maybe_concat_string_literal(s.to_string()), )), diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9276f91ded..02a54f08ff 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1606,6 +1606,20 @@ impl<'a> Tokenizer<'a> { return Ok(Some(Token::Number(s, false))); } + if s == "0" && self.dialect.supports_non_decimal_integer_literals() { + let radix = match chars.peek() { + Some('x' | 'X') => Some((16, "a hexadecimal")), + Some('o' | 'O') => Some((8, "an octal")), + Some('b' | 'B') => Some((2, "a binary")), + _ => None, + }; + if let Some((radix, name)) = radix { + return self + .tokenize_non_decimal_integer(chars, radix, name) + .map(Some); + } + } + // match binary literal that starts with 0x if s == "0" && chars.peek() == Some(&'x') { chars.next(); @@ -2302,6 +2316,32 @@ impl<'a> Tokenizer<'a> { Ok(s) } + /// Tokenizes an integer literal such as `0x1F` after its leading `0`, + /// keeping the spelling as written. + fn tokenize_non_decimal_integer( + &self, + chars: &mut State, + radix: u32, + name: &str, + ) -> Result { + let mut s = String::from('0'); + s.extend(chars.next()); + // An underscore may directly follow the prefix, as in `0x_1F`. + if chars.peek() == Some(&'_') && self.dialect.supports_numeric_literal_underscores() { + chars.next(); + s.push('_'); + } + let digits = self.tokenize_number_part(chars, |ch| ch.is_digit(radix))?; + if digits.is_empty() { + return self.tokenizer_error( + chars.location(), + format!("Expected {name} digit after '{}'", &s[..2]), + ); + } + s.push_str(&digits); + Ok(Token::Number(s, false)) + } + // Consume characters until newline fn tokenize_single_line_comment(&self, chars: &mut State) -> String { peeking_take_while(chars, |ch| match ch { @@ -3035,10 +3075,20 @@ mod tests { ], ); - numeric_underscore_dialects.tokenizes_to( + let hex_string_dialects = all_dialects_where(|dialect| { + dialect.supports_numeric_literal_underscores() + && !dialect.supports_non_decimal_integer_literals() + }); + hex_string_dialects.tokenizes_to( "0xFF_FF", vec![Token::HexStringLiteral("FF_FF".to_string())], ); + for dialect in &hex_string_dialects.dialects { + let err = Tokenizer::new(&**dialect, "SELECT 0x_1") + .tokenize() + .unwrap_err(); + assert_eq!("Unexpected character '_'", err.message); + } for dialect in &numeric_underscore_dialects.dialects { for sql in [ @@ -3050,7 +3100,6 @@ mod tests { "SELECT 1e_1", "SELECT 1e1_", "SELECT 1e1__0", - "SELECT 0x_1", "SELECT 0x1_", ] { let err = Tokenizer::new(&**dialect, sql).tokenize().unwrap_err(); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index facc859fe6..18fb0a726a 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -11351,3 +11351,57 @@ fn parse_drop_foreign_table_rejects_modifiers() { ); } } + +#[test] +fn parse_non_decimal_integer_literals() { + let sql = "SELECT 0x1F, 0X1F, 0o17, 0O17, 0b101, 0B101, 0x_1F, 0xFF_FF, 0xFFFFFFFFFFFFFFFFFFFF"; + let canonical = if cfg!(feature = "bigdecimal") { + "SELECT 31, 31, 15, 15, 5, 5, 31, 65535, 1208925819614629174706175" + } else { + sql + }; + let select = pg().verified_only_select_with_canonical(sql, canonical); + for item in &select.projection { + assert!( + matches!( + item, + SelectItem::UnnamedExpr(Expr::Value(ValueWithSpan { + value: Value::Number(_, false), + .. + })) + ), + "{item:?}" + ); + } + + assert_eq!( + expr_from_projection(&pg().verified_only_select("SELECT X'1F'").projection[0]), + &Expr::Value(Value::HexStringLiteral("1F".to_string()).with_empty_span()) + ); + + for (sql, message) in [ + ("SELECT 0x", "Expected a hexadecimal digit after '0x'"), + ("SELECT 0Xg", "Expected a hexadecimal digit after '0X'"), + ("SELECT 0o8", "Expected an octal digit after '0o'"), + ("SELECT 0b2", "Expected a binary digit after '0b'"), + ("SELECT 0x_", "Expected a hexadecimal digit after '0x'"), + ("SELECT 0x__1", "Unexpected character '_'"), + ("SELECT 0x1_", "Unexpected character '_'"), + ] { + let err = pg().parse_sql_statements(sql).unwrap_err().to_string(); + assert!(err.contains(message), "{sql}: {err}"); + } +} + +#[cfg(feature = "bigdecimal")] +#[test] +fn parse_non_decimal_integer_token_with_invalid_digits() { + let err = Parser::new(&PostgreSqlDialect {}) + .with_tokens(vec![Token::Number("0xZZ".to_string(), false)]) + .parse_value() + .unwrap_err(); + assert_eq!( + err, + ParserError::ParserError("Could not parse '0xZZ' as an integer".to_string()) + ); +}