Fix tokenization of order keywords (fixes #532).

Author: andialbrecht

CHANGELOG

@@ -13,11 +13,11 @@ Bug Fixes
   Note: In some cases you might want to add `strip_whitespace=True` where you
   previously used just `strip_comments=True`. `strip_comments` did some of the
   work that `strip_whitespace` should do.
-
 * Fix error when splitting statements that contain multiple CASE clauses
   within a BEGIN block (issue784).
-
 * Fix whitespace removal with nested expressions (issue782).
+* Fix parsing and formatting of ORDER clauses containing NULLS FIRST or
+  NULLS LAST (issue532).
 
 
 Release 0.5.0 (Apr 13, 2024)

sqlparse/keywords.py

@@ -71,7 +71,9 @@
      r'|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
     (r'END(\s+IF|\s+LOOP|\s+WHILE)?\b', tokens.Keyword),
     (r'NOT\s+NULL\b', tokens.Keyword),
-    (r'NULLS\s+(FIRST|LAST)\b', tokens.Keyword),
+    (r'(ASC|DESC)(\s+NULLS\s+(FIRST|LAST))?\b', tokens.Keyword.Order),
+    (r'(ASC|DESC)\b', tokens.Keyword.Order),
+    (r'NULLS\s+(FIRST|LAST)\b', tokens.Keyword.Order),
     (r'UNION\s+ALL\b', tokens.Keyword),
     (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
     (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
@@ -114,7 +116,6 @@
     'ANY': tokens.Keyword,
     'ARRAYLEN': tokens.Keyword,
     'ARE': tokens.Keyword,
-    'ASC': tokens.Keyword.Order,
     'ASENSITIVE': tokens.Keyword,
     'ASSERTION': tokens.Keyword,
     'ASSIGNMENT': tokens.Keyword,
@@ -227,7 +228,6 @@
     'DELIMITER': tokens.Keyword,
     'DELIMITERS': tokens.Keyword,
     'DEREF': tokens.Keyword,
-    'DESC': tokens.Keyword.Order,
     'DESCRIBE': tokens.Keyword,
     'DESCRIPTOR': tokens.Keyword,
     'DESTROY': tokens.Keyword,

tests/test_tokenize.py

@@ -169,11 +169,16 @@ def test_parse_endifloop(s):
     assert p.tokens[0].ttype is T.Keyword
 
 
-@pytest.mark.parametrize('s', ['NULLS FIRST', 'NULLS LAST'])
-def test_parse_nulls(s):  # issue487
+@pytest.mark.parametrize('s', [
+    'ASC', 'DESC',
+    'NULLS FIRST', 'NULLS LAST',
+    'ASC NULLS FIRST', 'ASC NULLS LAST',
+    'DESC NULLS FIRST', 'DESC NULLS LAST',
+])
+def test_parse_order(s):  # issue487
     p = sqlparse.parse(s)[0]
     assert len(p.tokens) == 1
-    assert p.tokens[0].ttype is T.Keyword
+    assert p.tokens[0].ttype is T.Keyword.Order
 
 
 @pytest.mark.parametrize('s', [