From d70c0e16a0dc2f37144266d0797c364e07855697 Mon Sep 17 00:00:00 2001 From: ikatyal21 Date: Wed, 8 Jul 2026 17:21:43 -0500 Subject: [PATCH] Fix transform to convert empty strings to NULL when changing column to integer or float type --- sqlite_utils/db.py | 16 +++++++++++++++- tests/test_transform.py | 20 +++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3033a36f2..fed0dca94 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2820,10 +2820,24 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: if "rowid" not in new_cols: new_cols.insert(0, "rowid") old_cols.insert(0, "rowid") + # Columns explicitly converted to a numeric type need NULLIF(col, '') so + # that empty strings stored in a previously TEXT column become NULL rather + # than being coerced to 0 or raising a type error. + _numeric_kws = ("INT", "REAL", "FLOA", "DOUB", "NUMERIC", "DECIMAL") + + def _col_expr(from_, to_): + if from_ in types: + raw = COLUMN_TYPE_MAPPING.get(types[from_]) + if raw is None and isinstance(types[from_], str): + raw = types[from_] + if raw and any(kw in raw.upper() for kw in _numeric_kws): + return "NULLIF({}, '')".format(quote_identifier(from_)) + return quote_identifier(from_) + copy_sql = "INSERT INTO {} ({new_cols})\n SELECT {old_cols} FROM {};".format( quote_identifier(new_table_name), quote_identifier(self.name), - old_cols=", ".join(quote_identifier(col) for col in old_cols), + old_cols=", ".join(_col_expr(f, t) for f, t in zip(old_cols, new_cols)), new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) diff --git a/tests/test_transform.py b/tests/test_transform.py index 71518bed9..f4708ebc6 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -21,7 +21,7 @@ {"types": {"age": int}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], @@ -51,7 +51,7 @@ {"types": {"age": int}, "rename": {"age": "dog_age"}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], @@ -144,7 +144,7 @@ def tracer(sql, params): {"types": {"age": int}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], @@ -669,6 +669,20 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): ) +def test_transform_converts_empty_strings_to_null_for_numeric_types(fresh_db): + # Regression test for: transform should convert '' to NULL when changing + # a TEXT column to an INTEGER or FLOAT type (issue #488). + fresh_db["test"].insert_all([ + {"id": "1", "age": "3", "weight": "2.5", "name": "Alice"}, + {"id": "2", "age": "", "weight": "", "name": ""}, + ]) + fresh_db["test"].transform(types={"age": int, "weight": float}) + rows = list(fresh_db["test"].rows) + assert rows[0] == {"id": "1", "age": 3, "weight": 2.5, "name": "Alice"} + # Empty strings in numeric columns become NULL; text columns are unchanged + assert rows[1] == {"id": "2", "age": None, "weight": None, "name": ""} + + def test_transform_with_unique_constraint_implicit_index(fresh_db): dogs = fresh_db["dogs"] # Create a table with a UNIQUE constraint on 'name', which creates an implicit index