diff --git a/docs/changelog.rst b/docs/changelog.rst index d2c07da12..3d919421d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Unreleased ---------- +- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's SQLite strict mode. Omitting the option, or passing ``strict=None``, preserves the existing mode. (:issue:`787`) +- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's SQLite strict mode. Omitting both options preserves the existing mode. (:issue:`787`) - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 71e8377ba..9fafe2891 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -508,6 +508,8 @@ See :ref:`cli_transform_table`. Add a foreign key constraint from a column to another table with another column --drop-foreign-key TEXT Drop foreign key constraint for this column + --strict / --no-strict Enable or disable STRICT mode (default: + preserve current mode) --sql Output SQL without executing it --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/docs/cli.rst b/docs/cli.rst index c446a72e6..dce36d9d2 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2182,7 +2182,7 @@ Use ``--ignore`` to ignore the error if the table does not exist. Transforming tables =================== -The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. The ``transform`` command preserves a table's ``STRICT`` mode. +The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. By default, the ``transform`` command preserves a table's ``STRICT`` mode. .. code-block:: bash @@ -2228,6 +2228,12 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--add-foreign-key column other_table other_column`` Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``. +``--strict`` + Convert the table to a `SQLite STRICT table `__. The command fails if the available SQLite version does not support strict tables. If existing rows contain values that are incompatible with their declared column types the transformation fails and the original table is left unchanged. + +``--no-strict`` + Convert a strict table back to a regular non-strict table. + If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index d444806e5..8b0767772 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1753,6 +1753,29 @@ To alter the type of a column, use the ``types=`` argument: See :ref:`python_api_add_column` for a list of available types. +.. _python_api_transform_strict: + +Changing strict mode +-------------------- + +The optional ``strict=`` parameter can change whether a table uses `SQLite STRICT mode `__. Pass ``strict=True`` to convert a regular table to a strict table: + +.. code-block:: python + + table.transform(strict=True) + +Pass ``strict=False`` to convert a strict table back to a regular non-strict table: + +.. code-block:: python + + table.transform(strict=False) + +The default is ``strict=None``, which preserves the table's existing strict mode. + +Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. + +Converting to a strict table validates all existing rows as they are copied into the replacement table. If a value is incompatible with its declared column type, SQLite raises ``sqlite3.IntegrityError`` and the transformation is rolled back, leaving the original table and its data unchanged. + .. _python_api_transform_rename_columns: Renaming columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7fab72b18..e0b896985 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2718,6 +2718,11 @@ def schema( multiple=True, help="Drop foreign key constraint for this column", ) +@click.option( + "--strict/--no-strict", + default=None, + help="Enable or disable STRICT mode (default: preserve current mode)", +) @click.option("--sql", is_flag=True, help="Output SQL without executing it") @load_extension_option def transform( @@ -2735,6 +2740,7 @@ def transform( default_none, add_foreign_keys, drop_foreign_keys, + strict, sql, load_extension, ): @@ -2796,6 +2802,7 @@ def transform( defaults=default_dict, drop_foreign_keys=drop_foreign_keys_value, add_foreign_keys=add_foreign_keys_value, + strict=strict, ): click.echo(line) else: @@ -2809,6 +2816,7 @@ def transform( defaults=default_dict, drop_foreign_keys=drop_foreign_keys_value, add_foreign_keys=add_foreign_keys_value, + strict=strict, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 62e365654..d709fb9cb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2514,6 +2514,7 @@ def transform( foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, keep_table: Optional[str] = None, + strict: Optional[bool] = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -2536,6 +2537,8 @@ def transform( to use when creating the table :param keep_table: If specified, the existing table will be renamed to this and will not be dropped + :param strict: Set to ``True`` to make the table strict or ``False`` to make it + non-strict. Defaults to ``None``, which preserves the existing strict mode. """ if not self.exists(): raise ValueError("Cannot transform a table that doesn't exist yet") @@ -2551,6 +2554,7 @@ def transform( foreign_keys=foreign_keys, column_order=column_order, keep_table=keep_table, + strict=strict, ) pragma_foreign_keys_was_on = bool( self.db.execute("PRAGMA foreign_keys").fetchone()[0] @@ -2587,6 +2591,8 @@ def transform( self.db.execute("PRAGMA defer_foreign_keys=OFF;") if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=1;") + if strict is not None: + self._defaults["strict"] = strict return self def transform_sql( @@ -2604,6 +2610,7 @@ def transform_sql( column_order: Optional[List[str]] = None, tmp_suffix: Optional[str] = None, keep_table: Optional[str] = None, + strict: Optional[bool] = None, ) -> List[str]: """ Return a list of SQL statements that should be executed in order to apply this transformation. @@ -2624,7 +2631,11 @@ def transform_sql( :param tmp_suffix: Suffix to use for the temporary table name :param keep_table: If specified, the existing table will be renamed to this and will not be dropped + :param strict: Set to ``True`` to make the table strict or ``False`` to make it + non-strict. Defaults to ``None``, which preserves the existing strict mode. """ + if strict is True and not self.db.supports_strict: + raise TransformError("SQLite does not support STRICT tables") types = types or {} rename = rename or {} drop = drop or set() @@ -2806,7 +2817,7 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: defaults=create_table_defaults, foreign_keys=create_table_foreign_keys, column_order=column_order, - strict=self.strict, + strict=self.strict if strict is None else strict, ).strip() ) diff --git a/tests/test_cli.py b/tests/test_cli.py index a828c70ed..a2135b087 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ from click.testing import CliRunner from pathlib import Path import subprocess +import sqlite3 import sys import json import os @@ -1939,6 +1940,64 @@ def test_transform_sql(db_path): assert db["dogs"].schema == original_schema +@pytest.mark.parametrize( + "initial_strict,args,expected_strict", + ( + (False, [], False), + (True, [], True), + (False, ["--strict"], True), + (True, ["--no-strict"], False), + ), +) +def test_transform_strict_option(db_path, initial_strict, args, expected_strict): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db["dogs"].create({"id": int}, strict=initial_strict) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) + + assert result.exit_code == 0, result.output + assert db["dogs"].strict is expected_strict + + +@pytest.mark.parametrize( + "initial_strict,flag,sql_is_strict", + ( + (False, "--strict", True), + (True, "--no-strict", False), + ), +) +def test_transform_strict_option_sql(db_path, initial_strict, flag, sql_is_strict): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db["dogs"].create({"id": int}, strict=initial_strict) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", flag, "--sql"]) + + assert result.exit_code == 0, result.output + assert (") STRICT;" in result.output) is sql_is_strict + assert db["dogs"].strict is initial_strict + + +def test_transform_strict_option_with_invalid_data(db_path): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = db["dogs"] + dogs.create({"id": int}) + dogs.insert({"id": "not-an-integer"}) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", "--strict"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, sqlite3.IntegrityError) + assert dogs.strict is False + assert list(dogs.rows) == [{"id": "not-an-integer"}] + assert not any(name.startswith("dogs_new_") for name in db.table_names()) + + @pytest.mark.parametrize( "extra_args,expected_schema", ( diff --git a/tests/test_transform.py b/tests/test_transform.py index 71518bed9..f0f5019a6 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,3 +1,5 @@ +import sqlite3 + from sqlite_utils.db import ForeignKey, TransformError from sqlite_utils.utils import OperationalError import pytest @@ -566,13 +568,63 @@ def test_transform_preserves_rowids(fresh_db, table_type): assert previous_rows == next_rows -@pytest.mark.parametrize("strict", (False, True)) -def test_transform_strict(fresh_db, strict): - dogs = fresh_db.table("dogs", strict=strict) +@pytest.mark.parametrize( + "initial_strict,transform_strict,expected_strict", + ( + (False, None, False), + (True, None, True), + (False, True, True), + (True, False, False), + ), +) +def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_strict): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = fresh_db.table("dogs", strict=initial_strict) dogs.insert({"id": 1, "name": "Cleo"}) - assert dogs.strict == strict or not fresh_db.supports_strict - dogs.transform(not_null={"name"}) - assert dogs.strict == strict or not fresh_db.supports_strict + assert dogs.strict is initial_strict + dogs.transform(strict=transform_strict) + assert dogs.strict is expected_strict + + +def test_transform_to_strict_with_invalid_data(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = fresh_db["dogs"] + dogs.create({"id": int}) + dogs.insert({"id": "not-an-integer"}) + + with pytest.raises(sqlite3.IntegrityError): + dogs.transform(strict=True) + + assert dogs.strict is False + assert list(dogs.rows) == [{"id": "not-an-integer"}] + assert fresh_db.table_names() == ["dogs"] + + +def test_transform_strict_updates_default(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + table = fresh_db.table("items", strict=True) + table.create({"id": int}) + + table.transform(strict=False) + assert table.strict is False + + table.create({"id": int}, replace=True) + assert table.strict is False + + +@pytest.mark.parametrize("method_name", ("transform", "transform_sql")) +def test_transform_to_strict_not_supported(fresh_db, method_name): + table = fresh_db["items"] + table.create({"id": int}) + fresh_db._supports_strict = False + + with pytest.raises(TransformError, match="SQLite does not support STRICT tables"): + getattr(table, method_name)(strict=True) + + assert table.strict is False @pytest.mark.parametrize(