fix: small refactors following code review

Author: stevenhsd

pyproject.toml

@@ -52,7 +52,6 @@ include-groups = [
 [tool.poetry.group.dev.dependencies]
 commitizen = "4.9.1"
 pre-commit = "4.3.0"
-ipykernel = "^7.1.0"
 
 [tool.poetry.group.test]
 optional = true

src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -277,12 +277,12 @@ def get_all_registered_udfs(connection: DuckDBPyConnection) -> set[str]:
 
 def duckdb_rel_to_dictionaries(
     entity: DuckDBPyRelation, batch_size=1000
-) -> Iterator[list[dict[str, Any]]]:
+) -> Iterator[dict[str, Any]]:
     """Iterator converting DuckDBPyRelation to lists of dictionaries.
     Avoids issues where dates are getting converted to datetimes using polars as intermediate."""
     # TODO - look into float conversion - floats that can't be stored exactly in binary
     # TODO - are given to nearest approximation. Tried Decimal, causes issues in arrays
     # TODO - with templating (as in complex fields, repr used when str called in jinja templating).
-    cols: tuple[str] = tuple(entity.columns) # type: ignore
+    cols: tuple[str] = tuple(entity.columns)  # type: ignore
     while rows := entity.fetchmany(batch_size):
-        yield [dict(zip(cols, rw)) for rw in rows]
+        yield from (dict(zip(cols, rw)) for rw in rows)

src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -512,25 +512,23 @@ def notify(self, entities: DuckDBEntities, *, config: Notification) -> Messages:
         if config.excluded_columns:
             matched = matched.select(StarExpression(exclude=config.excluded_columns))
 
-        for chunk in duckdb_rel_to_dictionaries(matched):
-            for record in chunk:
-                # NOTE: only templates using values directly accessible in record - nothing nested
-                # more complex extraction done in reporting module
-                messages.append(
-                    FeedbackMessage(
-                        entity=config.reporting.reporting_entity_override or config.entity_name,
-                        original_entity=config.entity_name,
-                        record=record,  # type: ignore
-                        error_location=config.reporting.legacy_location,
-                        error_message=template_object(config.reporting.message,
-                                                      record), # type: ignore
-                        failure_type=config.reporting.legacy_error_type,
-                        error_type=config.reporting.legacy_error_type,
-                        error_code=config.reporting.code,
-                        reporting_field=config.reporting.legacy_reporting_field,
-                        reporting_field_name=config.reporting.reporting_field_override,
-                        is_informational=config.reporting.emit in ("warning", "info"),
-                        category=config.reporting.category,
-                    )
+        for record in duckdb_rel_to_dictionaries(matched):
+            # NOTE: only templates using values directly accessible in record - nothing nested
+            # more complex extraction done in reporting module
+            messages.append(
+                FeedbackMessage(
+                    entity=config.reporting.reporting_entity_override or config.entity_name,
+                    original_entity=config.entity_name,
+                    record=record,  # type: ignore
+                    error_location=config.reporting.legacy_location,
+                    error_message=template_object(config.reporting.message, record),  # type: ignore
+                    failure_type=config.reporting.legacy_error_type,
+                    error_type=config.reporting.legacy_error_type,
+                    error_code=config.reporting.code,
+                    reporting_field=config.reporting.legacy_reporting_field,
+                    reporting_field_name=config.reporting.reporting_field_override,
+                    is_informational=config.reporting.emit in ("warning", "info"),
+                    category=config.reporting.category,
                 )
+            )
         return messages

tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py

@@ -90,8 +90,8 @@ def test_duckdb_rel_to_dictionaries(temp_ddb_conn: DuckDBPyConnection,
     test_rel = con.query("select dta.* from (select unnest($data) as dta)",
                                                   params={"data": data})
     res: list = []
-    for chunk in duckdb_rel_to_dictionaries(test_rel):
-        res.extend(chunk)
+    for chunk in duckdb_rel_to_dictionaries(test_rel, 1):
+        res.append(chunk)
 
     assert res == data
 

tests/testdata/movies/movies.dischema.json

@@ -21,7 +21,7 @@
                     },
                     "duration_minutes": "int",
                     "ratings": {
-                        "type": "float",
+                        "type": "NonNegativeFloat",
                         "is_array": true
                     },
                     "cast": {