Merge pull request #46 from NHSDigital/release_v061

build: upgrade werkzeug to 3.1.5 to mitigate vuln & lock poetry to 2.2.1
fix: issue with si filename handling when the filename contains special chars
fix: included submission status (with additional processing failure check) in error report population to reduce chance of incorrect status
fix: ensure that captured errors during business rule evaluation are being captured and logged
bump: version 0.6.0 → 0.6.1

Author: georgeRobertson

.mise.toml

@@ -1,4 +1,4 @@
 [tools]
 python="3.11"
-poetry="2.2"
+poetry="2.2.1"
 java="liberica-1.8.0"

.tool-versions

@@ -1,3 +1,3 @@
 python 3.11.14
-poetry 2.2.0
+poetry 2.2.1
 java liberica-1.8.0

CHANGELOG.md

@@ -1,3 +1,11 @@
+## v0.6.1 (2026-02-19)
+
+### Fix
+
+- ensure that captured errors during business rule evaluation are being captured and logged
+- included submission status (with additional processing failure check) in error report population to reduce chance of incorrect status
+- issue with si filename handling when the filename contains special chars
+
 ## v0.6.0 (2026-02-16)
 
 ### Feat

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nhs_dve"
-version = "0.6.0"
+version = "0.6.1"
 description = "`nhs data validation engine` is a framework used to validate data"
 authors = ["NHS England <england.contactus@nhs.net>"]
 readme = "README.md"
@@ -51,7 +51,7 @@ faker = "18.11.1"
 behave = "1.3.3"
 coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.0.13"}
-Werkzeug = "3.0.6"  # Dependency of moto which needs 3.0.6 for security vuln mitigation
+Werkzeug = "3.1.5"
 pytest = "8.4.2"
 pytest-lazy-fixtures = "1.4.0"  # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
 xlsx2csv = "0.8.2"

pyproject.toml

@@ -172,7 +172,7 @@ def _step_metadata_to_location(step_metadata: "AbstractStep") -> str:
 
     def _handle_rule_error(self, error: Exception, config: AbstractStep) -> Messages:
         """Log an error and create appropriate error messages."""
-        return render_error(error, self._step_metadata_to_location(config))
+        return render_error(error, self._step_metadata_to_location(config), self.logger)
 
     def evaluate(self, entities, *, config: AbstractStep) -> tuple[Messages, StageSuccessful]:
         """Evaluate a step definition, applying it to the entities."""
@@ -411,7 +411,7 @@ def apply_sync_filters(
                                     CriticalProcessingError(
                                         "Issue occurred while applying filter logic",
                                         messages=[
-                                            msg.error_message
+                                            msg.error_message  # type: ignore
                                             for msg in temp_messages
                                             if msg.error_message
                                         ],
@@ -439,7 +439,10 @@ def apply_sync_filters(
                                 [
                                     CriticalProcessingError(
                                         "Issue occurred while generating FeedbackMessages",
-                                        [msg.error_message for msg in temp_messages],
+                                        messages=[
+                                            msg.error_message  # type: ignore
+                                            for msg in temp_messages
+                                        ],
                                     )
                                 ],
                             )
@@ -467,7 +470,10 @@ def apply_sync_filters(
                                 [
                                     CriticalProcessingError(
                                         "Issue occurred while generating FeedbackMessages",
-                                        [msg.error_message for msg in temp_messages],
+                                        messages=[
+                                            msg.error_message  # type: ignore
+                                            for msg in temp_messages
+                                        ],
                                     )
                                 ],
                             )
@@ -505,7 +511,9 @@ def apply_sync_filters(
                             [
                                 CriticalProcessingError(
                                     "Issue occurred while filtering error records",
-                                    [msg.error_message for msg in temp_messages],
+                                    messages=[
+                                        msg.error_message for msg in temp_messages  # type: ignore
+                                    ],
                                 )
                             ],
                         )
@@ -533,7 +541,10 @@ def apply_sync_filters(
                                 [
                                     CriticalProcessingError(
                                         "Issue occurred while generating FeedbackMessages",
-                                        [msg.error_message for msg in temp_messages],
+                                        messages=[
+                                            msg.error_message  # type: ignore
+                                            for msg in temp_messages
+                                        ],
                                     )
                                 ],
                             )
@@ -592,7 +603,9 @@ def apply_rules(
                         [
                             CriticalProcessingError(
                                 "Issue occurred while applying pre filter steps",
-                                [msg.error_message for msg in stage_messages],
+                                messages=[
+                                    msg.error_message for msg in stage_messages  # type: ignore
+                                ],
                             )
                         ],
                     )
@@ -644,7 +657,9 @@ def apply_rules(
                         [
                             CriticalProcessingError(
                                 "Issue occurred while applying post filter steps",
-                                [msg.error_message for msg in stage_messages],
+                                messages=[
+                                    msg.error_message for msg in stage_messages  # type: ignore
+                                ],
                             )
                         ],
                     )

src/dve/core_engine/backends/base/rules.py

@@ -8,7 +8,7 @@
 import os
 import uuid
 from collections.abc import MutableMapping
-from pathlib import Path, PurePath
+from pathlib import Path
 from typing import Any, Optional
 
 from pydantic import UUID4, BaseModel, Field, FilePath, root_validator, validator
@@ -64,16 +64,9 @@ class SubmissionInfo(AuditRecord):
     datetime_received: Optional[dt.datetime] = None  # type: ignore
     """The datetime the file was received."""
 
-    @validator("file_name")
-    def _ensure_metadata_extension_removed(cls, filename):  # pylint: disable=no-self-argument
-        path = PurePath(filename)
-        return path.stem
-
     @validator("file_extension")
     def _ensure_just_file_stem(cls, extension: str):  # pylint: disable=no-self-argument
-        if "." in extension:
-            return extension.split(".")[-1]
-        return extension
+        return extension.rsplit(".", 1)[-1]
 
     @property
     def file_name_with_ext(self):

src/dve/core_engine/models.py

@@ -20,6 +20,7 @@
     dump_feedback_errors,
     dump_processing_errors,
     get_feedback_errors_uri,
+    get_processing_errors_uri,
     load_feedback_messages,
 )
 from dve.core_engine.backends.base.auditing import BaseAuditingManager
@@ -769,6 +770,13 @@ def error_report(
                 "error_report", submission_info.submission_id
             )
 
+        if not submission_status.processing_failed:
+            submission_status.processing_failed = fh.get_resource_exists(
+                get_processing_errors_uri(
+                    fh.joinuri(self.processed_files_path, submission_info.submission_id)
+                )
+            )
+
         if not self.processed_files_path:
             raise AttributeError("processed files path not provided")
 
@@ -797,6 +805,7 @@ def error_report(
             if value is not None and not key.endswith("_updated")
         }
         summary_items = er.SummaryItems(
+            submission_status=submission_status,
             summary_dict=summary_dict,
             row_headings=["Submission Failure", "Warning"],
         )