Merge pull request #45 from NHSDigital/bugfix/gr-ndit-943-deal_with_incorrect_filename_handling_in_si_object

fix: issue with si filename handling when the filename contains special chars
build: upgrade werkzeug to 3.1.5 to mitigate vuln & lock poetry to 2.2.1

Author: georgeRobertson

.mise.toml

@@ -1,4 +1,4 @@
 [tools]
 python="3.11"
-poetry="2.2"
+poetry="2.2.1"
 java="liberica-1.8.0"

.tool-versions

@@ -1,3 +1,3 @@
 python 3.11.14
-poetry 2.2.0
+poetry 2.2.1
 java liberica-1.8.0

poetry.lock

@@ -51,7 +51,7 @@ faker = "18.11.1"
 behave = "1.3.3"
 coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.0.13"}
-Werkzeug = "3.0.6"  # Dependency of moto which needs 3.0.6 for security vuln mitigation
+Werkzeug = "3.1.5"
 pytest = "8.4.2"
 pytest-lazy-fixtures = "1.4.0"  # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
 xlsx2csv = "0.8.2"

pyproject.toml

@@ -8,7 +8,7 @@
 import os
 import uuid
 from collections.abc import MutableMapping
-from pathlib import Path, PurePath
+from pathlib import Path
 from typing import Any, Optional
 
 from pydantic import UUID4, BaseModel, Field, FilePath, root_validator, validator
@@ -64,16 +64,9 @@ class SubmissionInfo(AuditRecord):
     datetime_received: Optional[dt.datetime] = None  # type: ignore
     """The datetime the file was received."""
 
-    @validator("file_name")
-    def _ensure_metadata_extension_removed(cls, filename):  # pylint: disable=no-self-argument
-        path = PurePath(filename)
-        return path.stem
-
     @validator("file_extension")
     def _ensure_just_file_stem(cls, extension: str):  # pylint: disable=no-self-argument
-        if "." in extension:
-            return extension.split(".")[-1]
-        return extension
+        return extension.rsplit(".", 1)[-1]
 
     @property
     def file_name_with_ext(self):

src/dve/core_engine/models.py

@@ -219,7 +219,7 @@ def submit_file_for_processing(context: Context, dataset: str, file_name: str):
     sub_info = {
         "submission_id": uuid4().hex,
         "dataset_id": dataset,
-        "file_name": file_name,
+        "file_name": file_name.rsplit(".", 1)[0],
         "file_extension": Path(file_name).suffix,
         "reporting_period_start": "2025-11-01 00:00:00",
         "reporting_period_end": "2025-11-30 23:59:59"

tests/features/steps/steps_pipeline.py

@@ -18,7 +18,7 @@
                 "submitted": {
                     "submission_id": CONSTANT_SUBMISSION_ID,
                     "dataset_id": "test0",
-                    "file_name": "my_file.csv",
+                    "file_name": "my_file",
                     "file_extension": "csv",
                 },
                 "expected": {
@@ -35,13 +35,13 @@
                 "submitted": {
                     "submission_id": CONSTANT_SUBMISSION_ID,
                     "dataset_id": "test1",
-                    "file_name": "my_file.csv.csv",
-                    "file_extension": "csv",
+                    "file_name": "my_file",
+                    "file_extension": ".csv.csv",
                 },
                 "expected": {
                     "submission_id": CONSTANT_SUBMISSION_ID,
                     "dataset_id": "test1",
-                    "file_name": "my_file.csv",
+                    "file_name": "my_file",
                     "file_extension": "csv",
                 },
             },
@@ -52,7 +52,7 @@
                 "submitted": {
                     "submission_id": CONSTANT_SUBMISSION_ID,
                     "dataset_id": "test2",
-                    "file_name": "my_file.xml",
+                    "file_name": "my_file",
                     "file_extension": "csv.csv.xml",
                 },
                 "expected": {
@@ -63,6 +63,40 @@
                 },
             },
         ),
+        # submission with multiple file extensions
+        (
+            {
+                "submitted": {
+                    "submission_id": CONSTANT_SUBMISSION_ID,
+                    "dataset_id": "test2",
+                    "file_name": "my.file.perfect",
+                    "file_extension": "csv",
+                },
+                "expected": {
+                    "submission_id": CONSTANT_SUBMISSION_ID,
+                    "dataset_id": "test2",
+                    "file_name": "my.file.perfect",
+                    "file_extension": "csv",
+                },
+            },
+        ),
+        # submission with multiple file extensions
+        (
+            {
+                "submitted": {
+                    "submission_id": CONSTANT_SUBMISSION_ID,
+                    "dataset_id": "test2",
+                    "file_name": "m_y%fil\ne.perfect",
+                    "file_extension": "csv",
+                },
+                "expected": {
+                    "submission_id": CONSTANT_SUBMISSION_ID,
+                    "dataset_id": "test2",
+                    "file_name": "m_y%fil\ne.perfect",
+                    "file_extension": "csv",
+                },
+            },
+        ),
     ],
 )
 def test_submission_info(  # pylint: disable=missing-function-docstring