Merge branch 'develop_v06' of https://github.com/NHSDigital/data-validation-engine into feature/ndit-655_refdata_arrow_integration

Author: stevenhsd

.github/workflows/ci_linting.yml

@@ -3,8 +3,6 @@ name: CI Formatting & Linting
 on:
   pull_request:
     types: [opened, reopened, synchronize]
-    branches:
-      - main
 
 
 jobs:
@@ -30,17 +28,9 @@ jobs:
       - name: ensure poetry using desired python version
         run: poetry env use $(asdf which python)
 
-      - name: Cache Poetry virtualenv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-poetry-
-
       - name: Install lint dependencies
         run: |
-          make install
+          poetry install --sync --no-interaction --with lint
       
       - name: Run black
         run: poetry run black src

.github/workflows/ci_testing.yml

@@ -30,18 +30,10 @@ jobs:
 
       - name: ensure poetry using desired python version
         run: poetry env use $(asdf which python)
-    
-      - name: Cache Poetry virtualenv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-poetry-
 
       - name: Install test dependencies
         run: |
-          make install
+          poetry install --sync --no-interaction --with test
 
       - name: Run pytest and coverage
         run: |

.gitignore

@@ -32,7 +32,6 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
-poetry.lock
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

poetry.lock

@@ -18,29 +18,19 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.12"
-boto3 = "1.34.162"
-botocore = "1.34.162"
-delta-spark = "2.4.0"
-duckdb = "1.1.0" # mitigates security vuln in < 1.1.0
-formulas = "1.2.4"
-idna = "3.7"   # Downstream dep of requests but has security vuln < 3.7
-Jinja2 = "3.1.6"   # mitigates security vuln in < 3.1.6
-lxml = "4.9.1"
-openpyxl = "3.1.0"
-pandas = "2.2.2"
-polars = "0.20.14"
-pyarrow = "17.0.0"
-pydantic = "1.10.15"  # Mitigates security vuln in < 1.10.13
-pymongo = "4.6.3"
-pyspark = "3.4.4"
-pytz = "2022.1"
-PyYAML = "6.0.3"
-requests = "2.32.4"  # Mitigates security vuln in < 2.31.0
-schedula = "1.2.19"
-sqlalchemy = "2.0.19"
-typing_extensions = "4.6.2"
-urllib3 = "2.6.3"  # Mitigates security vuln in < 2.6.0
-xmltodict = "0.13.0"
+boto3 = ">=1.34.162,<1.36"  # breaking change beyond 1.36
+botocore = ">=1.34.162,<1.36"  # breaking change beyond 1.36
+delta-spark = "2.4.*"
+duckdb = "1.1.*"  # breaking changes beyond 1.1
+Jinja2 = "3.1.*"
+lxml = "^4.9.1"
+openpyxl = "^3.1"
+pandas = "^2.2.2"
+polars = "0.20.*"
+pyarrow = "^17.0.0"
+pydantic = "1.10.15"
+pyspark = "3.4.*"
+typing_extensions = "^4.6.2"
 
 [tool.poetry.group.dev]
 optional = true
@@ -62,7 +52,6 @@ behave = "1.3.3"
 coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.0.13"}
 Werkzeug = "3.0.6"  # Dependency of moto which needs 3.0.6 for security vuln mitigation
-mongomock = "4.1.2"
 pytest = "8.4.2"
 pytest-lazy-fixtures = "1.4.0"  # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
 xlsx2csv = "0.8.2"
@@ -98,10 +87,9 @@ ignore = ["B028", "D213", "D203", "D205", "D107", "D105"]
 
 [tool.mypy]
 plugins = ["pydantic.mypy"]
-enable_recursive_aliases = true
 
 [[tool.mypy.overrides]]
-module = "polars"
+module = "polars.*"
 follow_imports = "skip"
 # ^language server knows what's going on, but mypy can't find attributes on Self? type
 

pyproject.toml

@@ -17,4 +17,4 @@ def check_latest_audit_record_for_submission(context: Context, status: str):
     proc_status = pipeline._audit_tables.get_current_processing_info(
         get_submission_info(context).submission_id
     )
-    assert proc_status.processing_status == status
+    assert proc_status.processing_status == status, proc_status.processing_status