From 9a9494a4053654f2999396c7c4a3aaa2e6490243 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Thu, 6 Aug 2026 10:37:09 +0100 Subject: [PATCH 01/12] Add readme to project config --- projects/configs/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 projects/configs/README.md diff --git a/projects/configs/README.md b/projects/configs/README.md new file mode 100644 index 000000000..f4a4feb3e --- /dev/null +++ b/projects/configs/README.md @@ -0,0 +1,8 @@ +# Project configurations for PIXL extractions + +This directory contains: + +- Configuration files for specific projects, including automated test cases, these reference the the sub-directories for their anonymisation rules. These are copied from [template_config.yaml](/template_config.yaml). See the Config file information in the base [README.md](/README.md#the-config-yaml-file) for more details +- [image-operations](./image-operations/): Methods for removing identifiable information from the pixel data of DICOM data +- [tag-operations](./tag-operations/): Allowlist for each DICOM tag. All projects are expected to use [tag-operations/base.yaml](./tag-operations/base.yaml), adding in extra configurations for each modality. Any DICOM tag not defined will be removed in anonymisation. +- In some cases, [tag-operations/manufacturer-overrides/](./tag-operations/manufacturer-overrides/) will be defined private DICOM tags are required from a manufacturer. \ No newline at end of file From 1dcc378a4c88155f930b0a479a02e1583582068a Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Thu, 6 Aug 2026 15:02:40 +0100 Subject: [PATCH 02/12] Update readme linkage --- projects/configs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/configs/README.md b/projects/configs/README.md index f4a4feb3e..5334dea40 100644 --- a/projects/configs/README.md +++ b/projects/configs/README.md @@ -2,7 +2,7 @@ This directory contains: -- Configuration files for specific projects, including automated test cases, these reference the the sub-directories for their anonymisation rules. These are copied from [template_config.yaml](/template_config.yaml). See the Config file information in the base [README.md](/README.md#the-config-yaml-file) for more details +- Configuration files for specific projects, including automated test cases, these reference the the sub-directories for their anonymisation rules. These are copied from [template_config.yaml](/template_config.yaml). Expand the [Setup PIXL in GAE section](/README.md#setup-pixl-in-gae) of the main readme find the `Configure a new project` section for more information - [image-operations](./image-operations/): Methods for removing identifiable information from the pixel data of DICOM data - [tag-operations](./tag-operations/): Allowlist for each DICOM tag. All projects are expected to use [tag-operations/base.yaml](./tag-operations/base.yaml), adding in extra configurations for each modality. Any DICOM tag not defined will be removed in anonymisation. - In some cases, [tag-operations/manufacturer-overrides/](./tag-operations/manufacturer-overrides/) will be defined private DICOM tags are required from a manufacturer. \ No newline at end of file From 80ea2a3235eb95ab89b8da6b2e46f1be827ad1d3 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Thu, 6 Aug 2026 15:07:10 +0100 Subject: [PATCH 03/12] Make config file a link in main readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73c00b6a8..473f6d02b 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ To configure a new project, follow these steps: git switch -c ``` -1. Copy the `template_config.yaml` file to a new file in the `projects/config` directory and fill +1. Copy the [template_config.yaml](./template_config.yaml) file to a new file in the `projects/config` directory and fill in the details. 1. The filename of the project config should be ``.yaml From cb8b7d9926f1735e1e3a1b296466cc768e6f4bf9 Mon Sep 17 00:00:00 2001 From: Cartner Date: Thu, 6 Aug 2026 15:21:44 +0100 Subject: [PATCH 04/12] add config for thrombosis AF project --- projects/configs/tag-operations/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/configs/tag-operations/base.yaml b/projects/configs/tag-operations/base.yaml index 8b7c3fc58..cad5fa48f 100644 --- a/projects/configs/tag-operations/base.yaml +++ b/projects/configs/tag-operations/base.yaml @@ -138,7 +138,7 @@ - name: "Scan Options" group: 0x0018 element: 0x0022 - op: "replace" + op: "keep" #CT, MR, X-Ray - name: Slice Thickness group: 0x0018 From bcea599fbd9e3d4984084e146cba9579c71c3954 Mon Sep 17 00:00:00 2001 From: Cartner Date: Thu, 6 Aug 2026 15:48:30 +0100 Subject: [PATCH 05/12] add config files for thrombosis af project --- .../configs/tag-operations/thrombosis-af.yaml | 25 +++++++++++ ...correlations-of-thrombosis-risk-in-af.yaml | 43 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 projects/configs/tag-operations/thrombosis-af.yaml create mode 100644 projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml diff --git a/projects/configs/tag-operations/thrombosis-af.yaml b/projects/configs/tag-operations/thrombosis-af.yaml new file mode 100644 index 000000000..dffb998ec --- /dev/null +++ b/projects/configs/tag-operations/thrombosis-af.yaml @@ -0,0 +1,25 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default `base.yaml` configuration file with commented links to their respective image modalities +# `base.yaml` configuration file is expected to be part of all projects configuration files + +#################################### 0008 Group ################################### +# +# + +- name: "Study Date" + group: 0x0008 + element: 0x0020 + op: "keep" diff --git a/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml new file mode 100644 index 000000000..324dc786f --- /dev/null +++ b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml @@ -0,0 +1,43 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project: + name: "uclh-investigating-correlations-of-thrombosis-risk-in-af" + modalities: ["CT"] # DICOM dataset modalities to retain + azure_kv_alias: null # Optional field to allow for a common secret prefix to be used + +tag_operation_files: # DICOM tag anonymisation operations + base: + - "base.yaml" # Base schema + - "ct.yaml" + - "thrombosis-af.yaml" + manufacturer_overrides: [] # Manufactuer-dependendent overrides + +allowed_manufacturers: + - regex: "*" + exclude_series_numbers: [] + +# Filter out any series with a single instance (e.g. PACS reports, screenshots) +min_instances_per_series: 6 + +series_filters: + - "localizer" + - "localiser" + - "scout" + - "positioning" + - "mip" # for PET studies these series have PID + +destination: + dicom: "ftps" # alternatives: "dicomweb", "xnat", "none" + parquet: "ftps" # alternatives: "none" From e5077eb198b6c8f00ee1be3909ffb5930b8caa53 Mon Sep 17 00:00:00 2001 From: Cartner Date: Thu, 6 Aug 2026 15:53:44 +0100 Subject: [PATCH 06/12] updating config file for thormbosis af --- ...h-investigating-correlations-of-thrombosis-risk-in-af.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml index 324dc786f..6a4912af3 100644 --- a/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml +++ b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml @@ -20,8 +20,8 @@ project: tag_operation_files: # DICOM tag anonymisation operations base: - "base.yaml" # Base schema - - "ct.yaml" - - "thrombosis-af.yaml" + - "ct.yaml" + - "thrombosis-af.yaml" manufacturer_overrides: [] # Manufactuer-dependendent overrides allowed_manufacturers: From 53177787023fbb612ec1ec5ec7260157cbfab55d Mon Sep 17 00:00:00 2001 From: Cartner Date: Thu, 6 Aug 2026 15:57:09 +0100 Subject: [PATCH 07/12] update thrombosis-af config --- ...clh-investigating-correlations-of-thrombosis-risk-in-af.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml index 6a4912af3..44a0d0e8d 100644 --- a/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml +++ b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml @@ -25,7 +25,7 @@ tag_operation_files: # DICOM tag anonymisation operations manufacturer_overrides: [] # Manufactuer-dependendent overrides allowed_manufacturers: - - regex: "*" + - regex: ".*" exclude_series_numbers: [] # Filter out any series with a single instance (e.g. PACS reports, screenshots) From e5a1fa95e73abd1a07be872bf79926d3f4b471db Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Wed, 19 Aug 2026 15:22:58 +0100 Subject: [PATCH 08/12] Fixes for work in prod Hopefully --- orthanc/orthanc-anon/plugin/pixl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/orthanc/orthanc-anon/plugin/pixl.py b/orthanc/orthanc-anon/plugin/pixl.py index 5cb0504ce..318e5fcda 100644 --- a/orthanc/orthanc-anon/plugin/pixl.py +++ b/orthanc/orthanc-anon/plugin/pixl.py @@ -495,12 +495,14 @@ def _anonymise_study_instances( message = f"All instances have been skipped for study: {dict(skipped_instance_counts)}" try: record_skip_reasons_for_study( - project_name=project_name, + project_slug=project_name, study_info=study_info, skip_reasons=dict(skipped_instance_counts), ) except PixlDiscardError as e: raise PixlDiscardError(message) from e + # Still raise the exception message + raise PixlDiscardError(message) with logger.contextualize(pseudo_study_uid=anonymised_study_uid): logger.debug( From be0a5ee66f2a9f244a930d5c6649c37590233df8 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Wed, 19 Aug 2026 17:47:42 +0100 Subject: [PATCH 09/12] Reduce debug noise --- orthanc/orthanc-anon/plugin/pixl.py | 2 +- pixl_dcmd/src/pixl_dcmd/main.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/orthanc/orthanc-anon/plugin/pixl.py b/orthanc/orthanc-anon/plugin/pixl.py index 318e5fcda..8ec92539c 100644 --- a/orthanc/orthanc-anon/plugin/pixl.py +++ b/orthanc/orthanc-anon/plugin/pixl.py @@ -386,7 +386,7 @@ def _anonymise_study_and_upload( with logger.contextualize(pseudo_study_uid=anonymised_study_uid): _upload_instances(anonymised_instances_bytes) - logger.info("Anonymised and uploaded study") + logger.info("Anonymised and uploaded study to Orthanc") return anonymised_study_uid diff --git a/pixl_dcmd/src/pixl_dcmd/main.py b/pixl_dcmd/src/pixl_dcmd/main.py index 9864a78ea..b4fe4d5d4 100644 --- a/pixl_dcmd/src/pixl_dcmd/main.py +++ b/pixl_dcmd/src/pixl_dcmd/main.py @@ -189,7 +189,7 @@ def anonymise_dicom( """ study_info = get_study_info(dataset) - logger.debug( + logger.trace( f"Processing instance for project {config.project.name}: {study_info}" ) @@ -205,7 +205,7 @@ def anonymise_dicom( msg = f"Dropping DICOM Modality: {dataset.Modality}" raise PixlSkipInstanceError(msg) - logger.debug("Anonymising instance for: {}", study_info) + logger.trace("Anonymising instance for: {}", study_info) # Apply any pixel cleaning prior to tag anonymisation # Do before anonymisation as some tag operations may rely on pixel data (e.g. burned in pixel detection). @@ -245,7 +245,6 @@ def _clean_dicom_image_pixels( :param config: Project config to use for pixel cleaning """ study_info = get_study_info(dataset) - logger.debug(f"Cleaning pixels for project {config.project.name}: {study_info}") image_operations = load_image_operations(config) deid_recipe_path = image_operations.deid_recipes @@ -254,11 +253,11 @@ def _clean_dicom_image_pixels( ) # current implementation permits only one recipe file if not deid_recipe_path: - logger.debug( + logger.trace( "No deid recipe provided for pixel cleaning, skipping pixel cleaning." ) return - + logger.debug(f"Cleaning pixels for project {config.project.name}: {study_info}") burned_pixels = has_burned_pixels(dataset, deid=deid_recipe) cleaned_pixels = clean_pixel_data(dicom_file=dataset, results=burned_pixels) @@ -330,8 +329,6 @@ def _secure_hash( el = tag[1] if tag in dataset: - message = f"Securely hashing: (0x{grp:04x},0x{el:04x})" - logger.debug(f"\t{message}") if dataset[grp, el].VR == "LO": pat_value = str(dataset[grp, el].value) hashed_value = _hash_values(pat_value, project_slug, hash_len=64) @@ -348,6 +345,8 @@ def _hash_values(pat_value: str, project_slug: str, hash_len: int = 0) -> str: """ Utility function for hashing values using the hasher API. """ + message = f"Securely hashing: (0x{grp:04x},0x{el:04x})" + logger.debug(f"\t{message}") HASHER_API_AZ_NAME = config("HASHER_API_AZ_NAME") HASHER_API_PORT = config("HASHER_API_PORT") hasher_req_url = f"http://{HASHER_API_AZ_NAME}:{HASHER_API_PORT}/hash" From 8eae79a01e8374dc15d6fc10c8541358c1e30459 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Thu, 20 Aug 2026 09:46:11 +0100 Subject: [PATCH 10/12] Fix logging --- pixl_dcmd/src/pixl_dcmd/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixl_dcmd/src/pixl_dcmd/main.py b/pixl_dcmd/src/pixl_dcmd/main.py index b4fe4d5d4..ffd4ed7fd 100644 --- a/pixl_dcmd/src/pixl_dcmd/main.py +++ b/pixl_dcmd/src/pixl_dcmd/main.py @@ -329,6 +329,8 @@ def _secure_hash( el = tag[1] if tag in dataset: + message = f"Securely hashing: (0x{grp:04x},0x{el:04x})" + logger.trace(message) if dataset[grp, el].VR == "LO": pat_value = str(dataset[grp, el].value) hashed_value = _hash_values(pat_value, project_slug, hash_len=64) @@ -345,8 +347,6 @@ def _hash_values(pat_value: str, project_slug: str, hash_len: int = 0) -> str: """ Utility function for hashing values using the hasher API. """ - message = f"Securely hashing: (0x{grp:04x},0x{el:04x})" - logger.debug(f"\t{message}") HASHER_API_AZ_NAME = config("HASHER_API_AZ_NAME") HASHER_API_PORT = config("HASHER_API_PORT") hasher_req_url = f"http://{HASHER_API_AZ_NAME}:{HASHER_API_PORT}/hash" From 66aa8181dc05d59d916cf6825b84ea1816b26efe Mon Sep 17 00:00:00 2001 From: Stefan Piatek Date: Wed, 26 Aug 2026 10:54:40 +0100 Subject: [PATCH 11/12] Use trace for stdout redirect --- pixl_dcmd/src/pixl_dcmd/dicom_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py b/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py index 8babad0d8..724b04b0c 100644 --- a/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py +++ b/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py @@ -97,7 +97,7 @@ def _redirect_stdout_to_debug(_logger: Logger) -> Generator[None, None, None]: thread_local.stdout.seek(0) output = thread_local.stdout.readlines() for line in output: - _logger.debug(line.strip()) + _logger.trace(line.strip()) @dataclass From 3d6cf5d8b8d00c51ce23e1db84615e4f560efc13 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Wed, 26 Aug 2026 15:07:33 +0100 Subject: [PATCH 12/12] Update success to after upload --- orthanc/orthanc-anon/plugin/pixl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orthanc/orthanc-anon/plugin/pixl.py b/orthanc/orthanc-anon/plugin/pixl.py index 8ec92539c..7d0640b84 100644 --- a/orthanc/orthanc-anon/plugin/pixl.py +++ b/orthanc/orthanc-anon/plugin/pixl.py @@ -386,7 +386,7 @@ def _anonymise_study_and_upload( with logger.contextualize(pseudo_study_uid=anonymised_study_uid): _upload_instances(anonymised_instances_bytes) - logger.info("Anonymised and uploaded study to Orthanc") + logger.success("Anonymised and uploaded study '{}', {}", project_name, study_info) return anonymised_study_uid @@ -517,7 +517,7 @@ def _anonymise_study_instances( "The anonymisation introduced the following validation errors:\n{}", parse_validation_results(dicom_validation_errors), ) - logger.success("Finished anonymising project: '{}', {}", project_name, study_info) + logger.info("Finished anonymising project '{}', {}", project_name, study_info) return anonymised_instances_bytes, anonymised_study_uid