diff --git a/README.md b/README.md index d8167aca6..89ee15fc9 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ To configure a new project, follow these steps: git switch -c ``` -1. Copy the `template_config.yaml` file to a new file in the `projects/config` directory and fill +1. Copy the [template_config.yaml](./template_config.yaml) file to a new file in the `projects/config` directory and fill in the details. 1. The filename of the project config should be ``.yaml diff --git a/orthanc/orthanc-anon/plugin/pixl.py b/orthanc/orthanc-anon/plugin/pixl.py index 318e5fcda..7d0640b84 100644 --- a/orthanc/orthanc-anon/plugin/pixl.py +++ b/orthanc/orthanc-anon/plugin/pixl.py @@ -386,7 +386,7 @@ def _anonymise_study_and_upload( with logger.contextualize(pseudo_study_uid=anonymised_study_uid): _upload_instances(anonymised_instances_bytes) - logger.info("Anonymised and uploaded study") + logger.success("Anonymised and uploaded study '{}', {}", project_name, study_info) return anonymised_study_uid @@ -517,7 +517,7 @@ def _anonymise_study_instances( "The anonymisation introduced the following validation errors:\n{}", parse_validation_results(dicom_validation_errors), ) - logger.success("Finished anonymising project: '{}', {}", project_name, study_info) + logger.info("Finished anonymising project '{}', {}", project_name, study_info) return anonymised_instances_bytes, anonymised_study_uid diff --git a/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py b/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py index 657718bd2..3e701b95b 100644 --- a/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py +++ b/pixl_dcmd/src/pixl_dcmd/dicom_helpers.py @@ -183,7 +183,7 @@ def _redirect_stdout_to_debug(_logger: Logger) -> Generator[None, None, None]: thread_local.stdout.seek(0) output = thread_local.stdout.readlines() for line in output: - _logger.debug(line.strip()) + _logger.trace(line.strip()) @dataclass diff --git a/pixl_dcmd/src/pixl_dcmd/main.py b/pixl_dcmd/src/pixl_dcmd/main.py index 04352dff9..70be1aa68 100644 --- a/pixl_dcmd/src/pixl_dcmd/main.py +++ b/pixl_dcmd/src/pixl_dcmd/main.py @@ -189,7 +189,7 @@ def anonymise_dicom( """ study_info = get_study_info(dataset) - logger.debug( + logger.trace( f"Processing instance for project {config.project.name}: {study_info}" ) @@ -205,7 +205,7 @@ def anonymise_dicom( msg = f"Dropping DICOM Modality: {dataset.Modality}" raise PixlSkipInstanceError(msg) - logger.debug("Anonymising instance for: {}", study_info) + logger.trace("Anonymising instance for: {}", study_info) # Apply any pixel cleaning prior to tag anonymisation # Do before anonymisation as some tag operations may rely on pixel data (e.g. burned in pixel detection). @@ -245,7 +245,6 @@ def _clean_dicom_image_pixels( :param config: Project config to use for pixel cleaning """ study_info = get_study_info(dataset) - logger.debug(f"Cleaning pixels for project {config.project.name}: {study_info}") image_operations = load_image_operations(config) deid_recipe_path = image_operations.deid_recipes @@ -254,11 +253,11 @@ def _clean_dicom_image_pixels( ) # current implementation permits only one recipe file if not deid_recipe_path: - logger.debug( + logger.trace( "No deid recipe provided for pixel cleaning, skipping pixel cleaning." ) return - + logger.debug(f"Cleaning pixels for project {config.project.name}: {study_info}") burned_pixels = has_burned_pixels(dataset, deid=deid_recipe) cleaned_pixels = clean_pixel_data(dicom_file=dataset, results=burned_pixels) @@ -331,7 +330,7 @@ def _secure_hash( if tag in dataset: message = f"Securely hashing: (0x{grp:04x},0x{el:04x})" - logger.debug(f"\t{message}") + logger.trace(message) if dataset[grp, el].VR == "LO": pat_value = str(dataset[grp, el].value) hashed_value = _hash_values(pat_value, project_slug, hash_len=64) diff --git a/projects/configs/README.md b/projects/configs/README.md new file mode 100644 index 000000000..5334dea40 --- /dev/null +++ b/projects/configs/README.md @@ -0,0 +1,8 @@ +# Project configurations for PIXL extractions + +This directory contains: + +- Configuration files for specific projects, including automated test cases, these reference the the sub-directories for their anonymisation rules. These are copied from [template_config.yaml](/template_config.yaml). Expand the [Setup PIXL in GAE section](/README.md#setup-pixl-in-gae) of the main readme find the `Configure a new project` section for more information +- [image-operations](./image-operations/): Methods for removing identifiable information from the pixel data of DICOM data +- [tag-operations](./tag-operations/): Allowlist for each DICOM tag. All projects are expected to use [tag-operations/base.yaml](./tag-operations/base.yaml), adding in extra configurations for each modality. Any DICOM tag not defined will be removed in anonymisation. +- In some cases, [tag-operations/manufacturer-overrides/](./tag-operations/manufacturer-overrides/) will be defined private DICOM tags are required from a manufacturer. \ No newline at end of file diff --git a/projects/configs/tag-operations/base.yaml b/projects/configs/tag-operations/base.yaml index 8b7c3fc58..cad5fa48f 100644 --- a/projects/configs/tag-operations/base.yaml +++ b/projects/configs/tag-operations/base.yaml @@ -138,7 +138,7 @@ - name: "Scan Options" group: 0x0018 element: 0x0022 - op: "replace" + op: "keep" #CT, MR, X-Ray - name: Slice Thickness group: 0x0018 diff --git a/projects/configs/tag-operations/thrombosis-af.yaml b/projects/configs/tag-operations/thrombosis-af.yaml new file mode 100644 index 000000000..dffb998ec --- /dev/null +++ b/projects/configs/tag-operations/thrombosis-af.yaml @@ -0,0 +1,25 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default `base.yaml` configuration file with commented links to their respective image modalities +# `base.yaml` configuration file is expected to be part of all projects configuration files + +#################################### 0008 Group ################################### +# +# + +- name: "Study Date" + group: 0x0008 + element: 0x0020 + op: "keep" diff --git a/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml new file mode 100644 index 000000000..44a0d0e8d --- /dev/null +++ b/projects/configs/uclh-investigating-correlations-of-thrombosis-risk-in-af.yaml @@ -0,0 +1,43 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project: + name: "uclh-investigating-correlations-of-thrombosis-risk-in-af" + modalities: ["CT"] # DICOM dataset modalities to retain + azure_kv_alias: null # Optional field to allow for a common secret prefix to be used + +tag_operation_files: # DICOM tag anonymisation operations + base: + - "base.yaml" # Base schema + - "ct.yaml" + - "thrombosis-af.yaml" + manufacturer_overrides: [] # Manufactuer-dependendent overrides + +allowed_manufacturers: + - regex: ".*" + exclude_series_numbers: [] + +# Filter out any series with a single instance (e.g. PACS reports, screenshots) +min_instances_per_series: 6 + +series_filters: + - "localizer" + - "localiser" + - "scout" + - "positioning" + - "mip" # for PET studies these series have PID + +destination: + dicom: "ftps" # alternatives: "dicomweb", "xnat", "none" + parquet: "ftps" # alternatives: "none"