From 75c43cbe5bab89a38b9fcb2472a481f5a8d6f77f Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 4 Mar 2026 23:00:38 +0530 Subject: [PATCH 1/5] Archive fieldcompare diff files in fieldcompare-diffs/ (fixes #441) --- changelog-entries/441.md | 1 + tools/tests/README.md | 2 +- tools/tests/systemtests/Systemtest.py | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 changelog-entries/441.md diff --git a/changelog-entries/441.md b/changelog-entries/441.md new file mode 100644 index 000000000..63cb05142 --- /dev/null +++ b/changelog-entries/441.md @@ -0,0 +1 @@ +- Archive fieldcompare diff VTK files into a `fieldcompare-diffs/` folder in each systemtest run directory so they are easy to find in CI artifacts when investigating comparison failures (fixes [#441](https://github.com/precice/tutorials/issues/441)). diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..d98cf1b36 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -105,7 +105,7 @@ In this case, building and running seems to work out, but the tests fail because The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. -Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files; these are copied into a `fieldcompare-diffs/` subfolder in the same run directory so you can open them (e.g. in ParaView) to see where results differ from the reference. ## Adding new tests diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6abc5a029..e6e8f4156 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -6,6 +6,8 @@ from pathlib import Path from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR, PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR +FIELDCOMPARE_DIFFS_DIR = "fieldcompare-diffs" + from metadata_parser.metdata import Tutorial, CaseCombination, Case, ReferenceResult from .SystemtestArguments import SystemtestArguments @@ -413,6 +415,24 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(1, stdout_data, stderr_data, self, elapsed_time) + def __archive_fieldcompare_diffs(self): + """ + Copy fieldcompare diff VTK files from precice-exports into fieldcompare-diffs/ + so they are easy to find in CI artifacts (issue #441). + """ + precice_exports = self.system_test_dir / PRECICE_REL_OUTPUT_DIR + if not precice_exports.exists(): + return + diff_files = list(precice_exports.glob("diff_*")) + list(precice_exports.glob("*_diff.*")) + if not diff_files: + return + dest_dir = self.system_test_dir / FIELDCOMPARE_DIFFS_DIR + dest_dir.mkdir(exist_ok=True) + for f in diff_files: + if f.is_file(): + shutil.copy2(f, dest_dir / f.name) + logging.debug(f"Archived {len(diff_files)} fieldcompare diff file(s) to {dest_dir} for {self}") + def _build_docker(self): """ Builds the docker image @@ -563,6 +583,7 @@ def run(self, run_directory: Path): fieldcompare_time=0) fieldcompare_result = self._run_field_compare() + self.__archive_fieldcompare_diffs() std_out.extend(fieldcompare_result.stdout_data) std_err.extend(fieldcompare_result.stderr_data) if fieldcompare_result.exit_code != 0: From a904f4f22a12743ac5c7b8e0b49bf781195b7776 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 7 Mar 2026 12:27:03 +0530 Subject: [PATCH 2/5] Archive fieldcompare diffs only on failure, use diff-results/ and extension-specific globs --- changelog-entries/441.md | 2 +- tools/tests/README.md | 2 +- tools/tests/systemtests/Systemtest.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/changelog-entries/441.md b/changelog-entries/441.md index 63cb05142..12774f63e 100644 --- a/changelog-entries/441.md +++ b/changelog-entries/441.md @@ -1 +1 @@ -- Archive fieldcompare diff VTK files into a `fieldcompare-diffs/` folder in each systemtest run directory so they are easy to find in CI artifacts when investigating comparison failures (fixes [#441](https://github.com/precice/tutorials/issues/441)). +- Archive fieldcompare diff VTK files into a `diff-results/` folder in each systemtest run directory on failure so they are easy to find in CI artifacts when investigating comparison failures (fixes [#441](https://github.com/precice/tutorials/issues/441)). diff --git a/tools/tests/README.md b/tools/tests/README.md index d98cf1b36..ba40a76f5 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -105,7 +105,7 @@ In this case, building and running seems to work out, but the tests fail because The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. -Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files; these are copied into a `fieldcompare-diffs/` subfolder in the same run directory so you can open them (e.g. in ParaView) to see where results differ from the reference. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files; these are copied into a `diff-results/` subfolder in the same run directory so you can open them (e.g. in ParaView) to see where results differ from the reference. ## Adding new tests diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index e6e8f4156..11000424e 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -6,7 +6,7 @@ from pathlib import Path from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR, PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR -FIELDCOMPARE_DIFFS_DIR = "fieldcompare-diffs" +DIFF_RESULTS_DIR = "diff-results" from metadata_parser.metdata import Tutorial, CaseCombination, Case, ReferenceResult from .SystemtestArguments import SystemtestArguments @@ -417,16 +417,18 @@ def _run_field_compare(self): def __archive_fieldcompare_diffs(self): """ - Copy fieldcompare diff VTK files from precice-exports into fieldcompare-diffs/ - so they are easy to find in CI artifacts (issue #441). + Copy fieldcompare diff VTK files from precice-exports into diff-results/ + so they are easy to find in CI artifacts when investigating failures (issue #441). """ precice_exports = self.system_test_dir / PRECICE_REL_OUTPUT_DIR if not precice_exports.exists(): return - diff_files = list(precice_exports.glob("diff_*")) + list(precice_exports.glob("*_diff.*")) + diff_files = [] + for pattern in ("*diff*.vtu", "*diff*.vtk", "*diff*.vtp"): + diff_files.extend(precice_exports.glob(pattern)) if not diff_files: return - dest_dir = self.system_test_dir / FIELDCOMPARE_DIFFS_DIR + dest_dir = self.system_test_dir / DIFF_RESULTS_DIR dest_dir.mkdir(exist_ok=True) for f in diff_files: if f.is_file(): @@ -583,10 +585,10 @@ def run(self, run_directory: Path): fieldcompare_time=0) fieldcompare_result = self._run_field_compare() - self.__archive_fieldcompare_diffs() std_out.extend(fieldcompare_result.stdout_data) std_err.extend(fieldcompare_result.stderr_data) if fieldcompare_result.exit_code != 0: + self.__archive_fieldcompare_diffs() self.__write_logs(std_out, std_err) logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") return SystemtestResult( From 56dd7866137e2570910e1444ece63c497c55659e Mon Sep 17 00:00:00 2001 From: Pranjal Date: Wed, 13 May 2026 23:12:59 +0530 Subject: [PATCH 3/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tools/tests/systemtests/Systemtest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index e0330e62c..e601a064b 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -469,7 +469,12 @@ def __archive_fieldcompare_diffs(self): for f in diff_files: if f.is_file(): shutil.copy2(f, dest_dir / f.name) - logging.debug(f"Archived {len(diff_files)} fieldcompare diff file(s) to {dest_dir} for {self}") + logging.debug( + "Archived %d fieldcompare diff file(s) to %s for %s", + len(diff_files), + dest_dir, + self, + ) def _build_docker(self): """ From aadc7649ffeec39fdeeb30b0e873badadff39ba8 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 13 May 2026 23:25:21 +0530 Subject: [PATCH 4/5] System tests: address Copilot review on diff archiving Move DIFF_RESULTS_DIR below imports; archive with rglob and preserve precice-exports/ paths under diff-results/; lazy debug log once after copy; README clarifies archiving only runs on fieldcompare failure. --- changelog-entries/441.md | 2 +- tools/tests/README.md | 2 +- tools/tests/systemtests/Systemtest.py | 61 +++++++++++++++++---------- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/changelog-entries/441.md b/changelog-entries/441.md index 12774f63e..3e3fa3638 100644 --- a/changelog-entries/441.md +++ b/changelog-entries/441.md @@ -1 +1 @@ -- Archive fieldcompare diff VTK files into a `diff-results/` folder in each systemtest run directory on failure so they are easy to find in CI artifacts when investigating comparison failures (fixes [#441](https://github.com/precice/tutorials/issues/441)). +- Archive fieldcompare diff VTK files into a `diff-results/` folder in each systemtest run directory on failure so they are easy to find in CI artifacts when investigating comparison failures (fixes [#441](https://github.com/precice/tutorials/issues/441)). Nested paths under `precice-exports/` are preserved under `diff-results/`. diff --git a/tools/tests/README.md b/tools/tests/README.md index 1d0dea205..7d2ba0de9 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -105,7 +105,7 @@ In this case, building and running seems to work out, but the tests fail because The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. -Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files; these are copied into a `diff-results/` subfolder in the same run directory so you can open them (e.g. in ParaView) to see where results differ from the reference. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files under `precice-exports/`; if the comparison fails, those files are copied into a `diff-results/` subfolder in the same run directory (mirroring any subpaths under `precice-exports/`) so you can open them (e.g. in ParaView) to see where results differ from the reference. On successful comparisons, `diff-results/` is therefore absent. ## Adding new tests diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index e601a064b..19d48ae7f 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -6,8 +6,6 @@ from pathlib import Path from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR, PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR -DIFF_RESULTS_DIR = "diff-results" - from metadata_parser.metdata import Tutorial, CaseCombination, Case, ReferenceResult from .SystemtestArguments import SystemtestArguments @@ -24,6 +22,8 @@ GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 900)) SHORT_TIMEOUT = 10 +DIFF_RESULTS_DIR = "diff-results" + def slugify(value, allow_unicode=False): """ @@ -451,30 +451,45 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(1, stdout_data, stderr_data, self, elapsed_time) - def __archive_fieldcompare_diffs(self): + def __archive_fieldcompare_diffs(self) -> None: """ - Copy fieldcompare diff VTK files from precice-exports into diff-results/ - so they are easy to find in CI artifacts when investigating failures (issue #441). + Copy fieldcompare diff VTK files from precice-exports/ into diff-results/, + preserving paths under precice-exports/ so nested outputs are not skipped + and identical basenames in different folders do not overwrite each other. """ - precice_exports = self.system_test_dir / PRECICE_REL_OUTPUT_DIR - if not precice_exports.exists(): - return - diff_files = [] - for pattern in ("*diff*.vtu", "*diff*.vtk", "*diff*.vtp"): - diff_files.extend(precice_exports.glob(pattern)) - if not diff_files: + exports_dir = self.system_test_dir / PRECICE_REL_OUTPUT_DIR + if not exports_dir.is_dir(): return - dest_dir = self.system_test_dir / DIFF_RESULTS_DIR - dest_dir.mkdir(exist_ok=True) - for f in diff_files: - if f.is_file(): - shutil.copy2(f, dest_dir / f.name) - logging.debug( - "Archived %d fieldcompare diff file(s) to %s for %s", - len(diff_files), - dest_dir, - self, - ) + suffixes = (".vtu", ".vtk", ".vtp") + dest_root = self.system_test_dir / DIFF_RESULTS_DIR + seen_resolved: set[Path] = set() + archived_count = 0 + for path in exports_dir.rglob("*"): + if not path.is_file(): + continue + if path.suffix.lower() not in suffixes: + continue + if "diff" not in path.name.lower(): + continue + resolved = path.resolve() + if resolved in seen_resolved: + continue + try: + rel = path.relative_to(exports_dir) + except ValueError: + continue + seen_resolved.add(resolved) + dest_path = dest_root / rel + dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, dest_path) + archived_count += 1 + if archived_count: + logging.debug( + "Archived %d fieldcompare diff file(s) to %s for %s", + archived_count, + dest_root, + self, + ) def _build_docker(self): """ From d9da063baa6e51f5403300038980b2cc114b174f Mon Sep 17 00:00:00 2001 From: Pranjal Date: Thu, 14 May 2026 11:13:13 +0530 Subject: [PATCH 5/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tools/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index 7d2ba0de9..3941bd6b4 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -105,7 +105,7 @@ In this case, building and running seems to work out, but the tests fail because The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. -Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files under `precice-exports/`; if the comparison fails, those files are copied into a `diff-results/` subfolder in the same run directory (mirroring any subpaths under `precice-exports/`) so you can open them (e.g. in ParaView) to see where results differ from the reference. On successful comparisons, `diff-results/` is therefore absent. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: `system-tests-stderr.log` and `system-tests-stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files under `precice-exports/`; if the comparison fails, those files are copied into a `diff-results/` subfolder in the same run directory (mirroring any subpaths under `precice-exports/`) so you can open them (e.g. in ParaView) to see where results differ from the reference. On successful comparisons, `diff-results/` is therefore absent. ## Adding new tests