From c228a2cb6a9a0e3d8fd28d03ef38e9044657601d Mon Sep 17 00:00:00 2001 From: Anabella Trigila <18577080+atrigila@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:08:38 +0000 Subject: [PATCH 1/4] add rclone check and checksum --- .../linux_amd64-bd-2ef33c5b9132aa97_1.txt | 81 +++++++++ .../linux_arm64-bd-351f8e39202b129a_1.txt | 75 ++++++++ modules/nf-core/rclone/check/environment.yml | 6 + modules/nf-core/rclone/check/main.nf | 57 ++++++ modules/nf-core/rclone/check/meta.yml | 161 +++++++++++++++++ .../nf-core/rclone/check/tests/main.nf.test | 85 +++++++++ .../rclone/check/tests/main.nf.test.snap | 106 +++++++++++ .../nf-core/rclone/checksum/environment.yml | 6 + modules/nf-core/rclone/checksum/main.nf | 59 +++++++ modules/nf-core/rclone/checksum/meta.yml | 166 ++++++++++++++++++ .../rclone/checksum/tests/main.nf.test | 97 ++++++++++ .../rclone/checksum/tests/main.nf.test.snap | 98 +++++++++++ 12 files changed, 997 insertions(+) create mode 100644 modules/nf-core/rclone/check/.conda-lock/linux_amd64-bd-2ef33c5b9132aa97_1.txt create mode 100644 modules/nf-core/rclone/check/.conda-lock/linux_arm64-bd-351f8e39202b129a_1.txt create mode 100644 modules/nf-core/rclone/check/environment.yml create mode 100644 modules/nf-core/rclone/check/main.nf create mode 100644 modules/nf-core/rclone/check/meta.yml create mode 100644 modules/nf-core/rclone/check/tests/main.nf.test create mode 100644 modules/nf-core/rclone/check/tests/main.nf.test.snap create mode 100644 modules/nf-core/rclone/checksum/environment.yml create mode 100644 modules/nf-core/rclone/checksum/main.nf create mode 100644 modules/nf-core/rclone/checksum/meta.yml create mode 100644 modules/nf-core/rclone/checksum/tests/main.nf.test create mode 100644 modules/nf-core/rclone/checksum/tests/main.nf.test.snap diff --git a/modules/nf-core/rclone/check/.conda-lock/linux_amd64-bd-2ef33c5b9132aa97_1.txt b/modules/nf-core/rclone/check/.conda-lock/linux_amd64-bd-2ef33c5b9132aa97_1.txt new file mode 100644 index 000000000000..d9222c8ce668 --- /dev/null +++ b/modules/nf-core/rclone/check/.conda-lock/linux_amd64-bd-2ef33c5b9132aa97_1.txt @@ -0,0 +1,81 @@ + +version: 6 +environments: +default: +channels: +- url: https://conda.anaconda.org/conda-forge/ +- url: https://conda.anaconda.org/bioconda/ +- url: https://conda.anaconda.org/conda-forge/ +options: +pypi-prerelease-mode: if-necessary-or-explicit +packages: +linux-64: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_19.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_19.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/procps-ng-4.0.6-h18c060e_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/rclone-1.74.3-h519d9b9_0.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda +build_number: 20 +sha256: 1dd3fffd892081df9726d7eb7e0dea6198962ba775bd88842135a4ddb4deb3c9 +md5: a9f577daf3de00bca7c3c76c0ecbd1de +depends: +- __glibc >=2.17,<3.0.a0 +- libgomp >=7.5.0 +constrains: +- openmp_impl <0.0a0 +license: BSD-3-Clause +license_family: BSD +size: 28948 +timestamp: 1770939786096 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_19.conda +sha256: 8e0a3b5e41272e5678499b5dfc4cddb673f9e935de01eb0767ce857001229f46 +md5: 57736f29cc2b0ec0b6c2952d3f101b6a +depends: +- __glibc >=2.17,<3.0.a0 +- _openmp_mutex >=4.5 +constrains: +- libgcc-ng ==15.2.0=*_19 +- libgomp 15.2.0 he0feb66_19 +license: GPL-3.0-only WITH GCC-exception-3.1 +license_family: GPL +size: 1041084 +timestamp: 1778269013026 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_19.conda +sha256: 5abe4ab9d93f6c9757d654f1969ae2267d4505315c1f2f8fe705fd60af084f1b +md5: faac990cb7aedc7f3a2224f2c9b0c26c +depends: +- __glibc >=2.17,<3.0.a0 +license: GPL-3.0-only WITH GCC-exception-3.1 +license_family: GPL +size: 603817 +timestamp: 1778268942614 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_0.conda +sha256: fc89f74bbe362fb29fa3c037697a89bec140b346a2469a90f7936d1d7ea4d8a3 +md5: fc21868a1a5aacc937e7a18747acb8a5 +depends: +- __glibc >=2.17,<3.0.a0 +- libgcc >=14 +license: X11 AND BSD-3-Clause +size: 918956 +timestamp: 1777422145199 +- conda: https://conda.anaconda.org/conda-forge/linux-64/procps-ng-4.0.6-h18c060e_0.conda +sha256: 4ce2e1ee31a6217998f78c31ce7dc0a3e0557d9238b51d49dd20c52d467a126d +md5: f2c23a77b25efcad57d377b34bd84941 +depends: +- __glibc >=2.17,<3.0.a0 +- libgcc >=14 +- ncurses >=6.5,<7.0a0 +license: GPL-2.0-or-later AND LGPL-2.0-or-later +license_family: GPL +size: 593603 +timestamp: 1769710381284 +- conda: https://conda.anaconda.org/conda-forge/linux-64/rclone-1.74.3-h519d9b9_0.conda +sha256: 74a49ffb12e8c974519e856e5cb19d2a3c0aa9538847fa616149a4e7576a8106 +md5: df1b9c8554e9fdb3e7610cde037b0757 +license: MIT +license_family: MIT +size: 45039813 +timestamp: 1780709814295 diff --git a/modules/nf-core/rclone/check/.conda-lock/linux_arm64-bd-351f8e39202b129a_1.txt b/modules/nf-core/rclone/check/.conda-lock/linux_arm64-bd-351f8e39202b129a_1.txt new file mode 100644 index 000000000000..d8994c6d61b1 --- /dev/null +++ b/modules/nf-core/rclone/check/.conda-lock/linux_arm64-bd-351f8e39202b129a_1.txt @@ -0,0 +1,75 @@ + +version: 6 +environments: +default: +channels: +- url: https://conda.anaconda.org/conda-forge/ +- url: https://conda.anaconda.org/bioconda/ +- url: https://conda.anaconda.org/conda-forge/ +options: +pypi-prerelease-mode: if-necessary-or-explicit +packages: +linux-aarch64: +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_19.conda +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_19.conda +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.6-hf8d1292_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/procps-ng-4.0.6-h1779866_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rclone-1.74.3-h8f1e559_0.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda +build_number: 20 +sha256: a2527b1d81792a0ccd2c05850960df119c2b6d8f5fdec97f2db7d25dc23b1068 +md5: 468fd3bb9e1f671d36c2cbc677e56f1d +depends: +- libgomp >=7.5.0 +constrains: +- openmp_impl <0.0a0 +license: BSD-3-Clause +license_family: BSD +size: 28926 +timestamp: 1770939656741 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_19.conda +sha256: 4592b096e553f67799ae70d4b6167eeda3ec74587d68c7aecbf4e7b1df136681 +md5: f35b3f52d0a2ec4ffe3c89ba135cdb9a +depends: +- _openmp_mutex >=4.5 +constrains: +- libgomp 15.2.0 h8acb6b2_19 +- libgcc-ng ==15.2.0=*_19 +license: GPL-3.0-only WITH GCC-exception-3.1 +license_family: GPL +size: 622462 +timestamp: 1778268755949 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_19.conda +sha256: 2370ef0ffcbae5bede3c4bf136add4abc257245eb91f724c99bb4a43116c5a83 +md5: c5e8a379c4a2ec2aea4ba22758c001d9 +license: GPL-3.0-only WITH GCC-exception-3.1 +license_family: GPL +size: 587387 +timestamp: 1778268674393 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.6-hf8d1292_0.conda +sha256: 369db85c5cd8d99dde364ce70725d76511d9c8199e5b820c740414091bf5bcca +md5: b2a43456aa56fe80c2477a5094899eff +depends: +- libgcc >=14 +license: X11 AND BSD-3-Clause +size: 960036 +timestamp: 1777422174534 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/procps-ng-4.0.6-h1779866_0.conda +sha256: e9cbcbc94e151ada3d6dc365380aaaf591f65012c16d9a2abaea4b9b90adc402 +md5: ab7288cc39545556d1bc5e71ab2df9a9 +depends: +- libgcc >=14 +- ncurses >=6.5,<7.0a0 +license: GPL-2.0-or-later AND LGPL-2.0-or-later +license_family: GPL +size: 636733 +timestamp: 1769712412683 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rclone-1.74.3-h8f1e559_0.conda +sha256: 6aef9a68af8b200db69a915073ba2285e01e95e363626921497074588fbd8493 +md5: d63c4d8337e3a0128716f6f515a1ed05 +license: MIT +license_family: MIT +size: 41497679 +timestamp: 1780709811705 diff --git a/modules/nf-core/rclone/check/environment.yml b/modules/nf-core/rclone/check/environment.yml new file mode 100644 index 000000000000..ea9631586d02 --- /dev/null +++ b/modules/nf-core/rclone/check/environment.yml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge +dependencies: + - "conda-forge::rclone=1.74.3" diff --git a/modules/nf-core/rclone/check/main.nf b/modules/nf-core/rclone/check/main.nf new file mode 100644 index 000000000000..2242c2dd2497 --- /dev/null +++ b/modules/nf-core/rclone/check/main.nf @@ -0,0 +1,57 @@ +process RCLONE_CHECK { + tag "${meta.id}" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5d/5dfd28fd0090c69f57c9bd93ea3235d8df194e8f269b5cb3027b6b59bff567d5/data' + : 'community.wave.seqera.io/library/rclone:1.74.3--2ef33c5b9132aa97' }" + + input: + tuple val(meta), path(source, stageAs: 'source/*') + tuple val(meta2), path(destination, stageAs: 'destination/*') + + output: + tuple val(meta), path("${prefix}.combined.txt") , emit: combined , optional: true + tuple val(meta), path("${prefix}.differ.txt") , emit: differ , optional: true + tuple val(meta), path("${prefix}.missing_on_dst.txt") , emit: missing_on_dst, optional: true + tuple val(meta), path("${prefix}.missing_on_src.txt") , emit: missing_on_src, optional: true + tuple val(meta), path("${prefix}.match.txt") , emit: match , optional: true + tuple val(meta), path("${prefix}.error.txt") , emit: error , optional: true + tuple val("${task.process}"), val('rclone'), eval("rclone --version | sed -n '1s/^rclone v//p'"), topic: versions, emit: versions_rclone + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + + """ + rclone check \\ + --copy-links \\ + $args \\ + --combined ${prefix}.combined.txt \\ + --differ ${prefix}.differ.txt \\ + --missing-on-dst ${prefix}.missing_on_dst.txt \\ + --missing-on-src ${prefix}.missing_on_src.txt \\ + --match ${prefix}.match.txt \\ + --error ${prefix}.error.txt \\ + --checkers $task.cpus \\ + source \\ + destination || true + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}" + + """ + touch \\ + ${prefix}.combined.txt \\ + ${prefix}.differ.txt \\ + ${prefix}.missing_on_dst.txt \\ + ${prefix}.missing_on_src.txt \\ + ${prefix}.match.txt \\ + ${prefix}.error.txt + """ +} diff --git a/modules/nf-core/rclone/check/meta.yml b/modules/nf-core/rclone/check/meta.yml new file mode 100644 index 000000000000..1d9e6d560930 --- /dev/null +++ b/modules/nf-core/rclone/check/meta.yml @@ -0,0 +1,161 @@ +name: "rclone_check" +description: Check that files in source and destination paths match +keywords: + - check + - checksum + - cloud + - sync +tools: + - "rclone": + description: "Rclone is a command-line program to manage files on cloud storage." + homepage: "https://rclone.org/" + documentation: "https://rclone.org/commands/rclone_check/" + tool_dev_url: "https://github.com/rclone/rclone" + licence: + - "MIT" + +input: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - source: + type: file + description: File or directory containing source files to compare. + pattern: "" + ontologies: + - edam: "http://edamontology.org/data_0006" + - - meta2: + type: map + description: | + Groovy Map containing destination information + e.g. [ id:'test_dst' ] + - destination: + type: file + description: File or directory containing destination files to compare. + pattern: "" + ontologies: + - edam: "http://edamontology.org/data_0006" +output: + combined: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - ${prefix}.combined.txt: + type: file + description: Combined report of matching, missing, differing, and error paths. + pattern: "*.combined.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + differ: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - ${prefix}.differ.txt: + type: file + description: Report of paths present in both source and destination but different. + pattern: "*.differ.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + missing_on_dst: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - ${prefix}.missing_on_dst.txt: + type: file + description: Report of paths present in source but missing from destination. + pattern: "*.missing_on_dst.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + missing_on_src: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - ${prefix}.missing_on_src.txt: + type: file + description: Report of paths present in destination but missing from source. + pattern: "*.missing_on_src.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + match: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - ${prefix}.match.txt: + type: file + description: Report of matching paths. + pattern: "*.match.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + error: + - - meta: + type: map + description: | + Groovy Map containing source information + e.g. [ id:'test' ] + - ${prefix}.error.txt: + type: file + description: Report of paths with read or hash errors. + pattern: "*.error.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + versions_rclone: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - rclone --version | sed -n '1s/^rclone v//p': + type: eval + description: The expression to obtain the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - rclone --version | sed -n '1s/^rclone v//p': + type: eval + description: The expression to obtain the version of the tool +authors: + - "@atrigila" +maintainers: + - "@atrigila" +containers: + docker: + linux/arm64: + name: community.wave.seqera.io/library/rclone:1.74.3--351f8e39202b129a + build_id: bd-351f8e39202b129a_1 + scan_id: sc-0e6358c70409ff39_1 + linux/amd64: + name: community.wave.seqera.io/library/rclone:1.74.3--2ef33c5b9132aa97 + build_id: bd-2ef33c5b9132aa97_1 + scan_id: sc-2cbce8a89ac463aa_1 + singularity: + linux/amd64: + name: oras://community.wave.seqera.io/library/rclone:1.74.3--af6486c6ac506f82 + build_id: bd-af6486c6ac506f82_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5d/5dfd28fd0090c69f57c9bd93ea3235d8df194e8f269b5cb3027b6b59bff567d5/data + linux/arm64: + name: oras://community.wave.seqera.io/library/rclone:1.74.3--49a53709aa05d28d + build_id: bd-49a53709aa05d28d_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/de/de06d232599f015aa253f63b84ba2e257bb542810dc11714b88ca9b4045b80ea/data + conda: + linux/amd64: + lock_file: modules/nf-core/rclone/check/.conda-lock/linux_amd64-bd-2ef33c5b9132aa97_1.txt + linux/arm64: + lock_file: modules/nf-core/rclone/check/.conda-lock/linux_arm64-bd-351f8e39202b129a_1.txt diff --git a/modules/nf-core/rclone/check/tests/main.nf.test b/modules/nf-core/rclone/check/tests/main.nf.test new file mode 100644 index 000000000000..b389a8b1c89a --- /dev/null +++ b/modules/nf-core/rclone/check/tests/main.nf.test @@ -0,0 +1,85 @@ +nextflow_process { + + name "Test Process RCLONE_CHECK" + script "../main.nf" + process "RCLONE_CHECK" + + tag "modules" + tag "modules_nfcore" + tag "rclone" + tag "rclone/check" + + test("sarscov2 - fastq") { + when { + process { + """ + input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([match: process.out.match, versions_rclone: process.out.versions_rclone]).match() } + ) + } + } + + test("sarscov2 - fastq - stub") { + options "-stub" + + when { + process { + """ + input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([versions_rclone: process.out.versions_rclone]).match() } + ) + } + } + + test("different files - fails") { + when { + process { + """ + input[0] = [[id:'test'], file(params.modules_testdata_base_path + 'delete_me/aracne3/README.md', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'delete_me/binette_checkm2/README.md', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([differ: process.out.differ, versions_rclone: process.out.versions_rclone]).match() } + ) + } + } + + test("destination subset - partial match and missing") { + when { + process { + """ + input[0] = [[id:'test'], [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true)]] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([missing_on_dst: process.out.missing_on_dst, match: process.out.match, versions_rclone: process.out.versions_rclone]).match() } + ) + } + } +} diff --git a/modules/nf-core/rclone/check/tests/main.nf.test.snap b/modules/nf-core/rclone/check/tests/main.nf.test.snap new file mode 100644 index 000000000000..be25bb3bd552 --- /dev/null +++ b/modules/nf-core/rclone/check/tests/main.nf.test.snap @@ -0,0 +1,106 @@ +{ + "sarscov2 - fastq - stub": { + "content": [ + { + "versions_rclone": [ + [ + "RCLONE_CHECK", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-06-30T14:08:13.090156298", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, + "different files - fails": { + "content": [ + { + "differ": [ + [ + { + "id": "test" + }, + "test.differ.txt:md5,ac036608c5c402f92926505d0f0ab0e4" + ] + ], + "versions_rclone": [ + [ + "RCLONE_CHECK", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-06-30T14:09:50.727076111", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, + "sarscov2 - fastq": { + "content": [ + { + "match": [ + [ + { + "id": "test" + }, + "test.match.txt:md5,6f2d815bc8fe93026183cc59946ffc46" + ] + ], + "versions_rclone": [ + [ + "RCLONE_CHECK", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-06-30T14:09:39.430160305", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, + "destination subset - partial match and missing": { + "content": [ + { + "missing_on_dst": [ + [ + { + "id": "test" + }, + "test.missing_on_dst.txt:md5,827cbdd053152d88295a331ab8bae87c" + ] + ], + "match": [ + [ + { + "id": "test" + }, + "test.match.txt:md5,6f2d815bc8fe93026183cc59946ffc46" + ] + ], + "versions_rclone": [ + [ + "RCLONE_CHECK", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-06-30T14:09:56.49214112", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file diff --git a/modules/nf-core/rclone/checksum/environment.yml b/modules/nf-core/rclone/checksum/environment.yml new file mode 100644 index 000000000000..ea9631586d02 --- /dev/null +++ b/modules/nf-core/rclone/checksum/environment.yml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge +dependencies: + - "conda-forge::rclone=1.74.3" diff --git a/modules/nf-core/rclone/checksum/main.nf b/modules/nf-core/rclone/checksum/main.nf new file mode 100644 index 000000000000..cffa8f60142e --- /dev/null +++ b/modules/nf-core/rclone/checksum/main.nf @@ -0,0 +1,59 @@ +process RCLONE_CHECKSUM { + tag "${meta.id}" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5d/5dfd28fd0090c69f57c9bd93ea3235d8df194e8f269b5cb3027b6b59bff567d5/data' + : 'community.wave.seqera.io/library/rclone:1.74.3--2ef33c5b9132aa97' }" + + input: + tuple val(meta), path(sumfile) + val hash + tuple val(meta2), path(destination, stageAs: 'destination/*') + + output: + tuple val(meta), path("${prefix}.combined.txt") , emit: combined , optional: true + tuple val(meta), path("${prefix}.differ.txt") , emit: differ , optional: true + tuple val(meta), path("${prefix}.missing_on_dst.txt") , emit: missing_on_dst, optional: true + tuple val(meta), path("${prefix}.missing_on_src.txt") , emit: missing_on_src, optional: true + tuple val(meta), path("${prefix}.match.txt") , emit: match , optional: true + tuple val(meta), path("${prefix}.error.txt") , emit: error , optional: true + tuple val("${task.process}"), val('rclone'), eval("rclone --version | sed -n '1s/^rclone v//p'"), topic: versions, emit: versions_rclone + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + + """ + rclone checksum \\ + --copy-links \\ + $args \\ + --combined ${prefix}.combined.txt \\ + --differ ${prefix}.differ.txt \\ + --missing-on-dst ${prefix}.missing_on_dst.txt \\ + --missing-on-src ${prefix}.missing_on_src.txt \\ + --match ${prefix}.match.txt \\ + --error ${prefix}.error.txt \\ + --checkers $task.cpus \\ + $hash \\ + $sumfile \\ + destination || true + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}" + + """ + touch \\ + ${prefix}.combined.txt \\ + ${prefix}.differ.txt \\ + ${prefix}.missing_on_dst.txt \\ + ${prefix}.missing_on_src.txt \\ + ${prefix}.match.txt \\ + ${prefix}.error.txt + """ +} diff --git a/modules/nf-core/rclone/checksum/meta.yml b/modules/nf-core/rclone/checksum/meta.yml new file mode 100644 index 000000000000..05de4e7cf691 --- /dev/null +++ b/modules/nf-core/rclone/checksum/meta.yml @@ -0,0 +1,166 @@ +name: "rclone_checksum" +description: Check files in a destination path against a checksum SUM file +keywords: + - checksum + - md5 + - sha1 + - cloud + - sync +tools: + - "rclone": + description: "Rclone is a command-line program to manage files on cloud storage." + homepage: "https://rclone.org/" + documentation: "https://rclone.org/commands/rclone_checksum/" + tool_dev_url: "https://github.com/rclone/rclone" + licence: + - "MIT" + identifier: "" + +input: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - sumfile: + type: file + description: Checksum SUM file containing hashes and destination paths to verify. + pattern: "*.{md5,sha1,sha256,sum,txt}" + ontologies: + - edam: "http://edamontology.org/data_3671" + - hash: + type: string + description: Hash type to check, for example MD5, SHA1, or SHA256. + - - meta2: + type: map + description: | + Groovy Map containing destination information + e.g. [ id:'test_dst' ] + - destination: + type: file + description: File or directory containing destination files to check against the SUM file. + pattern: "" + ontologies: + - edam: "http://edamontology.org/data_0006" +output: + combined: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.combined.txt: + type: file + description: Combined report of matching, missing, differing, and error paths. + pattern: "*.combined.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + differ: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.differ.txt: + type: file + description: Report of paths present in both source SUM file and destination but different. + pattern: "*.differ.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + missing_on_dst: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.missing_on_dst.txt: + type: file + description: Report of paths present in the SUM file but missing from the destination. + pattern: "*.missing_on_dst.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + missing_on_src: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.missing_on_src.txt: + type: file + description: Report of paths present in the destination but missing from the SUM file. + pattern: "*.missing_on_src.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + match: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.match.txt: + type: file + description: Report of matching paths. + pattern: "*.match.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + error: + - - meta: + type: map + description: | + Groovy Map containing checksum source information + e.g. [ id:'test' ] + - ${prefix}.error.txt: + type: file + description: Report of paths with read or hash errors. + pattern: "*.error.txt" + ontologies: + - edam: "http://edamontology.org/data_3671" + versions_rclone: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - rclone --version | sed -n '1s/^rclone v//p': + type: eval + description: The expression to obtain the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - rclone --version | sed -n '1s/^rclone v//p': + type: eval + description: The expression to obtain the version of the tool +authors: + - "@atrigila" +maintainers: + - "@atrigila" +containers: + docker: + linux/arm64: + name: community.wave.seqera.io/library/rclone:1.74.3--351f8e39202b129a + build_id: bd-351f8e39202b129a_1 + scan_id: sc-0e6358c70409ff39_1 + linux/amd64: + name: community.wave.seqera.io/library/rclone:1.74.3--2ef33c5b9132aa97 + build_id: bd-2ef33c5b9132aa97_1 + scan_id: sc-2cbce8a89ac463aa_1 + singularity: + linux/amd64: + name: oras://community.wave.seqera.io/library/rclone:1.74.3--af6486c6ac506f82 + build_id: bd-af6486c6ac506f82_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5d/5dfd28fd0090c69f57c9bd93ea3235d8df194e8f269b5cb3027b6b59bff567d5/data + linux/arm64: + name: oras://community.wave.seqera.io/library/rclone:1.74.3--49a53709aa05d28d + build_id: bd-49a53709aa05d28d_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/de/de06d232599f015aa253f63b84ba2e257bb542810dc11714b88ca9b4045b80ea/data + conda: + linux/amd64: + lock_file: modules/nf-core/rclone/checksum/.conda-lock/linux_amd64-bd-2ef33c5b9132aa97_1.txt + linux/arm64: + lock_file: modules/nf-core/rclone/checksum/.conda-lock/linux_arm64-bd-351f8e39202b129a_1.txt diff --git a/modules/nf-core/rclone/checksum/tests/main.nf.test b/modules/nf-core/rclone/checksum/tests/main.nf.test new file mode 100644 index 000000000000..afa81da7c76d --- /dev/null +++ b/modules/nf-core/rclone/checksum/tests/main.nf.test @@ -0,0 +1,97 @@ +nextflow_process { + + name "Test Process RCLONE_CHECKSUM" + script "../main.nf" + process "RCLONE_CHECKSUM" + + tag "modules" + tag "modules_nfcore" + tag "rclone" + tag "rclone/checksum" + + test("hello - md5") { + when { + process { + """ + file('hello.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' + + input[0] = [[id:'test'], file('hello.md5')] + input[1] = 'MD5' + input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([match: process.out.match, versions_rclone: process.out.versions_rclone]).match() } + ) + } + } + + test("hello - md5 - stub") { + options "-stub" + + when { + process { + """ + file('hello.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' + + input[0] = [[id:'test'], file('hello.md5')] + input[1] = 'MD5' + input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([versions_rclone: process.out.versions_rclone]).match() } + ) + } + } + + test("hello - differs") { + when { + process { + """ + file('hello_bad.md5').text = '00000000000000000000000000000000 hello.txt\\n' + + input[0] = [[id:'test'], file('hello_bad.md5')] + input[1] = 'MD5' + input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([differ: process.out.differ, versions_rclone: process.out.versions_rclone]).match() } + ) + } + } + + test("missing destination") { + when { + process { + """ + file('missing.md5').text = 'e59ff97941044f85df5297e1c302d260 missing.txt\\n' + + input[0] = [[id:'test'], file('missing.md5')] + input[1] = 'MD5' + input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot([missing_on_dst: process.out.missing_on_dst, versions_rclone: process.out.versions_rclone]).match() } + ) + } + } +} diff --git a/modules/nf-core/rclone/checksum/tests/main.nf.test.snap b/modules/nf-core/rclone/checksum/tests/main.nf.test.snap new file mode 100644 index 000000000000..d1a28b489595 --- /dev/null +++ b/modules/nf-core/rclone/checksum/tests/main.nf.test.snap @@ -0,0 +1,98 @@ +{ + "missing destination": { + "content": [ + { + "missing_on_dst": [ + [ + { + "id": "test" + }, + "test.missing_on_dst.txt:md5,9ddf8bbca47985a064c1d7163d0235da" + ] + ], + "versions_rclone": [ + [ + "RCLONE_CHECKSUM", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-07-06T18:38:08.501473956", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, + "hello - md5 - stub": { + "content": [ + { + "versions_rclone": [ + [ + "RCLONE_CHECKSUM", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-07-06T18:37:57.76302709", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, + "hello - md5": { + "content": [ + { + "match": [ + [ + { + "id": "test" + }, + "test.match.txt:md5,a438474115db37daf9d8e1307c06eb4a" + ] + ], + "versions_rclone": [ + [ + "RCLONE_CHECKSUM", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-07-06T18:37:52.325446193", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, + "hello - differs": { + "content": [ + { + "differ": [ + [ + { + "id": "test" + }, + "test.differ.txt:md5,a438474115db37daf9d8e1307c06eb4a" + ] + ], + "versions_rclone": [ + [ + "RCLONE_CHECKSUM", + "rclone", + "1.74.3-DEV" + ] + ] + } + ], + "timestamp": "2026-07-06T18:38:03.238466704", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file From 4fe37f2fdb036ccb77be891b479646a513f5d11a Mon Sep 17 00:00:00 2001 From: Anabella Trigila <18577080+atrigila@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:10:03 -0300 Subject: [PATCH 2/4] Update main.nf Co-authored-by: Simon Pearce <24893913+SPPearce@users.noreply.github.com> --- modules/nf-core/rclone/checksum/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/rclone/checksum/main.nf b/modules/nf-core/rclone/checksum/main.nf index cffa8f60142e..0ce8e379a519 100644 --- a/modules/nf-core/rclone/checksum/main.nf +++ b/modules/nf-core/rclone/checksum/main.nf @@ -9,8 +9,8 @@ process RCLONE_CHECKSUM { input: tuple val(meta), path(sumfile) - val hash tuple val(meta2), path(destination, stageAs: 'destination/*') + val hash output: tuple val(meta), path("${prefix}.combined.txt") , emit: combined , optional: true From 98b65d0ddffcc81e7126fce50277e817f23c3f61 Mon Sep 17 00:00:00 2001 From: Anabella Trigila <18577080+atrigila@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:43:32 +0000 Subject: [PATCH 3/4] change the order of inputs --- .../nf-core/rclone/checksum/tests/main.nf.test | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/rclone/checksum/tests/main.nf.test b/modules/nf-core/rclone/checksum/tests/main.nf.test index afa81da7c76d..eddc2419095c 100644 --- a/modules/nf-core/rclone/checksum/tests/main.nf.test +++ b/modules/nf-core/rclone/checksum/tests/main.nf.test @@ -16,8 +16,8 @@ nextflow_process { file('hello.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' input[0] = [[id:'test'], file('hello.md5')] - input[1] = 'MD5' - input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[2] = 'MD5' """ } } @@ -39,8 +39,8 @@ nextflow_process { file('hello.md5').text = 'e59ff97941044f85df5297e1c302d260 hello.txt\\n' input[0] = [[id:'test'], file('hello.md5')] - input[1] = 'MD5' - input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[2] = 'MD5' """ } } @@ -60,8 +60,8 @@ nextflow_process { file('hello_bad.md5').text = '00000000000000000000000000000000 hello.txt\\n' input[0] = [[id:'test'], file('hello_bad.md5')] - input[1] = 'MD5' - input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[2] = 'MD5' """ } } @@ -81,8 +81,8 @@ nextflow_process { file('missing.md5').text = 'e59ff97941044f85df5297e1c302d260 missing.txt\\n' input[0] = [[id:'test'], file('missing.md5')] - input[1] = 'MD5' - input[2] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[1] = [[id:'test_dst'], file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true)] + input[2] = 'MD5' """ } } From 15afb1e1e4b2b4115ea5edbc41b06c105898027c Mon Sep 17 00:00:00 2001 From: Anabella Trigila <18577080+atrigila@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:49:42 +0000 Subject: [PATCH 4/4] fix linting --- modules/nf-core/rclone/checksum/meta.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/rclone/checksum/meta.yml b/modules/nf-core/rclone/checksum/meta.yml index 05de4e7cf691..1b683280a597 100644 --- a/modules/nf-core/rclone/checksum/meta.yml +++ b/modules/nf-core/rclone/checksum/meta.yml @@ -28,9 +28,6 @@ input: pattern: "*.{md5,sha1,sha256,sum,txt}" ontologies: - edam: "http://edamontology.org/data_3671" - - hash: - type: string - description: Hash type to check, for example MD5, SHA1, or SHA256. - - meta2: type: map description: | @@ -42,6 +39,9 @@ input: pattern: "" ontologies: - edam: "http://edamontology.org/data_0006" + - hash: + type: string + description: Hash type to check, for example MD5, SHA1, or SHA256. output: combined: - - meta: