From 4d9c3599509ca9b4b45a32b403b4bcb585dd0d19 Mon Sep 17 00:00:00 2001 From: zxBIB Date: Thu, 9 Jul 2026 16:03:01 +0200 Subject: [PATCH 1/8] Include Rclone_copy module --- modules/nf-core/rclone/copy/environment.yml | 7 +++ modules/nf-core/rclone/copy/main.nf | 41 +++++++++++++ modules/nf-core/rclone/copy/meta.yml | 65 +++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 modules/nf-core/rclone/copy/environment.yml create mode 100644 modules/nf-core/rclone/copy/main.nf create mode 100644 modules/nf-core/rclone/copy/meta.yml diff --git a/modules/nf-core/rclone/copy/environment.yml b/modules/nf-core/rclone/copy/environment.yml new file mode 100644 index 000000000000..d6f401decc47 --- /dev/null +++ b/modules/nf-core/rclone/copy/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::rclone=1.65.0 diff --git a/modules/nf-core/rclone/copy/main.nf b/modules/nf-core/rclone/copy/main.nf new file mode 100644 index 000000000000..86d800ce2c5d --- /dev/null +++ b/modules/nf-core/rclone/copy/main.nf @@ -0,0 +1,41 @@ +process RCLONE_COPY { + + tag "${meta.id}" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c9/c947c1a7171daf074310295417d0f0afe879275e1543fa5fc2a9711e7c2c72ab/data' + : 'community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be'}" + + input: + tuple val(meta), val(source_path), val(destination_path) + path rclone_config + + output: + tuple val(meta), path("rclone-copy.log"), emit: log + tuple val("${task.process}"), val('rclone'), eval("rclone version | head -n1 | sed 's/rclone v//'"), topic: versions, emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def configArg = rclone_config ? "--config '${rclone_config}'" : '' + def transfers = task.ext.transfers ?: task.cpus + def checkers = task.ext.checkers ?: task.cpus + + """ + rclone ${configArg} copy ${args} \\ + --log-file rclone-copy.log \\ + --transfers ${transfers} \\ + --checkers ${checkers} \\ + "${source_path}" \\ + "${destination_path}" + """ + + stub: + """ + touch rclone-copy.log + """ +} \ No newline at end of file diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml new file mode 100644 index 000000000000..039a786ac513 --- /dev/null +++ b/modules/nf-core/rclone/copy/meta.yml @@ -0,0 +1,65 @@ +name: "rclone_copy" +description: Copy files or directories between local and/or remote storage using Rclone +keywords: + - rclone + - copy + - sync + - data-transfer +tools: + - "rclone": + description: "Rclone is a command line program to manage files on cloud storage" + homepage: "https://rclone.org/" + documentation: "https://rclone.org/docs/" + licence: + - "MIT" + identifier: "" + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test', single_end:false ] + - source_path: + type: string + description: | + Source path to copy from. This should be a path understood by Rclone, + such as a local path or a configured remote path. + Examples: `data/input`, `s3:bucket/path`, `gs:bucket/path`, + `remote:path/to/data`. + - destination_path: + type: string + description: | + Destination path to copy to. This should be the fully resolved target + path understood by Rclone, such as a local path or a configured remote + path. The module does not append sample IDs, basenames, or modify this + path internally. + Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, + `remote:path/to/output`. + - rclone_config: + type: file + description: | + Rclone configuration file defining the remotes used by source_path and/or + destination_path. Authentication and remote-specific options should be + configured in this file where possible. + +output: + log: + - - meta: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test', single_end:false ] + - "rclone-copy.log": + type: file + description: Rclone log file generated during the copy operation. + versions: + - - ${task.process}: + type: string + description: The name of the process. + - rclone: + type: string + description: The name of the tool. + - version: + type: string + description: The version of Rclone used. \ No newline at end of file From d70387b1eb315c4454f174579635f908f5565908 Mon Sep 17 00:00:00 2001 From: zxBIB Date: Thu, 9 Jul 2026 16:03:13 +0200 Subject: [PATCH 2/8] Include Rclone_copy module nf test --- .../nf-core/rclone/copy/tests/main.nf.test | 66 +++++++++++++++++++ .../nf-core/rclone/copy/tests/nextflow.config | 5 ++ 2 files changed, 71 insertions(+) create mode 100644 modules/nf-core/rclone/copy/tests/main.nf.test create mode 100644 modules/nf-core/rclone/copy/tests/nextflow.config diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test b/modules/nf-core/rclone/copy/tests/main.nf.test new file mode 100644 index 000000000000..45594c11a3e3 --- /dev/null +++ b/modules/nf-core/rclone/copy/tests/main.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test RCLONE_COPY" + script "../main.nf" + process "RCLONE_COPY" + + tag "modules" + tag "modules_nfcore" + tag "rclone" + tag "rclone/copy" + + test("homo_sapiens - gvcf - copy from https - dry-run") { + + config "./nextflow.config" + + when { + process { + """ + def test_file_url = params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' + + input[0] = [ + [ id:'test' ], + test_file_url, + test_file_url + ] + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.log).match("log") }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("homo_sapiens - gvcf - copy from https - stub") { + + options "-stub" + + when { + process { + """ + def test_file_url = params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' + + input[0] = [ + [ id:'test' ], + test_file_url, + test_file_url + ] + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} \ No newline at end of file diff --git a/modules/nf-core/rclone/copy/tests/nextflow.config b/modules/nf-core/rclone/copy/tests/nextflow.config new file mode 100644 index 000000000000..1245cff2a4cc --- /dev/null +++ b/modules/nf-core/rclone/copy/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: 'RCLONE_COPY' { + ext.args = '--dry-run' + } +} \ No newline at end of file From a8e0560be28d66ab6a4da32a3cdeeb38617e6243 Mon Sep 17 00:00:00 2001 From: Maria Antonia Saracco Date: Thu, 9 Jul 2026 14:52:58 +0000 Subject: [PATCH 3/8] update nf tests --- .../nf-core/rclone/copy/tests/main.nf.test | 16 ++--- .../rclone/copy/tests/main.nf.test.snap | 59 +++++++++++++++++++ .../nf-core/rclone/copy/tests/nextflow.config | 7 ++- 3 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 modules/nf-core/rclone/copy/tests/main.nf.test.snap diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test b/modules/nf-core/rclone/copy/tests/main.nf.test index 45594c11a3e3..77bf9aadc5c5 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test +++ b/modules/nf-core/rclone/copy/tests/main.nf.test @@ -16,12 +16,12 @@ nextflow_process { when { process { """ - def test_file_url = params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' + def test_file_path = params.modules_testdata_base_path.replaceFirst(/^https?:\\/\\/[^\\/]+\\//, '') + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' input[0] = [ [ id:'test' ], - test_file_url, - test_file_url + ':http:' + test_file_path, + '/tmp/test.genome.vcf.gz' ] input[1] = [] """ @@ -31,7 +31,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.log).match("log") }, + { assert process.out.log[0][1].endsWith("rclone-copy.log") }, { assert snapshot(process.out.versions).match("versions") } ) } @@ -44,12 +44,12 @@ nextflow_process { when { process { """ - def test_file_url = params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' + def test_file_path = params.modules_testdata_base_path.replaceFirst(/^https?:\\/\\/[^\\/]+\\//, '') + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' input[0] = [ [ id:'test' ], - test_file_url, - test_file_url + ':http:' + test_file_path, + '/tmp/test.genome.vcf.gz' ] input[1] = [] """ @@ -63,4 +63,4 @@ nextflow_process { ) } } -} \ No newline at end of file +} diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test.snap b/modules/nf-core/rclone/copy/tests/main.nf.test.snap new file mode 100644 index 000000000000..cd67611097f5 --- /dev/null +++ b/modules/nf-core/rclone/copy/tests/main.nf.test.snap @@ -0,0 +1,59 @@ +{ + "homo_sapiens - gvcf - copy from https - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "rclone-copy.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ], + "log": [ + [ + { + "id": "test" + }, + "rclone-copy.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ] + } + ], + "timestamp": "2026-07-09T14:37:53.802258515", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + }, + "versions": { + "content": [ + [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ] + ], + "timestamp": "2026-07-09T14:37:48.065056905", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + } +} \ No newline at end of file diff --git a/modules/nf-core/rclone/copy/tests/nextflow.config b/modules/nf-core/rclone/copy/tests/nextflow.config index 1245cff2a4cc..3234b6b8f554 100644 --- a/modules/nf-core/rclone/copy/tests/nextflow.config +++ b/modules/nf-core/rclone/copy/tests/nextflow.config @@ -1,5 +1,8 @@ process { withName: 'RCLONE_COPY' { - ext.args = '--dry-run' + ext.args = { + def base_url = (params.modules_testdata_base_path =~ /^(https?:\/\/[^\/]+)/)[0][1] + "--dry-run --http-url '${base_url}' --no-check-certificate" + } } -} \ No newline at end of file +} From 945c530e5a483ea1464747bcd609ab30be0724de Mon Sep 17 00:00:00 2001 From: Maria Antonia Saracco Date: Thu, 9 Jul 2026 14:53:16 +0000 Subject: [PATCH 4/8] update meta.yml file --- modules/nf-core/rclone/copy/meta.yml | 32 +++++++++++++++------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml index 039a786ac513..9a1ff5050d2e 100644 --- a/modules/nf-core/rclone/copy/meta.yml +++ b/modules/nf-core/rclone/copy/meta.yml @@ -36,12 +36,13 @@ input: path internally. Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, `remote:path/to/output`. - - rclone_config: - type: file - description: | - Rclone configuration file defining the remotes used by source_path and/or - destination_path. Authentication and remote-specific options should be - configured in this file where possible. + - - rclone_config: + type: file + description: | + Rclone configuration file defining the remotes used by source_path and/or + destination_path. Authentication and remote-specific options should be + configured in this file where possible. + pattern: "*.conf" output: log: @@ -53,13 +54,14 @@ output: - "rclone-copy.log": type: file description: Rclone log file generated during the copy operation. + pattern: "rclone-copy.log" versions: - - - ${task.process}: - type: string - description: The name of the process. - - rclone: - type: string - description: The name of the tool. - - version: - type: string - description: The version of Rclone used. \ No newline at end of file + - - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@antoniasaracco" +maintainers: + - "@antoniasaracco" From 310ef8db3294303518de74fab9107afa59fff0f9 Mon Sep 17 00:00:00 2001 From: Maria Antonia Saracco Date: Thu, 9 Jul 2026 15:22:15 +0000 Subject: [PATCH 5/8] fix versions emit --- modules/nf-core/rclone/copy/main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/rclone/copy/main.nf b/modules/nf-core/rclone/copy/main.nf index 86d800ce2c5d..1b81bc726d08 100644 --- a/modules/nf-core/rclone/copy/main.nf +++ b/modules/nf-core/rclone/copy/main.nf @@ -14,7 +14,7 @@ process RCLONE_COPY { output: tuple val(meta), path("rclone-copy.log"), emit: log - tuple val("${task.process}"), val('rclone'), eval("rclone version | head -n1 | sed 's/rclone v//'"), topic: versions, emit: versions + tuple val("${task.process}"), val('rclone'), eval("rclone version | head -n1 | sed 's/rclone v//'"), topic: versions, emit: versions_rclone when: task.ext.when == null || task.ext.when @@ -22,8 +22,8 @@ process RCLONE_COPY { script: def args = task.ext.args ?: '' def configArg = rclone_config ? "--config '${rclone_config}'" : '' - def transfers = task.ext.transfers ?: task.cpus - def checkers = task.ext.checkers ?: task.cpus + def transfers = task.cpus + def checkers = task.cpus """ rclone ${configArg} copy ${args} \\ @@ -38,4 +38,4 @@ process RCLONE_COPY { """ touch rclone-copy.log """ -} \ No newline at end of file +} From af75415372f49dddc6eecdab6a28d07c8eb30d22 Mon Sep 17 00:00:00 2001 From: Maria Antonia Saracco Date: Thu, 9 Jul 2026 15:22:25 +0000 Subject: [PATCH 6/8] fix meta.yml --- modules/nf-core/rclone/copy/meta.yml | 68 ++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml index 9a1ff5050d2e..96df3e8b38ae 100644 --- a/modules/nf-core/rclone/copy/meta.yml +++ b/modules/nf-core/rclone/copy/meta.yml @@ -36,14 +36,14 @@ input: path internally. Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, `remote:path/to/output`. - - - rclone_config: - type: file - description: | - Rclone configuration file defining the remotes used by source_path and/or - destination_path. Authentication and remote-specific options should be - configured in this file where possible. - pattern: "*.conf" - + - rclone_config: + type: file + description: | + Rclone configuration file defining the remotes used by source_path and/or + destination_path. Authentication and remote-specific options should be + configured in this file where possible. + pattern: "*.conf" + ontologies: [] output: log: - - meta: @@ -51,17 +51,59 @@ output: description: | Groovy Map containing sample information. e.g. [ id:'test', single_end:false ] - - "rclone-copy.log": + - rclone-copy.log: type: file description: Rclone log file generated during the copy operation. pattern: "rclone-copy.log" + ontologies: [] + versions_rclone: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - "rclone version | head -n1 | sed 's/rclone v//'": + type: eval + description: The expression to obtain the version of the tool + +topics: versions: - - - "versions.yml": - type: file - description: File containing software versions - pattern: "versions.yml" + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - "rclone version | head -n1 | sed 's/rclone v//'": + type: eval + description: The expression to obtain the version of the tool authors: - "@antoniasaracco" maintainers: - "@antoniasaracco" +containers: + docker: + linux/amd64: + name: community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + scan_id: sc-ff88b2e0040147be_1 + linux/arm64: + name: community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + scan_id: sc-ff88b2e0040147be_1 + singularity: + linux/amd64: + name: oras://community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c9/c947c1a7171daf074310295417d0f0afe879275e1543fa5fc2a9711e7c2c72ab/data + linux/arm64: + name: oras://community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c9/c947c1a7171daf074310295417d0f0afe879275e1543fa5fc2a9711e7c2c72ab/data + conda: + linux/amd64: + lock_file: modules/nf-core/rclone/copy/.conda-lock/linux_amd64-bd-ff88b2e0040147be_1.txt + linux/arm64: + lock_file: modules/nf-core/rclone/copy/.conda-lock/linux_arm64-bd-ff88b2e0040147be_1.txt From c8b4b06c33f50c0b4a06aeef3aba3937f90fb57a Mon Sep 17 00:00:00 2001 From: Maria Antonia Saracco Date: Thu, 9 Jul 2026 15:38:22 +0000 Subject: [PATCH 7/8] fix versions emit --- modules/nf-core/rclone/copy/main.nf | 3 +-- modules/nf-core/rclone/copy/tests/main.nf.test | 2 +- modules/nf-core/rclone/copy/tests/main.nf.test.snap | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/rclone/copy/main.nf b/modules/nf-core/rclone/copy/main.nf index 1b81bc726d08..242d52bc2e86 100644 --- a/modules/nf-core/rclone/copy/main.nf +++ b/modules/nf-core/rclone/copy/main.nf @@ -14,8 +14,7 @@ process RCLONE_COPY { output: tuple val(meta), path("rclone-copy.log"), emit: log - tuple val("${task.process}"), val('rclone'), eval("rclone version | head -n1 | sed 's/rclone v//'"), topic: versions, emit: versions_rclone - + tuple val("${task.process}"), val('rclone'), eval("rclone --version | sed -n '1s/^rclone v//p'"), topic: versions, emit: versions_rclone when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test b/modules/nf-core/rclone/copy/tests/main.nf.test index 77bf9aadc5c5..afca8dc7e760 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test +++ b/modules/nf-core/rclone/copy/tests/main.nf.test @@ -32,7 +32,7 @@ nextflow_process { assertAll( { assert process.success }, { assert process.out.log[0][1].endsWith("rclone-copy.log") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions_rclone).match("versions") } ) } } diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test.snap b/modules/nf-core/rclone/copy/tests/main.nf.test.snap index cd67611097f5..1a988535ce72 100644 --- a/modules/nf-core/rclone/copy/tests/main.nf.test.snap +++ b/modules/nf-core/rclone/copy/tests/main.nf.test.snap @@ -25,7 +25,7 @@ "rclone-copy.log:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "versions": [ + "versions_rclone": [ [ "RCLONE_COPY", "rclone", @@ -34,7 +34,7 @@ ] } ], - "timestamp": "2026-07-09T14:37:53.802258515", + "timestamp": "2026-07-09T15:32:10.498668888", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" @@ -50,7 +50,7 @@ ] ] ], - "timestamp": "2026-07-09T14:37:48.065056905", + "timestamp": "2026-07-09T15:36:57.123442141", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.4" From 47aed1251027def2d799b9d3322a42db9ea0f7d3 Mon Sep 17 00:00:00 2001 From: Maria Antonia Saracco Date: Thu, 9 Jul 2026 15:55:56 +0000 Subject: [PATCH 8/8] fix meta yml --- modules/nf-core/rclone/copy/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml index 96df3e8b38ae..4a9072f53d44 100644 --- a/modules/nf-core/rclone/copy/meta.yml +++ b/modules/nf-core/rclone/copy/meta.yml @@ -63,7 +63,7 @@ output: - rclone: type: string description: The name of the tool - - "rclone version | head -n1 | sed 's/rclone v//'": + - rclone --version | sed -n '1s/^rclone v//p': type: eval description: The expression to obtain the version of the tool @@ -75,7 +75,7 @@ topics: - rclone: type: string description: The name of the tool - - "rclone version | head -n1 | sed 's/rclone v//'": + - rclone --version | sed -n '1s/^rclone v//p': type: eval description: The expression to obtain the version of the tool