diff --git a/modules/nf-core/rclone/copy/environment.yml b/modules/nf-core/rclone/copy/environment.yml new file mode 100644 index 000000000000..d6f401decc47 --- /dev/null +++ b/modules/nf-core/rclone/copy/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::rclone=1.65.0 diff --git a/modules/nf-core/rclone/copy/main.nf b/modules/nf-core/rclone/copy/main.nf new file mode 100644 index 000000000000..242d52bc2e86 --- /dev/null +++ b/modules/nf-core/rclone/copy/main.nf @@ -0,0 +1,40 @@ +process RCLONE_COPY { + + tag "${meta.id}" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c9/c947c1a7171daf074310295417d0f0afe879275e1543fa5fc2a9711e7c2c72ab/data' + : 'community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be'}" + + input: + tuple val(meta), val(source_path), val(destination_path) + path rclone_config + + output: + tuple val(meta), path("rclone-copy.log"), emit: log + tuple val("${task.process}"), val('rclone'), eval("rclone --version | sed -n '1s/^rclone v//p'"), topic: versions, emit: versions_rclone + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def configArg = rclone_config ? "--config '${rclone_config}'" : '' + def transfers = task.cpus + def checkers = task.cpus + + """ + rclone ${configArg} copy ${args} \\ + --log-file rclone-copy.log \\ + --transfers ${transfers} \\ + --checkers ${checkers} \\ + "${source_path}" \\ + "${destination_path}" + """ + + stub: + """ + touch rclone-copy.log + """ +} diff --git a/modules/nf-core/rclone/copy/meta.yml b/modules/nf-core/rclone/copy/meta.yml new file mode 100644 index 000000000000..4a9072f53d44 --- /dev/null +++ b/modules/nf-core/rclone/copy/meta.yml @@ -0,0 +1,109 @@ +name: "rclone_copy" +description: Copy files or directories between local and/or remote storage using Rclone +keywords: + - rclone + - copy + - sync + - data-transfer +tools: + - "rclone": + description: "Rclone is a command line program to manage files on cloud storage" + homepage: "https://rclone.org/" + documentation: "https://rclone.org/docs/" + licence: + - "MIT" + identifier: "" + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test', single_end:false ] + - source_path: + type: string + description: | + Source path to copy from. This should be a path understood by Rclone, + such as a local path or a configured remote path. + Examples: `data/input`, `s3:bucket/path`, `gs:bucket/path`, + `remote:path/to/data`. + - destination_path: + type: string + description: | + Destination path to copy to. This should be the fully resolved target + path understood by Rclone, such as a local path or a configured remote + path. The module does not append sample IDs, basenames, or modify this + path internally. + Examples: `results/output`, `s3:bucket/output`, `gs:bucket/output`, + `remote:path/to/output`. + - rclone_config: + type: file + description: | + Rclone configuration file defining the remotes used by source_path and/or + destination_path. Authentication and remote-specific options should be + configured in this file where possible. + pattern: "*.conf" + ontologies: [] +output: + log: + - - meta: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test', single_end:false ] + - rclone-copy.log: + type: file + description: Rclone log file generated during the copy operation. + pattern: "rclone-copy.log" + ontologies: [] + versions_rclone: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - rclone --version | sed -n '1s/^rclone v//p': + type: eval + description: The expression to obtain the version of the tool + +topics: + versions: + - - ${task.process}: + type: string + description: The name of the process + - rclone: + type: string + description: The name of the tool + - rclone --version | sed -n '1s/^rclone v//p': + type: eval + description: The expression to obtain the version of the tool + +authors: + - "@antoniasaracco" +maintainers: + - "@antoniasaracco" +containers: + docker: + linux/amd64: + name: community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + scan_id: sc-ff88b2e0040147be_1 + linux/arm64: + name: community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + scan_id: sc-ff88b2e0040147be_1 + singularity: + linux/amd64: + name: oras://community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c9/c947c1a7171daf074310295417d0f0afe879275e1543fa5fc2a9711e7c2c72ab/data + linux/arm64: + name: oras://community.wave.seqera.io/library/rclone:1.65.0--ff88b2e0040147be + build_id: bd-ff88b2e0040147be_1 + https: https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c9/c947c1a7171daf074310295417d0f0afe879275e1543fa5fc2a9711e7c2c72ab/data + conda: + linux/amd64: + lock_file: modules/nf-core/rclone/copy/.conda-lock/linux_amd64-bd-ff88b2e0040147be_1.txt + linux/arm64: + lock_file: modules/nf-core/rclone/copy/.conda-lock/linux_arm64-bd-ff88b2e0040147be_1.txt diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test b/modules/nf-core/rclone/copy/tests/main.nf.test new file mode 100644 index 000000000000..afca8dc7e760 --- /dev/null +++ b/modules/nf-core/rclone/copy/tests/main.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test RCLONE_COPY" + script "../main.nf" + process "RCLONE_COPY" + + tag "modules" + tag "modules_nfcore" + tag "rclone" + tag "rclone/copy" + + test("homo_sapiens - gvcf - copy from https - dry-run") { + + config "./nextflow.config" + + when { + process { + """ + def test_file_path = params.modules_testdata_base_path.replaceFirst(/^https?:\\/\\/[^\\/]+\\//, '') + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' + + input[0] = [ + [ id:'test' ], + ':http:' + test_file_path, + '/tmp/test.genome.vcf.gz' + ] + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.log[0][1].endsWith("rclone-copy.log") }, + { assert snapshot(process.out.versions_rclone).match("versions") } + ) + } + } + + test("homo_sapiens - gvcf - copy from https - stub") { + + options "-stub" + + when { + process { + """ + def test_file_path = params.modules_testdata_base_path.replaceFirst(/^https?:\\/\\/[^\\/]+\\//, '') + 'genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz' + + input[0] = [ + [ id:'test' ], + ':http:' + test_file_path, + '/tmp/test.genome.vcf.gz' + ] + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/rclone/copy/tests/main.nf.test.snap b/modules/nf-core/rclone/copy/tests/main.nf.test.snap new file mode 100644 index 000000000000..1a988535ce72 --- /dev/null +++ b/modules/nf-core/rclone/copy/tests/main.nf.test.snap @@ -0,0 +1,59 @@ +{ + "homo_sapiens - gvcf - copy from https - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "rclone-copy.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ], + "log": [ + [ + { + "id": "test" + }, + "rclone-copy.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions_rclone": [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ] + } + ], + "timestamp": "2026-07-09T15:32:10.498668888", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + }, + "versions": { + "content": [ + [ + [ + "RCLONE_COPY", + "rclone", + "1.65.0-DEV" + ] + ] + ], + "timestamp": "2026-07-09T15:36:57.123442141", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.4" + } + } +} \ No newline at end of file diff --git a/modules/nf-core/rclone/copy/tests/nextflow.config b/modules/nf-core/rclone/copy/tests/nextflow.config new file mode 100644 index 000000000000..3234b6b8f554 --- /dev/null +++ b/modules/nf-core/rclone/copy/tests/nextflow.config @@ -0,0 +1,8 @@ +process { + withName: 'RCLONE_COPY' { + ext.args = { + def base_url = (params.modules_testdata_base_path =~ /^(https?:\/\/[^\/]+)/)[0][1] + "--dry-run --http-url '${base_url}' --no-check-certificate" + } + } +}