From a5af10edccf8fc5e58e20319e91342283016e6bb Mon Sep 17 00:00:00 2001 From: buehlere Date: Thu, 13 Apr 2023 14:52:42 -0400 Subject: [PATCH 01/50] adding mskcc modules --- .gitignore | 1 + .gitmodules | 3 + cookie | 4 + install_data.sh | 27 +++++++ modules/nf-core/mskccgbcms/main.nf | 41 ++++++++++ modules/nf-core/mskccgbcms/meta.yml | 39 ++++++++++ subworkflows/nf-core/mskccprepnucleo/main.nf | 74 +++++++++++++++++++ subworkflows/nf-core/mskccprepnucleo/meta.yml | 48 ++++++++++++ tests/config/pytest_modules.yml | 45 ++++++----- tests/modules/nf-core/mskccgbcms/main.nf | 25 +++++++ .../nf-core/mskccgbcms/nextflow.config | 15 ++++ tests/modules/nf-core/mskccgbcms/test.yml | 8 ++ .../nf-core/mskccprepnucleo/main.nf | 16 ++++ .../nf-core/mskccprepnucleo/nextflow.config | 5 ++ .../nf-core/mskccprepnucleo/test.yml | 12 +++ tools-test-dataset | 1 + 16 files changed, 347 insertions(+), 17 deletions(-) create mode 100644 .gitmodules create mode 100644 cookie create mode 100644 install_data.sh create mode 100644 modules/nf-core/mskccgbcms/main.nf create mode 100644 modules/nf-core/mskccgbcms/meta.yml create mode 100644 subworkflows/nf-core/mskccprepnucleo/main.nf create mode 100644 subworkflows/nf-core/mskccprepnucleo/meta.yml create mode 100644 tests/modules/nf-core/mskccgbcms/main.nf create mode 100644 tests/modules/nf-core/mskccgbcms/nextflow.config create mode 100644 tests/modules/nf-core/mskccgbcms/test.yml create mode 100644 tests/subworkflows/nf-core/mskccprepnucleo/main.nf create mode 100644 tests/subworkflows/nf-core/mskccprepnucleo/nextflow.config create mode 100644 tests/subworkflows/nf-core/mskccprepnucleo/test.yml create mode 160000 tools-test-dataset diff --git a/.gitignore b/.gitignore index 2b9d4996156..9b1b9bd6224 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ tests/data/ __pycache__ *.pyo *.pyc +test_nucleo/* \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..290c3921c1d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tools-test-dataset"] + path = tools-test-dataset + url = https://github.mskcc.org/MSKCC-Omics-Workflows/tools-test-dataset.git diff --git a/cookie b/cookie new file mode 100644 index 00000000000..c31d9899c33 --- /dev/null +++ b/cookie @@ -0,0 +1,4 @@ +# Netscape HTTP Cookie File +# https://curl.se/docs/http-cookies.html +# This file was generated by libcurl! Edit at your own risk. + diff --git a/install_data.sh b/install_data.sh new file mode 100644 index 00000000000..c4bb76d7a7e --- /dev/null +++ b/install_data.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + + +# Test data is hosted on Google Drive at: +# https://drive.google.com/file/d/1GtT8jsBGwRoQC-5wHh06r8RFkiFBuirp/view?usp=sharing + +fileid=1GtT8jsBGwRoQC-5wHh06r8RFkiFBuirp + +filename=test_nucleo.tar.gz +foldername=test_nucleo + +# Skip if already have test data +[[ -f $filename ]] && exit 0 +[[ -d $foldername ]] && exit 0 + +curl -c ./cookie -s -k -L "https://drive.google.com/uc?export=download&id=$fileid" > /dev/null + +curl -k -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=${fileid}" -o ${filename} + +# Suppress linux warnings for MacOS tar.gz files +if [[ "$OSTYPE" == "linux-gnu" ]]; then + tar --warning=no-unknown-keyword -xzvf $filename +elif [[ "$OSTYPE" == "darwin"* ]]; then + tar -xzvf $filename +fi + +rm $filename \ No newline at end of file diff --git a/modules/nf-core/mskccgbcms/main.nf b/modules/nf-core/mskccgbcms/main.nf new file mode 100644 index 00000000000..7294681f851 --- /dev/null +++ b/modules/nf-core/mskccgbcms/main.nf @@ -0,0 +1,41 @@ +process GBCMS { + tag "$meta.id" + label 'process_single' + container "ghcr.io/msk-access/gbcms:1.2.5" + + input: + tuple val(meta), path(fasta), path(fastafai), path(bam), path(bambai), path(variant_file), val(sample), val(output), val(options) + + output: + path('variant_file.{vcf,maf}') ,emit: variant_file + path("versions.yml") + + script: + // determine if input file is a maf of vcf + // the --maf and --vcf inputs are mutually input exclusive parameters. + // Both are part of the same subcommand. Thus warrants groovy scripting. + def input_ext = variant_file.getExtension() + def variant_input = '' + if(input_ext == 'maf') { + variant_input = '--maf ' + variant_file + } + if(input_ext == 'vcf'){ + variant_input = '--vcf ' + variant_file + } + // raise exception if file extension other than maf or vcf is passed + if(variant_input == ''){ + throw new Exception("Variant file must be maf or vcf, not ${input_ext}") + } + """ + GetBaseCountsMultiSample --fasta ${fasta} \\ + ${variant_input} \\ + --output ${output} \\ + --bam $sample:${bam} ${options.args} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + GBCM: \$(echo \$(GetBaseCountsMultiSample --help) | grep -oP '[0-9]\\.[0-9]\\.[0-9]') + END_VERSIONS + """ +} + diff --git a/modules/nf-core/mskccgbcms/meta.yml b/modules/nf-core/mskccgbcms/meta.yml new file mode 100644 index 00000000000..c6e674f88c5 --- /dev/null +++ b/modules/nf-core/mskccgbcms/meta.yml @@ -0,0 +1,39 @@ +name: "mskccgbcms" +## TODO nf-core: Add a description of the module and list keywords +description: write your description here +keywords: + - sort +tools: + - "mskccgbcms": + ## TODO nf-core: Add a description and other details for the software below + description: "" + homepage: "" + documentation: "" + tool_dev_url: "" + doi: "" + licence: "" + +## TODO nf-core: Add a description of all of the variables used as input +input: + # + ## TODO nf-core: Delete / customise this example input + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +## TODO nf-core: Add a description of all of the variables used as output +output: + # + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + ## TODO nf-core: Delete / customise this example output + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +authors: + - "None" diff --git a/subworkflows/nf-core/mskccprepnucleo/main.nf b/subworkflows/nf-core/mskccprepnucleo/main.nf new file mode 100644 index 00000000000..074dd17bbcd --- /dev/null +++ b/subworkflows/nf-core/mskccprepnucleo/main.nf @@ -0,0 +1,74 @@ +// TODO nf-core: If in doubt look at other nf-core/subworkflows to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/subworkflows +// You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace: +// https://nf-co.re/join +// TODO nf-core: A subworkflow SHOULD import at least two modules + +include { FGBIO_FASTQTOBAM } from '../../../modules/nf-core/fgbio/fastqtobam/main' +include { PICARD_MERGESAMFILES } from '../../../modules/nf-core/picard/mergesamfiles/' +include { GATK4_SAMTOFASTQ } from '../../../modules/nf-core/gatk4/samtofastq/main' +include { FASTP } from '../../../modules/nf-core/fastp/main' + + +workflow PREPNUCLEO { + + take: + // TODO nf-core: edit input (take) channels + ch_fastq // channel: [ val(meta), [ bam ] ] + + + main: + + ch_versions = Channel.empty() + + // FGBIO_FASTQTOBAM: get unmerged bams + // ch_fastq is a channel, which enables parallel + // channels enable parallel: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel + ch_fastq = Channel.fromList(ch_fastq) + FGBIO_FASTQTOBAM ( + ch_fastq + ) + FGBIO_FASTQTOBAM.out.bam.map{ + meta, bam -> + [bam] + }.collect().map{ + bams -> + [[id: 'unmerged_bams'], bams ] + }.set{unmerged_bams} + ch_versions = ch_versions.mix(FGBIO_FASTQTOBAM.out.versions) + + // PICARD_MERGESAMFILES: merge bams files + PICARD_MERGESAMFILES ( + unmerged_bams + ).bam.map { + meta, bam -> + new_id = 'merged_bam' + [[id: new_id], bam ] + }.set {merged_bam} + ch_versions = ch_versions.mix(PICARD_MERGESAMFILES.out.versions) + + // GATK4_SAMTOFASTQ: get fastqs from merged bam + GATK4_SAMTOFASTQ ( + merged_bam + ).fastq.map { + meta, fastq -> + new_id = 'merged_fastq' + [[id: new_id], fastq ] + }.set {merged_fastq} + ch_versions = ch_versions.mix(GATK4_SAMTOFASTQ.out.versions) + + // GATK4_SAMTOFASTQ: Run fastp on fastqs + FASTP ( + merged_fastq, [], false, false + ) + ch_versions = ch_versions.mix(FASTP.out.versions) + + // final emit + emit: + // TODO nf-core: edit emitted channels + bam = FASTP.out.reads + + versions = ch_versions // channel: [ versions.yml ] +} + + diff --git a/subworkflows/nf-core/mskccprepnucleo/meta.yml b/subworkflows/nf-core/mskccprepnucleo/meta.yml new file mode 100644 index 00000000000..df77b88c038 --- /dev/null +++ b/subworkflows/nf-core/mskccprepnucleo/meta.yml @@ -0,0 +1,48 @@ +name: "mskccprepnucleo" +## TODO nf-core: Add a description of the subworkflow and list keywords +description: Sort SAM/BAM/CRAM file +keywords: + - sort + - bam + - sam + - cram +## TODO nf-core: Add a list of the modules used in the subworkflow +modules: + - samtools/sort + - samtools/index +## TODO nf-core: List all of the variables used as input, including their types and descriptions +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" +## TODO nf-core: List all of the variables used as output, including their types and descriptions +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - bai: + type: file + description: BAM/CRAM/SAM samtools index + pattern: "*.{bai,crai,sai}" + - csi: + type: file + description: CSI samtools index + pattern: "*.csi" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "None" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 3cd89c514c7..5bfd54245c4 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -782,14 +782,14 @@ cooler/zoomify: - modules/nf-core/cooler/zoomify/** - tests/modules/nf-core/cooler/zoomify/** -crisprcleanr/normalize: - - modules/nf-core/crisprcleanr/normalize/** - - tests/modules/nf-core/crisprcleanr/normalize/** - coreograph: - modules/nf-core/coreograph/** - tests/modules/nf-core/coreograph/** +crisprcleanr/normalize: + - modules/nf-core/crisprcleanr/normalize/** + - tests/modules/nf-core/crisprcleanr/normalize/** + crumble: - modules/nf-core/crumble/** - tests/modules/nf-core/crumble/** @@ -2178,14 +2178,14 @@ methyldackel/mbias: - modules/nf-core/methyldackel/mbias/** - tests/modules/nf-core/methyldackel/mbias/** -mindagap/mindagap: - - modules/nf-core/mindagap/mindagap/** - - tests/modules/nf-core/mindagap/mindagap/** - midas/run: - modules/nf-core/midas/run/** - tests/modules/nf-core/midas/run/** +mindagap/mindagap: + - modules/nf-core/mindagap/mindagap/** + - tests/modules/nf-core/mindagap/mindagap/** + minia: - modules/nf-core/minia/** - tests/modules/nf-core/minia/** @@ -2274,6 +2274,14 @@ msisensorpro/scan: - modules/nf-core/msisensorpro/scan/** - tests/modules/nf-core/msisensorpro/scan/** +mskccgbcms: + - modules/nf-core/mskccgbcms/** + - tests/modules/nf-core/mskccgbcms/** + +mskccprepnucleo: + - modules/nf-core/mskccprepnucleo/** + - tests/modules/nf-core/mskccprepnucleo/** + mtnucratio: - modules/nf-core/mtnucratio/** - tests/modules/nf-core/mtnucratio/** @@ -2427,15 +2435,14 @@ pangolin: - modules/nf-core/pangolin/** - tests/modules/nf-core/pangolin/** -parabricks/fq2bam: - - modules/nf-core/parabricks/fq2bam/** - - tests/modules/nf-core/parabricks/fq2bam/** - -# GitHub Actions doesn't support GPUs, so stub parabricks/applybqsr: - modules/nf-core/parabricks/applybqsr/** - tests/modules/nf-core/parabricks/applybqsr/** +parabricks/fq2bam: + - modules/nf-core/parabricks/fq2bam/** + - tests/modules/nf-core/parabricks/fq2bam/** + paraclu: - modules/nf-core/paraclu/** - tests/modules/nf-core/paraclu/** @@ -3421,6 +3428,10 @@ subworkflows/homer/groseq: - subworkflows/nf-core/homer/groseq/** - tests/subworkflows/nf-core/homer/groseq/** +subworkflows/mskccprepnucleo: + - subworkflows/nf-core/mskccprepnucleo/** + - tests/subworkflows/nf-core/mskccprepnucleo/** + subworkflows/vcf_annotate_ensemblvep: - subworkflows/nf-core/vcf_annotate_ensemblvep/** - tests/subworkflows/nf-core/vcf_annotate_ensemblvep/** @@ -3722,14 +3733,14 @@ wisecondorx/convert: - modules/nf-core/wisecondorx/convert/** - tests/modules/nf-core/wisecondorx/convert/** -wisecondorx/predict: - - modules/nf-core/wisecondorx/predict/** - - tests/modules/nf-core/wisecondorx/predict/** - wisecondorx/newref: - modules/nf-core/wisecondorx/newref/** - tests/modules/nf-core/wisecondorx/newref/** +wisecondorx/predict: + - modules/nf-core/wisecondorx/predict/** + - tests/modules/nf-core/wisecondorx/predict/** + yahs: - modules/nf-core/yahs/** - tests/modules/nf-core/yahs/** diff --git a/tests/modules/nf-core/mskccgbcms/main.nf b/tests/modules/nf-core/mskccgbcms/main.nf new file mode 100644 index 00000000000..4ec3a8f2167 --- /dev/null +++ b/tests/modules/nf-core/mskccgbcms/main.nf @@ -0,0 +1,25 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { GBCMS } from '../../../../modules/nf-core/mskccgbcms/main.nf' + +workflow test_gbcms { + + input = [ + [ id:'test' ], // meta map + file(params.fasta, checkIfExists: true), + file(params.fastafai, checkIfExists: true), + file(params.bam, checkIfExists: true), + file(params.bambai, checkIfExists: true), + file("tools-test-dataset/getbasecountmultisample/chr22.maf"), + params.sample, + // customize test specific parameters here + "variant_file.maf", + [args: '--omaf'] + + ] + + GBCMS ( input ) +} + diff --git a/tests/modules/nf-core/mskccgbcms/nextflow.config b/tests/modules/nf-core/mskccgbcms/nextflow.config new file mode 100644 index 00000000000..b690fa5204f --- /dev/null +++ b/tests/modules/nf-core/mskccgbcms/nextflow.config @@ -0,0 +1,15 @@ +params { + enable_conda = false + fasta = "tools-test-dataset/getbasecountmultisample/chr22.fasta" + fastafai = "tools-test-dataset/getbasecountmultisample/chr22.fasta.fai" + bam = "tools-test-dataset/getbasecountmultisample/chr22.bam" + bambai = "tools-test-dataset/getbasecountmultisample/chr22.bam.bai" + sample = "C-9M47TU-L001-d" + outdir = "output" +} + +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/nf-core/mskccgbcms/test.yml b/tests/modules/nf-core/mskccgbcms/test.yml new file mode 100644 index 00000000000..9590f2139f0 --- /dev/null +++ b/tests/modules/nf-core/mskccgbcms/test.yml @@ -0,0 +1,8 @@ +- name: mskccgbcms test_gbcms + command: nextflow run ./tests/modules/nf-core/mskccgbcms -entry test_gbcms -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/mskccgbcms/nextflow.config + tags: + - mskccgbcms + files: + - path: output/gbcms/variant_file.maf + md5sum: 039df7916aa49258d63929173923154a + - path: output/gbcms/versions.yml diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/main.nf b/tests/subworkflows/nf-core/mskccprepnucleo/main.nf new file mode 100644 index 00000000000..1ff2a604397 --- /dev/null +++ b/tests/subworkflows/nf-core/mskccprepnucleo/main.nf @@ -0,0 +1,16 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { PREPNUCLEO } from '../../../../subworkflows/nf-core/mskccprepnucleo/main.nf' + +workflow test_prepnucleo { + + // channels enable parralle: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel + fastq = [ + [[id:'gene1', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ad.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ad.fastq.gz')]], + [[id:'gene2', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ae.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ae.fastq.gz')]] + ] + // fastq = Channel.fromFilePairs('test_nucleo/fastq/seracare_*_R{1,2}*.fastq.gz') + PREPNUCLEO ( fastq ) +} diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/nextflow.config b/tests/subworkflows/nf-core/mskccprepnucleo/nextflow.config new file mode 100644 index 00000000000..8730f1c4b93 --- /dev/null +++ b/tests/subworkflows/nf-core/mskccprepnucleo/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/test.yml b/tests/subworkflows/nf-core/mskccprepnucleo/test.yml new file mode 100644 index 00000000000..a233015047c --- /dev/null +++ b/tests/subworkflows/nf-core/mskccprepnucleo/test.yml @@ -0,0 +1,12 @@ +## TODO nf-core: Please run the following command to build this file: +# nf-core subworkflows create-test-yml +- name: "mskccprepnucleo" + command: nextflow run ./tests/subworkflows/nf-core/mskccprepnucleo -entry test_mskccprepnucleo -c ./tests/config/nextflow.config + tags: + - "subworkflows" + - "subworkflows/mskccprepnucleo" + files: + - path: "output/mskccprepnucleo/test.bam" + md5sum: e667c7caad0bc4b7ac383fd023c654fc + - path: output/mskccprepnucleo/versions.yml + md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b diff --git a/tools-test-dataset b/tools-test-dataset new file mode 160000 index 00000000000..1a13580d414 --- /dev/null +++ b/tools-test-dataset @@ -0,0 +1 @@ +Subproject commit 1a13580d414d039928f2764519f74accc7dd0a41 From 9edde93aed49fb26cf9f7852e22163a06c0569a4 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Thu, 13 Apr 2023 15:29:24 -0400 Subject: [PATCH 02/50] adding test yaml for mskccprepnucleo --- .../nf-core/mskccprepnucleo/test.yml | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/test.yml b/tests/subworkflows/nf-core/mskccprepnucleo/test.yml index a233015047c..c0a877a0aca 100644 --- a/tests/subworkflows/nf-core/mskccprepnucleo/test.yml +++ b/tests/subworkflows/nf-core/mskccprepnucleo/test.yml @@ -1,12 +1,36 @@ -## TODO nf-core: Please run the following command to build this file: -# nf-core subworkflows create-test-yml -- name: "mskccprepnucleo" - command: nextflow run ./tests/subworkflows/nf-core/mskccprepnucleo -entry test_mskccprepnucleo -c ./tests/config/nextflow.config +- name: mskccprepnucleo test_prepnucleo + command: nextflow run ./tests/subworkflows/nf-core/mskccprepnucleo -entry test_prepnucleo -c ./tests/config/nextflow.config tags: - - "subworkflows" - - "subworkflows/mskccprepnucleo" + - fastp + - fgbio + - fgbio/fastqtobam + - gatk4 + - gatk4/samtofastq + - picard + - picard/mergesamfiles + - subworkflows + - subworkflows/mskccprepnucleo files: - - path: "output/mskccprepnucleo/test.bam" - md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: output/mskccprepnucleo/versions.yml - md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + - path: output/fastp/merged_fastq.fastp.html + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + - path: output/fastp/merged_fastq.fastp.json + md5sum: 46bbed497953f1c7b3e4941771750f6d + - path: output/fastp/merged_fastq.fastp.log + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + - path: output/fastp/merged_fastq_1.fastp.fastq.gz + md5sum: 697ad371be463f83b095e85152aa3046 + - path: output/fastp/merged_fastq_2.fastp.fastq.gz + md5sum: 26328b6503c3c6ba480adfbdc9dbf54f + - path: output/fgbio/gene1.bam + md5sum: fb9749c10bbf3917dbf739de827276f7 + - path: output/fgbio/gene2.bam + md5sum: 4596c2c2a8f94d770410246fb03f3baa + - path: output/gatk4/merged_bam_1.fastq.gz + md5sum: 8d87d76a1a07892a4bd0b2709338831e + - path: output/gatk4/merged_bam_2.fastq.gz + md5sum: 94c76d2f77a71d81ac26adcf94720ecd + - path: output/picard/unmerged_bams.bam + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' From f07a6dec220a1fbc0065d6d0ddd74913ce767e04 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Thu, 20 Apr 2023 10:32:51 -0400 Subject: [PATCH 03/50] name change --- pytest.ini | 6 +++--- .../nf-core/{mskccprepnucleo => prepnucleo}/main.nf | 0 .../nf-core/{mskccprepnucleo => prepnucleo}/meta.yml | 2 +- tests/config/pytest_modules.yml | 12 ++++++------ .../nf-core/{mskccprepnucleo => prepnucleo}/main.nf | 2 +- .../{mskccprepnucleo => prepnucleo}/nextflow.config | 0 .../nf-core/{mskccprepnucleo => prepnucleo}/test.yml | 9 ++++----- 7 files changed, 15 insertions(+), 16 deletions(-) rename subworkflows/nf-core/{mskccprepnucleo => prepnucleo}/main.nf (100%) rename subworkflows/nf-core/{mskccprepnucleo => prepnucleo}/meta.yml (98%) rename tests/subworkflows/nf-core/{mskccprepnucleo => prepnucleo}/main.nf (87%) rename tests/subworkflows/nf-core/{mskccprepnucleo => prepnucleo}/nextflow.config (100%) rename tests/subworkflows/nf-core/{mskccprepnucleo => prepnucleo}/test.yml (78%) diff --git a/pytest.ini b/pytest.ini index 02a8acac0f2..56376e3b850 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ -[pytest] -filterwarnings = - ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 +; [pytest] +; filterwarnings = +; ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 diff --git a/subworkflows/nf-core/mskccprepnucleo/main.nf b/subworkflows/nf-core/prepnucleo/main.nf similarity index 100% rename from subworkflows/nf-core/mskccprepnucleo/main.nf rename to subworkflows/nf-core/prepnucleo/main.nf diff --git a/subworkflows/nf-core/mskccprepnucleo/meta.yml b/subworkflows/nf-core/prepnucleo/meta.yml similarity index 98% rename from subworkflows/nf-core/mskccprepnucleo/meta.yml rename to subworkflows/nf-core/prepnucleo/meta.yml index df77b88c038..a1091e74482 100644 --- a/subworkflows/nf-core/mskccprepnucleo/meta.yml +++ b/subworkflows/nf-core/prepnucleo/meta.yml @@ -1,4 +1,4 @@ -name: "mskccprepnucleo" +name: "prepnucleo" ## TODO nf-core: Add a description of the subworkflow and list keywords description: Sort SAM/BAM/CRAM file keywords: diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 5bfd54245c4..43279f42549 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -2278,9 +2278,9 @@ mskccgbcms: - modules/nf-core/mskccgbcms/** - tests/modules/nf-core/mskccgbcms/** -mskccprepnucleo: - - modules/nf-core/mskccprepnucleo/** - - tests/modules/nf-core/mskccprepnucleo/** +prepnucleo: + - modules/nf-core/prepnucleo/** + - tests/modules/nf-core/prepnucleo/** mtnucratio: - modules/nf-core/mtnucratio/** @@ -3428,9 +3428,9 @@ subworkflows/homer/groseq: - subworkflows/nf-core/homer/groseq/** - tests/subworkflows/nf-core/homer/groseq/** -subworkflows/mskccprepnucleo: - - subworkflows/nf-core/mskccprepnucleo/** - - tests/subworkflows/nf-core/mskccprepnucleo/** +subworkflows/prepnucleo: + - subworkflows/nf-core/prepnucleo/** + - tests/subworkflows/nf-core/prepnucleo/** subworkflows/vcf_annotate_ensemblvep: - subworkflows/nf-core/vcf_annotate_ensemblvep/** diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/main.nf b/tests/subworkflows/nf-core/prepnucleo/main.nf similarity index 87% rename from tests/subworkflows/nf-core/mskccprepnucleo/main.nf rename to tests/subworkflows/nf-core/prepnucleo/main.nf index 1ff2a604397..39d54f02c2b 100644 --- a/tests/subworkflows/nf-core/mskccprepnucleo/main.nf +++ b/tests/subworkflows/nf-core/prepnucleo/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { PREPNUCLEO } from '../../../../subworkflows/nf-core/mskccprepnucleo/main.nf' +include { PREPNUCLEO } from '../../../../subworkflows/nf-core/prepnucleo/main.nf' workflow test_prepnucleo { diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/nextflow.config b/tests/subworkflows/nf-core/prepnucleo/nextflow.config similarity index 100% rename from tests/subworkflows/nf-core/mskccprepnucleo/nextflow.config rename to tests/subworkflows/nf-core/prepnucleo/nextflow.config diff --git a/tests/subworkflows/nf-core/mskccprepnucleo/test.yml b/tests/subworkflows/nf-core/prepnucleo/test.yml similarity index 78% rename from tests/subworkflows/nf-core/mskccprepnucleo/test.yml rename to tests/subworkflows/nf-core/prepnucleo/test.yml index c0a877a0aca..010f761ffc1 100644 --- a/tests/subworkflows/nf-core/mskccprepnucleo/test.yml +++ b/tests/subworkflows/nf-core/prepnucleo/test.yml @@ -1,5 +1,5 @@ -- name: mskccprepnucleo test_prepnucleo - command: nextflow run ./tests/subworkflows/nf-core/mskccprepnucleo -entry test_prepnucleo -c ./tests/config/nextflow.config +- name: prepnucleo test_prepnucleo + command: nextflow run ./tests/subworkflows/nf-core/prepnucleo -entry test_prepnucleo -c ./tests/config/nextflow.config tags: - fastp - fgbio @@ -9,7 +9,7 @@ - picard - picard/mergesamfiles - subworkflows - - subworkflows/mskccprepnucleo + - subworkflows/prepnucleo files: - path: output/fastp/merged_fastq.fastp.html contains: @@ -17,8 +17,7 @@ - path: output/fastp/merged_fastq.fastp.json md5sum: 46bbed497953f1c7b3e4941771750f6d - path: output/fastp/merged_fastq.fastp.log - contains: - - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + md5sum: 2284445cf51519aca6fabce18dcaac33 - path: output/fastp/merged_fastq_1.fastp.fastq.gz md5sum: 697ad371be463f83b095e85152aa3046 - path: output/fastp/merged_fastq_2.fastp.fastq.gz From 043bec3675ddf4b85255eef8f07e122071ec885c Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Thu, 20 Apr 2023 11:31:03 -0400 Subject: [PATCH 04/50] removing redundant channel --- subworkflows/nf-core/prepnucleo/main.nf | 1 - tests/subworkflows/nf-core/prepnucleo/main.nf | 2 +- tests/subworkflows/nf-core/prepnucleo/test.yml | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/nf-core/prepnucleo/main.nf b/subworkflows/nf-core/prepnucleo/main.nf index 074dd17bbcd..eabec561586 100644 --- a/subworkflows/nf-core/prepnucleo/main.nf +++ b/subworkflows/nf-core/prepnucleo/main.nf @@ -24,7 +24,6 @@ workflow PREPNUCLEO { // FGBIO_FASTQTOBAM: get unmerged bams // ch_fastq is a channel, which enables parallel // channels enable parallel: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel - ch_fastq = Channel.fromList(ch_fastq) FGBIO_FASTQTOBAM ( ch_fastq ) diff --git a/tests/subworkflows/nf-core/prepnucleo/main.nf b/tests/subworkflows/nf-core/prepnucleo/main.nf index 39d54f02c2b..6f45d7c8da6 100644 --- a/tests/subworkflows/nf-core/prepnucleo/main.nf +++ b/tests/subworkflows/nf-core/prepnucleo/main.nf @@ -11,6 +11,6 @@ workflow test_prepnucleo { [[id:'gene1', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ad.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ad.fastq.gz')]], [[id:'gene2', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ae.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ae.fastq.gz')]] ] - // fastq = Channel.fromFilePairs('test_nucleo/fastq/seracare_*_R{1,2}*.fastq.gz') + fastq = ch_fastq = Channel.fromList(fastq) PREPNUCLEO ( fastq ) } diff --git a/tests/subworkflows/nf-core/prepnucleo/test.yml b/tests/subworkflows/nf-core/prepnucleo/test.yml index 010f761ffc1..a23b33323f2 100644 --- a/tests/subworkflows/nf-core/prepnucleo/test.yml +++ b/tests/subworkflows/nf-core/prepnucleo/test.yml @@ -17,7 +17,8 @@ - path: output/fastp/merged_fastq.fastp.json md5sum: 46bbed497953f1c7b3e4941771750f6d - path: output/fastp/merged_fastq.fastp.log - md5sum: 2284445cf51519aca6fabce18dcaac33 + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' - path: output/fastp/merged_fastq_1.fastp.fastq.gz md5sum: 697ad371be463f83b095e85152aa3046 - path: output/fastp/merged_fastq_2.fastp.fastq.gz From a830c376cd2c0a38ab323beebaec56f2ec225138 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Thu, 27 Apr 2023 12:13:21 -0400 Subject: [PATCH 05/50] re-doing first access subworkflow --- fastp/merged_fastq.fastp.html | 1 + fastp/merged_fastq.fastp.json | 1 + fastp/merged_fastq.fastp.log | 1 + fastp/merged_fastq_1.fastp.fastq.gz | 1 + fastp/merged_fastq_2.fastp.fastq.gz | 1 + fastp/versions.yml | 1 + fgbio/gene1.bam | 1 + fgbio/gene2.bam | 1 + fgbio/versions.yml | 1 + gatk4/merged_bam_1.fastq.gz | 1 + gatk4/merged_bam_2.fastq.gz | 1 + gatk4/versions.yml | 1 + picard/unmerged_bams.bam | 1 + picard/versions.yml | 1 + subworkflows/nf-core/extractumi/main.nf | 71 +++++++++++++++++++ subworkflows/nf-core/extractumi/meta.yml | 48 +++++++++++++ tests/config/pytest_modules.yml | 12 ++-- tests/subworkflows/nf-core/extractumi/main.nf | 16 +++++ .../nf-core/extractumi/nextflow.config | 5 ++ .../subworkflows/nf-core/extractumi/test.yml | 35 +++++++++ 20 files changed, 197 insertions(+), 4 deletions(-) create mode 120000 fastp/merged_fastq.fastp.html create mode 120000 fastp/merged_fastq.fastp.json create mode 120000 fastp/merged_fastq.fastp.log create mode 120000 fastp/merged_fastq_1.fastp.fastq.gz create mode 120000 fastp/merged_fastq_2.fastp.fastq.gz create mode 120000 fastp/versions.yml create mode 120000 fgbio/gene1.bam create mode 120000 fgbio/gene2.bam create mode 120000 fgbio/versions.yml create mode 120000 gatk4/merged_bam_1.fastq.gz create mode 120000 gatk4/merged_bam_2.fastq.gz create mode 120000 gatk4/versions.yml create mode 120000 picard/unmerged_bams.bam create mode 120000 picard/versions.yml create mode 100644 subworkflows/nf-core/extractumi/main.nf create mode 100644 subworkflows/nf-core/extractumi/meta.yml create mode 100644 tests/subworkflows/nf-core/extractumi/main.nf create mode 100644 tests/subworkflows/nf-core/extractumi/nextflow.config create mode 100644 tests/subworkflows/nf-core/extractumi/test.yml diff --git a/fastp/merged_fastq.fastp.html b/fastp/merged_fastq.fastp.html new file mode 120000 index 00000000000..029d816faf5 --- /dev/null +++ b/fastp/merged_fastq.fastp.html @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq.fastp.html \ No newline at end of file diff --git a/fastp/merged_fastq.fastp.json b/fastp/merged_fastq.fastp.json new file mode 120000 index 00000000000..af8fd01aa65 --- /dev/null +++ b/fastp/merged_fastq.fastp.json @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq.fastp.json \ No newline at end of file diff --git a/fastp/merged_fastq.fastp.log b/fastp/merged_fastq.fastp.log new file mode 120000 index 00000000000..bb752ce5030 --- /dev/null +++ b/fastp/merged_fastq.fastp.log @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq.fastp.log \ No newline at end of file diff --git a/fastp/merged_fastq_1.fastp.fastq.gz b/fastp/merged_fastq_1.fastp.fastq.gz new file mode 120000 index 00000000000..ca9b440171b --- /dev/null +++ b/fastp/merged_fastq_1.fastp.fastq.gz @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq_1.fastp.fastq.gz \ No newline at end of file diff --git a/fastp/merged_fastq_2.fastp.fastq.gz b/fastp/merged_fastq_2.fastp.fastq.gz new file mode 120000 index 00000000000..f1971eef0e2 --- /dev/null +++ b/fastp/merged_fastq_2.fastp.fastq.gz @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq_2.fastp.fastq.gz \ No newline at end of file diff --git a/fastp/versions.yml b/fastp/versions.yml new file mode 120000 index 00000000000..760a58e3ef5 --- /dev/null +++ b/fastp/versions.yml @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/versions.yml \ No newline at end of file diff --git a/fgbio/gene1.bam b/fgbio/gene1.bam new file mode 120000 index 00000000000..1f70c5c6134 --- /dev/null +++ b/fgbio/gene1.bam @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/6e/9a2bf920bcbc26f872eae76763ae65/gene1.bam \ No newline at end of file diff --git a/fgbio/gene2.bam b/fgbio/gene2.bam new file mode 120000 index 00000000000..479fc3a8414 --- /dev/null +++ b/fgbio/gene2.bam @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/b6/df527007cf5a649bdac5b3662c6423/gene2.bam \ No newline at end of file diff --git a/fgbio/versions.yml b/fgbio/versions.yml new file mode 120000 index 00000000000..2b76387d35e --- /dev/null +++ b/fgbio/versions.yml @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/6e/9a2bf920bcbc26f872eae76763ae65/versions.yml \ No newline at end of file diff --git a/gatk4/merged_bam_1.fastq.gz b/gatk4/merged_bam_1.fastq.gz new file mode 120000 index 00000000000..e7277f6a8a8 --- /dev/null +++ b/gatk4/merged_bam_1.fastq.gz @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/13/b0841f5ce1276b1fd2f7322a3e54bc/merged_bam_1.fastq.gz \ No newline at end of file diff --git a/gatk4/merged_bam_2.fastq.gz b/gatk4/merged_bam_2.fastq.gz new file mode 120000 index 00000000000..95f69545756 --- /dev/null +++ b/gatk4/merged_bam_2.fastq.gz @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/13/b0841f5ce1276b1fd2f7322a3e54bc/merged_bam_2.fastq.gz \ No newline at end of file diff --git a/gatk4/versions.yml b/gatk4/versions.yml new file mode 120000 index 00000000000..eb0cb8d2112 --- /dev/null +++ b/gatk4/versions.yml @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/13/b0841f5ce1276b1fd2f7322a3e54bc/versions.yml \ No newline at end of file diff --git a/picard/unmerged_bams.bam b/picard/unmerged_bams.bam new file mode 120000 index 00000000000..0b89a4cd80d --- /dev/null +++ b/picard/unmerged_bams.bam @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/b6/d55a70cdddae5b83cd74cd6be26896/unmerged_bams.bam \ No newline at end of file diff --git a/picard/versions.yml b/picard/versions.yml new file mode 120000 index 00000000000..d1ac40936d4 --- /dev/null +++ b/picard/versions.yml @@ -0,0 +1 @@ +/rtsess01/juno/home/buehlere/modules/b6/d55a70cdddae5b83cd74cd6be26896/versions.yml \ No newline at end of file diff --git a/subworkflows/nf-core/extractumi/main.nf b/subworkflows/nf-core/extractumi/main.nf new file mode 100644 index 00000000000..afa7be0c30d --- /dev/null +++ b/subworkflows/nf-core/extractumi/main.nf @@ -0,0 +1,71 @@ +// TODO nf-core: If in doubt look at other nf-core/subworkflows to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/subworkflows +// You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace: +// https://nf-co.re/join +// TODO nf-core: A subworkflow SHOULD import at least two modules + +include { FGBIO_FASTQTOBAM } from '../../../modules/nf-core/fgbio/fastqtobam/main' +include { PICARD_MERGESAMFILES } from '../../../modules/nf-core/picard/mergesamfiles/' +include { GATK4_SAMTOFASTQ } from '../../../modules/nf-core/gatk4/samtofastq/main' +include { FASTP } from '../../../modules/nf-core/fastp/main' + + +workflow EXTRACTUMI { + + take: + // TODO nf-core: edit input (take) channels + ch_fastq // channel: [ val(meta), [ bam ] ] + + + main: + + ch_versions = Channel.empty() + + // FGBIO_FASTQTOBAM: get unmerged bams + // ch_fastq is a channel, which enables parallel + // channels enable parallel: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel + FGBIO_FASTQTOBAM ( + ch_fastq + ) + FGBIO_FASTQTOBAM.out.bam.map{ + meta, bam -> + [bam] + }.collect().map{ + bams -> + [[id: 'unmerged_bams'], bams ] + }.set{unmerged_bams} + ch_versions = ch_versions.mix(FGBIO_FASTQTOBAM.out.versions) + + // PICARD_MERGESAMFILES: merge bams files + PICARD_MERGESAMFILES ( + unmerged_bams + ).bam.map { + meta, bam -> + new_id = 'merged_bam' + [[id: new_id], bam ] + }.set {merged_bam} + ch_versions = ch_versions.mix(PICARD_MERGESAMFILES.out.versions) + + // GATK4_SAMTOFASTQ: get fastqs from merged bam + GATK4_SAMTOFASTQ ( + merged_bam + ).fastq.map { + meta, fastq -> + new_id = 'merged_fastq' + [[id: new_id], fastq ] + }.set {merged_fastq} + ch_versions = ch_versions.mix(GATK4_SAMTOFASTQ.out.versions) + + // GATK4_SAMTOFASTQ: Run fastp on fastqs + FASTP ( + merged_fastq, [], false, false + ) + ch_versions = ch_versions.mix(FASTP.out.versions) + + // final emit + emit: + // TODO nf-core: edit emitted channels + bam = FASTP.out.reads + + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/extractumi/meta.yml b/subworkflows/nf-core/extractumi/meta.yml new file mode 100644 index 00000000000..6c57561ac1e --- /dev/null +++ b/subworkflows/nf-core/extractumi/meta.yml @@ -0,0 +1,48 @@ +name: "extractumi" +## TODO nf-core: Add a description of the subworkflow and list keywords +description: Sort SAM/BAM/CRAM file +keywords: + - sort + - bam + - sam + - cram +## TODO nf-core: Add a list of the modules used in the subworkflow +modules: + - samtools/sort + - samtools/index +## TODO nf-core: List all of the variables used as input, including their types and descriptions +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" +## TODO nf-core: List all of the variables used as output, including their types and descriptions +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - bai: + type: file + description: BAM/CRAM/SAM samtools index + pattern: "*.{bai,crai,sai}" + - csi: + type: file + description: CSI samtools index + pattern: "*.csi" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@buehlere" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 43279f42549..3f13de89edc 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -2278,10 +2278,6 @@ mskccgbcms: - modules/nf-core/mskccgbcms/** - tests/modules/nf-core/mskccgbcms/** -prepnucleo: - - modules/nf-core/prepnucleo/** - - tests/modules/nf-core/prepnucleo/** - mtnucratio: - modules/nf-core/mtnucratio/** - tests/modules/nf-core/mtnucratio/** @@ -2643,6 +2639,10 @@ porechop/porechop: - modules/nf-core/porechop/porechop/** - tests/modules/nf-core/porechop/porechop/** +prepnucleo: + - modules/nf-core/prepnucleo/** + - tests/modules/nf-core/prepnucleo/** + preseq/ccurve: - modules/nf-core/preseq/ccurve/** - tests/modules/nf-core/preseq/ccurve/** @@ -3344,6 +3344,10 @@ subworkflows/bedgraph_bedclip_bedgraphtobigwig: - subworkflows/nf-core/bedgraph_bedclip_bedgraphtobigwig/** - tests/subworkflows/nf-core/bedgraph_bedclip_bedgraphtobigwig/** +subworkflows/extractumi: + - subworkflows/nf-core/extractumi/** + - tests/subworkflows/nf-core/extractumi/** + subworkflows/fasta_binning_concoct: - subworkflows/nf-core/fasta_binning_concoct/** - tests/subworkflows/nf-core/fasta_binning_concoct/** diff --git a/tests/subworkflows/nf-core/extractumi/main.nf b/tests/subworkflows/nf-core/extractumi/main.nf new file mode 100644 index 00000000000..163afc02ca5 --- /dev/null +++ b/tests/subworkflows/nf-core/extractumi/main.nf @@ -0,0 +1,16 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { EXTRACTUMI } from '../../../../subworkflows/nf-core/extractumi/main.nf' + +workflow test_extractumi { + + // channels enable parralle: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel + fastq = [ + [[id:'gene1', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ad.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ad.fastq.gz')]], + [[id:'gene2', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ae.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ae.fastq.gz')]] + ] + fastq = ch_fastq = Channel.fromList(fastq) + EXTRACTUMI ( fastq ) +} diff --git a/tests/subworkflows/nf-core/extractumi/nextflow.config b/tests/subworkflows/nf-core/extractumi/nextflow.config new file mode 100644 index 00000000000..8730f1c4b93 --- /dev/null +++ b/tests/subworkflows/nf-core/extractumi/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} diff --git a/tests/subworkflows/nf-core/extractumi/test.yml b/tests/subworkflows/nf-core/extractumi/test.yml new file mode 100644 index 00000000000..d82e1082515 --- /dev/null +++ b/tests/subworkflows/nf-core/extractumi/test.yml @@ -0,0 +1,35 @@ +- name: extractumi test_extractumi + command: nextflow run ./tests/subworkflows/nf-core/extractumi -entry test_extractumi -c ./tests/config/nextflow.config + tags: + - fastp + - fgbio + - fgbio/fastqtobam + - gatk4 + - gatk4/samtofastq + - picard + - picard/mergesamfiles + - subworkflows + - subworkflows/extractumi + files: + - path: output/fastp/merged_fastq.fastp.html + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + - path: output/fastp/merged_fastq.fastp.json + md5sum: 46bbed497953f1c7b3e4941771750f6d + - path: output/fastp/merged_fastq.fastp.log + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + - path: output/fastp/merged_fastq_1.fastp.fastq.gz + md5sum: 697ad371be463f83b095e85152aa3046 + - path: output/fastp/merged_fastq_2.fastp.fastq.gz + md5sum: 26328b6503c3c6ba480adfbdc9dbf54f + - path: output/fgbio/gene1.bam + md5sum: fb9749c10bbf3917dbf739de827276f7 + - path: output/fgbio/gene2.bam + md5sum: 4596c2c2a8f94d770410246fb03f3baa + - path: output/gatk4/merged_bam_1.fastq.gz + md5sum: 8d87d76a1a07892a4bd0b2709338831e + - path: output/gatk4/merged_bam_2.fastq.gz + md5sum: 94c76d2f77a71d81ac26adcf94720ecd + - path: output/picard/unmerged_bams.bam + md5sum: a921915f4ea117e3a10daeb07d31abdb From 6503b000eb09c2490d434d5e74431475c0236741 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 2 May 2023 15:54:46 -0400 Subject: [PATCH 06/50] clean up old workflow, after re-name --- subworkflows/nf-core/prepnucleo/main.nf | 73 ------------------- subworkflows/nf-core/prepnucleo/meta.yml | 48 ------------ tests/config/pytest_modules.yml | 8 -- tests/subworkflows/nf-core/prepnucleo/main.nf | 16 ---- .../nf-core/prepnucleo/nextflow.config | 5 -- .../subworkflows/nf-core/prepnucleo/test.yml | 36 --------- 6 files changed, 186 deletions(-) delete mode 100644 subworkflows/nf-core/prepnucleo/main.nf delete mode 100644 subworkflows/nf-core/prepnucleo/meta.yml delete mode 100644 tests/subworkflows/nf-core/prepnucleo/main.nf delete mode 100644 tests/subworkflows/nf-core/prepnucleo/nextflow.config delete mode 100644 tests/subworkflows/nf-core/prepnucleo/test.yml diff --git a/subworkflows/nf-core/prepnucleo/main.nf b/subworkflows/nf-core/prepnucleo/main.nf deleted file mode 100644 index eabec561586..00000000000 --- a/subworkflows/nf-core/prepnucleo/main.nf +++ /dev/null @@ -1,73 +0,0 @@ -// TODO nf-core: If in doubt look at other nf-core/subworkflows to see how we are doing things! :) -// https://github.com/nf-core/modules/tree/master/subworkflows -// You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace: -// https://nf-co.re/join -// TODO nf-core: A subworkflow SHOULD import at least two modules - -include { FGBIO_FASTQTOBAM } from '../../../modules/nf-core/fgbio/fastqtobam/main' -include { PICARD_MERGESAMFILES } from '../../../modules/nf-core/picard/mergesamfiles/' -include { GATK4_SAMTOFASTQ } from '../../../modules/nf-core/gatk4/samtofastq/main' -include { FASTP } from '../../../modules/nf-core/fastp/main' - - -workflow PREPNUCLEO { - - take: - // TODO nf-core: edit input (take) channels - ch_fastq // channel: [ val(meta), [ bam ] ] - - - main: - - ch_versions = Channel.empty() - - // FGBIO_FASTQTOBAM: get unmerged bams - // ch_fastq is a channel, which enables parallel - // channels enable parallel: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel - FGBIO_FASTQTOBAM ( - ch_fastq - ) - FGBIO_FASTQTOBAM.out.bam.map{ - meta, bam -> - [bam] - }.collect().map{ - bams -> - [[id: 'unmerged_bams'], bams ] - }.set{unmerged_bams} - ch_versions = ch_versions.mix(FGBIO_FASTQTOBAM.out.versions) - - // PICARD_MERGESAMFILES: merge bams files - PICARD_MERGESAMFILES ( - unmerged_bams - ).bam.map { - meta, bam -> - new_id = 'merged_bam' - [[id: new_id], bam ] - }.set {merged_bam} - ch_versions = ch_versions.mix(PICARD_MERGESAMFILES.out.versions) - - // GATK4_SAMTOFASTQ: get fastqs from merged bam - GATK4_SAMTOFASTQ ( - merged_bam - ).fastq.map { - meta, fastq -> - new_id = 'merged_fastq' - [[id: new_id], fastq ] - }.set {merged_fastq} - ch_versions = ch_versions.mix(GATK4_SAMTOFASTQ.out.versions) - - // GATK4_SAMTOFASTQ: Run fastp on fastqs - FASTP ( - merged_fastq, [], false, false - ) - ch_versions = ch_versions.mix(FASTP.out.versions) - - // final emit - emit: - // TODO nf-core: edit emitted channels - bam = FASTP.out.reads - - versions = ch_versions // channel: [ versions.yml ] -} - - diff --git a/subworkflows/nf-core/prepnucleo/meta.yml b/subworkflows/nf-core/prepnucleo/meta.yml deleted file mode 100644 index a1091e74482..00000000000 --- a/subworkflows/nf-core/prepnucleo/meta.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: "prepnucleo" -## TODO nf-core: Add a description of the subworkflow and list keywords -description: Sort SAM/BAM/CRAM file -keywords: - - sort - - bam - - sam - - cram -## TODO nf-core: Add a list of the modules used in the subworkflow -modules: - - samtools/sort - - samtools/index -## TODO nf-core: List all of the variables used as input, including their types and descriptions -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test' ] - - bam: - type: file - description: BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" -## TODO nf-core: List all of the variables used as output, including their types and descriptions -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test' ] - - bam: - type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - - bai: - type: file - description: BAM/CRAM/SAM samtools index - pattern: "*.{bai,crai,sai}" - - csi: - type: file - description: CSI samtools index - pattern: "*.csi" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "None" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 3f13de89edc..56b375dfe77 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -2639,10 +2639,6 @@ porechop/porechop: - modules/nf-core/porechop/porechop/** - tests/modules/nf-core/porechop/porechop/** -prepnucleo: - - modules/nf-core/prepnucleo/** - - tests/modules/nf-core/prepnucleo/** - preseq/ccurve: - modules/nf-core/preseq/ccurve/** - tests/modules/nf-core/preseq/ccurve/** @@ -3432,10 +3428,6 @@ subworkflows/homer/groseq: - subworkflows/nf-core/homer/groseq/** - tests/subworkflows/nf-core/homer/groseq/** -subworkflows/prepnucleo: - - subworkflows/nf-core/prepnucleo/** - - tests/subworkflows/nf-core/prepnucleo/** - subworkflows/vcf_annotate_ensemblvep: - subworkflows/nf-core/vcf_annotate_ensemblvep/** - tests/subworkflows/nf-core/vcf_annotate_ensemblvep/** diff --git a/tests/subworkflows/nf-core/prepnucleo/main.nf b/tests/subworkflows/nf-core/prepnucleo/main.nf deleted file mode 100644 index 6f45d7c8da6..00000000000 --- a/tests/subworkflows/nf-core/prepnucleo/main.nf +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { PREPNUCLEO } from '../../../../subworkflows/nf-core/prepnucleo/main.nf' - -workflow test_prepnucleo { - - // channels enable parralle: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel - fastq = [ - [[id:'gene1', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ad.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ad.fastq.gz')]], - [[id:'gene2', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ae.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ae.fastq.gz')]] - ] - fastq = ch_fastq = Channel.fromList(fastq) - PREPNUCLEO ( fastq ) -} diff --git a/tests/subworkflows/nf-core/prepnucleo/nextflow.config b/tests/subworkflows/nf-core/prepnucleo/nextflow.config deleted file mode 100644 index 8730f1c4b93..00000000000 --- a/tests/subworkflows/nf-core/prepnucleo/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} diff --git a/tests/subworkflows/nf-core/prepnucleo/test.yml b/tests/subworkflows/nf-core/prepnucleo/test.yml deleted file mode 100644 index a23b33323f2..00000000000 --- a/tests/subworkflows/nf-core/prepnucleo/test.yml +++ /dev/null @@ -1,36 +0,0 @@ -- name: prepnucleo test_prepnucleo - command: nextflow run ./tests/subworkflows/nf-core/prepnucleo -entry test_prepnucleo -c ./tests/config/nextflow.config - tags: - - fastp - - fgbio - - fgbio/fastqtobam - - gatk4 - - gatk4/samtofastq - - picard - - picard/mergesamfiles - - subworkflows - - subworkflows/prepnucleo - files: - - path: output/fastp/merged_fastq.fastp.html - contains: - - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' - - path: output/fastp/merged_fastq.fastp.json - md5sum: 46bbed497953f1c7b3e4941771750f6d - - path: output/fastp/merged_fastq.fastp.log - contains: - - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' - - path: output/fastp/merged_fastq_1.fastp.fastq.gz - md5sum: 697ad371be463f83b095e85152aa3046 - - path: output/fastp/merged_fastq_2.fastp.fastq.gz - md5sum: 26328b6503c3c6ba480adfbdc9dbf54f - - path: output/fgbio/gene1.bam - md5sum: fb9749c10bbf3917dbf739de827276f7 - - path: output/fgbio/gene2.bam - md5sum: 4596c2c2a8f94d770410246fb03f3baa - - path: output/gatk4/merged_bam_1.fastq.gz - md5sum: 8d87d76a1a07892a4bd0b2709338831e - - path: output/gatk4/merged_bam_2.fastq.gz - md5sum: 94c76d2f77a71d81ac26adcf94720ecd - - path: output/picard/unmerged_bams.bam - contains: - - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' From db72d2c36aba93dd648b0b59ea61bf386818e667 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 9 May 2023 11:39:44 -0400 Subject: [PATCH 07/50] try fork syncing --- .github/workflows/sync-fork.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/sync-fork.yml diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml new file mode 100644 index 00000000000..a81897bf4eb --- /dev/null +++ b/.github/workflows/sync-fork.yml @@ -0,0 +1,20 @@ +name: Sync Fork + +on: + push: + branches: + - feature/extractumi + workflow_dispatch: # on button click + +jobs: + sync: + + runs-on: ubuntu-latest + + steps: + - uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + owner: llvm + base: master + head: master From fc28f6e74b07ab67760183cbffdfe0f18bac1581 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 9 May 2023 12:18:51 -0400 Subject: [PATCH 08/50] experimenting with auto sync --- .github/workflows/sync-fork.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index a81897bf4eb..6b16e6a3a91 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -14,7 +14,6 @@ jobs: steps: - uses: tgymnich/fork-sync@v1.8 with: - token: ${{ secrets.PERSONAL_TOKEN }} owner: llvm base: master - head: master + head: feature/extractumi From 7c9816d21d96cbe799db3fd5a4783faf1286558f Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 9 May 2023 12:54:37 -0400 Subject: [PATCH 09/50] testing gitactions --- .github/workflows/sync-fork.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 6b16e6a3a91..34e8978946c 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -14,6 +14,6 @@ jobs: steps: - uses: tgymnich/fork-sync@v1.8 with: - owner: llvm + token: ${{ secrets.PERSONAL_TOKEN }} base: master head: feature/extractumi From 228c7723dd5514f7f064b97e6288c292d9f40fc5 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 9 May 2023 13:05:01 -0400 Subject: [PATCH 10/50] gitactions test --- .github/workflows/sync-fork.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 34e8978946c..82cae081623 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -15,5 +15,5 @@ jobs: - uses: tgymnich/fork-sync@v1.8 with: token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: feature/extractumi + base: feature/extractumi + head: master From d612e60a0e92c74c72112bbbad2a057eefe809f0 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 9 May 2023 15:25:50 -0400 Subject: [PATCH 11/50] gitactions experiment --- .github/workflows/sync-fork.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 82cae081623..4a383e9a85c 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -7,13 +7,29 @@ on: workflow_dispatch: # on button click jobs: - sync: - + merge: runs-on: ubuntu-latest - steps: - - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: feature/extractumi - head: master + - uses: actions/checkout@v2 + - name: Merge upstream + run: | + # "git checkout master" is unnecessary, already here by default + git pull --unshallow # this option is very important, you would get + # complains about unrelated histories without it. + # (but actions/checkout@v2 can also be instructed + # to fetch all git depth right from the start) + + git remote add upstream https://github.com/nf-core/modules.git + git fetch upstream + + # Neither forget the -b opt, + # the feature/extractumi ref is ambiguous at this stage + git checkout -b feature/extractumi origin/feature/extractumi + git merge --no-edit upstream/feature/extractumi + git push origin feature/extractumi + + git checkout master + git merge --no-edit upstream/master + git push origin master + + # etc From a78e7ea976f6733cc46dc75ede75dd2ebeb6253d Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Tue, 9 May 2023 15:31:07 -0400 Subject: [PATCH 12/50] trying to fix gitactions --- .github/workflows/sync-fork.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 4a383e9a85c..2572aa7f73d 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -22,12 +22,6 @@ jobs: git remote add upstream https://github.com/nf-core/modules.git git fetch upstream - # Neither forget the -b opt, - # the feature/extractumi ref is ambiguous at this stage - git checkout -b feature/extractumi origin/feature/extractumi - git merge --no-edit upstream/feature/extractumi - git push origin feature/extractumi - git checkout master git merge --no-edit upstream/master git push origin master From 06b27e0e25a894eff1418ed760b89cc49fe7dae8 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 15:48:17 -0400 Subject: [PATCH 13/50] Create sync-fork.yml --- .github/workflows/sync-fork.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/sync-fork.yml diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml new file mode 100644 index 00000000000..1b68e43e34c --- /dev/null +++ b/.github/workflows/sync-fork.yml @@ -0,0 +1,26 @@ +name: Sync Fork + +on: + push: + branches: + - master + workflow_dispatch: # on button click + +jobs: + merge: + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v2 + with: + ref: master + - name: Fetch changes from upstream + run: git fetch upstream + env: + upstream: https://github.com/nf-core/modules.git + - name: Merge changes from upstream + run: git merge upstream/master --no-edit + - name: Push changes to fork + uses: ad-m/github-push-action@master + with: + branch: master \ No newline at end of file From f2c759d6718aef653b77913f5719bdb0fcced7e2 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 15:52:56 -0400 Subject: [PATCH 14/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 1b68e43e34c..7dc04722c82 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -16,8 +16,6 @@ jobs: ref: master - name: Fetch changes from upstream run: git fetch upstream - env: - upstream: https://github.com/nf-core/modules.git - name: Merge changes from upstream run: git merge upstream/master --no-edit - name: Push changes to fork From 7ff8837dbc105be340bbf2778ce71c6a1aa27ef4 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 15:53:39 -0400 Subject: [PATCH 15/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 7dc04722c82..dd5ed430869 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -15,7 +15,7 @@ jobs: with: ref: master - name: Fetch changes from upstream - run: git fetch upstream + run: git fetch upstream https://github.com/nf-core/modules.git - name: Merge changes from upstream run: git merge upstream/master --no-edit - name: Push changes to fork From 03ede3c6daff68325b0e4e24b7274c3b40a2d910 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:01:11 -0400 Subject: [PATCH 16/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index dd5ed430869..746034b4e4d 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -10,12 +10,12 @@ jobs: merge: runs-on: ubuntu-latest steps: + - name: Fetch changes from upstream + run: git fetch upstream https://github.com/nf-core/modules.git - name: Checkout branch uses: actions/checkout@v2 with: ref: master - - name: Fetch changes from upstream - run: git fetch upstream https://github.com/nf-core/modules.git - name: Merge changes from upstream run: git merge upstream/master --no-edit - name: Push changes to fork From 879199a56c9a7668bf0f4a87f30f03121ae902b8 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:03:42 -0400 Subject: [PATCH 17/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 746034b4e4d..389cf743afb 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -9,16 +9,23 @@ on: jobs: merge: runs-on: ubuntu-latest - steps: - - name: Fetch changes from upstream - run: git fetch upstream https://github.com/nf-core/modules.git - - name: Checkout branch + steps: + - name: Checkout repository uses: actions/checkout@v2 - with: - ref: master - - name: Merge changes from upstream - run: git merge upstream/master --no-edit - - name: Push changes to fork - uses: ad-m/github-push-action@master - with: - branch: master \ No newline at end of file + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/nf-core/modules.git + + - name: Fetch upstream changes + run: | + git fetch upstream + + - name: Merge upstream master into forked master + run: | + git checkout master + git merge upstream/master --no-edit + + - name: Push changes to forked repository + run: | + git push origin master \ No newline at end of file From c21e35b5ef6bc7a6e60e5f5a4cc44215af73767b Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:06:46 -0400 Subject: [PATCH 18/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 389cf743afb..e35dcaa527c 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -1,10 +1,8 @@ name: Sync Fork on: - push: - branches: - - master - workflow_dispatch: # on button click + schedule: + - cron: "0 0 * * *" jobs: merge: From cd0957950e9336f4674806e2916072a1a4efec57 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:10:12 -0400 Subject: [PATCH 19/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index e35dcaa527c..232e5bfaa23 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -7,7 +7,7 @@ on: jobs: merge: runs-on: ubuntu-latest - steps: + steps: - name: Checkout repository uses: actions/checkout@v2 @@ -26,4 +26,5 @@ jobs: - name: Push changes to forked repository run: | - git push origin master \ No newline at end of file + git push origin master + \ No newline at end of file From 31331e4e80efe1a7f872b912033994f1a0bb436d Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:13:37 -0400 Subject: [PATCH 20/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 232e5bfaa23..eef323780b7 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -27,4 +27,3 @@ jobs: - name: Push changes to forked repository run: | git push origin master - \ No newline at end of file From 9a1f635989bb95895d581b0a88d196465a312e59 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:24:12 -0400 Subject: [PATCH 21/50] remove output folders --- fastp/merged_fastq.fastp.html | 1 - fastp/merged_fastq.fastp.json | 1 - fastp/merged_fastq.fastp.log | 1 - fastp/merged_fastq_1.fastp.fastq.gz | 1 - fastp/merged_fastq_2.fastp.fastq.gz | 1 - fastp/versions.yml | 1 - fgbio/gene1.bam | 1 - fgbio/gene2.bam | 1 - fgbio/versions.yml | 1 - gatk4/merged_bam_1.fastq.gz | 1 - gatk4/merged_bam_2.fastq.gz | 1 - gatk4/versions.yml | 1 - picard/unmerged_bams.bam | 1 - picard/versions.yml | 1 - 14 files changed, 14 deletions(-) delete mode 120000 fastp/merged_fastq.fastp.html delete mode 120000 fastp/merged_fastq.fastp.json delete mode 120000 fastp/merged_fastq.fastp.log delete mode 120000 fastp/merged_fastq_1.fastp.fastq.gz delete mode 120000 fastp/merged_fastq_2.fastp.fastq.gz delete mode 120000 fastp/versions.yml delete mode 120000 fgbio/gene1.bam delete mode 120000 fgbio/gene2.bam delete mode 120000 fgbio/versions.yml delete mode 120000 gatk4/merged_bam_1.fastq.gz delete mode 120000 gatk4/merged_bam_2.fastq.gz delete mode 120000 gatk4/versions.yml delete mode 120000 picard/unmerged_bams.bam delete mode 120000 picard/versions.yml diff --git a/fastp/merged_fastq.fastp.html b/fastp/merged_fastq.fastp.html deleted file mode 120000 index 029d816faf5..00000000000 --- a/fastp/merged_fastq.fastp.html +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq.fastp.html \ No newline at end of file diff --git a/fastp/merged_fastq.fastp.json b/fastp/merged_fastq.fastp.json deleted file mode 120000 index af8fd01aa65..00000000000 --- a/fastp/merged_fastq.fastp.json +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq.fastp.json \ No newline at end of file diff --git a/fastp/merged_fastq.fastp.log b/fastp/merged_fastq.fastp.log deleted file mode 120000 index bb752ce5030..00000000000 --- a/fastp/merged_fastq.fastp.log +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq.fastp.log \ No newline at end of file diff --git a/fastp/merged_fastq_1.fastp.fastq.gz b/fastp/merged_fastq_1.fastp.fastq.gz deleted file mode 120000 index ca9b440171b..00000000000 --- a/fastp/merged_fastq_1.fastp.fastq.gz +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq_1.fastp.fastq.gz \ No newline at end of file diff --git a/fastp/merged_fastq_2.fastp.fastq.gz b/fastp/merged_fastq_2.fastp.fastq.gz deleted file mode 120000 index f1971eef0e2..00000000000 --- a/fastp/merged_fastq_2.fastp.fastq.gz +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/merged_fastq_2.fastp.fastq.gz \ No newline at end of file diff --git a/fastp/versions.yml b/fastp/versions.yml deleted file mode 120000 index 760a58e3ef5..00000000000 --- a/fastp/versions.yml +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/3b/757b03a7b4b38cdcdf4ea8e4ba3fd1/versions.yml \ No newline at end of file diff --git a/fgbio/gene1.bam b/fgbio/gene1.bam deleted file mode 120000 index 1f70c5c6134..00000000000 --- a/fgbio/gene1.bam +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/6e/9a2bf920bcbc26f872eae76763ae65/gene1.bam \ No newline at end of file diff --git a/fgbio/gene2.bam b/fgbio/gene2.bam deleted file mode 120000 index 479fc3a8414..00000000000 --- a/fgbio/gene2.bam +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/b6/df527007cf5a649bdac5b3662c6423/gene2.bam \ No newline at end of file diff --git a/fgbio/versions.yml b/fgbio/versions.yml deleted file mode 120000 index 2b76387d35e..00000000000 --- a/fgbio/versions.yml +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/6e/9a2bf920bcbc26f872eae76763ae65/versions.yml \ No newline at end of file diff --git a/gatk4/merged_bam_1.fastq.gz b/gatk4/merged_bam_1.fastq.gz deleted file mode 120000 index e7277f6a8a8..00000000000 --- a/gatk4/merged_bam_1.fastq.gz +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/13/b0841f5ce1276b1fd2f7322a3e54bc/merged_bam_1.fastq.gz \ No newline at end of file diff --git a/gatk4/merged_bam_2.fastq.gz b/gatk4/merged_bam_2.fastq.gz deleted file mode 120000 index 95f69545756..00000000000 --- a/gatk4/merged_bam_2.fastq.gz +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/13/b0841f5ce1276b1fd2f7322a3e54bc/merged_bam_2.fastq.gz \ No newline at end of file diff --git a/gatk4/versions.yml b/gatk4/versions.yml deleted file mode 120000 index eb0cb8d2112..00000000000 --- a/gatk4/versions.yml +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/13/b0841f5ce1276b1fd2f7322a3e54bc/versions.yml \ No newline at end of file diff --git a/picard/unmerged_bams.bam b/picard/unmerged_bams.bam deleted file mode 120000 index 0b89a4cd80d..00000000000 --- a/picard/unmerged_bams.bam +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/b6/d55a70cdddae5b83cd74cd6be26896/unmerged_bams.bam \ No newline at end of file diff --git a/picard/versions.yml b/picard/versions.yml deleted file mode 120000 index d1ac40936d4..00000000000 --- a/picard/versions.yml +++ /dev/null @@ -1 +0,0 @@ -/rtsess01/juno/home/buehlere/modules/b6/d55a70cdddae5b83cd74cd6be26896/versions.yml \ No newline at end of file From be48046b098a874cf222f26db8c4e05e8a738b22 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 10 May 2023 09:31:19 -0400 Subject: [PATCH 22/50] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index eef323780b7..7846d960096 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -1,6 +1,9 @@ name: Sync Fork on: + push: + branches: + - "master" schedule: - cron: "0 0 * * *" @@ -22,7 +25,7 @@ jobs: - name: Merge upstream master into forked master run: | git checkout master - git merge upstream/master --no-edit + git merge upstream/master --no-edit --allow-unrelated-histories - name: Push changes to forked repository run: | From caa71e9e3a8ef991a01abf55059fb5240042779d Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 10 May 2023 09:31:45 -0400 Subject: [PATCH 23/50] Revert "Update sync-fork.yml" This reverts commit be48046b098a874cf222f26db8c4e05e8a738b22. --- .github/workflows/sync-fork.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 7846d960096..eef323780b7 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -1,9 +1,6 @@ name: Sync Fork on: - push: - branches: - - "master" schedule: - cron: "0 0 * * *" @@ -25,7 +22,7 @@ jobs: - name: Merge upstream master into forked master run: | git checkout master - git merge upstream/master --no-edit --allow-unrelated-histories + git merge upstream/master --no-edit - name: Push changes to forked repository run: | From e0c8587417c69e1435ef0bc8d0625d77e53db2b5 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 10 May 2023 15:38:38 -0400 Subject: [PATCH 24/50] Create sync-action.yml --- .github/workflows/sync-action.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/sync-action.yml diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml new file mode 100644 index 00000000000..3b68c7e2169 --- /dev/null +++ b/.github/workflows/sync-action.yml @@ -0,0 +1,17 @@ +name: Sync Fork + +on: + schedule: + - cron: "0 0 * * *" # run once a night at midnight + workflow_dispatch: # on button click + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: master From 441d2d9b89aa9cd346450d63713be2040d79929d Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 12:36:14 -0400 Subject: [PATCH 25/50] remove this for separate PR --- modules/nf-core/mskccgbcms/main.nf | 41 ------------------- modules/nf-core/mskccgbcms/meta.yml | 39 ------------------ tests/config/pytest_modules.yml | 4 -- tests/modules/nf-core/mskccgbcms/main.nf | 25 ----------- .../nf-core/mskccgbcms/nextflow.config | 15 ------- tests/modules/nf-core/mskccgbcms/test.yml | 8 ---- 6 files changed, 132 deletions(-) delete mode 100644 modules/nf-core/mskccgbcms/main.nf delete mode 100644 modules/nf-core/mskccgbcms/meta.yml delete mode 100644 tests/modules/nf-core/mskccgbcms/main.nf delete mode 100644 tests/modules/nf-core/mskccgbcms/nextflow.config delete mode 100644 tests/modules/nf-core/mskccgbcms/test.yml diff --git a/modules/nf-core/mskccgbcms/main.nf b/modules/nf-core/mskccgbcms/main.nf deleted file mode 100644 index 7294681f851..00000000000 --- a/modules/nf-core/mskccgbcms/main.nf +++ /dev/null @@ -1,41 +0,0 @@ -process GBCMS { - tag "$meta.id" - label 'process_single' - container "ghcr.io/msk-access/gbcms:1.2.5" - - input: - tuple val(meta), path(fasta), path(fastafai), path(bam), path(bambai), path(variant_file), val(sample), val(output), val(options) - - output: - path('variant_file.{vcf,maf}') ,emit: variant_file - path("versions.yml") - - script: - // determine if input file is a maf of vcf - // the --maf and --vcf inputs are mutually input exclusive parameters. - // Both are part of the same subcommand. Thus warrants groovy scripting. - def input_ext = variant_file.getExtension() - def variant_input = '' - if(input_ext == 'maf') { - variant_input = '--maf ' + variant_file - } - if(input_ext == 'vcf'){ - variant_input = '--vcf ' + variant_file - } - // raise exception if file extension other than maf or vcf is passed - if(variant_input == ''){ - throw new Exception("Variant file must be maf or vcf, not ${input_ext}") - } - """ - GetBaseCountsMultiSample --fasta ${fasta} \\ - ${variant_input} \\ - --output ${output} \\ - --bam $sample:${bam} ${options.args} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - GBCM: \$(echo \$(GetBaseCountsMultiSample --help) | grep -oP '[0-9]\\.[0-9]\\.[0-9]') - END_VERSIONS - """ -} - diff --git a/modules/nf-core/mskccgbcms/meta.yml b/modules/nf-core/mskccgbcms/meta.yml deleted file mode 100644 index c6e674f88c5..00000000000 --- a/modules/nf-core/mskccgbcms/meta.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "mskccgbcms" -## TODO nf-core: Add a description of the module and list keywords -description: write your description here -keywords: - - sort -tools: - - "mskccgbcms": - ## TODO nf-core: Add a description and other details for the software below - description: "" - homepage: "" - documentation: "" - tool_dev_url: "" - doi: "" - licence: "" - -## TODO nf-core: Add a description of all of the variables used as input -input: - # - ## TODO nf-core: Delete / customise this example input - - bam: - type: file - description: BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - -## TODO nf-core: Add a description of all of the variables used as output -output: - # - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" - ## TODO nf-core: Delete / customise this example output - - bam: - type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - -authors: - - "None" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 559acb0b9b9..1e1223fdb0b 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -2350,10 +2350,6 @@ msisensorpro/scan: - modules/nf-core/msisensorpro/scan/** - tests/modules/nf-core/msisensorpro/scan/** -mskccgbcms: - - modules/nf-core/mskccgbcms/** - - tests/modules/nf-core/mskccgbcms/** - mtnucratio: - modules/nf-core/mtnucratio/** - tests/modules/nf-core/mtnucratio/** diff --git a/tests/modules/nf-core/mskccgbcms/main.nf b/tests/modules/nf-core/mskccgbcms/main.nf deleted file mode 100644 index 4ec3a8f2167..00000000000 --- a/tests/modules/nf-core/mskccgbcms/main.nf +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { GBCMS } from '../../../../modules/nf-core/mskccgbcms/main.nf' - -workflow test_gbcms { - - input = [ - [ id:'test' ], // meta map - file(params.fasta, checkIfExists: true), - file(params.fastafai, checkIfExists: true), - file(params.bam, checkIfExists: true), - file(params.bambai, checkIfExists: true), - file("tools-test-dataset/getbasecountmultisample/chr22.maf"), - params.sample, - // customize test specific parameters here - "variant_file.maf", - [args: '--omaf'] - - ] - - GBCMS ( input ) -} - diff --git a/tests/modules/nf-core/mskccgbcms/nextflow.config b/tests/modules/nf-core/mskccgbcms/nextflow.config deleted file mode 100644 index b690fa5204f..00000000000 --- a/tests/modules/nf-core/mskccgbcms/nextflow.config +++ /dev/null @@ -1,15 +0,0 @@ -params { - enable_conda = false - fasta = "tools-test-dataset/getbasecountmultisample/chr22.fasta" - fastafai = "tools-test-dataset/getbasecountmultisample/chr22.fasta.fai" - bam = "tools-test-dataset/getbasecountmultisample/chr22.bam" - bambai = "tools-test-dataset/getbasecountmultisample/chr22.bam.bai" - sample = "C-9M47TU-L001-d" - outdir = "output" -} - -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} \ No newline at end of file diff --git a/tests/modules/nf-core/mskccgbcms/test.yml b/tests/modules/nf-core/mskccgbcms/test.yml deleted file mode 100644 index 9590f2139f0..00000000000 --- a/tests/modules/nf-core/mskccgbcms/test.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: mskccgbcms test_gbcms - command: nextflow run ./tests/modules/nf-core/mskccgbcms -entry test_gbcms -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/mskccgbcms/nextflow.config - tags: - - mskccgbcms - files: - - path: output/gbcms/variant_file.maf - md5sum: 039df7916aa49258d63929173923154a - - path: output/gbcms/versions.yml From eaef95ae79ae05a963a7770975189efb4cf3ecf1 Mon Sep 17 00:00:00 2001 From: EricWilliam Buehler Date: Wed, 17 May 2023 13:13:22 -0400 Subject: [PATCH 26/50] clean up for extractumi --- .gitignore | 3 ++- tests/subworkflows/nf-core/extractumi/main.nf | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9b1b9bd6224..7f10e4d8cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ tests/data/ __pycache__ *.pyo *.pyc -test_nucleo/* \ No newline at end of file +test_nucleo/* +cookie diff --git a/tests/subworkflows/nf-core/extractumi/main.nf b/tests/subworkflows/nf-core/extractumi/main.nf index 163afc02ca5..7312d8732ca 100644 --- a/tests/subworkflows/nf-core/extractumi/main.nf +++ b/tests/subworkflows/nf-core/extractumi/main.nf @@ -5,7 +5,16 @@ nextflow.enable.dsl = 2 include { EXTRACTUMI } from '../../../../subworkflows/nf-core/extractumi/main.nf' workflow test_extractumi { - + // load test data + def bashScriptFile = new File('install_data.sh') + + def processBuilder = new ProcessBuilder('bash', bashScriptFile.toString()) + processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT) + processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT) + + def process = processBuilder.start() + process.waitFor() + // channels enable parralle: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel fastq = [ [[id:'gene1', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ad.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ad.fastq.gz')]], From 7215646e120cdb93f1d7dbf0b952523725370060 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:23:10 -0400 Subject: [PATCH 27/50] Update sync-action.yml --- .github/workflows/sync-action.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml index 3b68c7e2169..f20440aa039 100644 --- a/.github/workflows/sync-action.yml +++ b/.github/workflows/sync-action.yml @@ -10,8 +10,15 @@ jobs: runs-on: ubuntu-latest steps: - - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: master + - name: Sync master branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: master + - name: Sync develop branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: develop + head: develop From ab7ba9828e92e9aa8d3db3095c5856b85b0e80bf Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:28:04 -0400 Subject: [PATCH 28/50] Delete sync-fork.yml --- .github/workflows/sync-fork.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/sync-fork.yml diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml deleted file mode 100644 index eef323780b7..00000000000 --- a/.github/workflows/sync-fork.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Sync Fork - -on: - schedule: - - cron: "0 0 * * *" - -jobs: - merge: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Add upstream remote - run: | - git remote add upstream https://github.com/nf-core/modules.git - - - name: Fetch upstream changes - run: | - git fetch upstream - - - name: Merge upstream master into forked master - run: | - git checkout master - git merge upstream/master --no-edit - - - name: Push changes to forked repository - run: | - git push origin master From 35baf68d990c5aa31fcc1330e823d2ac5aacf282 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:37:06 -0400 Subject: [PATCH 29/50] Update sync-action.yml --- .github/workflows/sync-action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml index f20440aa039..67f9c9624f5 100644 --- a/.github/workflows/sync-action.yml +++ b/.github/workflows/sync-action.yml @@ -20,5 +20,5 @@ jobs: uses: tgymnich/fork-sync@v1.8 with: token: ${{ secrets.PERSONAL_TOKEN }} - base: develop + base: master head: develop From f2f4e041381c0388e718e38877d695c59f1a1667 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:37:40 -0400 Subject: [PATCH 30/50] Update sync-action.yml --- .github/workflows/sync-action.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml index 67f9c9624f5..a1f36450ec7 100644 --- a/.github/workflows/sync-action.yml +++ b/.github/workflows/sync-action.yml @@ -10,15 +10,15 @@ jobs: runs-on: ubuntu-latest steps: - - name: Sync master branch - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: master - - name: Sync develop branch - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: develop + - name: Sync master branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: master + - name: Sync develop branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: develop From 5912f2909a2274b926971ae3146a92b17d3d3064 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:43:03 -0400 Subject: [PATCH 31/50] Revert "Merge pull request #16 from nf-core/master" This reverts commit 13d45de834825776f339977b9e0ef34304b14128, reversing changes made to ab7ba9828e92e9aa8d3db3095c5856b85b0e80bf. --- .github/workflows/pytest-workflow.yml | 2 - modules/nf-core/bcftools/query/main.nf | 6 +- modules/nf-core/tabix/bgziptabix/main.nf | 6 +- modules/nf-core/tabix/bgziptabix/meta.yml | 5 - .../vcf_annotate_ensemblvep_snpeff/main.nf | 191 --------- .../vcf_annotate_ensemblvep_snpeff/meta.yml | 73 ---- tests/config/pytest_modules.yml | 4 - tests/modules/nf-core/bcftools/query/test.yml | 10 +- .../modules/nf-core/tabix/bgziptabix/main.nf | 7 +- .../nf-core/tabix/bgziptabix/nextflow.config | 6 +- .../modules/nf-core/tabix/bgziptabix/test.yml | 6 +- .../vcf_annotate_ensemblvep_snpeff/main.nf | 170 -------- .../nextflow.config | 21 - .../vcf_annotate_ensemblvep_snpeff/test.yml | 384 ------------------ 14 files changed, 14 insertions(+), 877 deletions(-) delete mode 100644 subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf delete mode 100644 subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml delete mode 100644 tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf delete mode 100644 tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config delete mode 100644 tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index a5c7979835c..76eec0da2eb 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -136,8 +136,6 @@ jobs: tags: gatk4/determinegermlinecontigploidy - profile: "conda" tags: subworkflows/bcl_demultiplex - - profile: "conda" - tags: subworkflows/vcf_annotate_ensemblvep_snpeff - profile: "conda" tags: subworkflows/vcf_annotate_ensemblvep - profile: "conda" diff --git a/modules/nf-core/bcftools/query/main.nf b/modules/nf-core/bcftools/query/main.nf index 0a775ebebb2..dbc00e00b16 100644 --- a/modules/nf-core/bcftools/query/main.nf +++ b/modules/nf-core/bcftools/query/main.nf @@ -1,6 +1,6 @@ process BCFTOOLS_QUERY { tag "$meta.id" - label 'process_single' + label 'process_medium' conda "bioconda::bcftools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -28,12 +28,12 @@ process BCFTOOLS_QUERY { def samples_file = samples ? "--samples-file ${samples}" : "" """ bcftools query \\ + --output ${prefix}.txt \\ $regions_file \\ $targets_file \\ $samples_file \\ $args \\ - $vcf \\ - > ${prefix}.txt + $vcf cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgziptabix/main.nf b/modules/nf-core/tabix/bgziptabix/main.nf index fef528fcdf8..fe8160bae4c 100644 --- a/modules/nf-core/tabix/bgziptabix/main.nf +++ b/modules/nf-core/tabix/bgziptabix/main.nf @@ -5,14 +5,13 @@ process TABIX_BGZIPTABIX { conda "bioconda::tabix=1.11" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0' : - 'quay.io/biocontainers/tabix:1.11--hdfd78af_0' }" + 'biocontainers/tabix:1.11--hdfd78af_0' }" input: tuple val(meta), path(input) output: - tuple val(meta), path("*.gz"), path("*.tbi"), optional: true, emit: gz_tbi - tuple val(meta), path("*.gz"), path("*.csi"), optional: true, emit: gz_csi + tuple val(meta), path("*.gz"), path("*.tbi"), emit: gz_tbi path "versions.yml" , emit: versions when: @@ -37,7 +36,6 @@ process TABIX_BGZIPTABIX { """ touch ${prefix}.${input.getExtension()}.gz touch ${prefix}.${input.getExtension()}.gz.tbi - touch ${prefix}.${input.getExtension()}.gz.csi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgziptabix/meta.yml b/modules/nf-core/tabix/bgziptabix/meta.yml index 2761e27183b..49c03289415 100644 --- a/modules/nf-core/tabix/bgziptabix/meta.yml +++ b/modules/nf-core/tabix/bgziptabix/meta.yml @@ -37,14 +37,9 @@ output: type: file description: tabix index file pattern: "*.{gz.tbi}" - - csi: - type: file - description: tabix alternate index file - pattern: "*.{gz.csi}" - versions: type: file description: File containing software versions pattern: "versions.yml" authors: - "@maxulysse" - - "@DLBPointon" diff --git a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf b/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf deleted file mode 100644 index 91cace4d1d9..00000000000 --- a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf +++ /dev/null @@ -1,191 +0,0 @@ -// -// Run VEP and/or SNPEFF to annotate VCF files -// - -include { ENSEMBLVEP_VEP } from '../../../modules/nf-core/ensemblvep/vep/main' -include { SNPEFF_SNPEFF } from '../../../modules/nf-core/snpeff/snpeff/main' -include { TABIX_TABIX } from '../../../modules/nf-core/tabix/tabix/main' -include { BCFTOOLS_PLUGINSCATTER } from '../../../modules/nf-core/bcftools/pluginscatter/main' -include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat/main' -include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort/main' - -workflow VCF_ANNOTATE_ENSEMBLVEP_SNPEFF { - take: - ch_vcf // channel: [ val(meta), path(vcf), path(tbi), [path(file1), path(file2)...] ] - ch_fasta // channel: [ val(meta2), path(fasta) ] (optional) - val_vep_genome // value: genome to use - val_vep_species // value: species to use - val_vep_cache_version // value: cache version to use - ch_vep_cache // channel: [ path(cache) ] (optional) - ch_vep_extra_files // channel: [ path(file1), path(file2)... ] (optional) - val_snpeff_db // value: the db version to use for snpEff - ch_snpeff_cache // channel: [ path(cache) ] (optional) - val_tools_to_use // value: a list of tools to use options are: ["ensemblvep", "snpeff"] - val_sites_per_chunk // value: the amount of variants per scattered VCF - - main: - ch_versions = Channel.empty() - - // Check if val_sites_per_chunk is set and scatter if it is - if(val_sites_per_chunk) { - // - // Prepare the input VCF channel for scattering (split VCFs from custom files) - // - - ch_input = ch_vcf - .multiMap { meta, vcf, tbi, custom_files -> - vcf: [ meta, vcf, tbi ] - custom: [ meta, custom_files ] - } - - // - // Scatter the input VCFs into multiple VCFs. These VCFs contain the amount of variants - // specified by `val_sites_per_chunk`. The lower this value is, the more files will be created - // - - BCFTOOLS_PLUGINSCATTER( - ch_input.vcf, - val_sites_per_chunk, - [], - [], - [], - [] - ) - ch_versions = ch_versions.mix(BCFTOOLS_PLUGINSCATTER.out.versions.first()) - - // - // Run the annotation with EnsemblVEP - // - - ch_scatter = BCFTOOLS_PLUGINSCATTER.out.scatter - .map { meta, vcfs -> - // This checks if multiple files were created using the scatter process - // If multiple files are created, a list will be made as output of the process - // So if the output isn't a list, there is always one file and if there is a list, - // the amount of files in the list gets counted by .size() - is_list = vcfs instanceof ArrayList - count = is_list ? vcfs.size() : 1 - [ meta, is_list ? vcfs : [vcfs], count ] - // Channel containing the list of VCFs and the size of this list - } - .transpose(by:1) // Transpose on the VCFs => Creates an entry for each VCF in the list - .combine(ch_input.custom, by: 0) // Re-add the sample specific custom files - .multiMap { meta, vcf, count, custom_files -> - // Define the new ID. The `_annotated` is to disambiguate the VEP output with its input - new_id = "${meta.id}${vcf.name.replace(meta.id,"").tokenize(".")[0]}_annotated" as String - new_meta = meta + [id:new_id] - - // Create channels: one with the VEP input and one with the original ID and count of scattered VCFs - input: [ new_meta, vcf, custom_files ] - count: [ new_meta, meta.id, count ] - } - - ch_vep_input = ch_scatter.input - } else { - // Use the normal input when no scattering has to be performed - ch_vep_input = ch_vcf.map { meta, vcf, tbi, files -> [ meta, vcf, files ] } - } - - // Annotate with ensemblvep if it's part of the requested tools - if("ensemblvep" in val_tools_to_use){ - ENSEMBLVEP_VEP( - ch_vep_input, - val_vep_genome, - val_vep_species, - val_vep_cache_version, - ch_vep_cache, - ch_fasta, - ch_vep_extra_files - ) - ch_versions = ch_versions.mix(ENSEMBLVEP_VEP.out.versions.first()) - - ch_vep_output = ENSEMBLVEP_VEP.out.vcf - ch_vep_reports = ENSEMBLVEP_VEP.out.report - } else { - ch_vep_output = ch_vep_input.map { meta, vcf, files -> [ meta, vcf ] } - ch_vep_reports = Channel.empty() - } - - // Annotate with snpeff if it's part of the requested tools - if("snpeff" in val_tools_to_use){ - SNPEFF_SNPEFF( - ch_vep_output, - val_snpeff_db, - ch_snpeff_cache - ) - ch_versions = ch_versions.mix(SNPEFF_SNPEFF.out.versions.first()) - - ch_snpeff_output = SNPEFF_SNPEFF.out.vcf - ch_snpeff_reports = SNPEFF_SNPEFF.out.report - ch_snpeff_html = SNPEFF_SNPEFF.out.summary_html - ch_snpeff_genes = SNPEFF_SNPEFF.out.genes_txt - } else { - ch_snpeff_output = ch_vep_output - ch_snpeff_reports = Channel.empty() - ch_snpeff_html = Channel.empty() - ch_snpeff_genes = Channel.empty() - } - - // Gather the files back together if they were scattered - if(val_sites_per_chunk) { - // - // Concatenate the VCFs back together with bcftools concat - // - - ch_concat_input = ch_snpeff_output - .join(ch_scatter.count, failOnDuplicate:true, failOnMismatch:true) - .map { meta, vcf, id, count -> - new_meta = meta + [id:id] - [ groupKey(new_meta, count), vcf ] - } - .groupTuple() // Group the VCFs which need to be concatenated - .map { it + [[]] } - - BCFTOOLS_CONCAT( - ch_concat_input - ) - ch_versions = ch_versions.mix(BCFTOOLS_CONCAT.out.versions.first()) - - // - // Sort the concatenate output (bcftools concat is unable to do this on its own) - // - - BCFTOOLS_SORT( - BCFTOOLS_CONCAT.out.vcf - ) - ch_versions = ch_versions.mix(BCFTOOLS_SORT.out.versions.first()) - - ch_ready_vcfs = BCFTOOLS_SORT.out.vcf - } else { - ch_ready_vcfs = ch_snpeff_output - } - - // - // Index the resulting bgzipped VCFs - // - - ch_tabix_input = ch_ready_vcfs - .branch { meta, vcf -> - // Split the bgzipped VCFs from the unzipped VCFs (only bgzipped VCFs should be indexed) - bgzip: vcf.extension == "gz" - unzip: true - return [ meta, vcf, [] ] - } - - TABIX_TABIX( - ch_tabix_input.bgzip - ) - ch_versions = ch_versions.mix(TABIX_TABIX.out.versions) - - ch_vcf_tbi = ch_tabix_input.bgzip - .join(TABIX_TABIX.out.tbi, failOnDuplicate: true, failOnMismatch: true) - .mix(ch_tabix_input.unzip) - - emit: - vcf_tbi = ch_vcf_tbi // channel: [ val(meta), path(vcf), path(tbi) ] - vep_reports = ch_vep_reports // channel: [ path(html) ] - snpeff_reports = ch_snpeff_reports // channel: [ path(csv) ] - snpeff_html = ch_snpeff_html // channel: [ path(html) ] - snpeff_genes = ch_snpeff_genes // channel: [ path(genes) ] - versions = ch_versions // channel: [ versions.yml ] -} diff --git a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml b/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml deleted file mode 100644 index 211144330be..00000000000 --- a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml +++ /dev/null @@ -1,73 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: vcf_annotate_ensemblvep_snpeff -description: | - Perform annotation with ensemblvep and/or snpeff and bgzip + tabix index the resulting VCF file. This subworkflow uses the scatter-gather method to run VEP/snpEff in parallel to increase throughput. The input VCF is split into multiple smaller VCFs of fixed size, which are annotated separately and concatenated back together to a single output file per sample. Only VCF/BCF outputs are currently supported. -keywords: - - vcf - - annotation - - ensemblvep - - snpeff -modules: - - ensemblvep/vep - - snpeff/snpeff - - tabix/tabix - - bcftools/pluginscatter - - bcftools/concat -input: - - ch_vcf: - description: | - vcf file to annotate - Structure: [ val(meta), path(vcf), path(tbi) ] - - ch_fasta: - description: | - Reference genome fasta file (optional) - Structure: [ val(meta2), path(fasta) ] - - val_vep_genome: - type: string - description: genome to use for ensemblvep - - val_vep_species: - type: string - description: species to use for ensemblvep - - val_vep_cache_version: - type: integer - description: cache version to use for ensemblvep - - ch_vep_cache: - description: | - the root cache folder for ensemblvep (optional) - Structure: [ path(cache) ] - - ch_vep_extra_files: - description: | - any extra files needed by plugins for ensemblvep (optional) - Structure: [ path(file1), path(file2)... ] - - val_snpeff_db: - type: string - description: database to use for snpeff - - ch_snpeff_cache: - description: | - the root cache folder for snpeff (optional) - Structure: [ path(cache) ] - - val_tools_to_use: - type: list - description: The tools to use. Options => '["ensemblvep", "snpeff"]' - - val_sites_per_chunk: - type: integer - description: | - The amount of variants per scattered VCF. - Set this value to `null`, `[]` or `false` to disable scattering. -output: - - vcf_tbi: - description: | - Compressed vcf file + tabix index - Structure: [ val(meta), path(vcf), path(tbi) ] - - reports: - type: file - description: html reports - pattern: "*.html" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@maxulysse" - - "@matthdsm" - - "@nvnieuwk" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index db13db14cb8..71c6735a89a 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -3544,10 +3544,6 @@ subworkflows/vcf_annotate_ensemblvep: - subworkflows/nf-core/vcf_annotate_ensemblvep/** - tests/subworkflows/nf-core/vcf_annotate_ensemblvep/** -subworkflows/vcf_annotate_ensemblvep_snpeff: - - subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/** - - tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/** - subworkflows/vcf_annotate_snpeff: - subworkflows/nf-core/vcf_annotate_snpeff/** - tests/subworkflows/nf-core/vcf_annotate_snpeff/** diff --git a/tests/modules/nf-core/bcftools/query/test.yml b/tests/modules/nf-core/bcftools/query/test.yml index 7acf1880dbd..829ee630e4d 100644 --- a/tests/modules/nf-core/bcftools/query/test.yml +++ b/tests/modules/nf-core/bcftools/query/test.yml @@ -1,19 +1,17 @@ -- name: bcftools query test_bcftools_query +- name: bcftools query command: nextflow run ./tests/modules/nf-core/bcftools/query -entry test_bcftools_query -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/bcftools/query/nextflow.config tags: - bcftools - bcftools/query files: - path: output/bcftools/out.txt - md5sum: 51d135de052f3bcef50dcd6b74806094 - - path: output/bcftools/versions.yml + md5sum: c32a6d28f185822d8fe1eeb7e42ec155 -- name: bcftools query test_bcftools_query_with_optional_files +- name: bcftools query with optional files command: nextflow run ./tests/modules/nf-core/bcftools/query -entry test_bcftools_query_with_optional_files -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/bcftools/query/nextflow.config tags: - bcftools - bcftools/query files: - path: output/bcftools/out.txt - md5sum: 1785d1957ba7206df852d0689b91753f - - path: output/bcftools/versions.yml + md5sum: 5a87e0865df2f0ab2884fc113ec2a70d diff --git a/tests/modules/nf-core/tabix/bgziptabix/main.nf b/tests/modules/nf-core/tabix/bgziptabix/main.nf index f4bc4754bf5..26e17101f8b 100644 --- a/tests/modules/nf-core/tabix/bgziptabix/main.nf +++ b/tests/modules/nf-core/tabix/bgziptabix/main.nf @@ -2,15 +2,12 @@ nextflow.enable.dsl = 2 -include { TABIX_BGZIPTABIX as TABIX_BGZIPTABIX_TBI } from '../../../../../modules/nf-core/tabix/bgziptabix/main.nf' -include { TABIX_BGZIPTABIX as TABIX_BGZIPTABIX_CSI } from '../../../../../modules/nf-core/tabix/bgziptabix/main.nf' +include { TABIX_BGZIPTABIX } from '../../../../../modules/nf-core/tabix/bgziptabix/main.nf' workflow test_tabix_bgziptabix { input = [ [ id:'test' ], // meta map [ file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) ] ] - TABIX_BGZIPTABIX_TBI ( input ) - TABIX_BGZIPTABIX_CSI ( input ) - + TABIX_BGZIPTABIX ( input ) } diff --git a/tests/modules/nf-core/tabix/bgziptabix/nextflow.config b/tests/modules/nf-core/tabix/bgziptabix/nextflow.config index 7830e915346..041bfa6acc1 100644 --- a/tests/modules/nf-core/tabix/bgziptabix/nextflow.config +++ b/tests/modules/nf-core/tabix/bgziptabix/nextflow.config @@ -2,12 +2,8 @@ process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: TABIX_BGZIPTABIX_TBI { + withName: TABIX_BGZIPTABIX { ext.args2 = '-p vcf' } - withName: TABIX_BGZIPTABIX_CSI { - ext.args2 = '-p vcf --csi' - } - } diff --git a/tests/modules/nf-core/tabix/bgziptabix/test.yml b/tests/modules/nf-core/tabix/bgziptabix/test.yml index 6bbfb09c798..78ce288b321 100644 --- a/tests/modules/nf-core/tabix/bgziptabix/test.yml +++ b/tests/modules/nf-core/tabix/bgziptabix/test.yml @@ -5,8 +5,6 @@ - tabix files: - path: output/tabix/test.vcf.gz - md5sum: 02aff9ae43faf2594f28e2cf87b32ba9 + md5sum: fc178eb342a91dc0d1d568601ad8f8e2 - path: output/tabix/test.vcf.gz.tbi - md5sum: d22e5b84e4fcd18792179f72e6da702e - - path: output/tabix/test.vcf.gz.csi - md5sum: 6f733d80ee760fcc8fdbe504a03f2640 + md5sum: 36e11bf96ed0af4a92caa91a68612d64 diff --git a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf b/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf deleted file mode 100644 index d6af5c2c999..00000000000 --- a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { VCF_ANNOTATE_ENSEMBLVEP_SNPEFF } from '../../../../subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf' - -workflow vcf_annotate_ensemblvep_snpeff_vep { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - [[],[]], - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - [], - [], - ["ensemblvep"], - 5 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_snpeff { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - [[],[]], - [], - [], - [], - [], - [], - "WBcel235.99", - [], - ["snpeff"], - 5 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_both { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - [[],[]], - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - "WBcel235.99", - [], - ["snpeff", "ensemblvep"], - 5 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_large_chunks { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - fasta = [ - [id:"fasta"], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - fasta, - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - [], - [], - ["ensemblvep"], - 100 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_no_scatter { - input = Channel.of([ - [ id:'test1' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - fasta = [ - [id:"fasta"], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - fasta, - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - [], - [], - ["ensemblvep"], - [] - ) -} diff --git a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config b/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config deleted file mode 100644 index e48fa306d67..00000000000 --- a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config +++ /dev/null @@ -1,21 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: ENSEMBLVEP_VEP { - container = 'docker.io/nfcore/vep:108.2.WBcel235' - ext.args = {[ - "--vcf --offline", - meta.id.startsWith("custom_test") ? "--custom test3.vcf.gz,,vcf,exact,0,TOPMED" : "" - ].join(" ")} - } - - withName: SNPEFF_SNPEFF { - container = 'docker.io/nfcore/snpeff:5.0.WBcel235' - } - - withName: BCFTOOLS_CONCAT { - ext.prefix = { "${meta.id}_concat" } - } - -} diff --git a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml b/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml deleted file mode 100644 index 92fa202cb90..00000000000 --- a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml +++ /dev/null @@ -1,384 +0,0 @@ -- name: vcf_annotate_ensemblvep_snpeff vcf_annotate_ensemblvep_snpeff_vep - command: nextflow run ./tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff -entry vcf_annotate_ensemblvep_snpeff_vep -c ./tests/config/nextflow.config - tags: - - bcftools - - bcftools/concat - - bcftools/pluginscatter - - bcftools/sort - - ensemblvep - - ensemblvep/vep - - snpeff - - snpeff/snpeff - - subworkflows - - subworkflows/vcf_annotate_ensemblvep_snpeff - - tabix - - tabix/tabix - files: - - path: output/bcftools/custom_test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/custom_test0_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/custom_test0_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/custom_test1_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/custom_test1_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/test0_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/test0_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/test1_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/test1_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/tabix/custom_test.vcf.gz.tbi - - path: output/tabix/test.vcf.gz.tbi -- name: vcf_annotate_ensemblvep_snpeff vcf_annotate_ensemblvep_snpeff_snpeff - command: nextflow run ./tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff -entry vcf_annotate_ensemblvep_snpeff_snpeff -c ./tests/config/nextflow.config - tags: - - bcftools - - bcftools/concat - - bcftools/pluginscatter - - bcftools/sort - - ensemblvep - - ensemblvep/vep - - snpeff - - snpeff/snpeff - - subworkflows - - subworkflows/vcf_annotate_ensemblvep_snpeff - - tabix - - tabix/tabix - files: - - path: output/bcftools/custom_test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/snpeff/custom_test0_annotated.ann.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/snpeff/custom_test0_annotated.csv - contains: - - "# Summary table" - - path: output/snpeff/custom_test0_annotated.genes.txt - md5sum: 130536bf0237d7f3f746d32aaa32840a - - path: output/snpeff/custom_test1_annotated.ann.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/snpeff/custom_test1_annotated.csv - contains: - - "# Summary table" - - path: output/snpeff/custom_test1_annotated.genes.txt - md5sum: 130536bf0237d7f3f746d32aaa32840a - - path: output/snpeff/snpEff_summary.html - contains: - - '