From f485b6f44cd44f80c9017091e88472ebcb32eeff Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 21 Sep 2023 20:06:57 -0400 Subject: [PATCH 01/16] bam_generation --- modules/{nf-core-test => mskcc}/.gitkeep | 0 modules/nf-core-test/fakeqc/main.nf | 1 - modules/nf-core-test/fakeqc/meta.yml | 1 - modules/nf-core-test/fastqc/main.nf | 59 -- modules/nf-core-test/fastqc/meta.yml | 52 -- modules/nf-core-test/multiqc/main.nf | 49 -- modules/nf-core-test/multiqc/meta.yml | 48 -- .../nf-core-test/bwa_markdup_bqsr/main.nf | 107 +++ .../bwa_markdup_bqsr/nextflow.config | 27 + tests/config/nextflow.config | 44 ++ tests/config/pytest_modules.yml | 4 + tests/config/test_data.config | 713 ++++++++++++++++++ .../nf-core-test/bwa_markdup_bqsr/main.nf | 41 + .../bwa_markdup_bqsr/nextflow.config | 5 + .../nf-core-test/bwa_markdup_bqsr/test.yml | 11 + tests/test_versions_yml.py | 79 ++ 16 files changed, 1031 insertions(+), 210 deletions(-) rename modules/{nf-core-test => mskcc}/.gitkeep (100%) delete mode 100644 modules/nf-core-test/fakeqc/main.nf delete mode 100644 modules/nf-core-test/fakeqc/meta.yml delete mode 100644 modules/nf-core-test/fastqc/main.nf delete mode 100644 modules/nf-core-test/fastqc/meta.yml delete mode 100644 modules/nf-core-test/multiqc/main.nf delete mode 100644 modules/nf-core-test/multiqc/meta.yml create mode 100755 subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf create mode 100755 subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config create mode 100644 tests/config/nextflow.config create mode 100644 tests/config/pytest_modules.yml create mode 100644 tests/config/test_data.config create mode 100644 tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf create mode 100644 tests/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config create mode 100644 tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml create mode 100644 tests/test_versions_yml.py diff --git a/modules/nf-core-test/.gitkeep b/modules/mskcc/.gitkeep similarity index 100% rename from modules/nf-core-test/.gitkeep rename to modules/mskcc/.gitkeep diff --git a/modules/nf-core-test/fakeqc/main.nf b/modules/nf-core-test/fakeqc/main.nf deleted file mode 100644 index 8b137891..00000000 --- a/modules/nf-core-test/fakeqc/main.nf +++ /dev/null @@ -1 +0,0 @@ - diff --git a/modules/nf-core-test/fakeqc/meta.yml b/modules/nf-core-test/fakeqc/meta.yml deleted file mode 100644 index 8b137891..00000000 --- a/modules/nf-core-test/fakeqc/meta.yml +++ /dev/null @@ -1 +0,0 @@ - diff --git a/modules/nf-core-test/fastqc/main.nf b/modules/nf-core-test/fastqc/main.nf deleted file mode 100644 index 7777e0aa..00000000 --- a/modules/nf-core-test/fastqc/main.nf +++ /dev/null @@ -1,59 +0,0 @@ -process FASTQC { - tag "$meta.id" - label 'process_medium' - - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'biocontainers/fastqc:0.11.9--0' }" - - input: - tuple val(meta), path(reads) - - output: - tuple val(meta), path("*.html"), emit: html - tuple val(meta), path("*.zip") , emit: zip - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - // Add soft-links to original FastQs for consistent naming in pipeline - def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } - - stub: - def prefix = task.ext.prefix ?: "${meta.id}" - """ - touch ${prefix}.html - touch ${prefix}.zip - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core-test/fastqc/meta.yml b/modules/nf-core-test/fastqc/meta.yml deleted file mode 100644 index 4da5bb5a..00000000 --- a/modules/nf-core-test/fastqc/meta.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: fastqc -description: Run FastQC on sequenced reads -keywords: - - quality control - - qc - - adapters - - fastq -tools: - - fastqc: - description: | - FastQC gives general quality metrics about your reads. - It provides information about the quality score distribution - across your reads, the per base sequence content (%A/C/G/T). - You get information about adapter contamination and other - overrepresented sequences. - homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ - documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ - licence: ["GPL-2.0-only"] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - html: - type: file - description: FastQC report - pattern: "*_{fastqc.html}" - - zip: - type: file - description: FastQC report archive - pattern: "*_{fastqc.zip}" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@grst" - - "@ewels" - - "@FelixKrueger" diff --git a/modules/nf-core-test/multiqc/main.nf b/modules/nf-core-test/multiqc/main.nf deleted file mode 100644 index c6d1fe52..00000000 --- a/modules/nf-core-test/multiqc/main.nf +++ /dev/null @@ -1,49 +0,0 @@ -process MULTIQC { - label 'process_medium' - - conda (params.enable_conda ? 'bioconda::multiqc=1.12' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'biocontainers/multiqc:1.12--pyhdfd78af_0' }" - - input: - path multiqc_files, stageAs: "?/*" - tuple path(multiqc_config), path(multiqc_logo) - - output: - path "*multiqc_report.html", emit: report - path "*_data" , emit: data - path "*_plots" , optional:true, emit: plots - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - def config = multiqc_config ? "--config $multiqc_config" : '' - """ - multiqc \\ - --force \\ - $config \\ - $args \\ - . - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ - - stub: - """ - touch multiqc_data - touch multiqc_plots - touch multiqc_report.html - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core-test/multiqc/meta.yml b/modules/nf-core-test/multiqc/meta.yml deleted file mode 100644 index d385b4d2..00000000 --- a/modules/nf-core-test/multiqc/meta.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: MultiQC -description: Aggregate results from bioinformatics analyses across many samples into a single report -keywords: - - QC - - bioinformatics tools - - Beautiful stand-alone HTML report -tools: - - multiqc: - description: | - MultiQC searches a given directory for analysis logs and compiles a HTML report. - It's a general use tool, perfect for summarising the output from numerous bioinformatics tools. - homepage: https://multiqc.info/ - documentation: https://multiqc.info/docs/ - licence: ["GPL-3.0-or-later"] -input: - - multiqc_files: - type: file - description: | - List of reports / files recognised by MultiQC, for example the html and zip output of FastQC - - multiqc_config: - type: file - description: Config yml for MultiQC - pattern: "*.{yml,yaml}" - - multiqc_logo: - type: file - description: Logo file for MultiQC - pattern: "*.{png}" -output: - - report: - type: file - description: MultiQC report file - pattern: "multiqc_report.html" - - data: - type: directory - description: MultiQC data dir - pattern: "multiqc_data" - - plots: - type: file - description: Plots created by MultiQC - pattern: "*_data" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@abhi18av" - - "@bunop" - - "@drpatelh" diff --git a/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf b/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf new file mode 100755 index 00000000..4c6e7d68 --- /dev/null +++ b/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf @@ -0,0 +1,107 @@ +include { BWA_MEM } from '../../../modules/nf-core/bwa/mem/main' +include { GATK4_MARKDUPLICATES } from '../../../modules/nf-core/gatk4/markduplicates/main' +include { GATK4_MARKDUPLICATES_SPARK } from '../../../modules/nf-core/gatk4/markduplicatesspark/main' +include { GATK4_APPLYBQSR } from '../../../modules/nf-core/gatk4/applybqsr/main' +include { GATK4_APPLYBQSR_SPARK } from '../../../modules/nf-core/gatk4/applybqsrspark/main' +include { GATK4_BASERECALIBRATOR } from '../../../modules/nf-core/gatk4/baserecalibrator/main' +include { GATK4_BASERECALIBRATOR_SPARK } from '../../../modules/nf-core/gatk4/baserecalibratorspark/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' + + +workflow BWA_MARKDUP_BQSR { + take: + reads + fasta + fai + bwa_index + dict + known_sites + known_sites_tbi + spark // true, false + + main: + + ch_versions = Channel.empty() + + // run once for each pair of reads + BWA_MEM( + reads, + bwa_index, + true + ) + ch_versions = ch_versions.mix(BWA_MEM.out.versions.first()) + + grouped_bam_ch = + BWA_MEM.out.bam + .map{meta, bam -> + def new_meta = meta - meta.subMap(['read_group']) + [ new_meta, bam ] + }.groupTuple(by:0) + + if (spark){ + GATK4_MARKDUPLICATES_SPARK( + grouped_bam_ch, + fasta, + fai + ) + GATK4_BASERECALIBRATOR_SPARK( + GATK4_MARKDUPLICATES_SPARK.out.output, + fasta, + fai, + dict, + known_sites, + known_sites_tbi + ) + GATK4_APPLYBQSR_SPARK( + GATK4_MARKDUPLICATES_SPARK.out.output + .join(GATK4_MARKDUPLICATES_SPARK.out.bam_index) + .join(GATK4_BASERECALIBRATOR_SPARK.out.table) + .map{meta, bam, bai, table -> + [ meta, bam, bai, table, [] ] // what is intervals? + }, + fasta, + fai, + dict + ) + bqsr_bam = GATK4_APPLYBQSR_SPARK.out.bam + + } else { + GATK4_MARKDUPLICATES( + grouped_bam_ch, + fasta, + fai + ) + GATK4_BASERECALIBRATOR( + GATK4_MARKDUPLICATES.out.output, + fasta, + fai, + dict, + known_sites, + known_sites_tbi + ) + GATK4_APPLYBQSR( + GATK4_MARKDUPLICATES.out.output + .join(GATK4_MARKDUPLICATES.out.bam_index) + .join(GATK4_BASERECALIBRATOR.out.table) + .map{meta, bam, bai, table -> + [ meta, bam, bai, table, [] ] // what is intervals? + }, + fasta, + fai, + dict + ) + bqsr_bam = GATK4_APPLYBQSR.out.bam + // bqsr_bai = GATK4_APPLYBQSR.out.bai + } + + SAMTOOLS_INDEX(bqsr_bam) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) + + bqsr_bai = SAMTOOLS_INDEX.out.bai + + emit: + bam = bqsr_bam + bai = bqsr_bai + versions = ch_versions + +} \ No newline at end of file diff --git a/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config b/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config new file mode 100755 index 00000000..e18f96ae --- /dev/null +++ b/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config @@ -0,0 +1,27 @@ +process { + withName: '.*:BWA_MARKDUP_BQSR:BWA_MEM' { + ext.args = "-R "\${meta.read_group}" + ext.prefix = { "${meta.id}@${meta.read_group}" } + } + + withName: '.*:BWA_MARKDUP_BQSR:GATK4_MARKDUPLICATES.*' { + ext.args = { + [ + "--ASSUME_SORT_ORDER coordinate", + "--CREATE_INDEX true" + ].join(" ") + } + } + + withName: '.*:BWA_MARKDUP_BQSR:GATK4_BASERECALIBRATOR.*' { + ext.args = "--verbosity INFO" + } + + withName: '.*:BWA_MARKDUP_BQSR:GATK4_APPLYBQSR' { + ext.args = "--create-output-bam-index true --create-output-bam-index true" + } + + withName: '.*:BWA_MARKDUP_BQSR:GATK4_APPLYBQSR_SPARK' { + ext.args = "--create-output-bam-index true" + } +} \ No newline at end of file diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config new file mode 100644 index 00000000..104f1d9e --- /dev/null +++ b/tests/config/nextflow.config @@ -0,0 +1,44 @@ +params { + outdir = "output/" + publish_dir_mode = "copy" + singularity_pull_docker_container = false + test_data_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules' +} + +process { + cpus = 2 + memory = 3.GB + time = 2.h +} + +if ("$PROFILE" == "singularity") { + singularity.enabled = true + singularity.autoMounts = true +} else if ("$PROFILE" == "conda") { + conda.enabled = true +} else if ("$PROFILE" == "mamba") { + conda.enabled = true + conda.useMamba = true +} else if ("$PROFILE" == "podman") { + podman.enabled = true + podman.userEmulation = true + podman.runOptions = "--runtime crun --platform linux/x86_64 --systemd=always" +} else { + docker.enabled = true + docker.userEmulation = true + docker.runOptions = "--platform linux/x86_64" +} + +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' + +// Increase time available to build Conda environment +conda { createTimeout = "120 min" } + +// Load test_data.config containing paths to test data +includeConfig 'test_data.config' + +manifest { + nextflowVersion = '!>=23.04.0' +} diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml new file mode 100644 index 00000000..75a0db83 --- /dev/null +++ b/tests/config/pytest_modules.yml @@ -0,0 +1,4 @@ +subworkflows/bwa_markdup_bqsr: + - subworkflows/nf-core-test/bwa_markdup_bqsr/** + - tests/subworkflows/nf-core-test/bwa_markdup_bqsr/** + diff --git a/tests/config/test_data.config b/tests/config/test_data.config new file mode 100644 index 00000000..b2191d73 --- /dev/null +++ b/tests/config/test_data.config @@ -0,0 +1,713 @@ +// README: +// https://github.com/nf-core/test-datasets/blob/modules/README.md + +params { + // Base directory for test data + test_data_base = "https://raw.githubusercontent.com/nf-core/test-datasets/modules" + + test_data { + 'sarscov2' { + 'genome' { + genome_fasta = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta" + genome_fasta_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta.gz" + genome_fasta_fai = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta.fai" + genome_fasta_txt_zst = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta.txt.zst" + genome_dict = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.dict" + genome_gff3 = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.gff3" + genome_gff3_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.gff3.gz" + genome_gtf = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.gtf" + genome_paf = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.paf" + genome_sizes = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.sizes" + transcriptome_fasta = "${params.test_data_base}/data/genomics/sarscov2/genome/transcriptome.fasta" + proteome_fasta = "${params.test_data_base}/data/genomics/sarscov2/genome/proteome.fasta" + transcriptome_paf = "${params.test_data_base}/data/genomics/sarscov2/genome/transcriptome.paf" + + test_bed = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test.bed" + test_bed_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test.bed.gz" + test2_bed = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test2.bed" + test_bed12 = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test.bed12" + baits_bed = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/baits.bed" + bed_autosql = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/bed6alt.as" + + reference_cnn = "${params.test_data_base}/data/genomics/sarscov2/genome/cnn/reference.cnn" + + kraken2 = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2" + kraken2_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2.tar.gz" + + kraken2_bracken = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2_bracken" + kraken2_bracken_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2_bracken.tar.gz" + + kaiju = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kaiju" + kaiju_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kaiju.tar.gz" + + kofamscan_profiles_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kofamscan/profiles.tar.gz" + kofamscan_ko_list_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kofamscan/ko_list.gz" + + ncbi_taxmap_zip = "${params.test_data_base}/data/genomics/sarscov2/genome/db/maltextract/ncbi_taxmap.zip" + taxon_list_txt = "${params.test_data_base}/data/genomics/sarscov2/genome/db/maltextract/taxon_list.txt" + + mmseqs_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/mmseqs.tar.gz" + + all_sites_fas = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/all_sites.fas" + informative_sites_fas = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/informative_sites.fas" + + contigs_genome_maf_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/last/contigs.genome.maf.gz" + contigs_genome_par = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/last/contigs.genome.par" + lastdb_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/last/lastdb.tar.gz" + + baits_interval_list = "${params.test_data_base}/data/genomics/sarscov2/genome/picard/baits.interval_list" + targets_interval_list = "${params.test_data_base}/data/genomics/sarscov2/genome/picard/targets.interval_list" + regions_txt = "${params.test_data_base}/data/genomics/sarscov2/genome/graphtyper/regions.txt" + } + 'illumina' { + test_single_end_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.single_end.bam" + test_single_end_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.single_end.sorted.bam" + test_single_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.single_end.sorted.bam.bai" + test_paired_end_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.bam" + test_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam" + test_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai" + test_paired_end_methylated_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.methylated.bam" + test_paired_end_methylated_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam" + test_paired_end_methylated_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai" + test_unaligned_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.unaligned.bam" + + test_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_1.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_2.fastq.gz" + test_interleaved_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz" + test_1_fastq_txt_zst = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_1.fastq.txt.zst" + test2_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test2_1.fastq.gz" + test2_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test2_2.fastq.gz" + test_methylated_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz" + test_methylated_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz" + + test_bedgraph = "${params.test_data_base}/data/genomics/sarscov2/illumina/bedgraph/test.bedgraph" + + test_bigwig = "${params.test_data_base}/data/genomics/sarscov2/illumina/bigwig/test.bigwig" + + test_wig_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/wig/test.wig.gz" + + test_baserecalibrator_table = "${params.test_data_base}/data/genomics/sarscov2/illumina/gatk/test.baserecalibrator.table" + + test_computematrix_mat_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/deeptools/test.computeMatrix.mat.gz" + + test_bcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.bcf" + + test_vcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.vcf" + test_vcf_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.vcf.gz" + test_vcf_gz_tbi = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.vcf.gz.tbi" + test2_vcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.vcf" + test2_vcf_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.vcf.gz" + test2_vcf_gz_tbi = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi" + test2_vcf_targets_tsv_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.targets.tsv.gz" + test3_vcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test3.vcf" + test3_vcf_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test3.vcf.gz" + test3_vcf_gz_tbi = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test3.vcf.gz.tbi" + + contigs_fasta = "${params.test_data_base}/data/genomics/sarscov2/illumina/fasta/contigs.fasta" + scaffolds_fasta = "${params.test_data_base}/data/genomics/sarscov2/illumina/fasta/scaffolds.fasta" + + assembly_gfa = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa" + assembly_gfa_bgz = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa.bgz" + assembly_gfa_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa.gz" + assembly_gfa_zst = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa.zst" + + test_single_end_bam_readlist_txt = "${params.test_data_base}/data/genomics/sarscov2/illumina/picard/test.single_end.bam.readlist.txt" + + SRR13255544_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR13255544.tar.gz" + SRR11140744_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR11140744.tar.gz" + } + 'nanopore' { + test_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/nanopore/bam/test.sorted.bam" + test_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/nanopore/bam/test.sorted.bam.bai" + + fast5_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/nanopore/fast5/fast5.tar.gz" + + test_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/nanopore/fastq/test.fastq.gz" + + test_sequencing_summary = "${params.test_data_base}/data/genomics/sarscov2/nanopore/sequencing_summary/test.sequencing_summary.txt" + } + 'metagenome' { + classified_reads_assignment = "${params.test_data_base}/data/genomics/sarscov2/metagenome/test_1.kraken2.reads.txt" + kraken_report = "${params.test_data_base}/data/genomics/sarscov2/metagenome/test_1.kraken2.report.txt" + krona_taxonomy = "${params.test_data_base}/data/genomics/sarscov2/metagenome/krona_taxonomy.tab" + seqid2taxid_map = "${params.test_data_base}/data/genomics/sarscov2/metagenome/seqid2taxid.map" + nodes_dmp = "${params.test_data_base}/data/genomics/sarscov2/metagenome/nodes.dmp" + names_dmp = "${params.test_data_base}/data/genomics/sarscov2/metagenome/names.dmp" + } + } + 'mus_musculus' { + 'genome' { + rnaseq_samplesheet = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.samplesheet.csv" + rnaseq_genemeta = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.gene_meta.tsv" + rnaseq_contrasts = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.contrasts.csv" + rnaseq_matrix = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.salmon.merged.gene_counts.top1000cov.tsv" + deseq_results = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.salmon.merged.deseq2.results.tsv" + } + 'illumina' { + test_1_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/mageck/ERR376998.small.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/mageck/ERR376999.small.fastq.gz" + } + 'csv' { + count_table = "${params.test_data_base}/data/genomics/mus_musculus/mageck/count_table.csv" + library = "${params.test_data_base}/data/genomics/mus_musculus/mageck/yusa_library.csv" + } + 'txt' { + design_matrix = "${params.test_data_base}/data/genomics/mus_musculus/mageck/design_matrix.txt" + } + } + 'homo_sapiens' { + '10xgenomics' { + cellranger { + test_10x_10k_pbmc_5fb_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_5fb_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_5gex_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_5gex_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_b_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_b_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_t_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/tcell/subsampled_sc5p_v2_hs_PBMC_10k_t_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_t_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/tcell/subsampled_sc5p_v2_hs_PBMC_10k_t_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_feature_ref_csv = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/sc5p_v2_hs_PBMC_10k_multi_5gex_5fb_b_t_feature_ref.csv" + + test_10x_10k_pbmc_cmo_cmo_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_cmo_cmo_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex1_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex1_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex2_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_2/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_2_gex_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex2_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_2/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_2_gex_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_cmo_feature_ref_csv = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/10k_pbmc_cmo_count_feature_reference.csv" + + test_10x_5k_cmvpos_tcells_ab_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R1_001.fastq.gz" + test_10x_5k_cmvpos_tcells_ab_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R2_001.fastq.gz" + test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz" + test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz" + test_10x_5k_cmvpos_tcells_vdj_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R1_001.fastq.gz" + test_10x_5k_cmvpos_tcells_vdj_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R2_001.fastq.gz" + test_10x_5k_cmvpos_tcells_feature_ref_csv = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv" + + test_10x_vdj_ref_json = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/reference.json" + test_10x_vdj_ref_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/regions.fa" + test_10x_vdj_ref_suppfasta = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/supp_regions.fa" + } + spaceranger { + test_10x_ffpe_cytassist_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_2_S1_L001_R1_001.fastq.gz" + test_10x_ffpe_cytassist_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_2_S1_L001_R2_001.fastq.gz" + test_10x_ffpe_cytassist_image = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_image.tif" + test_10x_ffpe_cytassist_probeset = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_probe_set.csv" + + test_10x_ffpe_v1_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-ovarian-cancer-1-standard_v1_ffpe/Visium_FFPE_Human_Ovarian_Cancer_S1_L001_R1_001.fastq.gz" + test_10x_ffpe_v1_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-ovarian-cancer-1-standard_v1_ffpe/Visium_FFPE_Human_Ovarian_Cancer_S1_L001_R2_001.fastq.gz" + test_10x_ffpe_v1_image = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-ovarian-cancer-1-standard_v1_ffpe/Visium_FFPE_Human_Ovarian_Cancer_image.jpg" + } + } + 'genome' { + genome_elfasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.elfasta" + genome_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta" + genome_fasta_fai = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.fai" + genome_fasta_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.gz" + genome_fasta_gz_fai = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.gz.fai" + genome_fasta_gz_gzi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.gz.gzi" + genome_strtablefile = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome_strtablefile.zip" + genome_dict = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.dict" + genome_gff3 = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.gff3" + genome_gtf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.gtf" + genome_interval_list = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.interval_list" + genome_multi_interval_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.multi_intervals.bed" + genome_blacklist_interval_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.blacklist_intervals.bed" + genome_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.sizes" + genome_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.bed" + genome_header = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.header" + genome_bed_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.bed.gz" + genome_bed_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.bed.gz.tbi" + genome_elsites = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.elsites" + transcriptome_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/transcriptome.fasta" + genome2_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome2.fasta" + genome_chain_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.chain.gz" + genome_annotated_interval_tsv = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.annotated_intervals.tsv" + genome_mt_gb = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.NC_012920_1.gb" + genome_preprocessed_count_tsv = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.preprocessed_intervals.counts.tsv" + genome_preprocessed_interval_list = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.preprocessed_intervals.interval_list" + genome_ploidy_model = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.ploidy_model.tar.gz" + genome_ploidy_calls = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.ploidy_calls.tar.gz" + genome_germline_cnv_model = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.germline_cnv_model.tar.gz" + genome_germline_cnv_calls = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.germline_cnv_calls.tar.gz" + genome_21_sdf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome_sdf.tar.gz" + genome_21_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta" + genome_21_fasta_fai = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta.fai" + genome_21_gencode_gtf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/chr21_gencode.gtf" + genome_21_dict = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.dict" + genome_21_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.sizes" + genome_21_interval_list = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.interval_list" + genome_21_annotated_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/annotated.bed" + genome_21_multi_interval_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed" + genome_21_multi_interval_antitarget_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.antitarget.bed" + genome_21_multi_interval_bed_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed.gz" + genome_21_multi_interval_bed_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed.gz.tbi" + genome_21_chromosomes_dir = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/chromosomes.tar.gz" + genome_21_reference_cnn = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/reference_chr21.cnn" + genome_21_eigenstrat_snp = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/chr_21.snp" + genome_21_stitch_posfile = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/dbsnp_138.hg38.first_10_biallelic_sites.tsv" + + dbsnp_146_hg38_elsites = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.elsites" + dbsnp_146_hg38_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz" + dbsnp_146_hg38_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz.tbi" + gnomad_r2_1_1_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz" + gnomad_r2_1_1_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz.tbi" + mills_and_1000g_indels_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz" + mills_and_1000g_indels_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz.tbi" + syntheticvcf_short_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.vcf.gz" + syntheticvcf_short_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.vcf.gz.tbi" + syntheticvcf_short_score = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.score" + gnomad_r2_1_1_sv_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1-sv.vcf.gz" + gnomad2_r2_1_1_sv_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD2.r2.1.1-sv.vcf.gz" + + hapmap_3_3_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/hapmap_3.3.hg38.vcf.gz" + hapmap_3_3_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/hapmap_3.3.hg38.vcf.gz.tbi" + res_1000g_omni2_5_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_omni2.5.hg38.vcf.gz" + res_1000g_omni2_5_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_omni2.5.hg38.vcf.gz.tbi" + res_1000g_phase1_snps_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_phase1.snps.hg38.vcf.gz" + res_1000g_phase1_snps_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_phase1.snps.hg38.vcf.gz.tbi" + dbsnp_138_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbsnp_138.hg38.vcf.gz" + dbsnp_138_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbsnp_138.hg38.vcf.gz.tbi" + gnomad_r2_1_1_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/gnomAD.r2.1.1.vcf.gz" + gnomad_r2_1_1_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/gnomAD.r2.1.1.vcf.gz.tbi" + mills_and_1000g_indels_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/mills_and_1000G.indels.hg38.vcf.gz" + mills_and_1000g_indels_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/mills_and_1000G.indels.hg38.vcf.gz.tbi" + haplotype_map = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/haplotype_map.txt" + dbNSFP_4_1a_21_hg38_txt_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbNSFP4.1a.21.txt.gz" + dbNSFP_4_1a_21_hg38_txt_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbNSFP4.1a.21.txt.gz.tbi" + + index_salmon = "${params.test_data_base}/data/genomics/homo_sapiens/genome/index/salmon" + repeat_expansions = "${params.test_data_base}/data/genomics/homo_sapiens/genome/loci/repeat_expansions.json" + justhusky_ped = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/ped/justhusky.ped" + justhusky_minimal_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/ped/justhusky_minimal.vcf.gz" + justhusky_minimal_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/ped/justhusky_minimal.vcf.gz.tbi" + + vcfanno_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/vcfanno/vcfanno_grch38_module_test.tar.gz" + vcfanno_toml = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/vcfanno/vcfanno.toml" + updsites_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/updsites.bed" + + prg_input = "${params.test_data_base}/data/genomics/homo_sapiens/genome/PRG_test.zip" + crispr_functional_counts = "${params.test_data_base}/data/genomics/homo_sapiens/genome/tsv/functional_genomics_counts.tsv" + crispr_functional_library = "${params.test_data_base}/data/genomics/homo_sapiens/genome/tsv/library_functional_genomics.tsv" + + vep_cache = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vep.tar.gz" + affy_array_samplesheet = "${params.test_data_base}/data/genomics/homo_sapiens/array_expression/GSE38751.csv" + affy_array_celfiles_tar = "${params.test_data_base}/data/genomics/homo_sapiens/array_expression/GSE38751_RAW.tar" + + } + 'pangenome' { + pangenome_fa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa" + pangenome_fa_bgzip = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa.gz" + pangenome_fa_bgzip_fai = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa.gz.fai" + pangenome_fa_bgzip_gzi = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa.gz.gzi" + pangenome_paf = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.paf" + pangenome_paf_gz = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.paf.gz" + pangenome_seqwish_gfa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.seqwish.gfa" + pangenome_smoothxg_gfa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.smoothxg.gfa" + pangenome_gfaffix_gfa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.gfaffix.gfa" + 'odgi' { + pangenome_og = "${params.test_data_base}/data/pangenomics/homo_sapiens/odgi/pangenome.og" + pangenome_lay = "${params.test_data_base}/data/pangenomics/homo_sapiens/odgi/pangenome.lay" + } + } + 'illumina' { + test_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam" + test_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai" + test_paired_end_name_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.name.sorted.bam" + test_paired_end_markduplicates_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.markduplicates.sorted.bam" + test_paired_end_markduplicates_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.markduplicates.sorted.bam.bai" + test_paired_end_markduplicates_sorted_referencesn_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.markduplicates.sorted.referencesn.txt" + test_paired_end_recalibrated_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.recalibrated.sorted.bam" + test_paired_end_recalibrated_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.recalibrated.sorted.bam.bai" + test_paired_end_umi_consensus_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_consensus.bam" + test_paired_end_umi_converted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_converted.bam" + test_paired_end_umi_grouped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_grouped.bam" + test_paired_end_umi_histogram_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_histogram.txt" + test_paired_end_umi_unsorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_unsorted.bam" + test_paired_end_umi_unsorted_tagged_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.unsorted_tagged.bam" + test_paired_end_hla = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/example_hla_pe.bam" + test_paired_end_hla_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/example_hla_pe.sorted.bam" + test_paired_end_hla_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/example_hla_pe.sorted.bam.bai" + test_rna_paired_end_sorted_chr6_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.rna.paired_end.sorted.chr6.bam" + test_rna_paired_end_sorted_chr6_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.rna.paired_end.sorted.chr6.bam.bai" + + test2_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam" + test2_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai" + test2_paired_end_name_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.name.sorted.bam" + test2_paired_end_markduplicates_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.markduplicates.sorted.bam" + test2_paired_end_markduplicates_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.markduplicates.sorted.bam.bai" + test2_paired_end_recalibrated_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam" + test2_paired_end_recalibrated_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam.bai" + test2_paired_end_umi_consensus_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_consensus.bam" + test2_paired_end_umi_converted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_converted.bam" + test2_paired_end_umi_grouped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_grouped.bam" + test2_paired_end_umi_histogram_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_histogram.txt" + test2_paired_end_umi_unsorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_unsorted.bam" + test2_paired_end_umi_unsorted_tagged_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.unsorted_tagged.bam" + test_paired_end_duplex_umi_unmapped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_unmapped.bam" + test_paired_end_duplex_umi_mapped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_mapped.bam" + test_paired_end_duplex_umi_mapped_tagged_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_mapped_tagged.bam" + test_paired_end_duplex_umi_grouped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_grouped.bam" + test_paired_end_duplex_umi_duplex_consensus_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_duplex_consensus.bam" + + mitochon_standin_recalibrated_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/mitochon_standin.recalibrated.sorted.bam" + mitochon_standin_recalibrated_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/mitochon_standin.recalibrated.sorted.bam.bai" + test_illumina_mt_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test_illumina_mt.bam" + test_illumina_mt_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test_illumina_mt.bam.bai" + + test3_single_end_markduplicates_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test3.single_end.markduplicates.sorted.bam" + + read_group_settings_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/read_group_settings.txt" + + test_paired_end_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram" + test_paired_end_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram.crai" + test_paired_end_markduplicates_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.markduplicates.sorted.cram" + test_paired_end_markduplicates_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.markduplicates.sorted.cram.crai" + test_paired_end_recalibrated_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram" + test_paired_end_recalibrated_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram.crai" + + test2_paired_end_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.sorted.cram" + test2_paired_end_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.sorted.cram.crai" + test2_paired_end_markduplicates_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.markduplicates.sorted.cram" + test2_paired_end_markduplicates_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.markduplicates.sorted.cram.crai" + test2_paired_end_recalibrated_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram" + test2_paired_end_recalibrated_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram.crai" + test3_paired_end_recalibrated_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test3.paired_end.recalibrated.sorted.cram" + test3_paired_end_recalibrated_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test3.paired_end.recalibrated.sorted.cram.crai" + + test_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz" + test_umi_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test.umi_1.fastq.gz" + test_umi_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test.umi_2.fastq.gz" + test2_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2_1.fastq.gz" + test2_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2_2.fastq.gz" + test2_umi_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2.umi_1.fastq.gz" + test2_umi_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2.umi_2.fastq.gz" + test_rnaseq_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_rnaseq_1.fastq.gz" + test_rnaseq_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_rnaseq_2.fastq.gz" + test_paired_end_duplex_umi_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_duplex_umi_1.fastq.gz" + test_paired_end_duplex_umi_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_duplex_umi_2.fastq.gz" + + test_baserecalibrator_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test.baserecalibrator.table" + test2_baserecalibrator_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test2.baserecalibrator.table" + test_pileups_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test.pileups.table" + test2_pileups_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test2.pileups.table" + + test_paired_end_sorted_dragstrmodel = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_paired_end_sorted_dragstrmodel.txt" + + test_genomicsdb_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_genomicsdb.tar.gz" + test_pon_genomicsdb_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_pon_genomicsdb.tar.gz" + + test2_haplotc_ann_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.ann.vcf.gz" + test2_haplotc_ann_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.ann.vcf.gz.tbi" + test_haplotc_cnn_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test_haplotcaller.cnn.vcf.gz" + test_haplotc_cnn_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test_haplotcaller.cnn.vcf.gz.tbi" + + test2_haplotc_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.vcf.gz" + test2_haplotc_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.vcf.gz.tbi" + + test2_recal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2.recal" + test2_recal_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2.recal.idx" + test2_tranches = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2.tranches" + test2_allele_specific_recal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2_allele_specific.recal" + test2_allele_specific_recal_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2_allele_specific.recal.idx" + test2_allele_specific_tranches = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2_allele_specific.tranches" + + test_test2_paired_mutect2_calls_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz" + test_test2_paired_mutect2_calls_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz.tbi" + test_test2_paired_mutect2_calls_vcf_gz_stats = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz.stats" + test_test2_paired_mutect2_calls_f1r2_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.f1r2.tar.gz" + test_test2_paired_mutect2_calls_artifact_prior_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_test2_paired_mutect2_calls.artifact-prior.tar.gz" + test_test2_paired_segmentation_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_test2_paired.segmentation.table" + test_test2_paired_contamination_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_test2_paired.contamination.table" + + test_genome_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf" + test_genome_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz" + test_genome_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz.tbi" + test_genome_vcf_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.idx" + + test_genome_vcf_ud = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/svd/test.genome.vcf.UD" + test_genome_vcf_mu = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/svd/test.genome.vcf.mu" + test_genome_vcf_bed = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/svd/test.genome.vcf.bed" + + test2_genome_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf" + test2_genome_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf.gz" + test2_genome_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf.gz.tbi" + test2_genome_vcf_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf.idx" + + test_genome21_indels_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test.genome_21.somatic_sv.vcf.gz" + test_genome21_indels_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test.genome_21.somatic_sv.vcf.gz.tbi" + + test_mpileup = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/mpileup/test.mpileup.gz" + test2_mpileup = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/mpileup/test2.mpileup.gz" + + test_broadpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/broadpeak/test.broadPeak" + test2_broadpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/broadpeak/test2.broadPeak" + + test_narrowpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/narrowpeak/test.narrowPeak" + test2_narrowpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/narrowpeak/test2.narrowPeak" + + test_yak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/yak/test.yak" + test2_yak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/yak/test2.yak" + + cutandrun_bedgraph_test_1 = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bedgraph/cutandtag_h3k27me3_test_1.bedGraph" + cutandrun_bedgraph_test_2 = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bedgraph/cutandtag_igg_test_1.bedGraph" + + empty_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/empty.vcf.gz" + empty_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/empty.vcf.gz.tbi" + + test_rnaseq_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test.rnaseq.vcf" + test_sv_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/sv_query.vcf.gz" + test_sv_vcf_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/sv_query.vcf.gz.tbi" + genmod_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/genmod.vcf.gz" + genmod_annotate_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test_annotate.vcf.gz" + genmod_models_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test_models.vcf.gz" + genmod_score_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test_score.vcf.gz" + + test_mito_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/NA12878_chrM.vcf.gz" + + test_pytor = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/pytor/test.pytor" + rank_model = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/genmod/svrank_model_-v1.8-.ini" + + test_flowcell = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bcl/flowcell.tar.gz" + test_flowcell_samplesheet = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bcl/flowcell_samplesheet.csv" + + varlociraptor_scenario = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/varlociraptor/scenario.yml" + + contig_ploidy_priors_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/contig_ploidy_priors_table.tsv" + + purecn_ex1_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1.bam" + purecn_ex1_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1.bam.bai" + purecn_ex1_interval = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1_intervals.txt" + purecn_ex1_normal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1_normal.txt.gz" + purecn_ex2_normal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex2_normal.txt.gz" + purecn_normalpanel_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_normalpanel.vcf.gz" + purecn_normalpanel_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_normalpanel.vcf.gz.tbi" + } + 'pacbio' { + primers = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fasta/primers.fasta" + alz = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.bam" + alzpbi = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.bam.pbi" + ccs = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.bam" + ccs_fa = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fasta/alz.ccs.fasta" + ccs_fa_gz = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fasta/alz.ccs.fasta.gz" + ccs_fq = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fastq/alz.ccs.fastq" + ccs_fq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fastq/alz.ccs.fastq.gz" + ccs_xml = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/xml/alz.ccs.consensusreadset.xml" + hifi = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fastq/test_hifi.fastq.gz" + lima = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.bam" + refine = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.bam" + cluster = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.bam" + singletons = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.bam" + aligned = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned.bam" + alignedbai = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned.bam.bai" + genemodel1 = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bed/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned_tc.bed" + genemodel2 = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bed/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned_tc.2.bed" + filelist = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/txt/filelist.txt" + } + 'scramble' { + fasta = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.fa" + fasta_fai = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.fa.fai" + bam = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.bam" + bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.bam.bai" + cram = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.cram" + cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.cram.crai" + bed = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.bed" + } + 'gene_set_analysis' { + gct = "${params.test_data_base}/data/genomics/homo_sapiens/gene_set_analysis/P53_6samples_collapsed_symbols.gct" + cls = "${params.test_data_base}/data/genomics/homo_sapiens/gene_set_analysis/P53_6samples.cls" + gmx = "${params.test_data_base}/data/genomics/homo_sapiens/gene_set_analysis/c1.symbols.reduced.gmx" + } + 'cnvkit' { + amplicon_cnr = "https://raw.githubusercontent.com/etal/cnvkit/v0.9.9/test/formats/amplicon.cnr" + amplicon_cns = "https://raw.githubusercontent.com/etal/cnvkit/v0.9.9/test/formats/amplicon.cns" + } + } + 'bacteroides_fragilis' { + 'genome' { + genome_fna_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz" + genome_gbff_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.gbff.gz" + genome_paf = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.paf" + genome_gff_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.gff.gz" + + } + 'hamronization' { + genome_abricate_tsv = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/hamronization/genome.abricate.tsv" + genome_mapping_potential_arg = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/hamronization/genome.mapping.potential.ARG" + } + 'illumina' { + test1_contigs_fa_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fasta/test1.contigs.fa.gz" + test1_1_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz" + test1_2_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz" + test2_1_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz" + test2_2_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz" + test1_paired_end_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test1.bam" + test1_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test1.sorted.bam" + test1_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test1.sorted.bam.bai" + test2_paired_end_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test2.bam" + test2_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test2.sorted.bam" + test2_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test2.sorted.bam.bai" + } + 'nanopore' { + test_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/nanopore/fastq/test.fastq.gz" + overlap_paf = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/nanopore/overlap.paf" + } + } + 'candidatus_portiera_aleyrodidarum' { + 'genome' { + genome_fasta = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.fasta" + genome_sizes = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.sizes" + genome_aln_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.aln.gz" + genome_aln_nwk = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.aln.nwk" + proteome_fasta = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/proteome.fasta" + test1_gff = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test1.gff" + test2_gff = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test2.gff" + test3_gff = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test3.gff" + } + 'illumina' { + test_1_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_1.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_2.fastq.gz" + test_se_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_se.fastq.gz" + } + 'nanopore' { + test_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/nanopore/fastq/test.fastq.gz" + } + } + 'haemophilus_influenzae' { + 'genome' { + genome_fna_gz = "${params.test_data_base}/data/genomics/prokaryotes/haemophilus_influenzae/genome/genome.fna.gz" + genome_aln_gz = "${params.test_data_base}/data/genomics/prokaryotes/haemophilus_influenzae/genome/genome.aln.gz" + genome_aln_nwk = "${params.test_data_base}/data/genomics/prokaryotes/haemophilus_influenzae/genome/genome.aln.nwk" + } + } + 'generic' { + 'csv' { + test_csv = "${params.test_data_base}/data/generic/csv/test.csv" + } + 'notebooks' { + rmarkdown = "${params.test_data_base}/data/generic/notebooks/rmarkdown/rmarkdown_notebook.Rmd" + ipython_md = "${params.test_data_base}/data/generic/notebooks/jupyter/ipython_notebook.md" + ipython_ipynb = "${params.test_data_base}/data/generic/notebooks/jupyter/ipython_notebook.ipynb" + } + 'tar' { + tar_gz = "${params.test_data_base}/data/generic/tar/hello.tar.gz" + } + 'tsv' { + test_tsv = "${params.test_data_base}/data/generic/tsv/test.tsv" + } + 'txt' { + hello = "${params.test_data_base}/data/generic/txt/hello.txt" + } + 'cooler'{ + test_pairix_pair_gz = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.blksrt.txt.gz" + test_pairix_pair_gz_px2 = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.blksrt.txt.gz.px2" + test_pairs_pair = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.sample1.pairs" + test_tabix_pair_gz = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.sorted.possrt.txt.gz" + test_tabix_pair_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.sorted.possrt.txt.gz.tbi" + hg19_chrom_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.chrom.sizes" + test_merge_cool = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/merge/toy/toy.symm.upper.2.cool" + test_merge_cool_cp2 = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/merge/toy/toy.symm.upper.2.cp2.cool" + + } + 'pairtools' { + mock_4dedup_pairsam = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.4dedup.pairsam" + mock_4flip_pairs = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.4flip.pairs" + mock_chrom_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.chrom.sizes" + mock_pairsam = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.pairsam" + mock_sam = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.sam" + frag_bed = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/frag.bed" + } + 'config' { + ncbi_user_settings = "${params.test_data_base}/data/generic/config/ncbi_user_settings.mkfg" + } + 'unsorted_data' { + 'unsorted_text' { + genome_file = "${params.test_data_base}/data/generic/unsorted_data/unsorted_text/test.genome" + intervals = "${params.test_data_base}/data/generic/unsorted_data/unsorted_text/test.bed" + numbers_csv = "${params.test_data_base}/data/generic/unsorted_data/unsorted_text/test.csv" + } + } + } + 'proteomics' { + 'msspectra' { + ups_file1 = "${params.test_data_base}/data/proteomics/msspectra/OVEMB150205_12.raw" + ups_file2 = "${params.test_data_base}/data/proteomics/msspectra/OVEMB150205_14.raw" + } + 'database' { + yeast_ups = "${params.test_data_base}/data/proteomics/database/yeast_UPS.fasta" + } + 'maxquant' { + mq_contrasts = "${params.test_data_base}/data/proteomics/maxquant/MaxQuant_contrasts.csv" + mq_proteingroups = "${params.test_data_base}/data/proteomics/maxquant/MaxQuant_proteinGroups.txt" + mq_samplesheet = "${params.test_data_base}/data/proteomics/maxquant/MaxQuant_samplesheet.tsv" + + } + 'parameter' { + maxquant = "${params.test_data_base}/data/proteomics/parameter/mqpar.xml" + } + 'idfile' { + openms_idxml = "${params.test_data_base}/data/proteomics/openms_idxml/BSA_QC_file.idXML" + } + } + 'galaxea_fascicularis' { + hic { + pretext = "${params.test_data_base}/data/genomics/eukaryotes/galaxea_fascicularis/hic/jaGalFasc40_2.pretext" + } + } + 'deilephila_porcellus' { + 'mito' { + ref_fa = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/MW539688.1.fasta" + ref_gb = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/MW539688.1.gb" + hifi_reads = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/ilDeiPorc1.HiFi.reads.fa" + contigs = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/ilDeiPorc1.contigs.fa" + } + } + 'imaging' { + 'h5' { + plant_wga = "${params.test_data_base}/data/imaging/h5/plant_wga.h5" + plant_wga_prob = "${params.test_data_base}/data/imaging/h5/plant_wga_probabilities.h5" + } + 'ilp' { + plant_wga_multicut = "${params.test_data_base}/data/imaging/ilp/plant_wga.multicut.ilp" + plant_wga_pixel_class = "${params.test_data_base}/data/imaging/ilp/plant_wga.pixel_prob.ilp" + } + 'tiff' { + mouse_heart_wga = "${params.test_data_base}/data/imaging/tiff/mindagap.mouse_heart.wga.tiff" + } + 'ome-tiff' { + cycif_tonsil_channels = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-channels.csv" + cycif_tonsil_cycle1 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + cycif_tonsil_cycle2 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle2.ome.tif" + cycif_tonsil_cycle3 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle3.ome.tif" + cycif_tonsil_dfp = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-dfp.ome.tif" + cycif_tonsil_ffp = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-ffp.ome.tif" + } + 'registration' { + markers = "${params.test_data_base}/data/imaging/registration/markers.csv" + cycle1 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + cycle2 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle2.ome.tif" + } + 'segmentation' { + markers = "${params.test_data_base}/data/imaging/segmentation/markers.csv" + image = "${params.test_data_base}/data/imaging/segmentation/cycif_tonsil_registered.ome.tif" + } + 'quantification' { + markers = "${params.test_data_base}/data/imaging/quantification/markers.csv" + image = "${params.test_data_base}/data/imaging/quantification/cycif_tonsil_registered.ome.tif" + mask = "${params.test_data_base}/data/imaging/quantification/cell.ome.tif" + } + 'downstream' { + markers = "${params.test_data_base}/data/imaging/downstream/markers.csv" + cell_feature_array = "${params.test_data_base}/data/imaging/downstream/cycif_tonsil_cell.csv" + } + 'background_subtraction' { + markers = "${params.test_data_base}/data/imaging/background_subtraction/markers.csv" + image = "${params.test_data_base}/data/imaging/background_subtraction/cycif_tonsil_registered.ome.tif" + } + 'core_detection' { + image = "${params.test_data_base}/data/imaging/core_detection/single_core_dapi.tif" + } + } + } +} diff --git a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf new file mode 100644 index 00000000..0fcc5201 --- /dev/null +++ b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf @@ -0,0 +1,41 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BWA_INDEX } from '../../../../../modules/nf-core/samtools/faidx/main.nf' +include { GATK4_CREATESEQUENCEDICTIONARY } from '../../../../../modules/nf-core/samtools/faidx/main.nf' +include { SAMTOOLS_FAIDX } from '../../../../../modules/nf-core/samtools/faidx/main.nf' +include { BWA_MARKDUP_BQSR } from '../../../../subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf' + +workflow test_bwa_markdup_bqsr { + + input = [ + [ id:'test', single_end:true ], + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) + ] + ] + + fasta = [ + [id: 'testfa'], + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + + BWA_INDEX ( fasta ) + SAMTOOLS_FAIDX ( fasta,[[],[]] ) + GATK4_CREATESEQUENCEDICTIONARY(fasta) + + BWA_MARKDUP_BQSR( + input, + fasta, + SAMTOOLS_FAIDX.out.fai, + BWA_INDEX.out.index, + GATK4_CREATESEQUENCEDICTIONARY.out.dict, + [], + [], + false + + + ) +} diff --git a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config new file mode 100644 index 00000000..8730f1c4 --- /dev/null +++ b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} diff --git a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml new file mode 100644 index 00000000..72230942 --- /dev/null +++ b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml @@ -0,0 +1,11 @@ +## TODO nf-core: Please run the following command to build this file: +# nf-core subworkflows create-test-yml +- name: "test_bwa_markdup_bqsr" + command: nextflow run ./tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf -entry test_bwa_markdup_bqsr -c ./tests/config/nextflow.config + tags: + - "subworkflows/bwa_markdup_bqsr" + files: + - path: "output//test.bam" + md5sum: e667c7caad0bc4b7ac383fd023c654fc + - path: output//versions.yml + md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py new file mode 100644 index 00000000..e81b8f3c --- /dev/null +++ b/tests/test_versions_yml.py @@ -0,0 +1,79 @@ +from pathlib import Path +import pytest +import yaml +import re +from textwrap import dedent + + +def _get_workflow_names(): + """Get all names of all workflows which have a test.yml in the tests directory. + + To do so, recursively finds all test.yml files and parses their content. + """ + here = Path(__file__).parent.resolve() + pytest_workflow_files = here.glob("modules/**/test.yml") + yield "dummy" + for f in pytest_workflow_files: + # test_config = yaml.safe_load(f.read_text()) + test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader) + for workflow in test_config: + # https://github.com/nf-core/modules/pull/1242 - added to cover tests + # that expect an error and therefore will not generate a versions.yml + if 'exit_code' not in workflow: + yield workflow["name"] + + +@pytest.mark.workflow(*_get_workflow_names()) +def test_ensure_valid_version_yml(workflow_dir): + workflow_dir = Path(workflow_dir) + software_name = workflow_dir.name.split("_")[0].lower() + try: + versions_yml_file = workflow_dir / f"output/{software_name}/versions.yml" + versions_yml = versions_yml_file.read_text() + except FileNotFoundError: + raise AssertionError( + dedent( + f"""\ + `versions.yml` not found in the output directory. + Expected path: `{versions_yml_file}` + + This can have multiple reasons: + * The test-workflow failed before a `versions.yml` could be generated. + * The workflow name in `test.yml` does not start with the tool name. + """ + ) + ) + + assert ( + "END_VERSIONS" not in versions_yml + ), "END_VERSIONS detected in versions.yml. This is a sign of an ill-formatted HEREDOC" + + # Raises an exception if yaml is not valid + versions = yaml.safe_load(versions_yml) + assert ( + len(versions) == 1 + ), "The top-level of versions.yml must contain exactly one entry: the process name as dict key" + software_versions = next(iter(versions.values())) + assert len(software_versions), "There must be at least one version emitted." + for tool, version in software_versions.items(): + assert re.match( + r"^\d.*|^[a-f0-9]{40}$", str(version) + ), f"Version number for {tool} must start with a number, or be a Git SHA commit id. " + +@pytest.fixture(scope="session", autouse=True) +def link_nfcore_components(request): + print(request) + workflow_dir = Path(workflow_dir) + addfile = workflow_dir / f"testfile.txt" + Path(addfile).touch() + yield + pass + #Path(addfile).unlink() + +#@pytest.mark.workflow(*_get_workflow_names()) +def unlink_nfcore_components(workflow_dir): + workflow_dir = Path(workflow_dir) + addfile = workflow_dir / f"testfile.txt" + assert Path(addfile).is_file() + Path(addfile).unlink() + assert not Path(addfile).is_file() From 8c2ca87dbc3872d7374c109afc1051c31b683d36 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 21 Sep 2023 20:10:26 -0400 Subject: [PATCH 02/16] add gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ebafb86d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/modules/nf-core/ +/subworkflows/nf-core/ From 45db09da237c86779d71b221ae6cf8e6279b89f6 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 21 Sep 2023 20:34:48 -0400 Subject: [PATCH 03/16] fix bugs and create-test-yml --- .../nf-core-test/bwa_markdup_bqsr/main.nf | 6 +-- .../nf-core-test/bwa_markdup_bqsr/main.nf | 21 ++++----- .../nf-core-test/bwa_markdup_bqsr/test.yml | 43 +++++++++++++++---- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf b/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf index 4c6e7d68..746bfe39 100755 --- a/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf +++ b/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf @@ -72,7 +72,7 @@ workflow BWA_MARKDUP_BQSR { fai ) GATK4_BASERECALIBRATOR( - GATK4_MARKDUPLICATES.out.output, + GATK4_MARKDUPLICATES.out.bam, fasta, fai, dict, @@ -80,8 +80,8 @@ workflow BWA_MARKDUP_BQSR { known_sites_tbi ) GATK4_APPLYBQSR( - GATK4_MARKDUPLICATES.out.output - .join(GATK4_MARKDUPLICATES.out.bam_index) + GATK4_MARKDUPLICATES.out.bam + .join(GATK4_MARKDUPLICATES.out.bai) .join(GATK4_BASERECALIBRATOR.out.table) .map{meta, bam, bai, table -> [ meta, bam, bai, table, [] ] // what is intervals? diff --git a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf index 0fcc5201..8685aff0 100644 --- a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf +++ b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf @@ -2,9 +2,9 @@ nextflow.enable.dsl = 2 -include { BWA_INDEX } from '../../../../../modules/nf-core/samtools/faidx/main.nf' -include { GATK4_CREATESEQUENCEDICTIONARY } from '../../../../../modules/nf-core/samtools/faidx/main.nf' -include { SAMTOOLS_FAIDX } from '../../../../../modules/nf-core/samtools/faidx/main.nf' +include { BWA_INDEX } from '../../../../modules/nf-core/bwa/index/main.nf' +include { GATK4_CREATESEQUENCEDICTIONARY } from '../../../../modules/nf-core/gatk4/createsequencedictionary/main.nf' +include { SAMTOOLS_FAIDX } from '../../../../modules/nf-core/samtools/faidx/main.nf' include { BWA_MARKDUP_BQSR } from '../../../../subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf' workflow test_bwa_markdup_bqsr { @@ -17,21 +17,18 @@ workflow test_bwa_markdup_bqsr { ] ] - fasta = [ - [id: 'testfa'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] + fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - BWA_INDEX ( fasta ) - SAMTOOLS_FAIDX ( fasta,[[],[]] ) - GATK4_CREATESEQUENCEDICTIONARY(fasta) + BWA_INDEX ( [[id: 'testfa'],fasta] ) + SAMTOOLS_FAIDX ( [[id: 'testfa'],fasta],[[],[]] ) + GATK4_CREATESEQUENCEDICTIONARY([[id: 'testfa'],fasta]) BWA_MARKDUP_BQSR( input, fasta, - SAMTOOLS_FAIDX.out.fai, + SAMTOOLS_FAIDX.out.fai.map{ it[1] }.first(), BWA_INDEX.out.index, - GATK4_CREATESEQUENCEDICTIONARY.out.dict, + GATK4_CREATESEQUENCEDICTIONARY.out.dict.map{ it[1] }.first(), [], [], false diff --git a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml index 72230942..4dbc601f 100644 --- a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml +++ b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml @@ -1,11 +1,36 @@ -## TODO nf-core: Please run the following command to build this file: -# nf-core subworkflows create-test-yml -- name: "test_bwa_markdup_bqsr" - command: nextflow run ./tests/subworkflows/nf-core-test/bwa_markdup_bqsr/main.nf -entry test_bwa_markdup_bqsr -c ./tests/config/nextflow.config +- name: bwa_markdup_bqsr test_bwa_markdup_bqsr + command: nextflow run ./tests/subworkflows/nf-core-test/bwa_markdup_bqsr -entry test_bwa_markdup_bqsr -c ./tests/config/nextflow.config tags: - - "subworkflows/bwa_markdup_bqsr" + - bwa + - bwa/mem + - gatk4 + - gatk4/applybqsr + - gatk4/applybqsr/spark + - gatk4/baserecalibrator + - gatk4/baserecalibrator/spark + - gatk4/markduplicates + - gatk4/markduplicates/spark + - samtools + - samtools/index + - subworkflows + - subworkflows/bwa_markdup_bqsr files: - - path: "output//test.bam" - md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: output//versions.yml - md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + - path: output/bwa/bwa/genome.amb + md5sum: 3a68b8b2287e07dd3f5f95f4344ba76e + - path: output/bwa/bwa/genome.ann + md5sum: c32e11f6c859f166c7525a9c1d583567 + - path: output/bwa/bwa/genome.bwt + md5sum: 0469c30a1e239dd08f68afe66fde99da + - path: output/bwa/bwa/genome.pac + md5sum: 983e3d2cd6f36e2546e6d25a0da78d66 + - path: output/bwa/bwa/genome.sa + md5sum: ab3952cabf026b48cd3eb5bccbb636d1 + - path: output/bwa/test.bam + md5sum: cf74076b67bb4c9b9c7b2d88964bf5d9 + - path: output/gatk4/genome.dict + md5sum: 7362679f176e0f52add03c08f457f646 + - path: output/gatk4/test.bam.metrics + contains: + - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + - path: output/samtools/genome.fasta.fai + md5sum: 9da2a56e2853dc8c0b86a9e7229c9fe5 From bfb487c2a945f057a64510fb7e9d1d0aac16e223 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 22 Sep 2023 14:18:40 -0400 Subject: [PATCH 04/16] fixed pytests setup and teardown --- tests/conftest.py | 38 ++++++++++++++++++++++++++++++++++++++ tests/test_versions_yml.py | 18 ------------------ 2 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..9cb7f8d0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,38 @@ +from pathlib import Path +import pytest +import git + +repository_url = "https://github.com/nf-core/modules.git" +remote_name = "nf-core-repo" +subfolder_path = "modules/nf-core" + +def pytest_sessionstart(session): + # Open the local Git repository + local_repository_path = Path(__file__).parent.parent.resolve() + repo = git.Repo(local_repository_path) + + # Add the remote origin + has_remote = False + for remote in repo.remotes: + if remote.name == "nf-core-repo": + has_remote = True + if not has_remote: + repository_url = "https://github.com/nf-core/modules.git" + remote = repo.create_remote(remote_name, repository_url, track="master") + remote.fetch() + master_branch = repo.create_head("master", remote.refs.master) + + # Print the remote URLs to verify + for remote in repo.remotes: + print(f"Remote '{remote.name}': {remote.url}") + + # Add subfolder + repo.git.read_tree("--prefix=" + subfolder_path, "-u", f"{remote_name}/master:{subfolder_path}") + + +def pytest_sessionfinish(session, exitstatus): + local_repository_path = Path(__file__).parent.parent.resolve() + repo = git.Repo(local_repository_path) + repo.git.rm(subfolder_path, r=True,f=True,q=True) + repo.delete_remote(remote_name) + diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py index e81b8f3c..dda94a14 100644 --- a/tests/test_versions_yml.py +++ b/tests/test_versions_yml.py @@ -4,7 +4,6 @@ import re from textwrap import dedent - def _get_workflow_names(): """Get all names of all workflows which have a test.yml in the tests directory. @@ -60,20 +59,3 @@ def test_ensure_valid_version_yml(workflow_dir): r"^\d.*|^[a-f0-9]{40}$", str(version) ), f"Version number for {tool} must start with a number, or be a Git SHA commit id. " -@pytest.fixture(scope="session", autouse=True) -def link_nfcore_components(request): - print(request) - workflow_dir = Path(workflow_dir) - addfile = workflow_dir / f"testfile.txt" - Path(addfile).touch() - yield - pass - #Path(addfile).unlink() - -#@pytest.mark.workflow(*_get_workflow_names()) -def unlink_nfcore_components(workflow_dir): - workflow_dir = Path(workflow_dir) - addfile = workflow_dir / f"testfile.txt" - assert Path(addfile).is_file() - Path(addfile).unlink() - assert not Path(addfile).is_file() From bbd7e6a9ce0c3c74a37aab7b2d87a5237d6088b5 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 22 Sep 2023 14:19:10 -0400 Subject: [PATCH 05/16] fixed test.yml --- tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml index 4dbc601f..c18b2e2b 100644 --- a/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml +++ b/tests/subworkflows/nf-core-test/bwa_markdup_bqsr/test.yml @@ -31,6 +31,6 @@ md5sum: 7362679f176e0f52add03c08f457f646 - path: output/gatk4/test.bam.metrics contains: - - ' # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ' + - 'picard.sam.DuplicationMetrics' - path: output/samtools/genome.fasta.fai md5sum: 9da2a56e2853dc8c0b86a9e7229c9fe5 From 603cf3ef66fcd2c15d8a298d16311de0d95dfde8 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 22 Sep 2023 14:32:56 -0400 Subject: [PATCH 06/16] moved module folder --- modules/{mskcc => nf-core-test}/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename modules/{mskcc => nf-core-test}/.gitkeep (100%) diff --git a/modules/mskcc/.gitkeep b/modules/nf-core-test/.gitkeep similarity index 100% rename from modules/mskcc/.gitkeep rename to modules/nf-core-test/.gitkeep From 513b54967acf0abe4ebb2e46319da04b646b8ee5 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 22 Sep 2023 16:40:57 -0400 Subject: [PATCH 07/16] Adding github CI --- .github/workflows/test.yml | 232 +++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100755 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100755 index 00000000..1e352f72 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,232 @@ +name: test +on: + pull_request: + branches: [main] + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + +jobs: + prettier: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Install NodeJS + uses: actions/setup-node@v3 + + - name: Install Prettier + run: npm install -g prettier + + - name: Run Prettier --check + run: prettier --check . + + editorconfig: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + + - name: Install editorconfig-checker + run: npm install -g editorconfig-checker + + - name: Run ECLint check + run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) + + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + + nf-core-lint: + runs-on: ubuntu-20.04 + name: nf-core-lint + needs: [pytest-changes] + if: needs.pytest-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + exclude: + - tags: "nf-test" + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install pip + run: python -m pip install --upgrade pip + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v1 + + - name: Lint module ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + if: startsWith( matrix.tags, 'module' ) + + - name: Lint subworkflow ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} + if: startsWith( matrix.tags, 'subworkflow' ) + + pytest: + runs-on: ubuntu-20.04 + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + profile: ["docker", "singularity", "conda"] + env: + NXF_ANSI_LOG: false + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - name: Setup Nextflow ${{ matrix.NXF_VER }} + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.7.1 + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@v6 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity + !${{ github.workspace }}/.singularity + + confirm-pass: + runs-on: ubuntu-latest + needs: [prettier, editorconfig, pytest-changes, nf-core-lint, pytest] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" \ No newline at end of file From 78538439f82ebcaddcdce17d2f0f8ec28d8fcd01 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 28 Sep 2023 19:39:54 -0400 Subject: [PATCH 08/16] Empty-Commit From fabfe09b94817a27e5ac79eb02072d661fe01a85 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 28 Sep 2023 20:02:32 -0400 Subject: [PATCH 09/16] fix linting with prettier --- .github/workflows/test.yml | 4 +- modules/yaml-schema.json | 298 +++++++++--------- .../conftest.cpython-310-pytest-7.2.0.pyc | Bin 0 -> 1351 bytes ..._versions_yml.cpython-310-pytest-7.2.0.pyc | Bin 0 -> 3900 bytes tests/config/pytest_modules.yml | 1 - .../nf-core-test/bwa_markdup_bqsr/test.yml | 2 +- 6 files changed, 159 insertions(+), 146 deletions(-) create mode 100644 tests/__pycache__/conftest.cpython-310-pytest-7.2.0.pyc create mode 100644 tests/__pycache__/test_versions_yml.cpython-310-pytest-7.2.0.pyc diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e352f72..9fe20abb 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: pull_request: branches: [main] - + # Cancel if a newer run is started concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -229,4 +229,4 @@ jobs: if: always() run: | echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" \ No newline at end of file + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/modules/yaml-schema.json b/modules/yaml-schema.json index 24516d54..fcb05aef 100644 --- a/modules/yaml-schema.json +++ b/modules/yaml-schema.json @@ -1,146 +1,160 @@ { - "$schema": "http://json-schema.org/draft-07/schema", - "title": "Meta yaml", - "description": "Validate the meta yaml file for an nf-core module", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Name of the module" - }, - "description": { - "type": "string", - "description": "Description of the module" - }, - "keywords": { - "type": "array", - "description": "Keywords for the module", - "items": { - "type": "string" - }, - "minItems": 3 - }, - "authors": { - "type": "array", - "description": "Authors of the module", - "items": { - "type": "string" - } - }, - "input": { - "type": "array", - "description": "Input channels for the module", + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Meta yaml", + "description": "Validate the meta yaml file for an nf-core module", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the module" + }, + "description": { + "type": "string", + "description": "Description of the module" + }, + "keywords": { + "type": "array", + "description": "Keywords for the module", + "items": { + "type": "string" + }, + "minItems": 3 + }, + "authors": { + "type": "array", + "description": "Authors of the module", + "items": { + "type": "string" + } + }, + "input": { + "type": "array", + "description": "Input channels for the module", - "items": { - "type": "object", - "patternProperties": { - ".*": { - "type": "object", - "properties": { - "type": { - "type": "string", - "description": "Type of the input channel", - "enum": ["map", "file", "directory", "string", "integer", "float"] - }, - "description": { - "type": "string", - "description": "Description of the input channel" - }, - "pattern": { - "type": "string", - "description": "Pattern of the input channel, given in Java glob syntax" - }, - "default": { - "type": ["string", "number", "boolean", "array", "object"], - "description": "Default value for the input channel" - } - }, - "required": ["type", "description"] - } - } - } - }, - "output": { - "type": "array", - "description": "Output channels for the module", - "items": { - "type": "object", - "patternProperties": { - ".*": { - "type": "object", - "properties": { - "type": { - "type": "string", - "description": "Type of the output channel", - "enum": ["map", "file", "directory", "string", "integer", "float"] - }, - "description": { - "type": "string", - "description": "Description of the output channel" - }, - "pattern": { - "type": "string", - "description": "Pattern of the input channel, given in Java glob syntax" - } - }, - "required": ["type", "description"] - } - } - } - }, - "tools": { - "type": "array", - "description": "Tools used by the module", - "items": { - "type": "object", - "patternProperties": { - ".*": { - "type": "object", - "properties": { - "description": { - "type": "string", - "description": "Description of the output channel" - }, - "homepage": { - "type": "string", - "description": "Homepage of the tool", - "pattern": "^(http|https)://.*$" - }, - "documentation": { - "type": "string", - "description": "Documentation of the tool", - "pattern": "^(http|https)://.*$" - }, - "tool_dev_url": { - "type": "string", - "description": "URL of the development version of the tool's documentation", - "pattern": "^(http|https)://.*$" - }, - "doi": { - "type": "string", - "description": "DOI of the tool", - "pattern": "^10.\\d{4,9}\\/[^,]+$" - }, - "licence": { - "type": ["array", "string"], - "description": "Licence of the tool", - "items": { - "type": "string" - }, - "uniqueItems": true - } - }, - "required": ["description"], - "anyOf": [ - { "required": ["homepage"] }, - { "required": ["documentation"] }, - { "required": ["tool_dev_url"] }, - { "required": ["doi"] } - ] - } - } - } + "items": { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of the input channel", + "enum": [ + "map", + "file", + "directory", + "string", + "integer", + "float" + ] + }, + "description": { + "type": "string", + "description": "Description of the input channel" + }, + "pattern": { + "type": "string", + "description": "Pattern of the input channel, given in Java glob syntax" + }, + "default": { + "type": ["string", "number", "boolean", "array", "object"], + "description": "Default value for the input channel" + } + }, + "required": ["type", "description"] + } } + } }, - "required": ["name", "description", "keywords", "authors", "output", "tools"] -} \ No newline at end of file + "output": { + "type": "array", + "description": "Output channels for the module", + "items": { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of the output channel", + "enum": [ + "map", + "file", + "directory", + "string", + "integer", + "float" + ] + }, + "description": { + "type": "string", + "description": "Description of the output channel" + }, + "pattern": { + "type": "string", + "description": "Pattern of the input channel, given in Java glob syntax" + } + }, + "required": ["type", "description"] + } + } + } + }, + "tools": { + "type": "array", + "description": "Tools used by the module", + "items": { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "Description of the output channel" + }, + "homepage": { + "type": "string", + "description": "Homepage of the tool", + "pattern": "^(http|https)://.*$" + }, + "documentation": { + "type": "string", + "description": "Documentation of the tool", + "pattern": "^(http|https)://.*$" + }, + "tool_dev_url": { + "type": "string", + "description": "URL of the development version of the tool's documentation", + "pattern": "^(http|https)://.*$" + }, + "doi": { + "type": "string", + "description": "DOI of the tool", + "pattern": "^10.\\d{4,9}\\/[^,]+$" + }, + "licence": { + "type": ["array", "string"], + "description": "Licence of the tool", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": ["description"], + "anyOf": [ + { "required": ["homepage"] }, + { "required": ["documentation"] }, + { "required": ["tool_dev_url"] }, + { "required": ["doi"] } + ] + } + } + } + } + }, + "required": ["name", "description", "keywords", "authors", "output", "tools"] +} diff --git a/tests/__pycache__/conftest.cpython-310-pytest-7.2.0.pyc b/tests/__pycache__/conftest.cpython-310-pytest-7.2.0.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e790f20971f7252af5c34bb251651baf79e47f9 GIT binary patch literal 1351 zcmZ8hON$#v5bnolUMp!IUgxzC93SlHpq-;(ae^Ts$0ax>fq^iLrd!r%=e5-nkc!5}2KPW~Loe;h*ovWNSN;o${t z^&=8N1T9ERTh2<(2qmQx_o&sd!Y#enYrkpX$9`)S#DU;(D4cjt^y0p7h4(9o2f`Nt z@}ZzVkWu(I`X=WiYMk#=J-c}_)4EY-QS>_3v&G~ztIMdGj-TZ6`Q-?lAnXM zvvTI1U>6Kd1Yh&ZqZLUhK&raT znTpuK3>5Tiyj}x22H^`R;sCvfM1z!Os1}oHT?h$DgM%M=##2(Myspf{qR!F+V>PwP zbq&|8g+ZT~SU5}7E{)jTr8B%|YQqe=l9LIfRW|#bz%eFr^HVfmRCRP!!$p*3DynLz zt67>xx~>ZqVX->7h1w)7lh?A6kZQ~*!CdXNX;ELbb}Gv1YHDwsHrM7sbKPFv<*#%K zI>3nPb0h~Wqz(@$Qp(;sNDruwJLI3y#~eOFlg!TLcG*e0xLdS7$B+M^m9=PPh*rMh zb7$^qPtf+q-aCjvzCmnm5nC;G`1z|5Hxvw=8hSYjTL^&l5B4tsz9lwSuunnEpoD$< zF%Ytlck_lm%0>FW%@6g>*VhAY0|klO_h>hCvgh$3UVz?a3`|x?PMWdZU>Qq@hqr{ ze9~dw?G#<54=`w3QCn&|fi{tQHMG^Vh4|0Ar{!z=qA1&dC&b4AI?RsJXZ=t91vY4F AhyVZp literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_versions_yml.cpython-310-pytest-7.2.0.pyc b/tests/__pycache__/test_versions_yml.cpython-310-pytest-7.2.0.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ccdd590f7085d428485cce1eff68001f3e826ca9 GIT binary patch literal 3900 zcmaJ^&2!tv6$e0&1R;{5DA|_e##uW~CUh)H@>gR|ok8nuCRVd)BcW~=_?1F_u&>kJg7!>Uu)^=ZS-=6rfWU3mDh2^@C&_StEk$# zZ?!DNRce*g+iuzL&iOOFa;pqHCe1(9S`}vPRCi`s<%#w?jTUI}srE!~)u?_~YgpgF zjM`>HPt2dYvJ3A5Wt0W7jr8owLj!LliU;ucwI(&GKhbxLgWOn)wMaWKsXo?Yof`K6 zZ=~PSK3k3rncpcy#zAqcLCd1K-x_f)%E8!7Y*I5acXj?w1Spp@lzL1a*J%Mz6-R%* z%_MPsp9F4?2@-ZxXD{TBI)1n($X?fLcS+aXWyB?t3E3R>eBuQ}cA4r6LOssfGUUUi zZ4YgvFI-s7B$GTH^m;>d;?F{tGM<#!z9*e_NLgZ{C7X>BH^6_1tJ+)m7)+w&9C2R%uNGnYD&?aRdeu`5`rypd1LF5|3qqCZ3gPUan@StNR= zHKXVNyyHDgtZe+R8lMLY-)^n$41#bK$*b+QSPeoR23^-(m0{=$Wt9_jR@xzFtJ`kR zdB_6BUFn5^BL>@(=jdgxcjY2DcaXRzNIH|=2 zpjBxZBr2ygT8#>x4%8wbunuOU;#kd1HB*btI4^4lbEqk{WIeLL&ODusOHn>5Ju+fD zvc*>*Rid@X2268EM@+PSgyjz~UXgXc@<(Xd*a9qdI)B9R7vzX6;P@MIU<)|rRXab{ z)w={}7oM4ba|ZfVdII_<0sX0HMxifH(1TUzIfXtC7T9q~&dY_!Iyf?QS zDF3A{7L~R|(6$8HUZVaN&{Nd^#`%v>FK4JhHitY>WvqXxa|^k!4ak>O8<5k}&vMG& zsb>orHki9|aB{3c>4`oy;%YoAPwgzmHTr{iPA)~WfUX|ZqPeL4#E9pkd6?_>ptDL} zi>ir32AD2hJnASxolc{e(HWR@(Ho-mMQ24t+HPqp= zAh9w~4HK2F@g-2om*IikYonHmV%T5t8SKc2G80gHdT$`4+IwBFkL|l{3A=U}Fanz< zAKp-o@AI(D1b7f*<>J=gmgEr|9y9CR)U0<$@4%Fh9!fPNgDZ&-NhX!r$>fCfJoHd= z)O^J?5SjmPe=;Y+j@*My-O21|j(A#Dl78xg_YbngBRC6ce7YU3bd~(~9kygWQpQ|NJHG=pBS7aIU#vcoS5DyD$6m@ zm+*0gpGHTV#m6~#BqpTIT4PxqiZ~t(lDz2qo=oy^%wwEaku>)aEb7freuPD5gY%Fl z)els*!eK-8IUEYK#D0Hr5^{Mm=d|JY=y2BO;1xV$B_&6=9f*6MCuQJtwg(=Rm_Q^2 zKLloxEIOi#1wu(VcnouW2XhLvrAx+17fDY%gtN_r#P<8X8>n*;FC&Z+PUGOoji153 z8Q{sy^duYz3%jmAU?RywuP69<9M3C<^EaX0IFXdI%HzNRwpBSwCdAR2%Tfw!f;#7M zJW91@(4G|A+H~hq+DVaKNfcY!xyE0|xu*oLP7A)m`@?C$>--HAH1Uwg-#VsaeOlq= z>CQw)it&0HfE3C4x;l=gj|EitnmV?!Kn!pZxxPoUrG#eVDiQ=LlVvREhF;Ul@H5h% zQ8+YSm@kYM^{Qc14-LbZ+~%Q)W2$eew`r6o{rcoxKD6`8dMfvQ{Dtq)k?MLmk9|yM z?XZ@63K#*OL0}BWEzo04_)<)!aa<41f1C&BKTbO?m)W#6pc4grilBt?`@|aFNIWr%JwvcE&Mpy<8EIqM0J8s t%%001@j8erAX7ocuMe&wZT%?yjCv0@8c~DCTN-=|z^|-ZM!8 Date: Thu, 28 Sep 2023 20:03:29 -0400 Subject: [PATCH 10/16] add CI execution for PRs merging to develop --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9fe20abb..0db8ee67 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: test on: pull_request: - branches: [main] + branches: [main, develop] # Cancel if a newer run is started concurrency: From b83e8d28b1d7518f55899f5ebed9f834cd6327eb Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 28 Sep 2023 20:27:39 -0400 Subject: [PATCH 11/16] fix nf-core subworkflows lint CI job --- .github/workflows/test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0db8ee67..26300189 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,11 +97,15 @@ jobs: - name: Lint module ${{ matrix.tags }} run: nf-core modules lint ${{ matrix.tags }} - if: startsWith( matrix.tags, 'module' ) + if: ${{ !startsWith(matrix.tags, 'subworkflows/') }} + + - name: Remove substring from matrix.tags + id: remove_substring + run: echo subworkflow_names=$(echo "${{ matrix.tags }}" | sed 's/subworkflows\///g') >> $GITHUB_OUTPUT - name: Lint subworkflow ${{ matrix.tags }} - run: nf-core subworkflows lint ${{ matrix.tags }} - if: startsWith( matrix.tags, 'subworkflow' ) + run: nf-core subworkflows lint ${{ steps.remove_substring.outputs.subworkflow_names }} + if: startsWith( matrix.tags, 'subworkflows/' ) pytest: runs-on: ubuntu-20.04 From 97fc71e484d4ff1adab36dcdad77ac1fa7674e0b Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 9 Oct 2023 10:43:19 -0400 Subject: [PATCH 12/16] Empty-Commit From bb5c739deaa507be301db96bd2a9213d5cfdff16 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 9 Oct 2023 11:34:26 -0400 Subject: [PATCH 13/16] Added meta file --- .../nf-core-test/bwa_markdup_bqsr/meta.yml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml diff --git a/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml b/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml new file mode 100755 index 00000000..67f28c24 --- /dev/null +++ b/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml @@ -0,0 +1,76 @@ +name: bam_markdup_bqsr +description: A subworkflow for generating a BAM file from FASTQ +keywords: + - bam + - alignment + - markduplicates + - bqsr + - duplicates + +components: + - bwa/mem + - gatk4/markduplicates + - gatk4/applybqsr + - gatk4/baserecalibrator + - samtools/index + - gatk4/markduplicatesspark + - gatk4/applybqsrspark + - gatk4/baserecalibratorspark + +input: + - reads: + description: | + An input channel containing fastq.gz files + Structure: [ val(meta), path(reads)] + pattern: "*.{fastq.gz}" + - fasta: + description: | + A channel containing the reference FASTA file + Structure: [ path(fasta) ] + pattern: "*.{fasta,fa}" + - fai: + description: | + A channel containing the index of the reference FASTA file + Structure: [ path(fai) ] + pattern: "*.{fai}" + - bwa_index: + description: | + A channel containing bwa index reference, which can be created using bwa/index + Structure: [ val(meta2), path(bwa_index) ] + pattern: "*.{amb,ann,bwt,pac,sa}" + - dict: + description: | + A channel containing a sequence dictionary file (`dict`), which can be created using gatk4/createsequencedictionary + Structure: [ path(blacklist) ] + pattern: "*.{dict}" + - known_sites: + description: | + A channel containing one or more files containing known polymorphic sites that should be excluded during base recalibration. + Structure: [ path([known_sites_1, known_sites_2]) ] + pattern: "*.vcf.gz" + - known_sites_tbi: + description: | + A channel containing tabix index files of the known_sites files + Structure: [ path([known_sites_1_tbi, known_sites_2_tbi]) ] + pattern: "*.vcf.gz.tbi" + - spark: + type: boolean + description: | + If true use spark gatk4 modules: GATK4_MARKDUPLICATES_SPARK, GATK4_APPLYBQSR_SPARK, GATK4_BASERECALIBRATOR_SPARK + pattern: "true|false" + +output: + - bam: + description: | + BAM file with marked duplicates and BQSR + Structure: [ val(meta), path(bam) ] + - bai: + description: | + BAM index file + Structure: [ val(meta), path(bai) ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@anoronh4" From 629fa1f925d3a2b8c60f955a87701cb7fb9456f8 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 9 Oct 2023 11:42:03 -0400 Subject: [PATCH 14/16] Added installation of GitPython package in CI workflows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26300189..243b0809 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -137,7 +137,7 @@ jobs: ${{ runner.os }}-pip- - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography + run: python -m pip install --upgrade pip pytest-workflow cryptography GitPython - name: Setup Nextflow ${{ matrix.NXF_VER }} uses: nf-core/setup-nextflow@v1 From 55f110ed67902efa2099458b823090cf240c3fe8 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 12 Oct 2023 15:27:20 -0400 Subject: [PATCH 15/16] fix nf-core subworkflow lint issues --- subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml b/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml index 67f28c24..308c4646 100755 --- a/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml +++ b/subworkflows/nf-core-test/bwa_markdup_bqsr/meta.yml @@ -1,4 +1,4 @@ -name: bam_markdup_bqsr +name: bwa_markdup_bqsr description: A subworkflow for generating a BAM file from FASTQ keywords: - bam @@ -13,9 +13,9 @@ components: - gatk4/applybqsr - gatk4/baserecalibrator - samtools/index - - gatk4/markduplicatesspark - - gatk4/applybqsrspark - - gatk4/baserecalibratorspark + - gatk4/markduplicates/spark + - gatk4/applybqsr/spark + - gatk4/baserecalibrator/spark input: - reads: From 477f07d4d00d3d97356c9ef94316ce6663d7f809 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 12 Oct 2023 16:24:34 -0400 Subject: [PATCH 16/16] cleanup gitpython commands --- tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9cb7f8d0..47d4e353 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,13 +14,15 @@ def pytest_sessionstart(session): # Add the remote origin has_remote = False for remote in repo.remotes: - if remote.name == "nf-core-repo": + if remote.name == remote_name: has_remote = True if not has_remote: repository_url = "https://github.com/nf-core/modules.git" remote = repo.create_remote(remote_name, repository_url, track="master") remote.fetch() - master_branch = repo.create_head("master", remote.refs.master) + latest_commit = remote.refs['master'].commit + + master_branch = repo.create_head("master", commit=latest_commit) # Print the remote URLs to verify for remote in repo.remotes: @@ -30,7 +32,7 @@ def pytest_sessionstart(session): repo.git.read_tree("--prefix=" + subfolder_path, "-u", f"{remote_name}/master:{subfolder_path}") -def pytest_sessionfinish(session, exitstatus): +def pytest_sessionfinish(session, exitstatus): local_repository_path = Path(__file__).parent.parent.resolve() repo = git.Repo(local_repository_path) repo.git.rm(subfolder_path, r=True,f=True,q=True)