Skip to content

Commit c050526

Browse files
martensonmartenson
authored
Merge pull request #194 from martenson/unify-tpv
reject docker and unify tpv configs
2 parents 88ce32f + e666168 commit c050526

File tree

16 files changed

+121
-206
lines changed

16 files changed

+121
-206
lines changed

‎files/galaxy-qa1.galaxy.cloud.e-infra.cz/tpv_rules_local.yml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎files/galaxy-umsa.grid.cesnet.cz/config/tool_conf.xml‎

Lines changed: 8 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -3,41 +3,22 @@
33
<section id="upload_file" name="Upload Data">
44
<tool file="data_source/upload.xml" />
55
</section>
6-
<!-- <section id="data_source_tools" name="External Data">
7-
<tool file="data_source/ucsc_tablebrowser.xml" />
8-
<tool file="data_source/ucsc_tablebrowser_archaea.xml" />
9-
<tool file="data_source/ncbi_datasets.xml" />
10-
<tool file="data_source/ebi_sra.xml" />
11-
<tool file="data_source/sra.xml" />
12-
<tool file="data_source/microbial_import.xml" />
13-
<tool file="data_source/biomart.xml" />
14-
<tool file="data_source/cbi_rice_mart.xml" />
15-
<tool file="data_source/gramene_mart.xml" />
16-
<tool file="data_source/fly_modencode.xml" />
17-
<tool file="data_source/intermine.xml" />
18-
<tool file="data_source/flymine.xml" />
19-
<tool file="data_source/modmine.xml" />
20-
<tool file="data_source/mousemine.xml" />
21-
<tool file="data_source/ratmine.xml" />
22-
<tool file="data_source/yeastmine.xml" />
23-
<tool file="data_source/metabolicmine.xml" />
24-
<tool file="data_source/worm_modencode.xml" />
25-
<tool file="data_source/wormbase.xml" />
26-
<tool file="data_source/zebrafishmine.xml" />
27-
<tool file="data_source/eupathdb.xml" />
28-
<tool file="data_source/hbvar.xml" />
29-
</section> -->
306
<section id="send_data" name="Send Data">
31-
<tool file="data_export/send.xml" />
32-
<!-- <tool file="data_export/export_remote.xml" /> -->
7+
<tool file="data_export/export_remote.xml" />
338
</section>
349
<section id="collection_operations" name="Collection Operations">
3510
<tool file="${model_tools_path}/unzip_collection.xml" />
3611
<tool file="${model_tools_path}/zip_collection.xml" />
3712
<tool file="${model_tools_path}/filter_failed_collection.xml" />
13+
<tool file="${model_tools_path}/filter_failed_collection_1.1.0.xml" />
3814
<tool file="${model_tools_path}/filter_empty_collection.xml" />
15+
<tool file="${model_tools_path}/filter_empty_collection_1.1.0.xml" />
3916
<tool file="${model_tools_path}/filter_null.xml" />
17+
<tool file="${model_tools_path}/filter_null_1.1.0.xml" />
18+
<tool file="${model_tools_path}/keep_success_collection.xml" />
19+
<tool file="${model_tools_path}/keep_success_collection_1.1.0.xml" />
4020
<tool file="${model_tools_path}/flatten_collection.xml" />
21+
<tool file="${model_tools_path}/nest_collection.xml" />
4122
<tool file="${model_tools_path}/merge_collection.xml" />
4223
<tool file="${model_tools_path}/relabel_from_file.xml" />
4324
<tool file="${model_tools_path}/filter_from_file.xml" />
@@ -49,6 +30,7 @@
4930
<tool file="${model_tools_path}/apply_rules.xml" />
5031
<tool file="${model_tools_path}/build_list.xml" />
5132
<tool file="${model_tools_path}/build_list_1.2.0.xml" />
33+
<tool file="${model_tools_path}/sample_sheet_to_tabular.xml" />
5234
<tool file="${model_tools_path}/extract_dataset.xml" />
5335
<tool file="${model_tools_path}/duplicate_file_to_collection.xml" />
5436
</section>
@@ -80,77 +62,18 @@
8062
<tool file="filters/sorter.xml" />
8163
<tool file="filters/grep.xml" />
8264
<tool file="filters/grep_1.0.1.xml"/>
83-
<!-- <tool file="filters/gff/extract_GFF_Features.xml" />
84-
<tool file="filters/gff/gff_filter_by_attribute.xml" />
85-
<tool file="filters/gff/gff_filter_by_feature_count.xml" />
86-
<tool file="filters/gff/gtf_filter_by_attribute_values_list.xml" /> -->
8765
</section>
8866
<section id="table_manipulation" name="Table Manipulation">
8967
<tool file="filters/joiner.xml" />
9068
<tool file="filters/compare.xml" />
9169
<tool file="stats/grouping.xml" />
9270
</section>
93-
<!-- <label id="genomic_manip_label" text="Genomic File Manipulation" />
94-
<section id="convert_formats" name="Convert Formats">
95-
<tool file="filters/axt_to_concat_fasta.xml" />
96-
<tool file="filters/axt_to_fasta.xml" />
97-
<tool file="filters/axt_to_lav.xml" />
98-
<tool file="filters/bed2gff.xml" />
99-
<tool file="filters/gff2bed.xml" />
100-
<tool file="filters/lav_to_bed.xml" />
101-
<tool file="maf/maf_to_bed.xml" />
102-
<tool file="maf/maf_to_interval.xml" />
103-
<tool file="maf/maf_to_fasta.xml" />
104-
<tool file="filters/wiggle_to_simple.xml" />
105-
<tool file="filters/sff_extractor.xml" />
106-
<tool file="filters/gtf2bedgraph.xml" />
107-
<tool file="filters/wig_to_bigwig.xml" />
108-
<tool file="filters/bed_to_bigbed.xml" />
109-
<tool file="extract/liftOver_wrapper.xml" />
110-
<tool file="filters/ucsc_gene_bed_to_exon_bed.xml" />
111-
<tool file="stats/aggregate_binned_scores_in_intervals.xml" />
112-
</section> -->
113-
<!-- <section id="fetch_sequences___alignments" name="Fetch Sequences / Alignments">
114-
<tool file="maf/interval2maf_pairwise.xml" />
115-
<tool file="maf/interval2maf.xml" />
116-
<tool file="maf/maf_split_by_species.xml" />
117-
<tool file="maf/interval_maf_to_merged_fasta.xml" />
118-
<tool file="maf/genebed_maf_to_fasta.xml" />
119-
<tool file="maf/maf_stats.xml" />
120-
<tool file="maf/maf_thread_for_species.xml" />
121-
<tool file="maf/maf_limit_to_species.xml" />
122-
<tool file="maf/maf_limit_size.xml" />
123-
<tool file="maf/maf_by_block_number.xml" />
124-
<tool file="maf/maf_reverse_complement.xml" />
125-
<tool file="maf/maf_filter.xml" />
126-
</section> -->
127-
<!-- <section id="mapping" name="Mapping" version="">
128-
<tool file="sr_mapping/bfast_wrapper.xml" />
129-
<tool file="sr_mapping/PerM.xml" />
130-
<tool file="sr_mapping/srma_wrapper.xml" />
131-
<tool file="sr_mapping/mosaik.xml" />
132-
</section> -->
133-
<!-- <section id="phenotype_association" name="Phenotype Association">
134-
<tool file="phenotype_association/linkToGProfile.xml" />
135-
<tool file="phenotype_association/linkToDavid.xml" />
136-
<tool file="phenotype_association/ldtools.xml" />
137-
<tool file="phenotype_association/pass.xml" />
138-
<tool file="phenotype_association/gpass.xml" />
139-
<tool file="phenotype_association/beam.xml" />
140-
<tool file="phenotype_association/lps.xml" />
141-
<tool file="phenotype_association/master2pg.xml" />
142-
</section> -->
14371
<section id="statistics" name="Statistics">
14472
<tool file="stats/gsummary.xml" />
14573
<tool file="filters/uniq.xml" />
14674
</section>
147-
<!-- <section id="motif_tools" name="Motif Tools">
148-
<tool file="meme/meme.xml" hidden="True" />
149-
<tool file="meme/fimo.xml" hidden="True" />
150-
</section> -->
15175
<section id="visualization" name="Visualization">
15276
<tool file="plotting/bar_chart.xml" />
15377
<tool file="plotting/boxplot.xml" />
154-
<!-- <tool file="maf/vcf_to_maf_customtrack.xml" /> -->
15578
</section>
15679
</toolbox>

‎files/galaxy-umsa.grid.cesnet.cz/tpv_rules_local.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,6 @@ tools:
4444
mem: 16
4545

4646
.*/fastqc/.*:
47-
cores: 1
47+
cores: 1
4848
context:
4949
walltime: 2

‎files/galaxy/config/tool_conf.xml‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,22 @@
2828
<tool file="data_source/hbvar.xml" />
2929
</section>
3030
<section id="send_data" name="Send Data">
31-
<tool file="data_export/send.xml" />
32-
<!-- <tool file="data_export/export_remote.xml" /> -->
31+
<!-- <tool file="data_export/send.xml" /> -->
32+
<tool file="data_export/export_remote.xml" />
3333
</section>
3434
<section id="collection_operations" name="Collection Operations">
3535
<tool file="${model_tools_path}/unzip_collection.xml" />
3636
<tool file="${model_tools_path}/zip_collection.xml" />
3737
<tool file="${model_tools_path}/filter_failed_collection.xml" />
38+
<tool file="${model_tools_path}/filter_failed_collection_1.1.0.xml" />
3839
<tool file="${model_tools_path}/filter_empty_collection.xml" />
40+
<tool file="${model_tools_path}/filter_empty_collection_1.1.0.xml" />
3941
<tool file="${model_tools_path}/filter_null.xml" />
42+
<tool file="${model_tools_path}/filter_null_1.1.0.xml" />
43+
<tool file="${model_tools_path}/keep_success_collection.xml" />
44+
<tool file="${model_tools_path}/keep_success_collection_1.1.0.xml" />
4045
<tool file="${model_tools_path}/flatten_collection.xml" />
46+
<tool file="${model_tools_path}/nest_collection.xml" />
4147
<tool file="${model_tools_path}/merge_collection.xml" />
4248
<tool file="${model_tools_path}/relabel_from_file.xml" />
4349
<tool file="${model_tools_path}/filter_from_file.xml" />
@@ -49,6 +55,7 @@
4955
<tool file="${model_tools_path}/apply_rules.xml" />
5056
<tool file="${model_tools_path}/build_list.xml" />
5157
<tool file="${model_tools_path}/build_list_1.2.0.xml" />
58+
<tool file="${model_tools_path}/sample_sheet_to_tabular.xml" />
5259
<tool file="${model_tools_path}/extract_dataset.xml" />
5360
<tool file="${model_tools_path}/duplicate_file_to_collection.xml" />
5461
</section>
@@ -134,20 +141,12 @@
134141
<tool file="phenotype_association/linkToGProfile.xml" />
135142
<tool file="phenotype_association/linkToDavid.xml" />
136143
<tool file="phenotype_association/ldtools.xml" />
137-
<tool file="phenotype_association/pass.xml" />
138-
<tool file="phenotype_association/gpass.xml" />
139-
<tool file="phenotype_association/beam.xml" />
140-
<tool file="phenotype_association/lps.xml" />
141144
<tool file="phenotype_association/master2pg.xml" />
142145
</section>
143146
<section id="statistics" name="Statistics">
144147
<tool file="stats/gsummary.xml" />
145148
<tool file="filters/uniq.xml" />
146149
</section>
147-
<section id="motif_tools" name="Motif Tools">
148-
<tool file="meme/meme.xml" hidden="True" />
149-
<tool file="meme/fimo.xml" hidden="True" />
150-
</section>
151150
<section id="graph_display_data" name="Graph/Display Data">
152151
<tool file="plotting/bar_chart.xml" />
153152
<tool file="plotting/boxplot.xml" />

files/galaxy-qa2.galaxy.cloud.e-infra.cz/tpv_rules_local.yml renamed to files/galaxy/config/tpv_rules_local.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
1-
# prevent tpv lint complaints
21
global:
32
default_inherits: default

‎files/usegalaxy.cz/tpv_rules_local.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,70 @@
11
global:
22
default_inherits: default
3+
destinations:
4+
tpv_pulsar_rosettafold:
5+
inherits: tpv_pulsar
6+
runner: pulsar_tpv_runner
7+
params:
8+
singularity_enabled: false
9+
submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q galaxy_gpu@pbs-m1.metacentrum.cz -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
10+
scheduling:
11+
require:
12+
- rosettafold
13+
tpv_pulsar_rosettafold_sing:
14+
inherits: tpv_pulsar_rosettafold
15+
container_resolvers:
16+
- type: explicit_singularity
17+
- cache_directory: /cvmfs/singularity.metacentrum.cz/RoseTTAFold2/
18+
type: cached_mulled_singularity
19+
params:
20+
singularity_enabled: true
21+
singularity_run_extra_arguments: '--nv --env SCRATCHDIR="$SCRATCHDIR" --env SCRATCH=$SCRATCHDIR'
22+
singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,/scratch.ssd/galaxyeu/permanent/rosettafold_data/bfd:/opt/RoseTTAFold2/bfd,/scratch.ssd/galaxyeu/permanent/rosettafold_data/pdb100_2021Mar03:/opt/RoseTTAFold2/pdb100_2021Mar03,/scratch.ssd/galaxyeu/permanent/rosettafold_data/UniRef30_2020_06:/opt/RoseTTAFold2/UniRef30_2020_06,/scratch.ssd/galaxyeu/permanent/rosettafold_data/weights:/opt/RoseTTAFold2/network/weights'
23+
singularity_default_container_id: "/cvmfs/singularity.metacentrum.cz/RoseTTAFold2/rosettafold_image.sif"
24+
scheduling:
25+
require:
26+
- singularity
27+
tpv_pulsar_alphafold:
28+
inherits: tpv_pulsar
29+
max_accepted_gpus: 1
30+
max_gpus: 1
31+
runner: pulsar_tpv_runner
32+
params:
33+
singularity_run_extra_arguments: '--nv'
34+
singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data/2.3:ro'
35+
submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q {{ pulsar.pbs_gpu_queue }} -N {{ pulsar.nfs_prefix }}_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
36+
scheduling:
37+
require:
38+
- alphafold
39+
tools:
40+
testing_rosettafold2:
41+
cores: 8
42+
mem: 64
43+
gpus: 1
44+
context:
45+
walltime: 24
46+
scratch: 25
47+
gpu_mem: 16
48+
scheduling:
49+
require:
50+
- rosettafold
51+
rosettafold2:
52+
inherits: testing_rosettafold2
53+
scheduling:
54+
require:
55+
- singularity
56+
toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.*:
57+
cores: 8
58+
mem: 120
59+
gpus: 1
60+
context:
61+
walltime: 24
62+
scratch: 100
63+
gpu_mem: 16
64+
env:
65+
MPLCONFIGDIR: "$SCRATCHDIR"
66+
ALPHAFOLD_DB: "/scratch.ssd/galaxyeu/permanent/alphafold.db"
67+
ALPHAFOLD_USE_GPU: True
68+
scheduling:
69+
require:
70+
- alphafold

‎group_vars/galaxyservers.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,11 @@ galaxy_config_templates:
269269
- src: "{{ lookup('first_found', ['templates/'+inventory_hostname+'/config/file_source_templates.yml.j2', 'templates/galaxy/config/file_source_templates.yml.j2']) }}"
270270
dest: "{{ galaxy_config_dir }}/file_source_templates.yml"
271271

272+
galaxy_local_tools:
273+
- testing.xml
274+
- testing_pbs.xml
275+
- testing_html.xml
276+
272277
galaxy_extra_dirs:
273278
- "{{ tpv_config_dir }}"
274279
- "{{ galaxy_config_dir }}/plugins/activities"

‎host_vars/galaxy-qa1.galaxy.cloud.e-infra.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ pulsar:
2121
galaxy_config_files:
2222
- src: files/galaxy/themes.yml
2323
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
24-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
24+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
2525
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
2626
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
2727
dest: "{{ galaxy_config_dir }}/tool_conf.xml"

‎host_vars/galaxy-qa2.galaxy.cloud.e-infra.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ pulsar:
1717
galaxy_config_files:
1818
- src: files/galaxy/themes.yml
1919
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
20-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
20+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
2121
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
2222
- src: files/{{ inventory_hostname }}/file_sources_conf.yml
2323
dest: "{{ galaxy_config_dir }}/file_sources_conf.yml"

‎host_vars/galaxy-umsa.grid.cesnet.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ pulsar:
3232
galaxy_config_files:
3333
- src: files/galaxy/themes.yml
3434
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
35-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
35+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
3636
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
3737
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
3838
dest: "{{ galaxy_config_dir }}/tool_conf.xml"

0 commit comments

Comments
 (0)