feat: annotating with Orphanet diseases (#58)

holtgrewe · web-flow · commit 7d43865a48cf · 2023-08-27T07:34:25.000Z
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -67,12 +67,21 @@ jobs:
           cache-environment: true
           post-cleanup: none  # breaks otherwise
 
+      - name: Install qsv (not in conda-forge yet)
+        run: |
+           cd /tmp
+           wget https://github.com/jqnatividad/qsv/releases/download/0.112.0/qsv-0.112.0-x86_64-unknown-linux-musl.zip
+           unzip qsv-0.112.0-x86_64-unknown-linux-musl.zip
+           cp qsv_musl-1.2.3 /usr/local/bin/qsv
+
       - name: Install python package
         run: |
           pip install -e .
         shell: bash -el {0}
 
       - name: Run in test mode
         run: |
-          CI=true snakemake --cores=1 -p
+          CI=true snakemake --cores=1 -p all
         shell: bash -el {0}
+        env:
+         QSV_NO_UPDATE: "1"
diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -15,6 +15,7 @@ jobs:
         id: release
         with:
           release-type: simple
+          token: ${{ secrets.BOT_TOKEN }}
 
       - name: Checkout repository
         uses: actions/checkout@v3
diff --git a/Snakefile b/Snakefile
@@ -87,13 +87,15 @@ rule all:
         # genes
         f"work/download/genes/clingen/{DV.clingen_gene}/clingen.csv",
         f"work/download/genes/rcnv/2022/Collins_rCNV_2022.dosage_sensitivity_scores.tsv.gz",
+        f"work/download/genes/orphapacket/{DV.orphapacket}/orphapacket.tar.gz",
         f"work/genes/dbnsfp/{DV.dbnsfp}/genes.tsv.gz",
         f"work/genes/ensembl/{DV.ensembl}/ensembl_xlink.tsv",
         f"work/genes/enst_ensg/grch37/{DV.ensembl_37}/enst_ensg.tsv",
         f"work/genes/entrez/{DV.today}/gene_info.jsonl",
         f"work/genes/gnomad/{DV.gnomad_constraints}/gnomad_constraints.tsv",
         f"work/genes/hgnc/{DV.today}/hgnc_info.jsonl",
         f"work/genes/omim/{DV.hpo}+{DV.today}/omim_diseases.tsv",
+        f"work/genes/orphapacket/{DV.orphapacket}+{DV.today}/orpha_diseases.tsv",
         "work/genes/rcnv/2022/rcnv_collins_2022.tsv",
         "work/genes/shet/2019/shet_weghorn_2019.tsv",
         # reference-specific annotations
@@ -160,7 +162,7 @@ rule all:
         f"output/full/annonars/cons-grch37-{DV.ucsc_cons_37}+{PV.annonars}/rocksdb/IDENTITY",
         f"output/full/annonars/cons-grch38-{DV.ucsc_cons_38}+{PV.annonars}/rocksdb/IDENTITY",
         # ----- genes
-        f"output/full/annonars/genes-{DV.acmg_sf}+{DV.gnomad_constraints}+{DV.dbnsfp}+{DV.hpo}+{DV.today}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/full/annonars/genes-{DV.acmg_sf}+{DV.gnomad_constraints}+{DV.dbnsfp}+{DV.hpo}+{DV.orphapacket}+{DV.today}+{PV.annonars}/rocksdb/IDENTITY",
         # -- worker data
         f"output/full/worker/genes-regions-grch37-{DV.refseq_37}+{PV.worker}/refseq_genes.bin",
         f"output/full/worker/genes-regions-grch37-{DV.ensembl_37}+{PV.worker}/ensembl_genes.bin",
@@ -329,6 +331,7 @@ include: "rules/work/genes/hgnc.smk"
 include: "rules/work/genes/mehari_data_tx.smk"
 include: "rules/work/genes/ncbi.smk"
 include: "rules/work/genes/omim.smk"
+include: "rules/work/genes/orphapacket.smk"
 include: "rules/work/genes/rcnv.smk"
 include: "rules/work/genes/shet.smk"
 # Reference sequence--related rules.
diff --git a/download_urls.yml b/download_urls.yml
@@ -1,3 +1,8 @@
+- url: https://github.com/Orphanet/orphapacket/archive/refs/tags/v10.1.tar.gz
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
 - url: https://zenodo.org/record/6347673/files/Collins_rCNV_2022.dosage_sensitivity_scores.tsv.gz
 
 - comment: The curation activity summary report is built in real-time.
diff --git a/environment.yml b/environment.yml
@@ -40,7 +40,7 @@ dependencies:
   # Parallel (de)compression.
   - pigz
   # Varfish related
-  - annonars =0.14.1
+  - annonars =0.15.0
   - viguno =0.1.6
   - mehari =0.6.2
   - varfish-server-worker =0.10.1
diff --git a/excerpt-data/6045c008f1c0f370/url.txt b/excerpt-data/6045c008f1c0f370/url.txt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58b5f890e962de336ac264c788ae9136ad1eb69f52c1f536379c2984b85beef1
+size 71
diff --git a/excerpt-data/6045c008f1c0f370/v10.1.tar.gz b/excerpt-data/6045c008f1c0f370/v10.1.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38de2709b2f9a9c4e0b9ec18c5ebae337a47e97897808069dcee7ba5c39d3224
+size 1503498
diff --git a/rules/output/annonars/genes.smk b/rules/output/annonars/genes.smk
@@ -10,15 +10,16 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
         hgnc="work/genes/hgnc/{date}/hgnc_info.jsonl",
         ncbi="work/genes/entrez/{date}/gene_info.jsonl",
         omim="work/genes/omim/{v_hpo}+{date}/omim_diseases.tsv",
+        orpha="work/genes/orphapacket/{v_orpha}+{date}/orpha_diseases.tsv",
         rcnv="work/genes/rcnv/2022/rcnv_collins_2022.tsv",
         shet="work/genes/shet/2019/shet_weghorn_2019.tsv",
     output:
         rocksdb_identity=(
-            "output/full/annonars/genes-{v_acmg_sf}+{v_gnomad_constraints}+{v_dbnsfp}+{v_hpo}+{date}+{v_annonars}/"
+            "output/full/annonars/genes-{v_acmg_sf}+{v_gnomad_constraints}+{v_dbnsfp}+{v_hpo}+{v_orpha}+{date}+{v_annonars}/"
             "rocksdb/IDENTITY"
         ),
         spec_yaml=(
-            "output/full/annonars/genes-{v_acmg_sf}+{v_gnomad_constraints}+{v_dbnsfp}+{v_hpo}+{date}+{v_annonars}/"
+            "output/full/annonars/genes-{v_acmg_sf}+{v_gnomad_constraints}+{v_dbnsfp}+{v_hpo}+{v_orpha}+{date}+{v_annonars}/"
             "spec.yaml"
         ),
     wildcard_constraints:
@@ -42,6 +43,7 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
             --path-in-dbnsfp {input.dbnsfp} \
             --path-in-hgnc {input.hgnc} \
             --path-in-omim {input.omim} \
+            --path-in-orpha {input.orpha} \
             --path-in-ncbi {input.ncbi} \
             --path-in-rcnv {input.rcnv} \
             --path-in-shet {input.shet}
@@ -58,5 +60,6 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
             \
             --value v_annonars={wildcards.v_annonars} \
             --value v_downloader={PV.downloader} \
+            --value v_orphapacket={wildcards.v_orpha} \
         > {output.spec_yaml}
         """
diff --git a/rules/output/annonars/genes.spec.yaml b/rules/output/annonars/genes.spec.yaml
@@ -34,6 +34,8 @@ x-created-from:
     version: {{ today }}
   - name: OMIM
     version: {{ today }}
+  - name: ORDO
+    version: {{ v_orphapacket }}
   - name: rCNV pHaplo/pTriplo scores
     version: 2022-Collins-et-al
   - name: sHet scores
diff --git a/rules/work/genes/orphapacket.smk b/rules/work/genes/orphapacket.smk
@@ -0,0 +1,34 @@
+## Rules related to annotating genes with ORDO terms
+
+
+rule genes_orphapacket_download:  # -- download orphapacket file
+    output:
+        tar="work/download/genes/orphapacket/{version}/orphapacket.tar.gz",
+    shell:
+        r"""
+        wget -O {output.tar} \
+            https://github.com/Orphanet/orphapacket/archive/refs/tags/v10.1.tar.gz
+        """
+
+
+rule genes_orphapacket_diseases:  # -- postprocess file for HGNC gene IDs
+    input:
+        tar="work/download/genes/orphapacket/{version}/orphapacket.tar.gz",
+        xlink="output/full/mehari/genes-xlink-{date}/genes-xlink.tsv",
+    output:
+        tsv="work/genes/orphapacket/{version}+{date}/orpha_diseases.tsv",
+        tsv_md5="work/genes/orphapacket/{version}+{date}/orpha_diseases.tsv.md5",
+    shell:
+        """
+        export TMPDIR=$(mktemp -d)
+        trap "rm -rf $TMPDIR" ERR EXIT
+
+        tar -C $TMPDIR -xf $(readlink -f {input.tar})
+
+        python ./scripts/genes-orpha-diseases.py {input.xlink} $TMPDIR/orphapacket-*/json \
+        | qsv sort -d '\t' \
+        | qsv fmt -t '\t' \
+        > {output.tsv}
+
+        md5sum {output.tsv} > {output.tsv}.md5
+        """
diff --git a/scripts/genes-orpha-diseases.py b/scripts/genes-orpha-diseases.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Helper script to extract gene-disease association from orphapacket."""
+
+import csv
+import json
+import pathlib
+import sys
+
+
+def main():
+    symbol_to_hgnc = {}
+    with open(sys.argv[1], "rt") as inputf:
+        reader = csv.DictReader(inputf, delimiter="\t")
+        for record in reader:
+            symbol_to_hgnc[record["gene_symbol"]] = record["hgnc_id"]
+
+    print(f"# xlink entries: {len(symbol_to_hgnc)}", file=sys.stderr)
+
+    base_path = pathlib.Path(sys.argv[2])
+    print("\t".join(["hgnc_id", "orpha_id", "disease_name"]))
+    for json_path in sorted(base_path.glob("*.json")):
+        with json_path.open("rt") as inputf:
+            data = json.load(inputf)
+            elem_top = data["Orphapacket"]
+            if elem_top.get("DisorderType", {}).get("value") != "Disease":
+                continue  # skip categories
+            disease_name = elem_top["Label"]
+            orpha_id = elem_top["PURL"].replace("http://www.orpha.net/ORDO/Orphanet_", "ORPHA:")
+            elem_genes = elem_top.get("Genes", [])
+            for elem_gene in elem_genes:
+                gene_symbol = elem_gene["Gene"]["Symbol"]
+                hgnc_id = symbol_to_hgnc.get(gene_symbol)
+                if hgnc_id:  # skip if no HGNC ID exists, maybe withdrawn?
+                    print("\t".join(map(str, [hgnc_id, orpha_id, disease_name])))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/varfish_db_downloader/versions.py b/varfish_db_downloader/versions.py
@@ -94,6 +94,8 @@ class DataVersions:
     acmg_sf: str
     #: HPO
     hpo: str
+    #: OrphaPacket
+    orphapacket: str
     #: Pathogenic MMS
     patho_mms: str
     #: Mehari transcript data.
@@ -144,6 +146,7 @@ class DataVersions:
     dbsnp="b151",
     acmg_sf="3.1",
     hpo="20230606",
+    orphapacket="10.1",
     patho_mms="20220730",
     mehari_tx="0.2.2",
     clinvar_release=CLINVAR_RELEASE,

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:58b5f890e962de336ac264c788ae9136ad1eb69f52c1f536379c2984b85beef1`
	`3`	`+size 71`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:38de2709b2f9a9c4e0b9ec18c5ebae337a47e97897808069dcee7ba5c39d3224`
	`3`	`+size 1503498`