From 5e09047d02d86cacd9d7d449d1d31d56eae4ff19 Mon Sep 17 00:00:00 2001 From: Natasha Singh Date: Mon, 25 Sep 2023 17:02:01 -0400 Subject: [PATCH 1/3] :recycle: Dont use retry Sessions --- kf_utils/dataservice/delete.py | 3 ++- kf_utils/dataservice/patch.py | 3 ++- kf_utils/dataservice/scrape.py | 3 ++- kf_utils/dbgap/release.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kf_utils/dataservice/delete.py b/kf_utils/dataservice/delete.py index 4a8ec30..79fbe67 100644 --- a/kf_utils/dataservice/delete.py +++ b/kf_utils/dataservice/delete.py @@ -2,7 +2,8 @@ from pprint import pformat from urllib.parse import urlparse -from d3b_utils.requests_retry import Session +# from d3b_utils.requests_retry import Session +from requests import Session from kf_utils.dataservice.meta import get_endpoint from kf_utils.dataservice.scrape import yield_kfids diff --git a/kf_utils/dataservice/patch.py b/kf_utils/dataservice/patch.py index 5c727c8..5c8e94c 100644 --- a/kf_utils/dataservice/patch.py +++ b/kf_utils/dataservice/patch.py @@ -1,6 +1,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -from d3b_utils.requests_retry import Session +# from d3b_utils.requests_retry import Session +from requests import Session from kf_utils.dataservice.meta import get_endpoint diff --git a/kf_utils/dataservice/scrape.py b/kf_utils/dataservice/scrape.py index a39bad9..d63c23d 100755 --- a/kf_utils/dataservice/scrape.py +++ b/kf_utils/dataservice/scrape.py @@ -1,6 +1,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -from d3b_utils.requests_retry import Session +# from d3b_utils.requests_retry import Session +from requests import Session from kf_utils.dataservice.meta import get_endpoint from tqdm import tqdm diff --git a/kf_utils/dbgap/release.py b/kf_utils/dbgap/release.py index d2f9479..12155a2 100644 --- a/kf_utils/dbgap/release.py +++ b/kf_utils/dbgap/release.py @@ -1,5 +1,6 @@ import xmltodict -from d3b_utils.requests_retry import Session +# from d3b_utils.requests_retry import Session +from requests import Session from xml.etree import ElementTree # from defusedxml import ElementTree as DefusedET From 389054fa21a0a04aee94a53e11660a885401e297 Mon Sep 17 00:00:00 2001 From: Natasha Singh Date: Mon, 25 Sep 2023 17:15:01 -0400 Subject: [PATCH 2/3] :rotating_light: Fix linter errors --- kf_utils/dataservice/descendants.py | 4 ++-- kf_utils/dataservice/scrape.py | 2 +- kf_utils/dbgap/release.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/kf_utils/dataservice/descendants.py b/kf_utils/dataservice/descendants.py index 6edd6c1..bd38422 100644 --- a/kf_utils/dataservice/descendants.py +++ b/kf_utils/dataservice/descendants.py @@ -325,7 +325,7 @@ def _inner(parent_type, parent_kfids, descendants): if parent_type in done: return done.add(parent_type) - for (child_type, link_on_parent, link_on_child) in descendancy.get( + for child_type, link_on_parent, link_on_child in descendancy.get( parent_type, [] ): if use_api: @@ -387,7 +387,7 @@ def _inner(parent_type, parent_kfids, descendants): for k, v in descendants["genomic_file"].items() if k not in to_remove } - for (child_type, _, _) in descendancy.get(parent_type, []): + for child_type, _, _ in descendancy.get(parent_type, []): if descendants.get(child_type): _inner(child_type, descendants[child_type].keys(), descendants) diff --git a/kf_utils/dataservice/scrape.py b/kf_utils/dataservice/scrape.py index d63c23d..422bc47 100755 --- a/kf_utils/dataservice/scrape.py +++ b/kf_utils/dataservice/scrape.py @@ -53,7 +53,7 @@ def yield_entities_from_filter(host, endpoint, filters, show_progress=False): pbar.update() yield entity try: - for (key, i) in [("after", 1), ("after_uuid", 2)]: + for key, i in [("after", 1), ("after_uuid", 2)]: which[key] = j["_links"]["next"].split("=")[i].split("&")[0] except KeyError: break diff --git a/kf_utils/dbgap/release.py b/kf_utils/dbgap/release.py index 12155a2..207d95c 100644 --- a/kf_utils/dbgap/release.py +++ b/kf_utils/dbgap/release.py @@ -1,4 +1,5 @@ import xmltodict + # from d3b_utils.requests_retry import Session from requests import Session from xml.etree import ElementTree @@ -28,7 +29,8 @@ def get_latest_sample_status(phs_id, required_status="released"): print(f"Querying dbGaP for study {phs_string}") print(f"Manifest URL -> {url}") - data = Session(status_forcelist=(502, 503, 504)).get(url) + # data = Session(status_forcelist=(502, 503, 504)).get(url) + data = Session().get(url) if data.status_code != 200: tried[phs_string] = f"status {data.status_code}" raise Exception( From 5bf83bf8635d56bdcd861345a46807dd6ca1b776 Mon Sep 17 00:00:00 2001 From: Natasha Singh Date: Tue, 26 Sep 2023 12:52:55 -0400 Subject: [PATCH 3/3] :loud_sound: More details in patch message --- kf_utils/dataservice/patch.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kf_utils/dataservice/patch.py b/kf_utils/dataservice/patch.py index 5c8e94c..0652b9d 100644 --- a/kf_utils/dataservice/patch.py +++ b/kf_utils/dataservice/patch.py @@ -1,3 +1,4 @@ +from pprint import pformat from concurrent.futures import ThreadPoolExecutor, as_completed # from d3b_utils.requests_retry import Session @@ -16,10 +17,15 @@ def send_patches(host, patches): host = host.strip("/") def do_patch(url, patch): - msg = f"Patched {url} with {patch}" resp = Session().patch(url, json=patch) if not resp.ok: - raise Exception(f"{resp.status_code} -- {msg} -- {resp.json()}") + msg = f"Patched {url} with {patch}" + raise Exception( + f"{resp.status_code} -- {msg} -- Response:\n{resp.text}" + ) + else: + msg = f"Patched {url} with {patch}. Response:\n{pformat(resp.json())}" + return msg with ThreadPoolExecutor() as tpex: