Safetensors

.github/workflows/run-pytest.yml

@@ -14,6 +14,7 @@ on:
 
 env:
   PYTHON_PRIMARY_VERSION: '3.10'
+  SHHS_DATA_AVAILABLE: 'false'
 
 jobs:
   build:
@@ -66,40 +67,61 @@ jobs:
     - name: List installed Python packages
       run: |
         python -m pip list
-    - name: Install nsrr and download a samll part of SHHS to do test
+    - name: Install nsrr and download a small part of SHHS to do test
     # ref. https://github.com/DeepPSP/nsrr-automate
-      uses: gacts/run-and-post-run@v1
-      with:
-        # if ~/tmp/nsrr-data/shhs is empty (no files downloaded),
-        # fail and terminate the workflow
-        run: |
-          gem install nsrr --no-document
-          nsrr download shhs/polysomnography/edfs/shhs1/ --file="^shhs1\-20010.*\.edf" --token=${{ secrets.NSRR_TOKEN }}
-          nsrr download shhs/polysomnography/annotations-events-nsrr/shhs1/ --file="^shhs1\-20010.*\-nsrr\.xml" --token=${{ secrets.NSRR_TOKEN }}
-          nsrr download shhs/polysomnography/annotations-events-profusion/shhs1/ --file="^shhs1\-20010.*\-profusion\.xml" --token=${{ secrets.NSRR_TOKEN }}
-          nsrr download shhs/polysomnography/annotations-rpoints/shhs1/ --file="^shhs1\-20010.*\-rpoint\.csv" --token=${{ secrets.NSRR_TOKEN }}
-          nsrr download shhs/datasets/ --shallow --token=${{ secrets.NSRR_TOKEN }}
-          nsrr download shhs/datasets/hrv-analysis/ --token=${{ secrets.NSRR_TOKEN }}
-          mkdir -p ~/tmp/nsrr-data/
-          mv shhs/ ~/tmp/nsrr-data/
-          du -sh ~/tmp/nsrr-data/*
-          if [ "$(find ~/tmp/nsrr-data/shhs -type f | wc -l)" -eq 0 ]; \
-          then (echo "No files downloaded. Exiting..." && exit 1); \
-          else echo "Found $(find ~/tmp/nsrr-data/shhs -type f | wc -l) files"; fi
-        post: |
-          cd ~/tmp/ && du -sh $(ls -A)
-          rm -rf ~/tmp/nsrr-data/
-          cd ~/tmp/ && du -sh $(ls -A)
+      # uses: gacts/run-and-post-run@v1
+      continue-on-error: true
+      run: |
+        set -u -o pipefail
+        gem install nsrr --no-document
+        nsrr download shhs/polysomnography/edfs/shhs1/ --file="^shhs1\-20010.*\.edf" --token=${{ secrets.NSRR_TOKEN }} 2>&1 | sed -E '/^[[:space:]]*[Ss]kipped([[:space:]]|$)/d' || true
+        nsrr download shhs/polysomnography/annotations-events-nsrr/shhs1/ --file="^shhs1\-20010.*\-nsrr\.xml" --token=${{ secrets.NSRR_TOKEN }} 2>&1 | sed -E '/^[[:space:]]*[Ss]kipped([[:space:]]|$)/d' || true
+        nsrr download shhs/polysomnography/annotations-events-profusion/shhs1/ --file="^shhs1\-20010.*\-profusion\.xml" --token=${{ secrets.NSRR_TOKEN }} 2>&1 | sed -E '/^[[:space:]]*[Ss]kipped([[:space:]]|$)/d' || true
+        nsrr download shhs/polysomnography/annotations-rpoints/shhs1/ --file="^shhs1\-20010.*\-rpoint\.csv" --token=${{ secrets.NSRR_TOKEN }} 2>&1 | sed -E '/^[[:space:]]*[Ss]kipped([[:space:]]|$)/d' || true
+        nsrr download shhs/datasets/ --shallow --token=${{ secrets.NSRR_TOKEN }} 2>&1 | sed -E '/^[[:space:]]*[Ss]kipped([[:space:]]|$)/d' || true
+        nsrr download shhs/datasets/hrv-analysis/ --token=${{ secrets.NSRR_TOKEN }} 2>&1 | sed -E '/^[[:space:]]*[Ss]kipped([[:space:]]|$)/d' || true
+
+        mkdir -p ~/tmp/nsrr-data/
+        mv shhs/ ~/tmp/nsrr-data/
+        du -sh ~/tmp/nsrr-data/* || true
+
+        EDF_COUNT=$(find ~/tmp/nsrr-data/shhs -type f -name "*.edf" 2>/dev/null | wc -l | tr -d ' ')
+        echo "Detected SHHS EDF file count: $EDF_COUNT"
+
+        if [ "$EDF_COUNT" -eq 0 ]; then
+          echo "::error title=No SHHS EDF files downloaded::No .edf files were downloaded (token may be invalid or pattern mismatch). SHHS tests will be skipped."
+          echo "No SHHS EDF files downloaded; setting SHHS_DATA_AVAILABLE=false"
+          echo "SHHS_DATA_AVAILABLE=false" >> $GITHUB_ENV
+          exit 1
+        else
+          echo "Found $EDF_COUNT SHHS EDF files; setting SHHS_DATA_AVAILABLE=true"
+          echo "SHHS_DATA_AVAILABLE=true" >> $GITHUB_ENV
+        fi
+      # post: |
+      #   cd ~/tmp/ && du -sh $(ls -A)
+      #   rm -rf ~/tmp/nsrr-data/
+      #   cd ~/tmp/ && du -sh $(ls -A)
     - name: Run test with pytest and collect coverage
       run: |
-        pytest -vv -s \
-        --cov=torch_ecg \
-        --ignore=test/test_pipelines \
-        test
+        echo "SHHS_DATA_AVAILABLE at test step: $SHHS_DATA_AVAILABLE"
+        pytest --cov --junitxml=junit.xml -o junit_family=legacy
+      env:
+        SHHS_DATA_AVAILABLE: ${{ env.SHHS_DATA_AVAILABLE }}
     - name: Upload coverage to Codecov
       if: matrix.python-version == ${{ env.PYTHON_PRIMARY_VERSION }}
       uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true # optional (default = false)
         verbose: true # optional (default = false)
         token: ${{ secrets.CODECOV_TOKEN }} # required
+    - name: Upload test results to Codecov
+      if: ${{ !cancelled() }}
+      uses: codecov/test-results-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+    - name: Cleanup SHHS temp data
+      if: always() && true
+      run: |
+        cd ~/tmp/ && du -sh $(ls -A)
+        rm -rf ~/tmp/nsrr-data/
+        cd ~/tmp/ && du -sh $(ls -A)

CHANGELOG.rst

@@ -19,8 +19,13 @@ Changed
 
 - Make the function `remove_spikes_naive` in `torch_ecg.utils.utils_signal`
   support 2D and 3D input signals.
+- Use `save_file` and `load_file` from the `safetensors` package for saving
+  and loading files in place of `torch.save` and `torch.load` in the `CkptMixin`
+  class in `torch_ecg.utils.utils_nn`.
 - Add retry mechanism to the `http_get` function in
   `torch_ecg.utils.download` module.
+- Add length verification in the `http_get` function in
+  `torch_ecg.utils.download` module.
 
 Deprecated
 ~~~~~~~~~~
@@ -42,6 +47,14 @@ Fixed
 - Fix potential errors when deepcopying a `torch_ecg.cfg.CFG` object:
   previously, deepcopying such an object like `CFG({"a": {1: 0.1, 2: 0.2}})`
   would result in an error.
+- Fix potential bugs in contextmanager `torch_ecg.utils.timeout`: restore the previously
+  installed SIGALRM handler after use, cancel any pending alarm reliably in a finally block,
+  avoid installing a handler when duration <= 0 (preventing unintended global side-effects),
+  and thereby eliminate spurious `TimeoutError` exceptions that could be triggered later by
+  unrelated signal.alarm calls due to the old implementation not reinstating the original handler.
+- Fix bugs in utility function `torch_ecg.utils.make_serializable`: the previous implementation
+  does not drop some types of unserializable items correctly. Two additional parameters
+  `drop_unserializable` and `drop_paths` are added.
 
 Security
 ~~~~~~~~

benchmarks/train_crnn_cinc2020/model.py

@@ -96,8 +96,8 @@ def inference(
             _input = _input.unsqueeze(0)  # add a batch dimension
         prob = self.sigmoid(self.forward(_input))
         pred = (prob >= bin_pred_thr).int()
-        prob = prob.cpu().detach().numpy()
-        pred = pred.cpu().detach().numpy()
+        prob = prob.detach().cpu().numpy()
+        pred = pred.detach().cpu().numpy()
         for row_idx, row in enumerate(pred):
             row_max_prob = prob[row_idx, ...].max()
             if row_max_prob < ModelCfg.bin_pred_nsr_thr and nsr_cid is not None:

benchmarks/train_crnn_cinc2021/model.py

@@ -99,8 +99,8 @@ def inference(
         # batch_size, channels, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
         pred = (prob >= bin_pred_thr).int()
-        prob = prob.cpu().detach().numpy()
-        pred = pred.cpu().detach().numpy()
+        prob = prob.detach().cpu().numpy()
+        pred = pred.detach().cpu().numpy()
         for row_idx, row in enumerate(pred):
             row_max_prob = prob[row_idx, ...].max()
             if row_max_prob < ModelCfg.bin_pred_nsr_thr and nsr_cid is not None:

benchmarks/train_hybrid_cpsc2020/model.py

@@ -92,8 +92,8 @@ def inference(
             _input = _input.unsqueeze(0)  # add a batch dimension
         prob = self.sigmoid(self.forward(_input))
         pred = (prob >= bin_pred_thr).int()
-        prob = prob.cpu().detach().numpy()
-        pred = pred.cpu().detach().numpy()
+        prob = prob.detach().cpu().numpy()
+        pred = pred.detach().cpu().numpy()
         for row_idx, row in enumerate(pred):
             row_max_prob = prob[row_idx, ...].max()
             if row.sum() == 0:
@@ -190,14 +190,14 @@ def inference(
         if self.n_classes == 2:
             prob = self.sigmoid(prob)  # (batch_size, seq_len, 2)
             pred = (prob >= bin_pred_thr).int()
-            prob = prob.cpu().detach().numpy()
-            pred = pred.cpu().detach().numpy()
+            prob = prob.detach().cpu().numpy()
+            pred = pred.detach().cpu().numpy()
             # aux used to filter out potential simultaneous predictions of SPB and PVC
             aux = (prob == np.max(prob, axis=2, keepdims=True)).astype(int)
             pred = aux * pred
         elif self.n_classes == 3:
             prob = self.softmax(prob)  # (batch_size, seq_len, 3)
-            prob = prob.cpu().detach().numpy()
+            prob = prob.detach().cpu().numpy()
             pred = np.argmax(prob, axis=2)
 
         if rpeak_inds is not None:

benchmarks/train_hybrid_cpsc2021/entry_2021.py

@@ -379,12 +379,12 @@ def _detect_rpeaks(model, sig, siglen, overlap_len, config):
     for idx in range(batch_size // _BATCH_SIZE):
         pred = model.forward(sig[_BATCH_SIZE * idx : _BATCH_SIZE * (idx + 1), ...])
         pred = model.sigmoid(pred)
-        pred = pred.cpu().detach().numpy().squeeze(-1)
+        pred = pred.detach().cpu().numpy().squeeze(-1)
         l_pred.append(pred)
     if batch_size % _BATCH_SIZE != 0:
         pred = model.forward(sig[batch_size // _BATCH_SIZE * _BATCH_SIZE :, ...])
         pred = model.sigmoid(pred)
-        pred = pred.cpu().detach().numpy().squeeze(-1)
+        pred = pred.detach().cpu().numpy().squeeze(-1)
         l_pred.append(pred)
     pred = np.concatenate(l_pred)
 
@@ -473,12 +473,12 @@ def _main_task(model, sig, siglen, overlap_len, rpeaks, config):
     for idx in range(batch_size // _BATCH_SIZE):
         pred = model.forward(sig[_BATCH_SIZE * idx : _BATCH_SIZE * (idx + 1), ...])
         pred = model.sigmoid(pred)
-        pred = pred.cpu().detach().numpy().squeeze(-1)
+        pred = pred.detach().cpu().numpy().squeeze(-1)
         l_pred.append(pred)
     if batch_size % _BATCH_SIZE != 0:
         pred = model.forward(sig[batch_size // _BATCH_SIZE * _BATCH_SIZE :, ...])
         pred = model.sigmoid(pred)
-        pred = pred.cpu().detach().numpy().squeeze(-1)
+        pred = pred.detach().cpu().numpy().squeeze(-1)
         l_pred.append(pred)
     pred = np.concatenate(l_pred)
 

benchmarks/train_hybrid_cpsc2021/model.py

@@ -169,7 +169,7 @@ def _inference_qrs_detection(
             _input = _input.unsqueeze(0)  # add a batch dimension
         # batch_size, channels, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         # prob --> qrs mask --> qrs intervals --> rpeaks
         rpeaks = _qrs_detection_post_process(
@@ -226,7 +226,7 @@ def _inference_main_task(
             _input = _input.unsqueeze(0)  # add a batch dimension
         batch_size, n_leads, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         af_episodes, af_mask = _main_task_post_process(
             prob=prob,
@@ -368,7 +368,7 @@ def _inference_qrs_detection(
             _input = _input.unsqueeze(0)  # add a batch dimension
         # batch_size, channels, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         # prob --> qrs mask --> qrs intervals --> rpeaks
         rpeaks = _qrs_detection_post_process(
@@ -425,7 +425,7 @@ def _inference_main_task(
             _input = _input.unsqueeze(0)  # add a batch dimension
         batch_size, n_leads, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         af_episodes, af_mask = _main_task_post_process(
             prob=prob,
@@ -567,7 +567,7 @@ def _inference_qrs_detection(
             _input = _input.unsqueeze(0)  # add a batch dimension
         # batch_size, channels, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         # prob --> qrs mask --> qrs intervals --> rpeaks
         rpeaks = _qrs_detection_post_process(
@@ -624,7 +624,7 @@ def _inference_main_task(
             _input = _input.unsqueeze(0)  # add a batch dimension
         batch_size, n_leads, seq_len = _input.shape
         prob = self.sigmoid(self.forward(_input))
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         af_episodes, af_mask = _main_task_post_process(
             prob=prob,
@@ -721,7 +721,7 @@ def inference(
         prob = self.forward(_input)
         if self.config.clf.name != "crf":
             prob = self.sigmoid(prob)
-        prob = prob.cpu().detach().numpy().squeeze(-1)
+        prob = prob.detach().cpu().numpy().squeeze(-1)
 
         af_episodes, af_mask = _main_task_post_process(
             prob=prob,

benchmarks/train_mtl_cinc2022/models/crnn.py

@@ -200,31 +200,31 @@ def inference(
         prob = self.softmax(forward_output["murmur"])
         pred = torch.argmax(prob, dim=-1)
         bin_pred = (prob == prob.max(dim=-1, keepdim=True).values).to(int)
-        prob = prob.cpu().detach().numpy()
-        pred = pred.cpu().detach().numpy()
-        bin_pred = bin_pred.cpu().detach().numpy()
+        prob = prob.detach().cpu().numpy()
+        pred = pred.detach().cpu().numpy()
+        bin_pred = bin_pred.detach().cpu().numpy()
 
         murmur_output = ClassificationOutput(
             classes=self.classes,
             prob=prob,
             pred=pred,
             bin_pred=bin_pred,
-            forward_output=forward_output["murmur"].cpu().detach().numpy(),
+            forward_output=forward_output["murmur"].detach().cpu().numpy(),
         )
 
         if forward_output.get("outcome", None) is not None:
             prob = self.softmax(forward_output["outcome"])
             pred = torch.argmax(prob, dim=-1)
             bin_pred = (prob == prob.max(dim=-1, keepdim=True).values).to(int)
-            prob = prob.cpu().detach().numpy()
-            pred = pred.cpu().detach().numpy()
-            bin_pred = bin_pred.cpu().detach().numpy()
+            prob = prob.detach().cpu().numpy()
+            pred = pred.detach().cpu().numpy()
+            bin_pred = bin_pred.detach().cpu().numpy()
             outcome_output = ClassificationOutput(
                 classes=self.outcomes,
                 prob=prob,
                 pred=pred,
                 bin_pred=bin_pred,
-                forward_output=forward_output["outcome"].cpu().detach().numpy(),
+                forward_output=forward_output["outcome"].detach().cpu().numpy(),
             )
         else:
             outcome_output = None
@@ -238,13 +238,13 @@ def inference(
             else:
                 prob = self.sigmoid(forward_output["segmentation"])
                 pred = (prob > seg_thr).int() * (prob == prob.max(dim=-1, keepdim=True).values).int()
-            prob = prob.cpu().detach().numpy()
-            pred = pred.cpu().detach().numpy()
+            prob = prob.detach().cpu().numpy()
+            pred = pred.detach().cpu().numpy()
             segmentation_output = SequenceLabellingOutput(
                 classes=self.states,
                 prob=prob,
                 pred=pred,
-                forward_output=forward_output["segmentation"].cpu().detach().numpy(),
+                forward_output=forward_output["segmentation"].detach().cpu().numpy(),
             )
         else:
             segmentation_output = None

benchmarks/train_mtl_cinc2022/models/model_ml.py

@@ -170,9 +170,10 @@ def get_model(self, model_name: str, params: Optional[dict] = None) -> BaseEstim
 
         """
         model_cls = self.model_map[model_name]
+        params = params or {}
         if model_cls in [GradientBoostingClassifier, SVC]:
             params.pop("n_jobs", None)
-        return model_cls(**(params or {}))
+        return model_cls(**params)
 
     def save_model(
         self,
@@ -198,9 +199,13 @@ def save_model(
             path to save the model.
 
         """
+        if isinstance(model_path, bytes):
+            model_path = model_path.decode()
+        model_path = Path(model_path).expanduser().resolve()
+        model_path.parent.mkdir(parents=True, exist_ok=True)
         _config = deepcopy(config)
         _config.pop("db_dir", None)
-        Path(model_path).write_bytes(
+        model_path.write_bytes(
             pickle.dumps(
                 {
                     "config": _config,

benchmarks/train_mtl_cinc2022/models/seg.py

@@ -138,8 +138,8 @@ def inference(
         else:
             prob = self.sigmoid(self.forward(_input)["segmentation"])
             pred = (prob > bin_pred_threshold).int() * (prob == prob.max(dim=-1, keepdim=True).values).int()
-        prob = prob.cpu().detach().numpy()
-        pred = pred.cpu().detach().numpy()
+        prob = prob.detach().cpu().numpy()
+        pred = pred.detach().cpu().numpy()
 
         segmentation_output = SequenceLabellingOutput(
             classes=self.classes,
@@ -264,8 +264,8 @@ def inference(
         else:
             prob = self.sigmoid(self.forward(_input)["segmentation"])
             pred = (prob > bin_pred_threshold).int() * (prob == prob.max(dim=-1, keepdim=True).values).int()
-        prob = prob.cpu().detach().numpy()
-        pred = pred.cpu().detach().numpy()
+        prob = prob.detach().cpu().numpy()
+        pred = pred.detach().cpu().numpy()
 
         segmentation_output = SequenceLabellingOutput(
             classes=self.classes,