Merge pull request #782 from effigies/tox

Update CI to use build-and-inspect-python-package and tox-uv

Author: tclose

.coveragerc

@@ -4,3 +4,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
+    groups:
+      actions-infrastructure:
+        patterns:
+          - "actions/*"

.github/dependabot.yml

@@ -8,8 +8,10 @@ on:
     types: [published]
   push:
     branches:
-      - master
+      - main
   pull_request:
+    branches:
+      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,75 +20,77 @@ concurrency:
 permissions:
   contents: read
 
-jobs:
+env:
+  # Force tox and pytest to use color
+  FORCE_COLOR: true
 
+jobs:
   build:
+    name: Build & verify package
     runs-on: ubuntu-latest
+    permissions:
+      attestations: write
+      id-token: write
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3
-      - run: pip install --upgrade build twine
-      - run: python -m build
-      - run: twine check dist/*
-      - uses: actions/upload-artifact@v4
+      - uses: hynek/build-and-inspect-python-package@v2
         with:
-          name: dist
-          path: dist/
-      - name: Build archive
-        run: |
-          git clean -fxd
-          mkdir archive
-          git archive -o archive/pydra.zip HEAD
-      - uses: actions/upload-artifact@v4
-        with:
-          name: archive
-          path: archive/
+          attest-build-provenance-github: ${{ github.event_name != 'pull_request' }}
 
   test:
-    needs: ['build']
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ['3.11', '3.12', '3.13']
+        dependencies: [latest, pre]
+        include:
+          # Test minimum dependencies on oldest supported Python
+          - os: ubuntu-latest
+            python-version: "3.11"
+            dependencies: min
       fail-fast: false
-    runs-on: ${{ matrix.os }}
+
+    env:
+      DEPENDS: ${{ matrix.dependencies }}
+
     steps:
       - name: Fetch repository
         uses: actions/checkout@v4
-      - name: Fetch tags
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v5
       - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Update pip
-        run: python -m pip install --upgrade pip
-      - name: Install Pydra
-        run: pip install .[test]
-      - name: Print version
-        run: python -c "import pydra.engine; print(pydra.utils.__version__)"
-      - name: Disable etelemetry
-        run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
-      - name: Pytest
+      - name: Install tox
         run: |
-          pytest -vs -n auto pydra --doctest-modules --import-mode=importlib --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --rootdir pydra
+          uv tool install tox --with=tox-uv --with=tox-gh-actions
+      - name: Show tox config
+        run: tox c
+      - name: Run tox
+        run: tox -v --exit-and-dump-after 1200
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
   test-singularity:
-    needs: ['build']
     runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: ['3.11', '3.12', '3.13']
+        dependencies: [latest]
       fail-fast: False
+
+    env:
+      DEPENDS: ${{ matrix.dependencies }}
+
     steps:
       - name: Set env
         run: |
@@ -118,30 +122,32 @@ jobs:
         run: |
           echo ${{ github.ref }}
           singularity --version
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v5
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Update build tools
-        run: python -m pip install --upgrade pip
-      - name: Checkout Pydra repo
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}
-      - name: Fetch tags
-        run: git fetch --prune --unshallow
-      - name: Install pydra (test)
-        run: pip install -e ".[test]"
-      - name: Pytest
-        run: pytest -vs --import-mode=importlib --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/environments/tests/test_singularity.py pydra/environments/tests/test_environments.py --rootdir pydra
+      - name: Install tox
+        run: |
+          uv tool install tox --with=tox-uv --with=tox-gh-actions
+      - name: Show tox config
+        run: tox c
+      - name: Run tox
+        # Run test files with singularity tests; re-add the overridable "-n auto"
+        run: |
+          tox -v --exit-and-dump-after 1200 -- -n auto \
+            pydra/environments/tests/test_singularity.py pydra/environments/tests/test_environments.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
   test-slurm:
-    needs: ['build']
     strategy:
       matrix:
         python-version: [3.11.5]
@@ -153,8 +159,8 @@ jobs:
     - name: Disable etelemetry
       run: echo "NO_ET=TRUE" >> $GITHUB_ENV
     - uses: actions/checkout@v4
-    - name: Fetch tags
-      run: git fetch --prune --unshallow
+      with:
+        fetch-depth: 0
     - name: Pull docker image
       run: |
         docker pull $DOCKER_IMAGE
@@ -174,20 +180,17 @@ jobs:
         docker exec slurm bash -c "echo $NO_ET"
         docker exec slurm bash -c "ls -la && echo list top level dir"
         docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
-        if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then
-            docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
-        fi
+        docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v ${{ matrix.python-version }}"
         docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
         docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.utils.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest /pydra/pydra/workers/tests/test_worker.py --import-mode=importlib --rootdir /pydra/pydra --only-worker=slurm --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
+        docker exec slurm bash -c "cd /pydra; pytest pydra/workers/tests/test_worker.py --only-worker=slurm --color=yes"
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v5
       with:
         fail_ci_if_error: true
         token: ${{ secrets.CODECOV_TOKEN }}
-        files: ./cov.xml
 
   # test-sge:
   #   needs: ['build']
@@ -270,7 +273,6 @@ jobs:
   #       files: ./cov.xml
 
   build-docs:
-    needs: ['build']
     runs-on: ubuntu-latest
     # Set up the environment so that it finds conda
     defaults:
@@ -306,8 +308,8 @@ jobs:
         dbus-monitor --session &
         sleep 3
     - uses: actions/checkout@v4
-    - name: Fetch tags
-      run: git fetch --prune --unshallow
+      with:
+        fetch-depth: 0
     - name: Install Minconda
       uses: conda-incubator/setup-miniconda@v3
       with:
@@ -321,63 +323,44 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build twine
     - name: Install package
       run: pip install .[doc]
     - name: Install Python3 kernel
       run: python -m ipykernel install --user
     - name: Build docs
-      run: |
-        cd docs
-        make html
-        cd ..
+      run: make -C docs html
     - uses: actions/upload-artifact@v4
       with:
         name: docs
         path: docs/build/html
 
   deploy:
-    needs: [build-docs, test, test-singularity, test-slurm]
+    needs: [build, build-docs, test, test-singularity, test-slurm]
     runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    permissions:
+      attestations: write
+      id-token: write
     steps:
     - name: Download dist
       uses: actions/download-artifact@v4
       with:
-        name: dist
+        name: Packages
         path: dist
-    - name: Check for PyPI token on tag
-      id: deployable
-      if: github.event_name == 'release'
-      env:
-        PYPI_API_TOKEN: "${{ secrets.PYPI_API_TOKEN }}"
-      run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi
     - name: Upload to PyPI
-      if: steps.deployable.outputs.DEPLOY
       uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
 
   deploy-docs:
     needs: [build-docs, deploy]
     runs-on: ubuntu-latest
+    if: github.event_name == 'release'
     steps:
     - name: Download docs
       uses: actions/download-artifact@v4
       with:
         name: docs
         path: docs-build
-    - name: Check for GHPAGES_DEPLOY_KEY token
-      id: deployable
-      if: github.event_name == 'release'
-      env:
-        GHPAGES_DEPLOY_KEY: "${{ secrets.GHPAGES_DEPLOY_KEY }}"
-      run: if [ -n "$GHPAGES_DEPLOY_KEY" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi
     - name: Deploy Docs to GitHub Pages
-      if: steps.deployable.outputs.DEPLOY
       uses: peaceiris/actions-gh-pages@v4
       with:
         github_token: ${{ secrets.GHPAGES_DEPLOY_KEY }}

.github/workflows/ci-cd.yml

@@ -4,7 +4,7 @@ on:
   # ATM, this is the closest trigger to a PR merging
   push:
     branches:
-      - master
+      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/release.yml

@@ -10,8 +10,7 @@ __pycache__
 .vscode/
 
 .coverage*
-!.coveragerc
-cov.xml
+coverage.xml
 
 .*.swp
 *~
@@ -21,5 +20,7 @@ cov.xml
 .DS_Store
 .ipynb_checkpoints
 
+.tox
+
 # This can be generated in-tree. We never want to commit it.
 pydra/utils/_version.py

.gitignore

@@ -73,21 +73,20 @@ which can be installed similarly
 Developer installation
 ======================
 
-Pydra requires Python 3.11+. To install in developer mode:
-
-::
+Pydra requires Python 3.11+. To install in developer mode::
 
     git clone git@github.com:nipype/pydra.git
     cd pydra
     pip install -e ".[dev]"
 
+In order to run pydra's test locally::
 
-In order to run pydra's test locally:
-
-::
+    pytest pydra
 
-    pytest -vs pydra
+We use `tox <https://tox.wiki/>`_ to test versions and dependency sets.
+For example, to test on the minimum and latest dependencies, run::
 
+    tox -e py311-min -e py313-latest
 
 It is also useful to install pre-commit:
 

README.rst

@@ -2250,13 +2250,6 @@ class Outputs(shell.Outputs):
     assert outputs.out1.fspath.exists()
 
 
-@pytest.mark.xfail(
-    sys.version_info >= (3, 11),
-    reason=(
-        "Fails on Python 3.11 in some cases (presumably a typing thing with that specific "
-        "version of Python)"
-    ),
-)
 @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter])
 def test_shell_cmd_outputspec_7(tmp_path, worker, results_function):
     """

pydra/compose/shell/tests/test_shell_run.py

@@ -237,7 +237,9 @@ def copyfile_workflow(
         value = getattr(outputs, field.name)
         # if the field is a path or it can contain a path _copyfile_single_value is run
         # to move all files and directories to the workflow directory
-        new_value = copy_nested_files(value, wf_path, mode=FileSet.CopyMode.hardlink)
+        new_value = copy_nested_files(
+            value, wf_path, mode=FileSet.CopyMode.hardlink_or_copy
+        )
         setattr(outputs, field.name, new_value)
     return outputs