diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 10ba5faa6a..8d8aa55145 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.12] + python-version: [3.13] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cf0579a7e5..7f81361446 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -31,10 +31,10 @@ jobs: python-version: 3.12 - name: Install pypa/build and twine - run: python -m pip install --user build twine + run: python -m pip install --user --upgrade build twine pkginfo - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ + run: python -m build --sdist --outdir dist/ - name: Validate wheel and sdis for Pypi run: python -m twine check dist/* diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7a2d4d9b8c..4d347b7037 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: run_code_checks image_name: ubuntu-24.04 - python_versions: ['3.12'] + python_versions: ['3.13'] test_suites: all: make check @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: ubuntu24_cpython image_name: ubuntu-24.04 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,23 +51,15 @@ jobs: parameters: job_name: macos15_cpython image_name: macOS-15 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs - - template: etc/ci/azure-win.yml parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -75,6 +67,6 @@ jobs: parameters: job_name: win2025_cpython image_name: windows-2025 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index 5ef0e063b3..6d317d4c79 100755 --- a/configure +++ b/configure @@ -110,7 +110,7 @@ create_virtualenv() { fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ - --wheel embed --pip embed --setuptools embed \ + --pip embed --setuptools embed \ --seeder pip \ --never-download \ --no-periodic-update \ diff --git a/configure.bat b/configure.bat index 3e9881fb4f..15ab701548 100644 --- a/configure.bat +++ b/configure.bat @@ -110,7 +110,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml index 85b611d355..d80e8dfb81 100644 --- a/etc/ci/azure-container-deb.yml +++ b/etc/ci/azure-container-deb.yml @@ -21,7 +21,7 @@ jobs: - job: ${{ parameters.job_name }} pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-22.04' container: image: ${{ parameters.container }} diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml index 1e6657d0f2..a64138c9b8 100644 --- a/etc/ci/azure-container-rpm.yml +++ b/etc/ci/azure-container-rpm.yml @@ -1,6 +1,6 @@ parameters: job_name: '' - image_name: 'ubuntu-16.04' + image_name: 'ubuntu-22.04' container: '' python_path: '' python_version: '' diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index aafc1d69f0..6f812f090e 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -24,13 +25,14 @@ import packageurl import requests import saneyaml -import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version +import utils_pip_compatibility_tags + """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. @@ -91,8 +93,7 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as - {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: @@ -114,13 +115,14 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "37", "38", "39", "310" +PYTHON_VERSIONS = "39", "310", "311", "312", "313" PYTHON_DOT_VERSIONS_BY_VER = { - "37": "3.7", - "38": "3.8", "39": "3.9", "310": "3.10", + "311": "3.11", + "312": "3.12", + "313": "3.13", } @@ -132,10 +134,11 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "37": ["cp37", "cp37m", "abi3"], - "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], "310": ["cp310", "cp310m", "abi3"], + "311": ["cp311", "cp311m", "abi3"], + "312": ["cp312", "cp312m", "abi3"], + "313": ["cp313", "cp313m", "abi3"], } PLATFORMS_BY_OS = { @@ -553,8 +556,7 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - if not self.filename: - raise ValueError(f"self.filename has no value but is required: {self.filename!r}") + assert self.filename if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -822,9 +824,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [eln for eln in extra_lic_names if eln] + extra_lic_names = [ln for ln in extra_lic_names if ln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -844,7 +846,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except Exception: + except: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -857,9 +859,8 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except Exception: - msg = f"No text for license {filename} in expression " - f"{self.license_expression!r} from {self}" + except: + msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' print(msg) errors.append(msg) @@ -999,7 +1000,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [url for url in urls if url.endswith(f"/{filename}")] + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1288,7 +1289,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except Exception: + except: return False @@ -1484,7 +1485,8 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - yield from self.wheels + for wheel in self.wheels: + yield wheel def get_url_for_filename(self, filename): """ @@ -1613,8 +1615,7 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " - "available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" ), ) @@ -1628,8 +1629,7 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. " - "Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." ), ) @@ -1638,8 +1638,7 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - if not name: - raise ValueError(f"name is required: {name!r}") + assert name normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1694,7 +1693,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [link.partition("#sha256=") for link in links] + links = [l.partition("#sha256=") for l in links] links = [url for url, _, _sha256 in links] return links @@ -1915,7 +1914,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2134,7 +2133,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: stdouts = [] @@ -2199,7 +2198,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print("Downloading wheels using command:", " ".join(cli_args)) + print(f"Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2232,7 +2231,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2283,5 +2282,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(lic).lower() for lic in declared_licenses] + lics = [python_safe_name(l).lower() for l in declared_licenses] return " AND ".join(lics).lower()