From 273ad147c881e29ee6656cd4da22bcfbf76c0408 Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Thu, 21 Mar 2024 19:28:21 +0100 Subject: [PATCH] Bump dependencies (#6312) --- .github/workflows/run_pip_compile.yaml | 23 +++- .../workflows/scripts/compile_requirements.py | 23 +++- .../workflows/scripts/merge_requirements.py | 121 ++++++++++++++---- requirements/base.in | 1 - requirements/base.txt | 66 ++++------ requirements/extra-doc.txt | 26 +++- requirements/extra-postgres.txt | 6 +- requirements/extra-style.txt | 4 +- requirements/extra-test.in | 4 +- requirements/extra-test.txt | 28 ++-- 10 files changed, 199 insertions(+), 103 deletions(-) diff --git a/.github/workflows/run_pip_compile.yaml b/.github/workflows/run_pip_compile.yaml index 47a47083f..2c7dd856c 100644 --- a/.github/workflows/run_pip_compile.yaml +++ b/.github/workflows/run_pip_compile.yaml @@ -20,12 +20,27 @@ jobs: - name: Set up Python 3.8. uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: | + 3.11 + 3.10 + 3.9 + 3.8 - - name: Install dependencies + - name: Install dependencies on Linux/macOS + if: matrix.os != 'windows-latest' run: | - python -m pip install -U pip - python -m pip install -U pip-tools + python3.11 -m pip install -U pip pip-tools + python3.10 -m pip install -U pip pip-tools + python3.9 -m pip install -U pip pip-tools + python3.8 -m pip install -U pip pip-tools + + - name: Install dependencies on Windows + if: matrix.os == 'windows-latest' + run: | + py -3.11 -m pip install -U pip pip-tools + py -3.10 -m pip install -U pip pip-tools + py -3.9 -m pip install -U pip pip-tools + py -3.8 -m pip install -U pip pip-tools - name: Generate requirements files. id: compile_requirements diff --git a/.github/workflows/scripts/compile_requirements.py b/.github/workflows/scripts/compile_requirements.py index 83827e4c2..903f436ff 100644 --- a/.github/workflows/scripts/compile_requirements.py +++ b/.github/workflows/scripts/compile_requirements.py @@ -1,19 +1,26 @@ import os +import re import shutil import subprocess import sys from pathlib import Path +EXCLUDE_STEM_RE = re.compile(r".*-3\.(?!8-)(\d+)-extra-(doc|style)") GITHUB_OUTPUT = os.environ["GITHUB_OUTPUT"] REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements" os.chdir(REQUIREMENTS_FOLDER) -def pip_compile(name: str) -> None: +def pip_compile(version: str, name: str) -> None: + stem = f"{sys.platform}-{version}-{name}" + if EXCLUDE_STEM_RE.fullmatch(stem): + return + + executable = ("py", f"-{version}") if sys.platform == "win32" else (f"python{version}",) subprocess.check_call( ( - sys.executable, + *executable, "-m", "piptools", "compile", @@ -22,15 +29,17 @@ def pip_compile(name: str) -> None: "--verbose", f"{name}.in", "--output-file", - f"{sys.platform}-{name}.txt", + f"{stem}.txt", ) ) -pip_compile("base") -shutil.copyfile(f"{sys.platform}-base.txt", "base.txt") -for file in REQUIREMENTS_FOLDER.glob("extra-*.in"): - pip_compile(file.stem) +for minor in range(8, 11 + 1): + version = f"3.{minor}" + pip_compile(version, "base") + shutil.copyfile(f"{sys.platform}-{version}-base.txt", "base.txt") + for file in REQUIREMENTS_FOLDER.glob("extra-*.in"): + pip_compile(version, file.stem) with open(GITHUB_OUTPUT, "a", encoding="utf-8") as fp: fp.write(f"sys_platform={sys.platform}\n") diff --git a/.github/workflows/scripts/merge_requirements.py b/.github/workflows/scripts/merge_requirements.py index 4ad2822b2..01cbb13ba 100644 --- a/.github/workflows/scripts/merge_requirements.py +++ b/.github/workflows/scripts/merge_requirements.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import os from pathlib import Path -from typing import List, TextIO +from typing import Dict, Iterable, List, TextIO, Tuple from packaging.markers import Marker from packaging.requirements import Requirement @@ -15,6 +17,12 @@ class RequirementData: self.req = Requirement(requirement_string) self.comments = set() + def __hash__(self) -> int: + return hash(self.req) + + def __eq__(self, other: RequirementData) -> bool: + return self.req == other.req + @property def name(self) -> str: return self.req.name @@ -49,30 +57,47 @@ def get_requirements(fp: TextIO) -> List[RequirementData]: return requirements +def iter_envs(envs: Iterable[str]) -> Iterable[Tuple[str, str]]: + for env_name in envs: + platform, python_version = env_name.split("-", maxsplit=1) + yield (platform, python_version) + + names = ["base"] names.extend(file.stem for file in REQUIREMENTS_FOLDER.glob("extra-*.in")) -base_requirements = [] +base_requirements: List[RequirementData] = [] for name in names: - # {req_name: {sys_platform: RequirementData} - input_data = {} + # {req_data: {sys_platform: RequirementData} + input_data: Dict[RequirementData, Dict[str, RequirementData]] = {} + all_envs = set() all_platforms = set() + all_python_versions = set() for file in REQUIREMENTS_FOLDER.glob(f"*-{name}.txt"): - platform_name = file.stem.split("-", maxsplit=1)[0] + platform_name, python_version, _ = file.stem.split("-", maxsplit=2) + env_name = f"{platform_name}-{python_version}" + all_envs.add(env_name) all_platforms.add(platform_name) + all_python_versions.add(python_version) with file.open(encoding="utf-8") as fp: requirements = get_requirements(fp) for req in requirements: - platforms = input_data.setdefault(req.name, {}) - platforms[platform_name] = req + envs = input_data.setdefault(req, {}) + envs[env_name] = req output = base_requirements if name == "base" else [] - for req_name, platforms in input_data.items(): - req = next(iter(platforms.values())) - for other_req in platforms.values(): - if req.req != other_req.req: - raise RuntimeError(f"Incompatible requirements for {req_name}.") + for req, envs in input_data.items(): + # {platform: [python_versions...]} + python_versions_per_platform: Dict[str, List[str]] = {} + # {python_version: [platforms...]} + platforms_per_python_version: Dict[str, List[str]] = {} + platforms = python_versions_per_platform.keys() + python_versions = platforms_per_python_version.keys() + for env_name, other_req in envs.items(): + platform_name, python_version = env_name.split("-", maxsplit=1) + python_versions_per_platform.setdefault(platform_name, []).append(python_version) + platforms_per_python_version.setdefault(python_version, []).append(platform_name) req.comments.update(other_req.comments) @@ -84,30 +109,74 @@ for name in names: old_req_marker = req.marker req.marker = base_req.marker = None if base_req.req != req.req: - raise RuntimeError(f"Incompatible requirements for {req_name}.") + raise RuntimeError(f"Incompatible requirements for {req.name}.") base_req.marker = old_base_marker req.marker = old_req_marker if base_req.marker is None or base_req.marker == req.marker: continue - if len(platforms) == len(all_platforms): + if len(envs) == len(all_envs): output.append(req) continue - elif len(platforms) < len(all_platforms - platforms.keys()): - platform_marker = " or ".join( - f"sys_platform == '{platform}'" for platform in platforms - ) - else: - platform_marker = " and ".join( - f"sys_platform != '{platform}'" for platform in all_platforms - platforms.keys() + + # At this point I'm wondering why I didn't just go for + # a more generic boolean algebra simplification (sympy.simplify_logic())... + if ( + len(set(map(frozenset, python_versions_per_platform.values()))) == 1 + or len(set(map(frozenset, platforms_per_python_version.values()))) == 1 + ): + # Either all platforms have the same Python version set + # or all Python versions have the same platform set. + # We can generate markers for platform (platform_marker) and Python + # (python_version_marker) version sets separately and then simply require + # that both markers are fulfilled at the same time (env_marker). + + python_version_marker = ( + # Requirement present on less Python versions than not. + " or ".join( + f"python_version == '{python_version}'" for python_version in python_versions + ) + if len(python_versions) < len(all_python_versions - python_versions) + # Requirement present on more Python versions than not + # This may generate an empty string when Python version is irrelevant. + else " and ".join( + f"python_version != '{python_version}'" + for python_version in all_python_versions - python_versions + ) ) - new_marker = ( - f"({req.marker}) and ({platform_marker})" - if req.marker is not None - else platform_marker - ) + platform_marker = ( + # Requirement present on less platforms than not. + " or ".join(f"sys_platform == '{platform}'" for platform in platforms) + if len(platforms) < len(all_platforms - platforms) + # Requirement present on more platforms than not + # This may generate an empty string when platform is irrelevant. + else " and ".join( + f"sys_platform != '{platform}'" for platform in all_platforms - platforms + ) + ) + + if python_version_marker and platform_marker: + env_marker = f"({python_version_marker}) and ({platform_marker})" + else: + env_marker = python_version_marker or platform_marker + else: + # Fallback to generic case. + env_marker = ( + # Requirement present on less envs than not. + " or ".join( + f"(sys_platform == '{platform}' and python_version == '{python_version}')" + for platform, python_version in iter_envs(envs) + ) + if len(envs) < len(all_envs - envs.keys()) + else " and ".join( + f"(sys_platform != '{platform}' and python_version != '{python_version}')" + for platform, python_version in iter_envs(all_envs - envs.keys()) + ) + ) + + new_marker = f"({req.marker}) and ({env_marker})" if req.marker is not None else env_marker req.marker = Marker(new_marker) if base_req is not None and base_req.marker == req.marker: continue diff --git a/requirements/base.in b/requirements/base.in index cca18ecb7..4a16b9b73 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,4 +1,3 @@ -aiodns aiohttp aiohttp-json-rpc apsw diff --git a/requirements/base.txt b/requirements/base.txt index 34d7aca98..30c95d2df 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,4 @@ -aiodns==3.0.0 - # via -r base.in -aiohttp==3.8.5 +aiohttp==3.9.3 # via # -r base.in # aiohttp-json-rpc @@ -10,20 +8,14 @@ aiohttp-json-rpc==0.13.3 # via -r base.in aiosignal==1.3.1 # via aiohttp -apsw==3.43.1.0 +apsw==3.45.2.0 # via -r base.in -async-timeout==4.0.3 +attrs==23.2.0 # via aiohttp -attrs==23.1.0 - # via aiohttp -babel==2.12.1 +babel==2.14.0 # via -r base.in brotli==1.1.0 # via -r base.in -cffi==1.15.1 - # via pycares -charset-normalizer==3.2.0 - # via aiohttp click==8.1.7 # via -r base.in contextlib2==21.6.0 @@ -32,45 +24,37 @@ discord-py==2.3.2 # via # -r base.in # red-lavalink -frozenlist==1.4.0 +frozenlist==1.4.1 # via # aiohttp # aiosignal -idna==3.4 +idna==3.6 # via yarl -importlib-metadata==6.8.0 - # via markdown -markdown==3.4.4 +markdown==3.6 # via -r base.in markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -multidict==6.0.4 +multidict==6.0.5 # via # aiohttp # yarl -orjson==3.9.7 +orjson==3.9.15 # via -r base.in -packaging==23.1 +packaging==24.0 # via -r base.in -platformdirs==3.10.0 +platformdirs==4.2.0 # via -r base.in -psutil==5.9.5 +psutil==5.9.8 # via -r base.in -pycares==4.3.0 - # via aiodns -pycparser==2.21 - # via cffi -pygments==2.16.1 +pygments==2.17.2 # via rich -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via -r base.in -pytz==2023.3.post1 - # via babel pyyaml==6.0.1 # via -r base.in -rapidfuzz==3.3.0 +rapidfuzz==3.6.2 # via -r base.in red-commons==1.0.0 # via @@ -78,25 +62,31 @@ red-commons==1.0.0 # red-lavalink red-lavalink==0.11.0 # via -r base.in -rich==13.5.2 +rich==13.7.1 # via -r base.in schema==0.7.5 # via -r base.in six==1.16.0 # via python-dateutil -typing-extensions==4.7.1 +typing-extensions==4.10.0 # via # -r base.in # rich -yarl==1.9.2 +yarl==1.9.4 # via # -r base.in # aiohttp -zipp==3.16.2 - # via importlib-metadata +async-timeout==4.0.3; python_version != "3.11" + # via aiohttp colorama==0.4.6; sys_platform == "win32" # via click -distro==1.8.0; sys_platform == "linux" and sys_platform == "linux" +distro==1.9.0; sys_platform == "linux" and sys_platform == "linux" # via -r base.in -uvloop==0.17.0; (sys_platform != "win32" and platform_python_implementation == "CPython") and sys_platform != "win32" +importlib-metadata==7.1.0; python_version != "3.10" and python_version != "3.11" + # via markdown +pytz==2024.1; python_version == "3.8" + # via babel +uvloop==0.19.0; (sys_platform != "win32" and platform_python_implementation == "CPython") and sys_platform != "win32" # via -r base.in +zipp==3.18.1; python_version != "3.10" and python_version != "3.11" + # via importlib-metadata diff --git a/requirements/extra-doc.txt b/requirements/extra-doc.txt index 8242bb651..2a1def346 100644 --- a/requirements/extra-doc.txt +++ b/requirements/extra-doc.txt @@ -1,18 +1,28 @@ alabaster==0.7.13 # via sphinx -certifi==2023.7.22 +certifi==2024.2.2 # via requests -docutils==0.18.1 +charset-normalizer==3.3.2 + # via requests +docutils==0.20.1 # via # sphinx # sphinx-prompt # sphinx-rtd-theme imagesize==1.4.1 # via sphinx -jinja2==3.1.2 +importlib-metadata==7.1.0 + # via + # -c base.txt + # sphinx +jinja2==3.1.3 # via sphinx -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 +pytz==2024.1 + # via + # -c base.txt + # babel requests==2.31.0 # via sphinx snowballstemmer==2.2.0 @@ -26,7 +36,7 @@ sphinx==7.1.2 # sphinxcontrib-trio sphinx-prompt==1.7.0 # via -r extra-doc.in -sphinx-rtd-theme==1.3.0 +sphinx-rtd-theme==2.0.0 # via -r extra-doc.in sphinxcontrib-applehelp==1.0.4 # via sphinx @@ -44,5 +54,9 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r extra-doc.in -urllib3==2.0.4 +urllib3==2.2.1 # via requests +zipp==3.18.1 + # via + # -c base.txt + # importlib-metadata diff --git a/requirements/extra-postgres.txt b/requirements/extra-postgres.txt index 43938c530..294e85ff1 100644 --- a/requirements/extra-postgres.txt +++ b/requirements/extra-postgres.txt @@ -1,2 +1,6 @@ -asyncpg==0.28.0 +async-timeout==4.0.3 + # via + # -c base.txt + # asyncpg +asyncpg==0.29.0 # via -r extra-postgres.in diff --git a/requirements/extra-style.txt b/requirements/extra-style.txt index 18df03d5e..3d143da21 100644 --- a/requirements/extra-style.txt +++ b/requirements/extra-style.txt @@ -1,8 +1,8 @@ -black==23.9.1 +black==23.12.1 # via -r extra-style.in mypy-extensions==1.0.0 # via black -pathspec==0.11.2 +pathspec==0.12.1 # via black tomli==2.0.1 # via black diff --git a/requirements/extra-test.in b/requirements/extra-test.in index b726962fd..226c71b33 100644 --- a/requirements/extra-test.in +++ b/requirements/extra-test.in @@ -1,6 +1,6 @@ -c base.txt pylint -pytest -pytest-asyncio +pytest<8 +pytest-asyncio<0.22 pytest-mock diff --git a/requirements/extra-test.txt b/requirements/extra-test.txt index b45bd0cd5..33cbc4aee 100644 --- a/requirements/extra-test.txt +++ b/requirements/extra-test.txt @@ -1,35 +1,31 @@ -astroid==2.15.6 +astroid==3.1.0 # via pylint -dill==0.3.7 +dill==0.3.8 # via pylint -exceptiongroup==1.1.3 - # via pytest iniconfig==2.0.0 # via pytest -isort==5.12.0 +isort==5.13.2 # via pylint -lazy-object-proxy==1.9.0 - # via astroid mccabe==0.7.0 # via pylint -pluggy==1.3.0 +pluggy==1.4.0 # via pytest -pylint==2.17.5 +pylint==3.1.0 # via -r extra-test.in -pytest==7.4.2 +pytest==7.4.4 # via # -r extra-test.in # pytest-asyncio # pytest-mock pytest-asyncio==0.21.1 # via -r extra-test.in -pytest-mock==3.11.1 +pytest-mock==3.12.0 # via -r extra-test.in -tomli==2.0.1 +tomlkit==0.12.4 + # via pylint +exceptiongroup==1.2.0; python_version != "3.11" + # via pytest +tomli==2.0.1; python_version != "3.11" # via # pylint # pytest -tomlkit==0.12.1 - # via pylint -wrapt==1.15.0 - # via astroid