Modernize packaging-related things in Red (#5924)

.github/workflows/codeql-analysis.yml

@@ -26,7 +26,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install -U pip setuptools wheel
+        python -m pip install -U pip wheel
         python -m pip install -e .[all]
         # Set the `CODEQL-PYTHON` environment variable to the Python executable
         # that includes the dependencies

.github/workflows/run_pip_compile.yaml

@@ -0,0 +1,84 @@
+name: Generate requirements files with pip-compile.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  generate_requirements:
+    name: Generate requirements files for ${{ matrix.os }} platform.
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    steps:
+      - name: Checkout the repository.
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8.
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install -U pip-tools
+
+      - name: Generate requirements files.
+        id: compile_requirements
+        run: |
+          python .github/workflows/scripts/compile_requirements.py
+
+      - name: Upload requirements files.
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.compile_requirements.outputs.sys_platform }}
+          path: requirements/${{ steps.compile_requirements.outputs.sys_platform }}-*.txt
+
+  merge_requirements:
+    name: Merge requirements files.
+    needs: generate_requirements
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository.
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8.
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U "packaging>=22.0"
+
+      - name: Download Windows requirements.
+        uses: actions/download-artifact@v3
+        with:
+          name: win32
+          path: requirements
+      - name: Download Linux requirements.
+        uses: actions/download-artifact@v3
+        with:
+          name: linux
+          path: requirements
+      - name: Download macOS requirements.
+        uses: actions/download-artifact@v3
+        with:
+          name: darwin
+          path: requirements
+
+      - name: Merge requirements files.
+        run: |
+          python .github/workflows/scripts/merge_requirements.py
+
+      - name: Upload merged requirements files.
+        uses: actions/upload-artifact@v3
+        with:
+          name: merged
+          path: |
+            requirements/base.txt
+            requirements/extra-*.txt

.github/workflows/scripts/compile_requirements.py

@@ -0,0 +1,35 @@
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+GITHUB_OUTPUT = os.environ["GITHUB_OUTPUT"]
+REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements"
+os.chdir(REQUIREMENTS_FOLDER)
+
+
+def pip_compile(name: str) -> None:
+    subprocess.check_call(
+        (
+            sys.executable,
+            "-m",
+            "piptools",
+            "compile",
+            "--upgrade",
+            "--verbose",
+            f"{name}.in",
+            "--output-file",
+            f"{sys.platform}-{name}.txt",
+        )
+    )
+
+
+pip_compile("base")
+shutil.copyfile(f"{sys.platform}-base.txt", "base.txt")
+for file in REQUIREMENTS_FOLDER.glob("extra-*.in"):
+    pip_compile(file.stem)
+
+with open(GITHUB_OUTPUT, "a", encoding="utf-8") as fp:
+    fp.write(f"sys_platform={sys.platform}\n")

.github/workflows/scripts/merge_requirements.py

@@ -0,0 +1,134 @@
+import os
+from pathlib import Path
+from typing import List, TextIO
+
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+
+
+REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements"
+os.chdir(REQUIREMENTS_FOLDER)
+
+
+class RequirementData:
+    def __init__(self, requirement_string: str) -> None:
+        self.req = Requirement(requirement_string)
+        self.comments = set()
+
+    @property
+    def name(self) -> str:
+        return self.req.name
+
+    @property
+    def marker(self) -> Marker:
+        return self.req.marker
+
+    @marker.setter
+    def marker(self, value: Marker) -> None:
+        self.req.marker = value
+
+
+def get_requirements(fp: TextIO) -> List[RequirementData]:
+    requirements = []
+
+    current = None
+    for line in fp.read().splitlines():
+        annotation_prefix = "    # "
+        if line.startswith(annotation_prefix) and current is not None:
+            source = line[len(annotation_prefix) :].strip()
+            if source == "via":
+                continue
+            via_prefix = "via "
+            if source.startswith(via_prefix):
+                source = source[len(via_prefix) :]
+            current.comments.add(source)
+        elif line and not line.startswith(("#", " ")):
+            current = RequirementData(line)
+            requirements.append(current)
+
+    return requirements
+
+
+names = ["base"]
+names.extend(file.stem for file in REQUIREMENTS_FOLDER.glob("extra-*.in"))
+base_requirements = []
+
+for name in names:
+    # {req_name: {sys_platform: RequirementData}
+    input_data = {}
+    all_platforms = set()
+    for file in REQUIREMENTS_FOLDER.glob(f"*-{name}.txt"):
+        platform_name = file.stem.split("-", maxsplit=1)[0]
+        all_platforms.add(platform_name)
+        with file.open(encoding="utf-8") as fp:
+            requirements = get_requirements(fp)
+
+        for req in requirements:
+            platforms = input_data.setdefault(req.name, {})
+            platforms[platform_name] = req
+
+    output = base_requirements if name == "base" else []
+    for req_name, platforms in input_data.items():
+        req = next(iter(platforms.values()))
+        for other_req in platforms.values():
+            if req.req != other_req.req:
+                raise RuntimeError(f"Incompatible requirements for {req_name}.")
+
+            req.comments.update(other_req.comments)
+
+        base_req = next(
+            (base_req for base_req in base_requirements if base_req.name == req.name), None
+        )
+        if base_req is not None:
+            old_base_marker = base_req.marker
+            old_req_marker = req.marker
+            req.marker = base_req.marker = None
+            if base_req.req != req.req:
+                raise RuntimeError(f"Incompatible requirements for {req_name}.")
+
+            base_req.marker = old_base_marker
+            req.marker = old_req_marker
+            if base_req.marker is None or base_req.marker == req.marker:
+                continue
+
+        if len(platforms) == len(all_platforms):
+            output.append(req)
+            continue
+        elif len(platforms) < len(all_platforms - platforms.keys()):
+            platform_marker = " or ".join(
+                f"sys_platform == '{platform}'" for platform in platforms
+            )
+        else:
+            platform_marker = " and ".join(
+                f"sys_platform != '{platform}'" for platform in all_platforms - platforms.keys()
+            )
+
+        new_marker = (
+            f"({req.marker}) and ({platform_marker})"
+            if req.marker is not None
+            else platform_marker
+        )
+        req.marker = Marker(new_marker)
+        if base_req is not None and base_req.marker == req.marker:
+            continue
+
+        output.append(req)
+
+    output.sort(key=lambda req: (req.marker is not None, req.name))
+    with open(f"{name}.txt", "w+", encoding="utf-8") as fp:
+        for req in output:
+            fp.write(str(req.req))
+            fp.write("\n")
+            comments = sorted(req.comments)
+
+            if len(comments) == 1:
+                source = comments[0]
+                fp.write("    # via ")
+                fp.write(source)
+                fp.write("\n")
+            else:
+                fp.write("    # via\n")
+                for source in comments:
+                    fp.write("    #   ")
+                    fp.write(source)
+                    fp.write("\n")

.github/workflows/tests.yml

@@ -46,7 +46,7 @@ jobs:
         - name: Install tox
           run: |
             python -m pip install --upgrade pip
-            pip install tox
+            pip install 'tox<4'
         - name: Tox test
           env:
             TOXENV: ${{ matrix.tox_env }}
@@ -82,7 +82,7 @@ jobs:
         - name: Install tox
           run: |
             python -m pip install --upgrade pip
-            pip install tox
+            pip install 'tox<4'
         - name: Tox test
           env:
             TOXENV: postgres

.readthedocs.yml

@@ -9,4 +9,4 @@ python:
     - method: pip
       path: .
       extra_requirements:
-        - docs
+        - doc

MANIFEST.in

@@ -2,6 +2,12 @@
 include LICENSE
 recursive-include redbot *.LICENSE
 
+# include requirements files
+include requirements/base.in
+include requirements/base.txt
+include requirements/extra-*.in
+include requirements/extra-*.txt
+
 # include locale files
 recursive-include redbot locales/*.po
 

Makefile

@@ -52,7 +52,7 @@ bumpdeps:
 # Development environment
 newenv:
 	$(PYTHON) -m venv --clear .venv
-	.venv/bin/pip install -U pip setuptools wheel
+	.venv/bin/pip install -U pip wheel
 	$(MAKE) syncenv
 syncenv:
 	.venv/bin/pip install -Ur ./tools/dev-requirements.txt

docs/install_guides/_includes/install-and-setup-red-unix.rst

@@ -9,7 +9,7 @@ To install without additional config backend support:
 .. prompt:: bash
     :prompts: (redenv) $
 
-    python -m pip install -U pip setuptools wheel
+    python -m pip install -U pip wheel
     python -m pip install -U Red-DiscordBot
 
 Or, to install with PostgreSQL support:
@@ -17,7 +17,7 @@ Or, to install with PostgreSQL support:
 .. prompt:: bash
     :prompts: (redenv) $
 
-    python -m pip install -U pip setuptools wheel
+    python -m pip install -U pip wheel
     python -m pip install -U "Red-DiscordBot[postgres]"
 
 

docs/install_guides/windows.rst

@@ -134,7 +134,7 @@ Run **one** of the following set of commands, depending on what extras you want
     .. prompt:: batch
         :prompts: (redenv) C:\\>
 
-        python -m pip install -U pip setuptools wheel
+        python -m pip install -U pip wheel
         python -m pip install -U Red-DiscordBot
 
   * With PostgreSQL support:
@@ -142,7 +142,7 @@ Run **one** of the following set of commands, depending on what extras you want
     .. prompt:: batch
         :prompts: (redenv) C:\\>
 
-        python -m pip install -U pip setuptools wheel
+        python -m pip install -U pip wheel
         python -m pip install -U Red-DiscordBot[postgres]
 
 --------------------------

make.bat

@@ -24,7 +24,7 @@ goto:eof
 
 :newenv
 py -3.8 -m venv --clear .venv
-"%~dp0.venv\Scripts\python" -m pip install -U pip setuptools wheel
+"%~dp0.venv\Scripts\python" -m pip install -U pip wheel
 goto syncenv
 
 :syncenv

make.ps1

@@ -56,7 +56,7 @@ function stylediff() {
 
 function newenv() {
     py -3.8 -m venv --clear .venv
-    & $PSScriptRoot\.venv\Scripts\python.exe -m pip install -U pip setuptools
+    & $PSScriptRoot\.venv\Scripts\python.exe -m pip install -U pip wheel
     syncenv
 }
 

pyproject.toml

@@ -1,7 +1,45 @@
 [build-system]
-    requires = ["setuptools", "wheel"]
+requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "Red-DiscordBot"
+description = "A highly customisable Discord bot"
+readme = "README.md"
+authors = [{ name = "Cog Creators" }]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "Intended Audience :: End Users/Desktop",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Natural Language :: English",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Topic :: Communications :: Chat",
+]
+dynamic = ["version", "requires-python", "dependencies", "optional-dependencies"]
+
+[project.urls]
+"Homepage" = "https://github.com/Cog-Creators/Red-DiscordBot"
+"Discord Server" = "https://discord.gg/red"
+"Documentation" = "https://docs.discord.red"
+"Donate on Patreon" = "https://www.patreon.com/Red_Devs"
+"Issue Tracker" = "https://github.com/Cog-Creators/Red-DiscordBot/issues"
+"Source Code" = "https://github.com/Cog-Creators/Red-DiscordBot"
+
+[project.scripts]
+redbot = "redbot.__main__:main"
+redbot-setup = "redbot.setup:run_cli"
+redbot-launcher = "redbot.launcher:main"
+
+[project.entry-points.pytest11]
+red-discordbot = "redbot.pytest"
+
 [tool.black]
 line-length = 99
 required-version = '22.1.0'

redbot/__main__.py

@@ -10,7 +10,6 @@ import json
 import logging
 import os
 import pip
-import pkg_resources
 import platform
 import shutil
 import signal
@@ -335,6 +334,8 @@ async def run_bot(red: Red, cli_flags: Namespace) -> None:
         # `sys.path`, you must invoke the appropriate methods on the `working_set` instance
         # to keep it in sync."
         # Source: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#workingset-objects
+        pkg_resources = sys.modules.get("pkg_resources")
+        if pkg_resources is not None:
             pkg_resources.working_set.add_entry(str(LIB_PATH))
     sys.meta_path.insert(0, SharedLibImportWarner())
 

redbot/core/events.py

@@ -6,11 +6,12 @@ import codecs
 import logging
 import traceback
 from datetime import datetime, timedelta, timezone
+from typing import Tuple
 
 import aiohttp
 import discord
-import pkg_resources
-from pkg_resources import DistributionNotFound
+import importlib.metadata
+from packaging.requirements import Requirement
 from redbot.core import data_manager
 
 from redbot.core.commands import RedHelpFormatter, HelpSettings
@@ -54,6 +55,88 @@ ______         _           ______ _                       _  ______       _
 _ = Translator(__name__, __file__)
 
 
+def get_outdated_red_messages(pypi_version: str, py_version_req: str) -> Tuple[str, str]:
+    outdated_red_message = _(
+        "Your Red instance is out of date! {} is the current version, however you are using {}!"
+    ).format(pypi_version, red_version)
+    rich_outdated_message = (
+        f"[red]Outdated version![/red]\n"
+        f"[red]!!![/red]Version [cyan]{pypi_version}[/] is available, "
+        f"but you're using [cyan]{red_version}[/][red]!!![/red]"
+    )
+    current_python = platform.python_version()
+    extra_update = _(
+        "\n\nWhile the following command should work in most scenarios as it is "
+        "based on your current OS, environment, and Python version, "
+        "**we highly recommend you to read the update docs at <{docs}> and "
+        "make sure there is nothing else that "
+        "needs to be done during the update.**"
+    ).format(docs="https://docs.discord.red/en/stable/update_red.html")
+
+    if not expected_version(current_python, py_version_req):
+        extra_update += _(
+            "\n\nYou have Python `{py_version}` and this update "
+            "requires `{req_py}`; you cannot simply run the update command.\n\n"
+            "You will need to follow the update instructions in our docs above, "
+            "if you still need help updating after following the docs go to our "
+            "#support channel in <https://discord.gg/red>"
+        ).format(py_version=current_python, req_py=py_version_req)
+        outdated_red_message += extra_update
+        return outdated_red_message, rich_outdated_message
+
+    red_dist = importlib.metadata.distribution("Red-DiscordBot")
+    installed_extras = red_dist.metadata.get_all("Provides-Extra")
+    installed_extras.remove("dev")
+    installed_extras.remove("all")
+    distributions = {}
+    for req_str in red_dist.requires:
+        req = Requirement(req_str)
+        if req.marker is None or req.marker.evaluate():
+            continue
+        for extra in reversed(installed_extras):
+            if not req.marker.evaluate({"extra": extra}):
+                continue
+
+            # Check that the requirement is met.
+            # This is a bit simplified for our purposes and does not check
+            # whether the requirements of our requirements are met as well.
+            # This could potentially be an issue if we'll ever depend on
+            # a dependency's extra in our extra when we already depend on that
+            # in our base dependencies. However, considering that right now, all
+            # our dependencies are also fully pinned, this should not ever matter.
+            if req.name in distributions:
+                dist = distributions[req.name]
+            else:
+                try:
+                    dist = importlib.metadata.distribution(req.name)
+                except importlib.metadata.PackageNotFoundError:
+                    installed_extras.remove(extra)
+                    dist = None
+                distributions[req.name] = dist
+            if dist is None or not req.specifier.contains(dist.version, prereleases=True):
+                installed_extras.remove(extra)
+
+    if installed_extras:
+        package_extras = f"[{','.join(installed_extras)}]"
+    else:
+        package_extras = ""
+
+    extra_update += _(
+        "\n\nTo update your bot, first shutdown your bot"
+        " then open a window of {console} (Not as admin) and run the following:"
+        "{command_1}\n"
+        "Once you've started up your bot again, we recommend that"
+        " you update any installed 3rd-party cogs with this command in Discord:"
+        "{command_2}"
+    ).format(
+        console=_("Command Prompt") if platform.system() == "Windows" else _("Terminal"),
+        command_1=f'```"{sys.executable}" -m pip install -U "Red-DiscordBot{package_extras}"```',
+        command_2=f"```[p]cog update```",
+    )
+    outdated_red_message += extra_update
+    return outdated_red_message, rich_outdated_message
+
+
 def init_events(bot, cli_flags):
     @bot.event
     async def on_connect():
@@ -74,7 +157,6 @@ def init_events(bot, cli_flags):
 
         prefixes = cli_flags.prefix or (await bot._config.prefix())
         lang = await bot._config.locale()
-        red_pkg = pkg_resources.get_distribution("Red-DiscordBot")
         dpy_version = discord.__version__
 
         table_general_info = Table(show_edge=False, show_header=False, box=box.MINIMAL)
@@ -97,69 +179,9 @@ def init_events(bot, cli_flags):
             pypi_version, py_version_req = await fetch_latest_red_version_info()
             outdated = pypi_version and pypi_version > red_version_info
             if outdated:
-                outdated_red_message = _(
-                    "Your Red instance is out of date! {} is the current "
-                    "version, however you are using {}!"
-                ).format(pypi_version, red_version)
-                rich_outdated_message = (
-                    f"[red]Outdated version![/red]\n"
-                    f"[red]!!![/red]Version [cyan]{pypi_version}[/] is available, "
-                    f"but you're using [cyan]{red_version}[/][red]!!![/red]"
+                outdated_red_message, rich_outdated_message = get_outdated_red_messages(
+                    pypi_version, py_version_req
                 )
-                current_python = platform.python_version()
-                extra_update = _(
-                    "\n\nWhile the following command should work in most scenarios as it is "
-                    "based on your current OS, environment, and Python version, "
-                    "**we highly recommend you to read the update docs at <{docs}> and "
-                    "make sure there is nothing else that "
-                    "needs to be done during the update.**"
-                ).format(docs="https://docs.discord.red/en/stable/update_red.html")
-                if expected_version(current_python, py_version_req):
-                    installed_extras = []
-                    for extra, reqs in red_pkg._dep_map.items():
-                        if extra is None or extra in {"dev", "all"}:
-                            continue
-                        try:
-                            pkg_resources.require(req.name for req in reqs)
-                        except pkg_resources.DistributionNotFound:
-                            pass
-                        else:
-                            installed_extras.append(extra)
-
-                    if installed_extras:
-                        package_extras = f"[{','.join(installed_extras)}]"
-                    else:
-                        package_extras = ""
-
-                    extra_update += _(
-                        "\n\nTo update your bot, first shutdown your "
-                        "bot then open a window of {console} (Not as admin) and "
-                        "run the following:\n\n"
-                    ).format(
-                        console=_("Command Prompt")
-                        if platform.system() == "Windows"
-                        else _("Terminal")
-                    )
-                    extra_update += (
-                        '```"{python}" -m pip install -U Red-DiscordBot{package_extras}```'.format(
-                            python=sys.executable, package_extras=package_extras
-                        )
-                    )
-                    extra_update += _(
-                        "\nOnce you've started up your bot again, if you have any 3rd-party cogs"
-                        " installed we then highly recommend you update them with this command"
-                        " in Discord: `[p]cog update`"
-                    )
-
-                else:
-                    extra_update += _(
-                        "\n\nYou have Python `{py_version}` and this update "
-                        "requires `{req_py}`; you cannot simply run the update command.\n\n"
-                        "You will need to follow the update instructions in our docs above, "
-                        "if you still need help updating after following the docs go to our "
-                        "#support channel in <https://discord.gg/red>"
-                    ).format(py_version=current_python, req_py=py_version_req)
-                outdated_red_message += extra_update
 
         rich_console = rich.get_console()
         rich_console.print(INTRO, style="red", markup=False, highlight=False)

redbot/core/utils/_internal_utils.py

@@ -31,7 +31,7 @@ from typing import (
 
 import aiohttp
 import discord
-import pkg_resources
+from packaging.requirements import Requirement
 from fuzzywuzzy import fuzz, process
 from rich.progress import ProgressColumn
 from rich.progress_bar import ProgressBar
@@ -316,8 +316,8 @@ async def send_to_owners_with_prefix_replaced(bot: Red, content: str, **kwargs):
 
 
 def expected_version(current: str, expected: str) -> bool:
-    # `pkg_resources` needs a regular requirement string, so "x" serves as requirement's name here
-    return current in pkg_resources.Requirement.parse(f"x{expected}")
+    # Requirement needs a regular requirement string, so "x" serves as requirement's name here
+    return Requirement(f"x{expected}").specifier.contains(current, prereleases=True)
 
 
 async def fetch_latest_red_version_info() -> Tuple[Optional[VersionInfo], Optional[str]]:

redbot/launcher.py

@@ -12,7 +12,6 @@ import argparse
 import asyncio
 import aiohttp
 
-import pkg_resources
 from redbot import MIN_PYTHON_VERSION
 from redbot.setup import (
     basic_setup,

requirements/base.in

@@ -0,0 +1,23 @@
+aiohttp
+aiohttp-json-rpc
+aiosqlite
+appdirs
+apsw-wheels
+babel
+click
+colorama
+discord.py
+fuzzywuzzy
+markdown
+packaging
+psutil
+python-dateutil
+python-Levenshtein-wheels
+PyNaCl
+PyYAML
+Red-Commons
+Red-Lavalink>=0.11.0rc1
+rich
+schema
+distro; sys_platform == "linux"
+uvloop; sys_platform != "win32" and platform_python_implementation == "CPython"

requirements/base.txt

@@ -0,0 +1,88 @@
+aiohttp==3.7.4.post0
+    # via
+    #   -r base.in
+    #   aiohttp-json-rpc
+    #   discord-py
+    #   red-lavalink
+aiohttp-json-rpc==0.13.3
+    # via -r base.in
+aiosqlite==0.17.0
+    # via -r base.in
+appdirs==1.4.4
+    # via -r base.in
+apsw-wheels==3.36.0.post1
+    # via -r base.in
+async-timeout==3.0.1
+    # via aiohttp
+attrs==21.2.0
+    # via aiohttp
+babel==2.9.1
+    # via -r base.in
+cffi==1.14.6
+    # via pynacl
+chardet==4.0.0
+    # via aiohttp
+click==8.0.1
+    # via -r base.in
+colorama==0.4.4
+    # via
+    #   -r base.in
+    #   click
+commonmark==0.9.1
+    # via rich
+contextlib2==21.6.0
+    # via schema
+discord-py==2.1.0
+    # via
+    #   -r base.in
+    #   red-lavalink
+fuzzywuzzy==0.18.0
+    # via -r base.in
+idna==3.2
+    # via yarl
+markdown==3.3.4
+    # via -r base.in
+multidict==5.1.0
+    # via
+    #   aiohttp
+    #   yarl
+packaging==22.0
+    # via -r base.in
+psutil==5.8.0
+    # via -r base.in
+pycparser==2.20
+    # via cffi
+pygments==2.10.0
+    # via rich
+pynacl==1.4.0
+    # via -r base.in
+python-dateutil==2.8.2
+    # via -r base.in
+python-levenshtein-wheels==0.13.2
+    # via -r base.in
+pytz==2021.1
+    # via babel
+pyyaml==5.4.1
+    # via -r base.in
+red-commons==1.0.0
+    # via
+    #   -r base.in
+    #   red-lavalink
+red-lavalink==0.11.0rc1
+    # via -r base.in
+rich==10.9.0
+    # via -r base.in
+schema==0.7.4
+    # via -r base.in
+six==1.16.0
+    # via python-dateutil
+typing-extensions==3.10.0.2
+    # via rich
+yarl==1.6.3
+    # via
+    #   -r base.in
+    #   aiohttp
+distro==1.6.0; sys_platform == "linux"
+    # via -r base.in
+uvloop==0.16.0; sys_platform != "win32" and platform_python_implementation == "CPython"
+    # via -r base.in

requirements/extra-doc.in

@@ -0,0 +1,6 @@
+-c base.txt
+
+Sphinx
+sphinx-prompt
+sphinx_rtd_theme
+sphinxcontrib-trio

requirements/extra-doc.txt

@@ -0,0 +1,46 @@
+alabaster==0.7.12
+    # via sphinx
+certifi==2021.5.30
+    # via requests
+charset-normalizer==2.0.4
+    # via requests
+docutils==0.16
+    # via
+    #   sphinx
+    #   sphinx-rtd-theme
+imagesize==1.2.0
+    # via sphinx
+jinja2==3.0.1
+    # via sphinx
+markupsafe==2.0.1
+    # via jinja2
+requests==2.26.0
+    # via sphinx
+snowballstemmer==2.1.0
+    # via sphinx
+sphinx==4.1.2
+    # via
+    #   -r extra-doc.in
+    #   sphinx-prompt
+    #   sphinx-rtd-theme
+    #   sphinxcontrib-trio
+sphinx-prompt==1.5.0
+    # via -r extra-doc.in
+sphinx-rtd-theme==0.5.2
+    # via -r extra-doc.in
+sphinxcontrib-applehelp==1.0.2
+    # via sphinx
+sphinxcontrib-devhelp==1.0.2
+    # via sphinx
+sphinxcontrib-htmlhelp==2.0.0
+    # via sphinx
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.3
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.5
+    # via sphinx
+sphinxcontrib-trio==1.1.2
+    # via -r extra-doc.in
+urllib3==1.26.6
+    # via requests

requirements/extra-postgres.in

@@ -0,0 +1,3 @@
+-c base.txt
+
+asyncpg

requirements/extra-postgres.txt

@@ -0,0 +1,2 @@
+asyncpg==0.24.0
+    # via -r extra-postgres.in

requirements/extra-style.in

@@ -0,0 +1,3 @@
+-c base.txt
+
+black

requirements/extra-style.txt

@@ -0,0 +1,12 @@
+black==22.1.0
+    # via -r extra-style.in
+mypy-extensions==0.4.3
+    # via black
+pathspec==0.9.0
+    # via black
+regex==2021.8.28
+    # via black
+toml==0.10.2
+    # via black
+typed-ast==1.4.3
+    # via black

requirements/extra-test.in

@@ -0,0 +1,6 @@
+-c base.txt
+
+pylint
+pytest
+pytest-asyncio
+pytest-mock

requirements/extra-test.txt

@@ -0,0 +1,33 @@
+astroid==2.7.3
+    # via pylint
+iniconfig==1.1.1
+    # via pytest
+isort==5.9.3
+    # via pylint
+lazy-object-proxy==1.6.0
+    # via astroid
+mccabe==0.6.1
+    # via pylint
+platformdirs==2.3.0
+    # via pylint
+pluggy==1.0.0
+    # via pytest
+py==1.10.0
+    # via pytest
+pylint==2.10.2
+    # via -r extra-test.in
+pytest==6.2.5
+    # via
+    #   -r extra-test.in
+    #   pytest-asyncio
+    #   pytest-mock
+pytest-asyncio==0.15.1
+    # via -r extra-test.in
+pytest-mock==3.6.1
+    # via -r extra-test.in
+toml==0.10.2
+    # via
+    #   pylint
+    #   pytest
+wrapt==1.12.1
+    # via astroid

setup.cfg

@@ -1,146 +0,0 @@
-[metadata]
-name = Red-DiscordBot
-description = A highly customisable Discord bot
-license = GPL-3.0
-long_description = file: README.md
-long_description_content_type = text/markdown; charset=UTF-8; variant=GFM
-author = Cog-Creators
-author_email = cogcreators@gmail.com
-url = https://github.com/Cog-Creators/Red-DiscordBot
-project_urls =
-    Discord Server = https://discord.gg/red
-    Documentation = https://docs.discord.red
-    Donate on Patreon = https://www.patreon.com/Red_Devs
-    Issue Tracker = https://github.com/Cog-Creators/Red-DiscordBot/issues
-    Source Code = https://github.com/Cog-Creators/Red-DiscordBot
-classifiers =
-    # List at https://pypi.org/pypi?%3Aaction=list_classifiers
-    Development Status :: 5 - Production/Stable
-    Framework :: AsyncIO
-    Intended Audience :: Developers
-    Intended Audience :: End Users/Desktop
-    License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-    Natural Language :: English
-    Operating System :: MacOS :: MacOS X
-    Operating System :: Microsoft :: Windows
-    Operating System :: POSIX :: Linux
-    Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Topic :: Communications :: Chat
-license_files =
-    LICENSE
-    redbot/**/*.LICENSE
-
-[options]
-packages = find_namespace:
-python_requires = >=3.8.1,<3.10
-include_package_data = True
-install_requires =
-    aiohttp==3.7.4.post0
-    aiohttp-json-rpc==0.13.3
-    aiosqlite==0.17.0
-    appdirs==1.4.4
-    apsw-wheels==3.36.0.post1
-    async-timeout==3.0.1
-    attrs==21.2.0
-    Babel==2.9.1
-    cffi==1.14.6
-    chardet==4.0.0
-    click==8.0.1
-    colorama==0.4.4
-    commonmark==0.9.1
-    contextlib2==21.6.0
-    discord.py==2.1.0
-    distro==1.6.0; sys_platform == "linux"
-    fuzzywuzzy==0.18.0
-    idna==3.2
-    Markdown==3.3.4
-    multidict==5.1.0
-    psutil==5.8.0
-    pycparser==2.20
-    Pygments==2.10.0
-    PyNaCl==1.4.0
-    python-dateutil==2.8.2
-    python-Levenshtein-wheels==0.13.2
-    pytz==2021.1
-    PyYAML==5.4.1
-    Red-Commons==1.0.0
-    Red-Lavalink==0.11.0rc1
-    rich==10.9.0
-    schema==0.7.4
-    six==1.16.0
-    typing-extensions==3.10.0.2
-    uvloop==0.16.0; sys_platform != "win32" and platform_python_implementation == "CPython"
-    yarl==1.6.3
-
-[options.extras_require]
-docs =
-    alabaster==0.7.12
-    certifi==2021.5.30
-    charset-normalizer==2.0.4
-    docutils==0.16
-    imagesize==1.2.0
-    Jinja2==3.0.1
-    MarkupSafe==2.0.1
-    packaging==21.0
-    pyparsing==2.4.7
-    requests==2.26.0
-    snowballstemmer==2.1.0
-    Sphinx==4.1.2
-    sphinx-prompt==1.5.0
-    sphinx-rtd-theme==0.5.2
-    sphinxcontrib-applehelp==1.0.2
-    sphinxcontrib-devhelp==1.0.2
-    sphinxcontrib-htmlhelp==2.0.0
-    sphinxcontrib-jsmath==1.0.1
-    sphinxcontrib-qthelp==1.0.3
-    sphinxcontrib-serializinghtml==1.1.5
-    sphinxcontrib-trio==1.1.2
-    urllib3==1.26.6
-postgres =
-    asyncpg==0.24.0
-style =
-    black==22.1.0
-    mypy-extensions==0.4.3
-    pathspec==0.9.0
-    regex==2021.8.28
-    toml==0.10.2
-    typed-ast==1.4.3
-test =
-    astroid==2.7.3
-    iniconfig==1.1.1
-    isort==5.9.3
-    lazy-object-proxy==1.6.0
-    mccabe==0.6.1
-    packaging==21.0
-    platformdirs==2.3.0
-    pluggy==1.0.0
-    py==1.10.0
-    pylint==2.10.2
-    pyparsing==2.4.7
-    pytest==6.2.5
-    pytest-asyncio==0.15.1
-    pytest-mock==3.6.1
-    toml==0.10.2
-    wrapt==1.12.1
-all =
-    %(postgres)s
-dev =
-    %(all)s
-    %(docs)s
-    %(style)s
-    %(test)s
-
-[options.entry_points]
-console_scripts =
-    redbot=redbot.__main__:main
-    redbot-setup=redbot.setup:run_cli
-    redbot-launcher=redbot.launcher:main
-pytest11 =
-    red-discordbot=redbot.pytest
-
-[options.packages.find]
-include =
-    redbot
-    redbot.*

setup.py

@@ -1,17 +1,64 @@
 import os
 import sys
-from setuptools import setup
+from pathlib import Path
+
+from setuptools import find_namespace_packages, setup
+
+ROOT_FOLDER = Path(__file__).parent.absolute()
+REQUIREMENTS_FOLDER = ROOT_FOLDER / "requirements"
 
 # Since we're importing `redbot` package, we have to ensure that it's in sys.path.
-sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+sys.path.insert(0, str(ROOT_FOLDER))
 
 from redbot import VersionInfo
 
 version, _ = VersionInfo._get_version(ignore_installed=True)
 
-if os.getenv("TOX_RED", False) and sys.version_info >= (3, 10):
-    # We want to be able to test Python versions that we do not support yet.
-    setup(python_requires=">=3.8.1", version=version)
-else:
-    # Metadata and options defined in setup.cfg
-    setup(version=version)
+
+def get_requirements(fp):
+    return [
+        line.strip()
+        for line in fp.read().splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+
+
+def extras_combined(*extra_names):
+    return list(
+        {
+            req
+            for extra_name, extra_reqs in extras_require.items()
+            if not extra_names or extra_name in extra_names
+            for req in extra_reqs
+        }
+    )
+
+
+with open(REQUIREMENTS_FOLDER / "base.txt", encoding="utf-8") as fp:
+    install_requires = get_requirements(fp)
+
+extras_require = {}
+for file in REQUIREMENTS_FOLDER.glob("extra-*.txt"):
+    with file.open(encoding="utf-8") as fp:
+        extras_require[file.stem[len("extra-") :]] = get_requirements(fp)
+
+extras_require["dev"] = extras_combined()
+extras_require["all"] = extras_combined("postgres")
+
+
+python_requires = ">=3.8.1"
+if not os.getenv("TOX_RED", False) or sys.version_info < (3, 10):
+    python_requires += ",<3.10"
+
+# Metadata and options defined in pyproject.toml
+setup(
+    version=version,
+    python_requires=python_requires,
+    # TODO: use [tool.setuptools.dynamic] table once this feature gets out of beta
+    install_requires=install_requires,
+    extras_require=extras_require,
+    # TODO: use [project] table once PEP 639 gets accepted
+    license_files=["LICENSE", "redbot/**/*.LICENSE"],
+    # TODO: use [tool.setuptools.packages] table once this feature gets out of beta
+    packages=find_namespace_packages(include=["redbot", "redbot.*"]),
+)

tests/core/test_version.py

@@ -1,7 +1,7 @@
 import importlib.metadata
-import pkg_resources
 import os
 import sys
+from packaging.requirements import Requirement
 
 import pytest
 
@@ -55,9 +55,9 @@ def test_python_version_has_lower_bound():
     requires_python = importlib.metadata.metadata("Red-DiscordBot")["Requires-Python"]
     assert requires_python is not None
 
-    # `pkg_resources` needs a regular requirement string, so "x" serves as requirement's name here
-    req = pkg_resources.Requirement.parse(f"x{requires_python}")
-    assert any(op in (">", ">=") for op, version in req.specs)
+    # Requirement needs a regular requirement string, so "x" serves as requirement's name here
+    req = Requirement(f"x{requires_python}")
+    assert any(spec.operator in (">", ">=") for spec in req.specifier)
 
 
 @pytest.mark.skipif(
@@ -72,6 +72,6 @@ def test_python_version_has_upper_bound():
     requires_python = importlib.metadata.metadata("Red-DiscordBot")["Requires-Python"]
     assert requires_python is not None
 
-    # `pkg_resources` needs a regular requirement string, so "x" serves as requirement's name here
-    req = pkg_resources.Requirement.parse(f"x{requires_python}")
-    assert any(op in ("<", "<=") for op, version in req.specs)
+    # Requirement needs a regular requirement string, so "x" serves as requirement's name here
+    req = Requirement(f"x{requires_python}")
+    assert any(spec.operator in ("<", "<=") for spec in req.specifier)

tools/bumpdeps.py

@@ -1,171 +0,0 @@
-#!/usr/bin/env python3.8
-"""Script to bump pinned dependencies in setup.cfg.
-
-This script aims to help update our list of pinned primary and
-secondary dependencies in *setup.cfg*, using the unpinned primary
-dependencies listed in *primary_deps.ini*.
-
-This script will not work when run on Windows.
-
-What this script does
----------------------
-It prints to stdout all primary and secondary dependencies for Red,
-pinned to the latest possible version, within the constraints specified
-in ``primary_deps.ini``. The output should be suitable for copying and
-pasting into ``setup.cfg``. PEP 508 markers are preserved.
-
-How this script works
----------------------
-Overview:
-1. Primary dependencies are read from primary_deps.ini using
-setuptools' config parser.
-2. A clean virtual environment is created in a temporary directory.
-3. Core primary dependencies are passed to the ``pip install`` command
-for that virtual environment.
-4. Pinned primary dependencies are obtained by reading the output of
-``pip freeze`` in that virtual environment, and any PEP 508 markers
-shown with the requirement in ``primary_deps.ini`` are preserved.
-5. Steps 2-4 are repeated for each extra requirement, but care is taken
-not to duplicate core dependencies (primary or secondary) in the final
-pinned extra dependencies.
-
-This script makes use of the *packaging* library to parse version
-specifiers and environment markers.
-
-Known Limitations
------------------
-These limitations don't stop this script from being helpful, but
-hopefully help explain in which situations some dependencies may need
-to be listed manually in ``setup.cfg``.
-
-1. Whilst environment markers of any primary dependencies specified in
-``primary_deps.ini`` are preserved in the output, they will not be
-added to secondary dependencies. So for example, if some package
-*dep1* has a dependency *dep2*, and *dep1* is listed as a primary
-dependency in ``primary_deps.ini`` like follows::
-    dep1; sys_platform == "linux"
-
-Then the output will look like this::
-    dep1==1.1.1; sys_platform == "linux"
-    dep2==2.2.2
-
-So even though ``dep1`` and its dependencies should only be installed on
-Linux, in reality, its dependencies will be installed regardless. To
-work around this, simply list the secondary dependencies in
-``primary_deps.ini`` as well, with the environment markers.
-
-2. If a core requirement and an extra requirement have a common
-sub-dependency, there is a chance the sub-dependency will have a version
-conflict unless it is manually held back. This script will issue a
-warning to stderr when it thinks this might be happening.
-
-3. Environment markers which exclude dependencies from the system
-running this script will cause those dependencies to be excluded from
-the output. So for example, if a dependency has the environment marker
-``sys_platform == "darwin"``, and the script is being run on linux, then
-this dependency will be ignored, and must be added to ``setup.cfg``
-manually.
-"""
-import shlex
-import sys
-import subprocess as sp
-import tempfile
-import textwrap
-import venv
-from pathlib import Path
-from typing import Sequence, Iterable, Dict
-
-import packaging.requirements
-import setuptools.config
-
-THIS_DIRECTORY = Path(__file__).parent
-REQUIREMENTS_INI_PTH: Path = THIS_DIRECTORY / "primary_deps.ini"
-
-PIP_INSTALL_ARGS = ("install", "--upgrade")
-PIP_FREEZE_ARGS = ("freeze", "--no-color")
-
-
-def main() -> int:
-    if not REQUIREMENTS_INI_PTH.is_file():
-        print("No primary_deps.ini found in the same directory as bumpdeps.py", file=sys.stderr)
-        return 1
-
-    primary_reqs_cfg = setuptools.config.read_configuration(str(REQUIREMENTS_INI_PTH))
-
-    print("[options]")
-    print("install_requires =")
-    core_primary_deps = primary_reqs_cfg["options"]["install_requires"]
-    full_core_reqs = get_all_reqs(core_primary_deps)
-    print(textwrap.indent("\n".join(map(str, full_core_reqs)), " " * 4))
-    print()
-
-    print("[options.extras_require]")
-    for extra, extra_primary_deps in primary_reqs_cfg["options"]["extras_require"].items():
-        print(extra, "=")
-        full_extra_reqs = get_all_reqs(
-            extra_primary_deps, all_core_deps={r.name.lower(): r for r in full_core_reqs}
-        )
-        print(textwrap.indent("\n".join(map(str, full_extra_reqs)), " " * 4))
-
-    return 0
-
-
-def get_all_reqs(
-    primary_deps: Iterable[str], all_core_deps: Dict[str, packaging.requirements.Requirement] = ()
-) -> Sequence[packaging.requirements.Requirement]:
-    reqs_dict = {r.name.lower(): r for r in map(packaging.requirements.Requirement, primary_deps)}
-    with tempfile.TemporaryDirectory() as tmpdir:
-        venv.create(tmpdir, system_site_packages=False, clear=True, with_pip=True)
-        tmpdir_pth = Path(tmpdir)
-
-        pip_exe_pth = tmpdir_pth / "bin" / "pip"
-
-        # Upgrade pip to latest version
-        sp.run((pip_exe_pth, *PIP_INSTALL_ARGS, "pip"), stdout=sp.DEVNULL, check=True)
-
-        # Install the primary dependencies
-        sp.run(
-            (pip_exe_pth, *PIP_INSTALL_ARGS, *map(str, reqs_dict.values())),
-            stdout=sp.DEVNULL,
-            check=True,
-        )
-
-        # Get pinned primary+secondary dependencies from pip freeze
-        proc = sp.run(
-            (pip_exe_pth, *PIP_FREEZE_ARGS), stdout=sp.PIPE, check=True, encoding="utf-8"
-        )
-
-        # Return Requirement objects
-        ret = []
-        for req_obj in map(packaging.requirements.Requirement, proc.stdout.strip().split("\n")):
-            dep_name = req_obj.name.lower()
-            # Don't include core dependencies if these are extra dependencies
-            if dep_name in all_core_deps:
-                if req_obj.specifier != all_core_deps[dep_name].specifier:
-                    print(
-                        f"[WARNING] {dep_name} is listed as both a core requirement and an extra "
-                        f"requirement, and it's possible that their versions conflict!",
-                        file=sys.stderr,
-                    )
-                continue
-
-            # Preserve environment markers
-            if dep_name in reqs_dict:
-                req_obj.marker = reqs_dict[dep_name].marker
-
-            ret.append(req_obj)
-
-        return ret
-
-
-if __name__ == "__main__":
-    try:
-        exit_code = main()
-    except sp.CalledProcessError as exc:
-        cmd = " ".join(map(lambda c: shlex.quote(str(c)), exc.cmd))
-        print(
-            f"The following command failed with code {exc.returncode}:\n    ", cmd, file=sys.stderr
-        )
-        exit_code = 1
-
-    sys.exit(exit_code)

tools/dev-requirements.txt

@@ -1,3 +1,2 @@
-packaging
-tox
+tox<4
 -e .[dev]

tools/primary_deps.ini

@@ -1,46 +0,0 @@
-# primary_deps.ini
-# This file should list primary dependencies in terms of both core and
-# extras, in setup.cfg format. A primary dependency is one which is
-# used directly in Red, or otherwise is forced to be listed as a
-# dependency. Version specifiers should be as liberal as possible.
-
-[options]
-install_requires =
-    aiohttp
-    aiohttp-json-rpc
-    aiosqlite
-    appdirs
-    apsw-wheels
-    babel
-    click
-    colorama
-    discord.py
-    distro; sys_platform == "linux"
-    fuzzywuzzy
-    markdown
-    psutil
-    python-dateutil
-    python-Levenshtein-wheels
-    PyYAML
-    Red-Commons
-    Red-Lavalink
-    rich
-    schema
-    uvloop; sys_platform != "win32" and platform_python_implementation == "CPython"
-    PyNaCl
-
-[options.extras_require]
-docs =
-    Sphinx
-    sphinx-prompt
-    sphinx_rtd_theme
-    sphinxcontrib-trio
-postgres =
-    asyncpg
-style =
-    black
-test =
-    pylint
-    pytest
-    pytest-asyncio
-    pytest-mock

tox.ini

@@ -10,13 +10,14 @@ envlist =
     docs
     style
 skip_missing_interpreters = True
+isolated_build = True
 
 [testenv]
 description = Run tests and basic automatic issue checking.
 whitelist_externals =
     pytest
     pylint
-extras = voice, test
+extras = test
 setenv =
     TOX_RED = 1
 commands =
@@ -28,7 +29,7 @@ commands =
 description = Run pytest with PostgreSQL backend
 whitelist_externals =
     pytest
-extras = voice, test, postgres
+extras = test, postgres
 setenv =
     TOX_RED = 1
     RED_STORAGE_TYPE=postgres
@@ -53,7 +54,7 @@ setenv =
     # Prioritise make.bat over any make.exe which might be on PATH
     PATHEXT=.BAT;.EXE
 basepython = python3.8
-extras = docs
+extras = doc
 commands =
     sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out/html" -W --keep-going -bhtml
     sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out/doctest" -W --keep-going -bdoctest