diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6ec7bc566..cf4681d86 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | - python -m pip install -U pip setuptools wheel + python -m pip install -U pip wheel python -m pip install -e .[all] # Set the `CODEQL-PYTHON` environment variable to the Python executable # that includes the dependencies diff --git a/.github/workflows/run_pip_compile.yaml b/.github/workflows/run_pip_compile.yaml new file mode 100644 index 000000000..47a47083f --- /dev/null +++ b/.github/workflows/run_pip_compile.yaml @@ -0,0 +1,84 @@ +name: Generate requirements files with pip-compile. + +on: + workflow_dispatch: + +jobs: + generate_requirements: + name: Generate requirements files for ${{ matrix.os }} platform. + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Checkout the repository. + uses: actions/checkout@v3 + + - name: Set up Python 3.8. + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -U pip-tools + + - name: Generate requirements files. + id: compile_requirements + run: | + python .github/workflows/scripts/compile_requirements.py + + - name: Upload requirements files. + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.compile_requirements.outputs.sys_platform }} + path: requirements/${{ steps.compile_requirements.outputs.sys_platform }}-*.txt + + merge_requirements: + name: Merge requirements files. + needs: generate_requirements + runs-on: ubuntu-latest + steps: + - name: Checkout the repository. + uses: actions/checkout@v3 + + - name: Set up Python 3.8. + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python -m pip install -U "packaging>=22.0" + + - name: Download Windows requirements. + uses: actions/download-artifact@v3 + with: + name: win32 + path: requirements + - name: Download Linux requirements. + uses: actions/download-artifact@v3 + with: + name: linux + path: requirements + - name: Download macOS requirements. + uses: actions/download-artifact@v3 + with: + name: darwin + path: requirements + + - name: Merge requirements files. + run: | + python .github/workflows/scripts/merge_requirements.py + + - name: Upload merged requirements files. + uses: actions/upload-artifact@v3 + with: + name: merged + path: | + requirements/base.txt + requirements/extra-*.txt diff --git a/.github/workflows/scripts/compile_requirements.py b/.github/workflows/scripts/compile_requirements.py new file mode 100644 index 000000000..37aab01b8 --- /dev/null +++ b/.github/workflows/scripts/compile_requirements.py @@ -0,0 +1,35 @@ +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +GITHUB_OUTPUT = os.environ["GITHUB_OUTPUT"] +REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements" +os.chdir(REQUIREMENTS_FOLDER) + + +def pip_compile(name: str) -> None: + subprocess.check_call( + ( + sys.executable, + "-m", + "piptools", + "compile", + "--upgrade", + "--verbose", + f"{name}.in", + "--output-file", + f"{sys.platform}-{name}.txt", + ) + ) + + +pip_compile("base") +shutil.copyfile(f"{sys.platform}-base.txt", "base.txt") +for file in REQUIREMENTS_FOLDER.glob("extra-*.in"): + pip_compile(file.stem) + +with open(GITHUB_OUTPUT, "a", encoding="utf-8") as fp: + fp.write(f"sys_platform={sys.platform}\n") diff --git a/.github/workflows/scripts/merge_requirements.py b/.github/workflows/scripts/merge_requirements.py new file mode 100644 index 000000000..4ad2822b2 --- /dev/null +++ b/.github/workflows/scripts/merge_requirements.py @@ -0,0 +1,134 @@ +import os +from pathlib import Path +from typing import List, TextIO + +from packaging.markers import Marker +from packaging.requirements import Requirement + + +REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements" +os.chdir(REQUIREMENTS_FOLDER) + + +class RequirementData: + def __init__(self, requirement_string: str) -> None: + self.req = Requirement(requirement_string) + self.comments = set() + + @property + def name(self) -> str: + return self.req.name + + @property + def marker(self) -> Marker: + return self.req.marker + + @marker.setter + def marker(self, value: Marker) -> None: + self.req.marker = value + + +def get_requirements(fp: TextIO) -> List[RequirementData]: + requirements = [] + + current = None + for line in fp.read().splitlines(): + annotation_prefix = " # " + if line.startswith(annotation_prefix) and current is not None: + source = line[len(annotation_prefix) :].strip() + if source == "via": + continue + via_prefix = "via " + if source.startswith(via_prefix): + source = source[len(via_prefix) :] + current.comments.add(source) + elif line and not line.startswith(("#", " ")): + current = RequirementData(line) + requirements.append(current) + + return requirements + + +names = ["base"] +names.extend(file.stem for file in REQUIREMENTS_FOLDER.glob("extra-*.in")) +base_requirements = [] + +for name in names: + # {req_name: {sys_platform: RequirementData} + input_data = {} + all_platforms = set() + for file in REQUIREMENTS_FOLDER.glob(f"*-{name}.txt"): + platform_name = file.stem.split("-", maxsplit=1)[0] + all_platforms.add(platform_name) + with file.open(encoding="utf-8") as fp: + requirements = get_requirements(fp) + + for req in requirements: + platforms = input_data.setdefault(req.name, {}) + platforms[platform_name] = req + + output = base_requirements if name == "base" else [] + for req_name, platforms in input_data.items(): + req = next(iter(platforms.values())) + for other_req in platforms.values(): + if req.req != other_req.req: + raise RuntimeError(f"Incompatible requirements for {req_name}.") + + req.comments.update(other_req.comments) + + base_req = next( + (base_req for base_req in base_requirements if base_req.name == req.name), None + ) + if base_req is not None: + old_base_marker = base_req.marker + old_req_marker = req.marker + req.marker = base_req.marker = None + if base_req.req != req.req: + raise RuntimeError(f"Incompatible requirements for {req_name}.") + + base_req.marker = old_base_marker + req.marker = old_req_marker + if base_req.marker is None or base_req.marker == req.marker: + continue + + if len(platforms) == len(all_platforms): + output.append(req) + continue + elif len(platforms) < len(all_platforms - platforms.keys()): + platform_marker = " or ".join( + f"sys_platform == '{platform}'" for platform in platforms + ) + else: + platform_marker = " and ".join( + f"sys_platform != '{platform}'" for platform in all_platforms - platforms.keys() + ) + + new_marker = ( + f"({req.marker}) and ({platform_marker})" + if req.marker is not None + else platform_marker + ) + req.marker = Marker(new_marker) + if base_req is not None and base_req.marker == req.marker: + continue + + output.append(req) + + output.sort(key=lambda req: (req.marker is not None, req.name)) + with open(f"{name}.txt", "w+", encoding="utf-8") as fp: + for req in output: + fp.write(str(req.req)) + fp.write("\n") + comments = sorted(req.comments) + + if len(comments) == 1: + source = comments[0] + fp.write(" # via ") + fp.write(source) + fp.write("\n") + else: + fp.write(" # via\n") + for source in comments: + fp.write(" # ") + fp.write(source) + fp.write("\n") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a34ad0b72..4783b8b7c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,7 +46,7 @@ jobs: - name: Install tox run: | python -m pip install --upgrade pip - pip install tox + pip install 'tox<4' - name: Tox test env: TOXENV: ${{ matrix.tox_env }} @@ -82,7 +82,7 @@ jobs: - name: Install tox run: | python -m pip install --upgrade pip - pip install tox + pip install 'tox<4' - name: Tox test env: TOXENV: postgres diff --git a/.readthedocs.yml b/.readthedocs.yml index d180ab09a..477d6aec4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,4 +9,4 @@ python: - method: pip path: . extra_requirements: - - docs + - doc diff --git a/MANIFEST.in b/MANIFEST.in index 62ccfa1ef..10bddfaa1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,12 @@ include LICENSE recursive-include redbot *.LICENSE +# include requirements files +include requirements/base.in +include requirements/base.txt +include requirements/extra-*.in +include requirements/extra-*.txt + # include locale files recursive-include redbot locales/*.po diff --git a/Makefile b/Makefile index a094ae28c..0f92df923 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ bumpdeps: # Development environment newenv: $(PYTHON) -m venv --clear .venv - .venv/bin/pip install -U pip setuptools wheel + .venv/bin/pip install -U pip wheel $(MAKE) syncenv syncenv: .venv/bin/pip install -Ur ./tools/dev-requirements.txt diff --git a/docs/install_guides/_includes/install-and-setup-red-unix.rst b/docs/install_guides/_includes/install-and-setup-red-unix.rst index 25b403678..5e4a18b01 100644 --- a/docs/install_guides/_includes/install-and-setup-red-unix.rst +++ b/docs/install_guides/_includes/install-and-setup-red-unix.rst @@ -9,7 +9,7 @@ To install without additional config backend support: .. prompt:: bash :prompts: (redenv) $ - python -m pip install -U pip setuptools wheel + python -m pip install -U pip wheel python -m pip install -U Red-DiscordBot Or, to install with PostgreSQL support: @@ -17,7 +17,7 @@ Or, to install with PostgreSQL support: .. prompt:: bash :prompts: (redenv) $ - python -m pip install -U pip setuptools wheel + python -m pip install -U pip wheel python -m pip install -U "Red-DiscordBot[postgres]" diff --git a/docs/install_guides/windows.rst b/docs/install_guides/windows.rst index d6ce2fe1c..d35a00890 100644 --- a/docs/install_guides/windows.rst +++ b/docs/install_guides/windows.rst @@ -134,7 +134,7 @@ Run **one** of the following set of commands, depending on what extras you want .. prompt:: batch :prompts: (redenv) C:\\> - python -m pip install -U pip setuptools wheel + python -m pip install -U pip wheel python -m pip install -U Red-DiscordBot * With PostgreSQL support: @@ -142,7 +142,7 @@ Run **one** of the following set of commands, depending on what extras you want .. prompt:: batch :prompts: (redenv) C:\\> - python -m pip install -U pip setuptools wheel + python -m pip install -U pip wheel python -m pip install -U Red-DiscordBot[postgres] -------------------------- diff --git a/make.bat b/make.bat index 2204b0172..7f3d046b5 100644 --- a/make.bat +++ b/make.bat @@ -24,7 +24,7 @@ goto:eof :newenv py -3.8 -m venv --clear .venv -"%~dp0.venv\Scripts\python" -m pip install -U pip setuptools wheel +"%~dp0.venv\Scripts\python" -m pip install -U pip wheel goto syncenv :syncenv diff --git a/make.ps1 b/make.ps1 index d960a626e..370a21837 100644 --- a/make.ps1 +++ b/make.ps1 @@ -56,7 +56,7 @@ function stylediff() { function newenv() { py -3.8 -m venv --clear .venv - & $PSScriptRoot\.venv\Scripts\python.exe -m pip install -U pip setuptools + & $PSScriptRoot\.venv\Scripts\python.exe -m pip install -U pip wheel syncenv } diff --git a/pyproject.toml b/pyproject.toml index f3ec923f7..08529574e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,52 @@ [build-system] - requires = ["setuptools", "wheel"] - build-backend = "setuptools.build_meta" +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "Red-DiscordBot" +description = "A highly customisable Discord bot" +readme = "README.md" +authors = [{ name = "Cog Creators" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: AsyncIO", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Topic :: Communications :: Chat", +] +dynamic = ["version", "requires-python", "dependencies", "optional-dependencies"] + +[project.urls] +"Homepage" = "https://github.com/Cog-Creators/Red-DiscordBot" +"Discord Server" = "https://discord.gg/red" +"Documentation" = "https://docs.discord.red" +"Donate on Patreon" = "https://www.patreon.com/Red_Devs" +"Issue Tracker" = "https://github.com/Cog-Creators/Red-DiscordBot/issues" +"Source Code" = "https://github.com/Cog-Creators/Red-DiscordBot" + +[project.scripts] +redbot = "redbot.__main__:main" +redbot-setup = "redbot.setup:run_cli" +redbot-launcher = "redbot.launcher:main" + +[project.entry-points.pytest11] +red-discordbot = "redbot.pytest" [tool.black] - line-length = 99 - required-version = '22.1.0' - target-version = ['py38'] - include = '\.py$' - force-exclude = ''' - /( - redbot\/vendored - )/ - ''' +line-length = 99 +required-version = '22.1.0' +target-version = ['py38'] +include = '\.py$' +force-exclude = ''' +/( + redbot\/vendored +)/ +''' diff --git a/redbot/__main__.py b/redbot/__main__.py index 745b85fa7..f9de75e03 100644 --- a/redbot/__main__.py +++ b/redbot/__main__.py @@ -10,7 +10,6 @@ import json import logging import os import pip -import pkg_resources import platform import shutil import signal @@ -335,7 +334,9 @@ async def run_bot(red: Red, cli_flags: Namespace) -> None: # `sys.path`, you must invoke the appropriate methods on the `working_set` instance # to keep it in sync." # Source: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#workingset-objects - pkg_resources.working_set.add_entry(str(LIB_PATH)) + pkg_resources = sys.modules.get("pkg_resources") + if pkg_resources is not None: + pkg_resources.working_set.add_entry(str(LIB_PATH)) sys.meta_path.insert(0, SharedLibImportWarner()) if cli_flags.token: diff --git a/redbot/core/events.py b/redbot/core/events.py index beb1eabab..69cb64e68 100644 --- a/redbot/core/events.py +++ b/redbot/core/events.py @@ -6,11 +6,12 @@ import codecs import logging import traceback from datetime import datetime, timedelta, timezone +from typing import Tuple import aiohttp import discord -import pkg_resources -from pkg_resources import DistributionNotFound +import importlib.metadata +from packaging.requirements import Requirement from redbot.core import data_manager from redbot.core.commands import RedHelpFormatter, HelpSettings @@ -54,6 +55,88 @@ ______ _ ______ _ _ ______ _ _ = Translator(__name__, __file__) +def get_outdated_red_messages(pypi_version: str, py_version_req: str) -> Tuple[str, str]: + outdated_red_message = _( + "Your Red instance is out of date! {} is the current version, however you are using {}!" + ).format(pypi_version, red_version) + rich_outdated_message = ( + f"[red]Outdated version![/red]\n" + f"[red]!!![/red]Version [cyan]{pypi_version}[/] is available, " + f"but you're using [cyan]{red_version}[/][red]!!![/red]" + ) + current_python = platform.python_version() + extra_update = _( + "\n\nWhile the following command should work in most scenarios as it is " + "based on your current OS, environment, and Python version, " + "**we highly recommend you to read the update docs at <{docs}> and " + "make sure there is nothing else that " + "needs to be done during the update.**" + ).format(docs="https://docs.discord.red/en/stable/update_red.html") + + if not expected_version(current_python, py_version_req): + extra_update += _( + "\n\nYou have Python `{py_version}` and this update " + "requires `{req_py}`; you cannot simply run the update command.\n\n" + "You will need to follow the update instructions in our docs above, " + "if you still need help updating after following the docs go to our " + "#support channel in " + ).format(py_version=current_python, req_py=py_version_req) + outdated_red_message += extra_update + return outdated_red_message, rich_outdated_message + + red_dist = importlib.metadata.distribution("Red-DiscordBot") + installed_extras = red_dist.metadata.get_all("Provides-Extra") + installed_extras.remove("dev") + installed_extras.remove("all") + distributions = {} + for req_str in red_dist.requires: + req = Requirement(req_str) + if req.marker is None or req.marker.evaluate(): + continue + for extra in reversed(installed_extras): + if not req.marker.evaluate({"extra": extra}): + continue + + # Check that the requirement is met. + # This is a bit simplified for our purposes and does not check + # whether the requirements of our requirements are met as well. + # This could potentially be an issue if we'll ever depend on + # a dependency's extra in our extra when we already depend on that + # in our base dependencies. However, considering that right now, all + # our dependencies are also fully pinned, this should not ever matter. + if req.name in distributions: + dist = distributions[req.name] + else: + try: + dist = importlib.metadata.distribution(req.name) + except importlib.metadata.PackageNotFoundError: + installed_extras.remove(extra) + dist = None + distributions[req.name] = dist + if dist is None or not req.specifier.contains(dist.version, prereleases=True): + installed_extras.remove(extra) + + if installed_extras: + package_extras = f"[{','.join(installed_extras)}]" + else: + package_extras = "" + + extra_update += _( + "\n\nTo update your bot, first shutdown your bot" + " then open a window of {console} (Not as admin) and run the following:" + "{command_1}\n" + "Once you've started up your bot again, we recommend that" + " you update any installed 3rd-party cogs with this command in Discord:" + "{command_2}" + ).format( + console=_("Command Prompt") if platform.system() == "Windows" else _("Terminal"), + command_1=f'```"{sys.executable}" -m pip install -U "Red-DiscordBot{package_extras}"```', + command_2=f"```[p]cog update```", + ) + outdated_red_message += extra_update + return outdated_red_message, rich_outdated_message + + def init_events(bot, cli_flags): @bot.event async def on_connect(): @@ -74,7 +157,6 @@ def init_events(bot, cli_flags): prefixes = cli_flags.prefix or (await bot._config.prefix()) lang = await bot._config.locale() - red_pkg = pkg_resources.get_distribution("Red-DiscordBot") dpy_version = discord.__version__ table_general_info = Table(show_edge=False, show_header=False, box=box.MINIMAL) @@ -97,69 +179,9 @@ def init_events(bot, cli_flags): pypi_version, py_version_req = await fetch_latest_red_version_info() outdated = pypi_version and pypi_version > red_version_info if outdated: - outdated_red_message = _( - "Your Red instance is out of date! {} is the current " - "version, however you are using {}!" - ).format(pypi_version, red_version) - rich_outdated_message = ( - f"[red]Outdated version![/red]\n" - f"[red]!!![/red]Version [cyan]{pypi_version}[/] is available, " - f"but you're using [cyan]{red_version}[/][red]!!![/red]" + outdated_red_message, rich_outdated_message = get_outdated_red_messages( + pypi_version, py_version_req ) - current_python = platform.python_version() - extra_update = _( - "\n\nWhile the following command should work in most scenarios as it is " - "based on your current OS, environment, and Python version, " - "**we highly recommend you to read the update docs at <{docs}> and " - "make sure there is nothing else that " - "needs to be done during the update.**" - ).format(docs="https://docs.discord.red/en/stable/update_red.html") - if expected_version(current_python, py_version_req): - installed_extras = [] - for extra, reqs in red_pkg._dep_map.items(): - if extra is None or extra in {"dev", "all"}: - continue - try: - pkg_resources.require(req.name for req in reqs) - except pkg_resources.DistributionNotFound: - pass - else: - installed_extras.append(extra) - - if installed_extras: - package_extras = f"[{','.join(installed_extras)}]" - else: - package_extras = "" - - extra_update += _( - "\n\nTo update your bot, first shutdown your " - "bot then open a window of {console} (Not as admin) and " - "run the following:\n\n" - ).format( - console=_("Command Prompt") - if platform.system() == "Windows" - else _("Terminal") - ) - extra_update += ( - '```"{python}" -m pip install -U Red-DiscordBot{package_extras}```'.format( - python=sys.executable, package_extras=package_extras - ) - ) - extra_update += _( - "\nOnce you've started up your bot again, if you have any 3rd-party cogs" - " installed we then highly recommend you update them with this command" - " in Discord: `[p]cog update`" - ) - - else: - extra_update += _( - "\n\nYou have Python `{py_version}` and this update " - "requires `{req_py}`; you cannot simply run the update command.\n\n" - "You will need to follow the update instructions in our docs above, " - "if you still need help updating after following the docs go to our " - "#support channel in " - ).format(py_version=current_python, req_py=py_version_req) - outdated_red_message += extra_update rich_console = rich.get_console() rich_console.print(INTRO, style="red", markup=False, highlight=False) diff --git a/redbot/core/utils/_internal_utils.py b/redbot/core/utils/_internal_utils.py index d6c1ed381..8407fdedf 100644 --- a/redbot/core/utils/_internal_utils.py +++ b/redbot/core/utils/_internal_utils.py @@ -31,7 +31,7 @@ from typing import ( import aiohttp import discord -import pkg_resources +from packaging.requirements import Requirement from fuzzywuzzy import fuzz, process from rich.progress import ProgressColumn from rich.progress_bar import ProgressBar @@ -316,8 +316,8 @@ async def send_to_owners_with_prefix_replaced(bot: Red, content: str, **kwargs): def expected_version(current: str, expected: str) -> bool: - # `pkg_resources` needs a regular requirement string, so "x" serves as requirement's name here - return current in pkg_resources.Requirement.parse(f"x{expected}") + # Requirement needs a regular requirement string, so "x" serves as requirement's name here + return Requirement(f"x{expected}").specifier.contains(current, prereleases=True) async def fetch_latest_red_version_info() -> Tuple[Optional[VersionInfo], Optional[str]]: diff --git a/redbot/launcher.py b/redbot/launcher.py index f7382078c..f726590a5 100644 --- a/redbot/launcher.py +++ b/redbot/launcher.py @@ -12,7 +12,6 @@ import argparse import asyncio import aiohttp -import pkg_resources from redbot import MIN_PYTHON_VERSION from redbot.setup import ( basic_setup, diff --git a/requirements/base.in b/requirements/base.in new file mode 100644 index 000000000..53bb09b0e --- /dev/null +++ b/requirements/base.in @@ -0,0 +1,23 @@ +aiohttp +aiohttp-json-rpc +aiosqlite +appdirs +apsw-wheels +babel +click +colorama +discord.py +fuzzywuzzy +markdown +packaging +psutil +python-dateutil +python-Levenshtein-wheels +PyNaCl +PyYAML +Red-Commons +Red-Lavalink>=0.11.0rc1 +rich +schema +distro; sys_platform == "linux" +uvloop; sys_platform != "win32" and platform_python_implementation == "CPython" diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 000000000..e69fa803b --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,88 @@ +aiohttp==3.7.4.post0 + # via + # -r base.in + # aiohttp-json-rpc + # discord-py + # red-lavalink +aiohttp-json-rpc==0.13.3 + # via -r base.in +aiosqlite==0.17.0 + # via -r base.in +appdirs==1.4.4 + # via -r base.in +apsw-wheels==3.36.0.post1 + # via -r base.in +async-timeout==3.0.1 + # via aiohttp +attrs==21.2.0 + # via aiohttp +babel==2.9.1 + # via -r base.in +cffi==1.14.6 + # via pynacl +chardet==4.0.0 + # via aiohttp +click==8.0.1 + # via -r base.in +colorama==0.4.4 + # via + # -r base.in + # click +commonmark==0.9.1 + # via rich +contextlib2==21.6.0 + # via schema +discord-py==2.1.0 + # via + # -r base.in + # red-lavalink +fuzzywuzzy==0.18.0 + # via -r base.in +idna==3.2 + # via yarl +markdown==3.3.4 + # via -r base.in +multidict==5.1.0 + # via + # aiohttp + # yarl +packaging==22.0 + # via -r base.in +psutil==5.8.0 + # via -r base.in +pycparser==2.20 + # via cffi +pygments==2.10.0 + # via rich +pynacl==1.4.0 + # via -r base.in +python-dateutil==2.8.2 + # via -r base.in +python-levenshtein-wheels==0.13.2 + # via -r base.in +pytz==2021.1 + # via babel +pyyaml==5.4.1 + # via -r base.in +red-commons==1.0.0 + # via + # -r base.in + # red-lavalink +red-lavalink==0.11.0rc1 + # via -r base.in +rich==10.9.0 + # via -r base.in +schema==0.7.4 + # via -r base.in +six==1.16.0 + # via python-dateutil +typing-extensions==3.10.0.2 + # via rich +yarl==1.6.3 + # via + # -r base.in + # aiohttp +distro==1.6.0; sys_platform == "linux" + # via -r base.in +uvloop==0.16.0; sys_platform != "win32" and platform_python_implementation == "CPython" + # via -r base.in diff --git a/requirements/extra-doc.in b/requirements/extra-doc.in new file mode 100644 index 000000000..2f14a30c1 --- /dev/null +++ b/requirements/extra-doc.in @@ -0,0 +1,6 @@ +-c base.txt + +Sphinx +sphinx-prompt +sphinx_rtd_theme +sphinxcontrib-trio diff --git a/requirements/extra-doc.txt b/requirements/extra-doc.txt new file mode 100644 index 000000000..eed9cb2e3 --- /dev/null +++ b/requirements/extra-doc.txt @@ -0,0 +1,46 @@ +alabaster==0.7.12 + # via sphinx +certifi==2021.5.30 + # via requests +charset-normalizer==2.0.4 + # via requests +docutils==0.16 + # via + # sphinx + # sphinx-rtd-theme +imagesize==1.2.0 + # via sphinx +jinja2==3.0.1 + # via sphinx +markupsafe==2.0.1 + # via jinja2 +requests==2.26.0 + # via sphinx +snowballstemmer==2.1.0 + # via sphinx +sphinx==4.1.2 + # via + # -r extra-doc.in + # sphinx-prompt + # sphinx-rtd-theme + # sphinxcontrib-trio +sphinx-prompt==1.5.0 + # via -r extra-doc.in +sphinx-rtd-theme==0.5.2 + # via -r extra-doc.in +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +sphinxcontrib-trio==1.1.2 + # via -r extra-doc.in +urllib3==1.26.6 + # via requests diff --git a/requirements/extra-postgres.in b/requirements/extra-postgres.in new file mode 100644 index 000000000..af77be5d5 --- /dev/null +++ b/requirements/extra-postgres.in @@ -0,0 +1,3 @@ +-c base.txt + +asyncpg diff --git a/requirements/extra-postgres.txt b/requirements/extra-postgres.txt new file mode 100644 index 000000000..945c075af --- /dev/null +++ b/requirements/extra-postgres.txt @@ -0,0 +1,2 @@ +asyncpg==0.24.0 + # via -r extra-postgres.in diff --git a/requirements/extra-style.in b/requirements/extra-style.in new file mode 100644 index 000000000..72cc61bd1 --- /dev/null +++ b/requirements/extra-style.in @@ -0,0 +1,3 @@ +-c base.txt + +black diff --git a/requirements/extra-style.txt b/requirements/extra-style.txt new file mode 100644 index 000000000..dde309b4d --- /dev/null +++ b/requirements/extra-style.txt @@ -0,0 +1,12 @@ +black==22.1.0 + # via -r extra-style.in +mypy-extensions==0.4.3 + # via black +pathspec==0.9.0 + # via black +regex==2021.8.28 + # via black +toml==0.10.2 + # via black +typed-ast==1.4.3 + # via black diff --git a/requirements/extra-test.in b/requirements/extra-test.in new file mode 100644 index 000000000..b726962fd --- /dev/null +++ b/requirements/extra-test.in @@ -0,0 +1,6 @@ +-c base.txt + +pylint +pytest +pytest-asyncio +pytest-mock diff --git a/requirements/extra-test.txt b/requirements/extra-test.txt new file mode 100644 index 000000000..ea453cbf3 --- /dev/null +++ b/requirements/extra-test.txt @@ -0,0 +1,33 @@ +astroid==2.7.3 + # via pylint +iniconfig==1.1.1 + # via pytest +isort==5.9.3 + # via pylint +lazy-object-proxy==1.6.0 + # via astroid +mccabe==0.6.1 + # via pylint +platformdirs==2.3.0 + # via pylint +pluggy==1.0.0 + # via pytest +py==1.10.0 + # via pytest +pylint==2.10.2 + # via -r extra-test.in +pytest==6.2.5 + # via + # -r extra-test.in + # pytest-asyncio + # pytest-mock +pytest-asyncio==0.15.1 + # via -r extra-test.in +pytest-mock==3.6.1 + # via -r extra-test.in +toml==0.10.2 + # via + # pylint + # pytest +wrapt==1.12.1 + # via astroid diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 93b7bc625..000000000 --- a/setup.cfg +++ /dev/null @@ -1,146 +0,0 @@ -[metadata] -name = Red-DiscordBot -description = A highly customisable Discord bot -license = GPL-3.0 -long_description = file: README.md -long_description_content_type = text/markdown; charset=UTF-8; variant=GFM -author = Cog-Creators -author_email = cogcreators@gmail.com -url = https://github.com/Cog-Creators/Red-DiscordBot -project_urls = - Discord Server = https://discord.gg/red - Documentation = https://docs.discord.red - Donate on Patreon = https://www.patreon.com/Red_Devs - Issue Tracker = https://github.com/Cog-Creators/Red-DiscordBot/issues - Source Code = https://github.com/Cog-Creators/Red-DiscordBot -classifiers = - # List at https://pypi.org/pypi?%3Aaction=list_classifiers - Development Status :: 5 - Production/Stable - Framework :: AsyncIO - Intended Audience :: Developers - Intended Audience :: End Users/Desktop - License :: OSI Approved :: GNU General Public License v3 (GPLv3) - Natural Language :: English - Operating System :: MacOS :: MacOS X - Operating System :: Microsoft :: Windows - Operating System :: POSIX :: Linux - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Topic :: Communications :: Chat -license_files = - LICENSE - redbot/**/*.LICENSE - -[options] -packages = find_namespace: -python_requires = >=3.8.1,<3.10 -include_package_data = True -install_requires = - aiohttp==3.7.4.post0 - aiohttp-json-rpc==0.13.3 - aiosqlite==0.17.0 - appdirs==1.4.4 - apsw-wheels==3.36.0.post1 - async-timeout==3.0.1 - attrs==21.2.0 - Babel==2.9.1 - cffi==1.14.6 - chardet==4.0.0 - click==8.0.1 - colorama==0.4.4 - commonmark==0.9.1 - contextlib2==21.6.0 - discord.py==2.1.0 - distro==1.6.0; sys_platform == "linux" - fuzzywuzzy==0.18.0 - idna==3.2 - Markdown==3.3.4 - multidict==5.1.0 - psutil==5.8.0 - pycparser==2.20 - Pygments==2.10.0 - PyNaCl==1.4.0 - python-dateutil==2.8.2 - python-Levenshtein-wheels==0.13.2 - pytz==2021.1 - PyYAML==5.4.1 - Red-Commons==1.0.0 - Red-Lavalink==0.11.0rc1 - rich==10.9.0 - schema==0.7.4 - six==1.16.0 - typing-extensions==3.10.0.2 - uvloop==0.16.0; sys_platform != "win32" and platform_python_implementation == "CPython" - yarl==1.6.3 - -[options.extras_require] -docs = - alabaster==0.7.12 - certifi==2021.5.30 - charset-normalizer==2.0.4 - docutils==0.16 - imagesize==1.2.0 - Jinja2==3.0.1 - MarkupSafe==2.0.1 - packaging==21.0 - pyparsing==2.4.7 - requests==2.26.0 - snowballstemmer==2.1.0 - Sphinx==4.1.2 - sphinx-prompt==1.5.0 - sphinx-rtd-theme==0.5.2 - sphinxcontrib-applehelp==1.0.2 - sphinxcontrib-devhelp==1.0.2 - sphinxcontrib-htmlhelp==2.0.0 - sphinxcontrib-jsmath==1.0.1 - sphinxcontrib-qthelp==1.0.3 - sphinxcontrib-serializinghtml==1.1.5 - sphinxcontrib-trio==1.1.2 - urllib3==1.26.6 -postgres = - asyncpg==0.24.0 -style = - black==22.1.0 - mypy-extensions==0.4.3 - pathspec==0.9.0 - regex==2021.8.28 - toml==0.10.2 - typed-ast==1.4.3 -test = - astroid==2.7.3 - iniconfig==1.1.1 - isort==5.9.3 - lazy-object-proxy==1.6.0 - mccabe==0.6.1 - packaging==21.0 - platformdirs==2.3.0 - pluggy==1.0.0 - py==1.10.0 - pylint==2.10.2 - pyparsing==2.4.7 - pytest==6.2.5 - pytest-asyncio==0.15.1 - pytest-mock==3.6.1 - toml==0.10.2 - wrapt==1.12.1 -all = - %(postgres)s -dev = - %(all)s - %(docs)s - %(style)s - %(test)s - -[options.entry_points] -console_scripts = - redbot=redbot.__main__:main - redbot-setup=redbot.setup:run_cli - redbot-launcher=redbot.launcher:main -pytest11 = - red-discordbot=redbot.pytest - -[options.packages.find] -include = - redbot - redbot.* diff --git a/setup.py b/setup.py index 0a6e94a23..58d1b8f9f 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,64 @@ import os import sys -from setuptools import setup +from pathlib import Path + +from setuptools import find_namespace_packages, setup + +ROOT_FOLDER = Path(__file__).parent.absolute() +REQUIREMENTS_FOLDER = ROOT_FOLDER / "requirements" # Since we're importing `redbot` package, we have to ensure that it's in sys.path. -sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) +sys.path.insert(0, str(ROOT_FOLDER)) from redbot import VersionInfo version, _ = VersionInfo._get_version(ignore_installed=True) -if os.getenv("TOX_RED", False) and sys.version_info >= (3, 10): - # We want to be able to test Python versions that we do not support yet. - setup(python_requires=">=3.8.1", version=version) -else: - # Metadata and options defined in setup.cfg - setup(version=version) + +def get_requirements(fp): + return [ + line.strip() + for line in fp.read().splitlines() + if line.strip() and not line.strip().startswith("#") + ] + + +def extras_combined(*extra_names): + return list( + { + req + for extra_name, extra_reqs in extras_require.items() + if not extra_names or extra_name in extra_names + for req in extra_reqs + } + ) + + +with open(REQUIREMENTS_FOLDER / "base.txt", encoding="utf-8") as fp: + install_requires = get_requirements(fp) + +extras_require = {} +for file in REQUIREMENTS_FOLDER.glob("extra-*.txt"): + with file.open(encoding="utf-8") as fp: + extras_require[file.stem[len("extra-") :]] = get_requirements(fp) + +extras_require["dev"] = extras_combined() +extras_require["all"] = extras_combined("postgres") + + +python_requires = ">=3.8.1" +if not os.getenv("TOX_RED", False) or sys.version_info < (3, 10): + python_requires += ",<3.10" + +# Metadata and options defined in pyproject.toml +setup( + version=version, + python_requires=python_requires, + # TODO: use [tool.setuptools.dynamic] table once this feature gets out of beta + install_requires=install_requires, + extras_require=extras_require, + # TODO: use [project] table once PEP 639 gets accepted + license_files=["LICENSE", "redbot/**/*.LICENSE"], + # TODO: use [tool.setuptools.packages] table once this feature gets out of beta + packages=find_namespace_packages(include=["redbot", "redbot.*"]), +) diff --git a/tests/core/test_version.py b/tests/core/test_version.py index 06242cd27..12c55c749 100644 --- a/tests/core/test_version.py +++ b/tests/core/test_version.py @@ -1,7 +1,7 @@ import importlib.metadata -import pkg_resources import os import sys +from packaging.requirements import Requirement import pytest @@ -55,9 +55,9 @@ def test_python_version_has_lower_bound(): requires_python = importlib.metadata.metadata("Red-DiscordBot")["Requires-Python"] assert requires_python is not None - # `pkg_resources` needs a regular requirement string, so "x" serves as requirement's name here - req = pkg_resources.Requirement.parse(f"x{requires_python}") - assert any(op in (">", ">=") for op, version in req.specs) + # Requirement needs a regular requirement string, so "x" serves as requirement's name here + req = Requirement(f"x{requires_python}") + assert any(spec.operator in (">", ">=") for spec in req.specifier) @pytest.mark.skipif( @@ -72,6 +72,6 @@ def test_python_version_has_upper_bound(): requires_python = importlib.metadata.metadata("Red-DiscordBot")["Requires-Python"] assert requires_python is not None - # `pkg_resources` needs a regular requirement string, so "x" serves as requirement's name here - req = pkg_resources.Requirement.parse(f"x{requires_python}") - assert any(op in ("<", "<=") for op, version in req.specs) + # Requirement needs a regular requirement string, so "x" serves as requirement's name here + req = Requirement(f"x{requires_python}") + assert any(spec.operator in ("<", "<=") for spec in req.specifier) diff --git a/tools/bumpdeps.py b/tools/bumpdeps.py deleted file mode 100755 index b6b38d0d0..000000000 --- a/tools/bumpdeps.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3.8 -"""Script to bump pinned dependencies in setup.cfg. - -This script aims to help update our list of pinned primary and -secondary dependencies in *setup.cfg*, using the unpinned primary -dependencies listed in *primary_deps.ini*. - -This script will not work when run on Windows. - -What this script does ---------------------- -It prints to stdout all primary and secondary dependencies for Red, -pinned to the latest possible version, within the constraints specified -in ``primary_deps.ini``. The output should be suitable for copying and -pasting into ``setup.cfg``. PEP 508 markers are preserved. - -How this script works ---------------------- -Overview: -1. Primary dependencies are read from primary_deps.ini using -setuptools' config parser. -2. A clean virtual environment is created in a temporary directory. -3. Core primary dependencies are passed to the ``pip install`` command -for that virtual environment. -4. Pinned primary dependencies are obtained by reading the output of -``pip freeze`` in that virtual environment, and any PEP 508 markers -shown with the requirement in ``primary_deps.ini`` are preserved. -5. Steps 2-4 are repeated for each extra requirement, but care is taken -not to duplicate core dependencies (primary or secondary) in the final -pinned extra dependencies. - -This script makes use of the *packaging* library to parse version -specifiers and environment markers. - -Known Limitations ------------------ -These limitations don't stop this script from being helpful, but -hopefully help explain in which situations some dependencies may need -to be listed manually in ``setup.cfg``. - -1. Whilst environment markers of any primary dependencies specified in -``primary_deps.ini`` are preserved in the output, they will not be -added to secondary dependencies. So for example, if some package -*dep1* has a dependency *dep2*, and *dep1* is listed as a primary -dependency in ``primary_deps.ini`` like follows:: - dep1; sys_platform == "linux" - -Then the output will look like this:: - dep1==1.1.1; sys_platform == "linux" - dep2==2.2.2 - -So even though ``dep1`` and its dependencies should only be installed on -Linux, in reality, its dependencies will be installed regardless. To -work around this, simply list the secondary dependencies in -``primary_deps.ini`` as well, with the environment markers. - -2. If a core requirement and an extra requirement have a common -sub-dependency, there is a chance the sub-dependency will have a version -conflict unless it is manually held back. This script will issue a -warning to stderr when it thinks this might be happening. - -3. Environment markers which exclude dependencies from the system -running this script will cause those dependencies to be excluded from -the output. So for example, if a dependency has the environment marker -``sys_platform == "darwin"``, and the script is being run on linux, then -this dependency will be ignored, and must be added to ``setup.cfg`` -manually. -""" -import shlex -import sys -import subprocess as sp -import tempfile -import textwrap -import venv -from pathlib import Path -from typing import Sequence, Iterable, Dict - -import packaging.requirements -import setuptools.config - -THIS_DIRECTORY = Path(__file__).parent -REQUIREMENTS_INI_PTH: Path = THIS_DIRECTORY / "primary_deps.ini" - -PIP_INSTALL_ARGS = ("install", "--upgrade") -PIP_FREEZE_ARGS = ("freeze", "--no-color") - - -def main() -> int: - if not REQUIREMENTS_INI_PTH.is_file(): - print("No primary_deps.ini found in the same directory as bumpdeps.py", file=sys.stderr) - return 1 - - primary_reqs_cfg = setuptools.config.read_configuration(str(REQUIREMENTS_INI_PTH)) - - print("[options]") - print("install_requires =") - core_primary_deps = primary_reqs_cfg["options"]["install_requires"] - full_core_reqs = get_all_reqs(core_primary_deps) - print(textwrap.indent("\n".join(map(str, full_core_reqs)), " " * 4)) - print() - - print("[options.extras_require]") - for extra, extra_primary_deps in primary_reqs_cfg["options"]["extras_require"].items(): - print(extra, "=") - full_extra_reqs = get_all_reqs( - extra_primary_deps, all_core_deps={r.name.lower(): r for r in full_core_reqs} - ) - print(textwrap.indent("\n".join(map(str, full_extra_reqs)), " " * 4)) - - return 0 - - -def get_all_reqs( - primary_deps: Iterable[str], all_core_deps: Dict[str, packaging.requirements.Requirement] = () -) -> Sequence[packaging.requirements.Requirement]: - reqs_dict = {r.name.lower(): r for r in map(packaging.requirements.Requirement, primary_deps)} - with tempfile.TemporaryDirectory() as tmpdir: - venv.create(tmpdir, system_site_packages=False, clear=True, with_pip=True) - tmpdir_pth = Path(tmpdir) - - pip_exe_pth = tmpdir_pth / "bin" / "pip" - - # Upgrade pip to latest version - sp.run((pip_exe_pth, *PIP_INSTALL_ARGS, "pip"), stdout=sp.DEVNULL, check=True) - - # Install the primary dependencies - sp.run( - (pip_exe_pth, *PIP_INSTALL_ARGS, *map(str, reqs_dict.values())), - stdout=sp.DEVNULL, - check=True, - ) - - # Get pinned primary+secondary dependencies from pip freeze - proc = sp.run( - (pip_exe_pth, *PIP_FREEZE_ARGS), stdout=sp.PIPE, check=True, encoding="utf-8" - ) - - # Return Requirement objects - ret = [] - for req_obj in map(packaging.requirements.Requirement, proc.stdout.strip().split("\n")): - dep_name = req_obj.name.lower() - # Don't include core dependencies if these are extra dependencies - if dep_name in all_core_deps: - if req_obj.specifier != all_core_deps[dep_name].specifier: - print( - f"[WARNING] {dep_name} is listed as both a core requirement and an extra " - f"requirement, and it's possible that their versions conflict!", - file=sys.stderr, - ) - continue - - # Preserve environment markers - if dep_name in reqs_dict: - req_obj.marker = reqs_dict[dep_name].marker - - ret.append(req_obj) - - return ret - - -if __name__ == "__main__": - try: - exit_code = main() - except sp.CalledProcessError as exc: - cmd = " ".join(map(lambda c: shlex.quote(str(c)), exc.cmd)) - print( - f"The following command failed with code {exc.returncode}:\n ", cmd, file=sys.stderr - ) - exit_code = 1 - - sys.exit(exit_code) diff --git a/tools/dev-requirements.txt b/tools/dev-requirements.txt index 962b6480a..d99485829 100644 --- a/tools/dev-requirements.txt +++ b/tools/dev-requirements.txt @@ -1,3 +1,2 @@ -packaging -tox +tox<4 -e .[dev] diff --git a/tools/primary_deps.ini b/tools/primary_deps.ini deleted file mode 100644 index f0617fb0c..000000000 --- a/tools/primary_deps.ini +++ /dev/null @@ -1,46 +0,0 @@ -# primary_deps.ini -# This file should list primary dependencies in terms of both core and -# extras, in setup.cfg format. A primary dependency is one which is -# used directly in Red, or otherwise is forced to be listed as a -# dependency. Version specifiers should be as liberal as possible. - -[options] -install_requires = - aiohttp - aiohttp-json-rpc - aiosqlite - appdirs - apsw-wheels - babel - click - colorama - discord.py - distro; sys_platform == "linux" - fuzzywuzzy - markdown - psutil - python-dateutil - python-Levenshtein-wheels - PyYAML - Red-Commons - Red-Lavalink - rich - schema - uvloop; sys_platform != "win32" and platform_python_implementation == "CPython" - PyNaCl - -[options.extras_require] -docs = - Sphinx - sphinx-prompt - sphinx_rtd_theme - sphinxcontrib-trio -postgres = - asyncpg -style = - black -test = - pylint - pytest - pytest-asyncio - pytest-mock diff --git a/tox.ini b/tox.ini index f613b080a..14f1479c4 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,14 @@ envlist = docs style skip_missing_interpreters = True +isolated_build = True [testenv] description = Run tests and basic automatic issue checking. whitelist_externals = pytest pylint -extras = voice, test +extras = test setenv = TOX_RED = 1 commands = @@ -28,7 +29,7 @@ commands = description = Run pytest with PostgreSQL backend whitelist_externals = pytest -extras = voice, test, postgres +extras = test, postgres setenv = TOX_RED = 1 RED_STORAGE_TYPE=postgres @@ -53,7 +54,7 @@ setenv = # Prioritise make.bat over any make.exe which might be on PATH PATHEXT=.BAT;.EXE basepython = python3.8 -extras = docs +extras = doc commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out/html" -W --keep-going -bhtml sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out/doctest" -W --keep-going -bdoctest