Add redbot-update command for updating Red (#6734)

.github/labeler.yml

@@ -204,6 +204,7 @@
   - docs/cog_guides/core.rst
 "Category: Core - Command-line Interfaces":
   - redbot/__main__.py
+  - redbot/_update/**/*
   - redbot/logging.py
   - redbot/core/_cli.py
   - redbot/core/_debuginfo.py

docs/update_red.rst

@@ -25,13 +25,57 @@ Updating differs depending on the version you currently have. Next sections will
     :depth: 1
 
 
-Red 3.5.0 or newer
+Red 3.5.25 or newer
-******************
+*******************
 
 Windows
 -------
 
-If you have Red 3.5.0 or newer, you can upgrade by following these steps:
+If you have Red 3.5.25 or newer, you can upgrade by following these steps:
+
+#. Shut your bot down.
+#. Activate your venv with the following command:
+
+    .. prompt:: batch
+
+        "%userprofile%\redenv\Scripts\activate.bat"
+#. Update Red with this command:
+
+    .. prompt:: batch
+        :prompts: (redenv) C:\\>
+
+        redbot-update
+#. Start your bot.
+
+Linux & Mac
+-----------
+
+If you have Red 3.5.25 or newer, you can upgrade by following these steps:
+
+#. Shut your bot down.
+#. Activate your virtual environment.
+  
+    If you used ``venv`` for your virtual environment, use:
+
+    .. prompt:: bash
+
+        source ~/redenv/bin/activate
+
+#. Update Red with this command:
+
+    .. prompt:: bash
+        :prompts: (redenv) $
+
+        redbot-update
+#. Start your bot.
+
+Red 3.5.0-3.5.24
+****************
+
+Windows
+-------
+
+If you have a Red version between 3.5.0 and 3.5.24, you can upgrade by following these steps:
 
 #. Shut your bot down.
 #. Activate your venv with the following command:
@@ -55,7 +99,7 @@ If you have Red 3.5.0 or newer, you can upgrade by following these steps:
 Linux & Mac
 -----------
 
-If you have Red 3.5.0 or newer, you can upgrade by following these steps:
+If you have a Red version between 3.5.0 and 3.5.24, you can upgrade by following these steps:
 
 #. Shut your bot down.
 #. Activate your virtual environment.

redbot/_update/__main__.py

@@ -0,0 +1,237 @@
+import asyncio
+from pathlib import Path
+from typing import Any, Final, Optional, Tuple
+
+import click
+from packaging.version import Version
+from python_discovery import PythonInfo
+
+from redbot.core._cli import asyncio_run
+
+from . import cmd, common, updater
+
+
+_CHECK_OTHER_PYTHON_INSTALLS_CMD_ARG_NAME: Final = "--check-other-python-installs"
+
+
+def _help_major_update_example() -> str:
+    version = common.get_current_red_version().__replace__(dev=None, local=None)
+    release = (version.major, version.minor + 1) + (0,) * (len(version.release) - 2)
+    next_major_version = version.__replace__(release=release)
+    return f"updating from Red {version} to Red {next_major_version}"
+
+
+def _help_minor_update_example() -> str:
+    version = common.get_current_red_version().__replace__(dev=None, local=None)
+    release = (version.major, version.minor, version.micro + 1) + (0,) * (len(version.release) - 3)
+    next_minor_version = version.__replace__(release=release)
+    return f"updating from Red {version} to Red {next_minor_version}"
+
+
+class _PythonInfoParamType(click.ParamType):
+    name = "Python interpreter"
+
+    def convert(
+        self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]
+    ) -> PythonInfo:
+        if isinstance(value, PythonInfo):
+            return value
+
+        try:
+            return PythonInfo.from_exe(value)
+        except RuntimeError:
+            self.fail(f"{value!r} is not a valid Python executable.", param, ctx)
+
+
+@click.group(invoke_without_command=True)
+# command-specific options
+@click.option(
+    "--include-instance",
+    "included_instances",
+    multiple=True,
+    type=click.Choice(common.INSTANCE_LIST),
+    help="The list of instances to backup and check cog compatibility for. If not specified,"
+    " all instances that use the current virtual environment will be backed up and checked.",
+)
+@click.option(
+    "--exclude-instance",
+    "excluded_instances",
+    multiple=True,
+    type=click.Choice(common.INSTANCE_LIST),
+    help="Exclude an instance from the list of instances to backup"
+    " and check cog compatibility for.",
+)
+@click.option(
+    "--backup-dir",
+    default=None,
+    type=click.Path(
+        dir_okay=True, file_okay=False, resolve_path=True, writable=True, path_type=Path
+    ),
+    help="The directory to place the backups of the virtual environment and instances.",
+)
+@click.option(
+    "--no-backup",
+    help="Do not make backups of the virtual environment and instances before update.",
+    is_flag=True,
+)
+@click.option(
+    "--red-version",
+    "--version",
+    type=common.VersionParamType(),
+    default=None,
+    help="Version of Red to update to instead of the latest.",
+)
+@click.option(
+    "--no-major-updates",
+    help=f"Skip major updates. For example: {_help_major_update_example()} is a major update"
+    f" but {_help_minor_update_example()} isn't.",
+    is_flag=True,
+)
+@click.option(
+    "--no-full-changelog",
+    help='Skip showing full changelog in a terminal user interface. The "Read before updating"'
+    " sections will still be printed.",
+    is_flag=True,
+)
+@click.option(
+    "--no-cog-compatibility-check",
+    help="Skip performing cog compatibility check before the update.",
+    is_flag=True,
+)
+@click.option(
+    "--new-python-interpreter",
+    type=_PythonInfoParamType(),
+    help="The new Python interpreter that should be used when creating a virtual environment"
+    " for Red. This can either be a path to a Python executable or a name of a Python executable"
+    " on the PATH.",
+)
+@click.option(
+    "--update-cogs/--no-update-cogs",
+    default=None,
+    help="When this option is used, it determines whether the cogs should be updated after Red"
+    " is updated. By default, you'll be asked, if you want to update.\n"
+    "In non-interactive mode, cogs will be updated unless this option is used to override"
+    " the default behavior.",
+)
+@click.option(
+    # `pip install` having an option with the same name is coincidental,
+    # this does not call `pip install` with the `--force-reinstall` option.
+    # Not that there would be any point in doing so - we create a fresh virtual environment.
+    "--force-reinstall",
+    type=bool,
+    is_flag=True,
+    help="Force the update process to proceed even, if there is no new version detected."
+    " This will essentially reinstall latest Red version into a fresh virtual environment. You can"
+    " combine it with the --new-python-interpreter option to change Red's Python interpreter.",
+)
+@click.option(
+    "--no-prompt",
+    "interactive",
+    type=bool,
+    is_flag=True,
+    default=True,
+    help="Don't ask for user input during the process (non-interactive mode).\n"
+    "NOTE: If you want to use this to automate Red updates, consider specifying --no-major-update"
+    " to avoid performing major updates without making an explicit decision to.\n"
+    "When performing a major update where the current Python interpreter is no longer compatible,"
+    " the --new-python-interpreter option has to be specified or the command will fail.",
+)
+# global options
+@click.option(
+    cmd.arg_names.DEBUG,
+    "--verbose",
+    "-v",
+    "logging_level",
+    count=True,
+    help=(
+        "Increase the verbosity of the logs, each usage of this flag increases the verbosity"
+        " level by 1."
+    ),
+)
+@click.option(
+    "--check-other-venvs",
+    _CHECK_OTHER_PYTHON_INSTALLS_CMD_ARG_NAME,
+    "ignore_prefix",
+    help="Check the compatibility of cogs for instances that are normally ran with"
+    " a different Python installation and/or virtual environment than the current one.",
+    is_flag=True,
+)
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    included_instances: Tuple[str, ...],
+    excluded_instances: Tuple[str, ...],
+    backup_dir: Optional[Path],
+    no_backup: bool,
+    red_version: Optional[Version],
+    no_major_updates: bool,
+    no_full_changelog: bool,
+    no_cog_compatibility_check: bool,
+    new_python_interpreter: Optional[PythonInfo],
+    update_cogs: Optional[bool],
+    force_reinstall: bool,
+    interactive: bool,
+    logging_level: int,
+    ignore_prefix: bool,
+) -> None:
+    common.ensure_supported_env()
+    common.configure_logging(logging_level)
+
+    ctx.ensure_object(dict)
+    ctx.obj["IGNORE_PREFIX"] = ignore_prefix
+
+    if ctx.invoked_subcommand is None:
+        if included_instances:
+            # de-duplicate with order intact
+            instances = list(dict.fromkeys(included_instances))
+        else:
+            instances = list(common.INSTANCE_LIST)
+        options = updater.UpdaterOptions(
+            instances=instances,
+            excluded_instances=set(excluded_instances),
+            ignore_prefix=ignore_prefix,
+            backup_dir=backup_dir,
+            no_backup=no_backup,
+            red_version=red_version,
+            no_major_updates=no_major_updates,
+            no_full_changelog=no_full_changelog,
+            no_cog_compatibility_check=no_cog_compatibility_check,
+            new_python_interpreter=new_python_interpreter,
+            update_cogs=update_cogs,
+            force_reinstall=force_reinstall,
+            interactive=interactive,
+        )
+        app = updater.Updater(options)
+        asyncio_run(app.run())
+    # these should not be available to subcommands
+    elif included_instances:
+        raise click.NoSuchOption("--include-instance", ctx=ctx)
+    elif excluded_instances:
+        raise click.NoSuchOption("--exclude-instance", ctx=ctx)
+    elif backup_dir is not None:
+        raise click.NoSuchOption("--backup-dir", ctx=ctx)
+    elif no_backup:
+        raise click.NoSuchOption("--no-backup", ctx=ctx)
+    elif red_version:
+        raise click.NoSuchOption("--red-version", ctx=ctx)
+    elif no_major_updates:
+        raise click.NoSuchOption("--no-major-updates", ctx=ctx)
+    elif no_cog_compatibility_check:
+        raise click.NoSuchOption("--no-cog-compatibility-check", ctx=ctx)
+    elif new_python_interpreter:
+        raise click.NoSuchOption("--new-python-interpreter", ctx=ctx)
+    elif update_cogs is True:
+        raise click.NoSuchOption("--update-cogs", ctx=ctx)
+    elif update_cogs is False:
+        raise click.NoSuchOption("--no-update-cogs", ctx=ctx)
+    elif not interactive:
+        raise click.NoSuchOption("--no-prompt", ctx=ctx)
+    elif force_reinstall:
+        raise click.NoSuchOption("--force-reinstall", ctx=ctx)
+
+
+cli.add_command(cmd.cog_compatibility.check_cog_compatibility)
+
+
+if __name__ == "__main__":
+    cli()

redbot/_update/changelog.py

@@ -0,0 +1,144 @@
+import dataclasses
+import datetime
+import functools
+import os
+import re
+from typing import Any, Dict, List
+
+import aiohttp
+import yarl
+from packaging.version import Version
+from typing_extensions import Self
+
+
+_CHANGELOG_PATTERN = re.compile(
+    r"\n<!--+ +RED-CHANGELOG-BEGIN: (?P<version>.+) +--+>\n"
+    r"(?P<content>[\s\S]+?)"
+    r"\n<!--+ +RED-CHANGELOG-END +--+>"
+)
+_RTD_CANONICAL_URL = os.getenv("_RED_RTD_CANONICAL_URL") or "https://docs.discord.red/en/stable/"
+
+
+@dataclasses.dataclass
+class VersionChangelog:
+    version: Version
+    content: str
+    _RELEASE_DATE_PATTERN = re.compile(
+        r"^<!--+ +RED-CHANGELOG-RELEASE-DATE: (\d{4})-(\d{2})-(\d{2}) +--+>$",
+        re.MULTILINE,
+    )
+    _CONTRIBUTORS_PATTERN = re.compile(
+        r"^<!--+ +RED-CHANGELOG-CONTRIBUTORS: (?P<contributors>.+) +--+>$",
+        re.MULTILINE,
+    )
+    _READ_BEFORE_UPDATING_SECTION_PATTERN = re.compile(
+        r"\n<!--+ +RED-CHANGELOG-READ-BEFORE-UPDATE-BEGIN +--+>\n"
+        r"(?P<content>[\s\S]+?)"
+        r"\n<!--+ +RED-CHANGELOG-READ-BEFORE-UPDATE-END +--+>"
+    )
+    _USER_CHANGELOG_SECTION_PATTERN = re.compile(
+        r"\n<!--+ +RED-CHANGELOG-USER-CHANGELOG-BEGIN +--+>\n"
+        r"(?P<content>[\s\S]+?)"
+        r"\n<!--+ +RED-CHANGELOG-USER-CHANGELOG-END +--+>"
+    )
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        return cls(version=Version(data["version"]), content=data["content"])
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {"version": str(self.version), "content": self.content}
+
+    @functools.cached_property
+    def release_date(self) -> datetime.date:
+        return datetime.date(*map(int, self._RELEASE_DATE_PATTERN.search(self.content).groups()))
+
+    @functools.cached_property
+    def contributors(self) -> List[str]:
+        match = self._CONTRIBUTORS_PATTERN.search(self.content)
+        if match is None:
+            return []
+        return match["contributors"].split()
+
+    @functools.cached_property
+    def read_before_updating_section(self) -> str:
+        return "\n".join(
+            match["content"].strip()
+            for match in self._READ_BEFORE_UPDATING_SECTION_PATTERN.finditer(self.content)
+        )
+
+    @functools.cached_property
+    def user_changelog_section(self) -> str:
+        return "\n".join(
+            match["content"].strip()
+            for match in self._USER_CHANGELOG_SECTION_PATTERN.finditer(self.content)
+        )
+
+
+Changelogs = Dict[Version, VersionChangelog]
+
+
+def parse_changelogs(content: str) -> Changelogs:
+    changelogs = {}
+    for match in _CHANGELOG_PATTERN.finditer(content):
+        changelog = VersionChangelog(Version(match["version"]), match["content"])
+        changelogs[changelog.version] = changelog
+
+    return changelogs
+
+
+def render_markdown(changelogs: Changelogs, *, minimal: bool = False) -> str:
+    if not changelogs:
+        return ""
+
+    parts = ["# Read before updating"]
+    for changelog in reversed(changelogs.values()):
+        parts.append(f"## {changelog.version}")
+        parts.append(changelog.read_before_updating_section)
+
+    contributors = sorted(
+        {
+            contributor
+            for changelog in changelogs.values()
+            for contributor in changelog.contributors
+        }
+    )
+    if contributors:
+        contributor_thanks = (
+            "  \n**The releases below were made with help from the following people:**  \n"
+        )
+        contributor_thanks += ", ".join(
+            f"[@{contributor}](https://github.com/sponsors/{contributor})"
+            for contributor in contributors
+        )
+        contributor_thanks += "  \n**Thank you** \N{HEAVY BLACK HEART}\N{VARIATION SELECTOR-16}"
+        parts.append(contributor_thanks)
+
+    # show the header both at the top and the bottom
+    parts.append(parts[0])
+
+    return "\n".join(parts)
+
+
+def get_changelogs_between(
+    changelogs: Changelogs, newer_than: Version, not_newer_than: Version
+) -> Changelogs:
+    return {
+        changelog_version: changelog
+        for changelog_version, changelog in changelogs.items()
+        if newer_than < changelog_version <= not_newer_than
+    }
+
+
+async def fetch_changelogs() -> Changelogs:
+    """
+    Fetch the Markdown-formatted changelog from Red's docs site.
+
+    Returns
+    -------
+    Dict[Version, VersionChangelog]
+        A dict mapping versions to their changelogs. Sorted by version, newest first.
+    """
+    async with aiohttp.ClientSession(raise_for_status=True) as session:
+        async with session.get(yarl.URL(_RTD_CANONICAL_URL) / "_markdown/changelog.md") as resp:
+            return parse_changelogs(await resp.text())

redbot/_update/cmd/__init__.py

@@ -0,0 +1,6 @@
+from . import arg_names, cog_compatibility
+
+__all__ = (
+    "arg_names",
+    "cog_compatibility",
+)

redbot/_update/cmd/arg_names.py

@@ -0,0 +1,6 @@
+from typing import Final
+
+DEBUG: Final = "--debug"
+RED_VERSION: Final = "--red-version"
+PYTHON_VERSION: Final = "--python-version"
+CHECK_OTHER_PYTHON_INSTALLS: Final = "--check-other-python-installs"

redbot/_update/cmd/cog_compatibility.py

@@ -0,0 +1,215 @@
+import asyncio
+import json
+import os
+import sys
+import tempfile
+from typing import Final, Optional, Tuple
+
+import click
+from packaging.version import Version
+from rich.text import Text
+
+from redbot._update import cog_compatibility_checker, common
+from redbot._update.cog_compatibility_checker import CompatibilitySummary
+from redbot.core import _drivers
+from redbot.core._cli import asyncio_run
+from redbot.core.utils._internal_utils import fetch_latest_red_version
+
+from . import arg_names
+
+
+EXIT_INSTANCE_SITE_PREFIX_MISMATCH: Final = 4
+EXIT_INSTANCE_BACKEND_UNSUPPORTED: Final = 5
+CMD_NAME: Final = "check-cog-compatibility"
+_COMPATIBILITY_RESULTS_ENV_VAR = "_RED_UPDATE_COMPATIBILITY_RESULTS_FILE"
+
+
+@click.command(CMD_NAME)
+@click.argument(
+    "instances",
+    nargs=-1,
+    type=click.Choice(common.INSTANCE_LIST),
+    default=None,
+    metavar="[INSTANCE_NAME]",
+)
+@click.option(
+    arg_names.RED_VERSION,
+    type=common.VersionParamType(),
+    default=None,
+    help="The Red version to check cog compatibility for."
+    " If not provided, the information about latest available version will be fetched"
+    " and the command will check whether installed cogs support that version.\n"
+    "If this option is provided, --python-version also has to be provided.",
+)
+@click.option(
+    arg_names.PYTHON_VERSION,
+    type=common.VersionParamType(),
+    default=None,
+    help="The Python version to check cog compatibility for."
+    " If not provided, the command will either use the current interpreter's version or,"
+    " if that version is not compatible with the latest Red version, it will try to"
+    " find the latest available CPython interpreter on the system and will check whether"
+    " installed cogs support it.\n"
+    "If this option is provided, --red-version also has to be provided.",
+)
+@click.pass_context
+def check_cog_compatibility(
+    ctx: click.Context,
+    instances: Tuple[str, ...],
+    red_version: Optional[Version],
+    python_version: Optional[Version],
+) -> None:
+    """
+    Check if the installed cogs are compatible with the given version.
+    """
+    if (red_version, python_version).count(None) == 1:
+        raise click.BadParameter(
+            "Either both --red-version and --python-version options"
+            " have to be specified or neither.",
+            param_hint=[arg_names.RED_VERSION, arg_names.PYTHON_VERSION],
+        )
+
+    asyncio_run(
+        _check_cog_compatibility_command_impl(
+            red_version=red_version,
+            python_version=python_version,
+            instances=instances,
+            ignore_prefix=ctx.obj["IGNORE_PREFIX"],
+        )
+    )
+
+
+async def _check_cog_compatibility_command_impl(
+    *,
+    red_version: Optional[Version],
+    python_version: Optional[Version],
+    instances: Tuple[str, ...] = (),
+    ignore_prefix: bool = False,
+) -> None:
+    console = common.get_console()
+    if red_version is None or python_version is None:
+        with console.status("Checking latest version..."):
+            latest = await fetch_latest_red_version()
+            red_version = latest.version
+
+        python_version = Version(".".join(map(str, sys.version_info[:3])))
+        if python_version not in latest.requires_python:
+            interpreters = common.search_for_interpreters(latest.requires_python)
+            _, python_version, _ = interpreters[0]
+
+    if len(instances) == 1:
+        results_file = os.getenv(_COMPATIBILITY_RESULTS_ENV_VAR, "")
+        try:
+            results = await cog_compatibility_checker.check_instance(
+                instances[0],
+                latest_version=red_version,
+                interpreter_version=python_version,
+                ignore_prefix=ignore_prefix,
+            )
+        except _drivers.MissingExtraRequirements:
+            if not results_file:
+                common.print_with_prefix_column(
+                    common.ICON_ERROR,
+                    Text(instances[0], style="bold"),
+                    " instance could not be checked as it uses a storage backend"
+                    " that is not supported by the current Red installation"
+                    " (some requirements are missing).",
+                )
+            raise SystemExit(EXIT_INSTANCE_BACKEND_UNSUPPORTED)
+        except cog_compatibility_checker.InstanceSitePrefixMismatchError as exc:
+            if not results_file:
+                common.print_with_prefix_column(
+                    common.ICON_ERROR,
+                    Text(exc.instance_name, style="bold"),
+                    " instance could not be checked as it is a part of"
+                    " a different Python installation and/or virtual environment.",
+                )
+            raise SystemExit(EXIT_INSTANCE_SITE_PREFIX_MISMATCH)
+        if results_file:
+            with open(results_file, "w", encoding="utf-8") as fp:
+                json.dump(results.to_json_dict(), fp)
+        return
+
+    if not instances:
+        instances = tuple(common.INSTANCE_LIST)
+    checked_instances = []
+    for instance_name in instances:
+        exit_code, _, _ = await call(
+            instance_name,
+            red_version=red_version,
+            python_version=python_version,
+            ignore_prefix=ignore_prefix,
+        )
+        if exit_code != EXIT_INSTANCE_SITE_PREFIX_MISMATCH:
+            if exit_code:
+                raise SystemExit(exit_code)
+            checked_instances.append(instance_name)
+
+    if not checked_instances:
+        common.print_with_prefix_column(
+            common.ICON_ERROR, "There were no instances to check cog compatibility for."
+        )
+        raise SystemExit(1)
+
+
+async def call(
+    instance_name: str,
+    *,
+    red_version: Version,
+    python_version: Version,
+    ignore_prefix: bool = False,
+    return_results: bool = False,
+    stdout: Optional[int] = None,
+) -> Tuple[int, Optional[str], Optional[CompatibilitySummary]]:
+    debug_args = (arg_names.DEBUG,) * common.get_log_cli_level()
+    args = [
+        "-m",
+        "redbot._update",
+        *debug_args,
+        CMD_NAME,
+        instance_name,
+        arg_names.RED_VERSION,
+        str(red_version),
+        arg_names.PYTHON_VERSION,
+        str(python_version),
+    ]
+    if ignore_prefix:
+        args.append(arg_names.CHECK_OTHER_PYTHON_INSTALLS)
+    env = os.environ.copy()
+
+    # terminal woes
+    console = common.get_console()
+    if console.is_terminal:
+        env["TTY_COMPATIBLE"] = "1"
+        # Rich only checks stdout for Windows console features:
+        # https://github.com/Textualize/rich/blob/fc41075a3206d2a5fd846c6f41c4d2becab814fa/rich/_windows.py#L46
+        env[common.INTERNAL_LEGACY_WINDOWS_ENV_VAR] = "1" if console.legacy_windows else "0"
+    else:
+        # Rich does not set legacy_windows correctly when is_terminal is False
+        # https://github.com/Textualize/rich/issues/3647
+        env[common.INTERNAL_LEGACY_WINDOWS_ENV_VAR] = "0"
+    env["PYTHONIOENCODING"] = sys.stdout.encoding
+
+    results = None
+    results_file = None
+    if return_results:
+        results_file = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        if results_file is not None:
+            results_file.close()
+            env[_COMPATIBILITY_RESULTS_ENV_VAR] = str(results_file.name)
+
+        proc = await asyncio.create_subprocess_exec(sys.executable, *args, env=env, stdout=stdout)
+        stdout_data, _ = await proc.communicate()
+        decoded_stdout = None
+        if stdout_data is not None:
+            decoded_stdout = stdout_data.decode()
+        exit_code = await proc.wait()
+        if not exit_code and results_file is not None:
+            with open(results_file.name, encoding="utf-8") as fp:
+                results = CompatibilitySummary.from_json_dict(json.load(fp))
+    finally:
+        if results_file is not None:
+            os.remove(results_file.name)
+
+    return exit_code, decoded_stdout, results

redbot/_update/cog_compatibility_checker.py

@@ -0,0 +1,551 @@
+import dataclasses
+import enum
+import functools
+import itertools
+import os
+import sys
+from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Set, Tuple
+
+import rich
+from packaging.version import Version
+from rich.text import Text
+from typing_extensions import Self
+
+from redbot.core import _downloader, _drivers, data_manager
+from redbot.core._cli import parse_cli_flags
+from redbot.core.bot import Red
+from redbot.core.utils._internal_utils import detailed_progress
+
+from . import common
+
+
+class InstanceSitePrefixMismatchError(Exception):
+    """The instance's last known sys.prefix is different from the current one."""
+
+    def __init__(self, instance_name: str, last_known_prefix: Optional[str]) -> None:
+        self.instance_name = instance_name
+        self.last_known_prefix = last_known_prefix
+        super().__init__(
+            f"The last known sys.prefix of {instance_name!r} is different from"
+            " current process's sys.prefix.",
+        )
+
+
+class SimpleCompatibilityStatus(common.OrderedEnum):
+    UNSUPPORTED = enum.auto()
+    POTENTIALLY_SUPPORTED = enum.auto()
+    EXPLICITLY_SUPPORTED = enum.auto()
+
+
+class CompatibilityStatus(enum.Enum):
+    # unsupported is <100, 200)
+    UNSUPPORTED_PYTHON_VERSION = 100
+    UNSUPPORTED_BOT_VERSION = 101
+    # potentially supported is <200, 300)
+    POTENTIALLY_SUPPORTED = 200
+    # explicitly supported is <300, 400)
+    EXPLICITLY_SUPPORTED_NON_BREAKING = 300
+    EXPLICITLY_SUPPORTED_MIN_BOT_VERSION = 301
+    EXPLICITLY_SUPPORTED_MAX_BOT_VERSION = 302
+    EXPLICITLY_SUPPORTED_READY_TAG = 303
+
+    @property
+    def simple_status(self) -> SimpleCompatibilityStatus:
+        if self.unsupported:
+            return SimpleCompatibilityStatus.UNSUPPORTED
+        if self.potentially_supported:
+            return SimpleCompatibilityStatus.POTENTIALLY_SUPPORTED
+        if self.explicitly_supported:
+            return SimpleCompatibilityStatus.EXPLICITLY_SUPPORTED
+        raise RuntimeError("unreachable")
+
+    @property
+    def unsupported(self) -> bool:
+        return 100 <= self.value < 200
+
+    @property
+    def potentially_supported(self) -> bool:
+        return 200 <= self.value < 300
+
+    @property
+    def explicitly_supported(self) -> bool:
+        return 300 <= self.value < 400
+
+    def __ge__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.simple_status >= other.simple_status
+        return NotImplemented
+
+    def __gt__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.simple_status > other.simple_status
+        return NotImplemented
+
+    def __le__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.simple_status <= other.simple_status
+        return NotImplemented
+
+    def __lt__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.simple_status < other.simple_status
+        return NotImplemented
+
+
+@dataclasses.dataclass
+class CogCompatibilityInfo:
+    name: str
+    repo_name: str
+    min_bot_version: Version
+    max_bot_version: Version
+    min_python_version: Version
+    tags: Tuple[str, ...]
+    compatibility_status: CompatibilityStatus = CompatibilityStatus.POTENTIALLY_SUPPORTED
+
+    @classmethod
+    def from_installable(cls, installable: _downloader.Installable) -> Self:
+        return cls(
+            name=installable.name,
+            repo_name=installable.repo_name,
+            min_bot_version=installable.min_bot_version,
+            max_bot_version=installable.max_bot_version,
+            min_python_version=installable.min_python_version,
+            tags=installable.tags,
+        )
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        return cls(
+            name=data["name"],
+            repo_name=data["repo_name"],
+            min_bot_version=Version(data["min_bot_version"]),
+            max_bot_version=Version(data["max_bot_version"]),
+            min_python_version=Version(data["min_python_version"]),
+            tags=tuple(data["tags"]),
+            compatibility_status=CompatibilityStatus(data["compatibility_status"]),
+        )
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "repo_name": self.repo_name,
+            "min_bot_version": str(self.min_bot_version),
+            "max_bot_version": str(self.max_bot_version),
+            "min_python_version": str(self.min_python_version),
+            "tags": self.tags,
+            "compatibility_status": self.compatibility_status.value,
+        }
+
+
+CogSupportDict = Dict[str, CogCompatibilityInfo]
+
+
+@dataclasses.dataclass(frozen=True)
+class CompatibilityResults(Mapping[str, CogCompatibilityInfo]):
+    latest_version: Version
+    interpreter_version: Version
+
+    explicitly_supported: CogSupportDict = dataclasses.field(default_factory=dict)
+    potentially_supported: CogSupportDict = dataclasses.field(default_factory=dict)
+    incompatible_python_version: CogSupportDict = dataclasses.field(default_factory=dict)
+    incompatible_bot_version: CogSupportDict = dataclasses.field(default_factory=dict)
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        return cls(
+            latest_version=Version(data["latest_version"]),
+            interpreter_version=Version(data["interpreter_version"]),
+            explicitly_supported={
+                cog_name: CogCompatibilityInfo.from_json_dict(info_data)
+                for cog_name, info_data in data["explicitly_supported"].items()
+            },
+            potentially_supported={
+                cog_name: CogCompatibilityInfo.from_json_dict(info_data)
+                for cog_name, info_data in data["potentially_supported"].items()
+            },
+            incompatible_python_version={
+                cog_name: CogCompatibilityInfo.from_json_dict(info_data)
+                for cog_name, info_data in data["incompatible_python_version"].items()
+            },
+            incompatible_bot_version={
+                cog_name: CogCompatibilityInfo.from_json_dict(info_data)
+                for cog_name, info_data in data["incompatible_bot_version"].items()
+            },
+        )
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {
+            "latest_version": str(self.latest_version),
+            "interpreter_version": str(self.interpreter_version),
+            "explicitly_supported": {
+                cog_name: info.to_json_dict()
+                for cog_name, info in self.explicitly_supported.items()
+            },
+            "potentially_supported": {
+                cog_name: info.to_json_dict()
+                for cog_name, info in self.potentially_supported.items()
+            },
+            "incompatible_python_version": {
+                cog_name: info.to_json_dict()
+                for cog_name, info in self.incompatible_python_version.items()
+            },
+            "incompatible_bot_version": {
+                cog_name: info.to_json_dict()
+                for cog_name, info in self.incompatible_bot_version.items()
+            },
+        }
+
+    def __getitem__(self, key: str) -> CogCompatibilityInfo:
+        for data in (
+            self.explicitly_supported,
+            self.potentially_supported,
+            self.incompatible_python_version,
+            self.incompatible_bot_version,
+        ):
+            try:
+                return data[key]
+            except KeyError:
+                pass
+        raise KeyError(key)
+
+    def __iter__(self) -> Iterator[str]:
+        return itertools.chain(
+            self.explicitly_supported.keys(),
+            self.potentially_supported.keys(),
+            self.incompatible_python_version.keys(),
+            self.incompatible_bot_version.keys(),
+        )
+
+    def __len__(self) -> int:
+        count = 0
+        for data in (
+            self.explicitly_supported,
+            self.potentially_supported,
+            self.incompatible_python_version,
+            self.incompatible_bot_version,
+        ):
+            count += len(data)
+        return count
+
+    def __bool__(self) -> bool:
+        return any(
+            (
+                self.explicitly_supported,
+                self.potentially_supported,
+                self.incompatible_python_version,
+                self.incompatible_bot_version,
+            )
+        )
+
+    def print(self) -> None:
+        major_version = Text(f"{self.latest_version.major}.{self.latest_version.minor}")
+        if self.explicitly_supported:
+            common.print_with_prefix_column(
+                common.ICON_SUCCESS,
+                "The following cogs are explicitly marked as supporting Red ",
+                major_version,
+                ":\n",
+                Text(", ").join(Text(cog, style="bold") for cog in self.explicitly_supported),
+            )
+        if self.potentially_supported:
+            common.print_with_prefix_column(
+                common.ICON_WARN,
+                "The following cogs may support Red ",
+                major_version,
+                " but they haven't been explicitly marked as such:\n",
+                Text(", ").join(Text(cog, style="bold") for cog in self.potentially_supported),
+            )
+        if self.incompatible_bot_version:
+            common.print_with_prefix_column(
+                common.ICON_ERROR,
+                "The following cogs do not support Red ",
+                Text(str(self.latest_version)),
+                ":\n",
+                Text(", ").join(Text(cog, style="bold") for cog in self.incompatible_bot_version),
+            )
+        if self.incompatible_python_version:
+            common.print_with_prefix_column(
+                common.ICON_ERROR,
+                "The following cogs do not support Python ",
+                Text(str(self.interpreter_version)),
+                ":\n",
+                Text(", ").join(
+                    Text(cog, style="bold") for cog in self.incompatible_python_version
+                ),
+            )
+        if not self.explicitly_supported and (
+            self.potentially_supported
+            or self.incompatible_bot_version
+            or self.incompatible_python_version
+        ):
+            common.print_with_prefix_column(
+                common.ICON_INFO,
+                "None of the checked cogs were explicitly marked as supporting Red ",
+                major_version,
+                ".",
+            )
+
+
+@dataclasses.dataclass(frozen=True)
+class CompatibilitySummary:
+    instance_name: str
+    before_update: CompatibilityResults
+    after_update: CompatibilityResults
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        return cls(
+            instance_name=data["instance_name"],
+            before_update=CompatibilityResults.from_json_dict(data["before_update"]),
+            after_update=CompatibilityResults.from_json_dict(data["after_update"]),
+        )
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {
+            "instance_name": self.instance_name,
+            "before_update": self.before_update.to_json_dict(),
+            "after_update": self.after_update.to_json_dict(),
+        }
+
+
+class CogCompatibilityChecker:
+    def __init__(
+        self,
+        bot: Red,
+        *,
+        latest_version: Version,
+        interpreter_version: Version,
+        ignore_prefix: bool = False,
+    ) -> None:
+        self.bot = bot
+        self.latest_version = latest_version
+        self.interpreter_version = interpreter_version
+        self.ignore_prefix = ignore_prefix
+        self._console = common.get_console(stderr=True)
+        self._stdout_console = common.get_console()
+
+    @functools.cached_property
+    def current_version(self) -> Version:
+        return common.get_current_red_version()
+
+    async def check(self) -> CompatibilitySummary:
+        instance_name = data_manager.instance_name()
+        if not self.ignore_prefix:
+            last_known_prefix = await self.bot._config.last_system_info.python_prefix()
+            same_install = False
+            if last_known_prefix is not None:
+                try:
+                    same_install = os.path.samefile(last_known_prefix, sys.prefix)
+                except OSError:
+                    pass
+            if not same_install:
+                raise InstanceSitePrefixMismatchError(instance_name, last_known_prefix)
+
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            "Started checking cog compatibility for the ",
+            Text(instance_name, style="bold"),
+            " instance.",
+            console=self._console,
+        )
+        status = Text.assemble(
+            "Checking compatibility of cogs installed on the ",
+            (instance_name, "bold"),
+            " instance...",
+        )
+        with self._console.status(status):
+            await _downloader._init_without_bot(self.bot._cog_mgr)
+
+            await self._update_repos()
+
+            installed_cogs = await _downloader.installed_cogs()
+            repo_unknown = []
+            to_check = set()
+
+            for cog in installed_cogs:
+                if cog.repo is None:
+                    repo_unknown.append(cog)
+                else:
+                    to_check.add(cog)
+
+            with self._console.status("Checking available cog updates..."):
+                update_check_result = await _downloader.check_cog_updates(
+                    cogs=to_check,
+                    update_repos=False,
+                    env=_downloader.Environment(
+                        red_version=self.latest_version, python_version=self.interpreter_version
+                    ),
+                )
+            self._console.print("Available cog updates checked.")
+
+            summary = CompatibilitySummary(
+                instance_name=instance_name,
+                before_update=self._evaluate_before_update_compatibility(to_check),
+                after_update=self._evaluate_after_update_compatibility(
+                    to_check, update_check_result
+                ),
+            )
+
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            "Finished checking cog compatibility for the ",
+            Text(instance_name, style="bold"),
+            " instance.",
+            console=self._console,
+        )
+
+        self._stdout_console.print()
+
+        # Note that when a cog can be updated
+        # and its up-to-date version does not support the Red version we're updating to,
+        # we don't check whether currently installed version of the cog supports that Red version.
+        # This is intentional - we want to allow cog creators to mark something incompatible
+        # after the fact.
+        summary.after_update.print()
+
+        return summary
+
+    async def _update_repos(self) -> None:
+        with detailed_progress(unit="repos", console=self._console) as progress:
+            task_id = progress.add_task(
+                "Updating repos", total=len(_downloader._repo_manager.repos)
+            )
+            updated_count = 0
+            already_up_to_date_count = 0
+            failed_count = 0
+            for repo in _downloader._repo_manager.repos:
+                progress.update(task_id, description=f"Updating {repo.name!r} repo")
+                try:
+                    old, new = await repo.update()
+                except _downloader.errors.UpdateError:
+                    common.print_with_prefix_column(
+                        common.ICON_WARN,
+                        "Could not update repo ",
+                        Text(repo.name, style="bold"),
+                        ", the results for cogs from it may be inaccurate.",
+                        console=self._console,
+                    )
+                    failed_count += 1
+                else:
+                    if old != new:
+                        updated_count += 1
+                        self._console.print("Updated repo", Text(repo.name, style="bold"))
+                    else:
+                        already_up_to_date_count += 1
+                        self._console.print(
+                            "Repo", Text(repo.name, style="bold"), "is already up-to-date."
+                        )
+                progress.advance(task_id)
+
+        self._stdout_console.print(
+            f"Successfully updated {updated_count} repos, failed to update {failed_count} repos.\n"
+            f"{already_up_to_date_count} repos were already up-to-date.",
+            highlight=True,
+        )
+
+    def _fill_compatibility_results(
+        self, results: CompatibilityResults, cogs: Iterable[_downloader.Installable]
+    ) -> None:
+        latest_version = self.latest_version
+        interpreter_version = self.interpreter_version
+        breaking_update = self.current_version.release[:2] != self.latest_version.release[:2]
+
+        for cog in cogs:
+            info = CogCompatibilityInfo.from_installable(cog)
+            if cog.min_python_version > interpreter_version:
+                info.compatibility_status = CompatibilityStatus.UNSUPPORTED_PYTHON_VERSION
+                results.incompatible_python_version[cog.name] = info
+            elif cog.min_bot_version > latest_version or (
+                # max version should be ignored when it's lower than min version
+                cog.min_bot_version <= cog.max_bot_version
+                and cog.max_bot_version < latest_version
+            ):
+                info.compatibility_status = CompatibilityStatus.UNSUPPORTED_BOT_VERSION
+                results.incompatible_bot_version[cog.name] = info
+            elif not breaking_update:
+                info.compatibility_status = CompatibilityStatus.EXPLICITLY_SUPPORTED_NON_BREAKING
+                results.explicitly_supported[cog.name] = info
+            elif latest_version.release[:2] == cog.min_bot_version.release[:2]:
+                # If cog creator explicitly set min_bot_version to 3.x.y,
+                # then 3.x is explicitly supported.
+                info.compatibility_status = (
+                    CompatibilityStatus.EXPLICITLY_SUPPORTED_MIN_BOT_VERSION
+                )
+                results.explicitly_supported[cog.name] = info
+            elif latest_version.release[:2] == cog.max_bot_version.release[:2]:
+                # If cog creator explicitly set max_bot_version to 3.x.y,
+                # then 3.x is explicitly supported.
+                info.compatibility_status = (
+                    CompatibilityStatus.EXPLICITLY_SUPPORTED_MAX_BOT_VERSION
+                )
+                results.explicitly_supported[cog.name] = info
+            elif f"red-{latest_version.major}-{latest_version.minor}-ready" in cog.tags:
+                # If cog creator explicitly added a "red-3.x-ready" tag,
+                # then 3.x is explicitly supported.
+                # This is similar to the meaning of "Programming Language :: Python :: 3.x"
+                # classifiers in Python packaging.
+                info.compatibility_status = CompatibilityStatus.EXPLICITLY_SUPPORTED_READY_TAG
+                results.explicitly_supported[cog.name] = info
+            else:
+                # If we don't have any explicit signals from the cog's metadata that
+                # Red 3.x is supported, the cog is only *potentially* supported by that version.
+                info.compatibility_status = CompatibilityStatus.POTENTIALLY_SUPPORTED
+                results.potentially_supported[cog.name] = info
+
+    def _evaluate_before_update_compatibility(
+        self, to_check: Iterable[_downloader.Installable]
+    ) -> CompatibilityResults:
+        results = CompatibilityResults(
+            latest_version=self.latest_version, interpreter_version=self.interpreter_version
+        )
+
+        self._fill_compatibility_results(results, to_check)
+
+        return results
+
+    def _evaluate_after_update_compatibility(
+        self,
+        to_check: Iterable[_downloader.Installable],
+        update_check_result: _downloader.CogUpdateCheckResult,
+    ) -> CompatibilityResults:
+        not_updatable = set(to_check)
+        results = CompatibilityResults(
+            latest_version=self.latest_version, interpreter_version=self.interpreter_version
+        )
+
+        not_updatable.difference_update(update_check_result.incompatible_python_version)
+        not_updatable.difference_update(update_check_result.incompatible_bot_version)
+        not_updatable.difference_update(update_check_result.updatable_cogs)
+
+        self._fill_compatibility_results(results, update_check_result.incompatible_python_version)
+        self._fill_compatibility_results(results, update_check_result.incompatible_bot_version)
+        self._fill_compatibility_results(results, update_check_result.updatable_cogs)
+
+        # not_updatable should now only have cogs that were not updateable. Those cogs
+        # are filled based on metadata of the currently installed ("before update") version.
+        self._fill_compatibility_results(results, not_updatable)
+
+        return results
+
+
+async def check_instance(
+    instance: str,
+    *,
+    latest_version: Version,
+    interpreter_version: Version,
+    ignore_prefix: bool = False,
+) -> CompatibilitySummary:
+    data_manager.load_basic_configuration(instance)
+    red = Red(cli_flags=parse_cli_flags([instance]))
+    driver_cls = _drivers.get_driver_class()
+    await driver_cls.initialize(**data_manager.storage_details())
+    try:
+        checker = CogCompatibilityChecker(
+            red,
+            latest_version=latest_version,
+            interpreter_version=interpreter_version,
+            ignore_prefix=ignore_prefix,
+        )
+        return await checker.check()
+    finally:
+        await driver_cls.teardown()

redbot/_update/common.py

@@ -0,0 +1,221 @@
+import enum
+import logging
+import os
+import sys
+from operator import itemgetter
+from typing import Any, Final, Iterable, List, Literal, Optional, Tuple, Union
+
+import click
+import rich
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+from python_discovery import PythonInfo, get_interpreter
+from rich.console import Console, RenderableType
+from rich.logging import RichHandler
+from rich.table import Table
+from rich.text import Text
+
+from redbot import __version__
+from redbot.core.utils._internal_utils import (
+    cli_level_to_log_level,
+    get_installed_extras,
+    log_level_to_cli_level,
+)
+from redbot.core import data_manager
+
+_instance_data = data_manager.load_existing_config()
+INSTANCE_LIST: Final = () if _instance_data is None else tuple(_instance_data.keys())
+
+
+ICON_SUCCESS = "[green]:white_heavy_check_mark-emoji:[/]"
+ICON_INFO = "[blue]:information-emoji:[/]"
+ICON_WARN = "[yellow]:warning-emoji:[/]"
+ICON_ERROR = "[red]:cross_mark-emoji:[/]"
+
+INTERNAL_LEGACY_WINDOWS_ENV_VAR = "_RED_UPDATE_INTERNAL_LEGACY_WINDOWS"
+INTERNAL_UPDATER_METADATA_ENV_VAR = "_RED_UPDATE_INTERNAL_UPDATER_METADATA"
+_STDERR_CONSOLE: Optional[Console] = None
+
+RUNNER_DIR_ENV_VAR: Final = "REDBOT_UPDATE_RUNNER_DIR"
+RUNNER_WRAPPER_EXE_ENV_VAR: Final = "REDBOT_UPDATE_RUNNER_WRAPPER_EXE"
+
+OLD_VENV_BACKUP_DIR_NAME: Final = "redbot-update-old-venv-backup"
+
+
+def get_red_dependency_specifier(version: Version, extras: Iterable[str]) -> str:
+    specifier_template = (
+        os.getenv("_RED_UPDATE_PRETEND_SPECIFIER_TEMPLATE")
+        or "Red-DiscordBot {extras} {versionspec}"
+    )
+    joined_extras = ",".join(extras)
+    return specifier_template.format(
+        extras=f"[{joined_extras}]" if joined_extras else "",
+        versionspec=f"=={version}",
+    )
+
+
+def get_current_red_version() -> Version:
+    return Version(os.getenv("_RED_UPDATE_PRETEND_VERSION") or __version__)
+
+
+def get_current_python_version() -> Version:
+    return Version(".".join(map(str, sys.version_info[:3])))
+
+
+def prefix_column(prefix: RenderableType, *parts: Union[str, Text]) -> Table:
+    output = Table.grid(padding=(0, 2))
+    output.add_column()
+    output.add_column()
+    text = Text()
+    for renderable in parts:
+        if isinstance(renderable, str):
+            text.append_text(Text.from_markup(renderable))
+        else:
+            text.append_text(renderable)
+    output.add_row(prefix, text)
+    return output
+
+
+def print_with_prefix_column(
+    prefix: RenderableType, *parts: Union[str, Text], console: Optional[Console] = None
+) -> None:
+    if console is None:
+        console = rich.get_console()
+    console.print(prefix_column(prefix, *parts))
+
+
+def _apply_legacy_windows_workaround() -> None:
+    # Rich does not properly support printing to stderr, when stdout is redirected...
+    # This monkeypatch should be enough to workaround this for our purposes.
+    # https://github.com/Textualize/rich/issues/4071
+    if sys.platform == "win32" and not sys.stdout.isatty():
+        import rich._win32_console
+
+        rich._win32_console.STDOUT = -12
+
+
+def configure_rich() -> None:
+    _apply_legacy_windows_workaround()
+    value = os.getenv(INTERNAL_LEGACY_WINDOWS_ENV_VAR, "")
+    legacy_windows = int(value) if value else None
+    rich.reconfigure(highlight=False, legacy_windows=legacy_windows)
+    global _STDERR_CONSOLE
+    _STDERR_CONSOLE = Console(highlight=False, stderr=True, legacy_windows=legacy_windows)
+
+
+def get_console(stderr: bool = False) -> Console:
+    global _STDERR_CONSOLE
+    if _STDERR_CONSOLE is None:
+        raise RuntimeError("_STDERR_CONSOLE is not set")
+    return _STDERR_CONSOLE if stderr else rich.get_console()
+
+
+def configure_logging(logging_level: int) -> None:
+    configure_rich()
+    level = cli_level_to_log_level(logging_level)
+    base_logger = logging.getLogger("red")
+    base_logger.setLevel(level)
+    base_logger.addHandler(RichHandler(console=get_console(stderr=True), show_path=False))
+
+
+def get_logging_level() -> int:
+    return logging.getLogger("red").level
+
+
+def get_log_cli_level() -> int:
+    return log_level_to_cli_level(logging.getLogger("red").level)
+
+
+def ensure_supported_env() -> None:
+    if sys.prefix == sys.base_prefix:
+        print("redbot-update cannot be used when Red is installed outside a virtual environment.")
+        raise SystemExit(1)
+    if not (
+        os.environ.get(RUNNER_DIR_ENV_VAR, "") and os.environ.get(RUNNER_WRAPPER_EXE_ENV_VAR, "")
+    ):
+        print("redbot-update was called incorrectly.")
+        raise SystemExit(1)
+
+
+def _get_system_interpreters(
+    requires_python: SpecifierSet,
+) -> List[Tuple[str, Version, PythonInfo]]:
+    interpreters = {}
+
+    def _append_interpreter(info: PythonInfo) -> Literal[False]:
+        version = Version(info.version_str)
+        if version in requires_python:
+            # realpath call is needed because get_interpreter lists
+            # /usr/bin and /bin as separate even though they're the same path
+            interpreters[os.path.realpath(info.executable)] = (version, info)
+        return False
+
+    get_interpreter("cpython", predicate=_append_interpreter)
+
+    ret = [(key, *value) for key, value in interpreters.items()]
+    ret.sort(key=itemgetter(1), reverse=True)
+    return ret
+
+
+def search_for_interpreters(
+    requires_python: SpecifierSet,
+) -> List[Tuple[str, Version, PythonInfo]]:
+    console = get_console()
+    with console.status("Searching for compatible Python interpreters on your system..."):
+        interpreters = _get_system_interpreters(requires_python)
+
+    if not interpreters:
+        url = "https://docs.discord.red/en/stable/install_guides/"
+        console.print(
+            f"{ICON_ERROR} Could not find a compatible Python interpreter!\n"
+            'Please follow the steps from the "Installing the pre-requirements" section'
+            " of the install guide for your system:"
+        )
+        console.print(Text(url, style=f"link {url}"))
+        console.print("Once you finish installing the pre-requirements, run this command again.")
+        raise SystemExit(1)
+
+    return interpreters
+
+
+class OrderedEnum(enum.Enum):
+    def __ge__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.value >= other.value
+        return NotImplemented
+
+    def __gt__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.value > other.value
+        return NotImplemented
+
+    def __le__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.value <= other.value
+        return NotImplemented
+
+    def __lt__(self, other: Any) -> bool:
+        if self.__class__ is other.__class__:
+            return self.value < other.value
+        return NotImplemented
+
+
+class VersionParamType(click.ParamType):
+    name = "version"
+
+    def convert(
+        self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]
+    ) -> Version:
+        if isinstance(value, Version):
+            if len(value.release) < 2:
+                self.fail(
+                    f"{value!r} needs to have at least 2 release components (major and minor).",
+                    param,
+                    ctx,
+                )
+            return value
+
+        try:
+            return self.convert(Version(value), param, ctx)
+        except ValueError:
+            self.fail(f"{value!r} is not a valid version number", param, ctx)

redbot/_update/internal.py

@@ -0,0 +1,453 @@
+import asyncio
+import os
+import subprocess
+import shutil
+import sys
+import sysconfig
+from pathlib import Path
+from typing import Tuple
+
+import click
+from rich.markdown import Markdown
+from rich.panel import Panel
+from rich.prompt import Confirm
+from rich.text import Text
+
+from redbot import __version__
+from redbot.core import _downloader, _drivers, data_manager
+from redbot.core._cli import asyncio_run, parse_cli_flags
+from redbot.core.bot import Red
+
+from . import changelog, cmd, common, runner
+from .updater import UpdaterMetadata, get_updater_metadata
+
+
+FINISH_UPDATE_CMD_NAME = "finish-update"
+_UPDATE_COGS_CMD_NAME = "update-cogs"
+_UPDATE_REPOS_OPTION_NAME = "--update-repos"
+_EXIT_INSTANCE_SITE_PREFIX_MISMATCH = 4
+_EXIT_INSTANCE_BACKEND_UNSUPPORTED = 5
+
+
+@click.group(invoke_without_command=True)
+@click.option(cmd.arg_names.DEBUG, "logging_level", count=True)
+def cli(logging_level: int) -> None:
+    common.ensure_supported_env()
+    common.configure_logging(logging_level)
+
+
+@cli.command(_UPDATE_COGS_CMD_NAME)
+@click.argument("instance_name")
+@click.option(_UPDATE_REPOS_OPTION_NAME, default=False, is_flag=True)
+def update_cogs(instance_name: str, update_repos: bool) -> None:
+    asyncio.run(_update_cogs(instance_name, update_repos))
+
+
+async def _update_cogs(instance: str, update_repos: bool) -> None:
+    data_manager.load_basic_configuration(instance)
+    red = Red(cli_flags=parse_cli_flags([instance]))
+    driver_cls = _drivers.get_driver_class()
+    await driver_cls.initialize(**data_manager.storage_details())
+    try:
+        await _run_cog_update(red, update_repos=update_repos)
+    except _drivers.MissingExtraRequirements:
+        raise SystemExit(_EXIT_INSTANCE_BACKEND_UNSUPPORTED)
+    finally:
+        await driver_cls.teardown()
+
+
+async def _run_cog_update(bot: Red, *, update_repos: bool) -> None:
+    stdout_console = common.get_console()
+    console = common.get_console(stderr=True)
+
+    instance_name = data_manager.instance_name()
+    last_known_prefix = await bot._config.last_system_info.python_prefix()
+    same_install = False
+    if last_known_prefix is not None:
+        try:
+            same_install = os.path.samefile(last_known_prefix, sys.prefix)
+        except OSError:
+            pass
+    if not same_install:
+        raise SystemExit(_EXIT_INSTANCE_SITE_PREFIX_MISMATCH)
+
+    common.print_with_prefix_column(
+        common.ICON_INFO,
+        "Started updating cogs for the ",
+        Text(instance_name, style="bold"),
+        " instance.",
+        console=console,
+    )
+    status = Text.assemble(
+        "Update cogs installed on the ", (instance_name, "bold"), " instance..."
+    )
+    with console.status(status):
+        await _downloader._init_without_bot(bot._cog_mgr)
+        result = await _downloader.update_cogs(update_repos=update_repos)
+
+    common.print_with_prefix_column(
+        common.ICON_INFO,
+        "Finished updating cogs for the ",
+        Text(instance_name, style="bold"),
+        " instance.",
+        console=console,
+    )
+
+    if not result.checked_cogs:
+        stdout_console.print("There were no cogs to check.")
+        return
+    if not result.updates_available:
+        stdout_console.print("All installed cogs are already up to date.")
+        return
+
+    current_cog_versions_map = {cog.name: cog for cog in result.checked_cogs}
+    if result.failed_reqs:
+        console.print(
+            "Failed to install requirements:",
+            Text(", ").join(Text(req, style="bold") for req in result.failed_reqs),
+        )
+        return
+
+    message = Text("Cog update completed successfully.")
+
+    if result.updated_cogs:
+        cogs_with_changed_eud_statement = set()
+        for cog in result.updated_cogs:
+            current_eud_statement = current_cog_versions_map[cog.name].end_user_data_statement
+            if current_eud_statement != cog.end_user_data_statement:
+                cogs_with_changed_eud_statement.add(cog.name)
+        message.append("\nUpdated: ")
+        message.append_text(
+            Text(", ").join(Text(cog.name, style="bold") for cog in result.updated_cogs)
+        )
+        if cogs_with_changed_eud_statement:
+            message.append("\nEnd user data statements of these cogs have changed: ")
+            message.append_text(
+                Text(", ").join(
+                    Text(cog_name, style="bold") for cog_name in cogs_with_changed_eud_statement
+                )
+            )
+            message.append("\nYou can use ")
+            message.append("[p]cog info <repo> <cog>", style="bold")
+            message.append(" to see the updated statements.\n")
+        # If the bot has any slash commands enabled, warn them to sync
+        enabled_slash = await bot.list_enabled_app_commands()
+        if any(enabled_slash.values()):
+            message.append("\nYou may need to resync your slash commands with ")
+            message.append("[p]slash sync")
+            message.append(".")
+    if result.failed_cogs:
+        message.append("\nFailed to update cogs: ")
+        message.append_text(
+            Text(", ").join(Text(cog.name, style="bold") for cog in result.failed_cogs)
+        )
+    if not result.outdated_cogs:
+        message = Text("No cogs were updated.")
+    if result.failed_libs:
+        message.append("\nFailed to install shared libraries: ")
+        message.append_text(
+            Text(", ").join(Text(lib.name, style="bold") for lib in result.failed_libs)
+        )
+
+    stdout_console.print(message)
+
+
+@cli.command(FINISH_UPDATE_CMD_NAME)
+def finish_update() -> None:
+    """
+    Entrypoint for finishing up the update that runs with the new version of Red.
+    """
+    asyncio_run(_finish_update())
+
+
+async def _finish_update() -> None:
+    assert runner.get_request_output().request_type is runner.RequestType.exec
+    updater_metadata = get_updater_metadata()
+    console = common.get_console()
+    console.print()
+
+    if updater_metadata.options.interactive and not updater_metadata.options.update_cogs:
+        msg = Text("It is highly recommended to update 3rd-party cogs after updating Red")
+        if updater_metadata.breaking_update:
+            msg.append(", especially after a major update")
+        msg.append(".")
+        console.print(msg)
+
+    cog_compatibility = updater_metadata.cog_compatibility
+    if cog_compatibility is not None:
+        unsupported_cogs = set()
+        cogs_with_improved_compatibility = set()
+        unaffected_cogs = set()
+        for summary in cog_compatibility.checked.values():
+            for before in summary.before_update.values():
+                cog_name = before.name
+                after = summary.after_update[cog_name]
+                if after.compatibility_status.unsupported:
+                    unsupported_cogs.add(cog_name)
+                elif after.compatibility_status.explicitly_supported:
+                    if before.compatibility_status.explicitly_supported:
+                        unaffected_cogs.add(cog_name)
+                    else:
+                        cogs_with_improved_compatibility.add(cog_name)
+                elif before.compatibility_status.unsupported:
+                    cogs_with_improved_compatibility.add(cog_name)
+                else:
+                    unaffected_cogs.add(cog_name)
+
+        if cogs_with_improved_compatibility:
+            common.print_with_prefix_column(
+                common.ICON_INFO,
+                "Updating will improve compatibility of ",
+                Text(str(len(cogs_with_improved_compatibility)), style="bold"),
+                " cogs.",
+            )
+        if unsupported_cogs:
+            common.print_with_prefix_column(
+                common.ICON_WARN,
+                Text(str(len(unsupported_cogs)), style="bold"),
+                " cogs will remain unsupported after updating:\n",
+                Text(", ").join(
+                    Text(cog_name, style="bold") for cog_name in sorted(unsupported_cogs)
+                ),
+            )
+
+    update_cogs = updater_metadata.options.update_cogs
+    if update_cogs is None:
+        if updater_metadata.options.interactive:
+            update_cogs = Confirm.ask("Do you want to update all your cogs?", default=True)
+        else:
+            update_cogs = True
+    if update_cogs:
+        await _handle_cog_updates(updater_metadata)
+
+    with console.status("Cleaning up..."):
+        backup_dir = Path(sys.prefix) / common.OLD_VENV_BACKUP_DIR_NAME
+        shutil.rmtree(backup_dir)
+
+    changelog_markdown = changelog.render_markdown(updater_metadata.changelogs)
+    if changelog_markdown:
+        console.print(Panel(Markdown(changelog_markdown)))
+
+    console.print()
+    common.print_with_prefix_column(
+        common.ICON_SUCCESS,
+        "Update to Red ",
+        Text(__version__, style="bold"),
+        " has been finished!",
+    )
+
+    if changelog_markdown:
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            'Remember to follow instructions from the "Read before updating" section,'
+            " if any were provided.",
+        )
+
+    if updater_metadata.backup_dir:
+        additional_text = ""
+        if not updater_metadata.options.backup_dir:
+            additional_text = (
+                "\nNote that this is a temporary directory and may eventually get auto-removed"
+                " by your system."
+            )
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            "If needed, you can find the backups of the virtual environment"
+            " and the instances at: ",
+            Text(str(updater_metadata.backup_dir), style="bold"),
+            additional_text,
+        )
+
+
+async def _handle_cog_updates(updater_metadata: UpdaterMetadata) -> None:
+    cog_compatibility = updater_metadata.cog_compatibility
+    console = common.get_console()
+
+    instances = (
+        list(cog_compatibility.checked)
+        if cog_compatibility is not None
+        else updater_metadata.options.instances
+    )
+    checked_instances = {}
+    failed_instances = []
+    unsupported_storage_instances = []
+    for instance_name in instances:
+        if instance_name in updater_metadata.options.excluded_instances:
+            continue
+        exit_code, stdout = await _call_cog_update(
+            instance_name, update_repos=cog_compatibility is None
+        )
+        if exit_code == _EXIT_INSTANCE_BACKEND_UNSUPPORTED:
+            unsupported_storage_instances.append(instance_name)
+        elif exit_code == _EXIT_INSTANCE_SITE_PREFIX_MISMATCH:
+            pass
+        elif exit_code:
+            failed_instances.append(instance_name)
+            print(stdout, end="")
+            Text.assemble(
+                "\N{UPWARDS ARROW} " * 3, "Failure for ", (instance_name, "bold"), " instance"
+            )
+            console.rule(
+                Text.assemble(
+                    "\N{UPWARDS ARROW} " * 3,
+                    "Failure for ",
+                    (instance_name, "bold"),
+                    " instance above",
+                    " \N{UPWARDS ARROW}" * 3,
+                ),
+                style="red",
+            )
+        else:
+            checked_instances[instance_name] = stdout
+        if stdout:
+            console.print()
+
+    if checked_instances:
+        for instance_name, stdout in checked_instances.items():
+            console.rule(Text(instance_name, style="bold"))
+            print(stdout, end="")
+        console.rule()
+
+    common.print_with_prefix_column(
+        common.ICON_INFO,
+        "Finished updating cogs.",
+        "\nThe results for each instance are shown above." if checked_instances else "",
+    )
+    if failed_instances:
+        common.print_with_prefix_column(
+            common.ICON_ERROR,
+            "Failure occurred while trying to perform update for following instances: ",
+            Text(", ").join(
+                Text(instance_name, style="bold") for instance_name in failed_instances
+            ),
+            "\nScroll above to find the errors.",
+        )
+    if unsupported_storage_instances:
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            "The following instances were skipped as they use a storage backend that is"
+            " not supported by the current Red installation (some requirements are missing): ",
+            Text(", ").join(
+                Text(instance_name, style="bold")
+                for instance_name in unsupported_storage_instances
+            ),
+        )
+    if not checked_instances:
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            "There were no",
+            (" other" if failed_instances or unsupported_storage_instances else ""),
+            " instances to update cogs for.",
+        )
+
+
+async def _call_cog_update(instance_name: str, *, update_repos: bool) -> Tuple[int, str]:
+    debug_args = (cmd.arg_names.DEBUG,) * common.get_log_cli_level()
+    args = [
+        "-m",
+        "redbot._update.internal",
+        *debug_args,
+        _UPDATE_COGS_CMD_NAME,
+        instance_name,
+    ]
+    if update_repos:
+        args.append(_UPDATE_REPOS_OPTION_NAME)
+    env = os.environ.copy()
+
+    # terminal woes
+    console = common.get_console()
+    if console.is_terminal:
+        env["TTY_COMPATIBLE"] = "1"
+        # Rich only checks stdout for Windows console features:
+        # https://github.com/Textualize/rich/blob/fc41075a3206d2a5fd846c6f41c4d2becab814fa/rich/_windows.py#L46
+        env[common.INTERNAL_LEGACY_WINDOWS_ENV_VAR] = "1" if console.legacy_windows else "0"
+    else:
+        # Rich does not set legacy_windows correctly when is_terminal is False
+        # https://github.com/Textualize/rich/issues/3647
+        env[common.INTERNAL_LEGACY_WINDOWS_ENV_VAR] = "0"
+    env["PYTHONIOENCODING"] = sys.stdout.encoding
+
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, *args, env=env, stdout=asyncio.subprocess.PIPE
+    )
+    stdout_data, _ = await proc.communicate()
+    decoded_stdout = stdout_data.decode()
+    exit_code = await proc.wait()
+
+    return exit_code, decoded_stdout
+
+
+@cli.command()
+@click.argument("base_executable")
+@click.argument("venv_dir", type=click.Path(path_type=Path))
+@click.argument("scripts_path", type=click.Path(path_type=Path))
+@click.argument("dependency_specifier")
+def reinstall(
+    base_executable: str, venv_dir: Path, scripts_path: Path, dependency_specifier: str
+) -> None:
+    assert runner.get_request_output().request_type is runner.RequestType.exec
+
+    console = common.get_console()
+    with console.status("Creating a new virtual environment..."):
+        subprocess.check_call((base_executable, "-m", "venv", str(venv_dir)))
+    console.print("Created a new virtual environment.")
+    executable = str(scripts_path / f"python{sysconfig.get_config_var('EXE')}")
+
+    common.print_with_prefix_column(common.ICON_INFO, "Starting the install process...")
+    try:
+        subprocess.check_call((executable, "-m", "pip", "install", "-U", "pip"))
+        subprocess.check_call((executable, "-m", "pip", "install", dependency_specifier))
+    except subprocess.CalledProcessError:
+        console.print()
+        common.print_with_prefix_column(
+            common.ICON_ERROR,
+            "Failed to install new version of Red.",
+        )
+        status = console.status("Attempting to restore old virtual environment...")
+        status.start()
+        try:
+            _remove_new_venv(venv_dir)
+        except Exception:
+            status.stop()
+            common.print_with_prefix_column(
+                common.ICON_ERROR, "Failed to remove newly created virtual environment."
+            )
+            raise SystemExit(1)
+        try:
+            _restore_old_venv(venv_dir)
+        except Exception:
+            status.stop()
+            common.print_with_prefix_column(
+                common.ICON_ERROR, "Failed to restore old virtual environment."
+            )
+        else:
+            common.print_with_prefix_column(
+                common.ICON_INFO, "The old virtual environment has been restored."
+            )
+        raise SystemExit(1)
+
+    # NOTE: this will run with the updated version of Red
+    runner.make_exec_request(executable, "finish-update")
+
+
+def _remove_new_venv(venv_dir: Path) -> None:
+    backup_dir = venv_dir / common.OLD_VENV_BACKUP_DIR_NAME
+    wrapper_exe = runner.get_wrapper_executable()
+
+    for path in venv_dir.iterdir():
+        if path == backup_dir or path == wrapper_exe:
+            continue
+        if path.is_dir():
+            shutil.rmtree(path)
+        else:
+            path.unlink()
+
+
+def _restore_old_venv(venv_dir: Path) -> None:
+    backup_dir = venv_dir / common.OLD_VENV_BACKUP_DIR_NAME
+    for path in backup_dir.iterdir():
+        path.rename(venv_dir / path.name)
+
+
+if __name__ == "__main__":
+    cli()

redbot/_update/runner.py

@@ -0,0 +1,119 @@
+import enum
+import dataclasses
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, ClassVar, Dict, Iterable, NoReturn, Optional, Tuple, Union
+
+from . import cmd, common
+
+_RUNNER_DIR = Path(os.environ.get(common.RUNNER_DIR_ENV_VAR, ""))
+
+
+class RequestType(enum.Enum):
+    exec = "exec"
+    spawn_command = "spawn_command"
+
+
+@dataclasses.dataclass(frozen=True)
+class RequestInput:
+    request_type: ClassVar[RequestType]
+    request_new_python_exe: str
+    request_new_start_args: Tuple[str, ...]
+    request_set_env_vars: Dict[str, Optional[str]]
+
+
+@dataclasses.dataclass(frozen=True)
+class RequestOutput:
+    request_type: RequestType
+
+
+@dataclasses.dataclass(frozen=True)
+class ExecRequestInput(RequestInput):
+    request_type: ClassVar = RequestType.exec
+
+
+@dataclasses.dataclass(frozen=True)
+class ExecRequestOutput(RequestOutput):
+    pass
+
+
+@dataclasses.dataclass(frozen=True)
+class SpawnProcessRequestInput(RequestInput):
+    request_type: ClassVar = RequestType.spawn_command
+    command: str
+    args: Tuple[str, ...]
+    env: Optional[Dict[str, str]]
+
+
+@dataclasses.dataclass(frozen=True)
+class SpawnProcessRequestOutput(RequestOutput):
+    exit_code: int
+    exited: bool
+    pid: int
+    sys: Any
+    sys_usage: Dict[str, Any]
+    system_time: int
+    user_time: int
+
+
+def make_request(request: RequestInput) -> NoReturn:
+    with open(_RUNNER_DIR / "request_input.json", "w", encoding="utf-8") as fp:
+        data = dataclasses.asdict(request)
+        data["request_type"] = request.request_type.value
+        json.dump(data, fp)
+    raise SystemExit(3)
+
+
+def get_request_output() -> Union[ExecRequestOutput, SpawnProcessRequestOutput]:
+    with open(_RUNNER_DIR / "request_output.json", encoding="utf-8") as fp:
+        data = json.load(fp)
+        request_type = RequestType(data.pop("request_type"))
+        if request_type == RequestType.exec:
+            return ExecRequestOutput(request_type=request_type)
+        elif request_type == RequestType.spawn_command:
+            return SpawnProcessRequestOutput(request_type=request_type, **data)
+        raise RuntimeError("unreachable code")
+
+
+def make_spawn_process_request(
+    command: str,
+    *args: str,
+    env: Optional[Dict[str, str]] = None,
+    new_start_args: Iterable[str],
+    new_python_exe: str = sys.executable,
+    set_env_vars: Optional[Dict[str, Optional[str]]] = None,
+) -> NoReturn:
+    if set_env_vars is None:
+        set_env_vars = {}
+    debug_args = (cmd.arg_names.DEBUG,) * common.get_log_cli_level()
+    request = SpawnProcessRequestInput(
+        request_new_python_exe=new_python_exe,
+        request_new_start_args=("-m", "redbot._update.internal", *debug_args, *new_start_args),
+        request_set_env_vars=set_env_vars,
+        command=command,
+        args=args,
+        env=env,
+    )
+    make_request(request)
+
+
+def make_exec_request(
+    new_python_exe: str,
+    *new_start_args: str,
+    set_env_vars: Optional[Dict[str, Optional[str]]] = None,
+) -> NoReturn:
+    if set_env_vars is None:
+        set_env_vars = {}
+    debug_args = (cmd.arg_names.DEBUG,) * common.get_log_cli_level()
+    request = ExecRequestInput(
+        request_new_python_exe=new_python_exe,
+        request_new_start_args=("-m", "redbot._update.internal", *debug_args, *new_start_args),
+        request_set_env_vars=set_env_vars,
+    )
+    make_request(request)
+
+
+def get_wrapper_executable() -> Path:
+    return Path(os.environ[common.RUNNER_WRAPPER_EXE_ENV_VAR])

redbot/_update/tui.py

@@ -0,0 +1,129 @@
+import enum
+
+from rich.text import Text
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.events import Click
+from textual.widgets import Footer, Markdown, MarkdownViewer, Static
+from typing_extensions import Self
+
+from .changelog import Changelogs
+
+
+# See https://github.com/Textualize/textual/discussions/6449
+class MarkdownLinkTooltip(Static, inherit_css=False):
+    DEFAULT_CSS = """
+    MarkdownLinkTooltip {
+        layer: _tooltips;
+        margin: 1 0;
+        padding: 1 2;
+        background: $panel;
+        width: auto;
+        height: auto;
+        constrain: inside inflect;
+        max-width: 40;
+        display: none;
+        offset-x: -50%;
+    }
+    """
+
+
+class _MarkdownViewer(MarkdownViewer):
+    DEFAULT_CSS = """
+    _MarkdownViewer {
+        layers: default _tooltips;
+    }
+    """
+
+    def compose(self) -> ComposeResult:
+        yield from super().compose()
+        yield MarkdownLinkTooltip()
+
+    def on_markdown_link_clicked(self, message: Markdown.LinkClicked) -> None:
+        # We don't want the default behavior of opening the browser/navigating to a file on click.
+        message.prevent_default()
+
+        tooltip = self.get_child_by_type(MarkdownLinkTooltip)
+        tooltip.display = True
+        # You can't cycle over the links in MarkdownViewer (see Textualize/textual#3555)
+        # so using mouse position is fine.
+        # Textualize/textual#3555: https://github.com/Textualize/textual/discussions/3555
+        tooltip.absolute_offset = self.app.mouse_position
+        # For some reason, links only render correctly when Text has a span over the whole text
+        # with a link but not when Text just has a style applied to it directly, i.e.:
+        #   Text(message.href, style=f"link {message.href}")
+        # will not work.
+        tooltip.update(Text().append(message.href, style=f"link {message.href}"))
+
+    def on_click(self, message: Click) -> None:
+        tooltip = self.get_child_by_type(MarkdownLinkTooltip)
+        tooltip.display = False
+
+
+class ChangelogReaderResult(enum.Enum):
+    QUIT = enum.auto()
+    CONTINUE = enum.auto()
+
+
+class ChangelogReaderApp(App[ChangelogReaderResult], inherit_bindings=False):
+    ENABLE_COMMAND_PALETTE = False
+    BINDINGS = [
+        Binding(key="ctrl+c", action="quit", description="Exit redbot-update"),
+        Binding(key="q", action="continue", description="Finish reading the changelog"),
+    ]
+
+    def __init__(self, markdown_content: str) -> None:
+        self.markdown_content = markdown_content
+        super().__init__()
+
+    @classmethod
+    def from_changelogs(cls, changelogs: Changelogs) -> Self:
+        if not changelogs:
+            return cls("")
+
+        parts = []
+        contributors = sorted(
+            {
+                contributor
+                for changelog in changelogs.values()
+                for contributor in changelog.contributors
+            }
+        )
+        if contributors:
+            contributor_thanks = (
+                "# Thanks to our contributors \N{HEAVY BLACK HEART}\N{VARIATION SELECTOR-16}\n"
+                "**The releases below were made with help from the following people:**  \n"
+            )
+            contributor_thanks += ", ".join(
+                f"[@{contributor}](https://github.com/sponsors/{contributor})"
+                for contributor in contributors
+            )
+            parts.append(contributor_thanks)
+
+        parts.append("# Read before updating")
+        for changelog in reversed(changelogs.values()):
+            if changelog.read_before_updating_section:
+                parts.append(f"## {changelog.version}")
+                parts.append(changelog.read_before_updating_section)
+
+        parts.append("# User changelog")
+        for changelog in reversed(changelogs.values()):
+            if changelog.user_changelog_section:
+                parts.append(f"## {changelog.version}")
+                parts.append(changelog.user_changelog_section)
+
+        return cls("\n".join(parts))
+
+    def compose(self) -> ComposeResult:
+        markdown_viewer = _MarkdownViewer(
+            self.markdown_content, show_table_of_contents=True, open_links=False
+        )
+        markdown_viewer.code_indent_guides = False
+        yield markdown_viewer
+        yield Footer()
+
+    def action_quit(self) -> None:
+        self.exit(ChangelogReaderResult.QUIT)
+
+    def action_continue(self) -> None:
+        self.exit(ChangelogReaderResult.CONTINUE)

redbot/_update/updater.py

@@ -0,0 +1,736 @@
+import asyncio
+import dataclasses
+import json
+import os
+import shutil
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, NoReturn, Optional, Set
+
+import click
+from packaging.version import Version
+from python_discovery import PythonInfo
+from rich.markdown import Markdown
+from rich.panel import Panel
+from rich.prompt import Confirm, IntPrompt, Prompt
+from rich.text import Text
+from typing_extensions import Self
+
+from redbot.core.utils._internal_utils import (
+    AvailableVersion,
+    detailed_progress,
+    fetch_available_red_versions,
+    get_installed_extras,
+)
+
+from . import changelog, cmd, common, runner
+from .cog_compatibility_checker import CompatibilitySummary
+from .tui import ChangelogReaderApp, ChangelogReaderResult
+
+
+@dataclasses.dataclass
+class UpdaterOptions:
+    """Update options specified by the user."""
+
+    instances: List[str]
+    excluded_instances: Set[str]
+    ignore_prefix: bool
+    backup_dir: Optional[Path]
+    no_backup: bool
+    red_version: Optional[Version]
+    no_major_updates: bool
+    no_full_changelog: bool
+    no_cog_compatibility_check: bool
+    new_python_interpreter: Optional[PythonInfo]
+    update_cogs: Optional[bool]
+    force_reinstall: bool
+    interactive: bool
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        backup_dir = data["backup_dir"]
+        red_version = data["red_version"]
+        return cls(
+            instances=data["instances"],
+            excluded_instances=set(data["excluded_instances"]),
+            ignore_prefix=data["ignore_prefix"],
+            backup_dir=backup_dir and Path(data["backup_dir"]),
+            no_backup=data["no_backup"],
+            red_version=red_version and Version(red_version),
+            no_major_updates=data["no_major_updates"],
+            no_full_changelog=data["no_full_changelog"],
+            no_cog_compatibility_check=data["no_cog_compatibility_check"],
+            new_python_interpreter=(
+                data["new_python_interpreter"]
+                and PythonInfo.from_dict(data["new_python_interpreter"])
+            ),
+            update_cogs=data["update_cogs"],
+            force_reinstall=data["force_reinstall"],
+            interactive=data["interactive"],
+        )
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        data = dataclasses.asdict(self)
+        data["excluded_instances"] = list(self.excluded_instances)
+        data["backup_dir"] = self.backup_dir and str(self.backup_dir)
+        data["red_version"] = self.red_version and str(self.red_version)
+        data["new_python_interpreter"] = (
+            self.new_python_interpreter and self.new_python_interpreter.to_dict()
+        )
+        return data
+
+
+@dataclasses.dataclass
+class UpdaterCompatibilitySummary:
+    checked: Dict[str, CompatibilitySummary]
+    failed: List[str]
+    skipped: List[str]
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        return cls(
+            checked={
+                instance_name: CompatibilitySummary.from_json_dict(results_data)
+                for instance_name, results_data in data["checked"].items()
+            },
+            failed=data["failed"],
+            skipped=data["skipped"],
+        )
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {
+            "checked": {
+                instance_name: results.to_json_dict()
+                for instance_name, results in self.checked.items()
+            },
+            "failed": self.failed,
+            "skipped": self.skipped,
+        }
+
+
+@dataclasses.dataclass
+class BackupResults:
+    checked: List[str]
+    failed: List[str]
+    skipped: List[str] = dataclasses.field(default_factory=list)
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, List[str]]) -> Self:
+        return cls(checked=data["checked"], failed=data["failed"], skipped=data["skipped"])
+
+    def to_json_dict(self) -> Dict[str, List[str]]:
+        return dataclasses.asdict(self)
+
+
+_PYTHON_VERSION_PLACEHOLDER = Version("0.0.dev0")
+
+
+@dataclasses.dataclass
+class UpdaterMetadata:
+    """Metadata about the update process."""
+
+    # options specified by the user
+    options: UpdaterOptions
+    # info about Red version to update to (latest available or latest non-major update)
+    latest: AvailableVersion
+    latest_major: AvailableVersion
+    # info about Red/Python versions that we're updating from
+    current_version: Version = dataclasses.field(default_factory=common.get_current_red_version)
+    current_python_version: Version = dataclasses.field(
+        default_factory=common.get_current_python_version
+    )
+    # details about the interpreter that will be used for the new venv
+    interpreter_info: PythonInfo = dataclasses.field(default_factory=PythonInfo.current_system)
+    interpreter_version: Version = _PYTHON_VERSION_PLACEHOLDER
+    interpreter_exe: str = ""
+    # changelogs for version in (current_version, latest> range
+    changelogs: changelog.Changelogs = dataclasses.field(default_factory=dict)
+    # cog compatibility check results
+    cog_compatibility: Optional[UpdaterCompatibilitySummary] = None
+    # backup info
+    to_backup: List[str] = dataclasses.field(default_factory=list)
+    backup_dir: Optional[Path] = None
+    backup_results: Optional[BackupResults] = None
+
+    def __post_init__(self) -> None:
+        if self.interpreter_version is _PYTHON_VERSION_PLACEHOLDER:
+            self.interpreter_version = Version(
+                ".".join(map(str, self.interpreter_info.version_info[:3]))
+            )
+        if not self.interpreter_exe:
+            self.interpreter_exe = self.interpreter_info.system_executable
+
+    @classmethod
+    def from_json_dict(cls, data: Dict[str, Any]) -> Self:
+        """
+        Make an instance of this class from a dictionary,
+        as returned by the `to_json_dict()` method.
+
+        This aims to maintain backwards compatibility with data generated by
+        earlier Red versions as it may be called with such data
+        after the last update step.
+        """
+        backup_dir = data.get("backup_dir")
+        return cls(
+            options=UpdaterOptions.from_json_dict(data["options"]),
+            latest=AvailableVersion.from_json_dict(data["latest"]),
+            latest_major=AvailableVersion.from_json_dict(data["latest_major"]),
+            current_version=Version(data["current_version"]),
+            current_python_version=Version(data["current_python_version"]),
+            interpreter_version=Version(data["interpreter_version"]),
+            interpreter_info=PythonInfo.from_dict(data["interpreter_info"]),
+            interpreter_exe=data["interpreter_exe"],
+            changelogs={
+                Version(raw_version): changelog.VersionChangelog.from_json_dict(raw_changelog)
+                for raw_version, raw_changelog in data["changelogs"].items()
+            },
+            cog_compatibility=UpdaterCompatibilitySummary.from_json_dict(
+                data["cog_compatibility"]
+            ),
+            to_backup=data["to_backup"],
+            backup_dir=backup_dir and Path(backup_dir),
+            backup_results=BackupResults.from_json_dict(data["backup_results"]),
+        )
+
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {
+            "options": self.options.to_json_dict(),
+            "latest": self.latest.to_json_dict(),
+            "latest_major": self.latest_major.to_json_dict(),
+            "current_version": str(self.current_version),
+            "current_python_version": str(self.current_python_version),
+            "interpreter_version": str(self.interpreter_version),
+            "interpreter_info": self.interpreter_info.to_dict(),
+            "interpreter_exe": self.interpreter_exe,
+            "changelogs": {str(v): c.to_json_dict() for v, c in self.changelogs.items()},
+            "cog_compatibility": self.cog_compatibility and self.cog_compatibility.to_json_dict(),
+            "to_backup": self.to_backup,
+            "backup_dir": self.backup_dir and str(self.backup_dir),
+            "backup_results": self.backup_results and self.backup_results.to_json_dict(),
+        }
+
+    @property
+    def breaking_update(self) -> bool:
+        return self.current_version.release[:2] != self.latest.version.release[:2]
+
+
+class Updater:
+    metadata: UpdaterMetadata
+
+    def __init__(self, options: UpdaterOptions) -> None:
+        self.options = options
+        self.console = common.get_console()
+
+    @property
+    def latest(self) -> AvailableVersion:
+        return self.metadata.latest
+
+    @property
+    def current_version(self) -> Version:
+        return self.metadata.current_version
+
+    async def run(self) -> None:
+        await self._prepare_metadata()
+
+        new_version_available = self.current_version < self.latest.version
+        if not self.options.force_reinstall and not new_version_available:
+            if self.current_version >= self.metadata.latest_major.version:
+                common.print_with_prefix_column(
+                    common.ICON_SUCCESS,
+                    "You are already running the latest available version of Red.",
+                )
+            else:
+                common.print_with_prefix_column(
+                    common.ICON_INFO,
+                    "There are no non-major updates available.\n",
+                    "There is a new major version available: ",
+                    Text(str(self.metadata.latest_major.version), style="bold"),
+                )
+            return
+
+        if new_version_available:
+            common.print_with_prefix_column(
+                common.ICON_SUCCESS,
+                "New version available: ",
+                Text(str(self.latest.version), style="bold"),
+            )
+
+        await self._show_changelog()
+        self._check_python_requires()
+        if self.options.no_cog_compatibility_check:
+            self.console.print(
+                "Will not make backups as --no-cog-compatibility-check option was passed."
+            )
+        else:
+            await self._check_cog_compatibility()
+
+        if self.options.no_backup:
+            common.print_with_prefix_column(
+                common.ICON_INFO, "Will not make backups as --no-backup option was passed."
+            )
+        else:
+            common.print_with_prefix_column(
+                common.ICON_INFO,
+                "The following instances will be backed up before performing the update: ",
+                Text(", ").join(
+                    Text(instance_name, style="bold") for instance_name in self.metadata.to_backup
+                ),
+            )
+        if self.metadata.breaking_update:
+            self.console.print(
+                "[b]Remember that this is a major release and it may have some breaking changes"
+                " that the bot or its cogs may be affected by.[/]"
+            )
+        if self.options.interactive and not Confirm.ask(
+            f"Do you want to continue with the update to [b]Red {self.latest.version}[/]?"
+        ):
+            return
+        self.console.print()
+
+        if self.options.no_backup:
+            self.console.print("Will not make backups as --no-backup option was passed.")
+        else:
+            await self._make_backups()
+
+        await self._update_with_fresh_venv()
+
+    async def _prepare_metadata(self) -> None:
+        interpreter_info = self.options.new_python_interpreter or PythonInfo.current_system()
+        with self.console.status("Checking latest version..."):
+            available_versions = await fetch_available_red_versions()
+            latest_major = available_versions[0]
+
+        self.metadata = UpdaterMetadata(
+            self.options,
+            latest=latest_major,
+            latest_major=latest_major,
+            interpreter_info=interpreter_info,
+        )
+
+        if self.options.red_version:
+            if self.options.red_version <= self.current_version:
+                common.print_with_prefix_column(
+                    common.ICON_ERROR, "You can only update to a newer version of Red."
+                )
+                raise SystemExit(2)
+            if (
+                self.options.no_major_updates
+                and self.options.red_version.release[:2] != self.current_version.release[:2]
+            ):
+                common.print_with_prefix_column(
+                    common.ICON_ERROR,
+                    "Updating to the specified version would be a major update"
+                    " but --no-major-updates option was specified.",
+                )
+                raise SystemExit(2)
+            for available_version in available_versions:
+                if available_version.version == self.options.red_version:
+                    break
+            else:
+                common.print_with_prefix_column(
+                    common.ICON_ERROR, "The provided version does not seem to exist."
+                )
+                raise SystemExit(2)
+            self.metadata.latest = available_version
+        elif self.options.no_major_updates:
+            for available_version in available_versions:
+                if available_version.version.release[:2] == self.current_version.release[:2]:
+                    self.metadata.latest = available_version
+                    break
+            else:
+                if self.current_version < latest_major.version:
+                    common.print_with_prefix_column(
+                        common.ICON_ERROR,
+                        "Could not find any version of Red that would not be a major update.",
+                    )
+                    raise SystemExit(1)
+
+    async def _show_changelog(self) -> None:
+        with self.console.status("Fetching changelogs..."):
+            changelogs = await changelog.fetch_changelogs()
+            self.metadata.changelogs = changelogs = changelog.get_changelogs_between(
+                changelogs, self.current_version, self.latest.version
+            )
+        common.print_with_prefix_column(common.ICON_SUCCESS, "Changelogs fetched.")
+
+        if not changelogs:
+            return
+
+        if not self.options.interactive or self.options.no_full_changelog:
+            self.console.print(Panel(Markdown(changelog.render_markdown(changelogs))))
+            if self.options.interactive and not Confirm.ask("Do you want to continue?"):
+                raise click.Abort()
+            return
+
+        first_changelog_version = min(changelogs)
+        last_changelog_version = max(changelogs)
+        parts = []
+        if first_changelog_version == last_changelog_version:
+            parts.append(
+                "You will now be presented with the changelog for"
+                f" [b]Red {first_changelog_version}[/]."
+            )
+        else:
+            parts.append(
+                "You will now be presented with the changelogs for"
+                f" [b]Red {first_changelog_version}[/]-[b]{last_changelog_version}[/]."
+            )
+        parts.append(
+            f"\n[bold]{common.ICON_WARN}"
+            '  Make sure to read through the [green]"Read before updating"[/] section'
+            f" before continuing. {common.ICON_WARN}[/bold]\n"
+        )
+        if self.metadata.breaking_update:
+            parts.append(
+                f"[bold]{common.ICON_WARN}"
+                "  Please note that this is a major release and it may have some changes that"
+                " your bot or its cogs are affected by.[/bold]\n"
+            )
+        parts.append(
+            "After the changelog is open and you're ready to continue, hit the [b]Q[/] key"
+            " to close the changelog and continue the update process.\n\n"
+            "Hit the [b]Enter[/] key to view the changelog."
+        )
+        self.console.input(Panel("".join(parts)), password=True)
+
+        viewer = ChangelogReaderApp.from_changelogs(changelogs)
+        result = await viewer.run_async()
+        if result is None:
+            raise RuntimeError("Unexpected state")
+        if result is ChangelogReaderResult.QUIT:
+            raise click.Abort()
+
+        self.console.print("Changelog has been closed.\n")
+
+    def _check_python_requires(self) -> None:
+        if self.metadata.interpreter_version in self.latest.requires_python:
+            return
+        if self.options.new_python_interpreter:
+            common.print_with_prefix_column(
+                common.ICON_ERROR,
+                "The latest version of Red requires a different Python version (",
+                Text(str(self.latest.requires_python), style="bold"),
+                ") from the version of the interpreter passed to with the --new-python-interpreter"
+                " option (",
+                Text(str(self.metadata.interpreter_version), style="bold"),
+                ")",
+            )
+            raise SystemExit(1)
+        common.print_with_prefix_column(
+            common.ICON_WARN if self.options.interactive else common.ICON_ERROR,
+            "The latest version of Red requires a different Python version (",
+            Text(str(self.latest.requires_python), style="bold"),
+            ") from the one that you are currently using (",
+            Text(str(self.metadata.interpreter_version), style="bold"),
+            ")",
+            (
+                "\nredbot-update will have to recreate the virtual environment"
+                " with a compatible version of Python."
+                if self.options.interactive
+                else ""
+            ),
+        )
+        if not self.options.interactive:
+            raise SystemExit(1)
+        interpreters = common.search_for_interpreters(self.latest.requires_python)
+
+        def _render_interpreter(interpreter_exe: str, interpreter_version: Version) -> Text:
+            return Text.assemble(
+                "CPython ",
+                (str(interpreter_version), "repr.number"),
+                " (",
+                (interpreter_exe, "log.path"),
+                ")",
+            )
+
+        text = Text("Found the following compatible Python interpreters on your system:")
+        for idx, (interpreter_exe, interpreter_version, python_info) in enumerate(interpreters, 1):
+            text.append_text(Text(f"\n{idx}. ", style="markdown.item.number"))
+            text.append_text(_render_interpreter(interpreter_exe, interpreter_version))
+        self.console.print(Panel(text))
+
+        while True:
+            result = IntPrompt.ask(
+                "\nEnter the number of the Python interpreter above that you want to use"
+                " or type 0 to input the path to it yourself. Generally, you should choose"
+                " the interpreter with the latest version on the above list.\n"
+                "Enter your selection",
+                default=1,
+            )
+            if result < 0 or result > len(interpreters):
+                self.console.print("[prompt.invalid] This is not a valid choice.")
+                continue
+
+            if result == 0:
+                response = Prompt.ask(
+                    "Please input the path to the Python interpreter that you want to use"
+                )
+                if not response:
+                    self.console.print("[prompt.invalid] No path was provided.")
+                    continue
+                info = PythonInfo.from_exe(response)
+                interpreter_version = Version(info.version_str)
+                if (
+                    info.implementation != "CPython"
+                    or interpreter_version not in self.latest.requires_python
+                ):
+                    self.console.print(
+                        "[prompt.invalid] The provided path points to an incompatible Python"
+                        " interpreter. Latest version requires CPython"
+                        f" {self.latest.requires_python} but the provided interpreter is"
+                        f" {info.implementation} {interpreter_version}."
+                    )
+                    continue
+                self.metadata.interpreter_version = interpreter_version
+                self.metadata.interpreter_info = info
+                self.metadata.interpreter_exe = info.executable
+            else:
+                (
+                    self.metadata.interpreter_exe,
+                    self.metadata.interpreter_version,
+                    self.metadata.interpreter_info,
+                ) = interpreters[result - 1]
+
+            self.console.print(
+                "\n[b]You selected:[/]",
+                _render_interpreter(
+                    self.metadata.interpreter_exe, self.metadata.interpreter_version
+                ),
+            )
+            if Confirm.ask("Do you want to continue with this choice?"):
+                self.console.print()
+                break
+
+    async def _check_cog_compatibility(self) -> None:
+        outputs = {}
+        checked_instances = {}
+        skipped_instances = []
+        failed_instances = []
+        unsupported_storage_instances = []
+        for instance_name in self.options.instances:
+            if instance_name in self.options.excluded_instances:
+                skipped_instances.append(instance_name)
+                continue
+            exit_code, stdout, results = await cmd.cog_compatibility.call(
+                instance_name,
+                red_version=self.latest.version,
+                python_version=self.metadata.interpreter_version,
+                ignore_prefix=self.options.ignore_prefix,
+                return_results=True,
+                stdout=asyncio.subprocess.PIPE,
+            )
+            if exit_code == cmd.cog_compatibility.EXIT_INSTANCE_BACKEND_UNSUPPORTED:
+                skipped_instances.append(instance_name)
+                unsupported_storage_instances.append(instance_name)
+            elif exit_code == cmd.cog_compatibility.EXIT_INSTANCE_SITE_PREFIX_MISMATCH:
+                skipped_instances.append(instance_name)
+            elif exit_code:
+                failed_instances.append(instance_name)
+                print(stdout, end="")
+                Text.assemble(
+                    "\N{UPWARDS ARROW} " * 3,
+                    "Failure for ",
+                    (instance_name, "bold"),
+                    " instance",
+                )
+                self.console.rule(
+                    Text.assemble(
+                        "\N{UPWARDS ARROW} " * 3,
+                        "Failure for ",
+                        (instance_name, "bold"),
+                        " instance above",
+                        " \N{UPWARDS ARROW}" * 3,
+                    ),
+                    style="red",
+                )
+            else:
+                assert results is not None
+                outputs[instance_name] = stdout
+                checked_instances[instance_name] = results
+            if stdout:
+                self.console.print()
+        self.console.print()
+        if not self.options.no_backup:
+            self.metadata.to_backup = [*checked_instances, *failed_instances]
+
+        if outputs:
+            for instance_name, stdout in outputs.items():
+                self.console.rule(Text(instance_name, style="bold"))
+                print(stdout, end="")
+            self.console.rule()
+
+        common.print_with_prefix_column(
+            common.ICON_INFO,
+            "Finished checking cog compatibility.",
+            (
+                "\nThe results for each of the checked instances are shown above."
+                if checked_instances
+                else ""
+            ),
+        )
+        if failed_instances:
+            common.print_with_prefix_column(
+                common.ICON_ERROR,
+                "Failure occurred while trying to check compatibility for following instances: ",
+                Text(", ").join(
+                    Text(instance_name, style="bold") for instance_name in failed_instances
+                ),
+                "\nScroll above to find the errors.",
+            )
+        if unsupported_storage_instances:
+            common.print_with_prefix_column(
+                common.ICON_INFO,
+                "The following instances were skipped as they use a storage backend that is"
+                " not supported by the current Red installation (some requirements are missing): ",
+                Text(", ").join(
+                    Text(instance_name, style="bold")
+                    for instance_name in unsupported_storage_instances
+                ),
+            )
+        if not checked_instances:
+            common.print_with_prefix_column(
+                common.ICON_INFO,
+                "There were no",
+                (" other" if failed_instances or unsupported_storage_instances else ""),
+                " instances to check cog compatibility for.",
+            )
+        self.console.print()
+
+        self.metadata.cog_compatibility = UpdaterCompatibilitySummary(
+            checked=checked_instances, failed=failed_instances, skipped=skipped_instances
+        )
+
+    async def _make_backups(self) -> None:
+        self.metadata.backup_dir = backup_dir = self.options.backup_dir or Path(
+            tempfile.mkdtemp(prefix="redbot-update-backup-")
+        )
+        console = common.get_console()
+        console.print("Backups will be created at:", Text(str(backup_dir), style="bold"))
+        venv_archive = backup_dir / "venv.tar.gz"
+        with console.status("Making a backup of the virtual environment directory..."):
+            venv_dir = Path(sys.prefix)
+            venv_files = []
+            for current_dir, _, filenames in os.walk(venv_dir):
+                target_dir = os.path.relpath(current_dir, venv_dir)
+                if target_dir == ".":
+                    target_dir = ""
+                for name in filenames:
+                    venv_files.append(
+                        (os.path.join(current_dir, name), os.path.join(target_dir, name))
+                    )
+            with tarfile.open(venv_archive, "w:gz", compresslevel=6) as tar:
+                with detailed_progress(unit="files") as progress:
+                    for src, arcname in progress.track(venv_files, description="Compressing..."):
+                        tar.add(src, arcname=arcname, recursive=False)
+        console.print(
+            "Created a backup of the virtual environment directory at:",
+            Text(str(venv_archive), style="bold"),
+        )
+
+        checked = []
+        failed = []
+        instance_backups_dir = backup_dir / "instance_backups"
+        instance_backups_dir.mkdir()
+        for instance_name in self.metadata.to_backup:
+            console.print(
+                "Making a backup of the", Text(instance_name, style="bold"), "instance..."
+            )
+            debug_args = (cmd.arg_names.DEBUG,) * common.get_log_cli_level()
+            proc = await asyncio.create_subprocess_exec(
+                sys.executable,
+                "-m",
+                "redbot.setup",
+                "backup",
+                *debug_args,
+                instance_name,
+                str(instance_backups_dir),
+            )
+            if await proc.wait():
+                failed.append(instance_name)
+            else:
+                checked.append(instance_name)
+
+        self.metadata.backup_results = BackupResults(checked=checked, failed=failed)
+        if self.metadata.cog_compatibility:
+            self.metadata.backup_results.skipped.extend(self.metadata.cog_compatibility.skipped)
+
+        if failed:
+            common.print_with_prefix_column(
+                common.ICON_ERROR,
+                "The following instances failed during backup: ",
+                Text(", ").join(Text(instance_name, style="bold") for instance_name in failed),
+                "\nScroll above to find the errors.",
+            )
+            # If a backup fails, we cannot allow non-interactive update to continue.
+            # The user can choose to use options such as `--no-backup`, `--instance`,
+            # and `--exclude-instance` to not have the backup step try to backup something
+            # that it can't.
+            if not self.options.interactive or not Confirm.ask(
+                "Do you want to continue with the update regardless?"
+            ):
+                raise SystemExit(1)
+
+    async def _update_with_fresh_venv(self) -> NoReturn:
+        console = common.get_console()
+        venv_dir = Path(sys.prefix)
+        backup_dir = venv_dir / common.OLD_VENV_BACKUP_DIR_NAME
+        try:
+            backup_dir.mkdir()
+        except FileExistsError:
+            console.print(
+                "Found that a partial backup of a virtual environment from a past failed update"
+                " exists at",
+                Text(str(backup_dir), style="bold"),
+                "\nThe update will not proceed to avoid overriding it. If you are certain that"
+                " you don't need to restore anything from it, remove it and try updating again.",
+            )
+            raise SystemExit(1)
+
+        with console.status("Determining extras to install..."):
+            try:
+                metadata = await self.latest.fetch_core_metadata()
+            except TypeError:
+                extras = get_installed_extras()
+            else:
+                known_extras = metadata.provides_extra or []
+                extras = [extra for extra in get_installed_extras() if extra in known_extras]
+        console.print("Extras to install have been determined.")
+
+        old_executable = Path(sys.executable)
+        rel_executable = old_executable.relative_to(venv_dir)
+        new_executable = backup_dir / rel_executable
+        wrapper_exe = runner.get_wrapper_executable()
+
+        with console.status("Moving old virtual environment..."):
+            for path in venv_dir.iterdir():
+                if path == backup_dir or path == wrapper_exe:
+                    continue
+                path.rename(backup_dir / path.name)
+        console.print("Old virtual environment moved.")
+
+        with tempfile.NamedTemporaryFile(
+            "w", encoding="utf-8", prefix="redbot-update-metadata-", suffix=".json", delete=False
+        ) as metadata_file:
+            json.dump(self.metadata.to_json_dict(), metadata_file)
+
+        console.print()
+        runner.make_exec_request(
+            str(new_executable),
+            "reinstall",
+            # base executable for venv creation
+            self.metadata.interpreter_exe,
+            # venv dir
+            str(venv_dir),
+            # scripts path
+            self.metadata.interpreter_info.sysconfig_path("scripts", {"base": str(venv_dir)}),
+            # Red dependency specifier
+            common.get_red_dependency_specifier(self.latest.version, extras),
+            set_env_vars={common.INTERNAL_UPDATER_METADATA_ENV_VAR: metadata_file.name},
+        )
+
+
+def get_updater_metadata() -> UpdaterMetadata:
+    with open(os.environ[common.INTERNAL_UPDATER_METADATA_ENV_VAR], encoding="utf-8") as fp:
+        return UpdaterMetadata.from_json_dict(json.load(fp))

redbot/core/_downloader/__init__.py

@@ -721,12 +721,15 @@ async def update_cogs(
     *,
     cogs: Optional[List[InstalledModule]] = None,
     repos: Optional[List[Repo]] = None,
+    update_repos: bool = True,
     env: Environment = Environment.current(),
 ) -> CogUpdateResult:
     if cogs is not None and repos is not None:
         raise ValueError("You can specify cogs or repos argument, not both")
 
-    cogs_to_check, failed_repos = await _get_cogs_to_check(repos=repos, cogs=cogs)
+    cogs_to_check, failed_repos = await _get_cogs_to_check(
+        repos=repos, cogs=cogs, update_repos=update_repos
+    )
     return await _update_cogs(cogs_to_check, failed_repos=failed_repos, env=env)
 
 
@@ -737,12 +740,14 @@ async def update_repo_cogs(
     cogs: Optional[List[InstalledModule]] = None,
     *,
     rev: Optional[str] = None,
+    update_repo: bool = True,
     env: Environment = Environment.current(),
 ) -> CogUpdateResult:
-    try:
+    if update_repo:
-        await repo.update()
+        try:
-    except errors.UpdateError:
+            await repo.update()
-        return await _update_cogs(set(), failed_repos=(repo.name,))
+        except errors.UpdateError:
+            return await _update_cogs(set(), failed_repos=(repo.name,))
 
     # TODO: should this be set to `repo.branch` when `rev` is None?
     commit = None

redbot/core/_downloader/info_schemas.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, Tuple, Union
 
@@ -21,6 +22,7 @@ class UseDefault:
 
 # sentinel value
 USE_DEFAULT = UseDefault()
+RED_TAG_READY_PATTERN = re.compile(r"^red-(?:[3-9]|[1-9][0-9]+)\.(?:[1-9][0-9]*)-ready$")
 
 
 def ensure_tuple_of_str(
@@ -203,6 +205,48 @@ def ensure_installable_type(
     return installable.InstallableType.UNKNOWN
 
 
+def ensure_tags(info_file: Path, key_name: str, value: Union[Any, UseDefault]) -> Tuple[str, ...]:
+    default: Tuple[str, ...] = ()
+    if value is USE_DEFAULT:
+        return default
+    if not isinstance(value, list):
+        log.warning(
+            "Invalid value of '%s' key (expected list, got %s)"
+            " in JSON information file at path: %s",
+            key_name,
+            type(value).__name__,
+            info_file,
+        )
+        return default
+    valid_tags = []
+    for item in value:
+        if not isinstance(item, str):
+            log.warning(
+                "Invalid item in '%s' list (expected str, got %s)"
+                " in JSON information file at path: %s",
+                key_name,
+                type(item).__name__,
+                info_file,
+            )
+            return default
+        # `red-` tags are reserved for informational metadata we only support a subset of tags
+        if not item.startswith("red-"):
+            valid_tags.append(item)
+            continue
+        if RED_TAG_READY_PATTERN.match(item):
+            valid_tags.append(item)
+        else:
+            log.warning(
+                "Invalid value in '%s' list (tag starts with the reserved 'red-' prefix"
+                " but does not use the only supported reserved tag format: 'red-X.Y-ready')"
+                " in JSON information file at path: %s",
+                key_name,
+                info_file,
+            )
+
+    return tuple(value)
+
+
 EnsureCallable = Callable[[Path, str, Union[Any, UseDefault]], Any]
 SchemaType = Dict[str, EnsureCallable]
 
@@ -224,7 +268,7 @@ INSTALLABLE_SCHEMA: SchemaType = {
     "disabled": ensure_bool,
     "required_cogs": ensure_required_cogs_mapping,
     "requirements": ensure_tuple_of_str,
-    "tags": ensure_tuple_of_str,
+    "tags": ensure_tags,
     "type": ensure_installable_type,
     "end_user_data_statement": ensure_str,
 }

redbot/core/_drivers/__init__.py

@@ -2,7 +2,7 @@ import enum
 from typing import Optional, Type
 
 from .. import data_manager
-from .base import IdentifierData, BaseDriver, ConfigCategory
+from .base import IdentifierData, BaseDriver, ConfigCategory, MissingExtraRequirements
 from .json import JsonDriver
 from .postgres import PostgresDriver
 
@@ -12,6 +12,7 @@ __all__ = [
     "get_driver_class_include_old",
     "ConfigCategory",
     "IdentifierData",
+    "MissingExtraRequirements",
     "BaseDriver",
     "JsonDriver",
     "PostgresDriver",

redbot/core/_events.py

@@ -1,6 +1,7 @@
 import asyncio
 import contextlib
 import platform
+import shlex
 import sys
 import logging
 import traceback
@@ -9,8 +10,7 @@ from typing import Tuple
 
 import aiohttp
 import discord
-import importlib.metadata
+import redbot_update
-from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 from redbot.core import data_manager
@@ -53,7 +53,7 @@ ______         _           ______ _                       _  ______       _
 _ = Translator(__name__, __file__)
 
 
-def get_outdated_red_messages(pypi_version: str, requires_python: SpecifierSet) -> Tuple[str, str]:
+def get_outdated_red_messages(pypi_version: str) -> Tuple[str, str]:
     outdated_red_message = _(
         "Your Red instance is out of date! {} is the current version, however you are using {}!"
     ).format(pypi_version, red_version)
@@ -62,7 +62,6 @@ def get_outdated_red_messages(pypi_version: str, requires_python: SpecifierSet)
         f"[red]!!![/red]Version [cyan]{pypi_version}[/] is available, "
         f"but you're using [cyan]{red_version}[/][red]!!![/red]"
     )
-    current_python = Version(platform.python_version())
     extra_update = _(
         "\n\nWhile the following command should work in most scenarios as it is "
         "based on your current OS, environment, and Python version, "
@@ -71,64 +70,15 @@ def get_outdated_red_messages(pypi_version: str, requires_python: SpecifierSet)
         "needs to be done during the update.**"
     ).format(docs="https://docs.discord.red/en/stable/update_red.html")
 
-    if current_python not in requires_python:
+    redbot_update_bin = redbot_update.find_redbot_update_bin()
-        extra_update += _(
+    is_windows = platform.system() == "Windows"
-            "\n\nYou have Python `{py_version}` and this update "
+    update_command = f'"{redbot_update_bin}"' if is_windows else shlex.quote(redbot_update_bin)
-            "requires `{req_py}`; you cannot simply run the update command.\n\n"
-            "You will need to follow the update instructions in our docs above, "
-            "if you still need help updating after following the docs go to our "
-            "#support channel in <https://discord.gg/red>"
-        ).format(py_version=current_python, req_py=requires_python)
-        outdated_red_message += extra_update
-        return outdated_red_message, rich_outdated_message
-
-    red_dist = importlib.metadata.distribution("Red-DiscordBot")
-    installed_extras = red_dist.metadata.get_all("Provides-Extra")
-    installed_extras.remove("dev")
-    installed_extras.remove("all")
-    distributions = {}
-    for req_str in red_dist.requires:
-        req = Requirement(req_str)
-        if req.marker is None or req.marker.evaluate():
-            continue
-        for extra in reversed(installed_extras):
-            if not req.marker.evaluate({"extra": extra}):
-                continue
-
-            # Check that the requirement is met.
-            # This is a bit simplified for our purposes and does not check
-            # whether the requirements of our requirements are met as well.
-            # This could potentially be an issue if we'll ever depend on
-            # a dependency's extra in our extra when we already depend on that
-            # in our base dependencies. However, considering that right now, all
-            # our dependencies are also fully pinned, this should not ever matter.
-            if req.name in distributions:
-                dist = distributions[req.name]
-            else:
-                try:
-                    dist = importlib.metadata.distribution(req.name)
-                except importlib.metadata.PackageNotFoundError:
-                    dist = None
-                distributions[req.name] = dist
-            if dist is None or not req.specifier.contains(dist.version, prereleases=True):
-                installed_extras.remove(extra)
-
-    if installed_extras:
-        package_extras = f"[{','.join(installed_extras)}]"
-    else:
-        package_extras = ""
-
     extra_update += _(
         "\n\nTo update your bot, first shutdown your bot"
-        " then open a window of {console} (Not as admin) and run the following:"
+        " then open a window of {console} (Not as admin) and run the following: {command}"
-        "{command_1}\n"
-        "Once you've started up your bot again, we recommend that"
-        " you update any installed 3rd-party cogs with this command in Discord:"
-        "{command_2}"
     ).format(
-        console=_("Command Prompt") if platform.system() == "Windows" else _("Terminal"),
+        console=_("Command Prompt") if is_windows else _("Terminal"),
-        command_1=f'```"{sys.executable}" -m pip install -U "Red-DiscordBot{package_extras}"```',
+        command=f"```{update_command}```",
-        command_2=f"```[p]cog update```",
     )
     outdated_red_message += extra_update
     return outdated_red_message, rich_outdated_message
@@ -224,7 +174,7 @@ def init_events(bot, cli_flags):
             outdated = latest.version > Version(red_version)
             if outdated:
                 outdated_red_message, rich_outdated_message = get_outdated_red_messages(
-                    latest.version, latest.requires_python
+                    latest.version
                 )
                 rich_console.print(rich_outdated_message)
                 await send_to_owners_with_prefix_replaced(bot, outdated_red_message)

redbot/core/utils/_internal_utils.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import asyncio
 import collections.abc
 import contextlib
+import importlib.metadata
 import json
 import logging
 import os
@@ -40,6 +41,7 @@ import aiohttp
 import discord
 import yarl
 from packaging.metadata import Metadata
+from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.utils import parse_sdist_filename
 from packaging.version import Version
@@ -587,6 +589,43 @@ async def fetch_latest_red_version(
     return available_versions[0]
 
 
+def get_installed_extras() -> List[str]:
+    red_dist = importlib.metadata.distribution("Red-DiscordBot")
+    installed_extras = red_dist.metadata.get_all("Provides-Extra")
+    if installed_extras is None:
+        return []
+    installed_extras.remove("dev")
+    installed_extras.remove("all")
+    distributions: Dict[str, Optional[importlib.metadata.Distribution]] = {}
+    for req_str in red_dist.requires or []:
+        req = Requirement(req_str)
+        if req.marker is None or req.marker.evaluate():
+            continue
+        for extra in reversed(installed_extras):
+            if not req.marker.evaluate({"extra": extra}):
+                continue
+
+            # Check that the requirement is met.
+            # This is a bit simplified for our purposes and does not check
+            # whether the requirements of our requirements are met as well.
+            # This could potentially be an issue if we'll ever depend on
+            # a dependency's extra in our extra when we already depend on that
+            # in our base dependencies. However, considering that right now, all
+            # our dependencies are also fully pinned, this should not ever matter.
+            if req.name in distributions:
+                dist = distributions[req.name]
+            else:
+                try:
+                    dist = importlib.metadata.distribution(req.name)
+                except importlib.metadata.PackageNotFoundError:
+                    dist = None
+                distributions[req.name] = dist
+            if dist is None or not req.specifier.contains(dist.version, prereleases=True):
+                installed_extras.remove(extra)
+
+    return installed_extras
+
+
 def deprecated_removed(
     deprecation_target: str,
     deprecation_version: str,
@@ -651,3 +690,15 @@ def cli_level_to_log_level(level: int) -> int:
     else:
         log_level = TRACE
     return log_level
+
+
+def log_level_to_cli_level(log_level: int) -> int:
+    if log_level == TRACE:
+        level = 3
+    elif log_level == VERBOSE:
+        level = 2
+    elif log_level == logging.DEBUG:
+        level = 1
+    else:
+        level = 0
+    return level

requirements/base.in

@@ -9,12 +9,15 @@ packaging
 platformdirs
 psutil
 python-dateutil
+python-discovery
 PyYAML
 rapidfuzz
 Red-Commons
 Red-Lavalink
+redbot-update
 rich
 schema
+textual
 typing_extensions
 yarl
 distro; sys_platform == "linux"

requirements/base.txt

@@ -22,18 +22,26 @@ discord-py==2.7.1
     # via
     #   -r base.in
     #   red-lavalink
+filelock==3.16.1
+    # via python-discovery
 frozenlist==1.5.0
     # via
     #   aiohttp
     #   aiosignal
 idna==3.11
     # via yarl
+linkify-it-py==2.0.3
+    # via markdown-it-py
 markdown==3.7
     # via -r base.in
 markdown-it-py==3.0.0
-    # via rich
+    # via
+    #   rich
+    #   textual
 mdurl==0.1.2
     # via markdown-it-py
+mdit-py-plugins==0.4.2
+    # via markdown-it-py
 multidict==6.1.0
     # via
     #   aiohttp
@@ -43,15 +51,21 @@ orjson==3.10.15
 packaging==26.0
     # via -r base.in
 platformdirs==4.3.6
-    # via -r base.in
+    # via
+    #   -r base.in
+    #   textual
 propcache==0.2.0
     # via yarl
 psutil==7.2.2
     # via -r base.in
 pygments==2.19.2
-    # via rich
+    # via
+    #   rich
+    #   textual
 python-dateutil==2.9.0.post0
     # via -r base.in
+python-discovery==1.2.1
+    # via -r base.in
 pyyaml==6.0.3
     # via -r base.in
 rapidfuzz==3.9.7
@@ -62,16 +76,25 @@ red-commons==1.0.0
     #   red-lavalink
 red-lavalink==0.11.1
     # via -r base.in
-rich==14.3.3
+redbot-update==1.1.0
     # via -r base.in
+rich==14.3.3
+    # via
+    #   -r base.in
+    #   textual
 schema==0.7.8
     # via -r base.in
 six==1.17.0
     # via python-dateutil
+textual==6.2.1
+    # via -r base.in
 typing-extensions==4.13.2
     # via
     #   -r base.in
     #   multidict
+    #   textual
+uc-micro-py==1.0.3
+    # via linkify-it-py
 yarl==1.15.2
     # via
     #   -r base.in

schema/red_cog.schema.json

@@ -71,7 +71,8 @@
       "description": "A list of strings that are related to the functionality of the cog. Used to aid in searching.",
       "uniqueItems": true,
       "items": {
-        "type": "string"
+        "type": "string",
+        "pattern": "^(?:(?!red-).+|red-(?:[3-9]|[1-9][0-9]+)\\.(?:[1-9][0-9]*)-ready)$"
       }
     },
     "type": {