[Core, Downloader] Clear lib folder on minor Python version change, add [p]cog reinstallreqs command (#3274)

* feat(downloader): add `[p]cog reinstallreqs` command

* enhance: clear lib folder on minor Python version change

* chore(changelog): add towncrier entries

* enhance: warn user about detected change in OS or arch

* enhance: use actual prefix instead of `[p]`

* Whoops...

Co-Authored-By: Michael H <michael@michaelhall.tech>

* enhance: wrap message sending in try except

Co-authored-by: Michael H <michael@michaelhall.tech>

changelog.d/3274.enhance.1.rst

@@ -0,0 +1 @@
+Lib folder is now cleared on minor Python version change. `[p]cog reinstallreqs` command in Downloader cog can be used to regenerate lib folder for new Python version.

changelog.d/3274.enhance.2.rst

@@ -0,0 +1 @@
+If Red detects operating system or architecture change, it will warn owner about possible problem with lib folder.

changelog.d/downloader/3167.feature.rst

@@ -0,0 +1 @@
+Added `[p]cog reinstallreqs` command that allows to reinstall cog requirements and shared libraries for all installed cogs.

redbot/cogs/downloader/downloader.py

@@ -50,13 +50,18 @@ class Downloader(commands.Cog):
         self.SHAREDLIB_PATH = self.LIB_PATH / "cog_shared"
         self.SHAREDLIB_INIT = self.SHAREDLIB_PATH / "__init__.py"
 
+        self._create_lib_folder()
+
+        self._repo_manager = RepoManager()
+
+    def _create_lib_folder(self, *, remove_first: bool = False) -> None:
+        if remove_first:
+            shutil.rmtree(str(self.LIB_PATH))
         self.SHAREDLIB_PATH.mkdir(parents=True, exist_ok=True)
         if not self.SHAREDLIB_INIT.exists():
             with self.SHAREDLIB_INIT.open(mode="w", encoding="utf-8") as _:
                 pass
 
-        self._repo_manager = RepoManager()
-
     async def initialize(self) -> None:
         await self._repo_manager.initialize()
         await self._maybe_update_config()
@@ -553,6 +558,59 @@ class Downloader(commands.Cog):
         """Cog installation management commands."""
         pass
 
+    @cog.command(name="reinstallreqs")
+    async def _cog_reinstallreqs(self, ctx: commands.Context) -> None:
+        """
+        This command will reinstall cog requirements and shared libraries for all installed cogs.
+
+        Red might ask user to use this when it clears contents of lib folder
+        because of change in minor version of Python.
+        """
+        async with ctx.typing():
+            self._create_lib_folder(remove_first=True)
+            installed_cogs = await self.installed_cogs()
+            cogs = []
+            repos = set()
+            for cog in installed_cogs:
+                if cog.repo is None:
+                    continue
+                repos.add(cog.repo)
+                cogs.append(cog)
+            failed_reqs = await self._install_requirements(cogs)
+            all_installed_libs: List[InstalledModule] = []
+            all_failed_libs: List[Installable] = []
+            for repo in repos:
+                installed_libs, failed_libs = await repo.install_libraries(
+                    target_dir=self.SHAREDLIB_PATH, req_target_dir=self.LIB_PATH
+                )
+                all_installed_libs += installed_libs
+                all_failed_libs += failed_libs
+        message = ""
+        if failed_reqs:
+            message += _("Failed to install requirements: ") + humanize_list(
+                tuple(map(inline, failed_reqs))
+            )
+        if all_failed_libs:
+            libnames = [lib.name for lib in failed_libs]
+            message += _("\nFailed to install shared libraries: ") + humanize_list(
+                tuple(map(inline, libnames))
+            )
+        if message:
+            await ctx.send(
+                _(
+                    "Cog requirements and shared libraries for all installed cogs"
+                    " have been reinstalled but there were some errors:\n"
+                )
+                + message
+            )
+        else:
+            await ctx.send(
+                _(
+                    "Cog requirements and shared libraries"
+                    " for all installed cogs have been reinstalled."
+                )
+            )
+
     @cog.command(name="install", usage="<repo_name> <cogs>")
     async def _cog_install(self, ctx: commands.Context, repo: Repo, *cog_names: str) -> None:
         """Install a cog from the given repo."""

redbot/core/bot.py

@@ -2,6 +2,8 @@ import asyncio
 import inspect
 import logging
 import os
+import platform
+import shutil
 import sys
 from collections import namedtuple
 from datetime import datetime
@@ -17,6 +19,7 @@ from discord.ext.commands import when_mentioned_or
 from . import Config, i18n, commands, errors, drivers, modlog, bank
 from .cog_manager import CogManager, CogManagerUI
 from .core_commands import license_info_command, Core
+from .data_manager import cog_data_path
 from .dev_commands import Dev
 from .events import init_events
 from .global_checks import init_global_checks
@@ -79,6 +82,9 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
             disabled_command_msg="That command is disabled.",
             extra_owner_destinations=[],
             owner_opt_out_list=[],
+            last_system_info__python_version=[3, 7],
+            last_system_info__machine=None,
+            last_system_info__system=None,
             schema_version=0,
         )
 
@@ -413,11 +419,70 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
 
         packages = []
 
-        if cli_flags.no_cogs is False:
-            packages.extend(await self._config.packages())
+        last_system_info = await self._config.last_system_info()
 
-        if cli_flags.load_cogs:
-            packages.extend(cli_flags.load_cogs)
+        async def notify_owners(content: str) -> None:
+            destinations = await self.get_owner_notification_destinations()
+            for destination in destinations:
+                prefixes = await self.get_valid_prefixes(getattr(destination, "guild", None))
+                prefix = prefixes[0]
+                try:
+                    await destination.send(content.format(prefix=prefix))
+                except Exception as _exc:
+                    log.exception(
+                        f"I could not send an owner notification to ({destination.id}){destination}"
+                    )
+
+        ver_info = list(sys.version_info[:2])
+        python_version_changed = False
+        LIB_PATH = cog_data_path(raw_name="Downloader") / "lib"
+        if ver_info != last_system_info["python_version"]:
+            await self._config.last_system_info.python_version.set(ver_info)
+            if any(LIB_PATH.iterdir()):
+                shutil.rmtree(str(LIB_PATH))
+                LIB_PATH.mkdir()
+                self.loop.create_task(
+                    notify_owners(
+                        "We detected a change in minor Python version"
+                        " and cleared packages in lib folder.\n"
+                        "The instance was started with no cogs, please load Downloader"
+                        " and use `{prefix}cog reinstallreqs` to regenerate lib folder."
+                        " After that, restart the bot to get"
+                        " all of your previously loaded cogs loaded again."
+                    )
+                )
+                python_version_changed = True
+        else:
+            if cli_flags.no_cogs is False:
+                packages.extend(await self._config.packages())
+
+            if cli_flags.load_cogs:
+                packages.extend(cli_flags.load_cogs)
+
+        system_changed = False
+        machine = platform.machine()
+        system = platform.system()
+        if last_system_info["machine"] is None:
+            await self._config.last_system_info.machine.set(machine)
+        elif last_system_info["machine"] != machine:
+            await self._config.last_system_info.machine.set(machine)
+            system_changed = True
+
+        if last_system_info["system"] is None:
+            await self._config.last_system_info.system.set(system)
+        elif last_system_info["system"] != system:
+            await self._config.last_system_info.system.set(system)
+            system_changed = True
+
+        if system_changed and not python_version_changed:
+            self.loop.create_task(
+                notify_owners(
+                    "We detected a possible change in machine's operating system"
+                    " or architecture. You might need to regenerate your lib folder"
+                    " if 3rd-party cogs stop working properly.\n"
+                    "To regenerate lib folder, load Downloader and use `{prefix}cog reinstallreqs`."
+                )
+            )
 
         if packages:
             # Load permissions first, for security reasons