[Core] Add deprecation warnings about removal of shared libraries. (#3106)

* feat: add deprecation warning when importing shared libs

* enhance(downloader): add shared libs deprecation warns

* enhance: add deprecation warning when (re)loading cogs

* docs(downloader): add deprecation note about shared libs

* chore(changelog): add towncrier entries

* style: split long tuple unpacks in multiple lines

* fix: argument to `humanize_list` has to be a sequence

changelog.d/3106.docs.rst

@@ -0,0 +1 @@
+Add deprecation note about shared libraries in Downloader Framework docs.

changelog.d/3106.misc.1.rst

@@ -0,0 +1 @@
+Send deprecation warning when using `[p]load` and `[p]reload` commands if the repos loaded cogs are from have shared libraries.

changelog.d/3106.misc.2.rst

@@ -0,0 +1 @@
+Print deprecation loading when some package tries importing from `cog_shared.*`.

changelog.d/3106.removal.rst

@@ -0,0 +1 @@
+Shared libraries are marked for removal in Red 3.3.

changelog.d/downloader/3106.misc.rst

@@ -0,0 +1 @@
+Send deprecation warning when using install and update commands if the repos installed/updated cogs are from have shared libraries.

docs/framework_downloader.rst

@@ -55,6 +55,9 @@ Keys specific to the cog info.json (case sensitive)
 - ``type`` (string) - Optional, defaults to ``COG``. Must be either ``COG`` or
   ``SHARED_LIBRARY``. If ``SHARED_LIBRARY`` then ``hidden`` will be ``True``.
 
+.. warning::
+    Shared libraries are deprecated since version 3.2 and are marked for removal in version 3.3.
+
 API Reference
 *************
 

redbot/__main__.py

@@ -33,6 +33,7 @@ from redbot.core.core_commands import Core, license_info_command
 from redbot.setup import get_data_dir, get_name, save_config
 from redbot.core.dev_commands import Dev
 from redbot.core import __version__, modlog, bank, data_manager, drivers
+from redbot.core._sharedlibdeprecation import SharedLibImportWarner
 from signal import SIGTERM
 
 
@@ -322,6 +323,7 @@ def main():
     LIB_PATH.mkdir(parents=True, exist_ok=True)
     if str(LIB_PATH) not in sys.path:
         sys.path.append(str(LIB_PATH))
+    sys.meta_path.insert(0, SharedLibImportWarner())
 
     red.add_cog(Core(red))
     red.add_cog(CogManagerUI())

redbot/cogs/downloader/downloader.py

@@ -27,6 +27,13 @@ from .repo_manager import RepoManager, Repo
 _ = Translator("Downloader", __file__)
 
 
+DEPRECATION_NOTICE = _(
+    "\n**WARNING:** The following repos are using shared libraries"
+    " which are marked for removal in Red 3.3: {repo_list}.\n"
+    " You should inform maintainers of these repos about this message."
+)
+
+
 @cog_i18n(_)
 class Downloader(commands.Cog):
     def __init__(self, bot: Red):
@@ -192,6 +199,16 @@ class Downloader(commands.Cog):
         await self.conf.installed_cogs.set(installed_cogs)
         await self.conf.installed_libraries.set(installed_libraries)
 
+    async def _shared_lib_load_check(self, cog_name: str) -> Optional[Repo]:
+        # remove in Red 3.3
+        is_installed, cog = await self.is_installed(cog_name)
+        # it's not gonna be None when `is_installed` is True
+        # if we'll use typing_extensions in future, `Literal` can solve this
+        cog = cast(InstalledModule, cog)
+        if is_installed and cog.repo is not None and cog.repo.available_libraries:
+            return cog.repo
+        return None
+
     async def _available_updates(
         self, cogs: Iterable[InstalledModule]
     ) -> Tuple[Tuple[Installable, ...], Tuple[Installable, ...]]:
@@ -584,6 +601,9 @@ class Downloader(commands.Cog):
 
                 installed_cogs, failed_cogs = await self._install_cogs(cogs)
 
+            deprecation_notice = ""
+            if repo.available_libraries:
+                deprecation_notice = DEPRECATION_NOTICE.format(repo_list=inline(repo.name))
             installed_libs, failed_libs = await repo.install_libraries(
                 target_dir=self.SHAREDLIB_PATH, req_target_dir=self.LIB_PATH
             )
@@ -622,7 +642,7 @@ class Downloader(commands.Cog):
                     + message
                 )
         # "---" added to separate cog install messages from Downloader's message
-        await ctx.send(f"{message}\n---")
+        await ctx.send(f"{message}{deprecation_notice}\n---")
         for cog in installed_cogs:
             if cog.install_msg:
                 await ctx.send(cog.install_msg.replace("[p]", ctx.prefix))
@@ -874,6 +894,14 @@ class Downloader(commands.Cog):
         if failed_repos:
             message += "\n" + self.format_failed_repos(failed_repos)
 
+        repos_with_libs = {
+            inline(module.repo.name)
+            for module in cogs_to_update + libs_to_update
+            if module.repo.available_libraries
+        }
+        if repos_with_libs:
+            message += DEPRECATION_NOTICE.format(repo_list=humanize_list(list(repos_with_libs)))
+
         await ctx.send(message)
 
         if updates_available and updated_cognames:

redbot/core/_sharedlibdeprecation.py

@@ -0,0 +1,29 @@
+from importlib.abc import MetaPathFinder
+import warnings
+
+
+class SharedLibDeprecationWarning(DeprecationWarning):
+    pass
+
+
+warnings.simplefilter("always", SharedLibDeprecationWarning)
+
+
+class SharedLibImportWarner(MetaPathFinder):
+    """
+    Deprecation warner for shared libraries. This class sits on `sys.meta_path`
+    and prints warning if imported module is a shared library
+    """
+
+    def find_spec(self, fullname, path, target=None) -> None:
+        """This is only supposed to print warnings, it won't ever return module spec."""
+        parts = fullname.split(".")
+        if parts[0] != "cog_shared" or len(parts) != 2:
+            return None
+        msg = (
+            "One of cogs uses shared libraries which are"
+            " deprecated and scheduled for removal in Red 3.3.\n"
+            "You should inform author of the cog about this message."
+        )
+        warnings.warn(msg, SharedLibDeprecationWarning, stacklevel=2)
+        return None

redbot/core/core_commands.py

@@ -14,7 +14,7 @@ from collections import namedtuple
 from pathlib import Path
 from random import SystemRandom
 from string import ascii_letters, digits
-from typing import TYPE_CHECKING, Union, Tuple, List, Optional, Iterable, Sequence, Dict
+from typing import TYPE_CHECKING, Union, Tuple, List, Optional, Iterable, Sequence, Dict, Set
 
 import aiohttp
 import discord
@@ -70,7 +70,7 @@ class CoreLogic:
 
     async def _load(
         self, cog_names: Iterable[str]
-    ) -> Tuple[List[str], List[str], List[str], List[str], List[Tuple[str, str]]]:
+    ) -> Tuple[List[str], List[str], List[str], List[str], List[Tuple[str, str]], Set[str]]:
         """
         Loads cogs by name.
         Parameters
@@ -87,6 +87,7 @@ class CoreLogic:
         notfound_packages = []
         alreadyloaded_packages = []
         failed_with_reason_packages = []
+        repos_with_shared_libs = set()
 
         bot = self.bot
 
@@ -125,6 +126,20 @@ class CoreLogic:
             else:
                 await bot.add_loaded_package(name)
                 loaded_packages.append(name)
+                # remove in Red 3.3
+                downloader = bot.get_cog("Downloader")
+                if downloader is None:
+                    continue
+                try:
+                    maybe_repo = await downloader._shared_lib_load_check(name)
+                except Exception:
+                    log.exception(
+                        "Shared library check failed,"
+                        " if you're not using modified Downloader, report this issue."
+                    )
+                    maybe_repo = None
+                if maybe_repo is not None:
+                    repos_with_shared_libs.add(maybe_repo.name)
 
         return (
             loaded_packages,
@@ -132,6 +147,7 @@ class CoreLogic:
             notfound_packages,
             alreadyloaded_packages,
             failed_with_reason_packages,
+            repos_with_shared_libs,
         )
 
     @staticmethod
@@ -186,14 +202,26 @@ class CoreLogic:
 
     async def _reload(
         self, cog_names: Sequence[str]
-    ) -> Tuple[List[str], List[str], List[str], List[str], List[Tuple[str, str]]]:
+    ) -> Tuple[List[str], List[str], List[str], List[str], List[Tuple[str, str]], Set[str]]:
         await self._unload(cog_names)
 
-        loaded, load_failed, not_found, already_loaded, load_failed_with_reason = await self._load(
+        (
-            cog_names
+            loaded,
-        )
+            load_failed,
+            not_found,
+            already_loaded,
+            load_failed_with_reason,
+            repos_with_shared_libs,
+        ) = await self._load(cog_names)
 
-        return loaded, load_failed, not_found, already_loaded, load_failed_with_reason
+        return (
+            loaded,
+            load_failed,
+            not_found,
+            already_loaded,
+            load_failed_with_reason,
+            repos_with_shared_libs,
+        )
 
     async def _name(self, name: Optional[str] = None) -> str:
         """
@@ -580,7 +608,14 @@ class Core(commands.Cog, CoreLogic):
             return await ctx.send_help()
         cogs = tuple(map(lambda cog: cog.rstrip(","), cogs))
         async with ctx.typing():
-            loaded, failed, not_found, already_loaded, failed_with_reason = await self._load(cogs)
+            (
+                loaded,
+                failed,
+                not_found,
+                already_loaded,
+                failed_with_reason,
+                repos_with_shared_libs,
+            ) = await self._load(cogs)
 
         output = []
 
@@ -636,6 +671,21 @@ class Core(commands.Cog, CoreLogic):
                 ).format(reasons=reasons)
             output.append(formed)
 
+        if repos_with_shared_libs:
+            if len(repos_with_shared_libs) == 1:
+                formed = _(
+                    "**WARNING**: The following repo is using shared libs"
+                    " which are marked for removal in Red 3.3: {repo}.\n"
+                    "You should inform maintainer of the repo about this message."
+                ).format(repo=inline(repos_with_shared_libs.pop()))
+            else:
+                formed = _(
+                    "**WARNING**: The following repos are using shared libs"
+                    " which are marked for removal in Red 3.3: {repos}.\n"
+                    "You should inform maintainers of these repos about this message."
+                ).format(repos=humanize_list([inline(repo) for repo in repos_with_shared_libs]))
+            output.append(formed)
+
         if output:
             total_message = "\n\n".join(output)
             for page in pagify(total_message):
@@ -687,9 +737,14 @@ class Core(commands.Cog, CoreLogic):
             return await ctx.send_help()
         cogs = tuple(map(lambda cog: cog.rstrip(","), cogs))
         async with ctx.typing():
-            loaded, failed, not_found, already_loaded, failed_with_reason = await self._reload(
+            (
-                cogs
+                loaded,
-            )
+                failed,
+                not_found,
+                already_loaded,
+                failed_with_reason,
+                repos_with_shared_libs,
+            ) = await self._reload(cogs)
 
         output = []
 
@@ -734,6 +789,21 @@ class Core(commands.Cog, CoreLogic):
                 ).format(reasons=reasons)
             output.append(formed)
 
+        if repos_with_shared_libs:
+            if len(repos_with_shared_libs) == 1:
+                formed = _(
+                    "**WARNING**: The following repo is using shared libs"
+                    " which are marked for removal in Red 3.3: {repo}.\n"
+                    "You should inform maintainers of these repos about this message."
+                ).format(repo=inline(repos_with_shared_libs.pop()))
+            else:
+                formed = _(
+                    "**WARNING**: The following repos are using shared libs"
+                    " which are marked for removal in Red 3.3: {repos}.\n"
+                    "You should inform maintainers of these repos about this message."
+                ).format(repos=humanize_list([inline(repo) for repo in repos_with_shared_libs]))
+            output.append(formed)
+
         if output:
             total_message = "\n\n".join(output)
             for page in pagify(total_message):