[V3 Cog/Data Manager] Bundled Cog Data (#1063)

* Refactor find_spec out of core_commands

* Fix version error when not installed

* initial

* Fix find_cogs call

* Enable copying

* Add helper method for cog creators

* Add warning

* My dpy skillz need work

redbot/core/__init__.py

@@ -5,4 +5,7 @@ from .context import RedContext
 
 __all__ = ["Config", "RedContext", "__version__"]
 
-__version__ = version = pkg_resources.require("Red-DiscordBot")[0].version
+try:
+    __version__ = pkg_resources.require("Red-DiscordBot")[0].version
+except pkg_resources.DistributionNotFound:
+    __version__ = "3.0.0"

redbot/core/cog_manager.py

@@ -1,8 +1,9 @@
+import contextlib
 import pkgutil
-from importlib import invalidate_caches
+from importlib import import_module, invalidate_caches
 from importlib.machinery import ModuleSpec
 from pathlib import Path
-from typing import Tuple, Union, List
+from typing import Tuple, Union, List, overload
 
 import redbot.cogs
 
@@ -187,6 +188,61 @@ class CogManager:
         str_paths = [str(p) for p in paths_]
         await self.conf.paths.set(str_paths)
 
+    async def _find_ext_cog(self, name: str) -> ModuleSpec:
+        """
+        Attempts to find a spec for a third party installed cog.
+
+        Parameters
+        ----------
+        name : str
+            Name of the cog package to look for.
+
+        Returns
+        -------
+        importlib.machinery.ModuleSpec
+            Module spec to be used for cog loading.
+
+        Raises
+        ------
+        RuntimeError
+            When no matching spec can be found.
+        """
+        resolved_paths = [str(p.resolve()) for p in await self.paths()]
+        for finder, module_name, _ in pkgutil.iter_modules(resolved_paths):
+            if name == module_name:
+                spec = finder.find_spec(name)
+                if spec:
+                    return spec
+
+        raise RuntimeError("No 3rd party module by the name of '{}' was found"
+                           " in any available path.".format(name))
+
+    async def _find_core_cog(self, name: str) -> ModuleSpec:
+        """
+        Attempts to find a spec for a core cog.
+
+        Parameters
+        ----------
+        name : str
+
+        Returns
+        -------
+        importlib.machinery.ModuleSpec
+
+        Raises
+        ------
+        RuntimeError
+            When no matching spec can be found.
+        """
+        real_name = ".{}".format(name)
+        try:
+            mod = import_module(real_name, package='redbot.cogs')
+        except ImportError as e:
+            raise RuntimeError("No core cog by the name of '{}' could"
+                               "be found.".format(name)) from e
+        return mod.__spec__
+
+    # noinspection PyUnreachableCode
     async def find_cog(self, name: str) -> ModuleSpec:
         """Find a cog in the list of available paths.
 
@@ -206,15 +262,13 @@ class CogManager:
             If there is no cog with the given name.
 
         """
-        resolved_paths = [str(p.resolve()) for p in await self.paths()]
+        with contextlib.suppress(RuntimeError):
-        for finder, module_name, _ in pkgutil.iter_modules(resolved_paths):
+            return await self._find_ext_cog(name)
-            if name == module_name:
-                spec = finder.find_spec(name)
-                if spec:
-                    return spec
 
-        raise RuntimeError("No module by the name of '{}' was found"
+        with contextlib.suppress(RuntimeError):
-                           " in any available path.".format(name))
+            return await self._find_core_cog(name)
+
+        raise RuntimeError("No cog with that name could be found.")
 
     async def available_modules(self) -> List[str]:
         """Finds the names of all available modules to load.

redbot/core/core_commands.py

@@ -14,9 +14,7 @@ from discord.ext import commands
 from redbot.core import checks
 from redbot.core import i18n
 
-import redbot.cogs  # Don't remove this line or core cogs won't load
+__all__ = ["Core"]
-
-__all__ = ["find_spec", "Core"]
 
 log = logging.getLogger("red")
 
@@ -26,20 +24,6 @@ OWNER_DISCLAIMER = ("⚠ **Only** the person who is hosting Red should be "
                     "system.** ⚠")
 
 
-async def find_spec(bot, cog_name: str):
-    try:
-        spec = await bot.cog_mgr.find_cog(cog_name)
-    except RuntimeError:
-        real_name = ".{}".format(cog_name)
-        try:
-            mod = importlib.import_module(real_name, package='redbot.cogs')
-        except ImportError:
-            spec = None
-        else:
-            spec = mod.__spec__
-    return spec
-
-
 _ = i18n.CogI18n("Core", __file__)
 
 
@@ -50,8 +34,9 @@ class Core:
     @checks.is_owner()
     async def load(self, ctx, *, cog_name: str):
         """Loads a package"""
-        spec = await find_spec(ctx.bot, cog_name)
+        try:
-        if spec is None:
+            spec = await ctx.bot.cog_mgr.find_cog(cog_name)
+        except RuntimeError:
             await ctx.send(_("No module by that name was found in any"
                              " cog path."))
             return
@@ -83,7 +68,7 @@ class Core:
         """Reloads a package"""
         ctx.bot.unload_extension(cog_name)
 
-        spec = await find_spec(ctx.bot, cog_name)
+        spec = await ctx.bot.cog_mgr.find_cog(cog_name)
         if spec is None:
             await ctx.send(_("No module by that name was found in any"
                              " cog path."))

redbot/core/data_manager.py

@@ -1,6 +1,9 @@
 import sys
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List
+import hashlib
+import shutil
+import logging
 
 import appdirs
 
@@ -10,7 +13,10 @@ if TYPE_CHECKING:
     from . import Config
 
 __all__ = ['load_basic_configuration', 'cog_data_path', 'core_data_path',
-           'storage_details', 'storage_type']
+           'load_bundled_data', 'bundled_data_path', 'storage_details',
+           'storage_type']
+
+log = logging.getLogger("red.data_manager")
 
 jsonio = None
 basic_config = None
@@ -108,6 +114,151 @@ def core_data_path() -> Path:
     return core_path.resolve()
 
 
+def _find_data_files(init_location: str) -> (Path, List[Path]):
+    """
+    Discovers all files in the bundled data folder of an installed cog.
+
+    Parameters
+    ----------
+    init_location
+
+    Returns
+    -------
+    (pathlib.Path, list of pathlib.Path)
+    """
+    init_file = Path(init_location)
+    if not init_file.is_file():
+        return []
+
+    package_folder = init_file.parent.resolve() / 'data'
+    if not package_folder.is_dir():
+        return []
+
+    all_files = list(package_folder.rglob("*"))
+
+    return package_folder, [p.resolve()
+                            for p in all_files
+                            if p.is_file()]
+
+
+def _compare_and_copy(to_copy: List[Path], bundled_data_dir: Path, cog_data_dir: Path):
+    """
+    Filters out files from ``to_copy`` that already exist, and are the
+    same, in ``data_dir``. The files that are different are copied into
+    ``data_dir``.
+
+    Parameters
+    ----------
+    to_copy : list of pathlib.Path
+    bundled_data_dir : pathlib.Path
+    cog_data_dir : pathlib.Path
+    """
+
+    def hash_bytestr_iter(bytesiter, hasher, ashexstr=False):
+        for block in bytesiter:
+            hasher.update(block)
+        return hasher.hexdigest() if ashexstr else hasher.digest()
+
+    def file_as_blockiter(afile, blocksize=65536):
+        with afile:
+            block = afile.read(blocksize)
+            while len(block) > 0:
+                yield block
+                block = afile.read(blocksize)
+
+    lookup = {p: cog_data_dir.joinpath(p.relative_to(bundled_data_dir))
+              for p in to_copy}
+
+    for orig, poss_existing in lookup.items():
+        if not poss_existing.is_file():
+            poss_existing.parent.mkdir(exist_ok=True, parents=True)
+            exists_checksum = None
+        else:
+            exists_checksum = hash_bytestr_iter(file_as_blockiter(
+                poss_existing.open('rb')), hashlib.sha256())
+
+        orig_checksum = ...
+        if exists_checksum is not None:
+            orig_checksum = hash_bytestr_iter(file_as_blockiter(
+                orig.open('rb')), hashlib.sha256())
+
+        if exists_checksum != orig_checksum:
+            shutil.copy(str(orig), str(poss_existing))
+            log.debug("Copying {} to {}".format(
+                orig, poss_existing
+            ))
+
+
+def load_bundled_data(cog_instance, init_location: str):
+    """
+    This function copies (and overwrites) data from the ``data/`` folder
+    of the installed cog.
+
+    .. important::
+
+        This function MUST be called from the ``setup()`` function of your
+        cog.
+
+    Examples
+    --------
+    >>> from redbot.core import data_manager
+    >>>
+    >>> def setup(bot):
+    >>>     cog = MyCog()
+    >>>     data_manager.load_bundled_data(cog, __file__)
+    >>>     bot.add_cog(cog)
+
+    Parameters
+    ----------
+    cog_instance
+        An instance of your cog class.
+    init_location : str
+        The ``__file__`` attribute of the file where your ``setup()``
+        function exists.
+    """
+    bundled_data_folder, to_copy = _find_data_files(init_location)
+
+    cog_data_folder = cog_data_path(cog_instance) / 'bundled_data'
+
+    _compare_and_copy(to_copy, bundled_data_folder, cog_data_folder)
+
+
+def bundled_data_path(cog_instance) -> Path:
+    """
+    The "data" directory that has been copied from installed cogs.
+
+    .. important::
+
+        You should *NEVER* write to this directory. Data manager will
+        overwrite files in this directory each time `load_bundled_data`
+        is called. You should instead write to the directory provided by
+        `cog_data_path`.
+
+    Parameters
+    ----------
+    cog_instance
+
+    Returns
+    -------
+    pathlib.Path
+        Path object to the bundled data folder.
+
+    Raises
+    ------
+    FileNotFoundError
+        If no bundled data folder exists or if it hasn't been loaded yet.
+    """
+
+    bundled_path = cog_data_path(cog_instance) / 'bundled_data'
+
+    if not bundled_path.is_dir():
+        raise FileNotFoundError("No such directory {}".format(
+            bundled_path
+        ))
+
+    return bundled_path
+
+
 def storage_type() -> str:
     """Gets the storage type as a string.
 

redbot/core/events.py

@@ -11,7 +11,6 @@ from discord.ext import commands
 
 from .data_manager import storage_type
 from .utils.chat_formatting import inline, bordered
-from .core_commands import find_spec
 from colorama import Fore, Style
 
 log = logging.getLogger("red")
@@ -48,7 +47,7 @@ def init_events(bot, cli_flags):
 
             for package in packages:
                 try:
-                    spec = await find_spec(bot, package)
+                    spec = await bot.cog_mgr.find_cog(package)
                     bot.load_extension(spec)
                 except Exception as e:
                     log.exception("Failed to load package {}".format(package),