diff --git a/redbot/core/__init__.py b/redbot/core/__init__.py index 7060c1627..663ac1939 100644 --- a/redbot/core/__init__.py +++ b/redbot/core/__init__.py @@ -5,4 +5,7 @@ from .context import RedContext __all__ = ["Config", "RedContext", "__version__"] -__version__ = version = pkg_resources.require("Red-DiscordBot")[0].version +try: + __version__ = pkg_resources.require("Red-DiscordBot")[0].version +except pkg_resources.DistributionNotFound: + __version__ = "3.0.0" diff --git a/redbot/core/cog_manager.py b/redbot/core/cog_manager.py index 5bc80741e..5fb98d04a 100644 --- a/redbot/core/cog_manager.py +++ b/redbot/core/cog_manager.py @@ -1,8 +1,9 @@ +import contextlib import pkgutil -from importlib import invalidate_caches +from importlib import import_module, invalidate_caches from importlib.machinery import ModuleSpec from pathlib import Path -from typing import Tuple, Union, List +from typing import Tuple, Union, List, overload import redbot.cogs @@ -187,6 +188,61 @@ class CogManager: str_paths = [str(p) for p in paths_] await self.conf.paths.set(str_paths) + async def _find_ext_cog(self, name: str) -> ModuleSpec: + """ + Attempts to find a spec for a third party installed cog. + + Parameters + ---------- + name : str + Name of the cog package to look for. + + Returns + ------- + importlib.machinery.ModuleSpec + Module spec to be used for cog loading. + + Raises + ------ + RuntimeError + When no matching spec can be found. + """ + resolved_paths = [str(p.resolve()) for p in await self.paths()] + for finder, module_name, _ in pkgutil.iter_modules(resolved_paths): + if name == module_name: + spec = finder.find_spec(name) + if spec: + return spec + + raise RuntimeError("No 3rd party module by the name of '{}' was found" + " in any available path.".format(name)) + + async def _find_core_cog(self, name: str) -> ModuleSpec: + """ + Attempts to find a spec for a core cog. + + Parameters + ---------- + name : str + + Returns + ------- + importlib.machinery.ModuleSpec + + Raises + ------ + RuntimeError + When no matching spec can be found. + """ + real_name = ".{}".format(name) + try: + mod = import_module(real_name, package='redbot.cogs') + except ImportError as e: + raise RuntimeError("No core cog by the name of '{}' could" + "be found.".format(name)) from e + return mod.__spec__ + + # noinspection PyUnreachableCode async def find_cog(self, name: str) -> ModuleSpec: """Find a cog in the list of available paths. @@ -206,15 +262,13 @@ class CogManager: If there is no cog with the given name. """ - resolved_paths = [str(p.resolve()) for p in await self.paths()] - for finder, module_name, _ in pkgutil.iter_modules(resolved_paths): - if name == module_name: - spec = finder.find_spec(name) - if spec: - return spec + with contextlib.suppress(RuntimeError): + return await self._find_ext_cog(name) - raise RuntimeError("No module by the name of '{}' was found" - " in any available path.".format(name)) + with contextlib.suppress(RuntimeError): + return await self._find_core_cog(name) + + raise RuntimeError("No cog with that name could be found.") async def available_modules(self) -> List[str]: """Finds the names of all available modules to load. diff --git a/redbot/core/core_commands.py b/redbot/core/core_commands.py index 8c31d048a..496c3b92b 100644 --- a/redbot/core/core_commands.py +++ b/redbot/core/core_commands.py @@ -14,9 +14,7 @@ from discord.ext import commands from redbot.core import checks from redbot.core import i18n -import redbot.cogs # Don't remove this line or core cogs won't load - -__all__ = ["find_spec", "Core"] +__all__ = ["Core"] log = logging.getLogger("red") @@ -26,20 +24,6 @@ OWNER_DISCLAIMER = ("⚠ **Only** the person who is hosting Red should be " "system.** ⚠") -async def find_spec(bot, cog_name: str): - try: - spec = await bot.cog_mgr.find_cog(cog_name) - except RuntimeError: - real_name = ".{}".format(cog_name) - try: - mod = importlib.import_module(real_name, package='redbot.cogs') - except ImportError: - spec = None - else: - spec = mod.__spec__ - return spec - - _ = i18n.CogI18n("Core", __file__) @@ -50,8 +34,9 @@ class Core: @checks.is_owner() async def load(self, ctx, *, cog_name: str): """Loads a package""" - spec = await find_spec(ctx.bot, cog_name) - if spec is None: + try: + spec = await ctx.bot.cog_mgr.find_cog(cog_name) + except RuntimeError: await ctx.send(_("No module by that name was found in any" " cog path.")) return @@ -83,7 +68,7 @@ class Core: """Reloads a package""" ctx.bot.unload_extension(cog_name) - spec = await find_spec(ctx.bot, cog_name) + spec = await ctx.bot.cog_mgr.find_cog(cog_name) if spec is None: await ctx.send(_("No module by that name was found in any" " cog path.")) diff --git a/redbot/core/data_manager.py b/redbot/core/data_manager.py index 855b0adac..b8b2c5386 100644 --- a/redbot/core/data_manager.py +++ b/redbot/core/data_manager.py @@ -1,6 +1,9 @@ import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List +import hashlib +import shutil +import logging import appdirs @@ -10,7 +13,10 @@ if TYPE_CHECKING: from . import Config __all__ = ['load_basic_configuration', 'cog_data_path', 'core_data_path', - 'storage_details', 'storage_type'] + 'load_bundled_data', 'bundled_data_path', 'storage_details', + 'storage_type'] + +log = logging.getLogger("red.data_manager") jsonio = None basic_config = None @@ -108,6 +114,151 @@ def core_data_path() -> Path: return core_path.resolve() +def _find_data_files(init_location: str) -> (Path, List[Path]): + """ + Discovers all files in the bundled data folder of an installed cog. + + Parameters + ---------- + init_location + + Returns + ------- + (pathlib.Path, list of pathlib.Path) + """ + init_file = Path(init_location) + if not init_file.is_file(): + return [] + + package_folder = init_file.parent.resolve() / 'data' + if not package_folder.is_dir(): + return [] + + all_files = list(package_folder.rglob("*")) + + return package_folder, [p.resolve() + for p in all_files + if p.is_file()] + + +def _compare_and_copy(to_copy: List[Path], bundled_data_dir: Path, cog_data_dir: Path): + """ + Filters out files from ``to_copy`` that already exist, and are the + same, in ``data_dir``. The files that are different are copied into + ``data_dir``. + + Parameters + ---------- + to_copy : list of pathlib.Path + bundled_data_dir : pathlib.Path + cog_data_dir : pathlib.Path + """ + + def hash_bytestr_iter(bytesiter, hasher, ashexstr=False): + for block in bytesiter: + hasher.update(block) + return hasher.hexdigest() if ashexstr else hasher.digest() + + def file_as_blockiter(afile, blocksize=65536): + with afile: + block = afile.read(blocksize) + while len(block) > 0: + yield block + block = afile.read(blocksize) + + lookup = {p: cog_data_dir.joinpath(p.relative_to(bundled_data_dir)) + for p in to_copy} + + for orig, poss_existing in lookup.items(): + if not poss_existing.is_file(): + poss_existing.parent.mkdir(exist_ok=True, parents=True) + exists_checksum = None + else: + exists_checksum = hash_bytestr_iter(file_as_blockiter( + poss_existing.open('rb')), hashlib.sha256()) + + orig_checksum = ... + if exists_checksum is not None: + orig_checksum = hash_bytestr_iter(file_as_blockiter( + orig.open('rb')), hashlib.sha256()) + + if exists_checksum != orig_checksum: + shutil.copy(str(orig), str(poss_existing)) + log.debug("Copying {} to {}".format( + orig, poss_existing + )) + + +def load_bundled_data(cog_instance, init_location: str): + """ + This function copies (and overwrites) data from the ``data/`` folder + of the installed cog. + + .. important:: + + This function MUST be called from the ``setup()`` function of your + cog. + + Examples + -------- + >>> from redbot.core import data_manager + >>> + >>> def setup(bot): + >>> cog = MyCog() + >>> data_manager.load_bundled_data(cog, __file__) + >>> bot.add_cog(cog) + + Parameters + ---------- + cog_instance + An instance of your cog class. + init_location : str + The ``__file__`` attribute of the file where your ``setup()`` + function exists. + """ + bundled_data_folder, to_copy = _find_data_files(init_location) + + cog_data_folder = cog_data_path(cog_instance) / 'bundled_data' + + _compare_and_copy(to_copy, bundled_data_folder, cog_data_folder) + + +def bundled_data_path(cog_instance) -> Path: + """ + The "data" directory that has been copied from installed cogs. + + .. important:: + + You should *NEVER* write to this directory. Data manager will + overwrite files in this directory each time `load_bundled_data` + is called. You should instead write to the directory provided by + `cog_data_path`. + + Parameters + ---------- + cog_instance + + Returns + ------- + pathlib.Path + Path object to the bundled data folder. + + Raises + ------ + FileNotFoundError + If no bundled data folder exists or if it hasn't been loaded yet. + """ + + bundled_path = cog_data_path(cog_instance) / 'bundled_data' + + if not bundled_path.is_dir(): + raise FileNotFoundError("No such directory {}".format( + bundled_path + )) + + return bundled_path + + def storage_type() -> str: """Gets the storage type as a string. diff --git a/redbot/core/events.py b/redbot/core/events.py index 49a1d65f4..5448c9097 100644 --- a/redbot/core/events.py +++ b/redbot/core/events.py @@ -11,7 +11,6 @@ from discord.ext import commands from .data_manager import storage_type from .utils.chat_formatting import inline, bordered -from .core_commands import find_spec from colorama import Fore, Style log = logging.getLogger("red") @@ -48,7 +47,7 @@ def init_events(bot, cli_flags): for package in packages: try: - spec = await find_spec(bot, package) + spec = await bot.cog_mgr.find_cog(package) bot.load_extension(spec) except Exception as e: log.exception("Failed to load package {}".format(package),