[Core V3] Make the bot data path configurable (#879)

* Initial commit

* Fix sentry

* Make cog manager install path work relative to the bot's dir

* Fix downloader to save data relative to the defined data folder

* Fix sentry test

* Fix downloader tests

* Change logfile location

* Add another line to codeowners

* Basic tests

* Fix versioning

* Add in FutureWarning for config file changes

* Add reference to issue

.github/CODEOWNERS

@@ -4,6 +4,7 @@
 # Core
 core/config.py              @tekulvw
 core/cog_manager.py         @tekulvw
+core/data_manager.py        @tekulvw
 core/drivers/*              @tekulvw
 core/sentry_setup.py        @Kowlin @tekulvw
 

cogs/downloader/repo_manager.py

@@ -13,6 +13,7 @@ import functools
 from discord.ext import commands
 
 from core import Config
+from core import data_manager
 from .errors import *
 from .installable import Installable, InstallableType
 from .log import log
@@ -443,13 +444,16 @@ class RepoManager:
     def __init__(self, downloader_config: Config):
         self.downloader_config = downloader_config
 
-        self.repos_folder = Path(__file__).parent / 'repos'
-
         self._repos = {}
 
         loop = asyncio.get_event_loop()
         loop.run_until_complete(self._load_repos(set=True))  # str_name: Repo
 
+    @property
+    def repos_folder(self) -> Path:
+        data_folder = data_manager.cog_data_path(self)
+        return data_folder / 'repos'
+
     def does_repo_exist(self, name: str) -> bool:
         return name in self._repos
 

core/__init__.py

@@ -1,6 +1,7 @@
 from core.config import Config
 from subprocess import run, PIPE
 from collections import namedtuple
+from main import determine_main_folder
 
 __all__ = ["Config", "__version__"]
 version_info = namedtuple("VersionInfo", "major minor patch")
@@ -9,10 +10,12 @@ BASE_VERSION = version_info(3, 0, 0)
 
 
 def get_latest_version():
+    main_folder = determine_main_folder()
     try:
         p = run(
             "git describe --abbrev=0 --tags".split(),
-            stdout=PIPE
+            stdout=PIPE,
+            cwd=str(main_folder)
         )
     except FileNotFoundError:
         # No git

core/cli.py

@@ -102,6 +102,9 @@ def parse_cli_flags():
     parser.add_argument("--dev",
                         action="store_true",
                         help="Enables developer mode")
+    parser.add_argument("config",
+                        nargs='?',
+                        help="Path to config generated on initial setup.")
 
     args = parser.parse_args()
 

core/cog_manager.py

@@ -270,6 +270,8 @@ class CogManagerUI:
         No installed cogs will be transferred in the process.
         """
         if path:
+            if not path.is_absolute():
+                path = (ctx.bot.main_dir / path).resolve()
             try:
                 await ctx.bot.cog_mgr.set_install_path(path)
             except ValueError:

core/config.py

@@ -8,6 +8,7 @@ from copy import deepcopy
 from pathlib import Path
 
 from .drivers.red_json import JSON as JSONDriver
+from core.data_manager import cog_data_path, core_data_path
 
 log = logging.getLogger("red.config")
 
@@ -359,6 +360,7 @@ class MemberGroup(Group):
         return guild_member.get(self.identifiers[-2], {})
 
 
+
 class Config:
     """
     You should always use :func:`get_conf` or :func:`get_core_conf` to initialize a Config object.
@@ -431,10 +433,11 @@ class Config:
         :return:
             A new config object.
         """
-        cog_name = cog_instance.__class__.__name__
+        cog_path_override = cog_data_path(cog_instance)
+        cog_name = cog_path_override.stem
         uuid = str(hash(identifier))
 
-        spawner = JSONDriver(cog_name)
+        spawner = JSONDriver(cog_name, data_path_override=cog_path_override)
         return cls(cog_name=cog_name, unique_identifier=uuid,
                    force_registration=force_registration,
                    driver_spawn=spawner)
@@ -451,8 +454,7 @@ class Config:
             See :py:attr:`force_registration`
         :type force_registration: Optional[bool]
         """
-        core_data_path = Path.cwd() / 'core' / '.data'
-        driver_spawn = JSONDriver("Core", data_path_override=core_data_path)
+        driver_spawn = JSONDriver("Core", data_path_override=core_data_path())
         return cls(cog_name="Core", driver_spawn=driver_spawn,
                    unique_identifier='0',
                    force_registration=force_registration)

core/data_manager.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+
+from core.json_io import JsonIO
+
+jsonio = None
+basic_config = None
+
+basic_config_default = {
+    "DATA_PATH": None,
+    "COG_PATH_APPEND": "cogs",
+    "CORE_PATH_APPEND": "core"
+}
+
+
+def load_basic_configuration(path: Path):
+    global jsonio
+    global basic_config
+
+    jsonio = JsonIO(path)
+    basic_config = jsonio._load_json()
+
+
+def _base_data_path() -> Path:
+    if basic_config is None:
+        raise RuntimeError("You must load the basic config before you"
+                           " can get the base data path.")
+    path = basic_config['DATA_PATH']
+    return Path(path).resolve()
+
+
+def cog_data_path(cog_instance=None) -> Path:
+    """
+    Gets the base cog data path. If you want to get the folder with
+        which to store your own cog's data please pass in an instance
+        of your cog class.
+    :param cog_instance:
+    :return:
+    """
+    try:
+        base_data_path = Path(_base_data_path())
+    except RuntimeError as e:
+        raise RuntimeError("You must load the basic config before you"
+                           " can get the cog data path.") from e
+    cog_path = base_data_path / basic_config['COG_PATH_APPEND']
+    if cog_instance:
+        cog_path = cog_path / cog_instance.__class__.__name__
+    cog_path.mkdir(exist_ok=True, parents=True)
+
+    return cog_path.resolve()
+
+
+def core_data_path() -> Path:
+    try:
+        base_data_path = Path(_base_data_path())
+    except RuntimeError as e:
+        raise RuntimeError("You must load the basic config before you"
+                           " can get the core data path.") from e
+    core_path = base_data_path / basic_config['CORE_PATH_APPEND']
+    core_path.mkdir(exist_ok=True, parents=True)
+
+    return core_path.resolve()

core/sentry_setup.py

@@ -27,12 +27,12 @@ include_paths = (
 client = None
 
 
-def init_sentry_logging(logger):
+def init_sentry_logging(bot, logger):
     global client
     client = Client(
         dsn=("https://27f3915ba0144725a53ea5a99c9ae6f3:87913fb5d0894251821dcf06e5e9cfe6@"
              "sentry.telemetry.red/19?verify_ssl=0"),
-        release=fetch_git_sha(str(Path.cwd()))
+        release=fetch_git_sha(str(bot.main_dir))
     )
 
     breadcrumbs.ignore_logger("websockets")

main.py

@@ -11,6 +11,7 @@ if discord.version_info.major < 1:
 
 from core.bot import Red, ExitCodes
 from core.cog_manager import CogManagerUI
+from core.data_manager import load_basic_configuration
 from core.global_checks import init_global_checks
 from core.events import init_events
 from core.sentry_setup import init_sentry_logging
@@ -22,6 +23,7 @@ import logging.handlers
 import logging
 import os
 from pathlib import Path
+from warnings import warn
 
 #
 #               Red - Discord Bot v3
@@ -56,8 +58,10 @@ def init_loggers(cli_flags):
     else:
         logger.setLevel(logging.WARNING)
 
+    from core.data_manager import core_data_path
+    logfile_path = core_data_path() / 'red.log'
     fhandler = logging.handlers.RotatingFileHandler(
-        filename='red.log', encoding='utf-8', mode='a',
+        filename=str(logfile_path), encoding='utf-8', mode='a',
         maxBytes=10**7, backupCount=5)
     fhandler.setFormatter(red_format)
 
@@ -88,6 +92,24 @@ async def _get_prefix_and_token(red, indict):
 
 if __name__ == '__main__':
     cli_flags = parse_cli_flags()
+
+    if cli_flags.config:
+        load_basic_configuration(Path(cli_flags.config).resolve())
+    else:
+        warn("Soon you will need to change the way you load the bot."
+             " The new method of loading has yet to be decided upon but"
+             " will be made clear in announcements from the support server"
+             " and from documentation. Please see issue #938 for further"
+             " discussion on this topic.",
+             category=FutureWarning)
+        import core.data_manager
+        defaults = core.data_manager.basic_config_default.copy()
+        defaults['DATA_PATH'] = str(determine_main_folder())
+        defaults['CORE_PATH_APPEND'] = 'core/.data'
+        defaults['COG_PATH_APPEND'] = 'cogs/.data'
+
+        core.data_manager.basic_config = defaults
+
     log, sentry_log = init_loggers(cli_flags)
     description = "Red v3 - Alpha"
     bot_dir = determine_main_folder()
@@ -126,7 +148,7 @@ if __name__ == '__main__':
     loop.run_until_complete(_get_prefix_and_token(red, tmp_data))
 
     if tmp_data['enable_sentry']:
-        init_sentry_logging(sentry_log)
+        init_sentry_logging(red, sentry_log)
 
     cleanup_tasks = True
 

tests/cogs/downloader/test_downloader.py

@@ -31,7 +31,7 @@ def patch_relative_to(monkeysession):
 def repo_manager(tmpdir_factory, config):
     config.register_global(repos={})
     rm = RepoManager(config)
-    rm.repos_folder = Path(str(tmpdir_factory.getbasetemp())) / 'repos'
+    # rm.repos_folder = Path(str(tmpdir_factory.getbasetemp())) / 'repos'
     return rm
 
 

tests/conftest.py

@@ -17,6 +17,13 @@ def monkeysession(request):
     mpatch.undo()
 
 
+@pytest.fixture(autouse=True)
+def override_data_path(tmpdir):
+    from core import data_manager
+    data_manager.basic_config = data_manager.basic_config_default
+    data_manager.basic_config['DATA_PATH'] = str(tmpdir)
+
+
 @pytest.fixture()
 def json_driver(tmpdir_factory):
     import uuid

tests/core/test_data_manager.py

@@ -0,0 +1,41 @@
+import json
+from pathlib import Path
+
+from core import data_manager
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def cleanup_datamanager():
+    data_manager.basic_config = None
+    data_manager.jsonio = None
+
+
+@pytest.fixture()
+def data_mgr_config(tmpdir):
+    default = data_manager.basic_config_default.copy()
+    default['BASE_DIR'] = str(tmpdir)
+    return default
+
+
+@pytest.fixture()
+def cog_instance():
+    thing = type('CogTest', (object, ), {})
+    return thing()
+
+
+def test_no_basic(cog_instance):
+    with pytest.raises(RuntimeError):
+        data_manager.core_data_path()
+
+    with pytest.raises(RuntimeError):
+        data_manager.cog_data_path(cog_instance)
+
+
+def test_core_path(data_mgr_config, tmpdir):
+    conf_path = tmpdir.join('config.json')
+    conf_path.write(json.dumps(data_mgr_config))
+
+    data_manager.load_basic_configuration(Path(str(conf_path)))
+
+    assert data_manager.core_data_path().parent == Path(data_mgr_config['BASE_DIR'])

tests/core/test_sentry.py

@@ -2,9 +2,9 @@ from core import sentry_setup
 import logging
 
 
-def test_sentry_capture():
+def test_sentry_capture(red):
     log = logging.getLogger(__name__)
-    sentry_setup.init_sentry_logging(log)
+    sentry_setup.init_sentry_logging(red, log)
 
     assert sentry_setup.client is not None