From 7775b161994cd33f460b228395fb33954b62efb1 Mon Sep 17 00:00:00 2001 From: palmtree5 <3577255+palmtree5@users.noreply.github.com> Date: Sun, 27 May 2018 20:56:28 -0800 Subject: [PATCH] [V3] Optimize the backup command (#1666) * [V3 Core] Enhance [p]backup to exclude some files * Backup the repo list too * Lol Sinbad's pre-commit hook * Add option of sending the backup to the owner via DM * Drop an unnecessary config object in RepoManager * Move the backup functionality in redbot-setup to the new stuff * More work on implementation, including backing up the instance data --- redbot/cogs/downloader/downloader.py | 2 +- redbot/cogs/downloader/repo_manager.py | 3 +- redbot/core/core_commands.py | 58 +++++++++++++++++--- redbot/setup.py | 69 +++++++++++++++--------- tests/cogs/downloader/test_downloader.py | 5 +- 5 files changed, 99 insertions(+), 38 deletions(-) diff --git a/redbot/cogs/downloader/downloader.py b/redbot/cogs/downloader/downloader.py index 34f5d76e0..0527926cc 100644 --- a/redbot/cogs/downloader/downloader.py +++ b/redbot/cogs/downloader/downloader.py @@ -50,7 +50,7 @@ class Downloader: if str(self.LIB_PATH) not in syspath: syspath.insert(1, str(self.LIB_PATH)) - self._repo_manager = RepoManager(self.conf) + self._repo_manager = RepoManager() async def cog_install_path(self): """Get the current cog install path. diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index a617778f5..a686959e1 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -496,8 +496,7 @@ class Repo(RepoJSONMixin): class RepoManager: - def __init__(self, downloader_config: Config): - self.downloader_config = downloader_config + def __init__(self): self._repos = {} diff --git a/redbot/core/core_commands.py b/redbot/core/core_commands.py index 14d47eb96..a90bef4e9 100644 --- a/redbot/core/core_commands.py +++ b/redbot/core/core_commands.py @@ -2,6 +2,7 @@ import asyncio import datetime import importlib import itertools +import json import logging import os import sys @@ -951,7 +952,7 @@ class Core: @commands.command() @checks.is_owner() - async def backup(self, ctx): + async def backup(self, ctx, backup_path: str = None): """Creates a backup of all data for the instance.""" from redbot.core.data_manager import basic_config, instance_name from redbot.core.drivers.red_json import JSON @@ -979,14 +980,59 @@ class Core: instance_name, ctx.message.created_at.strftime("%Y-%m-%d %H-%M-%S") ) if data_dir.exists(): - home = data_dir.home() - backup_file = home / backup_filename - os.chdir(str(data_dir.parent)) + if not backup_path: + backup_pth = data_dir.home() + else: + backup_pth = Path(backup_path) + backup_file = backup_pth / backup_filename + + to_backup = [] + exclusions = [ + "__pycache__", + "Lavalink.jar", + os.path.join("Downloader", "lib"), + os.path.join("CogManager", "cogs"), + os.path.join("RepoManager", "repos"), + ] + downloader_cog = ctx.bot.get_cog("Downloader") + if downloader_cog and hasattr(downloader_cog, "_repo_manager"): + repo_output = [] + repo_mgr = downloader_cog._repo_manager + for n, repo in repo_mgr._repos: + repo_output.append( + {{"url": repo.url, "name": repo.name, "branch": repo.branch}} + ) + repo_filename = data_dir / "cogs" / "RepoManager" / "repos.json" + with open(str(repo_filename), "w") as f: + f.write(json.dumps(repo_output, indent=4)) + instance_data = {instance_name: basic_config} + instance_file = data_dir / "instance.json" + with open(str(instance_file), "w") as instance_out: + instance_out.write(json.dumps(instance_data, indent=4)) + for f in data_dir.glob("**/*"): + if not any(ex in str(f) for ex in exclusions): + to_backup.append(f) with tarfile.open(str(backup_file), "w:gz") as tar: - tar.add(data_dir.stem) + for f in to_backup: + tar.add(str(f), recursive=False) + print(str(backup_file)) await ctx.send( - _("A backup has been made of this instance. It is at {}.").format(backup_file) + _("A backup has been made of this instance. It is at {}.").format((backup_file)) ) + await ctx.send(_("Would you like to receive a copy via DM? (y/n)")) + + def same_author_check(m): + return m.author == ctx.author and m.channel == ctx.channel + + try: + msg = await ctx.bot.wait_for("message", check=same_author_check, timeout=60) + except asyncio.TimeoutError: + await ctx.send(_("Ok then.")) + else: + if msg.content.lower().strip() == "y": + await ctx.author.send( + _("Here's a copy of the backup"), file=discord.File(str(backup_file)) + ) else: await ctx.send(_("That directory doesn't seem to exist...")) diff --git a/redbot/setup.py b/redbot/setup.py index bd2729bde..58ea85879 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -12,7 +12,12 @@ from pathlib import Path import appdirs from redbot.core.cli import confirm -from redbot.core.data_manager import basic_config_default +from redbot.core.data_manager import ( + basic_config_default, + load_basic_configuration, + instance_name, + basic_config, +) from redbot.core.json_io import JsonIO from redbot.core.utils import safe_delete from redbot.core.drivers.red_json import JSON @@ -287,34 +292,46 @@ async def edit_instance(): async def create_backup(selected, instance_data): if confirm("Would you like to make a backup of the data for this instance? (y/n)"): + load_basic_configuration(selected) if instance_data["STORAGE_TYPE"] == "MongoDB": - print("Backing up the instance's data...") await mongo_to_json(instance_data["DATA_PATH"], instance_data["STORAGE_DETAILS"]) - backup_filename = "redv3-{}-{}.tar.gz".format( - selected, dt.utcnow().strftime("%Y-%m-%d %H-%M-%S") - ) - pth = Path(instance_data["DATA_PATH"]) - if pth.exists(): - home = pth.home() - backup_file = home / backup_filename - os.chdir(str(pth.parent)) - with tarfile.open(str(backup_file), "w:gz") as tar: - tar.add(pth.stem) - print("A backup of {} has been made. It is at {}".format(selected, backup_file)) + print("Backing up the instance's data...") + backup_filename = "redv3-{}-{}.tar.gz".format( + selected, dt.utcnow().strftime("%Y-%m-%d %H-%M-%S") + ) + pth = Path(instance_data["DATA_PATH"]) + if pth.exists(): + backup_pth = pth.home() + backup_file = backup_pth / backup_filename - else: - print("Backing up the instance's data...") - backup_filename = "redv3-{}-{}.tar.gz".format( - selected, dt.utcnow().strftime("%Y-%m-%d %H-%M-%S") - ) - pth = Path(instance_data["DATA_PATH"]) - if pth.exists(): - home = pth.home() - backup_file = home / backup_filename - os.chdir(str(pth.parent)) # str is used here because 3.5 support - with tarfile.open(str(backup_file), "w:gz") as tar: - tar.add(pth.stem) # add all files in that directory - print("A backup of {} has been made. It is at {}".format(selected, backup_file)) + to_backup = [] + exclusions = [ + "__pycache__", + "Lavalink.jar", + os.path.join("Downloader", "lib"), + os.path.join("CogManager", "cogs"), + os.path.join("RepoManager", "repos"), + ] + from redbot.cogs.downloader.repo_manager import RepoManager + + repo_mgr = RepoManager() + repo_output = [] + for _, repo in repo_mgr._repos: + repo_output.append({"url": repo.url, "name": repo.name, "branch": repo.branch}) + repo_filename = pth / "cogs" / "RepoManager" / "repos.json" + with open(str(repo_filename), "w") as f: + f.write(json.dumps(repo_output, indent=4)) + instance_data = {instance_name: basic_config} + instance_file = pth / "instance.json" + with open(str(instance_file), "w") as instance_out: + instance_out.write(json.dumps(instance_data, indent=4)) + for f in pth.glob("**/*"): + if not any(ex in str(f) for ex in exclusions): + to_backup.append(f) + with tarfile.open(str(backup_file), "w:gz") as tar: + for f in to_backup: + tar.add(str(f), recursive=False) + print("A backup of {} has been made. It is at {}".format(selected, backup_file)) async def remove_instance(selected, instance_data): diff --git a/tests/cogs/downloader/test_downloader.py b/tests/cogs/downloader/test_downloader.py index 6cb9ba245..2fc2ccf1e 100644 --- a/tests/cogs/downloader/test_downloader.py +++ b/tests/cogs/downloader/test_downloader.py @@ -33,9 +33,8 @@ def patch_relative_to(monkeysession): @pytest.fixture -def repo_manager(tmpdir_factory, config): - config.register_global(repos={}) - rm = RepoManager(config) +def repo_manager(tmpdir_factory): + rm = RepoManager() # rm.repos_folder = Path(str(tmpdir_factory.getbasetemp())) / 'repos' return rm