diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ad3f5bf23..aceaac94b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,6 +21,7 @@ redbot/core/rpc.py @tekulvw redbot/core/sentry_setup.py @Kowlin @tekulvw redbot/core/utils/chat_formatting.py @tekulvw redbot/core/utils/mod.py @palmtree5 +redbot/core/utils/data_converter.py @mikeshardmind # Cogs redbot/cogs/admin/* @tekulvw @@ -38,6 +39,7 @@ redbot/cogs/mod/* @palmtree5 redbot/cogs/modlog/* @palmtree5 redbot/cogs/streams/* @Twentysix26 @palmtree5 redbot/cogs/trivia/* @Tobotimus +redbot/cogs/dataconverter/* @mikeshardmind # Docs docs/* @tekulvw @palmtree5 diff --git a/docs/cog_dataconverter.rst b/docs/cog_dataconverter.rst new file mode 100644 index 000000000..4f5e23b4f --- /dev/null +++ b/docs/cog_dataconverter.rst @@ -0,0 +1,62 @@ +.. Importing data from a V2 install + +================================ +Importing data from a V2 install +================================ + +---------------- +What you'll need +---------------- + +1. A Running V3 bot +2. The path where your V2 bot is installed + +-------------- +Importing data +-------------- + +.. important:: + + Unless otherwise specified, the V2 data will take priority over V3 data for the same entires + +.. important:: + + For the purposes of this guide, your prefix will be denoted as + [p] + + You should swap whatever you made your prefix in for this. + All of the below are commands to be entered in discord where the bot can + see them. + +The dataconverter cog is not loaded by default. To start, load it with + +.. code-block:: none + + [p]load dataconverter + +Next, you'll need to give it the path where your V2 install is. + +On linux and OSX, it may look something like: + +.. code-block:: none + + /home/username/Red-DiscordBot/ + +On Windows it will look something like: + +.. code-block:: none + + C:\Users\yourusername\Red-DiscordBot + +Once you have that path, give it to the bot with the following command +(make sure to swap your own path in) + +.. code-block:: none + + [p]convertdata /home/username/Red-DiscordBot/ + + +From here, if the path is correct, you will be prompted with an interactive menu asking you +what data you would like to import + +You can select an entry by number, or quit with any of 'quit', 'exit', 'q', '-1', or 'cancel' diff --git a/docs/framework_utils.rst b/docs/framework_utils.rst index 55c13e55c..3ff1f4e17 100644 --- a/docs/framework_utils.rst +++ b/docs/framework_utils.rst @@ -21,3 +21,9 @@ Mod Helpers .. automodule:: redbot.core.utils.mod :members: + +V2 Data Conversion +================== + +.. automodule:: redbot.core.utils.data_converter + :members: DataConverter \ No newline at end of file diff --git a/docs/guide_data_conversion.rst b/docs/guide_data_conversion.rst new file mode 100644 index 000000000..8c7254af1 --- /dev/null +++ b/docs/guide_data_conversion.rst @@ -0,0 +1,154 @@ +.. Converting Data from a V2 cog + +.. role:: python(code) + :language: python + +============================ +Importing Data From a V2 Cog +============================ + +This guide serves as a tutorial on using the DataConverter class +to import settings from a V2 cog. + +------------------ +Things you'll need +------------------ + +1. The path where each file holding related settings in v2 is +2. A conversion function to take the data and transform it to conform to Config + +----------------------- +Getting your file paths +----------------------- + +You should probably not try to find the files manually. +Asking the user for the base install path and using a relative path to where the +data should be, then testing that the file exists there is safer. This is especially +True if your cog has multiple settings files + +Example + +.. code-block:: python + + from discord.ext import commands + from pathlib import Path + + @commands.command(name="filefinder") + async def file_finding_command(self, ctx, filepath): + """ + this finds a file based on a user provided input and a known relative path + """ + + base_path = Path(filepath) + fp = base_path / 'data' / 'mycog' / 'settings.json' + if not fp.is_file(): + pass + # fail, prompting user + else: + pass + # do something with the file + +--------------- +Converting data +--------------- + +Once you've gotten your v2 settings file, you'll want to be able to import it +There are a couple options available depending on how you would like to convert +the data. + +The first one takes a data path, and a conversion function and does the rest for you. +This is great for simple data that just needs to quickly be imported without much +modification. + + +Here's an example of that in use: + +.. code-block:: python + + from pathlib import Path + from discord.ext import commands + + from redbot.core.utils.data_converter import DataConverter as dc + from redbot.core.config import Config + + ... + + + async def import_v2(self, file_path: Path): + """ + to be called from a command limited to owner + + This should be a coroutine as the convert function will + need to be awaited + """ + + # First we give the converter out cog's Config instance. + converter = dc(self.config) + + # next we design a way to get all of the data into Config's internal + # format. This should be a generator, but you can also return a single + # list with identical results outside of memory usage + def conversion_spec(v2data): + for guild_id in v2.data.keys(): + yield {(Config.GUILD, guild_id): {('blacklisted',): True}} + # This is yielding a dictionary that is designed for config's set_raw. + # The keys should be a tuple of Config scopes + the needed Identifiers. The + # values should be another dictionary whose keys are tuples representing + # config settings, the value should be the value to set for that. + + # Then we pass the file and the conversion function + await converter.convert(file_path, conversion_spec) + # From here, our data should be imported + + +You can also choose to convert all of your data and pass it as a single dict +This can be useful if you want finer control over the dataconversion or want to +preserve any data from v3 that may share the same entry and set it aside to prompt +a user + +.. code-block:: python + + from pathlib import Path + from discord.ext import commands + + from redbot.core.utils.data_converter import DataConverter as dc + from redbot.core.config import Config + + ... + + await dc(config_instance).dict_import(some_processed_dict) + + +The format of the items of the dict is the same as in the above example + + +----------------------------------- +Config Scopes and their Identifiers +----------------------------------- + +This section is provided as a quick reference for the identifiers for default +scopes available in Config. This does not cover usage of custom scopes, though the +data converter is compatible with those as well. + +Global:: + :code:`(Config.GLOBAL,)` +Guild:: + :code:`(Config.GUILD, guild_id)` +Channel:: + :code:`(Config.CHANNEL, channel_id)` +User:: + :code:`(Config.USER, user_id)` +Member:: + :code:`(Config.MEMBER, guild_id, user_id)` +Role:: + :code:`(Config.ROLE, role_id)` + + +----------------------------- +More information and Examples +----------------------------- + +For a more in depth look at how all of these commands function +You may want to take a look at how core data is being imported + +:code:`redbot/cogs/dataconverter/core_specs.py` \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 65d4f835b..256dad0c2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ Welcome to Red - Discord Bot's documentation! install_debian install_centos install_raspbian + cog_dataconverter .. toctree:: :maxdepth: 2 @@ -29,6 +30,7 @@ Welcome to Red - Discord Bot's documentation! guide_migration guide_cog_creation + guide_data_conversion framework_bank framework_bot framework_cogmanager diff --git a/redbot/cogs/dataconverter/__init__.py b/redbot/cogs/dataconverter/__init__.py new file mode 100644 index 000000000..24b5d0f3a --- /dev/null +++ b/redbot/cogs/dataconverter/__init__.py @@ -0,0 +1,6 @@ +from redbot.core.bot import Red +from .dataconverter import DataConverter + + +def setup(bot: Red): + bot.add_cog(DataConverter(bot)) diff --git a/redbot/cogs/dataconverter/core_specs.py b/redbot/cogs/dataconverter/core_specs.py new file mode 100644 index 000000000..123664922 --- /dev/null +++ b/redbot/cogs/dataconverter/core_specs.py @@ -0,0 +1,184 @@ +from itertools import chain, starmap +from pathlib import Path +from datetime import datetime + +from redbot.core.bot import Red +from redbot.core.utils.data_converter import DataConverter as dc +from redbot.core.config import Config + + +class SpecResolver(object): + """ + Resolves Certain things for DataConverter + """ + + def __init__(self, path: Path): + self.v2path = path + self.resolved = set() + self.available_core_conversions = { + 'Bank Accounts': { + 'cfg': ('Bank', None, 384734293238749), + 'file': self.v2path / 'data' / 'economy' / 'bank.json', + 'converter': self.bank_accounts_conv_spec + }, + 'Economy Settings': { + 'cfg': ('Economy', 'config', 1256844281), + 'file': self.v2path / 'data' / 'economy' / 'settings.json', + 'converter': self.economy_conv_spec + }, + 'Mod Log Cases': { + 'cfg': ('ModLog', None, 1354799444), + 'file': self.v2path / 'data' / 'mod' / 'modlog.json', + 'converter': None # prevents from showing as available + }, + 'Filter': { + 'cfg': ('Filter', 'settings', 4766951341), + 'file': self.v2path / 'data' / 'mod' / 'filter.json', + 'converter': self.filter_conv_spec + }, + 'Past Names': { + 'cfg': ('Mod', 'settings', 4961522000), + 'file': self.v2path / 'data' / 'mod' / 'past_names.json', + 'converter': self.past_names_conv_spec + }, + 'Past Nicknames': { + 'cfg': ('Mod', 'settings', 4961522000), + 'file': self.v2path / 'data' / 'mod' / 'past_nicknames.json', + 'converter': self.past_nicknames_conv_spec + }, + 'Custom Commands': { + 'cfg': ('CustomCommands', 'config', 414589031223512), + 'file': self.v2path / 'data' / 'customcom' / 'commands.json', + 'converter': self.customcom_conv_spec + } + } + + @property + def available(self): + return sorted( + k for k, v in self.available_core_conversions.items() + if v['file'].is_file() and v['converter'] is not None + and k not in self.resolved + ) + + def unpack(self, parent_key, parent_value): + """Unpack one level of nesting in a dictionary""" + try: + items = parent_value.items() + except AttributeError: + yield (parent_key, parent_value) + else: + for key, value in items: + yield (parent_key + (key,), value) + + def flatten_dict(self, dictionary: dict): + """Flatten a nested dictionary structure""" + dictionary = {(key,): value for key, value in dictionary.items()} + while True: + dictionary = dict( + chain.from_iterable( + starmap(self.unpack, dictionary.items()) + ) + ) + if not any( + isinstance(value, dict) + for value in dictionary.values() + ): + break + return dictionary + + def apply_scope(self, scope: str, data: dict): + return {(scope,) + k: v for k, v in data.items()} + + def bank_accounts_conv_spec(self, data: dict): + flatscoped = self.apply_scope(Config.MEMBER, self.flatten_dict(data)) + ret = {} + for k, v in flatscoped.items(): + outerkey, innerkey = tuple(k[:-1]), (k[-1],) + if outerkey not in ret: + ret[outerkey] = {} + if innerkey[0] == 'created_at': + x = int( + datetime.strptime( + v, "%Y-%m-%d %H:%M:%S").timestamp() + ) + ret[outerkey].update({innerkey: x}) + else: + ret[outerkey].update({innerkey: v}) + return ret + + def economy_conv_spec(self, data: dict): + flatscoped = self.apply_scope(Config.GUILD, self.flatten_dict(data)) + ret = {} + for k, v in flatscoped.items(): + outerkey, innerkey = (*k[:-1],), (k[-1],) + if outerkey not in ret: + ret[outerkey] = {} + ret[outerkey].update({innerkey: v}) + return ret + + def mod_log_cases(self, data: dict): + raise NotImplementedError("This one isn't ready yet") + + def filter_conv_spec(self, data: dict): + return { + (Config.GUILD, k): {('filter',): v} + for k, v in data.items() + } + + def past_names_conv_spec(self, data: dict): + return { + (Config.USER, k): {('past_names',): v} + for k, v in data.items() + } + + def past_nicknames_conv_spec(self, data: dict): + flatscoped = self.apply_scope(Config.MEMBER, self.flatten_dict(data)) + ret = {} + for k, v in flatscoped.items(): + outerkey, innerkey = (*k[:-1],), (k[-1],) + if outerkey not in ret: + ret[outerkey] = {} + ret[outerkey].update({innerkey: v}) + return ret + + def customcom_conv_spec(self, data: dict): + flatscoped = self.apply_scope(Config.GUILD, self.flatten_dict(data)) + ret = {} + for k, v in flatscoped.items(): + outerkey, innerkey = (*k[:-1],), ('commands', k[-1]) + if outerkey not in ret: + ret[outerkey] = {} + + ccinfo = { + 'author': { + 'id': 42, + 'name': 'Converted from a v2 instance' + }, + 'command': k[-1], + 'created_at': '{:%d/%m/%Y %H:%M:%S}'.format(datetime.utcnow()), + 'editors': [], + 'response': v + } + ret[outerkey].update({innerkey: ccinfo}) + return ret + + async def convert(self, bot: Red, prettyname: str): + if prettyname not in self.available: + raise NotImplementedError("No Conversion Specs for this") + + info = self.available_core_conversions[prettyname] + filepath, converter = info['file'], info['converter'] + (cogname, attr, _id) = info['cfg'] + try: + config = getattr(bot.get_cog(cogname), attr) + except (TypeError, AttributeError): + config = Config.get_conf(cogname, _id) + + try: + items = converter(dc.json_load(filepath)) + await dc(config).dict_import(items) + except Exception: + raise + else: + self.resolved.add(prettyname) diff --git a/redbot/cogs/dataconverter/dataconverter.py b/redbot/cogs/dataconverter/dataconverter.py new file mode 100644 index 000000000..37de1d9c7 --- /dev/null +++ b/redbot/cogs/dataconverter/dataconverter.py @@ -0,0 +1,82 @@ +from pathlib import Path +import asyncio + +from discord.ext import commands + +from redbot.core import checks, RedContext +from redbot.core.bot import Red +from redbot.core.i18n import CogI18n +from redbot.cogs.dataconverter.core_specs import SpecResolver +from redbot.core.utils.chat_formatting import box + +_ = CogI18n('DataConverter', __file__) + + +class DataConverter: + """ + Cog for importing Red v2 Data + """ + + def __init__(self, bot: Red): + self.bot = bot + + @checks.is_owner() + @commands.command(name="convertdata") + async def dataconversioncommand(self, ctx: RedContext, v2path: str): + """ + Interactive prompt for importing data from Red v2 + + Takes the path where the v2 install is + + Overwrites values which have entries in both v2 and v3, + use with caution. + """ + resolver = SpecResolver(Path(v2path.strip())) + + if not resolver.available: + return await ctx.send( + _("There don't seem to be any data files I know how to " + "handle here. Are you sure you gave me the base " + "installation path?") + ) + while resolver.available: + menu = _("Please select a set of data to import by number" + ", or 'exit' to exit") + for index, entry in enumerate(resolver.available, 1): + menu += "\n{}. {}".format(index, entry) + + menu_message = await ctx.send(box(menu)) + + def pred(m): + return m.channel == ctx.channel and m.author == ctx.author + + try: + message = await self.bot.wait_for( + 'message', check=pred, timeout=60 + ) + except asyncio.TimeoutError: + return await ctx.send( + _('Try this again when you are more ready')) + else: + if message.content.strip().lower() in [ + 'quit', 'exit', '-1', 'q', 'cancel' + ]: + return await ctx.tick() + try: + message = int(message.content.strip()) + to_conv = resolver.available[message - 1] + except (ValueError, IndexError): + await ctx.send( + _("That wasn't a valid choice.") + ) + continue + else: + async with ctx.typing(): + await resolver.convert(self.bot, to_conv) + await ctx.send(_("{} converted.").format(to_conv)) + await menu_message.delete() + else: + return await ctx.send( + _("There isn't anything else I know how to convert here." + "\nThere might be more things I can convert in the future.") + ) diff --git a/redbot/cogs/dataconverter/locales/messages.pot b/redbot/cogs/dataconverter/locales/messages.pot new file mode 100644 index 000000000..e81f6ee58 --- /dev/null +++ b/redbot/cogs/dataconverter/locales/messages.pot @@ -0,0 +1,43 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2018-03-12 04:35+EDT\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: ENCODING\n" +"Generated-By: pygettext.py 1.5\n" + + +#: ../dataconverter.py:38 +msgid "There don't seem to be any data files I know how to handle here. Are you sure you gave me the base installation path?" +msgstr "" + +#: ../dataconverter.py:43 +msgid "Please select a set of data to import by number, or 'exit' to exit" +msgstr "" + +#: ../dataconverter.py:59 +msgid "Try this again when you are more ready" +msgstr "" + +#: ../dataconverter.py:70 +msgid "That wasn't a valid choice." +msgstr "" + +#: ../dataconverter.py:76 +msgid "{} converted." +msgstr "" + +#: ../dataconverter.py:80 +msgid "" +"There isn't anything else I know how to convert here.\n" +"There might be more things I can convert in the future." +msgstr "" + diff --git a/redbot/cogs/dataconverter/locales/regen_messages.py b/redbot/cogs/dataconverter/locales/regen_messages.py new file mode 100644 index 000000000..ecca5345b --- /dev/null +++ b/redbot/cogs/dataconverter/locales/regen_messages.py @@ -0,0 +1,15 @@ +import subprocess + +TO_TRANSLATE = [ + '../dataconverter.py' +] + + +def regen_messages(): + subprocess.run( + ['pygettext', '-n'] + TO_TRANSLATE + ) + + +if __name__ == "__main__": + regen_messages() diff --git a/redbot/core/utils/data_converter.py b/redbot/core/utils/data_converter.py new file mode 100644 index 000000000..bc4029827 --- /dev/null +++ b/redbot/core/utils/data_converter.py @@ -0,0 +1,134 @@ +import json +from pathlib import Path +from redbot.core import Config + + +class DataConverter: + """ + Class for moving v2 data to v3 + """ + + def __init__(self, config_instance: Config): + self.config = config_instance + + @staticmethod + def json_load(file_path: Path): + """Utility function for quickly grabbing data from a JSON file + + Parameters + ---------- + file_path: `pathlib.Path` + The path to the file to grabdata from + + Raises + ------ + FileNotFoundError + The file doesn't exist + json.JsonDecodeError + The file isn't valid JSON + """ + try: + with open(file_path, mode='r') as f: + data = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + raise + else: + return data + + async def convert(self, file_path: Path, conversion_spec: object): + """Converts v2 data to v3 format. If your v2 data uses multiple files + you will need to call this for each file. + + Parameters + ---------- + file_path : `pathlib.Path` + This should be the path to a JSON settings file from v2 + conversion_spec : `object` + This should be a function which takes a single argument argument + (the loaded JSON) and from it either + returns or yields one or more `dict` + whose items are in the form: + + .. code-block:: python + + {(SCOPE, *IDENTIFIERS): {(key_tuple): value}} + + an example of a possible entry of that dict: + + .. code-block:: python + + {(Config.MEMBER, '133049272517001216', '78631113035100160'): + {('balance',): 9001}} + + + This allows for any amount of entries at each level + in each of the nested dictionaries returned by conversion_spec + but the nesting cannot be different to this and still get the + expected results + see documentation for Config for more details on scopes + and the identifiers they need + + Returns + ------- + None + + Raises + ------ + FileNotFoundError + No such file at the specified path + json.JSONDecodeError + File is not valid JSON + AttributeError + Something goes wrong with your conversion and it provides + data in the wrong format + """ + + v2data = self.json_load(file_path) + + for entryset in conversion_spec(v2data): + for scope_id, values in entryset.items(): + base = self.config._get_base_group(*scope_id) + for inner_k, inner_v in values.items(): + await base.set_raw(*inner_k, value=inner_v) + + async def dict_import(self, entrydict: dict): + """This imports a dictionary in the correct format into Config + + Parameters + ---------- + entrydict : `dict` + This should be a dictionary of values to set. + This is provided as an alternative + to providing a file and conversion specification + the dictionary should be in the following format + + .. code-block:: python + + {(SCOPE, *IDENTIFIERS): {(key_tuple): value}}` + + an example of a possible entry of that dict: + + .. code-block:: python + + {(Config.MEMBER, '133049272517001216', '78631113035100160'): + {('balance',): 9001}} + + This allows for any amount of entries at each level + in each of the nested dictionaries returned by conversion_spec + but the nesting cannot be different to this and still get the + expected results + + Returns + ------- + None + + Raises + ------ + AttributeError + Data not in the correct format. + """ + + for scope_id, values in entrydict.items(): + base = self.config._get_base_group(*scope_id) + for inner_k, inner_v in values.items(): + await base.set_raw(*inner_k, value=inner_v)