[V3] Data Converter (#1293)

* More docstrings

* still not ready...

* push this untested pile of code.

* working, menu needs cleaning up though, modlog converter not here yet

* menu cleanup

* add note about the fact that values are overwritten

* add i18n

* User friendlier quitting

* Better naming of a function

* setup automodule for dataconverter

* More documentation

* use Config.MEMBER (etc) instead of 'MEMBER' (etc)

redbot/cogs/dataconverter/__init__.py

@@ -0,0 +1,6 @@
+from redbot.core.bot import Red
+from .dataconverter import DataConverter
+
+
+def setup(bot: Red):
+    bot.add_cog(DataConverter(bot))

redbot/cogs/dataconverter/core_specs.py

@@ -0,0 +1,184 @@
+from itertools import chain, starmap
+from pathlib import Path
+from datetime import datetime
+
+from redbot.core.bot import Red
+from redbot.core.utils.data_converter import DataConverter as dc
+from redbot.core.config import Config
+
+
+class SpecResolver(object):
+    """
+    Resolves Certain things for DataConverter
+    """
+
+    def __init__(self, path: Path):
+        self.v2path = path
+        self.resolved = set()
+        self.available_core_conversions = {
+            'Bank Accounts': {
+                'cfg': ('Bank', None, 384734293238749),
+                'file': self.v2path / 'data' / 'economy' / 'bank.json',
+                'converter': self.bank_accounts_conv_spec
+            },
+            'Economy Settings': {
+                'cfg': ('Economy', 'config', 1256844281),
+                'file': self.v2path / 'data' / 'economy' / 'settings.json',
+                'converter': self.economy_conv_spec
+            },
+            'Mod Log Cases': {
+                'cfg': ('ModLog', None, 1354799444),
+                'file': self.v2path / 'data' / 'mod' / 'modlog.json',
+                'converter': None  # prevents from showing as available
+            },
+            'Filter': {
+                'cfg': ('Filter', 'settings', 4766951341),
+                'file': self.v2path / 'data' / 'mod' / 'filter.json',
+                'converter': self.filter_conv_spec
+            },
+            'Past Names': {
+                'cfg': ('Mod', 'settings', 4961522000),
+                'file': self.v2path / 'data' / 'mod' / 'past_names.json',
+                'converter': self.past_names_conv_spec
+            },
+            'Past Nicknames': {
+                'cfg': ('Mod', 'settings', 4961522000),
+                'file': self.v2path / 'data' / 'mod' / 'past_nicknames.json',
+                'converter': self.past_nicknames_conv_spec
+            },
+            'Custom Commands': {
+                'cfg': ('CustomCommands', 'config', 414589031223512),
+                'file': self.v2path / 'data' / 'customcom' / 'commands.json',
+                'converter': self.customcom_conv_spec
+            }
+        }
+
+    @property
+    def available(self):
+        return sorted(
+            k for k, v in self.available_core_conversions.items()
+            if v['file'].is_file() and v['converter'] is not None
+            and k not in self.resolved
+        )
+
+    def unpack(self, parent_key, parent_value):
+        """Unpack one level of nesting in a dictionary"""
+        try:
+            items = parent_value.items()
+        except AttributeError:
+            yield (parent_key, parent_value)
+        else:
+            for key, value in items:
+                yield (parent_key + (key,), value)
+
+    def flatten_dict(self, dictionary: dict):
+        """Flatten a nested dictionary structure"""
+        dictionary = {(key,): value for key, value in dictionary.items()}
+        while True:
+            dictionary = dict(
+                chain.from_iterable(
+                    starmap(self.unpack, dictionary.items())
+                )
+            )
+            if not any(
+                isinstance(value, dict)
+                for value in dictionary.values()
+            ):
+                break
+        return dictionary
+
+    def apply_scope(self, scope: str, data: dict):
+        return {(scope,) + k: v for k, v in data.items()}
+
+    def bank_accounts_conv_spec(self, data: dict):
+        flatscoped = self.apply_scope(Config.MEMBER, self.flatten_dict(data))
+        ret = {}
+        for k, v in flatscoped.items():
+            outerkey, innerkey = tuple(k[:-1]), (k[-1],)
+            if outerkey not in ret:
+                ret[outerkey] = {}
+            if innerkey[0] == 'created_at':
+                x = int(
+                    datetime.strptime(
+                        v, "%Y-%m-%d %H:%M:%S").timestamp()
+                )
+                ret[outerkey].update({innerkey: x})
+            else:
+                ret[outerkey].update({innerkey: v})
+        return ret
+
+    def economy_conv_spec(self, data: dict):
+        flatscoped = self.apply_scope(Config.GUILD, self.flatten_dict(data))
+        ret = {}
+        for k, v in flatscoped.items():
+            outerkey, innerkey = (*k[:-1],), (k[-1],)
+            if outerkey not in ret:
+                ret[outerkey] = {}
+            ret[outerkey].update({innerkey: v})
+        return ret
+
+    def mod_log_cases(self, data: dict):
+        raise NotImplementedError("This one isn't ready yet")
+
+    def filter_conv_spec(self, data: dict):
+        return {
+            (Config.GUILD, k): {('filter',): v}
+            for k, v in data.items()
+        }
+
+    def past_names_conv_spec(self, data: dict):
+        return {
+            (Config.USER, k): {('past_names',): v}
+            for k, v in data.items()
+        }
+
+    def past_nicknames_conv_spec(self, data: dict):
+        flatscoped = self.apply_scope(Config.MEMBER, self.flatten_dict(data))
+        ret = {}
+        for k, v in flatscoped.items():
+            outerkey, innerkey = (*k[:-1],), (k[-1],)
+            if outerkey not in ret:
+                ret[outerkey] = {}
+            ret[outerkey].update({innerkey: v})
+        return ret
+
+    def customcom_conv_spec(self, data: dict):
+        flatscoped = self.apply_scope(Config.GUILD, self.flatten_dict(data))
+        ret = {}
+        for k, v in flatscoped.items():
+            outerkey, innerkey = (*k[:-1],), ('commands', k[-1])
+            if outerkey not in ret:
+                ret[outerkey] = {}
+
+            ccinfo = {
+                'author': {
+                    'id': 42,
+                    'name': 'Converted from a v2 instance'
+                },
+                'command': k[-1],
+                'created_at': '{:%d/%m/%Y %H:%M:%S}'.format(datetime.utcnow()),
+                'editors': [],
+                'response': v
+            }
+            ret[outerkey].update({innerkey: ccinfo})
+        return ret
+
+    async def convert(self, bot: Red, prettyname: str):
+        if prettyname not in self.available:
+            raise NotImplementedError("No Conversion Specs for this")
+
+        info = self.available_core_conversions[prettyname]
+        filepath, converter = info['file'], info['converter']
+        (cogname, attr, _id) = info['cfg']
+        try:
+            config = getattr(bot.get_cog(cogname), attr)
+        except (TypeError, AttributeError):
+            config = Config.get_conf(cogname, _id)
+
+        try:
+            items = converter(dc.json_load(filepath))
+            await dc(config).dict_import(items)
+        except Exception:
+            raise
+        else:
+            self.resolved.add(prettyname)

redbot/cogs/dataconverter/dataconverter.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+import asyncio
+
+from discord.ext import commands
+
+from redbot.core import checks, RedContext
+from redbot.core.bot import Red
+from redbot.core.i18n import CogI18n
+from redbot.cogs.dataconverter.core_specs import SpecResolver
+from redbot.core.utils.chat_formatting import box
+
+_ = CogI18n('DataConverter', __file__)
+
+
+class DataConverter:
+    """
+    Cog for importing Red v2 Data
+    """
+
+    def __init__(self, bot: Red):
+        self.bot = bot
+
+    @checks.is_owner()
+    @commands.command(name="convertdata")
+    async def dataconversioncommand(self, ctx: RedContext, v2path: str):
+        """
+        Interactive prompt for importing data from Red v2
+
+        Takes the path where the v2 install is
+
+        Overwrites values which have entries in both v2 and v3,
+        use with caution.
+        """
+        resolver = SpecResolver(Path(v2path.strip()))
+
+        if not resolver.available:
+            return await ctx.send(
+                _("There don't seem to be any data files I know how to "
+                  "handle here. Are you sure you gave me the base "
+                  "installation path?")
+            )
+        while resolver.available:
+            menu = _("Please select a set of data to import by number"
+                     ", or 'exit' to exit")
+            for index, entry in enumerate(resolver.available, 1):
+                menu += "\n{}. {}".format(index, entry)
+
+            menu_message = await ctx.send(box(menu))
+
+            def pred(m):
+                return m.channel == ctx.channel and m.author == ctx.author
+
+            try:
+                message = await self.bot.wait_for(
+                    'message', check=pred, timeout=60
+                )
+            except asyncio.TimeoutError:
+                return await ctx.send(
+                    _('Try this again when you are more ready'))
+            else:
+                if message.content.strip().lower() in [
+                    'quit', 'exit', '-1', 'q', 'cancel'
+                ]:
+                    return await ctx.tick()
+                try:
+                    message = int(message.content.strip())
+                    to_conv = resolver.available[message - 1]
+                except (ValueError, IndexError):
+                    await ctx.send(
+                        _("That wasn't a valid choice.")
+                    )
+                    continue
+                else:
+                    async with ctx.typing():
+                        await resolver.convert(self.bot, to_conv)
+                    await ctx.send(_("{} converted.").format(to_conv))
+            await menu_message.delete()
+        else:
+            return await ctx.send(
+                _("There isn't anything else I know how to convert here."
+                  "\nThere might be more things I can convert in the future.")
+            )

redbot/cogs/dataconverter/locales/messages.pot

@@ -0,0 +1,43 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2018-03-12 04:35+EDT\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=CHARSET\n"
+"Content-Transfer-Encoding: ENCODING\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: ../dataconverter.py:38
+msgid "There don't seem to be any data files I know how to handle here. Are you sure you gave me the base installation path?"
+msgstr ""
+
+#: ../dataconverter.py:43
+msgid "Please select a set of data to import by number, or 'exit' to exit"
+msgstr ""
+
+#: ../dataconverter.py:59
+msgid "Try this again when you are more ready"
+msgstr ""
+
+#: ../dataconverter.py:70
+msgid "That wasn't a valid choice."
+msgstr ""
+
+#: ../dataconverter.py:76
+msgid "{} converted."
+msgstr ""
+
+#: ../dataconverter.py:80
+msgid ""
+"There isn't anything else I know how to convert here.\n"
+"There might be more things I can convert in the future."
+msgstr ""
+

redbot/cogs/dataconverter/locales/regen_messages.py

@@ -0,0 +1,15 @@
+import subprocess
+
+TO_TRANSLATE = [
+    '../dataconverter.py'
+]
+
+
+def regen_messages():
+    subprocess.run(
+        ['pygettext', '-n'] + TO_TRANSLATE
+    )
+
+
+if __name__ == "__main__":
+    regen_messages()

redbot/core/utils/data_converter.py

@@ -0,0 +1,134 @@
+import json
+from pathlib import Path
+from redbot.core import Config
+
+
+class DataConverter:
+    """
+    Class for moving v2 data to v3
+    """
+
+    def __init__(self, config_instance: Config):
+        self.config = config_instance
+
+    @staticmethod
+    def json_load(file_path: Path):
+        """Utility function for quickly grabbing data from a JSON file
+
+        Parameters
+        ----------
+        file_path: `pathlib.Path`
+            The path to the file to grabdata from
+
+        Raises
+        ------
+        FileNotFoundError
+            The file doesn't exist
+        json.JsonDecodeError
+            The file isn't valid JSON
+        """
+        try:
+            with open(file_path, mode='r') as f:
+                data = json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError):
+            raise
+        else:
+            return data
+
+    async def convert(self, file_path: Path, conversion_spec: object):
+        """Converts v2 data to v3 format. If your v2 data uses multiple files
+        you will need to call this for each file.
+
+        Parameters
+        ----------
+        file_path : `pathlib.Path`
+            This should be the path to a JSON settings file from v2
+        conversion_spec : `object`
+            This should be a function which takes a single argument argument
+            (the loaded JSON) and from it either
+            returns or yields one or more `dict`
+            whose items are in the form:
+
+            .. code-block:: python
+
+                {(SCOPE, *IDENTIFIERS): {(key_tuple): value}}
+
+            an example of a possible entry of that dict:
+
+            .. code-block:: python
+
+                {(Config.MEMBER, '133049272517001216', '78631113035100160'):
+                    {('balance',): 9001}}
+
+
+            This allows for any amount of entries at each level
+            in each of the nested dictionaries returned by conversion_spec
+            but the nesting cannot be different to this and still get the
+            expected results
+            see documentation for Config for more details on scopes
+            and the identifiers they need
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        FileNotFoundError
+            No such file at the specified path
+        json.JSONDecodeError
+            File is not valid JSON
+        AttributeError
+            Something goes wrong with your conversion and it provides
+            data in the wrong format
+        """
+
+        v2data = self.json_load(file_path)
+
+        for entryset in conversion_spec(v2data):
+            for scope_id, values in entryset.items():
+                base = self.config._get_base_group(*scope_id)
+                for inner_k, inner_v in values.items():
+                    await base.set_raw(*inner_k, value=inner_v)
+
+    async def dict_import(self, entrydict: dict):
+        """This imports a dictionary in the correct format into Config
+
+        Parameters
+        ----------
+        entrydict : `dict`
+            This should be a dictionary of values to set.
+            This is provided as an alternative
+            to providing a file and conversion specification
+            the dictionary should be in the following format
+
+            .. code-block:: python
+
+                {(SCOPE, *IDENTIFIERS): {(key_tuple): value}}`
+
+            an example of a possible entry of that dict:
+
+            .. code-block:: python
+
+                {(Config.MEMBER, '133049272517001216', '78631113035100160'):
+                    {('balance',): 9001}}
+
+            This allows for any amount of entries at each level
+            in each of the nested dictionaries returned by conversion_spec
+            but the nesting cannot be different to this and still get the
+            expected results
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        AttributeError
+            Data not in the correct format.
+        """
+
+        for scope_id, values in entrydict.items():
+            base = self.config._get_base_group(*scope_id)
+            for inner_k, inner_v in values.items():
+                await base.set_raw(*inner_k, value=inner_v)