[V3 Setup] Overhaul backend conversion process through setup scripts (#2579)

* swap to click for setup

* Initial changes

* expose some stuff to allow for per-driver optimizations

* overwrite base config

* add red log

* add one print juuuust in case

* fix this

* thanks kowlin

* damn

* oops

* fix thing

* partial commit

* Working mongo -> json conversion, it sucks tho

* remove unused line

* Wrote initial optimized json importer

* optimized json importer

* remove useless line

* update mongo to json converter

* lets try writing the correct entry

* oops

* style fix

* add some garbage data filters going from old mongo to json

* ignore garbage data in mongov2 conversions

* simplify code a bit and add a completion message

* missed one

* Update pipfile lock

* Lock click version

redbot/core/drivers/__init__.py

@@ -1,6 +1,14 @@
+import enum
+
 from .red_base import IdentifierData
 
-__all__ = ["get_driver", "IdentifierData"]
+__all__ = ["get_driver", "IdentifierData", "BackendType"]
+
+
+class BackendType(enum.Enum):
+    JSON = "JSON"
+    MONGO = "MongoDBV2"
+    MONGOV1 = "MongoDB"
 
 
 def get_driver(type, *args, **kwargs):

redbot/core/drivers/red_base.py

@@ -1,8 +1,18 @@
+import enum
 from typing import Tuple
 
 __all__ = ["BaseDriver", "IdentifierData"]
 
 
+class ConfigCategory(enum.Enum):
+    GLOBAL = "GLOBAL"
+    GUILD = "GUILD"
+    CHANNEL = "TEXTCHANNEL"
+    ROLE = "ROLE"
+    USER = "USER"
+    MEMBER = "MEMBER"
+
+
 class IdentifierData:
     def __init__(
         self,
@@ -72,6 +82,9 @@ class BaseDriver:
         self.cog_name = cog_name
         self.unique_cog_identifier = identifier
 
+    async def has_valid_connection(self) -> bool:
+        raise NotImplementedError
+
     async def get(self, identifier_data: IdentifierData):
         """
         Finds the value indicate by the given identifiers.
@@ -121,3 +134,75 @@ class BaseDriver:
         identifier_data
         """
         raise NotImplementedError
+
+    def _get_levels(self, category, custom_group_data):
+        if category == ConfigCategory.GLOBAL.value:
+            return 0
+        elif category in (
+            ConfigCategory.USER.value,
+            ConfigCategory.GUILD.value,
+            ConfigCategory.CHANNEL.value,
+            ConfigCategory.ROLE.value,
+        ):
+            return 1
+        elif category == ConfigCategory.MEMBER.value:
+            return 2
+        elif category in custom_group_data:
+            return custom_group_data[category]
+        else:
+            raise RuntimeError(f"Cannot convert due to group: {category}")
+
+    def _split_primary_key(self, category, custom_group_data, data):
+        levels = self._get_levels(category, custom_group_data)
+        if levels == 0:
+            return (((), data),)
+
+        def flatten(levels_remaining, currdata, parent_key=()):
+            items = []
+            for k, v in currdata.items():
+                new_key = parent_key + (k,)
+                if levels_remaining > 1:
+                    items.extend(flatten(levels_remaining - 1, v, new_key).items())
+                else:
+                    items.append((new_key, v))
+            return dict(items)
+
+        ret = []
+        for k, v in flatten(levels, data).items():
+            ret.append((k, v))
+        return tuple(ret)
+
+    async def export_data(self, custom_group_data):
+        categories = [c.value for c in ConfigCategory]
+        categories.extend(custom_group_data.keys())
+
+        ret = []
+        for c in categories:
+            ident_data = IdentifierData(
+                self.unique_cog_identifier,
+                c,
+                (),
+                (),
+                custom_group_data.get(c, {}),
+                is_custom=c in custom_group_data,
+            )
+            try:
+                data = await self.get(ident_data)
+            except KeyError:
+                continue
+            ret.append((c, data))
+        return ret
+
+    async def import_data(self, cog_data, custom_group_data):
+        for category, all_data in cog_data:
+            splitted_pkey = self._split_primary_key(category, custom_group_data, all_data)
+            for pkey, data in splitted_pkey:
+                ident_data = IdentifierData(
+                    self.unique_cog_identifier,
+                    category,
+                    pkey,
+                    (),
+                    custom_group_data.get(category, {}),
+                    is_custom=category in custom_group_data,
+                )
+                await self.set(ident_data, data)

redbot/core/drivers/red_json.py

@@ -69,6 +69,9 @@ class JSON(BaseDriver):
 
         self._load_data()
 
+    async def has_valid_connection(self) -> bool:
+        return True
+
     @property
     def data(self):
         return _shared_datastore.get(self.cog_name)
@@ -123,5 +126,29 @@ class JSON(BaseDriver):
         else:
             await self.jsonIO._threadsafe_save_json(self.data)
 
+    async def import_data(self, cog_data, custom_group_data):
+        def update_write_data(identifier_data: IdentifierData, data):
+            partial = self.data
+            idents = identifier_data.to_tuple()
+            for ident in idents[:-1]:
+                if ident not in partial:
+                    partial[ident] = {}
+                partial = partial[ident]
+            partial[idents[-1]] = data
+
+        for category, all_data in cog_data:
+            splitted_pkey = self._split_primary_key(category, custom_group_data, all_data)
+            for pkey, data in splitted_pkey:
+                ident_data = IdentifierData(
+                    self.unique_cog_identifier,
+                    category,
+                    pkey,
+                    (),
+                    custom_group_data.get(category, {}),
+                    is_custom=category in custom_group_data,
+                )
+                update_write_data(ident_data, data)
+        await self.jsonIO._threadsafe_save_json(self.data)
+
     def get_config_details(self):
         return

redbot/core/drivers/red_mongo.py

@@ -49,6 +49,10 @@ class Mongo(BaseDriver):
         if _conn is None:
             _initialize(**kwargs)
 
+    async def has_valid_connection(self) -> bool:
+        # Maybe fix this?
+        return True
+
     @property
     def db(self) -> motor.core.Database:
         """