Begin work on a data request API (#4045)

[Core] Data Deletion And Disclosure APIs

 - Adds a Data Deletion API
   - Deletion comes in a few forms based on who is requesting
   - Deletion must be handled by 3rd party
 - Adds a Data Collection Disclosure Command
   - Provides a dynamically generated statement from 3rd party
   extensions
 - Modifies the always available commands to be cog compatible
   - Also prevents them from being unloaded accidentally

redbot/core/bot.py

@@ -6,6 +6,8 @@ import platform
 import shutil
 import sys
 import contextlib
+import weakref
+import functools
 from collections import namedtuple
 from datetime import datetime
 from enum import IntEnum
@@ -22,6 +24,8 @@ from typing import (
     Callable,
     Awaitable,
     Any,
+    Literal,
+    MutableMapping,
 )
 from types import MappingProxyType
 
@@ -31,7 +35,7 @@ from discord.ext.commands import when_mentioned_or
 
 from . import Config, i18n, commands, errors, drivers, modlog, bank
 from .cog_manager import CogManager, CogManagerUI
-from .core_commands import license_info_command, Core
+from .core_commands import Core
 from .data_manager import cog_data_path
 from .dev_commands import Dev
 from .events import init_events
@@ -45,7 +49,7 @@ from .settings_caches import (
 )
 
 from .rpc import RPCMixin
-from .utils import common_filters
+from .utils import common_filters, AsyncIter
 from .utils._internal_utils import send_to_owners_with_prefix_replaced
 
 CUSTOM_GROUPS = "CUSTOM_GROUPS"
@@ -57,6 +61,8 @@ __all__ = ["RedBase", "Red", "ExitCodes"]
 
 NotMessage = namedtuple("NotMessage", "guild")
 
+DataDeletionResults = namedtuple("DataDeletionResults", "failed_modules failed_cogs unhandled")
+
 PreInvokeCoroutine = Callable[[commands.Context], Awaitable[Any]]
 T_BIC = TypeVar("T_BIC", bound=PreInvokeCoroutine)
 
@@ -117,6 +123,8 @@ class RedBase(
             last_system_info__machine=None,
             last_system_info__system=None,
             schema_version=0,
+            datarequests__allow_user_requests=True,
+            datarequests__user_requests_are_strict=True,
         )
 
         self._config.register_guild(
@@ -198,6 +206,8 @@ class RedBase(
         self._red_ready = asyncio.Event()
         self._red_before_invoke_objs: Set[PreInvokeCoroutine] = set()
 
+        self._deletion_requests: MutableMapping[int, asyncio.Lock] = weakref.WeakValueDictionary()
+
     def get_command(self, name: str) -> Optional[commands.Command]:
         com = super().get_command(name)
         assert com is None or isinstance(com, commands.Command)
@@ -219,7 +229,7 @@ class RedBase(
 
     async def _red_before_invoke_method(self, ctx):
         await self.wait_until_red_ready()
-        return_exceptions = isinstance(ctx.command, commands.commands._AlwaysAvailableCommand)
+        return_exceptions = isinstance(ctx.command, commands.commands._RuleDropper)
         if self._red_before_invoke_objs:
             await asyncio.gather(
                 *(coro(ctx) for coro in self._red_before_invoke_objs),
@@ -666,7 +676,6 @@ class RedBase(
 
         self.add_cog(Core(self))
         self.add_cog(CogManagerUI())
-        self.add_command(license_info_command)
         if cli_flags.dev:
             self.add_cog(Dev())
 
@@ -1040,7 +1049,7 @@ class RedBase(
 
     def remove_cog(self, cogname: str):
         cog = self.get_cog(cogname)
-        if cog is None:
+        if cog is None or isinstance(cog, commands.commands._RuleDropper):
             return
 
         for cls in inspect.getmro(cog.__class__):
@@ -1197,6 +1206,9 @@ class RedBase(
                     subcommand.requires.ready_event.set()
 
     def remove_command(self, name: str) -> None:
+        command = self.get_command(name)
+        if isinstance(command, commands.commands._RuleDropper):
+            return
         command = super().remove_command(name)
         if not command:
             return
@@ -1395,6 +1407,124 @@ class RedBase(
         await self.logout()
         sys.exit(self._shutdown_mode)
 
+    async def _core_data_deletion(
+        self,
+        *,
+        requester: Literal["discord_deleted_user", "owner", "user", "user_strict"],
+        user_id: int,
+    ):
+        if requester != "discord_deleted_user":
+            return
+
+        await self._config.user_from_id(user_id).clear()
+        all_guilds = await self._config.all_guilds()
+
+        async for guild_id, guild_data in AsyncIter(all_guilds.items(), steps=100):
+            if user_id in guild_data.get("autoimmune_ids", []):
+                async with self._config.guild_from_id(guild_id).autoimmune_ids() as ids:
+                    # prevent a racy crash here without locking
+                    # up the vals in all guilds first
+                    with contextlib.suppress(ValueError):
+                        ids.remove(user_id)
+
+        await self._whiteblacklist_cache.discord_deleted_user(user_id)
+
+    async def handle_data_deletion_request(
+        self,
+        *,
+        requester: Literal["discord_deleted_user", "owner", "user", "user_strict"],
+        user_id: int,
+    ) -> DataDeletionResults:
+        """
+        This tells each cog and extension, as well as any APIs in Red
+        to go delete data
+
+        Calling this should be limited to interfaces designed for it.
+
+        See ``redbot.core.commands.Cog.delete_data_for_user``
+        for details about the parameters and intent.
+
+        Parameters
+        ----------
+        requester
+        user_id
+
+        Returns
+        -------
+        DataDeletionResults
+            A named tuple ``(failed_modules, failed_cogs, unhandled)``
+            containing lists with names of failed modules, failed cogs,
+            and cogs that didn't handle data deletion request.
+        """
+        await self.wait_until_red_ready()
+        lock = self._deletion_requests.setdefault(user_id, asyncio.Lock())
+        async with lock:
+            return await self._handle_data_deletion_request(requester=requester, user_id=user_id)
+
+    async def _handle_data_deletion_request(
+        self,
+        *,
+        requester: Literal["discord_deleted_user", "owner", "user", "user_strict"],
+        user_id: int,
+    ) -> DataDeletionResults:
+        """
+        Actual interface for the above.
+
+        Parameters
+        ----------
+        requester
+        user_id
+
+        Returns
+        -------
+        DataDeletionResults
+        """
+        extension_handlers = {
+            extension_name: handler
+            for extension_name, extension in self.extensions.items()
+            if (handler := getattr(extension, "red_delete_data_for_user", None))
+        }
+
+        cog_handlers = {
+            cog_qualname: cog.red_delete_data_for_user for cog_qualname, cog in self.cogs.items()
+        }
+
+        special_handlers = {
+            "Red Core Modlog API": modlog._process_data_deletion,
+            "Red Core Bank API": bank._process_data_deletion,
+            "Red Core Bot Data": self._core_data_deletion,
+        }
+
+        failures = {
+            "extension": [],
+            "cog": [],
+            "unhandled": [],
+        }
+
+        async def wrapper(func, stype, sname):
+            try:
+                await func(requester=requester, user_id=user_id)
+            except commands.commands.RedUnhandledAPI:
+                log.warning(f"{stype}.{sname} did not handle data deletion ")
+                failures["unhandled"].append(sname)
+            except Exception as exc:
+                log.exception(f"{stype}.{sname} errored when handling data deletion ")
+                failures[stype].append(sname)
+
+        handlers = [
+            *(wrapper(coro, "extension", name) for name, coro in extension_handlers.items()),
+            *(wrapper(coro, "cog", name) for name, coro in cog_handlers.items()),
+            *(wrapper(coro, "extension", name) for name, coro in special_handlers.items()),
+        ]
+
+        await asyncio.gather(*handlers)
+
+        return DataDeletionResults(
+            failed_modules=failures["extension"],
+            failed_cogs=failures["cog"],
+            unhandled=failures["unhandled"],
+        )
+
 
 # This can be removed, and the parent class renamed as a breaking change
 class Red(RedBase):