From c0b1e50a5f10c48e5ce7c6f1cafd4fd0ac3d10dd Mon Sep 17 00:00:00 2001 From: Michael H Date: Mon, 3 Aug 2020 09:09:07 -0400 Subject: [PATCH] Begin work on a data request API (#4045) [Core] Data Deletion And Disclosure APIs - Adds a Data Deletion API - Deletion comes in a few forms based on who is requesting - Deletion must be handled by 3rd party - Adds a Data Collection Disclosure Command - Provides a dynamically generated statement from 3rd party extensions - Modifies the always available commands to be cog compatible - Also prevents them from being unloaded accidentally --- docs/framework_commands.rst | 8 + docs/guide_cog_creation.rst | 18 + docs/index.rst | 1 + docs/red_core_data_statement.rst | 87 +++ redbot/cogs/admin/admin.py | 4 + redbot/cogs/alias/__init__.py | 2 +- redbot/cogs/alias/alias.py | 107 +++- redbot/cogs/alias/alias_entry.py | 24 + redbot/cogs/audio/apis/playlist_wrapper.py | 11 + redbot/cogs/audio/core/events/red.py | 37 +- redbot/cogs/audio/sql_statements.py | 29 + redbot/cogs/bank/bank.py | 4 + redbot/cogs/cleanup/cleanup.py | 4 + redbot/cogs/customcom/customcom.py | 54 +- redbot/cogs/downloader/downloader.py | 4 + redbot/cogs/economy/economy.py | 21 +- redbot/cogs/filter/filter.py | 18 +- redbot/cogs/general/general.py | 4 + redbot/cogs/image/image.py | 4 + redbot/cogs/mod/mod.py | 30 +- redbot/cogs/modlog/modlog.py | 4 + redbot/cogs/permissions/converters.py | 19 +- redbot/cogs/permissions/permissions.py | 85 +-- redbot/cogs/reports/reports.py | 35 +- redbot/cogs/streams/streams.py | 4 + redbot/cogs/trivia/trivia.py | 18 +- redbot/cogs/warnings/warnings.py | 51 +- redbot/core/bank.py | 31 +- redbot/core/bot.py | 140 ++++- redbot/core/cog_manager.py | 4 + redbot/core/commands/__init__.py | 1 + redbot/core/commands/commands.py | 213 +++++++- redbot/core/core_commands.py | 602 +++++++++++++++++++-- redbot/core/dev_commands.py | 7 + redbot/core/modlog.py | 53 +- redbot/core/settings_caches.py | 243 +++++---- setup.cfg | 1 + tools/primary_deps.ini | 1 + 38 files changed, 1761 insertions(+), 222 deletions(-) create mode 100644 docs/red_core_data_statement.rst diff --git a/docs/framework_commands.rst b/docs/framework_commands.rst index 26f8b32d9..d5e61b240 100644 --- a/docs/framework_commands.rst +++ b/docs/framework_commands.rst @@ -13,6 +13,14 @@ extend functionalities used throughout the bot, as outlined below. .. autofunction:: redbot.core.commands.group +.. autoclass:: redbot.core.commands.Cog + + .. automethod:: format_help_for_context + + .. automethod:: red_get_data_for_user + + .. automethod:: red_delete_data_for_user + .. autoclass:: redbot.core.commands.Command :members: :inherited-members: format_help_for_context diff --git a/docs/guide_cog_creation.rst b/docs/guide_cog_creation.rst index 632f452ec..6ce4239d8 100644 --- a/docs/guide_cog_creation.rst +++ b/docs/guide_cog_creation.rst @@ -98,6 +98,7 @@ Open :code:`__init__.py`. In that file, place the following: from .mycog import Mycog + def setup(bot): bot.add_cog(Mycog()) @@ -238,3 +239,20 @@ Not all of these are strict requirements (some are) but are all generally advisa but a cog which takes actions based on messages should not. 15. Respect settings when treating non command messages as commands. + +16. Handle user data responsibly + + - Don't do unexpected things with user data. + - Don't expose user data to additional audiences without permission. + - Don't collect data your cogs don't need. + - Don't store data in unexpected locations. + Utilize the cog data path, Config, or if you need something more + prompt the owner to provide it. + +17. Utilize the data deletion and statement APIs + + - See `redbot.core.commands.Cog.red_delete_data_for_user` + - Make a statement about what data your cogs use with the module level + variable ``__red_end_user_data_statement__``. + This should be a string containing a user friendly explanation of what data + your cog stores and why. diff --git a/docs/index.rst b/docs/index.rst index 64408672c..b19dd2459 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ Welcome to Red - Discord Bot's documentation! :caption: User guides: getting_started + red_core_data_statement .. toctree:: :maxdepth: 2 diff --git a/docs/red_core_data_statement.rst b/docs/red_core_data_statement.rst new file mode 100644 index 000000000..088517b42 --- /dev/null +++ b/docs/red_core_data_statement.rst @@ -0,0 +1,87 @@ +.. Red Core Data Statement + +===================== +Red and End User Data +===================== + +Notes for everyone +****************** + +What data Red collects +---------------------- + +Red and the cogs included with it collect some amount of data +about users for the bot's normal operations. + +In particular the bot will keep track of a short history of usernames/nicknames +which actions refer to your Discord account (such as creating a playlist) +as well as the content of specific messages used directly as commands for the bot +(such as reports sent to servers). + +By default, Red will not collect any more data than it needs, and will not use it +for anything other than the portion of the Red's functionality that necessitated it. + +3rd party extensions may store additional data beyond what Red does by default. +You can use the command ``[p]mydata 3rdparty`` +to view statements about how extensions use your data made by the authors of +the specific 3rd party extensions an instance of Red has installed. + +How can I delete data Red has about me? +--------------------------------------- + +The command ``[p]mydata forgetme`` provides a way for users to remove +large portions of their own data from the bot. This command will not +remove operational data, such as a record that your +Discord account was the target of a moderation action. + +3rd party extensions to Red are able to delete data when this command +is used as well, but this is something each extension must implement. +If a loaded extension does not implement this, the user will be informed. + +Additional Notes for Bot Owners and Hosts +***************************************** + +How to comply with a request from Discord Trust & Safety +-------------------------------------------------------- + +There are a handful of these available to bot owners in the command group +``[p]mydata ownermanagement``. + +The most pertinent one if asked to delete data by a member of Trust & Safety +is + +``[p]mydata ownermanagement processdiscordrequest`` + +This will cause the bot to get rid of or disassociate all data +from the specified user ID. + +.. warning:: + + You should not use this unless + Discord has specifically requested this with regard to a deleted user. + This will remove the user from various anti-abuse measures. + If you are processing a manual request from a user, read the next section + + +How to process deletion requests from users +------------------------------------------- + +You can point users to the command ``[p]mydata forgetme`` as a first step. + +If users cannot use that for some reason, the command + +``[p]mydata ownermanagement deleteforuser`` + +exists as a way to handle this as if the user had done it themselves. + +Be careful about using the other owner level deletion options on behalf of users, +as this may also result in losing operational data such as data used to prevent spam. + +What owners and hosts are responsible for +----------------------------------------- + +Owners and hosts must comply both with Discord's terms of service and any applicable laws. +Owners and hosts are responsible for all actions their bot takes. + +We cannot give specific guidance on this, but recommend that if there are any issues +you be forthright with users, own up to any mistakes, and do your best to handle it. diff --git a/redbot/cogs/admin/admin.py b/redbot/cogs/admin/admin.py index 2896df2cd..077756389 100644 --- a/redbot/cogs/admin/admin.py +++ b/redbot/cogs/admin/admin.py @@ -87,6 +87,10 @@ class Admin(commands.Cog): async def cog_before_invoke(self, ctx: commands.Context): await self._ready.wait() + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + async def handle_migrations(self): lock = self.config.get_guilds_lock() diff --git a/redbot/cogs/alias/__init__.py b/redbot/cogs/alias/__init__.py index c4ff8ea95..3bdd12415 100644 --- a/redbot/cogs/alias/__init__.py +++ b/redbot/cogs/alias/__init__.py @@ -4,5 +4,5 @@ from redbot.core.bot import Red async def setup(bot: Red): cog = Alias(bot) - await cog.initialize() bot.add_cog(cog) + cog.sync_init() diff --git a/redbot/cogs/alias/alias.py b/redbot/cogs/alias/alias.py index d7a2f58a7..dc40dc308 100644 --- a/redbot/cogs/alias/alias.py +++ b/redbot/cogs/alias/alias.py @@ -1,7 +1,9 @@ +import asyncio +import logging from copy import copy from re import search from string import Formatter -from typing import Dict, List +from typing import Dict, List, Literal import discord from redbot.core import Config, commands, checks @@ -14,6 +16,8 @@ from .alias_entry import AliasEntry, AliasCache, ArgParseError _ = Translator("Alias", __file__) +log = logging.getLogger("red.cogs.alias") + class _TrackingFormatter(Formatter): def __init__(self): @@ -38,24 +42,107 @@ class Alias(commands.Cog): and append them to the stored alias. """ - default_global_settings: Dict[str, list] = {"entries": []} - - default_guild_settings: Dict[str, list] = {"entries": []} # Going to be a list of dicts - def __init__(self, bot: Red): super().__init__() self.bot = bot self.config = Config.get_conf(self, 8927348724) - self.config.register_global(**self.default_global_settings) - self.config.register_guild(**self.default_guild_settings) + self.config.register_global(entries=[], handled_string_creator=False) + self.config.register_guild(entries=[]) self._aliases: AliasCache = AliasCache(config=self.config, cache_enabled=True) + self._ready_event = asyncio.Event() + + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + await self._ready_event.wait() + await self._aliases.anonymize_aliases(user_id) + + async def cog_before_invoke(self, ctx): + await self._ready_event.wait() + + async def _maybe_handle_string_keys(self): + # This isn't a normal schema migration because it's being added + # after the fact for GH-3788 + if await self.config.handled_string_creator(): + return + + async with self.config.entries() as alias_list: + bad_aliases = [] + for a in alias_list: + for keyname in ("creator", "guild"): + if isinstance((val := a.get(keyname)), str): + try: + a[keyname] = int(val) + except ValueError: + # Because migrations weren't created as changes were made, + # and the prior form was a string of an ID, + # if this fails, there's nothing to go back to + bad_aliases.append(a) + break + + for a in bad_aliases: + alias_list.remove(a) + + # if this was using a custom group of (guild_id, aliasname) it would be better but... + all_guild_aliases = await self.config.all_guilds() + + for guild_id, guild_data in all_guild_aliases.items(): + + to_set = [] + modified = False + + for a in guild_data.get("entries", []): + + for keyname in ("creator", "guild"): + if isinstance((val := a.get(keyname)), str): + try: + a[keyname] = int(val) + except ValueError: + break + finally: + modified = True + else: + to_set.append(a) + + if modified: + await self.config.guild_from_id(guild_id).entries.set(to_set) + + await asyncio.sleep(0) + # control yielded per loop since this is most likely to happen + # at bot startup, where this is most likely to have a performance + # hit. + + await self.config.handled_string_creator.set(True) + + def sync_init(self): + t = asyncio.create_task(self._initialize()) + + def done_callback(fut: asyncio.Future): + try: + t.result() + except Exception as exc: + log.exception("Failed to load alias cog", exc_info=exc) + # Maybe schedule extension unloading with message to owner in future + + t.add_done_callback(done_callback) + + async def _initialize(self): + """ Should only ever be a task """ + + await self._maybe_handle_string_keys() - async def initialize(self): - # This can be where we set the cache_enabled attribute later if not self._aliases._loaded: await self._aliases.load_aliases() + self._ready_event.set() + def is_command(self, alias_name: str) -> bool: """ The logic here is that if this returns true, the name should not be used for an alias @@ -327,6 +414,8 @@ class Alias(commands.Cog): @commands.Cog.listener() async def on_message_without_command(self, message: discord.Message): + await self._ready_event.wait() + if message.guild is not None: if await self.bot.cog_disabled_in_guild(self, message.guild): return diff --git a/redbot/cogs/alias/alias_entry.py b/redbot/cogs/alias/alias_entry.py index 78e1026ba..2ac5a2779 100644 --- a/redbot/cogs/alias/alias_entry.py +++ b/redbot/cogs/alias/alias_entry.py @@ -90,6 +90,30 @@ class AliasCache: self._loaded = False self._aliases: Dict[Optional[int], Dict[str, AliasEntry]] = {None: {}} + async def anonymize_aliases(self, user_id: int): + + async with self.config.entries() as global_aliases: + for a in global_aliases: + if a.get("creator", 0) == user_id: + a["creator"] = 0xDE1 + if self._cache_enabled: + self._aliases[None][a["name"]] = AliasEntry.from_json(a) + + all_guilds = await self.config.all_guilds() + async for guild_id, guild_data in AsyncIter(all_guilds.items(), steps=100): + for a in guild_data["entries"]: + if a.get("creator", 0) == user_id: + break + else: + continue + # basically, don't build a context manager wihout a need. + async with self.config.guild_from_id(guild_id).entries() as entry_list: + for a in entry_list: + if a.get("creator", 0) == user_id: + a["creator"] = 0xDE1 + if self._cache_enabled: + self._aliases[guild_id][a["name"]] = AliasEntry.from_json(a) + async def load_aliases(self): if not self._cache_enabled: self._loaded = True diff --git a/redbot/cogs/audio/apis/playlist_wrapper.py b/redbot/cogs/audio/apis/playlist_wrapper.py index 106ca6c8f..11f11e0b7 100644 --- a/redbot/cogs/audio/apis/playlist_wrapper.py +++ b/redbot/cogs/audio/apis/playlist_wrapper.py @@ -27,6 +27,7 @@ from ..sql_statements import ( PRAGMA_SET_read_uncommitted, PRAGMA_SET_temp_store, PRAGMA_SET_user_version, + HANDLE_DISCORD_DATA_DELETION_QUERY, ) from ..utils import PlaylistScope from .api_utils import PlaylistFetchResult @@ -58,6 +59,8 @@ class PlaylistWrapper: self.statement.get_all_with_filter = PLAYLIST_FETCH_ALL_WITH_FILTER self.statement.get_all_converter = PLAYLIST_FETCH_ALL_CONVERTER + self.statement.drop_user_playlists = HANDLE_DISCORD_DATA_DELETION_QUERY + async def init(self) -> None: """Initialize the Playlist table""" with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: @@ -247,3 +250,11 @@ class PlaylistWrapper: "tracks": json.dumps(tracks), }, ) + + async def handle_playlist_user_id_deletion(self, user_id: int): + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + executor.submit( + self.database.cursor().execute, + self.statement.drop_user_playlists, + {"user_id": user_id}, + ) diff --git a/redbot/cogs/audio/core/events/red.py b/redbot/cogs/audio/core/events/red.py index 562f80432..5010cc9b9 100644 --- a/redbot/cogs/audio/core/events/red.py +++ b/redbot/cogs/audio/core/events/red.py @@ -1,5 +1,6 @@ +import asyncio import logging -from typing import Mapping +from typing import Literal, Mapping from redbot.core import commands from ..abc import MixinMeta @@ -19,3 +20,37 @@ class RedEvents(MixinMeta, metaclass=CompositeMetaClass): self.api_interface.spotify_api.update_token(api_tokens) elif service_name == "audiodb": self.api_interface.global_cache_api.update_token(api_tokens) + + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + + await self.cog_ready_event.wait() + + if requester in ("discord_deleted_user", "owner"): + await self.playlist_api.handle_playlist_user_id_deletion(user_id) + + all_equalizers = await self.config.custom("EQUALIZER").all() + + collected_for_removal = [] + + c = 0 + for guild_id, guild_equalizers in all_equalizers.items(): + c += 1 + if not c % 100: + await asyncio.sleep(0) + + for preset_name, preset in guild_equalizers.get("eq_presets", {}).items(): + c += 1 + if not c % 100: + await asyncio.sleep(0) + + if preset.get("author", 0) == user_id: + collected_for_removal.append((guild_id, preset_name)) + + async with self.config.custom("EQUALIZER").all() as all_eqs: + for guild_id, preset_name in collected_for_removal: + all_eqs[str(guild_id)]["eq_presets"][preset_name]["author"] = 0xDE1 diff --git a/redbot/cogs/audio/sql_statements.py b/redbot/cogs/audio/sql_statements.py index dcf55663c..e37448d48 100644 --- a/redbot/cogs/audio/sql_statements.py +++ b/redbot/cogs/audio/sql_statements.py @@ -10,6 +10,8 @@ __all__ = [ "PRAGMA_SET_read_uncommitted", "PRAGMA_FETCH_user_version", "PRAGMA_SET_user_version", + # Data Deletion statement + "HANDLE_DISCORD_DATA_DELETION_QUERY", # Playlist table statements "PLAYLIST_CREATE_TABLE", "PLAYLIST_DELETE", @@ -82,6 +84,33 @@ PRAGMA_SET_user_version: Final[ pragma user_version=3; """ +# Data Deletion +# This is intentionally 2 seperate transactions due to concerns +# Draper had. This should prevent it from being a large issue, +# as this is no different than triggering a bulk deletion now. +HANDLE_DISCORD_DATA_DELETION_QUERY: Final[ + str +] = """ +BEGIN TRANSACTION; + +UPDATE playlists +SET deleted = true +WHERE scope_id = :user_id ; + +UPDATE playlists +SET author_id = 0xde1 +WHERE author_id = :user_id ; + +COMMIT TRANSACTION; + +BEGIN TRANSACTION; + +DELETE FROM PLAYLISTS +WHERE deleted=true; + +COMMIT TRANSACTION; +""" + # Playlist table statements PLAYLIST_CREATE_TABLE: Final[ str diff --git a/redbot/cogs/bank/bank.py b/redbot/cogs/bank/bank.py index 41bfeb261..f02d86fd5 100644 --- a/redbot/cogs/bank/bank.py +++ b/redbot/cogs/bank/bank.py @@ -134,3 +134,7 @@ class Bank(commands.Cog): ) # ENDSECTION + + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return diff --git a/redbot/cogs/cleanup/cleanup.py b/redbot/cogs/cleanup/cleanup.py index 2cd2d37e8..c8311647c 100644 --- a/redbot/cogs/cleanup/cleanup.py +++ b/redbot/cogs/cleanup/cleanup.py @@ -25,6 +25,10 @@ class Cleanup(commands.Cog): super().__init__() self.bot = bot + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + @staticmethod async def check_100_plus(ctx: commands.Context, number: int) -> bool: """ diff --git a/redbot/cogs/customcom/customcom.py b/redbot/cogs/customcom/customcom.py index 0976e1832..a94548bb8 100644 --- a/redbot/cogs/customcom/customcom.py +++ b/redbot/cogs/customcom/customcom.py @@ -1,9 +1,10 @@ +import asyncio import re import random from datetime import datetime, timedelta from inspect import Parameter from collections import OrderedDict -from typing import Iterable, List, Mapping, Tuple, Dict, Set +from typing import Iterable, List, Mapping, Tuple, Dict, Set, Literal from urllib.parse import quote_plus import discord @@ -11,7 +12,7 @@ from fuzzywuzzy import process from redbot.core import Config, checks, commands from redbot.core.i18n import Translator, cog_i18n -from redbot.core.utils import menus +from redbot.core.utils import menus, AsyncIter from redbot.core.utils.chat_formatting import box, pagify, escape, humanize_list from redbot.core.utils.predicates import MessagePredicate @@ -40,15 +41,35 @@ class OnCooldown(CCError): class CommandObj: def __init__(self, **kwargs): - config = kwargs.get("config") + self.config = kwargs.get("config") self.bot = kwargs.get("bot") - self.db = config.guild + self.db = self.config.guild @staticmethod async def get_commands(config) -> dict: _commands = await config.commands() return {k: v for k, v in _commands.items() if _commands[k]} + async def redact_author_ids(self, user_id: int): + + all_guilds = await self.config.all_guilds() + + for guild_id in all_guilds.keys(): + await asyncio.sleep(0) + async with self.config.guild_from_id(guild_id).commands() as all_commands: + async for com_name, com_info in AsyncIter(all_commands.items(), steps=100): + if not com_info: + continue + + if com_info.get("author", {}).get("id", 0) == user_id: + com_info["author"]["id"] = 0xDE1 + com_info["author"]["name"] = "Deleted User" + + if editors := com_info.get("editors", None): + for index, editor_id in enumerate(editors): + if editor_id == user_id: + editors[index] = 0xDE1 + async def get_responses(self, ctx): intro = _( "Welcome to the interactive random {cc} maker!\n" @@ -200,6 +221,17 @@ class CustomCommands(commands.Cog): self.commandobj = CommandObj(config=self.config, bot=self.bot) self.cooldowns = {} + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + await self.commandobj.redact_author_ids(user_id) + @commands.group(aliases=["cc"]) @commands.guild_only() async def customcom(self, ctx: commands.Context): @@ -209,7 +241,7 @@ class CustomCommands(commands.Cog): @customcom.command(name="raw") async def cc_raw(self, ctx: commands.Context, command: str.lower): """Get the raw response of a custom command, to get the proper markdown. - + This is helpful for copy and pasting.""" commands = await self.config.guild(ctx.guild).commands() if command not in commands: @@ -472,12 +504,14 @@ class CustomCommands(commands.Cog): if isinstance(responses, str): responses = [responses] - author = ctx.guild.get_member(cmd["author"]["id"]) - # If the author is still in the server, show their current name - if author: - author = "{} ({})".format(author, cmd["author"]["id"]) + _aid = cmd["author"]["id"] + + if _aid == 0xDE1: + author = _("Deleted User") + elif member := ctx.guild.get_member(_aid): + author = f"{member} ({_aid})" else: - author = "{} ({})".format(cmd["author"]["name"], cmd["author"]["id"]) + author = f"{cmd['author']['name']} ({_aid})" _type = _("Random") if len(responses) > 1 else _("Normal") diff --git a/redbot/cogs/downloader/downloader.py b/redbot/cogs/downloader/downloader.py index ef4138387..1c8666650 100644 --- a/redbot/cogs/downloader/downloader.py +++ b/redbot/cogs/downloader/downloader.py @@ -91,6 +91,10 @@ class Downloader(commands.Cog): if self._init_task is not None: self._init_task.cancel() + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + def create_init_task(self): def _done_callback(task: asyncio.Task) -> None: exc = task.exception() diff --git a/redbot/cogs/economy/economy.py b/redbot/cogs/economy/economy.py index c05195d8d..0e9cfca47 100644 --- a/redbot/cogs/economy/economy.py +++ b/redbot/cogs/economy/economy.py @@ -3,7 +3,7 @@ import logging import random from collections import defaultdict, deque, namedtuple from enum import Enum -from typing import cast, Iterable, Union +from typing import cast, Iterable, Union, Literal import discord @@ -11,6 +11,7 @@ from redbot.cogs.bank import is_owner_if_bank_global from redbot.cogs.mod.converters import RawUserIds from redbot.core import Config, bank, commands, errors, checks from redbot.core.i18n import Translator, cog_i18n +from redbot.core.utils import AsyncIter from redbot.core.utils.chat_formatting import box, humanize_number from redbot.core.utils.menus import menu, DEFAULT_CONTROLS @@ -136,7 +137,6 @@ class Economy(commands.Cog): def __init__(self, bot: Red): super().__init__() self.bot = bot - self.file_path = "data/economy/settings.json" self.config = Config.get_conf(self, 1256844281) self.config.register_guild(**self.default_guild_settings) self.config.register_global(**self.default_global_settings) @@ -145,6 +145,23 @@ class Economy(commands.Cog): self.config.register_role(**self.default_role_settings) self.slot_register = defaultdict(dict) + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + await self.config.user_from_id(user_id).clear() + + all_members = await self.config.all_members() + + async for guild_id, guild_data in AsyncIter(all_members.items(), steps=100): + if user_id in guild_data: + await self.config.member_from_ids(guild_id, user_id).clear() + @guild_only_check() @commands.group(name="bank") async def _bank(self, ctx: commands.Context): diff --git a/redbot/cogs/filter/filter.py b/redbot/cogs/filter/filter.py index a25c89635..318aa1d94 100644 --- a/redbot/cogs/filter/filter.py +++ b/redbot/cogs/filter/filter.py @@ -1,10 +1,11 @@ import discord import re -from typing import Union, Set +from typing import Union, Set, Literal from redbot.core import checks, Config, modlog, commands from redbot.core.bot import Red from redbot.core.i18n import Translator, cog_i18n +from redbot.core.utils import AsyncIter from redbot.core.utils.chat_formatting import pagify, humanize_list _ = Translator("Filter", __file__) @@ -33,6 +34,21 @@ class Filter(commands.Cog): self.register_task = self.bot.loop.create_task(self.register_filterban()) self.pattern_cache = {} + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + all_members = await self.config.all_members() + + async for guild_id, guild_data in AsyncIter(all_members.items(), steps=100): + if user_id in guild_data: + await self.config.member_from_ids(guild_id, user_id).clear() + def cog_unload(self): self.register_task.cancel() diff --git a/redbot/cogs/general/general.py b/redbot/cogs/general/general.py index 943660193..951b8bb31 100644 --- a/redbot/cogs/general/general.py +++ b/redbot/cogs/general/general.py @@ -75,6 +75,10 @@ class General(commands.Cog): super().__init__() self.stopwatches = {} + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + @commands.command() async def choose(self, ctx, *choices): """Choose between multiple options. diff --git a/redbot/cogs/image/image.py b/redbot/cogs/image/image.py index f1c30ed76..c5918b273 100644 --- a/redbot/cogs/image/image.py +++ b/redbot/cogs/image/image.py @@ -26,6 +26,10 @@ class Image(commands.Cog): def cog_unload(self): self.session.detach() + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + async def initialize(self) -> None: """Move the API keys from cog stored config to core bot config if they exist.""" imgur_token = await self.config.imgur_client_id() diff --git a/redbot/cogs/mod/mod.py b/redbot/cogs/mod/mod.py index 75995376e..99ab02340 100644 --- a/redbot/cogs/mod/mod.py +++ b/redbot/cogs/mod/mod.py @@ -3,7 +3,7 @@ import logging import re from abc import ABC from collections import defaultdict -from typing import List, Tuple +from typing import List, Tuple, Literal import discord from redbot.core.utils import AsyncIter @@ -83,6 +83,34 @@ class Mod( self._ready = asyncio.Event() + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + all_members = await self.config.all_members() + + async for guild_id, guild_data in AsyncIter(all_members.items(), steps=100): + if user_id in guild_data: + await self.config.member_from_ids(guild_id, user_id).clear() + + await self.config.user_from_id(user_id).clear() + + guild_data = await self.config.all_guilds() + + async for guild_id, guild_data in AsyncIter(guild_data.items(), steps=100): + if user_id in guild_data["current_tempbans"]: + async with self.config.guild_from_id(guild_id).current_tempbans() as tbs: + try: + tbs.remove(user_id) + except ValueError: + pass + # possible with a context switch between here and getting all guilds + async def initialize(self): await self._maybe_update_config() self._ready.set() diff --git a/redbot/cogs/modlog/modlog.py b/redbot/cogs/modlog/modlog.py index bdf3cfffc..0d1979b8e 100644 --- a/redbot/cogs/modlog/modlog.py +++ b/redbot/cogs/modlog/modlog.py @@ -20,6 +20,10 @@ class ModLog(commands.Cog): super().__init__() self.bot = bot + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + @commands.group() @checks.guildowner_or_permissions(administrator=True) async def modlogset(self, ctx: commands.Context): diff --git a/redbot/cogs/permissions/converters.py b/redbot/cogs/permissions/converters.py index ab0257eb9..cd90f5319 100644 --- a/redbot/cogs/permissions/converters.py +++ b/redbot/cogs/permissions/converters.py @@ -138,12 +138,19 @@ class CogOrCommand(NamedTuple): # noinspection PyArgumentList @classmethod async def convert(cls, ctx: commands.Context, arg: str) -> "CogOrCommand": - cog = ctx.bot.get_cog(arg) - if cog: - return cls(type="COG", name=cog.__class__.__name__, obj=cog) - cmd = ctx.bot.get_command(arg) - if cmd: - return cls(type="COMMAND", name=cmd.qualified_name, obj=cmd) + ret = None + if cog := ctx.bot.get_cog(arg): + ret = cls(type="COG", name=cog.qualified_name, obj=cog) + + elif cmd := ctx.bot.get_command(arg): + ret = cls(type="COMMAND", name=cmd.qualified_name, obj=cmd) + + if ret: + if isinstance(ret.obj, commands.commands._RuleDropper): + raise commands.BadArgument( + "You cannot apply permission rules to this cog or command." + ) + return ret raise commands.BadArgument( _( diff --git a/redbot/cogs/permissions/permissions.py b/redbot/cogs/permissions/permissions.py index b93c3ff9e..68a5bd95d 100644 --- a/redbot/cogs/permissions/permissions.py +++ b/redbot/cogs/permissions/permissions.py @@ -2,7 +2,7 @@ import asyncio import io import textwrap from copy import copy -from typing import Union, Optional, Dict, List, Tuple, Any, Iterator, ItemsView, cast +from typing import Union, Optional, Dict, List, Tuple, Any, Iterator, ItemsView, Literal, cast import discord import yaml @@ -10,6 +10,7 @@ from schema import And, Or, Schema, SchemaError, Optional as UseOptional from redbot.core import checks, commands, config from redbot.core.bot import Red from redbot.core.i18n import Translator, cog_i18n +from redbot.core.utils import AsyncIter from redbot.core.utils.chat_formatting import box from redbot.core.utils.menus import start_adding_reactions from redbot.core.utils.predicates import ReactionPredicate, MessagePredicate @@ -114,6 +115,56 @@ class Permissions(commands.Cog): self.config.init_custom(COMMAND, 1) self.config.register_custom(COMMAND) + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + count = 0 + + _uid = str(user_id) + + # The dict as returned here as string keys. Above is for comparison, + # there's a below recast to int where needed for guild ids + + for typename, getter in ((COG, self.bot.get_cog), (COMMAND, self.bot.get_command)): + + obj_type_rules = await self.config.custom(typename).all() + + count += 1 + if not count % 100: + await asyncio.sleep(0) + + for obj_name, rules_dict in obj_type_rules.items(): + + count += 1 + if not count % 100: + await asyncio.sleep(0) + + obj = getter(obj_name) + + for guild_id, guild_rules in rules_dict.items(): + + count += 1 + if not count % 100: + await asyncio.sleep(0) + + if _uid in guild_rules: + if obj: + # delegate to remove rule here + await self._remove_rule( + CogOrCommand(typename, obj.qualified_name, obj), + user_id, + int(guild_id), + ) + else: + grp = self.config.custom(typename, obj_name) + await grp.clear_raw(guild_id, user_id) + async def __permissions_hook(self, ctx: commands.Context) -> Optional[bool]: """ Purpose of this hook is to prevent guild owner lockouts of permissions specifically @@ -345,14 +396,6 @@ class Permissions(commands.Cog): if not who_or_what: await ctx.send_help() return - if isinstance(cog_or_command.obj, commands.commands._AlwaysAvailableCommand): - await ctx.send( - _( - "This command is designated as being always available and " - "cannot be modified by permission rules." - ) - ) - return for w in who_or_what: await self._add_rule( rule=cast(bool, allow_or_deny), @@ -388,14 +431,6 @@ class Permissions(commands.Cog): if not who_or_what: await ctx.send_help() return - if isinstance(cog_or_command.obj, commands.commands._AlwaysAvailableCommand): - await ctx.send( - _( - "This command is designated as being always available and " - "cannot be modified by permission rules." - ) - ) - return for w in who_or_what: await self._add_rule( rule=cast(bool, allow_or_deny), @@ -473,14 +508,6 @@ class Permissions(commands.Cog): `` is the cog or command to set the default rule for. This is case sensitive. """ - if isinstance(cog_or_command.obj, commands.commands._AlwaysAvailableCommand): - await ctx.send( - _( - "This command is designated as being always available and " - "cannot be modified by permission rules." - ) - ) - return await self._set_default_rule( rule=cast(Optional[bool], allow_or_deny), cog_or_cmd=cog_or_command, @@ -504,14 +531,6 @@ class Permissions(commands.Cog): `` is the cog or command to set the default rule for. This is case sensitive. """ - if isinstance(cog_or_command.obj, commands.commands._AlwaysAvailableCommand): - await ctx.send( - _( - "This command is designated as being always available and " - "cannot be modified by permission rules." - ) - ) - return await self._set_default_rule( rule=cast(Optional[bool], allow_or_deny), cog_or_cmd=cog_or_command, guild_id=GLOBAL ) diff --git a/redbot/cogs/reports/reports.py b/redbot/cogs/reports/reports.py index 30cbf5de3..fd2a9f564 100644 --- a/redbot/cogs/reports/reports.py +++ b/redbot/cogs/reports/reports.py @@ -1,6 +1,6 @@ import logging import asyncio -from typing import Union, List +from typing import Union, List, Literal from datetime import timedelta from copy import copy import contextlib @@ -60,6 +60,39 @@ class Reports(commands.Cog): # (guild, ticket#): # {'tun': Tunnel, 'msgs': List[int]} + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + all_reports = await self.config.custom("REPORT").all() + + steps = 0 + paths = [] + + # this doesn't use async iter intentionally due to the nested iterations + for guild_id_str, tickets in all_reports.items(): + for ticket_number, ticket in tickets.items(): + steps += 1 + if not steps % 100: + await asyncio.sleep(0) # yield context + + if ticket.get("report", {}).get("user_id", 0) == user_id: + paths.append((guild_id_str, ticket_number)) + + async with self.config.custom("REPORT").all() as all_reports: + async for guild_id_str, ticket_number in AsyncIter(paths, steps=100): + r = all_reports[guild_id_str][ticket_number]["report"] + r["user_id"] = 0xDE1 + # this might include EUD, and a report of a deleted user + # that's been unhandled for long enough for the + # user to be deleted and the bot recieve a request like this... + r["report"] = "[REPORT DELETED DUE TO DISCORD REQUEST]" + @property def tunnels(self): return [x["tun"] for x in self.tunnel_store.values()] diff --git a/redbot/cogs/streams/streams.py b/redbot/cogs/streams/streams.py index 24bfd142f..cde398c11 100644 --- a/redbot/cogs/streams/streams.py +++ b/redbot/cogs/streams/streams.py @@ -79,6 +79,10 @@ class Streams(commands.Cog): self._ready_event: asyncio.Event = asyncio.Event() self._init_task: asyncio.Task = self.bot.loop.create_task(self.initialize()) + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete """ + return + def check_name_or_id(self, data: str) -> bool: matched = self.yt_cid_pattern.fullmatch(data) if matched is None: diff --git a/redbot/cogs/trivia/trivia.py b/redbot/cogs/trivia/trivia.py index d2bfb251c..b428605fd 100644 --- a/redbot/cogs/trivia/trivia.py +++ b/redbot/cogs/trivia/trivia.py @@ -3,7 +3,7 @@ import asyncio import math import pathlib from collections import Counter -from typing import List +from typing import List, Literal import io import yaml @@ -13,6 +13,7 @@ from redbot.core import Config, commands, checks from redbot.cogs.bank import is_owner_if_bank_global from redbot.core.data_manager import cog_data_path from redbot.core.i18n import Translator, cog_i18n +from redbot.core.utils import AsyncIter from redbot.core.utils.chat_formatting import box, pagify, bold from redbot.core.utils.menus import start_adding_reactions from redbot.core.utils.predicates import MessagePredicate, ReactionPredicate @@ -56,6 +57,21 @@ class Trivia(commands.Cog): self.config.register_member(wins=0, games=0, total_score=0) + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + all_members = await self.config.all_members() + + async for guild_id, guild_data in AsyncIter(all_members.items(), steps=100): + if user_id in guild_data: + await self.config.member_from_ids(guild_id, user_id).clear() + @commands.group() @commands.guild_only() @checks.mod_or_permissions(administrator=True) diff --git a/redbot/cogs/warnings/warnings.py b/redbot/cogs/warnings/warnings.py index 93ddbaf75..e23ad6fd6 100644 --- a/redbot/cogs/warnings/warnings.py +++ b/redbot/cogs/warnings/warnings.py @@ -1,7 +1,8 @@ +import asyncio import contextlib from collections import namedtuple from copy import copy -from typing import Union, Optional +from typing import Union, Optional, Literal import discord @@ -14,6 +15,7 @@ from redbot.cogs.warnings.helpers import ( from redbot.core import Config, checks, commands, modlog from redbot.core.bot import Red from redbot.core.i18n import Translator, cog_i18n +from redbot.core.utils import AsyncIter from redbot.core.utils.chat_formatting import warning, pagify from redbot.core.utils.menus import menu, DEFAULT_CONTROLS @@ -45,6 +47,41 @@ class Warnings(commands.Cog): self.bot = bot self.registration_task = self.bot.loop.create_task(self.register_warningtype()) + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + all_members = await self.config.all_members() + + c = 0 + + for guild_id, guild_data in all_members.items(): + c += 1 + if not c % 100: + await asyncio.sleep(0) + + if user_id in guild_data: + await self.config.member_from_ids(guild_id, user_id).clear() + + for remaining_user, user_warns in guild_data.items(): + c += 1 + if not c % 100: + await asyncio.sleep(0) + + for warn_id, warning in user_warns.get("warnings", {}).items(): + c += 1 + if not c % 100: + await asyncio.sleep(0) + + if warning.get("mod", 0) == user_id: + grp = self.config.member_from_ids(guild_id, remaining_user) + await grp.set_raw("warnings", warn_id, "mod", value=0xDE1) + # We're not utilising modlog yet - no need to register a casetype @staticmethod async def register_warningtype(): @@ -489,7 +526,11 @@ class Warnings(commands.Cog): else: for key in user_warnings.keys(): mod_id = user_warnings[key]["mod"] - mod = ctx.bot.get_user(mod_id) or _("Unknown Moderator ({})").format(mod_id) + if mod_id == 0xDE1: + mod = _("Deleted Moderator") + else: + bot = ctx.bot + mod = bot.get_user(mod_id) or _("Unknown Moderator ({})").format(mod_id) msg += _( "{num_points} point warning {reason_name} issued by {user} for " "{description}\n" @@ -519,7 +560,11 @@ class Warnings(commands.Cog): else: for key in user_warnings.keys(): mod_id = user_warnings[key]["mod"] - mod = ctx.bot.get_user(mod_id) or _("Unknown Moderator ({})").format(mod_id) + if mod_id == 0xDE1: + mod = _("Deleted Moderator") + else: + bot = ctx.bot + mod = bot.get_user(mod_id) or _("Unknown Moderator ({})").format(mod_id) msg += _( "{num_points} point warning {reason_name} issued by {user} for " "{description}\n" diff --git a/redbot/core/bank.py b/redbot/core/bank.py index baf0a4c8a..adee5b322 100644 --- a/redbot/core/bank.py +++ b/redbot/core/bank.py @@ -2,17 +2,18 @@ from __future__ import annotations import asyncio import datetime -from typing import Union, List, Optional, TYPE_CHECKING +import logging +from typing import Union, List, Optional, TYPE_CHECKING, Literal from functools import wraps import discord +from redbot.core.utils import AsyncIter from redbot.core.utils.chat_formatting import humanize_number from . import Config, errors, commands from .i18n import Translator from .errors import BankPruneError -from .utils import AsyncIter if TYPE_CHECKING: from .bot import Red @@ -67,6 +68,10 @@ _DEFAULT_USER = _DEFAULT_MEMBER _config: Config = None +log = logging.getLogger("red.core.bank") + +_data_deletion_lock = asyncio.Lock() + def _init(): global _config @@ -77,6 +82,28 @@ def _init(): _config.register_user(**_DEFAULT_USER) +async def _process_data_deletion( + *, requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], user_id: int +): + """ + Bank has no reason to keep any of this data + if the user doesn't want it kept, + we won't special case any request type + """ + if requester not in ("discord_deleted_user", "owner", "user", "user_strict"): + log.warning( + "Got unknown data request type `{req_type}` for user, deleting anyway", + req_type=requester, + ) + + async with _data_deletion_lock: + await _config.user_from_id(user_id).clear() + all_members = await _config.all_members() + async for guild_id, member_dict in AsyncIter(all_members.items(), steps=100): + if user_id in member_dict: + await _config.member_from_ids(guild_id, user_id).clear() + + class Account: """A single account. diff --git a/redbot/core/bot.py b/redbot/core/bot.py index 6b3956247..1a8e9fc28 100644 --- a/redbot/core/bot.py +++ b/redbot/core/bot.py @@ -6,6 +6,8 @@ import platform import shutil import sys import contextlib +import weakref +import functools from collections import namedtuple from datetime import datetime from enum import IntEnum @@ -22,6 +24,8 @@ from typing import ( Callable, Awaitable, Any, + Literal, + MutableMapping, ) from types import MappingProxyType @@ -31,7 +35,7 @@ from discord.ext.commands import when_mentioned_or from . import Config, i18n, commands, errors, drivers, modlog, bank from .cog_manager import CogManager, CogManagerUI -from .core_commands import license_info_command, Core +from .core_commands import Core from .data_manager import cog_data_path from .dev_commands import Dev from .events import init_events @@ -45,7 +49,7 @@ from .settings_caches import ( ) from .rpc import RPCMixin -from .utils import common_filters +from .utils import common_filters, AsyncIter from .utils._internal_utils import send_to_owners_with_prefix_replaced CUSTOM_GROUPS = "CUSTOM_GROUPS" @@ -57,6 +61,8 @@ __all__ = ["RedBase", "Red", "ExitCodes"] NotMessage = namedtuple("NotMessage", "guild") +DataDeletionResults = namedtuple("DataDeletionResults", "failed_modules failed_cogs unhandled") + PreInvokeCoroutine = Callable[[commands.Context], Awaitable[Any]] T_BIC = TypeVar("T_BIC", bound=PreInvokeCoroutine) @@ -117,6 +123,8 @@ class RedBase( last_system_info__machine=None, last_system_info__system=None, schema_version=0, + datarequests__allow_user_requests=True, + datarequests__user_requests_are_strict=True, ) self._config.register_guild( @@ -198,6 +206,8 @@ class RedBase( self._red_ready = asyncio.Event() self._red_before_invoke_objs: Set[PreInvokeCoroutine] = set() + self._deletion_requests: MutableMapping[int, asyncio.Lock] = weakref.WeakValueDictionary() + def get_command(self, name: str) -> Optional[commands.Command]: com = super().get_command(name) assert com is None or isinstance(com, commands.Command) @@ -219,7 +229,7 @@ class RedBase( async def _red_before_invoke_method(self, ctx): await self.wait_until_red_ready() - return_exceptions = isinstance(ctx.command, commands.commands._AlwaysAvailableCommand) + return_exceptions = isinstance(ctx.command, commands.commands._RuleDropper) if self._red_before_invoke_objs: await asyncio.gather( *(coro(ctx) for coro in self._red_before_invoke_objs), @@ -666,7 +676,6 @@ class RedBase( self.add_cog(Core(self)) self.add_cog(CogManagerUI()) - self.add_command(license_info_command) if cli_flags.dev: self.add_cog(Dev()) @@ -1040,7 +1049,7 @@ class RedBase( def remove_cog(self, cogname: str): cog = self.get_cog(cogname) - if cog is None: + if cog is None or isinstance(cog, commands.commands._RuleDropper): return for cls in inspect.getmro(cog.__class__): @@ -1197,6 +1206,9 @@ class RedBase( subcommand.requires.ready_event.set() def remove_command(self, name: str) -> None: + command = self.get_command(name) + if isinstance(command, commands.commands._RuleDropper): + return command = super().remove_command(name) if not command: return @@ -1395,6 +1407,124 @@ class RedBase( await self.logout() sys.exit(self._shutdown_mode) + async def _core_data_deletion( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + if requester != "discord_deleted_user": + return + + await self._config.user_from_id(user_id).clear() + all_guilds = await self._config.all_guilds() + + async for guild_id, guild_data in AsyncIter(all_guilds.items(), steps=100): + if user_id in guild_data.get("autoimmune_ids", []): + async with self._config.guild_from_id(guild_id).autoimmune_ids() as ids: + # prevent a racy crash here without locking + # up the vals in all guilds first + with contextlib.suppress(ValueError): + ids.remove(user_id) + + await self._whiteblacklist_cache.discord_deleted_user(user_id) + + async def handle_data_deletion_request( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ) -> DataDeletionResults: + """ + This tells each cog and extension, as well as any APIs in Red + to go delete data + + Calling this should be limited to interfaces designed for it. + + See ``redbot.core.commands.Cog.delete_data_for_user`` + for details about the parameters and intent. + + Parameters + ---------- + requester + user_id + + Returns + ------- + DataDeletionResults + A named tuple ``(failed_modules, failed_cogs, unhandled)`` + containing lists with names of failed modules, failed cogs, + and cogs that didn't handle data deletion request. + """ + await self.wait_until_red_ready() + lock = self._deletion_requests.setdefault(user_id, asyncio.Lock()) + async with lock: + return await self._handle_data_deletion_request(requester=requester, user_id=user_id) + + async def _handle_data_deletion_request( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ) -> DataDeletionResults: + """ + Actual interface for the above. + + Parameters + ---------- + requester + user_id + + Returns + ------- + DataDeletionResults + """ + extension_handlers = { + extension_name: handler + for extension_name, extension in self.extensions.items() + if (handler := getattr(extension, "red_delete_data_for_user", None)) + } + + cog_handlers = { + cog_qualname: cog.red_delete_data_for_user for cog_qualname, cog in self.cogs.items() + } + + special_handlers = { + "Red Core Modlog API": modlog._process_data_deletion, + "Red Core Bank API": bank._process_data_deletion, + "Red Core Bot Data": self._core_data_deletion, + } + + failures = { + "extension": [], + "cog": [], + "unhandled": [], + } + + async def wrapper(func, stype, sname): + try: + await func(requester=requester, user_id=user_id) + except commands.commands.RedUnhandledAPI: + log.warning(f"{stype}.{sname} did not handle data deletion ") + failures["unhandled"].append(sname) + except Exception as exc: + log.exception(f"{stype}.{sname} errored when handling data deletion ") + failures[stype].append(sname) + + handlers = [ + *(wrapper(coro, "extension", name) for name, coro in extension_handlers.items()), + *(wrapper(coro, "cog", name) for name, coro in cog_handlers.items()), + *(wrapper(coro, "extension", name) for name, coro in special_handlers.items()), + ] + + await asyncio.gather(*handlers) + + return DataDeletionResults( + failed_modules=failures["extension"], + failed_cogs=failures["cog"], + unhandled=failures["unhandled"], + ) + # This can be removed, and the parent class renamed as a breaking change class Red(RedBase): diff --git a/redbot/core/cog_manager.py b/redbot/core/cog_manager.py index a93bc7d5a..4929ad691 100644 --- a/redbot/core/cog_manager.py +++ b/redbot/core/cog_manager.py @@ -311,6 +311,10 @@ _ = Translator("CogManagerUI", __file__) class CogManagerUI(commands.Cog): """Commands to interface with Red's cog manager.""" + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete (Core Config is handled in a bot method ) """ + return + @commands.command() @checks.is_owner() async def paths(self, ctx: commands.Context): diff --git a/redbot/core/commands/__init__.py b/redbot/core/commands/__init__.py index 51a10932c..cfaa2669b 100644 --- a/redbot/core/commands/__init__.py +++ b/redbot/core/commands/__init__.py @@ -15,6 +15,7 @@ from .commands import ( GroupMixin as GroupMixin, command as command, group as group, + RedUnhandledAPI as RedUnhandledAPI, RESERVED_COMMAND_NAMES as RESERVED_COMMAND_NAMES, ) from .context import Context as Context, GuildContext as GuildContext, DMContext as DMContext diff --git a/redbot/core/commands/commands.py b/redbot/core/commands/commands.py index e2215958f..54283e732 100644 --- a/redbot/core/commands/commands.py +++ b/redbot/core/commands/commands.py @@ -6,19 +6,23 @@ be used instead of those from the `discord.ext.commands` module. from __future__ import annotations import inspect +import io import re import functools import weakref from typing import ( + Any, Awaitable, Callable, Dict, List, + Literal, Optional, Tuple, Union, MutableMapping, TYPE_CHECKING, + cast, ) import discord @@ -55,6 +59,7 @@ __all__ = [ "command", "group", "RESERVED_COMMAND_NAMES", + "RedUnhandledAPI", ] #: The following names are reserved for various reasons @@ -66,6 +71,12 @@ _ = Translator("commands.commands", __file__) DisablerDictType = MutableMapping[discord.Guild, Callable[["Context"], Awaitable[bool]]] +class RedUnhandledAPI(Exception): + """ An exception which can be raised to signal a lack of handling specific APIs """ + + pass + + class CogCommandMixin: """A mixin for cogs and commands.""" @@ -731,6 +742,7 @@ class CogGroupMixin: whether or not the rule was changed as a result of this call. + :meta private: """ cur_rule = self.requires.get_rule(model_id, guild_id=guild_id) if cur_rule not in (PermState.NORMAL, PermState.ACTIVE_ALLOW, PermState.ACTIVE_DENY): @@ -809,6 +821,136 @@ class CogMixin(CogGroupMixin, CogCommandMixin): if doc: return inspect.cleandoc(translator(doc)) + async def red_get_data_for_user(self, *, user_id: int) -> MutableMapping[str, io.BytesIO]: + """ + + .. note:: + + This method is documented provisionally + and may have minor changes made to it. + It is not expected to undergo major changes, + but nothing utilizes this method yet and the inclusion of this method + in documentation in advance is solely to allow cog creators time to prepare. + + + This should be overridden by all cogs. + + Overridden implementations should return a mapping of filenames to io.BytesIO + containing a human-readable version of the data + the cog has about the specified user_id or an empty mapping + if the cog does not have end user data. + + The data should be easily understood for what it represents to + most users of age to use Discord. + + You may want to include a readme file + which explains specifics about the data. + + This method may also be implemented for an extension. + + Parameters + ---------- + user_id: int + + Returns + ------- + MutableMapping[str, io.BytesIO] + A mapping of filenames to BytesIO objects + suitable to send as a files or as part of an archive to a user. + + This may be empty if you don't have data for users. + + Raises + ------ + RedUnhandledAPI + If the method was not overriden, + or an overriden implementation is not handling this + + """ + raise RedUnhandledAPI() + + async def red_delete_data_for_user( + self, + *, + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], + user_id: int, + ): + """ + This should be overridden by all cogs. + + If your cog does not store data, overriding and doing nothing should still + be done to indicate that this has been considered. + + .. note:: + This may receive other strings in the future without warning + you should safely handle + any string value (log a warning if needed) + as additional requester types may be added + in the future without prior warning. + (see what this method can raise for details) + + + This method can currently be passed one of these strings: + + + - ``"discord_deleted_user"``: + + The request should be processed as if + Discord has asked for the data removal + This then additionally must treat the + user ID itself as something to be deleted. + The user ID is no longer operational data + as the ID no longer refers to a valid user. + + - ``"owner"``: + + The request was made by the bot owner. + If removing the data requested by the owner + would be an operational hazard + (such as removing a user id from a blocked user list) + you may elect to inform the user of an alternative way + to remove that ID to ensure the process can not be abused + by users to bypass anti-abuse measures, + but there must remain a way for them to process this request. + + - ``"user_strict"``: + + The request was made by a user, + the bot settings allow a user to request their own data + be deleted, and the bot is configured to respect this + at the cost of functionality. + Cogs may retain data needed for anti abuse measures + such as IDs and timestamps of interactions, + but should not keep EUD such + as user nicknames if receiving a request of this nature. + + - ``"user"``: + + The request was made by a user, + the bot settings allow a user to request their own data + be deleted, and the bot is configured to let cogs keep + data needed for operation. + Under this case, you may elect to retain data which is + essential to the functionality of the cog. This case will + only happen if the bot owner has opted into keeping + minimal EUD needed for cog functionality. + + + Parameters + ---------- + requester: Literal["discord_deleted_user", "owner", "user", "user_strict"] + See above notes for details about this parameter + user_id: int + The user ID which needs deletion handling + + Raises + ------ + RedUnhandledAPI + If the method was not overriden, + or an overriden implementation is not handling this + """ + raise RedUnhandledAPI() + async def can_run(self, ctx: "Context", **kwargs) -> bool: """ This really just exists to allow easy use with other methods using can_run @@ -826,6 +968,8 @@ class CogMixin(CogGroupMixin, CogCommandMixin): ------- bool ``True`` if this cog is usable in the given context. + + :meta private: """ try: @@ -854,6 +998,7 @@ class CogMixin(CogGroupMixin, CogCommandMixin): bool ``True`` if this cog is visible in the given context. + :meta private: """ return await self.can_run(ctx) @@ -873,6 +1018,8 @@ class Cog(CogMixin, DPYCog, metaclass=DPYCogMeta): """ This does not have identical behavior to Group.all_commands but should return what you expect + + :meta private: """ return {cmd.name: cmd for cmd in self.__cog_commands__} @@ -917,13 +1064,15 @@ def get_command_disabler(guild: discord.Guild) -> Callable[["Context"], Awaitabl return disabler -# This is intentionally left out of `__all__` as it is not intended for general use -class _AlwaysAvailableCommand(Command): +# The below are intentionally left out of `__all__` +# as they are not intended for general use +class _AlwaysAvailableMixin: """ - This should be used only for informational commands + This should be used for commands which should not be disabled or removed - These commands cannot belong to a cog. + These commands cannot belong to any cog except Core (core_commands.py) + to prevent issues with the appearance of certain behavior. These commands do not respect most forms of checks, and should only be used with that in mind. @@ -931,10 +1080,56 @@ class _AlwaysAvailableCommand(Command): This particular class is not supported for 3rd party use """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if self.cog is not None: - raise TypeError("This command may not be added to a cog") - async def can_run(self, ctx, *args, **kwargs) -> bool: return not ctx.author.bot + + can_see = can_run + + +class _RuleDropper(CogCommandMixin): + """ + Objects inheriting from this, be they command or cog, + should not be interfered with operation except by their own rules, + or by global checks which are not tailored for these objects but instead + on global abuse prevention + (such as a check that disallows blocked users and bots from interacting.) + + This should not be used by 3rd-party extensions directly for their own objects. + """ + + def allow_for(self, model_id: Union[int, str], guild_id: int) -> None: + """ This will do nothing. """ + + def deny_to(self, model_id: Union[int, str], guild_id: int) -> None: + """ This will do nothing. """ + + def clear_rule_for( + self, model_id: Union[int, str], guild_id: int + ) -> Tuple[PermState, PermState]: + """ + This will do nothing, except return a compatible rule + """ + cur_rule = self.requires.get_rule(model_id, guild_id=guild_id) + return cur_rule, cur_rule + + def set_default_rule(self, rule: Optional[bool], guild_id: int) -> None: + """ This will do nothing. """ + + +class _AlwaysAvailableCommand(_AlwaysAvailableMixin, _RuleDropper, Command): + pass + + +class _AlwaysAvailableGroup(_AlwaysAvailableMixin, _RuleDropper, Group): + pass + + +class _ForgetMeSpecialCommand(_RuleDropper, Command): + """ + We need special can_run behavior here + """ + + async def can_run(self, ctx, *args, **kwargs) -> bool: + return await ctx.bot._config.datarequests.allow_user_requests() + + can_see = can_run diff --git a/redbot/core/core_commands.py b/redbot/core/core_commands.py index 4cae03451..bb50b8826 100644 --- a/redbot/core/core_commands.py +++ b/redbot/core/core_commands.py @@ -4,6 +4,9 @@ import datetime import importlib import itertools import logging +import io +import random +import markdown import os import re import sys @@ -11,15 +14,12 @@ import platform import getpass import pip import traceback -from collections import namedtuple from pathlib import Path -from random import SystemRandom from string import ascii_letters, digits from typing import TYPE_CHECKING, Union, Tuple, List, Optional, Iterable, Sequence, Dict, Set import aiohttp import discord -import pkg_resources from babel import Locale as BabelLocale, UnknownLocaleError from redbot.core.data_manager import storage_type @@ -29,10 +29,8 @@ from . import ( VersionInfo, checks, commands, - drivers, errors, i18n, - config, ) from .utils import AsyncIter from .utils._internal_utils import fetch_latest_red_version_info @@ -49,6 +47,43 @@ from .utils.chat_formatting import ( from .commands.requires import PrivilegeLevel +_entities = { + "*": "*", + "\\": "\", + "`": "`", + "!": "!", + "{": "{", + "[": "[", + "_": "_", + "(": "(", + "#": "#", + ".": ".", + "+": "+", + "}": "}", + "]": "]", + ")": ")", +} + +PRETTY_HTML_HEAD = """ + + + + +3rd Party Data Statements + +""" # This ends up being a small bit extra that really makes a difference. + +HTML_CLOSING = "" + + +def entity_transformer(statement: str) -> str: + return "".join(_entities.get(c, c) for c in statement) + + if TYPE_CHECKING: from redbot.core.bot import Red @@ -300,9 +335,13 @@ class CoreLogic: @i18n.cog_i18n(_) -class Core(commands.Cog, CoreLogic): +class Core(commands.commands._RuleDropper, commands.Cog, CoreLogic): """Commands related to core functions.""" + async def red_delete_data_for_user(self, **kwargs): + """ Nothing to delete (Core Config is handled in a bot method ) """ + return + @commands.command(hidden=True) async def ping(self, ctx: commands.Context): """Pong.""" @@ -443,6 +482,502 @@ class Core(commands.Cog, CoreLogic): ) ) + @commands.group(cls=commands.commands._AlwaysAvailableGroup) + async def mydata(self, ctx: commands.Context): + """ Commands which interact with the data [botname] has about you """ + + # 1/10 minutes. It's a static response, but the inability to lock + # will annoy people if it's spammable + @commands.cooldown(1, 600, commands.BucketType.user) + @mydata.command(cls=commands.commands._AlwaysAvailableCommand, name="whatdata") + async def mydata_whatdata(self, ctx: commands.Context): + """ Find out what type of data [botname] stores and why """ + + ver = "latest" if red_version_info.dev_release else "stable" + link = f"https://docs.discord.red/en/{ver}/red_core_data_statement.html" + await ctx.send( + _( + "This bot stores some data about users as necessary to function. " + "This is mostly the ID your user is assigned by Discord, linked to " + "a handful of things depending on what you interact with in the bot. " + "There are a few commands which store it to keep track of who created " + "something. (such as playlists) " + "For full details about this as well as more in depth details of what " + "is stored and why, see {link}.\n\n" + "Additionally, 3rd party addons loaded by the bot's owner may or " + "may not store additional things. " + "You can use `{prefix}mydata 3rdparty` " + "to view the statements provided by each 3rd-party addition." + ).format(link=link, prefix=ctx.clean_prefix) + ) + + # 1/30 minutes. It's not likely to change much and uploads a standalone webpage. + @commands.cooldown(1, 1800, commands.BucketType.user) + @mydata.command(cls=commands.commands._AlwaysAvailableCommand, name="3rdparty") + async def mydata_3rd_party(self, ctx: commands.Context): + """ View the End User Data statements of each 3rd-party module. """ + + # Can't check this as a command check, and want to prompt DMs as an option. + if not ctx.channel.permissions_for(ctx.me).attach_files: + ctx.command.reset_cooldown(ctx) + return await ctx.send(_("I need to be able to attach files (try in DMs?)")) + + statements = { + ext_name: getattr(ext, "__red_end_user_data_statement__", None) + for ext_name, ext in ctx.bot.extensions.items() + if not (ext.__package__ and ext.__package__.startswith("redbot.")) + } + + if not statements: + return await ctx.send( + _("This instance does not appear to have any 3rd-party extensions loaded.") + ) + + parts = [] + + formatted_statements = [] + + no_statements = [] + + for ext_name, statement in sorted(statements.items()): + if not statement: + no_statements.append(ext_name) + else: + formatted_statements.append( + f"### {entity_transformer(ext_name)}\n\n{entity_transformer(statement)}" + ) + + if formatted_statements: + parts.append( + "## " + + _("3rd party End User Data statements") + + "\n\n" + + _("The following are statements provided by 3rd-party extensions.") + ) + parts.extend(formatted_statements) + + if no_statements: + parts.append("## " + _("3rd-party extensions without statements\n")) + for ext in no_statements: + parts.append(f"\n - {entity_transformer(ext)}") + + generated = markdown.markdown("\n".join(parts), output_format="html") + + html = "\n".join((PRETTY_HTML_HEAD, generated, HTML_CLOSING)) + + fp = io.BytesIO(html.encode()) + + await ctx.send( + _("Here's a generated page with the statements provided by 3rd-party extensions"), + file=discord.File(fp, filename="3rd-party.html"), + ) + + async def get_serious_confirmation(self, ctx: commands.Context, prompt: str) -> bool: + + confirm_token = "".join(random.choices((*ascii_letters, *digits), k=8)) + + await ctx.send(f"{prompt}\n\n{confirm_token}") + try: + message = await ctx.bot.wait_for( + "message", + check=lambda m: m.channel.id == ctx.channel.id and m.author.id == ctx.author.id, + timeout=30, + ) + except asyncio.TimeoutError: + await ctx.send(_("Did not get confirmation, cancelling.")) + else: + if message.content.strip() == confirm_token: + return True + else: + await ctx.send(_("Did not get a matching confirmation, cancelling.")) + + return False + + # 1 per day, not stored to config to avoid this being more stored data. + # large bots shouldn't be restarting so often that this is an issue, + # and small bots that do restart often don't have enough + # users for this to be an issue. + @commands.cooldown(1, 86400, commands.BucketType.user) + @mydata.command(cls=commands.commands._ForgetMeSpecialCommand, name="forgetme") + async def mydata_forgetme(self, ctx: commands.Context): + """ + Have [botname] forget what it knows about you. + + This may not remove all data about you, data needed for operation, + such as command cooldowns will be kept until no longer necessary. + + Further interactions with [botname] may cause it to learn about you again. + """ + if ctx.assume_yes: + # lol, no, we're not letting users schedule deletions every day to thrash the bot. + ctx.command.reset_cooldown(ctx) # We will however not let that lock them out either. + return await ctx.send( + _("This command ({command}) does not support non-interactive usage").format( + command=ctx.command.qualified_name + ) + ) + + if not await self.get_serious_confirmation( + ctx, + _( + "This will cause the bot to get rid of and/or disassociate " + "data from you. It will not get rid of operational data such " + "as modlog entries, warnings, or mutes. " + "If you are sure this is what you want, " + "please respond with the following:" + ), + ): + ctx.command.reset_cooldown(ctx) + return + await ctx.send(_("This may take some time")) + + if await ctx.bot._config.datarequests.user_requests_are_strict(): + requester = "user_strict" + else: + requester = "user" + + results = await self.bot.handle_data_deletion_request( + requester=requester, user_id=ctx.author.id + ) + + if results.failed_cogs and results.failed_modules: + await ctx.send( + _( + "I tried to delete all non-operational data about you " + "(that I know how to delete) " + "{mention}, however the following modules errored: {modules}. " + "Additionally, the following cogs errored: {cogs}\n" + "Please contact the owner of this bot to address this.\n" + "Note: Outside of these failures, data should have been deleted." + ).format( + mention=ctx.author.mention, + cogs=humanize_list(results.failed_cogs), + modules=humanize_list(results.failed_modules), + ) + ) + elif results.failed_cogs: + await ctx.send( + _( + "I tried to delete all non-operational data about you " + "(that I know how to delete) " + "{mention}, however the following cogs errored: {cogs}.\n" + "Please contact the owner of this bot to address this.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(mention=ctx.author.mention, cogs=humanize_list(results.failed_cogs)) + ) + elif results.failed_modules: + await ctx.send( + _( + "I tried to delete all non-operational data about you " + "(that I know how to delete) " + "{mention}, however the following modules errored: {modules}.\n" + "Please contact the owner of this bot to address this.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(mention=ctx.author.mention, modules=humanize_list(results.failed_modules)) + ) + else: + await ctx.send( + _( + "I've deleted any non-operational data about you " + "(that I know how to delete) {mention}" + ).format(mention=ctx.author.mention) + ) + + if results.unhandled: + await ctx.send( + _("{mention} The following cogs did not handle deletion:\n{cogs}").format( + mention=ctx.author.mention, cogs=humanize_list(results.unhandled) + ) + ) + + # The cooldown of this should be longer once actually implemented + # This is a couple hours, and lets people occasionally check status, I guess. + @commands.cooldown(1, 7200, commands.BucketType.user) + @mydata.command(cls=commands.commands._AlwaysAvailableCommand, name="getmydata") + async def mydata_getdata(self, ctx: commands.Context): + """ [Coming Soon] Get what data [botname] has about you. """ + await ctx.send( + _( + "This command doesn't do anything yet, " + "but we're working on adding support for this." + ) + ) + + @checks.is_owner() + @mydata.group(name="ownermanagement") + async def mydata_owner_management(self, ctx: commands.Context): + """ + Commands for more complete data handling. + """ + + @mydata_owner_management.command(name="allowuserdeletions") + async def mydata_owner_allow_user_deletions(self, ctx): + """ + Set the bot to allow users to request a data deletion. + + This is on by default. + """ + await ctx.bot._config.datarequests.allow_user_requests.set(True) + await ctx.send( + _( + "User can delete their own data. " + "This will not include operational data such as blocked users." + ) + ) + + @mydata_owner_management.command(name="disallowuserdeletions") + async def mydata_owner_disallow_user_deletions(self, ctx): + """ + Set the bot to not allow users to request a data deletion. + """ + await ctx.bot._config.datarequests.allow_user_requests.set(False) + await ctx.send(_("User can not delete their own data.")) + + @mydata_owner_management.command(name="setuserdeletionlevel") + async def mydata_owner_user_deletion_level(self, ctx, level: int): + """ + Sets how user deletions are treated. + + Level: + 0: What users can delete is left entirely up to each cog. + 1: Cogs should delete anything the cog doesn't need about the user. + """ + + if level == 1: + await ctx.bot._config.datarequests.user_requests_are_strict.set(True) + await ctx.send( + _( + "Cogs will be instructed to remove all non operational " + "data upon a user request." + ) + ) + elif level == 0: + await ctx.bot._config.datarequests.user_requests_are_strict.set(False) + await ctx.send( + _( + "Cogs will be informed a user has made a data deletion request, " + "and the details of what to delete will be left to the " + "discretion of the cog author." + ) + ) + else: + await ctx.send_help() + + @mydata_owner_management.command(name="processdiscordrequest") + async def mydata_discord_deletion_request(self, ctx, user_id: int): + """ + Handle a deletion request from discord. + """ + + if not await self.get_serious_confirmation( + ctx, + _( + "This will cause the bot to get rid of or disassociate all data " + "from the specified user ID. You should not use this unless " + "Discord has specifically requested this with regard to a deleted user. " + "This will remove the user from various anti-abuse measures. " + "If you are processing a manual request from a user, you may want " + "`{prefix}{command_name}` instead" + "\n\nIf you are sure this is what you intend to do " + "please respond with the following:" + ).format(prefix=ctx.clean_prefix, command_name="mydata ownermanagement deleteforuser"), + ): + return + results = await self.bot.handle_data_deletion_request( + requester="discord_deleted_user", user_id=user_id + ) + + if results.failed_cogs and results.failed_modules: + await ctx.send( + _( + "I tried to delete all data about that user, " + "(that I know how to delete) " + "however the following modules errored: {modules}. " + "Additionally, the following cogs errored: {cogs}\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format( + cogs=humanize_list(results.failed_cogs), + modules=humanize_list(results.failed_modules), + ) + ) + elif results.failed_cogs: + await ctx.send( + _( + "I tried to delete all data about that user, " + "(that I know how to delete) " + "however the following cogs errored: {cogs}.\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(cogs=humanize_list(results.failed_cogs)) + ) + elif results.failed_modules: + await ctx.send( + _( + "I tried to delete all data about that user, " + "(that I know how to delete) " + "however the following modules errored: {modules}.\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(modules=humanize_list(results.failed_modules)) + ) + else: + await ctx.send(_("I've deleted all data about that user that I know how to delete.")) + + if results.unhandled: + await ctx.send( + _("{mention} The following cogs did not handle deletion:\n{cogs}").format( + mention=ctx.author.mention, cogs=humanize_list(results.unhandled) + ) + ) + + @mydata_owner_management.command(name="deleteforuser") + async def mydata_user_deletion_request_by_owner(self, ctx, user_id: int): + """ Delete data [botname] has about a user for a user. """ + if not await self.get_serious_confirmation( + ctx, + _( + "This will cause the bot to get rid of or disassociate " + "a lot of non-operational data from the " + "specified user. Users have access to " + "different command for this unless they can't interact with the bot at all. " + "This is a mostly safe operation, but you should not use it " + "unless processing a request from this " + "user as it may impact their usage of the bot. " + "\n\nIf you are sure this is what you intend to do " + "please respond with the following:" + ), + ): + return + + if await ctx.bot._config.datarequests.user_requests_are_strict(): + requester = "user_strict" + else: + requester = "user" + + results = await self.bot.handle_data_deletion_request(requester=requester, user_id=user_id) + + if results.failed_cogs and results.failed_modules: + await ctx.send( + _( + "I tried to delete all non-operational data about that user, " + "(that I know how to delete) " + "however the following modules errored: {modules}. " + "Additionally, the following cogs errored: {cogs}\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format( + cogs=humanize_list(results.failed_cogs), + modules=humanize_list(results.failed_modules), + ) + ) + elif results.failed_cogs: + await ctx.send( + _( + "I tried to delete all non-operational data about that user, " + "(that I know how to delete) " + "however the following cogs errored: {cogs}.\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(cogs=humanize_list(results.failed_cogs)) + ) + elif results.failed_modules: + await ctx.send( + _( + "I tried to delete all non-operational data about that user, " + "(that I know how to delete) " + "however the following modules errored: {modules}.\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(modules=humanize_list(results.failed_modules)) + ) + else: + await ctx.send( + _( + "I've deleted all non-operational data about that user " + "that I know how to delete." + ) + ) + + if results.unhandled: + await ctx.send( + _("{mention} The following cogs did not handle deletion:\n{cogs}").format( + mention=ctx.author.mention, cogs=humanize_list(results.unhandled) + ) + ) + + @mydata_owner_management.command(name="deleteuserasowner") + async def mydata_user_deletion_by_owner(self, ctx, user_id: int): + """ Delete data [botname] has about a user. """ + if not await self.get_serious_confirmation( + ctx, + _( + "This will cause the bot to get rid of or disassociate " + "a lot of data about the specified user. " + "This may include more than just end user data, including " + "anti abuse records." + "\n\nIf you are sure this is what you intend to do " + "please respond with the following:" + ), + ): + return + results = await self.bot.handle_data_deletion_request(requester="owner", user_id=user_id) + + if results.failed_cogs and results.failed_modules: + await ctx.send( + _( + "I tried to delete all data about that user, " + "(that I know how to delete) " + "however the following modules errored: {modules}. " + "Additionally, the following cogs errored: {cogs}\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format( + cogs=humanize_list(results.failed_cogs), + modules=humanize_list(results.failed_modules), + ) + ) + elif results.failed_cogs: + await ctx.send( + _( + "I tried to delete all data about that user, " + "(that I know how to delete) " + "however the following cogs errored: {cogs}.\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(cogs=humanize_list(results.failed_cogs)) + ) + elif results.failed_modules: + await ctx.send( + _( + "I tried to delete all data about that user, " + "(that I know how to delete) " + "however the following modules errored: {modules}.\n" + "Please check your logs and contact the creators of " + "these cogs and modules.\n" + "Note: Outside of these failures, data should have been deleted." + ).format(modules=humanize_list(results.failed_modules)) + ) + else: + await ctx.send( + _("I've deleted all data about that user " "that I know how to delete.") + ) + + if results.unhandled: + await ctx.send( + _("{mention} The following cogs did not handle deletion:\n{cogs}").format( + mention=ctx.author.mention, cogs=humanize_list(results.unhandled) + ) + ) + @commands.group() async def embedset(self, ctx: commands.Context): """ @@ -2184,7 +2719,7 @@ class Core(commands.Cog, CoreLogic): cog = self.bot.get_cog(cogname) if not cog: return await ctx.send(_("Cog with the given name doesn't exist.")) - if cog == self: + if isinstance(cog, commands.commands._RuleDropper): return await ctx.send(_("You can't disable this cog by default.")) await self.bot._disabled_cog_cache.default_disable(cogname) await ctx.send(_("{cogname} has been set as disabled by default.").format(cogname=cogname)) @@ -2206,7 +2741,7 @@ class Core(commands.Cog, CoreLogic): cog = self.bot.get_cog(cogname) if not cog: return await ctx.send(_("Cog with the given name doesn't exist.")) - if cog == self: + if isinstance(cog, commands.commands._RuleDropper): return await ctx.send(_("You can't disable this cog as you would lock yourself out.")) if await self.bot._disabled_cog_cache.disable_cog_in_guild(cogname, ctx.guild.id): await ctx.send(_("{cogname} has been disabled in this guild.").format(cogname=cogname)) @@ -2328,7 +2863,7 @@ class Core(commands.Cog, CoreLogic): ) return - if isinstance(command_obj, commands.commands._AlwaysAvailableCommand): + if isinstance(command_obj, commands.commands._RuleDropper): await ctx.send( _("This command is designated as being always available and cannot be disabled.") ) @@ -2362,7 +2897,7 @@ class Core(commands.Cog, CoreLogic): ) return - if isinstance(command_obj, commands.commands._AlwaysAvailableCommand): + if isinstance(command_obj, commands.commands._RuleDropper): await ctx.send( _("This command is designated as being always available and cannot be disabled.") ) @@ -2748,6 +3283,28 @@ class Core(commands.Cog, CoreLogic): ) return msg + # Removing this command from forks is a violation of the GPLv3 under which it is licensed. + # Otherwise interfering with the ability for this command to be accessible is also a violation. + @commands.command( + cls=commands.commands._AlwaysAvailableCommand, + name="licenseinfo", + aliases=["licenceinfo"], + i18n=_, + ) + async def license_info_command(ctx): + """ + Get info about Red's licenses. + """ + + message = ( + "This bot is an instance of Red-DiscordBot (hereafter referred to as Red)\n" + "Red is a free and open source application made available to the public and " + "licensed under the GNU GPLv3. The full text of this license is available to you at " + "" + ) + await ctx.send(message) + # We need a link which contains a thank you to other projects which we use at some point. + # DEP-WARN: CooldownMapping should have a method `from_cooldown` # which accepts (number, number, bucket) @@ -2764,30 +3321,7 @@ class LicenseCooldownMapping(commands.CooldownMapping): return (msg.channel.id, msg.author.id) -# Removing this command from forks is a violation of the GPLv3 under which it is licensed. -# Otherwise interfering with the ability for this command to be accessible is also a violation. -@commands.command( - cls=commands.commands._AlwaysAvailableCommand, - name="licenseinfo", - aliases=["licenceinfo"], - i18n=_, -) -async def license_info_command(ctx): - """ - Get info about Red's licenses. - """ - - message = ( - "This bot is an instance of Red-DiscordBot (hereafter referred to as Red)\n" - "Red is a free and open source application made available to the public and " - "licensed under the GNU GPLv3. The full text of this license is available to you at " - "" - ) - await ctx.send(message) - # We need a link which contains a thank you to other projects which we use at some point. - - # DEP-WARN: command objects should store a single cooldown mapping as `._buckets` -license_info_command._buckets = LicenseCooldownMapping.from_cooldown( +Core.license_info_command._buckets = LicenseCooldownMapping.from_cooldown( 1, 180, commands.BucketType.member # pick a random bucket,it wont get used. ) diff --git a/redbot/core/dev_commands.py b/redbot/core/dev_commands.py index 626b254e4..96ca2845c 100644 --- a/redbot/core/dev_commands.py +++ b/redbot/core/dev_commands.py @@ -33,6 +33,13 @@ START_CODE_BLOCK_RE = re.compile(r"^((```py)(?=\s)|(```))") class Dev(commands.Cog): """Various development focused utilities.""" + async def red_delete_data_for_user(self, **kwargs): + """ + Because despite my best efforts to advise otherwise, + people use ``--dev`` in production + """ + return + def __init__(self): super().__init__() self._last_result = None diff --git a/redbot/core/modlog.py b/redbot/core/modlog.py index 5ccd07472..a25a3f1e6 100644 --- a/redbot/core/modlog.py +++ b/redbot/core/modlog.py @@ -3,12 +3,12 @@ from __future__ import annotations import asyncio import logging from datetime import datetime, timedelta -from typing import List, Union, Optional, cast, TYPE_CHECKING +from typing import List, Literal, Union, Optional, cast, TYPE_CHECKING import discord from redbot.core import Config - +from .utils import AsyncIter from .utils.common_filters import ( filter_invites, filter_mass_mentions, @@ -47,10 +47,41 @@ _CASETYPES = "CASETYPES" _CASES = "CASES" _SCHEMA_VERSION = 4 +_data_deletion_lock = asyncio.Lock() _ = Translator("ModLog", __file__) +async def _process_data_deletion( + *, requester: Literal["discord_deleted_user", "owner", "user", "user_strict"], user_id: int +): + if requester != "discord_deleted_user": + return + + # Oh, how I wish it was as simple as I wanted... + + key_paths = [] + + async with _data_deletion_lock: + all_cases = await _config.custom(_CASES).all() + async for guild_id_str, guild_cases in AsyncIter(all_cases.items(), steps=100): + async for case_num_str, case in AsyncIter(guild_cases.items(), steps=100): + for keyname in ("user", "moderator", "amended_by"): + if (case.get(keyname, 0) or 0) == user_id: # this could be None... + key_paths.append((guild_id_str, case_num_str)) + + async with _config.custom(_CASES).all() as all_cases: + for guild_id_str, case_num_str in key_paths: + case = all_cases[guild_id_str][case_num_str] + if (case.get("user", 0) or 0) == user_id: + case["user"] = 0xDE1 + case.pop("last_known_username", None) + if (case.get("moderator", 0) or 0) == user_id: + case["moderator"] = 0xDE1 + if (case.get("amended_by", 0) or 0) == user_id: + case["amended_by"] = 0xDE1 + + async def _init(bot: Red): global _config global _bot_ref @@ -310,8 +341,11 @@ class Case: moderator = _("Unknown") elif isinstance(self.moderator, int): # can't use _() inside f-string expressions, see bpo-36310 and red#3818 - translated = _("Unknown or Deleted User") - moderator = f"[{translated}] ({self.moderator})" + if self.moderator == 0xDE1: + moderator = _("Deleted User.") + else: + translated = _("Unknown or Deleted User") + moderator = f"[{translated}] ({self.moderator})" else: moderator = escape_spoilers(f"{self.moderator} ({self.moderator.id})") until = None @@ -329,8 +363,11 @@ class Case: amended_by = None elif isinstance(self.amended_by, int): # can't use _() inside f-string expressions, see bpo-36310 and red#3818 - translated = _("Unknown or Deleted User") - amended_by = f"[{translated}] ({self.amended_by})" + if self.amended_by == 0xDE1: + amended_by = _("Deleted User.") + else: + translated = _("Unknown or Deleted User") + amended_by = f"[{translated}] ({self.amended_by})" else: amended_by = escape_spoilers(f"{self.amended_by} ({self.amended_by.id})") @@ -341,7 +378,9 @@ class Case: ) if isinstance(self.user, int): - if self.last_known_username is None: + if self.user == 0xDE1: + user = _("Deleted User.") + elif self.last_known_username is None: # can't use _() inside f-string expressions, see bpo-36310 and red#3818 translated = _("Unknown or Deleted User") user = f"[{translated}] ({self.user})" diff --git a/redbot/core/settings_caches.py b/redbot/core/settings_caches.py index ad87a2fd5..8d5635379 100644 --- a/redbot/core/settings_caches.py +++ b/redbot/core/settings_caches.py @@ -1,12 +1,14 @@ from __future__ import annotations from typing import Dict, List, Optional, Union, Set, Iterable, Tuple +import asyncio from argparse import Namespace from collections import defaultdict import discord from .config import Config +from .utils import AsyncIter class PrefixManager: @@ -125,136 +127,185 @@ class WhitelistBlacklistManager: self._config: Config = config self._cached_whitelist: Dict[Optional[int], Set[int]] = {} self._cached_blacklist: Dict[Optional[int], Set[int]] = {} + # because of discord deletion + # we now have sync and async access that may need to happen at the + # same time. + # blame discord for this. + self._access_lock = asyncio.Lock() + + async def discord_deleted_user(self, user_id: int): + + async with self._access_lock: + + async for guild_id_or_none, ids in AsyncIter( + self._cached_whitelist.items(), steps=100 + ): + ids.discard(user_id) + + async for guild_id_or_none, ids in AsyncIter( + self._cached_blacklist.items(), steps=100 + ): + ids.discard(user_id) + + for grp in (self._config.whitelist, self._config.blacklist): + async with grp() as ul: + try: + ul.remove(user_id) + except ValueError: + pass + + # don't use this in extensions, it's optimized and controlled for here, + # but can't be safe in 3rd party use + + async with self._config._get_base_group("GUILD").all() as abuse: + for guild_str, guild_data in abuse.items(): + for l_name in ("whitelist", "blacklist"): + try: + guild_data[l_name].remove(user_id) + except (ValueError, KeyError): + pass # this is raw access not filled with defaults async def get_whitelist(self, guild: Optional[discord.Guild] = None) -> Set[int]: - ret: Set[int] - - gid: Optional[int] = guild.id if guild else None - - if gid in self._cached_whitelist: - ret = self._cached_whitelist[gid].copy() - else: - if gid is not None: - ret = set(await self._config.guild_from_id(gid).whitelist()) + async with self._access_lock: + ret: Set[int] + gid: Optional[int] = guild.id if guild else None + if gid in self._cached_whitelist: + ret = self._cached_whitelist[gid].copy() else: - ret = set(await self._config.whitelist()) + if gid is not None: + ret = set(await self._config.guild_from_id(gid).whitelist()) + else: + ret = set(await self._config.whitelist()) - self._cached_whitelist[gid] = ret.copy() + self._cached_whitelist[gid] = ret.copy() - return ret + return ret async def add_to_whitelist(self, guild: Optional[discord.Guild], role_or_user: Iterable[int]): - gid: Optional[int] = guild.id if guild else None - role_or_user = role_or_user or [] - if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): - raise TypeError("`role_or_user` must be an iterable of `int`s.") + async with self._access_lock: + gid: Optional[int] = guild.id if guild else None + role_or_user = role_or_user or [] + if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): + raise TypeError("`role_or_user` must be an iterable of `int`s.") - if gid is None: - if gid not in self._cached_whitelist: - self._cached_whitelist[gid] = set(await self._config.whitelist()) - self._cached_whitelist[gid].update(role_or_user) - await self._config.whitelist.set(list(self._cached_whitelist[gid])) + if gid is None: + if gid not in self._cached_whitelist: + self._cached_whitelist[gid] = set(await self._config.whitelist()) + self._cached_whitelist[gid].update(role_or_user) + await self._config.whitelist.set(list(self._cached_whitelist[gid])) - else: - if gid not in self._cached_whitelist: - self._cached_whitelist[gid] = set( - await self._config.guild_from_id(gid).whitelist() + else: + if gid not in self._cached_whitelist: + self._cached_whitelist[gid] = set( + await self._config.guild_from_id(gid).whitelist() + ) + self._cached_whitelist[gid].update(role_or_user) + await self._config.guild_from_id(gid).whitelist.set( + list(self._cached_whitelist[gid]) ) - self._cached_whitelist[gid].update(role_or_user) - await self._config.guild_from_id(gid).whitelist.set(list(self._cached_whitelist[gid])) async def clear_whitelist(self, guild: Optional[discord.Guild] = None): - gid: Optional[int] = guild.id if guild else None - self._cached_whitelist[gid] = set() - if gid is None: - await self._config.whitelist.clear() - else: - await self._config.guild_from_id(gid).whitelist.clear() + async with self._access_lock: + gid: Optional[int] = guild.id if guild else None + self._cached_whitelist[gid] = set() + if gid is None: + await self._config.whitelist.clear() + else: + await self._config.guild_from_id(gid).whitelist.clear() async def remove_from_whitelist( self, guild: Optional[discord.Guild], role_or_user: Iterable[int] ): - gid: Optional[int] = guild.id if guild else None - role_or_user = role_or_user or [] - if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): - raise TypeError("`role_or_user` must be an iterable of `int`s.") + async with self._access_lock: + gid: Optional[int] = guild.id if guild else None + role_or_user = role_or_user or [] + if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): + raise TypeError("`role_or_user` must be an iterable of `int`s.") - if gid is None: - if gid not in self._cached_whitelist: - self._cached_whitelist[gid] = set(await self._config.whitelist()) - self._cached_whitelist[gid].difference_update(role_or_user) - await self._config.whitelist.set(list(self._cached_whitelist[gid])) + if gid is None: + if gid not in self._cached_whitelist: + self._cached_whitelist[gid] = set(await self._config.whitelist()) + self._cached_whitelist[gid].difference_update(role_or_user) + await self._config.whitelist.set(list(self._cached_whitelist[gid])) - else: - if gid not in self._cached_whitelist: - self._cached_whitelist[gid] = set( - await self._config.guild_from_id(gid).whitelist() + else: + if gid not in self._cached_whitelist: + self._cached_whitelist[gid] = set( + await self._config.guild_from_id(gid).whitelist() + ) + self._cached_whitelist[gid].difference_update(role_or_user) + await self._config.guild_from_id(gid).whitelist.set( + list(self._cached_whitelist[gid]) ) - self._cached_whitelist[gid].difference_update(role_or_user) - await self._config.guild_from_id(gid).whitelist.set(list(self._cached_whitelist[gid])) async def get_blacklist(self, guild: Optional[discord.Guild] = None) -> Set[int]: - ret: Set[int] - - gid: Optional[int] = guild.id if guild else None - - if gid in self._cached_blacklist: - ret = self._cached_blacklist[gid].copy() - else: - if gid is not None: - ret = set(await self._config.guild_from_id(gid).blacklist()) + async with self._access_lock: + ret: Set[int] + gid: Optional[int] = guild.id if guild else None + if gid in self._cached_blacklist: + ret = self._cached_blacklist[gid].copy() else: - ret = set(await self._config.blacklist()) + if gid is not None: + ret = set(await self._config.guild_from_id(gid).blacklist()) + else: + ret = set(await self._config.blacklist()) - self._cached_blacklist[gid] = ret.copy() + self._cached_blacklist[gid] = ret.copy() - return ret + return ret async def add_to_blacklist(self, guild: Optional[discord.Guild], role_or_user: Iterable[int]): - gid: Optional[int] = guild.id if guild else None - role_or_user = role_or_user or [] - if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): - raise TypeError("`role_or_user` must be an iterable of `int`s.") - if gid is None: - if gid not in self._cached_blacklist: - self._cached_blacklist[gid] = set(await self._config.blacklist()) - self._cached_blacklist[gid].update(role_or_user) - await self._config.blacklist.set(list(self._cached_blacklist[gid])) - else: - if gid not in self._cached_blacklist: - self._cached_blacklist[gid] = set( - await self._config.guild_from_id(gid).blacklist() + async with self._access_lock: + gid: Optional[int] = guild.id if guild else None + role_or_user = role_or_user or [] + if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): + raise TypeError("`role_or_user` must be an iterable of `int`s.") + if gid is None: + if gid not in self._cached_blacklist: + self._cached_blacklist[gid] = set(await self._config.blacklist()) + self._cached_blacklist[gid].update(role_or_user) + await self._config.blacklist.set(list(self._cached_blacklist[gid])) + else: + if gid not in self._cached_blacklist: + self._cached_blacklist[gid] = set( + await self._config.guild_from_id(gid).blacklist() + ) + self._cached_blacklist[gid].update(role_or_user) + await self._config.guild_from_id(gid).blacklist.set( + list(self._cached_blacklist[gid]) ) - self._cached_blacklist[gid].update(role_or_user) - await self._config.guild_from_id(gid).blacklist.set(list(self._cached_blacklist[gid])) async def clear_blacklist(self, guild: Optional[discord.Guild] = None): - gid: Optional[int] = guild.id if guild else None - self._cached_blacklist[gid] = set() - if gid is None: - await self._config.blacklist.clear() - else: - await self._config.guild_from_id(gid).blacklist.clear() + async with self._access_lock: + gid: Optional[int] = guild.id if guild else None + self._cached_blacklist[gid] = set() + if gid is None: + await self._config.blacklist.clear() + else: + await self._config.guild_from_id(gid).blacklist.clear() async def remove_from_blacklist( self, guild: Optional[discord.Guild], role_or_user: Iterable[int] ): - gid: Optional[int] = guild.id if guild else None - role_or_user = role_or_user or [] - if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): - raise TypeError("`role_or_user` must be an iterable of `int`s.") - if gid is None: - if gid not in self._cached_blacklist: - self._cached_blacklist[gid] = set(await self._config.blacklist()) - self._cached_blacklist[gid].difference_update(role_or_user) - await self._config.blacklist.set(list(self._cached_blacklist[gid])) - else: - if gid not in self._cached_blacklist: - self._cached_blacklist[gid] = set( - await self._config.guild_from_id(gid).blacklist() + async with self._access_lock: + gid: Optional[int] = guild.id if guild else None + role_or_user = role_or_user or [] + if not all(isinstance(r_or_u, int) for r_or_u in role_or_user): + raise TypeError("`role_or_user` must be an iterable of `int`s.") + if gid is None: + if gid not in self._cached_blacklist: + self._cached_blacklist[gid] = set(await self._config.blacklist()) + self._cached_blacklist[gid].difference_update(role_or_user) + await self._config.blacklist.set(list(self._cached_blacklist[gid])) + else: + if gid not in self._cached_blacklist: + self._cached_blacklist[gid] = set( + await self._config.guild_from_id(gid).blacklist() + ) + self._cached_blacklist[gid].difference_update(role_or_user) + await self._config.guild_from_id(gid).blacklist.set( + list(self._cached_blacklist[gid]) ) - self._cached_blacklist[gid].difference_update(role_or_user) - await self._config.guild_from_id(gid).blacklist.set(list(self._cached_blacklist[gid])) class DisabledCogCache: diff --git a/setup.cfg b/setup.cfg index 1b97966f1..aed3fad14 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,7 @@ install_requires = distro==1.5.0; sys_platform == "linux" fuzzywuzzy==0.18.0 idna==2.10 + markdown==3.2.2 multidict==4.7.6 python-Levenshtein-wheels==0.13.1 pytz==2020.1 diff --git a/tools/primary_deps.ini b/tools/primary_deps.ini index bf9aa0bb1..f5ab6da25 100644 --- a/tools/primary_deps.ini +++ b/tools/primary_deps.ini @@ -17,6 +17,7 @@ install_requires = discord.py distro; sys_platform == "linux" fuzzywuzzy + markdown python-Levenshtein-wheels PyYAML Red-Lavalink