Begin work on a data request API (#4045)

[Core] Data Deletion And Disclosure APIs

 - Adds a Data Deletion API
   - Deletion comes in a few forms based on who is requesting
   - Deletion must be handled by 3rd party
 - Adds a Data Collection Disclosure Command
   - Provides a dynamically generated statement from 3rd party
   extensions
 - Modifies the always available commands to be cog compatible
   - Also prevents them from being unloaded accidentally
This commit is contained in:
Michael H
2020-08-03 09:09:07 -04:00
committed by GitHub
parent bb1a256295
commit c0b1e50a5f
38 changed files with 1761 additions and 222 deletions

View File

@@ -4,6 +4,9 @@ import datetime
import importlib
import itertools
import logging
import io
import random
import markdown
import os
import re
import sys
@@ -11,15 +14,12 @@ import platform
import getpass
import pip
import traceback
from collections import namedtuple
from pathlib import Path
from random import SystemRandom
from string import ascii_letters, digits
from typing import TYPE_CHECKING, Union, Tuple, List, Optional, Iterable, Sequence, Dict, Set
import aiohttp
import discord
import pkg_resources
from babel import Locale as BabelLocale, UnknownLocaleError
from redbot.core.data_manager import storage_type
@@ -29,10 +29,8 @@ from . import (
VersionInfo,
checks,
commands,
drivers,
errors,
i18n,
config,
)
from .utils import AsyncIter
from .utils._internal_utils import fetch_latest_red_version_info
@@ -49,6 +47,43 @@ from .utils.chat_formatting import (
from .commands.requires import PrivilegeLevel
_entities = {
"*": "*",
"\\": "\",
"`": "`",
"!": "!",
"{": "{",
"[": "[",
"_": "_",
"(": "(",
"#": "#",
".": ".",
"+": "+",
"}": "}",
"]": "]",
")": ")",
}
PRETTY_HTML_HEAD = """
<!DOCTYPE html>
<html>
<head><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>3rd Party Data Statements</title>
<style type="text/css">
body{margin:2em auto;max-width:800px;line-height:1.4;font-size:16px;
background-color=#EEEEEE;color:#454545;padding:1em;text-align:justify}
h1,h2,h3{line-height:1.2}
</style></head><body>
""" # This ends up being a small bit extra that really makes a difference.
HTML_CLOSING = "</body></html>"
def entity_transformer(statement: str) -> str:
return "".join(_entities.get(c, c) for c in statement)
if TYPE_CHECKING:
from redbot.core.bot import Red
@@ -300,9 +335,13 @@ class CoreLogic:
@i18n.cog_i18n(_)
class Core(commands.Cog, CoreLogic):
class Core(commands.commands._RuleDropper, commands.Cog, CoreLogic):
"""Commands related to core functions."""
async def red_delete_data_for_user(self, **kwargs):
""" Nothing to delete (Core Config is handled in a bot method ) """
return
@commands.command(hidden=True)
async def ping(self, ctx: commands.Context):
"""Pong."""
@@ -443,6 +482,502 @@ class Core(commands.Cog, CoreLogic):
)
)
@commands.group(cls=commands.commands._AlwaysAvailableGroup)
async def mydata(self, ctx: commands.Context):
""" Commands which interact with the data [botname] has about you """
# 1/10 minutes. It's a static response, but the inability to lock
# will annoy people if it's spammable
@commands.cooldown(1, 600, commands.BucketType.user)
@mydata.command(cls=commands.commands._AlwaysAvailableCommand, name="whatdata")
async def mydata_whatdata(self, ctx: commands.Context):
""" Find out what type of data [botname] stores and why """
ver = "latest" if red_version_info.dev_release else "stable"
link = f"https://docs.discord.red/en/{ver}/red_core_data_statement.html"
await ctx.send(
_(
"This bot stores some data about users as necessary to function. "
"This is mostly the ID your user is assigned by Discord, linked to "
"a handful of things depending on what you interact with in the bot. "
"There are a few commands which store it to keep track of who created "
"something. (such as playlists) "
"For full details about this as well as more in depth details of what "
"is stored and why, see {link}.\n\n"
"Additionally, 3rd party addons loaded by the bot's owner may or "
"may not store additional things. "
"You can use `{prefix}mydata 3rdparty` "
"to view the statements provided by each 3rd-party addition."
).format(link=link, prefix=ctx.clean_prefix)
)
# 1/30 minutes. It's not likely to change much and uploads a standalone webpage.
@commands.cooldown(1, 1800, commands.BucketType.user)
@mydata.command(cls=commands.commands._AlwaysAvailableCommand, name="3rdparty")
async def mydata_3rd_party(self, ctx: commands.Context):
""" View the End User Data statements of each 3rd-party module. """
# Can't check this as a command check, and want to prompt DMs as an option.
if not ctx.channel.permissions_for(ctx.me).attach_files:
ctx.command.reset_cooldown(ctx)
return await ctx.send(_("I need to be able to attach files (try in DMs?)"))
statements = {
ext_name: getattr(ext, "__red_end_user_data_statement__", None)
for ext_name, ext in ctx.bot.extensions.items()
if not (ext.__package__ and ext.__package__.startswith("redbot."))
}
if not statements:
return await ctx.send(
_("This instance does not appear to have any 3rd-party extensions loaded.")
)
parts = []
formatted_statements = []
no_statements = []
for ext_name, statement in sorted(statements.items()):
if not statement:
no_statements.append(ext_name)
else:
formatted_statements.append(
f"### {entity_transformer(ext_name)}\n\n{entity_transformer(statement)}"
)
if formatted_statements:
parts.append(
"## "
+ _("3rd party End User Data statements")
+ "\n\n"
+ _("The following are statements provided by 3rd-party extensions.")
)
parts.extend(formatted_statements)
if no_statements:
parts.append("## " + _("3rd-party extensions without statements\n"))
for ext in no_statements:
parts.append(f"\n - {entity_transformer(ext)}")
generated = markdown.markdown("\n".join(parts), output_format="html")
html = "\n".join((PRETTY_HTML_HEAD, generated, HTML_CLOSING))
fp = io.BytesIO(html.encode())
await ctx.send(
_("Here's a generated page with the statements provided by 3rd-party extensions"),
file=discord.File(fp, filename="3rd-party.html"),
)
async def get_serious_confirmation(self, ctx: commands.Context, prompt: str) -> bool:
confirm_token = "".join(random.choices((*ascii_letters, *digits), k=8))
await ctx.send(f"{prompt}\n\n{confirm_token}")
try:
message = await ctx.bot.wait_for(
"message",
check=lambda m: m.channel.id == ctx.channel.id and m.author.id == ctx.author.id,
timeout=30,
)
except asyncio.TimeoutError:
await ctx.send(_("Did not get confirmation, cancelling."))
else:
if message.content.strip() == confirm_token:
return True
else:
await ctx.send(_("Did not get a matching confirmation, cancelling."))
return False
# 1 per day, not stored to config to avoid this being more stored data.
# large bots shouldn't be restarting so often that this is an issue,
# and small bots that do restart often don't have enough
# users for this to be an issue.
@commands.cooldown(1, 86400, commands.BucketType.user)
@mydata.command(cls=commands.commands._ForgetMeSpecialCommand, name="forgetme")
async def mydata_forgetme(self, ctx: commands.Context):
"""
Have [botname] forget what it knows about you.
This may not remove all data about you, data needed for operation,
such as command cooldowns will be kept until no longer necessary.
Further interactions with [botname] may cause it to learn about you again.
"""
if ctx.assume_yes:
# lol, no, we're not letting users schedule deletions every day to thrash the bot.
ctx.command.reset_cooldown(ctx) # We will however not let that lock them out either.
return await ctx.send(
_("This command ({command}) does not support non-interactive usage").format(
command=ctx.command.qualified_name
)
)
if not await self.get_serious_confirmation(
ctx,
_(
"This will cause the bot to get rid of and/or disassociate "
"data from you. It will not get rid of operational data such "
"as modlog entries, warnings, or mutes. "
"If you are sure this is what you want, "
"please respond with the following:"
),
):
ctx.command.reset_cooldown(ctx)
return
await ctx.send(_("This may take some time"))
if await ctx.bot._config.datarequests.user_requests_are_strict():
requester = "user_strict"
else:
requester = "user"
results = await self.bot.handle_data_deletion_request(
requester=requester, user_id=ctx.author.id
)
if results.failed_cogs and results.failed_modules:
await ctx.send(
_(
"I tried to delete all non-operational data about you "
"(that I know how to delete) "
"{mention}, however the following modules errored: {modules}. "
"Additionally, the following cogs errored: {cogs}\n"
"Please contact the owner of this bot to address this.\n"
"Note: Outside of these failures, data should have been deleted."
).format(
mention=ctx.author.mention,
cogs=humanize_list(results.failed_cogs),
modules=humanize_list(results.failed_modules),
)
)
elif results.failed_cogs:
await ctx.send(
_(
"I tried to delete all non-operational data about you "
"(that I know how to delete) "
"{mention}, however the following cogs errored: {cogs}.\n"
"Please contact the owner of this bot to address this.\n"
"Note: Outside of these failures, data should have been deleted."
).format(mention=ctx.author.mention, cogs=humanize_list(results.failed_cogs))
)
elif results.failed_modules:
await ctx.send(
_(
"I tried to delete all non-operational data about you "
"(that I know how to delete) "
"{mention}, however the following modules errored: {modules}.\n"
"Please contact the owner of this bot to address this.\n"
"Note: Outside of these failures, data should have been deleted."
).format(mention=ctx.author.mention, modules=humanize_list(results.failed_modules))
)
else:
await ctx.send(
_(
"I've deleted any non-operational data about you "
"(that I know how to delete) {mention}"
).format(mention=ctx.author.mention)
)
if results.unhandled:
await ctx.send(
_("{mention} The following cogs did not handle deletion:\n{cogs}").format(
mention=ctx.author.mention, cogs=humanize_list(results.unhandled)
)
)
# The cooldown of this should be longer once actually implemented
# This is a couple hours, and lets people occasionally check status, I guess.
@commands.cooldown(1, 7200, commands.BucketType.user)
@mydata.command(cls=commands.commands._AlwaysAvailableCommand, name="getmydata")
async def mydata_getdata(self, ctx: commands.Context):
""" [Coming Soon] Get what data [botname] has about you. """
await ctx.send(
_(
"This command doesn't do anything yet, "
"but we're working on adding support for this."
)
)
@checks.is_owner()
@mydata.group(name="ownermanagement")
async def mydata_owner_management(self, ctx: commands.Context):
"""
Commands for more complete data handling.
"""
@mydata_owner_management.command(name="allowuserdeletions")
async def mydata_owner_allow_user_deletions(self, ctx):
"""
Set the bot to allow users to request a data deletion.
This is on by default.
"""
await ctx.bot._config.datarequests.allow_user_requests.set(True)
await ctx.send(
_(
"User can delete their own data. "
"This will not include operational data such as blocked users."
)
)
@mydata_owner_management.command(name="disallowuserdeletions")
async def mydata_owner_disallow_user_deletions(self, ctx):
"""
Set the bot to not allow users to request a data deletion.
"""
await ctx.bot._config.datarequests.allow_user_requests.set(False)
await ctx.send(_("User can not delete their own data."))
@mydata_owner_management.command(name="setuserdeletionlevel")
async def mydata_owner_user_deletion_level(self, ctx, level: int):
"""
Sets how user deletions are treated.
Level:
0: What users can delete is left entirely up to each cog.
1: Cogs should delete anything the cog doesn't need about the user.
"""
if level == 1:
await ctx.bot._config.datarequests.user_requests_are_strict.set(True)
await ctx.send(
_(
"Cogs will be instructed to remove all non operational "
"data upon a user request."
)
)
elif level == 0:
await ctx.bot._config.datarequests.user_requests_are_strict.set(False)
await ctx.send(
_(
"Cogs will be informed a user has made a data deletion request, "
"and the details of what to delete will be left to the "
"discretion of the cog author."
)
)
else:
await ctx.send_help()
@mydata_owner_management.command(name="processdiscordrequest")
async def mydata_discord_deletion_request(self, ctx, user_id: int):
"""
Handle a deletion request from discord.
"""
if not await self.get_serious_confirmation(
ctx,
_(
"This will cause the bot to get rid of or disassociate all data "
"from the specified user ID. You should not use this unless "
"Discord has specifically requested this with regard to a deleted user. "
"This will remove the user from various anti-abuse measures. "
"If you are processing a manual request from a user, you may want "
"`{prefix}{command_name}` instead"
"\n\nIf you are sure this is what you intend to do "
"please respond with the following:"
).format(prefix=ctx.clean_prefix, command_name="mydata ownermanagement deleteforuser"),
):
return
results = await self.bot.handle_data_deletion_request(
requester="discord_deleted_user", user_id=user_id
)
if results.failed_cogs and results.failed_modules:
await ctx.send(
_(
"I tried to delete all data about that user, "
"(that I know how to delete) "
"however the following modules errored: {modules}. "
"Additionally, the following cogs errored: {cogs}\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(
cogs=humanize_list(results.failed_cogs),
modules=humanize_list(results.failed_modules),
)
)
elif results.failed_cogs:
await ctx.send(
_(
"I tried to delete all data about that user, "
"(that I know how to delete) "
"however the following cogs errored: {cogs}.\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(cogs=humanize_list(results.failed_cogs))
)
elif results.failed_modules:
await ctx.send(
_(
"I tried to delete all data about that user, "
"(that I know how to delete) "
"however the following modules errored: {modules}.\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(modules=humanize_list(results.failed_modules))
)
else:
await ctx.send(_("I've deleted all data about that user that I know how to delete."))
if results.unhandled:
await ctx.send(
_("{mention} The following cogs did not handle deletion:\n{cogs}").format(
mention=ctx.author.mention, cogs=humanize_list(results.unhandled)
)
)
@mydata_owner_management.command(name="deleteforuser")
async def mydata_user_deletion_request_by_owner(self, ctx, user_id: int):
""" Delete data [botname] has about a user for a user. """
if not await self.get_serious_confirmation(
ctx,
_(
"This will cause the bot to get rid of or disassociate "
"a lot of non-operational data from the "
"specified user. Users have access to "
"different command for this unless they can't interact with the bot at all. "
"This is a mostly safe operation, but you should not use it "
"unless processing a request from this "
"user as it may impact their usage of the bot. "
"\n\nIf you are sure this is what you intend to do "
"please respond with the following:"
),
):
return
if await ctx.bot._config.datarequests.user_requests_are_strict():
requester = "user_strict"
else:
requester = "user"
results = await self.bot.handle_data_deletion_request(requester=requester, user_id=user_id)
if results.failed_cogs and results.failed_modules:
await ctx.send(
_(
"I tried to delete all non-operational data about that user, "
"(that I know how to delete) "
"however the following modules errored: {modules}. "
"Additionally, the following cogs errored: {cogs}\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(
cogs=humanize_list(results.failed_cogs),
modules=humanize_list(results.failed_modules),
)
)
elif results.failed_cogs:
await ctx.send(
_(
"I tried to delete all non-operational data about that user, "
"(that I know how to delete) "
"however the following cogs errored: {cogs}.\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(cogs=humanize_list(results.failed_cogs))
)
elif results.failed_modules:
await ctx.send(
_(
"I tried to delete all non-operational data about that user, "
"(that I know how to delete) "
"however the following modules errored: {modules}.\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(modules=humanize_list(results.failed_modules))
)
else:
await ctx.send(
_(
"I've deleted all non-operational data about that user "
"that I know how to delete."
)
)
if results.unhandled:
await ctx.send(
_("{mention} The following cogs did not handle deletion:\n{cogs}").format(
mention=ctx.author.mention, cogs=humanize_list(results.unhandled)
)
)
@mydata_owner_management.command(name="deleteuserasowner")
async def mydata_user_deletion_by_owner(self, ctx, user_id: int):
""" Delete data [botname] has about a user. """
if not await self.get_serious_confirmation(
ctx,
_(
"This will cause the bot to get rid of or disassociate "
"a lot of data about the specified user. "
"This may include more than just end user data, including "
"anti abuse records."
"\n\nIf you are sure this is what you intend to do "
"please respond with the following:"
),
):
return
results = await self.bot.handle_data_deletion_request(requester="owner", user_id=user_id)
if results.failed_cogs and results.failed_modules:
await ctx.send(
_(
"I tried to delete all data about that user, "
"(that I know how to delete) "
"however the following modules errored: {modules}. "
"Additionally, the following cogs errored: {cogs}\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(
cogs=humanize_list(results.failed_cogs),
modules=humanize_list(results.failed_modules),
)
)
elif results.failed_cogs:
await ctx.send(
_(
"I tried to delete all data about that user, "
"(that I know how to delete) "
"however the following cogs errored: {cogs}.\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(cogs=humanize_list(results.failed_cogs))
)
elif results.failed_modules:
await ctx.send(
_(
"I tried to delete all data about that user, "
"(that I know how to delete) "
"however the following modules errored: {modules}.\n"
"Please check your logs and contact the creators of "
"these cogs and modules.\n"
"Note: Outside of these failures, data should have been deleted."
).format(modules=humanize_list(results.failed_modules))
)
else:
await ctx.send(
_("I've deleted all data about that user " "that I know how to delete.")
)
if results.unhandled:
await ctx.send(
_("{mention} The following cogs did not handle deletion:\n{cogs}").format(
mention=ctx.author.mention, cogs=humanize_list(results.unhandled)
)
)
@commands.group()
async def embedset(self, ctx: commands.Context):
"""
@@ -2184,7 +2719,7 @@ class Core(commands.Cog, CoreLogic):
cog = self.bot.get_cog(cogname)
if not cog:
return await ctx.send(_("Cog with the given name doesn't exist."))
if cog == self:
if isinstance(cog, commands.commands._RuleDropper):
return await ctx.send(_("You can't disable this cog by default."))
await self.bot._disabled_cog_cache.default_disable(cogname)
await ctx.send(_("{cogname} has been set as disabled by default.").format(cogname=cogname))
@@ -2206,7 +2741,7 @@ class Core(commands.Cog, CoreLogic):
cog = self.bot.get_cog(cogname)
if not cog:
return await ctx.send(_("Cog with the given name doesn't exist."))
if cog == self:
if isinstance(cog, commands.commands._RuleDropper):
return await ctx.send(_("You can't disable this cog as you would lock yourself out."))
if await self.bot._disabled_cog_cache.disable_cog_in_guild(cogname, ctx.guild.id):
await ctx.send(_("{cogname} has been disabled in this guild.").format(cogname=cogname))
@@ -2328,7 +2863,7 @@ class Core(commands.Cog, CoreLogic):
)
return
if isinstance(command_obj, commands.commands._AlwaysAvailableCommand):
if isinstance(command_obj, commands.commands._RuleDropper):
await ctx.send(
_("This command is designated as being always available and cannot be disabled.")
)
@@ -2362,7 +2897,7 @@ class Core(commands.Cog, CoreLogic):
)
return
if isinstance(command_obj, commands.commands._AlwaysAvailableCommand):
if isinstance(command_obj, commands.commands._RuleDropper):
await ctx.send(
_("This command is designated as being always available and cannot be disabled.")
)
@@ -2748,6 +3283,28 @@ class Core(commands.Cog, CoreLogic):
)
return msg
# Removing this command from forks is a violation of the GPLv3 under which it is licensed.
# Otherwise interfering with the ability for this command to be accessible is also a violation.
@commands.command(
cls=commands.commands._AlwaysAvailableCommand,
name="licenseinfo",
aliases=["licenceinfo"],
i18n=_,
)
async def license_info_command(ctx):
"""
Get info about Red's licenses.
"""
message = (
"This bot is an instance of Red-DiscordBot (hereafter referred to as Red)\n"
"Red is a free and open source application made available to the public and "
"licensed under the GNU GPLv3. The full text of this license is available to you at "
"<https://github.com/Cog-Creators/Red-DiscordBot/blob/V3/develop/LICENSE>"
)
await ctx.send(message)
# We need a link which contains a thank you to other projects which we use at some point.
# DEP-WARN: CooldownMapping should have a method `from_cooldown`
# which accepts (number, number, bucket)
@@ -2764,30 +3321,7 @@ class LicenseCooldownMapping(commands.CooldownMapping):
return (msg.channel.id, msg.author.id)
# Removing this command from forks is a violation of the GPLv3 under which it is licensed.
# Otherwise interfering with the ability for this command to be accessible is also a violation.
@commands.command(
cls=commands.commands._AlwaysAvailableCommand,
name="licenseinfo",
aliases=["licenceinfo"],
i18n=_,
)
async def license_info_command(ctx):
"""
Get info about Red's licenses.
"""
message = (
"This bot is an instance of Red-DiscordBot (hereafter referred to as Red)\n"
"Red is a free and open source application made available to the public and "
"licensed under the GNU GPLv3. The full text of this license is available to you at "
"<https://github.com/Cog-Creators/Red-DiscordBot/blob/V3/develop/LICENSE>"
)
await ctx.send(message)
# We need a link which contains a thank you to other projects which we use at some point.
# DEP-WARN: command objects should store a single cooldown mapping as `._buckets`
license_info_command._buckets = LicenseCooldownMapping.from_cooldown(
Core.license_info_command._buckets = LicenseCooldownMapping.from_cooldown(
1, 180, commands.BucketType.member # pick a random bucket,it wont get used.
)