Merge remote-tracking branch 'release/V3/develop' into V3/develop

Pipfile

@@ -4,8 +4,8 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-red-discordbot = {path = ".",editable = true,extras = ['mongo', 'voice']}
+red-discordbot = {path = ".",editable = true,extras = ['mongo']}
 
 [dev-packages]
 tox = "*"
-red-discordbot = {path = ".",editable = true,extras = ['docs', 'test', 'style']}
+red-discordbot = {path = ".",editable = true,extras = ['docs', 'test', 'style', 'mongo']}

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b9f385e4c53c659dd76e8722d1fb69c244d3a76e4b0dfc40956ff2493277c1f6"
+            "sha256": "d71d118bb7fd8ed744bd9f98d3b9f22ccb589d1c45cd92ea2cbd721446fe6002"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -97,6 +97,13 @@
             ],
             "version": "==1.0.1"
         },
+        "distro": {
+            "hashes": [
+                "sha256:362dde65d846d23baee4b5c058c8586f219b5a54be1cf5fc6ff55c4578392f57",
+                "sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4"
+            ],
+            "version": "==1.4.0"
+        },
         "dnspython": {
             "hashes": [
                 "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
@@ -261,17 +268,16 @@
         "red-discordbot": {
             "editable": true,
             "extras": [
-                "mongo",
+                "mongo"
-                "voice"
             ],
             "path": "."
         },
         "red-lavalink": {
             "hashes": [
-                "sha256:13e1a3f91b990be9582cba039d9a32ec4cef760da1e7e6952143116ec83d4302",
+                "sha256:2a2f469c1feb72c2604795053a8823757ace85ed752eaf573c1d0daba29d1180",
-                "sha256:3dd0d73b4a908bbe9cfb703d2563dad1d1a58f8eea5896a0dacdf37d54a39d9c"
+                "sha256:4bc685a5d89660875d07f50060bacc820e69a763a581ce69375c792e16df4081"
             ],
-            "version": "==0.2.3"
+            "version": "==0.3.0"
         },
         "schema": {
             "hashes": [
@@ -442,6 +448,20 @@
             ],
             "version": "==1.0.1"
         },
+        "distro": {
+            "hashes": [
+                "sha256:362dde65d846d23baee4b5c058c8586f219b5a54be1cf5fc6ff55c4578392f57",
+                "sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4"
+            ],
+            "version": "==1.4.0"
+        },
+        "dnspython": {
+            "hashes": [
+                "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
+                "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
+            ],
+            "version": "==1.16.0"
+        },
         "docutils": {
             "hashes": [
                 "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
@@ -531,6 +551,13 @@
             ],
             "version": "==6.0.0"
         },
+        "motor": {
+            "hashes": [
+                "sha256:462fbb824f4289481c158227a2579d6adaf1ec7c70cf7ebe60ed6ceb321e5869",
+                "sha256:d035c09ab422bc50bf3efb134f7405694cae76268545bd21e14fb22e2638f84e"
+            ],
+            "version": "==2.0.0"
+        },
         "multidict": {
             "hashes": [
                 "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f",
@@ -593,6 +620,45 @@
             ],
             "version": "==2.3.1"
         },
+        "pymongo": {
+            "hashes": [
+                "sha256:025f94fc1e1364f00e50badc88c47f98af20012f23317234e51a11333ef986e6",
+                "sha256:02aa7fb282606331aefbc0586e2cf540e9dbe5e343493295e7f390936ad2738e",
+                "sha256:057210e831573e932702cf332012ed39da78edf0f02d24a3f0b213264a87a397",
+                "sha256:0d946b79c56187fe139276d4c8ed612a27a616966c8b9779d6b79e2053587c8b",
+                "sha256:104790893b928d310aae8a955e0bdbaa442fb0ac0a33d1bbb0741c791a407778",
+                "sha256:15527ef218d95a8717486106553b0d54ff2641e795b65668754e17ab9ca6e381",
+                "sha256:1826527a0b032f6e20e7ac7f72d7c26dd476a5e5aa82c04aa1c7088a59fded7d",
+                "sha256:22e3aa4ce1c3eebc7f70f9ca7fd4ce1ea33e8bdb7b61996806cd312f08f84a3a",
+                "sha256:244e1101e9a48615b9a16cbd194f73c115fdfefc96894803158608115f703b26",
+                "sha256:24b8c04fdb633a84829d03909752c385faef249c06114cc8d8e1700b95aae5c8",
+                "sha256:2c276696350785d3104412cbe3ac70ab1e3a10c408e7b20599ee41403a3ed630",
+                "sha256:2d8474dc833b1182b651b184ace997a7bd83de0f51244de988d3c30e49f07de3",
+                "sha256:3119b57fe1d964781e91a53e81532c85ed1701baaddec592e22f6b77a9fdf3df",
+                "sha256:3bee8e7e0709b0fcdaa498a3e513bde9ffc7cd09dbceb11e425bd91c89dbd5b6",
+                "sha256:436c071e01a464753d30dbfc8768dd93aecf2a8e378e5314d130b95e77b4d612",
+                "sha256:46635e3f19ad04d5a7d7cf23d232388ddbfccf46d9a3b7436b6abadda4e84813",
+                "sha256:4772e0b679717e7ac4608d996f57b6f380748a919b457cb05bb941467b888b22",
+                "sha256:4e2cd80e16f481a62c3175b607373200e714ed29025f21559ebf7524f295689f",
+                "sha256:52732960efa0e003ca1c092dc0a3c65276e897681287a788a01ca78dda3b41f0",
+                "sha256:55a7de51ec7d1731b2431886d0349146645f2816e5b8eb982d7c49f89472c9f3",
+                "sha256:5f8ed5934197a2d4b2087646e98de3e099a237099dcf498b9e38dd3465f74ef4",
+                "sha256:64b064124fcbc8eb04a155117dc4d9a336e3cda3f069958fbc44fe70c3c3d1e9",
+                "sha256:65958b8e4319f992e85dad59d8081888b97fcdbde5f0d14bc28f2848b92d3ef1",
+                "sha256:7683428862e20c6a790c19e64f8ccf487f613fbc83d47e3d532df9c81668d451",
+                "sha256:78566d5570c75a127c2491e343dc006798a384f06be588fe9b0cbe5595711559",
+                "sha256:7d1cb00c093dbf1d0b16ccf123e79dee3b82608e4a2a88947695f0460eef13ff",
+                "sha256:8c74e2a9b594f7962c62cef7680a4cb92a96b4e6e3c2f970790da67cc0213a7e",
+                "sha256:8e60aa7699170f55f4b0f56ee6f8415229777ac7e4b4b1aa41fc61eec08c1f1d",
+                "sha256:9447b561529576d89d3bf973e5241a88cf76e45bd101963f5236888713dea774",
+                "sha256:970055bfeb0be373f2f5299a3db8432444bad3bc2f198753ee6c2a3a781e0959",
+                "sha256:a6344b8542e584e140dc3c651d68bde51270e79490aa9320f9e708f9b2c39bd5",
+                "sha256:ce309ca470d747b02ba6069d286a17b7df8e9c94d10d727d9cf3a64e51d85184",
+                "sha256:cfbd86ed4c2b2ac71bbdbcea6669bf295def7152e3722ddd9dda94ac7981f33d",
+                "sha256:d7929c513732dff093481f4a0954ed5ff16816365842136b17caa0b4992e49d3"
+            ],
+            "version": "==3.7.2"
+        },
         "pyparsing": {
             "hashes": [
                 "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a",
@@ -678,17 +744,16 @@
         "red-discordbot": {
             "editable": true,
             "extras": [
-                "mongo",
+                "mongo"
-                "voice"
             ],
             "path": "."
         },
         "red-lavalink": {
             "hashes": [
-                "sha256:13e1a3f91b990be9582cba039d9a32ec4cef760da1e7e6952143116ec83d4302",
+                "sha256:2a2f469c1feb72c2604795053a8823757ace85ed752eaf573c1d0daba29d1180",
-                "sha256:3dd0d73b4a908bbe9cfb703d2563dad1d1a58f8eea5896a0dacdf37d54a39d9c"
+                "sha256:4bc685a5d89660875d07f50060bacc820e69a763a581ce69375c792e16df4081"
             ],
-            "version": "==0.2.3"
+            "version": "==0.3.0"
         },
         "requests": {
             "hashes": [
@@ -754,11 +819,11 @@
         },
         "tox": {
             "hashes": [
-                "sha256:1b166b93d2ce66bb7b253ba944d2be89e0c9d432d49eeb9da2988b4902a4684e",
+                "sha256:5d6b9e7ad99a93b00ecd509e13552600d38eedd2b035ba24709f850b23f51254",
-                "sha256:665cbdd99f5c196dd80d1d8db8c8cf5d48b1ae1f778bccd1bdf14d5aaf4ca0fc"
+                "sha256:fee5b4fa2fb1638b57879a1fcaefbfd16201d8d7ecb9956406855a85d518ac4c"
             ],
             "index": "pypi",
-            "version": "==3.9.0"
+            "version": "==3.10.0"
         },
         "urllib3": {
             "hashes": [
@@ -769,10 +834,10 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:6aebaf4dd2568a0094225ebbca987859e369e3e5c22dc7d52e5406d504890417",
+                "sha256:15ee248d13e4001a691d9583948ad3947bcb8a289775102e4c4aa98a8b7a6d73",
-                "sha256:984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39"
+                "sha256:bfc98bb9b42a3029ee41b96dc00a34c2f254cbf7716bec824477b2c82741a5c4"
             ],
-            "version": "==16.4.3"
+            "version": "==16.5.0"
         },
         "websockets": {
             "hashes": [

docs/changelog_3_1_0.rst

@@ -66,6 +66,7 @@ Audio
 Core
 ----
 
+ * Warn on usage of ``yaml.load`` (`#2326`_)
  * New Event dispatch: ``on_message_without_command`` (`#2338`_)
  * Improve output format of cooldown messages (`#2412`_)
  * Delete cooldown messages when expired (`#2469`_)
@@ -73,6 +74,7 @@ Core
  * ``[p]set locale`` now only accepts actual locales (`#2553`_)
  * ``[p]listlocales`` now displays ``en-US`` (`#2553`_)
  * ``redbot --version`` will now give you current version of Red (`#2567`_)
+ * Redesign help and related formatter (`#2628`_)
  * Default locale changed from ``en`` to ``en-US`` (`#2642`_)
  * New command ``[p]datapath`` that prints the bot's datapath (`#2652`_)
 
@@ -116,6 +118,12 @@ Filter
 
  * Filter performs significantly better on large servers. (`#2509`_)
 
+--------
+Launcher
+--------
+
+* Fixed extras in the launcher (`#2588`_)
+
 ---
 Mod
 ---
@@ -166,6 +174,7 @@ Utility Functions
  * ``Tunnel`` - fixed behavior of ``react_close()``, now when tunnel closes message will be sent to other end (`#2507`_)
  * ``chat_formatting.humanize_list`` - Improved error handling of empty lists (`#2597`_)
 
+.. _#2326: https://github.com/Cog-Creators/Red-DiscordBot/pull/2326
 .. _#2328: https://github.com/Cog-Creators/Red-DiscordBot/pull/2328
 .. _#2338: https://github.com/Cog-Creators/Red-DiscordBot/pull/2338
 .. _#2412: https://github.com/Cog-Creators/Red-DiscordBot/pull/2412
@@ -205,6 +214,7 @@ Utility Functions
 .. _#2579: https://github.com/Cog-Creators/Red-DiscordBot/pull/2579
 .. _#2586: https://github.com/Cog-Creators/Red-DiscordBot/pull/2586
 .. _#2587: https://github.com/Cog-Creators/Red-DiscordBot/pull/2587
+.. _#2588: https://github.com/Cog-Creators/Red-DiscordBot/pull/2588
 .. _#2590: https://github.com/Cog-Creators/Red-DiscordBot/pull/2590
 .. _#2591: https://github.com/Cog-Creators/Red-DiscordBot/pull/2591
 .. _#2592: https://github.com/Cog-Creators/Red-DiscordBot/pull/2592
@@ -216,6 +226,7 @@ Utility Functions
 .. _#2605: https://github.com/Cog-Creators/Red-DiscordBot/pull/2605
 .. _#2606: https://github.com/Cog-Creators/Red-DiscordBot/pull/2606
 .. _#2620: https://github.com/Cog-Creators/Red-DiscordBot/pull/2620
+.. _#2628: https://github.com/Cog-Creators/Red-DiscordBot/pull/2628
 .. _#2639: https://github.com/Cog-Creators/Red-DiscordBot/pull/2639
 .. _#2642: https://github.com/Cog-Creators/Red-DiscordBot/pull/2642
 .. _#2652: https://github.com/Cog-Creators/Red-DiscordBot/pull/2652

redbot/__init__.py

@@ -174,7 +174,7 @@ class VersionInfo:
         )
 
 
-__version__ = "3.0.2"
+__version__ = "3.1.0"
 version_info = VersionInfo.from_str(__version__)
 
 # Filter fuzzywuzzy slow sequence matcher warning

redbot/cogs/audio/manager.py

@@ -14,7 +14,7 @@ import aiohttp
 from redbot.core import data_manager
 
 JAR_VERSION = "3.2.0.3"
-JAR_BUILD = 751
+JAR_BUILD = 772
 LAVALINK_DOWNLOAD_URL = (
     f"https://github.com/Cog-Creators/Lavalink-Jars/releases/download/{JAR_VERSION}_{JAR_BUILD}/"
     f"Lavalink.jar"

redbot/core/__init__.py

@@ -1,8 +1,10 @@
 import colorama as _colorama
 import discord as _discord
+import yaml as _yaml
 
 from .. import __version__, version_info, VersionInfo
 from .config import Config
+from .utils.safety import warn_unsafe as _warn_unsafe
 
 __all__ = ["Config", "__version__", "version_info", "VersionInfo"]
 
@@ -10,3 +12,6 @@ _colorama.init()
 
 # Prevent discord PyNaCl missing warning
 _discord.voice_client.VoiceClient.warn_nacl = False
+
+# Warn on known unsafe usage of dependencies
+_yaml.load = _warn_unsafe(_yaml.load, "Use yaml.safe_load instead. See CVE-2017-18342")

redbot/core/bot.py

@@ -115,10 +115,22 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):
 
         self.cog_mgr = CogManager()
 
-        super().__init__(*args, help_command=commands.DefaultHelpCommand(), **kwargs)
+        super().__init__(*args, help_command=None, **kwargs)
+        # Do not manually use the help formatter attribute here, see `send_help_for`,
+        # for a documented API. The internals of this object are still subject to change.
+        self._help_formatter = commands.help.RedHelpFormatter()
+        self.add_command(commands.help.red_help)
 
         self._permissions_hooks: List[commands.CheckPredicate] = []
 
+    async def send_help_for(
+        self, ctx: commands.Context, help_for: Union[commands.Command, commands.GroupMixin, str]
+    ):
+        """
+        Invokes Red's helpformatter for a given context and object.
+        """
+        return await self._help_formatter.send_help(ctx, help_for)
+
     async def _dict_abuse(self, indict):
         """
         Please blame <@269933075037814786> for this.

redbot/core/commands/commands.py

@@ -537,6 +537,10 @@ class Group(GroupMixin, Command, CogGroupMixin, commands.Group):
         super().__init__(*args, **kwargs)
 
     async def invoke(self, ctx: "Context"):
+        # we skip prepare in some cases to avoid some things
+        # We still always want this part of the behavior though
+        ctx.command = self
+        # Our re-ordered behavior below.
         view = ctx.view
         previous = view.index
         view.skip_ws()
@@ -557,6 +561,7 @@ class Group(GroupMixin, Command, CogGroupMixin, commands.Group):
             # how our permissions system works, we don't want it to skip the checks
             # as well.
             await self._verify_checks(ctx)
+            # this is actually why we don't prepare earlier.
 
         await super().invoke(ctx)
 
@@ -565,8 +570,60 @@ class CogMixin(CogGroupMixin, CogCommandMixin):
     """Mixin class for a cog, intended for use with discord.py's cog class"""
 
     @property
-    def all_commands(self) -> Dict[str, Command]:
+    def help(self):
-        return {cmd.name: cmd for cmd in self.__cog_commands__}
+        doc = self.__doc__
+        translator = getattr(self, "__translator__", lambda s: s)
+        if doc:
+            return inspect.cleandoc(translator(doc))
+
+    async def can_run(self, ctx: "Context", **kwargs) -> bool:
+        """
+        This really just exists to allow easy use with other methods using can_run
+        on commands and groups such as help formatters.
+        
+        kwargs used in that won't apply here as they don't make sense to,
+        but will be swallowed silently for a compatible signature for ease of use.
+
+        Parameters
+        ----------
+        ctx : `Context`
+            The invocation context to check with.
+
+        Returns
+        -------
+        bool
+            ``True`` if this cog is usable in the given context.
+        """
+
+        try:
+            can_run = await self.requires.verify(ctx)
+        except commands.CommandError:
+            return False
+
+        return can_run
+
+    async def can_see(self, ctx: "Context") -> bool:
+        """Check if this cog is visible in the given context.
+
+        In short, this will verify whether
+        the user is allowed to access the cog by permissions.
+
+        This has an identical signature to the one used by commands, and groups,
+        but needs a different underlying mechanism.
+
+        Parameters
+        ----------
+        ctx : `Context`
+            The invocation context to check with.
+
+        Returns
+        -------
+        bool
+            ``True`` if this cog is visible in the given context.
+
+        """
+
+        return await self.can_run(ctx)
 
 
 class Cog(CogMixin, commands.Cog):

redbot/core/commands/context.py

@@ -62,10 +62,12 @@ class Context(commands.Context):
 
         return await super().send(content=content, **kwargs)
 
-    async def send_help(self) -> List[discord.Message]:
+    async def send_help(self, command=None):
         """ Send the command help message. """
-        command = self.invoked_subcommand or self.command
+        # This allows people to manually use this similarly
-        await super().send_help(command)
+        # to the upstream d.py version, while retaining our use.
+        command = command or self.command
+        await self.bot.send_help_for(self, command)
 
     async def tick(self) -> bool:
         """Add a tick reaction to the command message.

redbot/core/commands/help.py

@@ -1,23 +1,515 @@
-from discord.ext import commands
+# This is a full replacement of discord.py's help command
-from .commands import Command
+# Signatures are not guaranteed to be unchanging in this file.
+# At a later date when this is more set in stone, this warning will be removed.
+# At said later date, there should also be things added to support extra formatter
+# registration from 3rd party cogs.
+#
+# This exists due to deficiencies in discord.py which conflict
+# with our needs for per-context help settings
+# see https://github.com/Rapptz/discord.py/issues/2123
+#
+# While the issue above discusses this as theoretical, merely interacting with config within
+# the help command preparation was enough to cause
+# demonstrable breakage in 150 help invokes in a 2 minute window.
+# This is not an unreasonable volume on some already existing Red instances,
+# especially since help is invoked for command groups
+# automatically when subcommands are not provided correctly as user feedback.
+#
+# The implemented fix is in
+# https://github.com/Rapptz/discord.py/commit/ad5beed8dd75c00bd87492cac17fe877033a3ea1
+#
+# While this fix would handle our immediate specific issues, it's less appropriate to use
+# Where we do not have a downstream consumer to consider.
+# Simply modifying the design to not be susceptible to the issue,
+# rather than adding copy and deepcopy use in multiple places is better for us
+#
+# Additionally, this gives our users a bit more customization options including by
+# 3rd party cogs down the road.
 
-__all__ = ["HelpCommand", "DefaultHelpCommand", "MinimalHelpCommand"]
+from collections import namedtuple
+from typing import Union, List, AsyncIterator, Iterable, cast
+
+import discord
+from discord.ext import commands as dpy_commands
+
+from . import commands
+from .context import Context
+from ..i18n import Translator
+from ..utils import menus, fuzzy_command_search, format_fuzzy_results
+from ..utils.chat_formatting import box, pagify
+
+__all__ = ["red_help", "RedHelpFormatter"]
+
+T_ = Translator("Help", __file__)
+
+HelpTarget = Union[commands.Command, commands.Group, commands.Cog, dpy_commands.bot.BotBase, str]
+
+# The below could be a protocol if we pulled in typing_extensions from mypy.
+SupportsCanSee = Union[commands.Command, commands.Group, dpy_commands.bot.BotBase, commands.Cog]
+
+EmbedField = namedtuple("EmbedField", "name value inline")
+EMPTY_STRING = "\N{ZERO WIDTH SPACE}"
 
 
-class _HelpCommandImpl(Command, commands.help._HelpCommandImpl):
+class NoCommand(Exception):
     pass
 
 
-class HelpCommand(commands.help.HelpCommand):
+class NoSubCommand(Exception):
-    def _add_to_bot(self, bot):
+    def __init__(self, *, last, not_found):
-        command = _HelpCommandImpl(self, self.command_callback, **self.command_attrs)
+        self.last = last
-        bot.add_command(command)
+        self.not_found = not_found
-        self._command_impl = command
 
 
-class DefaultHelpCommand(HelpCommand, commands.help.DefaultHelpCommand):
+class RedHelpFormatter:
-    pass
+    """
+    Red's help implementation
+
+    This is intended to be overridable in parts to only change some behavior.
+
+    While currently, there is a global formatter, later plans include a context specific
+    formatter selector as well as an API for cogs to register/un-register a formatter with the bot.
+
+    When implementing your own formatter, at minimum you must provide an implementation of 
+    `send_help` with identical signature.
+
+    While this exists as a class for easy partial overriding, most implementations
+    should not need or want a shared state.
+    """
+
+    # Class vars for things which should be configurable at a later date but aren't now
+    # Technically, someone can just use a cog to switch these in real time for now.
+
+    USE_MENU = False
+    CONFIRM_UNAVAILABLE_COMMAND_EXISTENCES = False
+    SHOW_HIDDEN = False
+    VERIFY_CHECKS = True
+
+    async def send_help(self, ctx: Context, help_for: HelpTarget = None):
+        """ 
+        This delegates to other functions. 
+        
+        For most cases, you should use this and only this directly.
+        """
+        if help_for is None or isinstance(help_for, dpy_commands.bot.BotBase):
+            await self.format_bot_help(ctx)
+            return
+
+        if isinstance(help_for, str):
+            try:
+                help_for = self.parse_command(ctx, help_for)
+            except NoCommand:
+                await self.command_not_found(ctx, help_for)
+                return
+            except NoSubCommand as exc:
+                if self.CONFIRM_UNAVAILABLE_COMMAND_EXISTENCES:
+                    await self.subcommand_not_found(ctx, exc.last, exc.not_found)
+                    return
+                help_for = exc.last
+
+        if isinstance(help_for, commands.Cog):
+            await self.format_cog_help(ctx, help_for)
+        else:
+            await self.format_command_help(ctx, help_for)
+
+    async def get_cog_help_mapping(self, ctx: Context, obj: commands.Cog):
+        iterator = filter(lambda c: c.parent is None and c.cog is obj, ctx.bot.commands)
+        return {com.name: com async for com in self.help_filter_func(ctx, iterator)}
+
+    async def get_group_help_mapping(self, ctx: Context, obj: commands.Group):
+        return {
+            com.name: com async for com in self.help_filter_func(ctx, obj.all_commands.values())
+        }
+
+    async def get_bot_help_mapping(self, ctx):
+        sorted_iterable = []
+        for cogname, cog in (*sorted(ctx.bot.cogs.items()), (None, None)):
+            cm = await self.get_cog_help_mapping(ctx, cog)
+            if cm:
+                sorted_iterable.append((cogname, cm))
+        return sorted_iterable
+
+    @staticmethod
+    def get_default_tagline(ctx: Context):
+        return (
+            f"Type {ctx.clean_prefix}help <command> for more info on a command. "
+            f"You can also type {ctx.clean_prefix}help <category> for more info on a category."
+        )
+
+    async def format_command_help(self, ctx: Context, obj: commands.Command):
+
+        send = self.CONFIRM_UNAVAILABLE_COMMAND_EXISTENCES
+        if not send:
+            async for _ in self.help_filter_func(ctx, (obj,), bypass_hidden=True):
+                # This is a really lazy option for not
+                # creating a separate single case version.
+                # It is efficient though
+                #
+                # We do still want to bypass the hidden requirement on
+                # a specific command explicitly invoked here.
+                send = True
+
+        if not send:
+            return
+
+        command = obj
+
+        description = command.description or ""
+        tagline = (await ctx.bot.db.help.tagline()) or self.get_default_tagline(ctx)
+        signature = f"`Syntax: {ctx.clean_prefix}{command.qualified_name} {command.signature}`"
+        subcommands = None
+
+        if hasattr(command, "all_commands"):
+            grp = cast(commands.Group, command)
+            subcommands = await self.get_group_help_mapping(ctx, grp)
+
+        if await ctx.embed_requested():
+            emb = {"embed": {"title": "", "description": ""}, "footer": {"text": ""}, "fields": []}
+
+            if description:
+                emb["embed"]["title"] = f"*{description[:2044]}*"
+
+            emb["footer"]["text"] = tagline
+            emb["embed"]["description"] = signature
+
+            if command.help:
+                splitted = command.help.split("\n\n")
+                name = "__{0}__".format(splitted[0])
+                value = "\n\n".join(splitted[1:]).replace("[p]", ctx.clean_prefix)
+                if not value:
+                    value = EMPTY_STRING
+                field = EmbedField(name[:252], value[:1024], False)
+                emb["fields"].append(field)
+
+            if subcommands:
+                subtext = "\n".join(
+                    f"**{name}** {command.short_doc}"
+                    for name, command in sorted(subcommands.items())
+                )
+                for i, page in enumerate(pagify(subtext, page_length=1000, shorten_by=0)):
+                    if i == 0:
+                        title = "**__Subcommands:__**"
+                    else:
+                        title = "**__Subcommands:__** (continued)"
+                    field = EmbedField(title, page, False)
+                    emb["fields"].append(field)
+
+            await self.make_and_send_embeds(ctx, emb)
+
+        else:  # Code blocks:
+
+            subtext = None
+            subtext_header = None
+            if subcommands:
+                subtext_header = "Subcommands:"
+                max_width = max(discord.utils._string_width(name) for name in subcommands.keys())
+
+                def width_maker(cmds):
+                    doc_max_width = 80 - max_width
+                    for nm, com in sorted(cmds):
+                        width_gap = discord.utils._string_width(nm) - len(nm)
+                        doc = command.short_doc
+                        if len(doc) > doc_max_width:
+                            doc = doc[: doc_max_width - 3] + "..."
+                        yield nm, doc, max_width - width_gap
+
+                subtext = "\n".join(
+                    f"  {name:<{width}} {doc}"
+                    for name, doc, width in width_maker(subcommands.items())
+                )
+
+            to_page = "\n\n".join(
+                filter(None, (description, signature[1:-1], command.help, subtext_header, subtext))
+            )
+            pages = [box(p) for p in pagify(to_page)]
+            await self.send_pages(ctx, pages, embed=False)
+
+    @staticmethod
+    def group_embed_fields(fields: List[EmbedField], max_chars=1000):
+        curr_group = []
+        ret = []
+        for f in fields:
+            curr_group.append(f)
+            if sum(len(f.value) for f in curr_group) > max_chars:
+                ret.append(curr_group)
+                curr_group = []
+
+        if len(curr_group) > 0:
+            ret.append(curr_group)
+
+        return ret
+
+    async def make_and_send_embeds(self, ctx, embed_dict: dict):
+
+        pages = []
+
+        page_char_limit = await ctx.bot.db.help.page_char_limit()
+        field_groups = self.group_embed_fields(embed_dict["fields"], page_char_limit)
+
+        color = await ctx.embed_color()
+        page_count = len(field_groups)
+
+        author_info = {"name": f"{ctx.me.display_name} Help Menu", "icon_url": ctx.me.avatar_url}
+
+        for i, group in enumerate(field_groups, 1):
+            embed = discord.Embed(color=color, **embed_dict["embed"])
+
+            if page_count > 1:
+                description = f"{embed.description} *Page {i} of {page_count}*"
+                embed.description = description
+
+            embed.set_author(**author_info)
+
+            for field in group:
+                embed.add_field(**field._asdict())
+
+            embed.set_footer(**embed_dict["footer"])
+
+            pages.append(embed)
+
+        await self.send_pages(ctx, pages, embed=True)
+
+    async def format_cog_help(self, ctx: Context, obj: commands.Cog):
+
+        commands = await self.get_cog_help_mapping(ctx, obj)
+        if not (commands or self.CONFIRM_UNAVAILABLE_COMMAND_EXISTENCES):
+            return
+
+        description = obj.help
+        tagline = (await ctx.bot.db.help.tagline()) or self.get_default_tagline(ctx)
+
+        if await ctx.embed_requested():
+            emb = {"embed": {"title": "", "description": ""}, "footer": {"text": ""}, "fields": []}
+
+            emb["footer"]["text"] = tagline
+            if description:
+                emb["embed"]["title"] = f"*{description[:2044]}*"
+
+            if commands:
+                command_text = "\n".join(
+                    f"**{name}** {command.short_doc}" for name, command in sorted(commands.items())
+                )
+                for i, page in enumerate(pagify(command_text, page_length=1000, shorten_by=0)):
+                    if i == 0:
+                        title = "**__Commands:__**"
+                    else:
+                        title = "**__Commands:__** (continued)"
+                    field = EmbedField(title, page, False)
+                    emb["fields"].append(field)
+
+            await self.make_and_send_embeds(ctx, emb)
+
+        else:
+            commands_text = None
+            commands_header = None
+            if commands:
+                subtext_header = "Commands:"
+                max_width = max(discord.utils._string_width(name) for name in commands.keys())
+
+                def width_maker(cmds):
+                    doc_max_width = 80 - max_width
+                    for nm, com in sorted(cmds):
+                        width_gap = discord.utils._string_width(nm) - len(nm)
+                        doc = com.short_doc
+                        if len(doc) > doc_max_width:
+                            doc = doc[: doc_max_width - 3] + "..."
+                        yield nm, doc, max_width - width_gap
+
+                subtext = "\n".join(
+                    f"  {name:<{width}} {doc}"
+                    for name, doc, width in width_maker(commands.items())
+                )
+
+            to_page = "\n\n".join(
+                filter(None, (description, signature[1:-1], subtext_header, subtext))
+            )
+            pages = [box(p) for p in pagify(to_page)]
+            await self.send_pages(ctx, pages, embed=False)
+
+    async def format_bot_help(self, ctx: Context):
+
+        commands = await self.get_bot_help_mapping(ctx)
+        if not commands:
+            return
+
+        description = ctx.bot.description or ""
+        tagline = (await ctx.bot.db.help.tagline()) or self.get_default_tagline(ctx)
+
+        if await ctx.embed_requested():
+
+            emb = {"embed": {"title": "", "description": ""}, "footer": {"text": ""}, "fields": []}
+
+            emb["footer"]["text"] = tagline
+            if description:
+                emb["embed"]["title"] = f"*{description[:2044]}*"
+
+            for cog_name, data in commands:
+
+                if cog_name:
+                    title = f"**__{cog_name}:__**"
+                else:
+                    title = f"**__No Category:__**"
+
+                cog_text = "\n".join(
+                    f"**{name}** {command.short_doc}" for name, command in sorted(data.items())
+                )
+
+                for i, page in enumerate(pagify(cog_text, page_length=1000, shorten_by=0)):
+                    title = title if i < 1 else f"{title} (continued)"
+                    field = EmbedField(title, page, False)
+                    emb["fields"].append(field)
+
+            await self.make_and_send_embeds(ctx, emb)
+
+        else:
+            if description:
+                to_join = [f"{description}\n"]
+
+            names = []
+            for k, v in commands:
+                names.extend(list(v.name for v in v.values()))
+
+            max_width = max(
+                discord.utils._string_width((name or "No Category:")) for name in names
+            )
+
+            def width_maker(cmds):
+                doc_max_width = 80 - max_width
+                for nm, com in cmds:
+                    width_gap = discord.utils._string_width(nm) - len(nm)
+                    doc = com.short_doc
+                    if len(doc) > doc_max_width:
+                        doc = doc[: doc_max_width - 3] + "..."
+                    yield nm, doc, max_width - width_gap
+
+            for cog_name, data in commands:
+
+                title = f"{cog_name}:" if cog_name else "No Category:"
+                to_join.append(title)
+
+                for name, doc, width in width_maker(sorted(data.items())):
+                    to_join.append(f"  {name:<{width}} {doc}")
+
+            to_join.append(f"\n{tagline}")
+            to_page = "\n".join(to_join)
+            pages = [box(p) for p in pagify(to_page)]
+            await self.send_pages(ctx, pages, embed=False)
+
+    async def help_filter_func(
+        self, ctx, objects: Iterable[SupportsCanSee], bypass_hidden=False
+    ) -> AsyncIterator[SupportsCanSee]:
+        """
+        This does most of actual filtering.
+        """
+        # TODO: Settings for this in core bot db
+        for obj in objects:
+            if self.VERIFY_CHECKS and not (self.SHOW_HIDDEN or bypass_hidden):
+                # Default Red behavior, can_see includes a can_run check.
+                if await obj.can_see(ctx):
+                    yield obj
+            elif self.VERIFY_CHECKS:
+                if await obj.can_run(ctx):
+                    yield obj
+            elif not (self.SHOW_HIDDEN or bypass_hidden):
+                if getattr(obj, "hidden", False):  # Cog compatibility
+                    yield obj
+            else:
+                yield obj
+
+    async def command_not_found(self, ctx, help_for):
+        """
+        Sends an error, fuzzy help, or stays quiet based on settings
+        """
+        coms = [c async for c in self.help_filter_func(ctx, ctx.bot.walk_commands())]
+        fuzzy_commands = await fuzzy_command_search(ctx, help_for, commands=coms, min_score=75)
+        use_embeds = await ctx.embed_requested()
+        if fuzzy_commands:
+            ret = await format_fuzzy_results(ctx, fuzzy_commands, embed=use_embeds)
+            if use_embeds:
+                ret.set_author()
+                tagline = (await ctx.bot.db.help.tagline()) or self.get_default_tagline(ctx)
+                ret.set_footer(text=tagline)
+                await ctx.send(embed=ret)
+            else:
+                await ctx.send(ret)
+        elif self.CONFIRM_UNAVAILABLE_COMMAND_EXISTENCES:
+            ret = T_("Command *{command_name}* not found.").format(command_name=command_name)
+            if use_embeds:
+                emb = discord.Embed(color=(await ctx.embed_color()), description=ret)
+                emb.set_author(name=f"{ctx.me.display_name} Help Menu", icon_url=ctx.me.avatar_url)
+                tagline = (await ctx.bot.db.help.tagline()) or self.get_default_tagline(ctx)
+                ret.set_footer(text=tagline)
+                await ctx.send(embed=ret)
+            else:
+                await ctx.send(ret)
+
+    async def subcommand_not_found(self, ctx, command, not_found):
+        """
+        Sends an error
+        """
+        ret = T_("Command *{command_name}* has no subcommands.").format(
+            command_name=command.qualified_name
+        )
+        await ctx.send(ret)
+
+    @staticmethod
+    def parse_command(ctx, help_for: str):
+        """
+        Handles parsing
+        """
+
+        maybe_cog = ctx.bot.get_cog(help_for)
+        if maybe_cog:
+            return maybe_cog
+
+        com = ctx.bot
+        last = None
+
+        clist = help_for.split()
+
+        for index, item in enumerate(clist):
+            try:
+                com = com.all_commands[item]
+                # TODO: This doesn't handle valid command aliases.
+                # swap parsing method to use get_command.
+            except (KeyError, AttributeError):
+                if last:
+                    raise NoSubCommand(last=last, not_found=clist[index:]) from None
+                else:
+                    raise NoCommand() from None
+            else:
+                last = com
+
+        return com
+
+    async def send_pages(
+        self, ctx: Context, pages: List[Union[str, discord.Embed]], embed: bool = True
+    ):
+        """
+        Sends pages based on settings.
+        """
+
+        if not self.USE_MENU:
+
+            max_pages_in_guild = await ctx.bot.db.help.max_pages_in_guild()
+            destination = ctx.author if len(pages) > max_pages_in_guild else ctx
+
+            if embed:
+                for page in pages:
+                    await destination.send(embed=page)
+            else:
+                for page in pages:
+                    await destination.send(page)
+        else:
+            await menus.menu(ctx, pages, menus.DEFAULT_CONTROLS)
 
 
-class MinimalHelpCommand(HelpCommand, commands.help.MinimalHelpCommand):
+@commands.command(name="help", hidden=True, i18n=T_)
-    pass
+async def red_help(ctx: Context, *, thing_to_get_help_for: str = None):
+    """
+    I need somebody
+    (Help) not just anybody
+    (Help) you know I need someone
+    (Help!)
+    """
+    await ctx.bot.send_help_for(ctx, thing_to_get_help_for)

redbot/core/drivers/red_mongo.py

@@ -92,6 +92,7 @@ class Mongo(BaseDriver):
         async for doc in cursor:
             pkeys = doc["_id"]["RED_primary_key"]
             del doc["_id"]
+            doc = self._unescape_dict_keys(doc)
             if len(pkeys) == 0:
                 # Global data
                 ret.update(**doc)

redbot/core/utils/__init__.py

@@ -176,7 +176,11 @@ async def async_enumerate(
 
 
 async def fuzzy_command_search(
-    ctx: commands.Context, term: Optional[str] = None, *, min_score: int = 80
+    ctx: commands.Context,
+    term: Optional[str] = None,
+    *,
+    commands: Optional[list] = None,
+    min_score: int = 80,
 ) -> Optional[List[commands.Command]]:
     """Search for commands which are similar in name to the one invoked.
 
@@ -230,7 +234,9 @@ async def fuzzy_command_search(
             return
 
     # Do the scoring. `extracted` is a list of tuples in the form `(command, score)`
-    extracted = process.extract(term, ctx.bot.walk_commands(), limit=5, scorer=fuzz.QRatio)
+    extracted = process.extract(
+        term, (commands or ctx.bot.walk_commands()), limit=5, scorer=fuzz.QRatio
+    )
     if not extracted:
         return
 

redbot/core/utils/safety.py

@@ -0,0 +1,49 @@
+import warnings
+import functools
+
+
+def unsafe(f, message=None):
+    """
+    Decorator form for marking a function as unsafe.
+
+    This form may not get used much, but there are a few cases
+    we may want to add something unsafe generally, but safe in specific uses.
+
+    The warning can be supressed in the safe context with warnings.catch_warnings
+    This should be used sparingly at most.
+    """
+
+    def wrapper(func):
+        @functools.wraps(func)
+        def get_wrapped(*args, **kwargs):
+            actual_message = message or f"{func.__name__} is unsafe for use"
+            warnings.warn(actual_message, stacklevel=3, category=RuntimeWarning)
+            return func(*args, **kwargs)
+
+        return get_wrapped
+
+    return wrapper
+
+
+def warn_unsafe(f, message=None):
+    """
+    Function to mark function from dependencies as unsafe for use.
+
+    Warning: There is no check that a function has already been modified.
+    This form should only be used in init, if you want to mark an internal function
+    as unsafe, use the decorator form above.
+
+    The warning can be suppressed in safe contexts with warnings.catch_warnings
+    This should be used sparingly at most.
+    """
+
+    def wrapper(func):
+        @functools.wraps(func)
+        def get_wrapped(*args, **kwargs):
+            actual_message = message or f"{func.__name__} is unsafe for use"
+            warnings.warn(actual_message, stacklevel=3, category=RuntimeWarning)
+            return func(*args, **kwargs)
+
+        return get_wrapped
+
+    return wrapper(f)