Merge remote-tracking branch 'release/V3/develop' into V3/develop

.codeclimate.yml

@@ -0,0 +1,44 @@
+version: "2"  # required to adjust maintainability checks
+checks:
+  argument-count:
+    config:
+      threshold: 6
+  complex-logic:
+    enabled: false  # Disabled in favor of using Radon for this
+    config:
+      threshold: 4
+  file-lines:
+    config:
+      threshold: 1000  # I would set this lower if not for cogs as command containers.
+  method-complexity:
+    enabled: false  # Disabled in favor of using Radon for this
+    config:
+      threshold: 5
+  method-count:
+    enabled: false  # I would set this lower if not for cogs as command containers.
+    config:
+      threshold: 20
+  method-lines:
+    enabled: false
+    config:
+      threshold: 25  # I'm fine with long methods, cautious about the complexity of a single method.
+  nested-control-flow:
+    config:
+      threshold: 4
+  return-statements:
+    config:
+      threshold: 6
+  similar-code:
+    enabled: false
+    config:
+      threshold: # language-specific defaults. an override will affect all languages.
+  identical-code:
+    config:
+      threshold: # language-specific defaults. an override will affect all languages.
+plugins:
+  bandit:
+    enabled: true
+  radon: 
+    enabled: true
+    config: 
+      threshold: "D"

.github/CODEOWNERS

@@ -33,7 +33,7 @@ redbot/cogs/audio/*                 @aikaterna
 redbot/cogs/bank/*                  @tekulvw
 redbot/cogs/cleanup/*               @palmtree5
 redbot/cogs/customcom/*             @palmtree5
-redbot/cogs/downloader/*            @tekulvw
+redbot/cogs/downloader/*            @tekulvw @jack1142
 redbot/cogs/economy/*               @palmtree5
 redbot/cogs/filter/*                @palmtree5
 redbot/cogs/general/*               @palmtree5
@@ -49,6 +49,9 @@ redbot/cogs/warnings/*              @palmtree5
 # Docs
 docs/*                              @tekulvw @palmtree5
 
+# Tests
+tests/cogs/downloader/*             @jack1142
+
 # Setup, instance setup, and running the bot
 setup.py                            @tekulvw
 redbot/__init__.py                  @tekulvw

.github/ISSUE_TEMPLATE/command_bug.md

@@ -1,6 +1,9 @@
 ---
 name: Bug reports for commands
 about: For bugs that involve commands found within Red
+title: ''
+labels: 'Type: Bug'
+assignees: ''
 
 ---
 

.github/ISSUE_TEMPLATE/feature_req.md

@@ -1,6 +1,9 @@
 ---
 name: Feature request
 about: For feature requests regarding Red itself.
+title: ''
+labels: 'Type: Feature'
+assignees: ''
 
 ---
 

.github/ISSUE_TEMPLATE/other_bug.md

@@ -1,6 +1,9 @@
 ---
 name: Bug report
 about: For bugs that don't involve a command.
+title: ''
+labels: 'Type: Bug'
+assignees: ''
 
 ---
 

.readthedocs.yml

@@ -14,4 +14,3 @@ python:
       path: .
       extra_requirements:
         - docs
-        - mongo

.travis.yml

@@ -27,10 +27,6 @@ jobs:
         postgresql: "10"
       before_script:
         - psql -c 'create database red_db;' -U postgres
-    - env: TOXENV=mongo
-      services: mongodb
-      before_script:
-        - mongo red_db --eval 'db.createUser({user:"red",pwd:"red",roles:["readWrite"]});'
     # These jobs only occur on tag creation if the prior ones succeed
     - stage: PyPi Deployment
       if: tag IS present

changelog.d/2105.docs.rst

@@ -0,0 +1 @@
+Added documentation for PM2 support.

changelog.d/2571.misc.rst

@@ -0,0 +1 @@
+Tests now use same event loop policy as Red's code.

changelog.d/2962.enhance.rst

@@ -0,0 +1 @@
+```redbot-setup delete`` now has the option to leave Red's data untouched on database backends.

changelog.d/3005.docs.rst

@@ -0,0 +1 @@
+Adds autostart documentation for Red users who installed it inside a virtual environment.

changelog.d/3045.enhance.rst

@@ -0,0 +1 @@
+Bot now handles more things prior to connecting to discord to reduce issues with initial load

changelog.d/3060.enhance.rst

@@ -0,0 +1 @@
+All ``y/n`` confirmations in cli commands are now unified.

changelog.d/3060.feature.rst

@@ -0,0 +1 @@
+Added ``redbot --edit`` cli flag that can be used to edit instance name, token, owner and datapath.

changelog.d/3060.fix.rst

@@ -0,0 +1 @@
+Arguments ``--co-owner`` and ``--load-cogs`` now properly require at least one argument to be passed.

changelog.d/3073.breaking.rst

@@ -0,0 +1 @@
+``bot.wait_until_ready`` should no longer be used during extension setup

changelog.d/3079.docs.rst

@@ -0,0 +1 @@
+Word using dev during install more strongly, to try to avoid end users using dev.

changelog.d/3083.docs.rst

@@ -0,0 +1 @@
+Fix some typos and wording, add MS Azure to host list

changelog.d/3090.feature.rst

@@ -0,0 +1 @@
+adds a licenseinfo command

changelog.d/3099.breaking.rst

@@ -0,0 +1 @@
+Removes the mongo driver.

changelog.d/3100.bugfix.rst

@@ -0,0 +1 @@
+fix ``is_automod_immune`` handling of guild check and support for checking webhooks

changelog.d/3105.docs.rst

@@ -0,0 +1 @@
+Update docs footer copyright to 2019.

changelog.d/3110.docs.rst

@@ -0,0 +1 @@
+Update apikey framework documentation. Change bot.get_shared_api_keys() to bot.get_shared_api_tokens().

changelog.d/3118.feature.rst

@@ -0,0 +1 @@
+Adds a command to list disabled commands globally or per guild.

changelog.d/3121.enhance.rst

@@ -0,0 +1 @@
+Change ``[p]info`` to say "This bot is an..." instead of "This is an..." for clarity.

changelog.d/3124.docs.rst

@@ -0,0 +1 @@
+Add information about ``info.json``'s ``min_python_version`` key in Downloader Framework docs.

changelog.d/3134.docs.rst

@@ -0,0 +1 @@
+Add event reference for ``on_red_api_tokens_update`` event in Shared API Keys docs.

changelog.d/3134.feature.rst

@@ -0,0 +1 @@
+New event ``on_red_api_tokens_update`` is now dispatched when shared api keys for the service are updated.

changelog.d/3172.enhance.rst

@@ -0,0 +1 @@
+Clarified that ``[p]backup`` saves the **bot's** data in the help text.

changelog.d/3174.bugfix.rst

@@ -0,0 +1 @@
+``--owner`` and ``-p`` cli flags now work when added from launcher.

changelog.d/admin/3166.bugfix.rst

@@ -0,0 +1 @@
+Fixed ``[p]announce`` failing after encountering an error attempting to message the bot owner.

changelog.d/audio/3048.bugfix.rst

@@ -0,0 +1 @@
+Unify capitalisation in ``[p]help playlist``.

changelog.d/audio/3050.bugfix.rst

@@ -0,0 +1 @@
+Bot's status is now properly cleared on emptydisconnect.

changelog.d/audio/3051.enhance.rst

@@ -0,0 +1 @@
+Improved explanation in help string for ``[p]audioset emptydisconnect``.

changelog.d/audio/3058.enhancement.rst

@@ -0,0 +1 @@
+Add typing indicator to playlist dedupe

changelog.d/audio/3085.enhance.1.rst

@@ -0,0 +1 @@
+Expose FriendlyExceptions to users on the play command.

changelog.d/audio/3104.misc.1.rst

@@ -0,0 +1 @@
+Fix an issue where some YouTube playlists were being recognised as single tracks.

changelog.d/downloader/1866.enhance.rst

@@ -0,0 +1 @@
+Downloader will now check if Python and bot version match requirements in ``info.json`` during update.

changelog.d/downloader/2527.docs.rst

@@ -0,0 +1 @@
+Added :func:`redbot.cogs.downloader.installable.InstalledModule` to Downloader's framework docs.

changelog.d/downloader/2527.enhance.1.rst

@@ -0,0 +1 @@
+User can now pass multiple cog names to ``[p]cog install``.

changelog.d/downloader/2527.enhance.2.rst

@@ -0,0 +1 @@
+When passing cogs to ``[p]cog update`` command, it will now only update those cogs, not all cogs from the repo these cogs are from.

changelog.d/downloader/2527.feature.1.rst

@@ -0,0 +1 @@
+Added ``[p]repo update [repos]`` command that allows you to update repos without updating cogs from them.

changelog.d/downloader/2527.feature.2.rst

@@ -0,0 +1 @@
+Added ``[p]cog installversion <repo_name> <revision> <cogs>`` command that allows you to install cogs from specified revision (commit, tag) of given repo. When using this command, the cog will automatically be pinned.

changelog.d/downloader/2527.feature.3.rst

@@ -0,0 +1 @@
+Added ``[p]cog pin <cogs>`` and ``[p]cog unpin <cogs>`` for pinning cogs. Cogs that are pinned will not be updated when using update commands.

changelog.d/downloader/2527.feature.4.rst

@@ -0,0 +1 @@
+Added ``[p]cog checkforupdates`` command that will tell which cogs can be updated (including pinned cog) without updating them.

changelog.d/downloader/2527.feature.5.rst

@@ -0,0 +1 @@
+Added ``[p]cog updateallfromrepos <repos>`` command that will update all cogs from given repos.

changelog.d/downloader/2527.feature.6.rst

@@ -0,0 +1 @@
+Added ``[p]cog updatetoversion <repo_name> <revision> [cogs]`` command that updates all cogs or ones of user's choosing to chosen revision of given repo.

changelog.d/downloader/2527.misc.1.rst

@@ -0,0 +1,4 @@
+Added :func:`redbot.cogs.downloader.installable.InstalledModule` which is used instead of :func:`redbot.cogs.downloader.installable.Installable` when we refer to installed cog or shared library.
+Therefore:
+    - ``to_json`` and ``from_json`` methods were moved from :func:`redbot.cogs.downloader.installable.Installable` to :func:`redbot.cogs.downloader.installable.InstalledModule`
+    - return types changed for :func:`redbot.cogs.downloader.converters.InstalledCog.convert`, :func:`redbot.cogs.downloader.downloader.Downloader.installed_cogs`, :func:`redbot.cogs.downloader.repo_manager.Repo.install_cog` to use :func:`redbot.cogs.downloader.installable.InstalledModule`.

changelog.d/downloader/2571.bugfix.1.rst

@@ -0,0 +1 @@
+Made regex for repo names use raw string to stop ``DeprecationWarning`` about invalid escape sequence.

changelog.d/downloader/2571.bugfix.2.rst

@@ -0,0 +1 @@
+Downloader will no longer allow to install cog that is already installed.

changelog.d/downloader/2571.dep.rst

@@ -0,0 +1 @@
+Added ``pytest-mock`` requirement to ``tests`` extra.

changelog.d/downloader/2571.enhance.rst

@@ -0,0 +1 @@
+Added error messages for failures during installing/reinstalling requirements and copying cogs and shared libraries.

changelog.d/downloader/2571.misc.rst

@@ -0,0 +1 @@
+Added more Downloader tests for Repo logic and git integration. New git tests use a test repo file that can be generated using new tool at ``tools/edit_testrepo.py``.

changelog.d/downloader/2927.bugfix.rst

@@ -0,0 +1 @@
+Downloader will no longer allow to install cog with same name as other that is installed.

changelog.d/downloader/2936.bugfix.rst

@@ -0,0 +1 @@
+Catch errors if remote repository or branch is deleted, notify user which repository failed and continue updating others.

changelog.d/downloader/3080.misc.1.rst

@@ -0,0 +1 @@
+`RepoManager.update_all_repos` replaced by new method `update_repos` which additionally handles failing repositories.

changelog.d/downloader/3080.misc.2.rst

@@ -0,0 +1 @@
+Added `Downloader.format_failed_repos` for formatting error message of repos failing to update.

changelog.d/downloader/3129.enhance.rst

@@ -0,0 +1 @@
+Use sanitized url (without HTTP Basic Auth fragments) in `[p]findcog` command.

changelog.d/downloader/3129.misc.rst

@@ -0,0 +1 @@
+Add `clean_url` property to :class:`redbot.cogs.downloader.repo_manager.Repo` which contains sanitized repo URL (without HTTP Basic Auth).

changelog.d/downloader/3141.bugfix.rst

@@ -0,0 +1 @@
+Make :attr:`redbot.cogs.downloader.repo_manager.Repo.clean_url` work with relative urls. This property uses `str` type now.

changelog.d/downloader/3153.bugfix.rst

@@ -0,0 +1 @@
+Fixed an error on repo add from empty string values for the `install_msg` info.json field.

changelog.d/downloader/3159.bugfix.rst

@@ -0,0 +1 @@
+Disable all git auth prompts when adding/updating repo with Downloader.

changelog.d/downloader/3160.misc.rst

@@ -0,0 +1 @@
+Ensure consistent output from git commands for purpose of parsing.

changelog.d/downloader/3177.bugfix.rst

@@ -0,0 +1 @@
+``[p]findcog`` now properly works for cogs with less typical folder structure.

changelog.d/permissions/3037.bugfix.rst

@@ -0,0 +1 @@
+defaults are cleared properly when clearing all rules

docs/autostart_pm2.rst

@@ -0,0 +1,42 @@
+.. pm2 service guide
+
+==============================================
+Setting up auto-restart using pm2 on Linux
+==============================================
+
+.. note:: This guide is for setting up PM2 on a Linux environment. This guide assumes that you already have a working Red instance.
+
+--------------
+Installing PM2
+--------------
+
+Start by installing Node.JS and NPM via your favorite package distributor. From there run the following command:
+
+:code:`npm install pm2 -g`
+
+After PM2 is installed, run the following command to enable your Red instance to be managed by PM2. Replace the brackets with the required information.
+You can add additional Red based arguments after the instance, such as :code:`--dev`.
+
+:code:`pm2 start redbot --name "<Insert a name here>" --interpreter "<Location to your Python Interpreter>" -- <Red Instance> --no-prompt`
+
+.. code-block:: none
+
+    Arguments to replace.
+
+    --name ""
+    A name to identify the bot within pm2, this is not your Red instance.
+
+    --interpreter ""
+    The location of your Python interpreter, to find out where that is use the following command:
+    which python3.6
+
+    <Red Instance>
+    The name of your Red instance.
+
+------------------------------
+Ensuring that PM2 stays online
+------------------------------
+
+To make sure that PM2 stays online and persistence between machine restarts, run the following commands:
+
+:code:`pm2 save` & :code:`pm2 startup`

docs/autostart_systemd.rst

@@ -8,11 +8,23 @@ Setting up auto-restart using systemd on Linux
 Creating the service file
 -------------------------
 
-Create the new service file:
+In order to create the service file, you will first need the location of your :code:`redbot` binary.
+
+.. code-block:: bash
+
+    # If redbot is installed in a virtualenv
+    source redenv/bin/activate
+
+    # If you are using pyenv
+    pyenv shell <name>
+
+    which redbot
+
+Then create the new service file:
 
 :code:`sudo -e /etc/systemd/system/red@.service`
 
-Paste the following and replace all instances of :code:`username` with the username your bot is running under (hopefully not root):
+Paste the following and replace all instances of :code:`username` with the username, and :code:`path` with the location you obtained above:
 
 .. code-block:: none
 
@@ -21,7 +33,7 @@ Paste the following and replace all instances of :code:`username` with the usern
     After=multi-user.target
 
     [Service]
-    ExecStart=/home/username/.local/bin/redbot %I --no-prompt
+    ExecStart=path %I --no-prompt
     User=username
     Group=username
     Type=idle

docs/conf.py

@@ -58,7 +58,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "Red - Discord Bot"
-copyright = "2018, Cog Creators"
+copyright = "2018-2019, Cog Creators"
 author = "Cog Creators"
 
 # The version info for the project you're documenting, acts as replacement for

docs/framework_apikeys.rst

@@ -18,7 +18,7 @@ and when accessed in the code it should be done by
 
 .. code-block:: python
 
-    await self.bot.get_shared_api_keys("twitch")
+    await self.bot.get_shared_api_tokens("twitch")
 
 Each service has its own dict of key, value pairs for each required key type. If there's only one key required then a name for the key is still required for storing and accessing.
 
@@ -30,7 +30,7 @@ and when accessed in the code it should be done by
 
 .. code-block:: python
 
-    await self.bot.get_shared_api_keys("youtube")
+    await self.bot.get_shared_api_tokens("youtube")
 
 
 ***********
@@ -42,7 +42,21 @@ Basic Usage
     class MyCog:
         @commands.command()
         async def youtube(self, ctx, user: str):
-            youtube_keys = await self.bot.get_shared_api_keys("youtube")
+            youtube_keys = await self.bot.get_shared_api_tokens("youtube")
             if youtube_keys.get("api_key") is None:
                 return await ctx.send("The YouTube API key has not been set.")
             # Use the API key to access content as you normally would
+
+
+***************
+Event Reference
+***************
+
+.. function:: on_red_api_tokens_update(service_name, api_tokens)
+
+    Dispatched when service's api keys are updated.
+
+    :param service_name: Name of the service.
+    :type service_name: :class:`str`
+    :param api_tokens: New Mapping of token names to tokens. This contains api tokens that weren't changed too.
+    :type api_tokens: Mapping[:class:`str`, :class:`str`]

docs/framework_config.rst

@@ -429,7 +429,3 @@ JSON Driver
 .. autoclass:: redbot.core.drivers.JsonDriver
     :members:
 
-Mongo Driver
-^^^^^^^^^^^^
-.. autoclass:: redbot.core.drivers.MongoDriver
-    :members:

docs/framework_downloader.rst

@@ -35,6 +35,9 @@ Keys specific to the cog info.json (case sensitive)
 - ``max_bot_version`` (string) - Max version number of Red in the format ``MAJOR.MINOR.MICRO``,
   if ``min_bot_version`` is newer than ``max_bot_version``, ``max_bot_version`` will be ignored
 
+- ``min_python_version`` (list of integers) - Min version number of Python
+  in the format ``[MAJOR, MINOR, PATCH]``
+
 - ``hidden`` (bool) - Determines if a cog is visible in the cog list for a repo.
 
 - ``disabled`` (bool) - Determines if a cog is available for install.
@@ -68,6 +71,12 @@ Installable
 .. autoclass:: Installable
     :members:
 
+InstalledModule
+^^^^^^^^^^^^^^^
+
+.. autoclass:: InstalledModule
+    :members:
+
 .. automodule:: redbot.cogs.downloader.repo_manager
 
 Repo

docs/getting_started.rst

@@ -115,8 +115,8 @@ to use one, do it like this: ``[p]cleanup messages 10``
 Cogs
 ----
 
-Red is built with cogs, fancy term for plugins. They are
+Red is built with cogs, a fancy term for plugins. They are
-modules that enhance the Red functionalities. They contain
+modules that add functionality to Red. They contain
 commands to use.
 
 Red comes with 19 cogs containing the basic features, such
@@ -162,10 +162,10 @@ there are hundreds of cogs available!
 
 .. 26-cogs not available, let's use my repo :3
 
-Cogs comes with repositories. A repository is a container of cogs
+Cogs come in repositories. A repository is a container of cogs
 that you can install. Let's suppose you want to install the ``say``
 cog from the repository ``Laggrons-Dumb-Cogs``. You'll first need
-to install the repository.
+to add the repository.
 
 .. code-block:: none
 
@@ -173,7 +173,7 @@ to install the repository.
 
 .. note:: You may need to specify a branch. If so, add its name after the link.
 
-Then you can add the cog
+Then you can install the cog
 
 .. code-block:: none
 
@@ -195,7 +195,7 @@ the level of permission needed for a command.
 Bot owner
 ~~~~~~~~~
 
-The bot owner can access all commands on every guild. He can also use
+The bot owner can access all commands on every guild. They can also use
 exclusive commands that can interact with the global settings
 or system files.
 
@@ -214,7 +214,7 @@ Administrator
 ~~~~~~~~~~~~~
 
 The administrator is defined by its roles. You can set multiple admin roles
-with the ``[p]addadminrole`` and ``[p]removeadminrole`` commands.
+with the ``[p]set addadminrole`` and ``[p]set removeadminrole`` commands.
 
 For example, in the mod cog, an admin can use the ``[p]modset`` command
 which defines the cog settings.
@@ -224,7 +224,7 @@ Moderator
 ~~~~~~~~~
 
 A moderator is a step above the average users. You can set multiple moderator
-roles with the ``[p]addmodrole`` and ``[p]removemodrole`` commands.
+roles with the ``[p]set addmodrole`` and ``[p]set removemodrole`` commands.
 
 For example, in the mod cog (again), a mod will be able to mute, kick and ban;
 but he won't be able to modify the cog settings with the ``[p]modset`` command.

docs/host-list.rst

@@ -56,6 +56,9 @@ Others
 |`Google Cloud                        |Same as AWS, but it's Google.                        |
 |<https://cloud.google.com/compute/>`_|                                                     |
 +-------------------------------------+-----------------------------------------------------+
+|`Microsoft Azure                     |Same as AWS, but it's Microsoft.                     |
+|<https://azure.microsoft.com>`_      |                                                     |
++-------------------------------------+-----------------------------------------------------+
 |`LowEndBox <http://lowendbox.com/>`_ |A curator for lower specced servers.                 |
 +-------------------------------------+-----------------------------------------------------+
 

docs/index.rst

@@ -16,6 +16,7 @@ Welcome to Red - Discord Bot's documentation!
     install_linux_mac
     venv_guide
     autostart_systemd
+    autostart_pm2
 
 .. toctree::
     :maxdepth: 2

docs/install_linux_mac.rst

@@ -265,18 +265,12 @@ Choose one of the following commands to install Red.
 
         python3.7 -m pip install --user -U Red-DiscordBot
 
-To install without MongoDB support:
+To install without additional config backend support:
 
 .. code-block:: none
 
     python3.7 -m pip install -U Red-DiscordBot
 
-Or, to install with MongoDB support:
-
-.. code-block:: none
-
-    python3.7 -m pip install -U Red-DiscordBot[mongo]
-
 Or, to install with PostgreSQL support:
 
 .. code-block:: none
@@ -286,7 +280,11 @@ Or, to install with PostgreSQL support:
 .. note::
 
   To install the development version, replace ``Red-DiscordBot`` in the above commands with the
-  following link:
+  link below. **The development version of the bot contains experimental changes. It is not
+  intended for normal users.** We will not support anyone using the development version in any
+  support channels. Using the development version may break third party cogs and not all core
+  commands may work. Downgrading to stable after installing the development version may cause
+  data loss, crashes or worse.
 
   .. code-block:: none
 

docs/install_windows.rst

@@ -76,12 +76,6 @@ Installing Red
 
         python -m pip install -U Red-DiscordBot
 
-  * With MongoDB support:
-
-    .. code-block:: none
-
-        python -m pip install -U Red-DiscordBot[mongo]
-
   * With PostgreSQL support:
 
     .. code-block:: none
@@ -91,7 +85,11 @@ Installing Red
   .. note::
 
       To install the development version, replace ``Red-DiscordBot`` in the above commands with the
-      following link:
+      link below. **The development version of the bot contains experimental changes. It is not
+      intended for normal users.** We will not support anyone using the development version in any
+      support channels. Using the development version may break third party cogs and not all core
+      commands may work. Downgrading to stable after installing the development version may cause
+      data loss, crashes or worse.
 
       .. code-block:: none
 

redbot/__init__.py

@@ -1,3 +1,4 @@
+import asyncio as _asyncio
 import re as _re
 import sys as _sys
 import warnings as _warnings
@@ -15,8 +16,13 @@ from typing import (
 
 MIN_PYTHON_VERSION = (3, 7, 0)
 
-__all__ = ["MIN_PYTHON_VERSION", "__version__", "version_info", "VersionInfo"]
+__all__ = [
-
+    "MIN_PYTHON_VERSION",
+    "__version__",
+    "version_info",
+    "VersionInfo",
+    "_update_event_loop_policy",
+]
 if _sys.version_info < MIN_PYTHON_VERSION:
     print(
         f"Python {'.'.join(map(str, MIN_PYTHON_VERSION))} is required to run Red, but you have "
@@ -173,7 +179,20 @@ class VersionInfo:
         )
 
 
-__version__ = "3.1.6"
+def _update_event_loop_policy():
+    if _sys.platform == "win32":
+        _asyncio.set_event_loop_policy(_asyncio.WindowsProactorEventLoopPolicy())
+    elif _sys.implementation.name == "cpython":
+        # Let's not force this dependency, uvloop is much faster on cpython
+        try:
+            import uvloop as _uvloop
+        except ImportError:
+            pass
+        else:
+            _asyncio.set_event_loop_policy(_uvloop.EventLoopPolicy())
+
+
+__version__ = "3.1.8"
 version_info = VersionInfo.from_str(__version__)
 
 # Filter fuzzywuzzy slow sequence matcher warning

redbot/__main__.py

@@ -6,24 +6,19 @@ import asyncio
 import json
 import logging
 import os
+import shutil
 import sys
+from copy import deepcopy
+from pathlib import Path
 
 import discord
 
 # Set the event loop policies here so any subsequent `get_event_loop()`
 # calls, in particular those as a result of the following imports,
 # return the correct loop object.
-if sys.platform == "win32":
+from redbot import _update_event_loop_policy
-    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
-elif sys.implementation.name == "cpython":
+_update_event_loop_policy()
-    # Let's not force this dependency, uvloop is much faster on cpython
-    try:
-        import uvloop
-    except ImportError:
-        uvloop = None
-        pass
-    else:
-        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
 import redbot.logging
 from redbot.core.bot import Red, ExitCodes
@@ -31,7 +26,8 @@ from redbot.core.cog_manager import CogManagerUI
 from redbot.core.global_checks import init_global_checks
 from redbot.core.events import init_events
 from redbot.core.cli import interactive_config, confirm, parse_cli_flags
-from redbot.core.core_commands import Core
+from redbot.core.core_commands import Core, license_info_command
+from redbot.setup import get_data_dir, get_name, save_config
 from redbot.core.dev_commands import Dev
 from redbot.core import __version__, modlog, bank, data_manager, drivers
 from signal import SIGTERM
@@ -56,6 +52,12 @@ async def _get_prefix_and_token(red, indict):
     indict["prefix"] = await red._config.prefix()
 
 
+def _get_instance_names():
+    with data_manager.config_file.open(encoding="utf-8") as fs:
+        data = json.load(fs)
+    return sorted(data.keys())
+
+
 def list_instances():
     if not data_manager.config_file.exists():
         print(
@@ -64,22 +66,164 @@ def list_instances():
         )
         sys.exit(1)
     else:
-        with data_manager.config_file.open(encoding="utf-8") as fs:
-            data = json.load(fs)
         text = "Configured Instances:\n\n"
-        for instance_name in sorted(data.keys()):
+        for instance_name in _get_instance_names():
             text += "{}\n".format(instance_name)
         print(text)
         sys.exit(0)
 
 
+def edit_instance(red, cli_flags):
+    no_prompt = cli_flags.no_prompt
+    token = cli_flags.token
+    owner = cli_flags.owner
+    old_name = cli_flags.instance_name
+    new_name = cli_flags.edit_instance_name
+    data_path = cli_flags.edit_data_path
+    copy_data = cli_flags.copy_data
+    confirm_overwrite = cli_flags.overwrite_existing_instance
+
+    if data_path is None and copy_data:
+        print("--copy-data can't be used without --edit-data-path argument")
+        sys.exit(1)
+    if new_name is None and confirm_overwrite:
+        print("--overwrite-existing-instance can't be used without --edit-instance-name argument")
+        sys.exit(1)
+    if no_prompt and all(to_change is None for to_change in (token, owner, new_name, data_path)):
+        print(
+            "No arguments to edit were provided. Available arguments (check help for more "
+            "information): --edit-instance-name, --edit-data-path, --copy-data, --owner, --token"
+        )
+        sys.exit(1)
+
+    _edit_token(red, token, no_prompt)
+    _edit_owner(red, owner, no_prompt)
+
+    data = deepcopy(data_manager.basic_config)
+    name = _edit_instance_name(old_name, new_name, confirm_overwrite, no_prompt)
+    _edit_data_path(data, data_path, copy_data, no_prompt)
+
+    save_config(name, data)
+    if old_name != name:
+        save_config(old_name, {}, remove=True)
+
+
+def _edit_token(red, token, no_prompt):
+    if token:
+        if not len(token) >= 50:
+            print(
+                "The provided token doesn't look a valid Discord bot token."
+                " Instance's token will remain unchanged.\n"
+            )
+            return
+        red.loop.run_until_complete(red._config.token.set(token))
+    elif not no_prompt and confirm("Would you like to change instance's token?", default=False):
+        interactive_config(red, False, True, print_header=False)
+        print("Token updated.\n")
+
+
+def _edit_owner(red, owner, no_prompt):
+    if owner:
+        if not (15 <= len(str(owner)) <= 21):
+            print(
+                "The provided owner id doesn't look like a valid Discord user id."
+                " Instance's owner will remain unchanged."
+            )
+            return
+        red.loop.run_until_complete(red._config.owner.set(owner))
+    elif not no_prompt and confirm("Would you like to change instance's owner?", default=False):
+        print(
+            "Remember:\n"
+            "ONLY the person who is hosting Red should be owner."
+            " This has SERIOUS security implications."
+            " The owner can access any data that is present on the host system.\n"
+        )
+        if confirm("Are you sure you want to change instance's owner?", default=False):
+            print("Please enter a Discord user id for new owner:")
+            while True:
+                owner_id = input("> ").strip()
+                if not (15 <= len(owner_id) <= 21 and owner_id.isdecimal()):
+                    print("That doesn't look like a valid Discord user id.")
+                    continue
+                owner_id = int(owner_id)
+                red.loop.run_until_complete(red._config.owner.set(owner_id))
+                print("Owner updated.")
+                break
+        else:
+            print("Instance's owner will remain unchanged.")
+        print()
+
+
+def _edit_instance_name(old_name, new_name, confirm_overwrite, no_prompt):
+    if new_name:
+        name = new_name
+        if name in _get_instance_names() and not confirm_overwrite:
+            name = old_name
+            print(
+                "An instance with this name already exists.\n"
+                "If you want to remove the existing instance and replace it with this one,"
+                " run this command with --overwrite-existing-instance flag."
+            )
+    elif not no_prompt and confirm("Would you like to change the instance name?", default=False):
+        name = get_name()
+        if name in _get_instance_names():
+            print(
+                "WARNING: An instance already exists with this name. "
+                "Continuing will overwrite the existing instance config."
+            )
+            if not confirm(
+                "Are you absolutely certain you want to continue with this instance name?",
+                default=False,
+            ):
+                print("Instance name will remain unchanged.")
+                name = old_name
+            else:
+                print("Instance name updated.")
+            print()
+    else:
+        name = old_name
+    return name
+
+
+def _edit_data_path(data, data_path, copy_data, no_prompt):
+    # This modifies the passed dict.
+    if data_path:
+        data["DATA_PATH"] = data_path
+        if copy_data and not _copy_data(data):
+            print("Can't copy data to non-empty location. Data location will remain unchanged.")
+            data["DATA_PATH"] = data_manager.basic_config["DATA_PATH"]
+    elif not no_prompt and confirm("Would you like to change the data location?", default=False):
+        data["DATA_PATH"] = get_data_dir()
+        if confirm(
+            "Do you want to copy the data from old location?", default=True
+        ) and not _copy_data(data):
+            print("Can't copy the data to non-empty location.")
+            if not confirm("Do you still want to use the new data location?"):
+                data["DATA_PATH"] = data_manager.basic_config["DATA_PATH"]
+                print("Data location will remain unchanged.")
+            else:
+                print("Data location updated.")
+
+
+def _copy_data(data):
+    if Path(data["DATA_PATH"]).exists():
+        if any(os.scandir(data["DATA_PATH"])):
+            return False
+        else:
+            # this is needed because copytree doesn't work when destination folder exists
+            # Python 3.8 has `dirs_exist_ok` option for that
+            os.rmdir(data["DATA_PATH"])
+    shutil.copytree(data_manager.basic_config["DATA_PATH"], data["DATA_PATH"])
+    return True
+
+
 async def sigterm_handler(red, log):
     log.info("SIGTERM received. Quitting...")
     await red.shutdown(restart=False)
 
 
 def main():
-    description = "Red V3 (c) Cog Creators"
+    description = "Red V3"
     cli_flags = parse_cli_flags(sys.argv[1:])
     if cli_flags.list_instances:
         list_instances()
@@ -87,7 +231,7 @@ def main():
         print(description)
         print("Current Version: {}".format(__version__))
         sys.exit(0)
-    elif not cli_flags.instance_name and not cli_flags.no_instance:
+    elif not cli_flags.instance_name and (not cli_flags.no_instance or cli_flags.edit):
         print("Error: No instance name was provided!")
         sys.exit(1)
     if cli_flags.no_instance:
@@ -116,6 +260,16 @@ def main():
         cli_flags=cli_flags, description=description, dm_help=None, fetch_offline_members=True
     )
     loop.run_until_complete(red._maybe_update_config())
+
+    if cli_flags.edit:
+        try:
+            edit_instance(red, cli_flags)
+        except (KeyboardInterrupt, EOFError):
+            print("Aborted!")
+        finally:
+            loop.run_until_complete(driver_cls.teardown())
+        sys.exit(0)
+
     init_global_checks(red)
     init_events(red, cli_flags)
 
@@ -128,6 +282,7 @@ def main():
 
     red.add_cog(Core(red))
     red.add_cog(CogManagerUI())
+    red.add_command(license_info_command)
     if cli_flags.dev:
         red.add_cog(Dev())
     # noinspection PyProtectedMember
@@ -157,13 +312,12 @@ def main():
         loop.run_until_complete(red.http.close())
         sys.exit(0)
     try:
-        loop.run_until_complete(red.start(token, bot=True))
+        loop.run_until_complete(red.start(token, bot=True, cli_flags=cli_flags))
     except discord.LoginFailure:
         log.critical("This token doesn't seem to be valid.")
         db_token = loop.run_until_complete(red._config.token())
         if db_token and not cli_flags.no_prompt:
-            print("\nDo you want to reset the token? (y/n)")
+            if confirm("\nDo you want to reset the token?"):
-            if confirm("> "):
                 loop.run_until_complete(red._config.token.set(""))
                 print("Token has been reset.")
     except KeyboardInterrupt:

redbot/cogs/admin/announcer.py

@@ -3,6 +3,7 @@ import asyncio
 import discord
 from redbot.core import commands
 from redbot.core.i18n import Translator
+from redbot.core.utils.chat_formatting import humanize_list, inline
 
 _ = Translator("Announcer", __file__)
 
@@ -53,7 +54,7 @@ class Announcer:
 
     async def announcer(self):
         guild_list = self.ctx.bot.guilds
-        bot_owner = (await self.ctx.bot.application_info()).owner
+        failed = []
         for g in guild_list:
             if not self.active:
                 return
@@ -66,9 +67,14 @@ class Announcer:
             try:
                 await channel.send(self.message)
             except discord.Forbidden:
-                await bot_owner.send(
+                failed.append(str(g.id))
-                    _("I could not announce to server: {server.id}").format(server=g)
-                )
             await asyncio.sleep(0.5)
 
+        msg = (
+            _("I could not announce to the following server: ")
+            if len(failed) == 1
+            else _("I could not announce to the following servers: ")
+        )
+        msg += humanize_list(tuple(map(inline, failed)))
+        await self.ctx.bot.send_to_owners(msg)
         self.active = False

redbot/cogs/audio/__init__.py

@@ -3,7 +3,6 @@ from redbot.core import commands
 from .audio import Audio
 
 
-async def setup(bot: commands.Bot):
+def setup(bot: commands.Bot):
     cog = Audio(bot)
-    await cog.initialize()
     bot.add_cog(cog)

redbot/cogs/audio/apis.py

@@ -9,7 +9,7 @@ import random
 import time
 import traceback
 from collections import namedtuple
-from typing import Callable, Dict, List, Mapping, NoReturn, Optional, Tuple, Union
+from typing import Callable, Dict, List, Mapping, Optional, Tuple, Union
 
 try:
     from sqlite3 import Error as SQLError
@@ -32,7 +32,7 @@ from lavalink.rest_api import LoadResult
 from redbot.core import Config, commands
 from redbot.core.bot import Red
 from redbot.core.i18n import Translator, cog_i18n
-from . import dataclasses
+from . import audio_dataclasses
 from .errors import InvalidTableError, SpotifyFetchError, YouTubeApiError
 from .playlists import get_playlist
 from .utils import CacheLevel, Notifier, is_allowed, queue_duration, track_limit
@@ -193,7 +193,7 @@ class SpotifyAPI:
                 )
             return await r.json()
 
-    async def _get_auth(self) -> NoReturn:
+    async def _get_auth(self):
         if self.client_id is None or self.client_secret is None:
             tokens = await self.bot.get_shared_api_tokens("spotify")
             self.client_id = tokens.get("client_id", "")
@@ -331,7 +331,7 @@ class MusicCache:
         self._lock: asyncio.Lock = asyncio.Lock()
         self.config: Optional[Config] = None
 
-    async def initialize(self, config: Config) -> NoReturn:
+    async def initialize(self, config: Config):
         if HAS_SQL:
             await self.database.connect()
 
@@ -348,12 +348,12 @@ class MusicCache:
             await self.database.execute(query=_CREATE_UNIQUE_INDEX_SPOTIFY_TABLE)
         self.config = config
 
-    async def close(self) -> NoReturn:
+    async def close(self):
         if HAS_SQL:
             await self.database.execute(query="PRAGMA optimize;")
             await self.database.disconnect()
 
-    async def insert(self, table: str, values: List[dict]) -> NoReturn:
+    async def insert(self, table: str, values: List[dict]):
         # if table == "spotify":
         #     return
         if HAS_SQL:
@@ -363,7 +363,7 @@ class MusicCache:
 
             await self.database.execute_many(query=query, values=values)
 
-    async def update(self, table: str, values: Dict[str, str]) -> NoReturn:
+    async def update(self, table: str, values: Dict[str, str]):
         # if table == "spotify":
         #     return
         if HAS_SQL:
@@ -746,7 +746,7 @@ class MusicCache:
                 if val:
                     try:
                         result, called_api = await self.lavalink_query(
-                            ctx, player, dataclasses.Query.process_input(val)
+                            ctx, player, audio_dataclasses.Query.process_input(val)
                         )
                     except (RuntimeError, aiohttp.ServerDisconnectedError):
                         lock(ctx, False)
@@ -805,7 +805,7 @@ class MusicCache:
                     ctx.guild,
                     (
                         f"{single_track.title} {single_track.author} {single_track.uri} "
-                        f"{str(dataclasses.Query.process_input(single_track))}"
+                        f"{str(audio_dataclasses.Query.process_input(single_track))}"
                     ),
                 ):
                     has_not_allowed = True
@@ -911,7 +911,7 @@ class MusicCache:
         self,
         ctx: commands.Context,
         player: lavalink.Player,
-        query: dataclasses.Query,
+        query: audio_dataclasses.Query,
         forced: bool = False,
     ) -> Tuple[LoadResult, bool]:
         """
@@ -925,7 +925,7 @@ class MusicCache:
             The context this method is being called under.
         player : lavalink.Player
             The player who's requesting the query.
-        query: dataclasses.Query
+        query: audio_dataclasses.Query
             The Query object for the query in question.
         forced:bool
             Whether or not to skip cache and call API first..
@@ -939,7 +939,7 @@ class MusicCache:
         )
         cache_enabled = CacheLevel.set_lavalink().is_subset(current_cache_level)
         val = None
-        _raw_query = dataclasses.Query.process_input(query)
+        _raw_query = audio_dataclasses.Query.process_input(query)
         query = str(_raw_query)
         if cache_enabled and not forced and not _raw_query.is_local:
             update = True
@@ -1003,14 +1003,10 @@ class MusicCache:
                     tasks = self._tasks[ctx.message.id]
                     del self._tasks[ctx.message.id]
                     await asyncio.gather(
-                        *[asyncio.ensure_future(self.insert(*a)) for a in tasks["insert"]],
+                        *[self.insert(*a) for a in tasks["insert"]], return_exceptions=True
-                        loop=self.bot.loop,
-                        return_exceptions=True,
                     )
                     await asyncio.gather(
-                        *[asyncio.ensure_future(self.update(*a)) for a in tasks["update"]],
+                        *[self.update(*a) for a in tasks["update"]], return_exceptions=True
-                        loop=self.bot.loop,
-                        return_exceptions=True,
                     )
                 log.debug(f"Completed database writes for {lock_id} " f"({lock_author})")
 
@@ -1025,14 +1021,10 @@ class MusicCache:
                 self._tasks = {}
 
                 await asyncio.gather(
-                    *[asyncio.ensure_future(self.insert(*a)) for a in tasks["insert"]],
+                    *[self.insert(*a) for a in tasks["insert"]], return_exceptions=True
-                    loop=self.bot.loop,
-                    return_exceptions=True,
                 )
                 await asyncio.gather(
-                    *[asyncio.ensure_future(self.update(*a)) for a in tasks["update"]],
+                    *[self.update(*a) for a in tasks["update"]], return_exceptions=True
-                    loop=self.bot.loop,
-                    return_exceptions=True,
                 )
             log.debug("Completed pending writes to database have finished")
 
@@ -1096,7 +1088,9 @@ class MusicCache:
             if not tracks:
                 ctx = namedtuple("Context", "message")
                 results, called_api = await self.lavalink_query(
-                    ctx(player.channel.guild), player, dataclasses.Query.process_input(_TOP_100_US)
+                    ctx(player.channel.guild),
+                    player,
+                    audio_dataclasses.Query.process_input(_TOP_100_US),
                 )
                 tracks = list(results.tracks)
         if tracks:
@@ -1107,7 +1101,7 @@ class MusicCache:
 
             while valid is False and multiple:
                 track = random.choice(tracks)
-                query = dataclasses.Query.process_input(track)
+                query = audio_dataclasses.Query.process_input(track)
                 if not query.valid:
                     continue
                 if query.is_local and not query.track.exists():
@@ -1116,7 +1110,7 @@ class MusicCache:
                     player.channel.guild,
                     (
                         f"{track.title} {track.author} {track.uri} "
-                        f"{str(dataclasses.Query.process_input(track))}"
+                        f"{str(audio_dataclasses.Query.process_input(track))}"
                     ),
                 ):
                     log.debug(

redbot/cogs/audio/audio.py

@@ -34,7 +34,7 @@ from redbot.core.utils.menus import (
     start_adding_reactions,
 )
 from redbot.core.utils.predicates import MessagePredicate, ReactionPredicate
-from . import dataclasses
+from . import audio_dataclasses
 from .apis import MusicCache, HAS_SQL, _ERROR
 from .checks import can_have_caching
 from .converters import ComplexScopeParser, ScopeParser, get_lazy_converter, get_playlist_converter
@@ -142,7 +142,11 @@ class Audio(commands.Cog):
         self.play_lock = {}
 
         self._manager: Optional[ServerManager] = None
-        self.bot.dispatch("red_audio_initialized", self)
+        # These has to be a task since this requires the bot to be ready
+        # If it waits for ready in startup, we cause a deadlock during initial load
+        # as initial load happens before the bot can ever be ready.
+        self._init_task = self.bot.loop.create_task(self.initialize())
+        self._ready_event = asyncio.Event()
 
     @property
     def owns_autoplay(self):
@@ -166,9 +170,14 @@ class Audio(commands.Cog):
         self._cog_id = None
 
     async def cog_before_invoke(self, ctx: commands.Context):
+        await self._ready_event.wait()
+        # check for unsupported arch
+        # Check on this needs refactoring at a later date
+        # so that we have a better way to handle the tasks
         if self.llsetup in [ctx.command, ctx.command.root_parent]:
             pass
-        elif self._connect_task.cancelled():
+
+        elif self._connect_task and self._connect_task.cancelled():
             await ctx.send(
                 "You have attempted to run Audio's Lavalink server on an unsupported"
                 " architecture. Only settings related commands will be available."
@@ -176,6 +185,7 @@ class Audio(commands.Cog):
             raise RuntimeError(
                 "Not running audio command due to invalid machine architecture for Lavalink."
             )
+
         dj_enabled = await self.config.guild(ctx.guild).dj_enabled()
         if dj_enabled:
             dj_role_obj = ctx.guild.get_role(await self.config.guild(ctx.guild).dj_role())
@@ -185,13 +195,13 @@ class Audio(commands.Cog):
                 await self._embed_msg(ctx, _("No DJ role found. Disabling DJ mode."))
 
     async def initialize(self):
-        pass_config_to_dependencies(self.config, self.bot, await self.config.localpath())
+        await self.bot.wait_until_ready()
+        # Unlike most cases, we want the cache to exit before migration.
         await self.music_cache.initialize(self.config)
-        asyncio.ensure_future(
+        await self._migrate_config(
-            self._migrate_config(
+            from_version=await self.config.schema_version(), to_version=_SCHEMA_VERSION
-                from_version=await self.config.schema_version(), to_version=_SCHEMA_VERSION
-            )
         )
+        pass_config_to_dependencies(self.config, self.bot, await self.config.localpath())
         self._restart_connect()
         self._disconnect_task = self.bot.loop.create_task(self.disconnect_timer())
         lavalink.register_event_listener(self.event_handler)
@@ -209,6 +219,9 @@ class Audio(commands.Cog):
                     await self.bot.send_to_owners(page)
             log.critical(error_message)
 
+        self._ready_event.set()
+        self.bot.dispatch("red_audio_initialized", self)
+
     async def _migrate_config(self, from_version: int, to_version: int):
         database_entries = []
         time_now = str(datetime.datetime.now(datetime.timezone.utc))
@@ -253,7 +266,7 @@ class Audio(commands.Cog):
                     cast(discord.Guild, discord.Object(id=guild_id))
                 ).clear_raw("playlists")
         if database_entries and HAS_SQL:
-            asyncio.ensure_future(self.music_cache.insert("lavalink", database_entries))
+            await self.music_cache.insert("lavalink", database_entries)
 
     def _restart_connect(self):
         if self._connect_task:
@@ -366,7 +379,9 @@ class Audio(commands.Cog):
         async def _players_check():
             try:
                 get_single_title = lavalink.active_players()[0].current.title
-                query = dataclasses.Query.process_input(lavalink.active_players()[0].current.uri)
+                query = audio_dataclasses.Query.process_input(
+                    lavalink.active_players()[0].current.uri
+                )
                 if get_single_title == "Unknown title":
                     get_single_title = lavalink.active_players()[0].current.uri
                     if not get_single_title.startswith("http"):
@@ -463,18 +478,18 @@ class Audio(commands.Cog):
                     )
                     await notify_channel.send(embed=embed)
 
-                query = dataclasses.Query.process_input(player.current.uri)
+                query = audio_dataclasses.Query.process_input(player.current.uri)
 
                 if query.is_local if player.current else False:
                     if player.current.title != "Unknown title":
                         description = "**{} - {}**\n{}".format(
                             player.current.author,
                             player.current.title,
-                            dataclasses.LocalPath(player.current.uri).to_string_hidden(),
+                            audio_dataclasses.LocalPath(player.current.uri).to_string_hidden(),
                         )
                     else:
                         description = "{}".format(
-                            dataclasses.LocalPath(player.current.uri).to_string_hidden()
+                            audio_dataclasses.LocalPath(player.current.uri).to_string_hidden()
                         )
                 else:
                     description = "**[{}]({})**".format(player.current.title, player.current.uri)
@@ -532,9 +547,9 @@ class Audio(commands.Cog):
             message_channel = player.fetch("channel")
             if message_channel:
                 message_channel = self.bot.get_channel(message_channel)
-                query = dataclasses.Query.process_input(player.current.uri)
+                query = audio_dataclasses.Query.process_input(player.current.uri)
                 if player.current and query.is_local:
-                    query = dataclasses.Query.process_input(player.current.uri)
+                    query = audio_dataclasses.Query.process_input(player.current.uri)
                     if player.current.title == "Unknown title":
                         description = "{}".format(query.track.to_string_hidden())
                     else:
@@ -590,7 +605,7 @@ class Audio(commands.Cog):
         player.store("channel", channel.id)
         player.store("guild", guild.id)
         await self._data_check(guild.me)
-        query = dataclasses.Query.process_input(query)
+        query = audio_dataclasses.Query.process_input(query)
         ctx = namedtuple("Context", "message")
         results, called_api = await self.music_cache.lavalink_query(ctx(guild), player, query)
 
@@ -985,7 +1000,7 @@ class Audio(commands.Cog):
     @audioset.command()
     @checks.mod_or_permissions(administrator=True)
     async def emptydisconnect(self, ctx: commands.Context, seconds: int):
-        """Auto-disconnection after x seconds while stopped. 0 to disable."""
+        """Auto-disconnect from channel when bot is alone in it for x seconds. 0 to disable."""
         if seconds < 0:
             return await self._embed_msg(ctx, _("Can't be less than zero."))
         if 10 > seconds > 0:
@@ -1094,7 +1109,7 @@ class Audio(commands.Cog):
             with contextlib.suppress(discord.HTTPException):
                 await info.delete()
             return
-        temp = dataclasses.LocalPath(local_path, forced=True)
+        temp = audio_dataclasses.LocalPath(local_path, forced=True)
         if not temp.exists() or not temp.is_dir():
             return await self._embed_msg(
                 ctx,
@@ -1536,7 +1551,7 @@ class Audio(commands.Cog):
                 int((datetime.datetime.utcnow() - connect_start).total_seconds())
             )
             try:
-                query = dataclasses.Query.process_input(p.current.uri)
+                query = audio_dataclasses.Query.process_input(p.current.uri)
                 if query.is_local:
                     if p.current.title == "Unknown title":
                         current_title = localtracks.LocalPath(p.current.uri).to_string_hidden()
@@ -1606,9 +1621,9 @@ class Audio(commands.Cog):
         bump_song = player.queue[bump_index]
         player.queue.insert(0, bump_song)
         removed = player.queue.pop(index)
-        query = dataclasses.Query.process_input(removed.uri)
+        query = audio_dataclasses.Query.process_input(removed.uri)
         if query.is_local:
-            localtrack = dataclasses.LocalPath(removed.uri)
+            localtrack = audio_dataclasses.LocalPath(removed.uri)
             if removed.title != "Unknown title":
                 description = "**{} - {}**\n{}".format(
                     removed.author, removed.title, localtrack.to_string_hidden()
@@ -1997,12 +2012,12 @@ class Audio(commands.Cog):
             await ctx.invoke(self.local_play, play_subfolders=play_subfolders)
         else:
             folder = folder.strip()
-            _dir = dataclasses.LocalPath.joinpath(folder)
+            _dir = audio_dataclasses.LocalPath.joinpath(folder)
             if not _dir.exists():
                 return await self._embed_msg(
                     ctx, _("No localtracks folder named {name}.").format(name=folder)
                 )
-            query = dataclasses.Query.process_input(_dir, search_subfolders=play_subfolders)
+            query = audio_dataclasses.Query.process_input(_dir, search_subfolders=play_subfolders)
             await self._local_play_all(ctx, query, from_search=False if not folder else True)
 
     @local.command(name="play")
@@ -2064,8 +2079,8 @@ class Audio(commands.Cog):
         all_tracks = await self._folder_list(
             ctx,
             (
-                dataclasses.Query.process_input(
+                audio_dataclasses.Query.process_input(
-                    dataclasses.LocalPath(
+                    audio_dataclasses.LocalPath(
                         await self.config.localpath()
                     ).localtrack_folder.absolute(),
                     search_subfolders=play_subfolders,
@@ -2081,18 +2096,18 @@ class Audio(commands.Cog):
         return await ctx.invoke(self.search, query=search_list)
 
     async def _localtracks_folders(self, ctx: commands.Context, search_subfolders=False):
-        audio_data = dataclasses.LocalPath(
+        audio_data = audio_dataclasses.LocalPath(
-            dataclasses.LocalPath(None).localtrack_folder.absolute()
+            audio_dataclasses.LocalPath(None).localtrack_folder.absolute()
         )
         if not await self._localtracks_check(ctx):
             return
 
         return audio_data.subfolders_in_tree() if search_subfolders else audio_data.subfolders()
 
-    async def _folder_list(self, ctx: commands.Context, query: dataclasses.Query):
+    async def _folder_list(self, ctx: commands.Context, query: audio_dataclasses.Query):
         if not await self._localtracks_check(ctx):
             return
-        query = dataclasses.Query.process_input(query)
+        query = audio_dataclasses.Query.process_input(query)
         if not query.track.exists():
             return
         return (
@@ -2102,12 +2117,12 @@ class Audio(commands.Cog):
         )
 
     async def _folder_tracks(
-        self, ctx, player: lavalink.player_manager.Player, query: dataclasses.Query
+        self, ctx, player: lavalink.player_manager.Player, query: audio_dataclasses.Query
     ):
         if not await self._localtracks_check(ctx):
             return
 
-        audio_data = dataclasses.LocalPath(None)
+        audio_data = audio_dataclasses.LocalPath(None)
         try:
             query.track.path.relative_to(audio_data.to_string())
         except ValueError:
@@ -2120,17 +2135,17 @@ class Audio(commands.Cog):
         return local_tracks
 
     async def _local_play_all(
-        self, ctx: commands.Context, query: dataclasses.Query, from_search=False
+        self, ctx: commands.Context, query: audio_dataclasses.Query, from_search=False
     ):
         if not await self._localtracks_check(ctx):
             return
         if from_search:
-            query = dataclasses.Query.process_input(
+            query = audio_dataclasses.Query.process_input(
                 query.track.to_string(), invoked_from="local folder"
             )
         await ctx.invoke(self.search, query=query)
 
-    async def _all_folder_tracks(self, ctx: commands.Context, query: dataclasses.Query):
+    async def _all_folder_tracks(self, ctx: commands.Context, query: audio_dataclasses.Query):
         if not await self._localtracks_check(ctx):
             return
 
@@ -2141,7 +2156,7 @@ class Audio(commands.Cog):
         )
 
     async def _localtracks_check(self, ctx: commands.Context):
-        folder = dataclasses.LocalPath(None)
+        folder = audio_dataclasses.LocalPath(None)
         if folder.localtrack_folder.exists():
             return True
         if ctx.invoked_with != "start":
@@ -2177,7 +2192,7 @@ class Audio(commands.Cog):
                 dur = "LIVE"
             else:
                 dur = lavalink.utils.format_time(player.current.length)
-            query = dataclasses.Query.process_input(player.current.uri)
+            query = audio_dataclasses.Query.process_input(player.current.uri)
             if query.is_local:
                 if not player.current.title == "Unknown title":
                     song = "**{track.author} - {track.title}**\n{uri}\n"
@@ -2189,8 +2204,8 @@ class Audio(commands.Cog):
             song += "\n\n{arrow}`{pos}`/`{dur}`"
             song = song.format(
                 track=player.current,
-                uri=dataclasses.LocalPath(player.current.uri).to_string_hidden()
+                uri=audio_dataclasses.LocalPath(player.current.uri).to_string_hidden()
-                if dataclasses.Query.process_input(player.current.uri).is_local
+                if audio_dataclasses.Query.process_input(player.current.uri).is_local
                 else player.current.uri,
                 arrow=arrow,
                 pos=pos,
@@ -2301,9 +2316,9 @@ class Audio(commands.Cog):
 
         if not player.current:
             return await self._embed_msg(ctx, _("Nothing playing."))
-        query = dataclasses.Query.process_input(player.current.uri)
+        query = audio_dataclasses.Query.process_input(player.current.uri)
         if query.is_local:
-            query = dataclasses.Query.process_input(player.current.uri)
+            query = audio_dataclasses.Query.process_input(player.current.uri)
             if player.current.title == "Unknown title":
                 description = "{}".format(query.track.to_string_hidden())
             else:
@@ -2436,7 +2451,7 @@ class Audio(commands.Cog):
             )
         if not await self._currency_check(ctx, guild_data["jukebox_price"]):
             return
-        query = dataclasses.Query.process_input(query)
+        query = audio_dataclasses.Query.process_input(query)
         if not query.valid:
             return await self._embed_msg(ctx, _("No tracks to play."))
         if query.is_spotify:
@@ -2593,7 +2608,7 @@ class Audio(commands.Cog):
             )
             playlists_search_page_list.append(embed)
         playlists_pick = await menu(ctx, playlists_search_page_list, playlist_search_controls)
-        query = dataclasses.Query.process_input(playlists_pick)
+        query = audio_dataclasses.Query.process_input(playlists_pick)
         if not query.valid:
             return await self._embed_msg(ctx, _("No tracks to play."))
         if not await self._currency_check(ctx, guild_data["jukebox_price"]):
@@ -2728,7 +2743,7 @@ class Audio(commands.Cog):
         elif player.current:
             await self._embed_msg(ctx, _("Adding a track to queue."))
 
-    async def _get_spotify_tracks(self, ctx: commands.Context, query: dataclasses.Query):
+    async def _get_spotify_tracks(self, ctx: commands.Context, query: audio_dataclasses.Query):
         if ctx.invoked_with in ["play", "genre"]:
             enqueue_tracks = True
         else:
@@ -2771,12 +2786,12 @@ class Audio(commands.Cog):
             self._play_lock(ctx, False)
             try:
                 if enqueue_tracks:
-                    new_query = dataclasses.Query.process_input(res[0])
+                    new_query = audio_dataclasses.Query.process_input(res[0])
                     new_query.start_time = query.start_time
                     return await self._enqueue_tracks(ctx, new_query)
                 else:
                     result, called_api = await self.music_cache.lavalink_query(
-                        ctx, player, dataclasses.Query.process_input(res[0])
+                        ctx, player, audio_dataclasses.Query.process_input(res[0])
                     )
                     tracks = result.tracks
                     if not tracks:
@@ -2808,7 +2823,9 @@ class Audio(commands.Cog):
                 ctx, _("This doesn't seem to be a supported Spotify URL or code.")
             )
 
-    async def _enqueue_tracks(self, ctx: commands.Context, query: Union[dataclasses.Query, list]):
+    async def _enqueue_tracks(
+        self, ctx: commands.Context, query: Union[audio_dataclasses.Query, list]
+    ):
         player = lavalink.get_player(ctx.guild.id)
         try:
             if self.play_lock[ctx.message.guild.id]:
@@ -2835,6 +2852,8 @@ class Audio(commands.Cog):
             if not tracks:
                 self._play_lock(ctx, False)
                 embed = discord.Embed(title=_("Nothing found."), colour=await ctx.embed_colour())
+                if result.exception_message:
+                    embed.set_footer(text=result.exception_message)
                 if await self.config.use_external_lavalink() and query.is_local:
                     embed.description = _(
                         "Local tracks will not work "
@@ -2861,7 +2880,7 @@ class Audio(commands.Cog):
                     ctx.guild,
                     (
                         f"{track.title} {track.author} {track.uri} "
-                        f"{str(dataclasses.Query.process_input(track))}"
+                        f"{str(audio_dataclasses.Query.process_input(track))}"
                     ),
                 ):
                     log.debug(f"Query is not allowed in {ctx.guild} ({ctx.guild.id})")
@@ -2921,7 +2940,7 @@ class Audio(commands.Cog):
                     ctx.guild,
                     (
                         f"{single_track.title} {single_track.author} {single_track.uri} "
-                        f"{str(dataclasses.Query.process_input(single_track))}"
+                        f"{str(audio_dataclasses.Query.process_input(single_track))}"
                     ),
                 ):
                     log.debug(f"Query is not allowed in {ctx.guild} ({ctx.guild.id})")
@@ -2954,17 +2973,17 @@ class Audio(commands.Cog):
                 return await self._embed_msg(
                     ctx, _("Nothing found. Check your Lavalink logs for details.")
                 )
-            query = dataclasses.Query.process_input(single_track.uri)
+            query = audio_dataclasses.Query.process_input(single_track.uri)
             if query.is_local:
                 if single_track.title != "Unknown title":
                     description = "**{} - {}**\n{}".format(
                         single_track.author,
                         single_track.title,
-                        dataclasses.LocalPath(single_track.uri).to_string_hidden(),
+                        audio_dataclasses.LocalPath(single_track.uri).to_string_hidden(),
                     )
                 else:
                     description = "{}".format(
-                        dataclasses.LocalPath(single_track.uri).to_string_hidden()
+                        audio_dataclasses.LocalPath(single_track.uri).to_string_hidden()
                     )
             else:
                 description = "**[{}]({})**".format(single_track.title, single_track.uri)
@@ -2985,7 +3004,11 @@ class Audio(commands.Cog):
         self._play_lock(ctx, False)
 
     async def _spotify_playlist(
-        self, ctx: commands.Context, stype: str, query: dataclasses.Query, enqueue: bool = False
+        self,
+        ctx: commands.Context,
+        stype: str,
+        query: audio_dataclasses.Query,
+        enqueue: bool = False,
     ):
 
         player = lavalink.get_player(ctx.guild.id)
@@ -3253,8 +3276,8 @@ class Audio(commands.Cog):
         ​ ​ ​ ​ ​ ​ ​ ​ Only editable by bot owner.
         ​ ​ ​ ​ **Guild**:
         ​ ​ ​ ​ ​ ​ ​ ​ Visible to all users in this guild.
-        ​ ​ ​ ​ ​ ​ ​ ​ Editable By Bot Owner, Guild Owner, Guild Admins,
+        ​ ​ ​ ​ ​ ​ ​ ​ Editable by bot owner, guild owner, guild admins,
-        ​ ​ ​ ​ ​ ​ ​ ​ Guild Mods, DJ Role and playlist creator.
+        ​ ​ ​ ​ ​ ​ ​ ​ guild mods, DJ role and playlist creator.
         ​ ​ ​ ​ **User**:
         ​ ​ ​ ​ ​ ​ ​ ​ Visible to all bot users, if --author is passed.
         ​ ​ ​ ​ ​ ​ ​ ​ Editable by bot owner and creator.
@@ -3338,7 +3361,7 @@ class Audio(commands.Cog):
             return
         player = lavalink.get_player(ctx.guild.id)
         to_append = await self._playlist_tracks(
-            ctx, player, dataclasses.Query.process_input(query)
+            ctx, player, audio_dataclasses.Query.process_input(query)
         )
         if not to_append:
             return await self._embed_msg(ctx, _("Could not find a track matching your query."))
@@ -3714,89 +3737,92 @@ class Audio(commands.Cog):
         ​ ​ ​ ​ [p]playlist dedupe MyGlobalPlaylist --scope Global
         ​ ​ ​ ​ [p]playlist dedupe MyPersonalPlaylist --scope User
         """
-        if scope_data is None:
+        async with ctx.typing():
-            scope_data = [PlaylistScope.GUILD.value, ctx.author, ctx.guild, False]
+            if scope_data is None:
-        scope, author, guild, specified_user = scope_data
+                scope_data = [PlaylistScope.GUILD.value, ctx.author, ctx.guild, False]
-        scope_name = humanize_scope(
+            scope, author, guild, specified_user = scope_data
-            scope, ctx=guild if scope == PlaylistScope.GUILD.value else author
+            scope_name = humanize_scope(
-        )
+                scope, ctx=guild if scope == PlaylistScope.GUILD.value else author
-
-        try:
-            playlist_id, playlist_arg = await self._get_correct_playlist_id(
-                ctx, playlist_matches, scope, author, guild, specified_user
-            )
-        except TooManyMatches as e:
-            return await self._embed_msg(ctx, str(e))
-        if playlist_id is None:
-            return await self._embed_msg(
-                ctx, _("Could not match '{arg}' to a playlist.").format(arg=playlist_arg)
             )
 
-        try:
+            try:
-            playlist = await get_playlist(playlist_id, scope, self.bot, guild, author)
+                playlist_id, playlist_arg = await self._get_correct_playlist_id(
-        except RuntimeError:
+                    ctx, playlist_matches, scope, author, guild, specified_user
-            return await self._embed_msg(
+                )
-                ctx,
+            except TooManyMatches as e:
-                _("Playlist {id} does not exist in {scope} scope.").format(
+                return await self._embed_msg(ctx, str(e))
-                    id=playlist_id, scope=humanize_scope(scope, the=True)
+            if playlist_id is None:
-                ),
+                return await self._embed_msg(
-            )
+                    ctx, _("Could not match '{arg}' to a playlist.").format(arg=playlist_arg)
-        except MissingGuild:
+                )
-            return await self._embed_msg(
-                ctx, _("You need to specify the Guild ID for the guild to lookup.")
-            )
 
-        if not await self.can_manage_playlist(scope, playlist, ctx, author, guild):
+            try:
-            return
+                playlist = await get_playlist(playlist_id, scope, self.bot, guild, author)
+            except RuntimeError:
+                return await self._embed_msg(
+                    ctx,
+                    _("Playlist {id} does not exist in {scope} scope.").format(
+                        id=playlist_id, scope=humanize_scope(scope, the=True)
+                    ),
+                )
+            except MissingGuild:
+                return await self._embed_msg(
+                    ctx, _("You need to specify the Guild ID for the guild to lookup.")
+                )
 
-        track_objects = playlist.tracks_obj
+            if not await self.can_manage_playlist(scope, playlist, ctx, author, guild):
-        original_count = len(track_objects)
+                return
-        unique_tracks = set()
-        unique_tracks_add = unique_tracks.add
-        track_objects = [
-            x for x in track_objects if not (x in unique_tracks or unique_tracks_add(x))
-        ]
 
-        tracklist = []
+            track_objects = playlist.tracks_obj
-        for track in track_objects:
+            original_count = len(track_objects)
-            track_keys = track._info.keys()
+            unique_tracks = set()
-            track_values = track._info.values()
+            unique_tracks_add = unique_tracks.add
-            track_id = track.track_identifier
+            track_objects = [
-            track_info = {}
+                x for x in track_objects if not (x in unique_tracks or unique_tracks_add(x))
-            for k, v in zip(track_keys, track_values):
+            ]
-                track_info[k] = v
-            keys = ["track", "info"]
-            values = [track_id, track_info]
-            track_obj = {}
-            for key, value in zip(keys, values):
-                track_obj[key] = value
-            tracklist.append(track_obj)
 
-        final_count = len(tracklist)
+            tracklist = []
-        if original_count - final_count != 0:
+            for track in track_objects:
-            update = {"tracks": tracklist, "url": None}
+                track_keys = track._info.keys()
-            await playlist.edit(update)
+                track_values = track._info.values()
+                track_id = track.track_identifier
+                track_info = {}
+                for k, v in zip(track_keys, track_values):
+                    track_info[k] = v
+                keys = ["track", "info"]
+                values = [track_id, track_info]
+                track_obj = {}
+                for key, value in zip(keys, values):
+                    track_obj[key] = value
+                tracklist.append(track_obj)
 
-        if original_count - final_count != 0:
+            final_count = len(tracklist)
-            await self._embed_msg(
+            if original_count - final_count != 0:
-                ctx,
+                update = {"tracks": tracklist, "url": None}
-                _(
+                await playlist.edit(update)
-                    "Removed {track_diff} duplicated "
+
-                    "tracks from {name} (`{id}`) [**{scope}**] playlist."
+            if original_count - final_count != 0:
-                ).format(
+                await self._embed_msg(
-                    name=playlist.name,
+                    ctx,
-                    id=playlist.id,
+                    _(
-                    track_diff=original_count - final_count,
+                        "Removed {track_diff} duplicated "
-                    scope=scope_name,
+                        "tracks from {name} (`{id}`) [**{scope}**] playlist."
-                ),
+                    ).format(
-            )
+                        name=playlist.name,
-        else:
+                        id=playlist.id,
-            await self._embed_msg(
+                        track_diff=original_count - final_count,
-                ctx,
+                        scope=scope_name,
-                _("{name} (`{id}`) [**{scope}**] playlist has no duplicate tracks.").format(
+                    ),
-                    name=playlist.name, id=playlist.id, scope=scope_name
+                )
-                ),
+                return
-            )
+            else:
+                await self._embed_msg(
+                    ctx,
+                    _("{name} (`{id}`) [**{scope}**] playlist has no duplicate tracks.").format(
+                        name=playlist.name, id=playlist.id, scope=scope_name
+                    ),
+                )
+                return
 
     @checks.is_owner()
     @playlist.command(name="download", usage="<playlist_name_OR_id> [v2=False] [args]")
@@ -3991,7 +4017,7 @@ class Audio(commands.Cog):
             spaces = "\N{EN SPACE}" * (len(str(len(playlist.tracks))) + 2)
             for track in playlist.tracks:
                 track_idx = track_idx + 1
-                query = dataclasses.Query.process_input(track["info"]["uri"])
+                query = audio_dataclasses.Query.process_input(track["info"]["uri"])
                 if query.is_local:
                     if track["info"]["title"] != "Unknown title":
                         msg += "`{}.` **{} - {}**\n{}{}\n".format(
@@ -4396,7 +4422,7 @@ class Audio(commands.Cog):
             return
         player = lavalink.get_player(ctx.guild.id)
         tracklist = await self._playlist_tracks(
-            ctx, player, dataclasses.Query.process_input(playlist_url)
+            ctx, player, audio_dataclasses.Query.process_input(playlist_url)
         )
         if tracklist is not None:
             playlist = await create_playlist(
@@ -4486,14 +4512,14 @@ class Audio(commands.Cog):
                     ctx.guild,
                     (
                         f"{track.title} {track.author} {track.uri} "
-                        f"{str(dataclasses.Query.process_input(track))}"
+                        f"{str(audio_dataclasses.Query.process_input(track))}"
                     ),
                 ):
                     log.debug(f"Query is not allowed in {ctx.guild} ({ctx.guild.id})")
                     continue
-                query = dataclasses.Query.process_input(track.uri)
+                query = audio_dataclasses.Query.process_input(track.uri)
                 if query.is_local:
-                    local_path = dataclasses.LocalPath(track.uri)
+                    local_path = audio_dataclasses.LocalPath(track.uri)
                     if not await self._localtracks_check(ctx):
                         pass
                     if not local_path.exists() and not local_path.is_file():
@@ -4779,7 +4805,7 @@ class Audio(commands.Cog):
             or not match_yt_playlist(uploaded_playlist_url)
             or not (
                 await self.music_cache.lavalink_query(
-                    ctx, player, dataclasses.Query.process_input(uploaded_playlist_url)
+                    ctx, player, audio_dataclasses.Query.process_input(uploaded_playlist_url)
                 )
             )[0].tracks
         ):
@@ -4964,7 +4990,7 @@ class Audio(commands.Cog):
                     }
                 )
         if database_entries and HAS_SQL:
-            asyncio.ensure_future(self.music_cache.insert("lavalink", database_entries))
+            await self.music_cache.insert("lavalink", database_entries)
 
     async def _load_v2_playlist(
         self,
@@ -4991,7 +5017,7 @@ class Audio(commands.Cog):
             track_count += 1
             try:
                 result, called_api = await self.music_cache.lavalink_query(
-                    ctx, player, dataclasses.Query.process_input(song_url)
+                    ctx, player, audio_dataclasses.Query.process_input(song_url)
                 )
                 track = result.tracks
             except Exception:
@@ -5039,7 +5065,7 @@ class Audio(commands.Cog):
             return [], [], playlist
         results = {}
         updated_tracks = await self._playlist_tracks(
-            ctx, player, dataclasses.Query.process_input(playlist.url)
+            ctx, player, audio_dataclasses.Query.process_input(playlist.url)
         )
         if not updated_tracks:
             # No Tracks available on url Lets set it to none to avoid repeated calls here
@@ -5104,7 +5130,7 @@ class Audio(commands.Cog):
         self,
         ctx: commands.Context,
         player: lavalink.player_manager.Player,
-        query: dataclasses.Query,
+        query: audio_dataclasses.Query,
     ):
         search = query.is_search
         tracklist = []
@@ -5173,7 +5199,7 @@ class Audio(commands.Cog):
             player.queue.insert(0, bump_song)
             player.queue.pop(queue_len)
             await player.skip()
-            query = dataclasses.Query.process_input(player.current.uri)
+            query = audio_dataclasses.Query.process_input(player.current.uri)
             if query.is_local:
 
                 if player.current.title == "Unknown title":
@@ -5225,7 +5251,7 @@ class Audio(commands.Cog):
                 else:
                     dur = lavalink.utils.format_time(player.current.length)
 
-                query = dataclasses.Query.process_input(player.current)
+                query = audio_dataclasses.Query.process_input(player.current)
 
                 if query.is_local:
                     if player.current.title != "Unknown title":
@@ -5238,8 +5264,8 @@ class Audio(commands.Cog):
                 song += "\n\n{arrow}`{pos}`/`{dur}`"
                 song = song.format(
                     track=player.current,
-                    uri=dataclasses.LocalPath(player.current.uri).to_string_hidden()
+                    uri=audio_dataclasses.LocalPath(player.current.uri).to_string_hidden()
-                    if dataclasses.Query.process_input(player.current.uri).is_local
+                    if audio_dataclasses.Query.process_input(player.current.uri).is_local
                     else player.current.uri,
                     arrow=arrow,
                     pos=pos,
@@ -5311,7 +5337,7 @@ class Audio(commands.Cog):
         else:
             dur = lavalink.utils.format_time(player.current.length)
 
-        query = dataclasses.Query.process_input(player.current)
+        query = audio_dataclasses.Query.process_input(player.current)
 
         if query.is_stream:
             queue_list += _("**Currently livestreaming:**\n")
@@ -5325,7 +5351,7 @@ class Audio(commands.Cog):
                     (
                         _("Playing: ")
                         + "**{current.author} - {current.title}**".format(current=player.current),
-                        dataclasses.LocalPath(player.current.uri).to_string_hidden(),
+                        audio_dataclasses.LocalPath(player.current.uri).to_string_hidden(),
                         _("Requested by: **{user}**\n").format(user=player.current.requester),
                         f"{arrow}`{pos}`/`{dur}`\n\n",
                     )
@@ -5334,7 +5360,7 @@ class Audio(commands.Cog):
                 queue_list += "\n".join(
                     (
                         _("Playing: ")
-                        + dataclasses.LocalPath(player.current.uri).to_string_hidden(),
+                        + audio_dataclasses.LocalPath(player.current.uri).to_string_hidden(),
                         _("Requested by: **{user}**\n").format(user=player.current.requester),
                         f"{arrow}`{pos}`/`{dur}`\n\n",
                     )
@@ -5355,13 +5381,13 @@ class Audio(commands.Cog):
                 track_title = track.title
             req_user = track.requester
             track_idx = i + 1
-            query = dataclasses.Query.process_input(track)
+            query = audio_dataclasses.Query.process_input(track)
 
             if query.is_local:
                 if track.title == "Unknown title":
                     queue_list += f"`{track_idx}.` " + ", ".join(
                         (
-                            bold(dataclasses.LocalPath(track.uri).to_string_hidden()),
+                            bold(audio_dataclasses.LocalPath(track.uri).to_string_hidden()),
                             _("requested by **{user}**\n").format(user=req_user),
                         )
                     )
@@ -5418,7 +5444,7 @@ class Audio(commands.Cog):
         for track in queue_list:
             queue_idx = queue_idx + 1
             if not match_url(track.uri):
-                query = dataclasses.Query.process_input(track)
+                query = audio_dataclasses.Query.process_input(track)
                 if track.title == "Unknown title":
                     track_title = query.track.to_string_hidden()
                 else:
@@ -5447,7 +5473,7 @@ class Audio(commands.Cog):
         ):
             track_idx = i + 1
             if type(track) is str:
-                track_location = dataclasses.LocalPath(track).to_string_hidden()
+                track_location = audio_dataclasses.LocalPath(track).to_string_hidden()
                 track_match += "`{}.` **{}**\n".format(track_idx, track_location)
             else:
                 track_match += "`{}.` **{}**\n".format(track[0], track[1])
@@ -5672,9 +5698,9 @@ class Audio(commands.Cog):
             )
         index -= 1
         removed = player.queue.pop(index)
-        query = dataclasses.Query.process_input(removed.uri)
+        query = audio_dataclasses.Query.process_input(removed.uri)
         if query.is_local:
-            local_path = dataclasses.LocalPath(removed.uri).to_string_hidden()
+            local_path = audio_dataclasses.LocalPath(removed.uri).to_string_hidden()
             if removed.title == "Unknown title":
                 removed_title = local_path
             else:
@@ -5760,7 +5786,7 @@ class Audio(commands.Cog):
         await self._data_check(ctx)
 
         if not isinstance(query, list):
-            query = dataclasses.Query.process_input(query)
+            query = audio_dataclasses.Query.process_input(query)
             if query.invoked_from == "search list" or query.invoked_from == "local folder":
                 if query.invoked_from == "search list":
                     result, called_api = await self.music_cache.lavalink_query(ctx, player, query)
@@ -5789,7 +5815,7 @@ class Audio(commands.Cog):
                         ctx.guild,
                         (
                             f"{track.title} {track.author} {track.uri} "
-                            f"{str(dataclasses.Query.process_input(track))}"
+                            f"{str(audio_dataclasses.Query.process_input(track))}"
                         ),
                     ):
                         log.debug(f"Query is not allowed in {ctx.guild} ({ctx.guild.id})")
@@ -5903,10 +5929,10 @@ class Audio(commands.Cog):
         except IndexError:
             search_choice = tracks[-1]
         try:
-            query = dataclasses.Query.process_input(search_choice.uri)
+            query = audio_dataclasses.Query.process_input(search_choice.uri)
             if query.is_local:
 
-                localtrack = dataclasses.LocalPath(search_choice.uri)
+                localtrack = audio_dataclasses.LocalPath(search_choice.uri)
                 if search_choice.title != "Unknown title":
                     description = "**{} - {}**\n{}".format(
                         search_choice.author, search_choice.title, localtrack.to_string_hidden()
@@ -5917,7 +5943,7 @@ class Audio(commands.Cog):
                 description = "**[{}]({})**".format(search_choice.title, search_choice.uri)
 
         except AttributeError:
-            search_choice = dataclasses.Query.process_input(search_choice)
+            search_choice = audio_dataclasses.Query.process_input(search_choice)
             if search_choice.track.exists() and search_choice.track.is_dir():
                 return await ctx.invoke(self.search, query=search_choice)
             elif search_choice.track.exists() and search_choice.track.is_file():
@@ -5933,7 +5959,7 @@ class Audio(commands.Cog):
             ctx.guild,
             (
                 f"{search_choice.title} {search_choice.author} {search_choice.uri} "
-                f"{str(dataclasses.Query.process_input(search_choice))}"
+                f"{str(audio_dataclasses.Query.process_input(search_choice))}"
             ),
         ):
             log.debug(f"Query is not allowed in {ctx.guild} ({ctx.guild.id})")
@@ -5982,12 +6008,12 @@ class Audio(commands.Cog):
             if search_track_num == 0:
                 search_track_num = 5
             try:
-                query = dataclasses.Query.process_input(track.uri)
+                query = audio_dataclasses.Query.process_input(track.uri)
                 if query.is_local:
                     search_list += "`{0}.` **{1}**\n[{2}]\n".format(
                         search_track_num,
                         track.title,
-                        dataclasses.LocalPath(track.uri).to_string_hidden(),
+                        audio_dataclasses.LocalPath(track.uri).to_string_hidden(),
                     )
                 else:
                     search_list += "`{0}.` **[{1}]({2})**\n".format(
@@ -5995,7 +6021,7 @@ class Audio(commands.Cog):
                     )
             except AttributeError:
                 # query = Query.process_input(track)
-                track = dataclasses.Query.process_input(track)
+                track = audio_dataclasses.Query.process_input(track)
                 if track.is_local and command != "search":
                     search_list += "`{}.` **{}**\n".format(
                         search_track_num, track.to_string_user()
@@ -6717,7 +6743,9 @@ class Audio(commands.Cog):
                     if (time.time() - stop_times[sid]) >= emptydc_timer:
                         stop_times.pop(sid)
                         try:
-                            await lavalink.get_player(sid).disconnect()
+                            player = lavalink.get_player(sid)
+                            await player.stop()
+                            await player.disconnect()
                         except Exception:
                             log.error("Exception raised in Audio's emptydc_timer.", exc_info=True)
                             pass
@@ -6888,6 +6916,7 @@ class Audio(commands.Cog):
     async def on_voice_state_update(
         self, member: discord.Member, before: discord.VoiceState, after: discord.VoiceState
     ):
+        await self._ready_event.wait()
         if after.channel != before.channel:
             try:
                 self.skip_votes[before.channel.guild].remove(member.id)
@@ -6905,6 +6934,9 @@ class Audio(commands.Cog):
             if self._connect_task:
                 self._connect_task.cancel()
 
+            if self._init_task:
+                self._init_task.cancel()
+
             lavalink.unregister_event_listener(self.event_handler)
             self.bot.loop.create_task(lavalink.close())
             if self._manager is not None:

redbot/cogs/audio/audio_dataclasses.py

@@ -381,14 +381,17 @@ class Query:
                             match = re.search(_re_youtube_index, track)
                             if match:
                                 returning["track_index"] = int(match.group(1)) - 1
-
                         if all(k in track for k in ["&list=", "watch?"]):
                             returning["track_index"] = 0
                             returning["playlist"] = True
                             returning["single"] = False
                         elif all(x in track for x in ["playlist?"]):
-                            returning["playlist"] = True if not _has_index else False
+                            returning["playlist"] = not _has_index
-                            returning["single"] = True if _has_index else False
+                            returning["single"] = _has_index
+                        elif any(k in track for k in ["list="]):
+                            returning["track_index"] = 0
+                            returning["playlist"] = True
+                            returning["single"] = False
                         else:
                             returning["single"] = True
                     elif url_domain == "spotify.com":

redbot/cogs/audio/manager.py

@@ -18,7 +18,7 @@ from redbot.core import data_manager
 from .errors import LavalinkDownloadFailed
 
 JAR_VERSION = "3.2.1"
-JAR_BUILD = 823
+JAR_BUILD = 846
 LAVALINK_DOWNLOAD_URL = (
     f"https://github.com/Cog-Creators/Lavalink-Jars/releases/download/{JAR_VERSION}_{JAR_BUILD}/"
     f"Lavalink.jar"

redbot/cogs/audio/utils.py

@@ -3,7 +3,6 @@ import contextlib
 import os
 import re
 import time
-from typing import NoReturn
 from urllib.parse import urlparse
 
 import discord
@@ -11,7 +10,7 @@ import lavalink
 
 from redbot.core import Config, commands
 from redbot.core.bot import Red
-from . import dataclasses
+from . import audio_dataclasses
 
 from .converters import _pass_config_to_converters
 
@@ -51,7 +50,7 @@ def pass_config_to_dependencies(config: Config, bot: Red, localtracks_folder: st
     _config = config
     _pass_config_to_playlist(config, bot)
     _pass_config_to_converters(config, bot)
-    dataclasses._pass_config_to_dataclasses(config, bot, localtracks_folder)
+    audio_dataclasses._pass_config_to_dataclasses(config, bot, localtracks_folder)
 
 
 def track_limit(track, maxlength):
@@ -168,7 +167,7 @@ async def clear_react(bot: Red, message: discord.Message, emoji: dict = None):
 
 async def get_description(track):
     if any(x in track.uri for x in [f"{os.sep}localtracks", f"localtracks{os.sep}"]):
-        local_track = dataclasses.LocalPath(track.uri)
+        local_track = audio_dataclasses.LocalPath(track.uri)
         if track.title != "Unknown title":
             return "**{} - {}**\n{}".format(
                 track.author, track.title, local_track.to_string_hidden()
@@ -389,7 +388,7 @@ class Notifier:
         key: str = None,
         seconds_key: str = None,
         seconds: str = None,
-    ) -> NoReturn:
+    ):
         """
         This updates an existing message.
         Based on the message found in :variable:`Notifier.updates` as per the `key` param
@@ -410,14 +409,14 @@ class Notifier:
         except discord.errors.NotFound:
             pass
 
-    async def update_text(self, text: str) -> NoReturn:
+    async def update_text(self, text: str):
         embed2 = discord.Embed(colour=self.color, title=text)
         try:
             await self.message.edit(embed=embed2)
         except discord.errors.NotFound:
             pass
 
-    async def update_embed(self, embed: discord.Embed) -> NoReturn:
+    async def update_embed(self, embed: discord.Embed):
         try:
             await self.message.edit(embed=embed)
             self.last_msg_time = time.time()

redbot/cogs/downloader/checks.py

@@ -21,7 +21,7 @@ REPO_INSTALL_MSG = _(
 _ = T_
 
 
-async def do_install_agreement(ctx: commands.Context):
+async def do_install_agreement(ctx: commands.Context) -> bool:
     downloader = ctx.cog
     if downloader is None or downloader.already_agreed:
         return True

redbot/cogs/downloader/converters.py

@@ -1,14 +1,14 @@
 import discord
 from redbot.core import commands
 from redbot.core.i18n import Translator
-from .installable import Installable
+from .installable import InstalledModule
 
 _ = Translator("Koala", __file__)
 
 
-class InstalledCog(Installable):
+class InstalledCog(InstalledModule):
     @classmethod
-    async def convert(cls, ctx: commands.Context, arg: str) -> Installable:
+    async def convert(cls, ctx: commands.Context, arg: str) -> InstalledModule:
         downloader = ctx.bot.get_cog("Downloader")
         if downloader is None:
             raise commands.CommandError(_("No Downloader cog found."))

redbot/cogs/downloader/errors.py

@@ -1,7 +1,16 @@
+from __future__ import annotations
+
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .repo_manager import Candidate
+
+
 __all__ = [
     "DownloaderException",
     "GitException",
     "InvalidRepoName",
+    "CopyingError",
     "ExistingGitRepo",
     "MissingGitRepo",
     "CloningError",
@@ -10,6 +19,8 @@ __all__ = [
     "UpdateError",
     "GitDiffError",
     "NoRemoteURL",
+    "UnknownRevision",
+    "AmbiguousRevision",
     "PipError",
 ]
 
@@ -37,6 +48,15 @@ class InvalidRepoName(DownloaderException):
     pass
 
 
+class CopyingError(DownloaderException):
+    """
+    Throw when there was an issue
+    during copying of module's files.
+    """
+
+    pass
+
+
 class ExistingGitRepo(DownloaderException):
     """
     Thrown when trying to clone into a folder where a
@@ -105,6 +125,24 @@ class NoRemoteURL(GitException):
     pass
 
 
+class UnknownRevision(GitException):
+    """
+    Thrown when specified revision cannot be found.
+    """
+
+    pass
+
+
+class AmbiguousRevision(GitException):
+    """
+    Thrown when specified revision is ambiguous.
+    """
+
+    def __init__(self, message: str, candidates: List[Candidate]) -> None:
+        super().__init__(message)
+        self.candidates = candidates
+
+
 class PipError(DownloaderException):
     """
     Thrown when pip returns a non-zero return code.

redbot/cogs/downloader/installable.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
+
 import json
 import distutils.dir_util
 import shutil
-from enum import Enum
+from enum import IntEnum
 from pathlib import Path
-from typing import MutableMapping, Any, TYPE_CHECKING
+from typing import MutableMapping, Any, TYPE_CHECKING, Optional, Dict, Union, Callable, Tuple, cast
 
 from .log import log
 from .json_mixins import RepoJSONMixin
@@ -11,10 +13,11 @@ from .json_mixins import RepoJSONMixin
 from redbot.core import __version__, version_info as red_version_info, VersionInfo
 
 if TYPE_CHECKING:
-    from .repo_manager import RepoManager
+    from .repo_manager import RepoManager, Repo
 
 
-class InstallableType(Enum):
+class InstallableType(IntEnum):
+    # using IntEnum, because hot-reload breaks its identity
     UNKNOWN = 0
     COG = 1
     SHARED_LIBRARY = 2
@@ -34,6 +37,10 @@ class Installable(RepoJSONMixin):
     ----------
     repo_name : `str`
         Name of the repository which this package belongs to.
+    repo : Repo, optional
+        Repo object of the Installable, if repo is missing this will be `None`
+    commit : `str`, optional
+        Installable's commit. This is not the same as ``repo.commit``
     author : `tuple` of `str`, optional
         Name(s) of the author(s).
     bot_version : `tuple` of `int`
@@ -58,30 +65,36 @@ class Installable(RepoJSONMixin):
 
     """
 
-    def __init__(self, location: Path):
+    def __init__(self, location: Path, repo: Optional[Repo] = None, commit: str = ""):
         """Base installable initializer.
 
         Parameters
         ----------
         location : pathlib.Path
             Location (file or folder) to the installable.
+        repo : Repo, optional
+            Repo object of the Installable, if repo is missing this will be `None`
+        commit : str
+            Installable's commit. This is not the same as ``repo.commit``
 
         """
         super().__init__(location)
 
         self._location = location
 
+        self.repo = repo
         self.repo_name = self._location.parent.stem
+        self.commit = commit
 
-        self.author = ()
+        self.author: Tuple[str, ...] = ()
         self.min_bot_version = red_version_info
         self.max_bot_version = red_version_info
         self.min_python_version = (3, 5, 1)
         self.hidden = False
         self.disabled = False
-        self.required_cogs = {}  # Cog name -> repo URL
+        self.required_cogs: Dict[str, str] = {}  # Cog name -> repo URL
-        self.requirements = ()
+        self.requirements: Tuple[str, ...] = ()
-        self.tags = ()
+        self.tags: Tuple[str, ...] = ()
         self.type = InstallableType.UNKNOWN
 
         if self._info_file.exists():
@@ -90,15 +103,15 @@ class Installable(RepoJSONMixin):
         if self._info == {}:
             self.type = InstallableType.COG
 
-    def __eq__(self, other):
+    def __eq__(self, other: Any) -> bool:
         # noinspection PyProtectedMember
         return self._location == other._location
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self._location)
 
     @property
-    def name(self):
+    def name(self) -> str:
         """`str` : The name of this package."""
         return self._location.stem
 
@@ -111,6 +124,7 @@ class Installable(RepoJSONMixin):
         :return: Status of installation
         :rtype: bool
         """
+        copy_func: Callable[..., Any]
         if self._location.is_file():
             copy_func = shutil.copy2
         else:
@@ -121,18 +135,20 @@ class Installable(RepoJSONMixin):
         # noinspection PyBroadException
         try:
             copy_func(src=str(self._location), dst=str(target_dir / self._location.stem))
-        except:
+        except:  # noqa: E722
             log.exception("Error occurred when copying path: {}".format(self._location))
             return False
         return True
 
-    def _read_info_file(self):
+    def _read_info_file(self) -> None:
         super()._read_info_file()
 
         if self._info_file.exists():
             self._process_info_file()
 
-    def _process_info_file(self, info_file_path: Path = None) -> MutableMapping[str, Any]:
+    def _process_info_file(
+        self, info_file_path: Optional[Path] = None
+    ) -> MutableMapping[str, Any]:
         """
         Processes an information file. Loads dependencies among other
         information into this object.
@@ -145,7 +161,7 @@ class Installable(RepoJSONMixin):
         if info_file_path is None or not info_file_path.is_file():
             raise ValueError("No valid information file path was found.")
 
-        info = {}
+        info: Dict[str, Any] = {}
         with info_file_path.open(encoding="utf-8") as f:
             try:
                 info = json.load(f)
@@ -174,7 +190,7 @@ class Installable(RepoJSONMixin):
         self.max_bot_version = max_bot_version
 
         try:
-            min_python_version = tuple(info.get("min_python_version", [3, 5, 1]))
+            min_python_version = tuple(info.get("min_python_version", (3, 5, 1)))
         except ValueError:
             min_python_version = self.min_python_version
         self.min_python_version = min_python_version
@@ -212,14 +228,51 @@ class Installable(RepoJSONMixin):
 
         return info
 
-    def to_json(self):
+
-        return {"repo_name": self.repo_name, "cog_name": self.name}
+class InstalledModule(Installable):
+    """Base class for installed modules,
+    this is basically instance of installed `Installable`
+    used by Downloader.
+
+    Attributes
+    ----------
+    pinned : `bool`
+        Whether or not this cog is pinned, always `False` if module is not a cog.
+    """
+
+    def __init__(
+        self,
+        location: Path,
+        repo: Optional[Repo] = None,
+        commit: str = "",
+        pinned: bool = False,
+        json_repo_name: str = "",
+    ):
+        super().__init__(location=location, repo=repo, commit=commit)
+        self.pinned: bool = pinned if self.type == InstallableType.COG else False
+        # this is here so that Downloader could use real repo name instead of "MISSING_REPO"
+        self._json_repo_name = json_repo_name
+
+    def to_json(self) -> Dict[str, Union[str, bool]]:
+        module_json: Dict[str, Union[str, bool]] = {
+            "repo_name": self.repo_name,
+            "module_name": self.name,
+            "commit": self.commit,
+        }
+        if self.type == InstallableType.COG:
+            module_json["pinned"] = self.pinned
+        return module_json
 
     @classmethod
-    def from_json(cls, data: dict, repo_mgr: "RepoManager"):
+    def from_json(
-        repo_name = data["repo_name"]
+        cls, data: Dict[str, Union[str, bool]], repo_mgr: RepoManager
-        cog_name = data["cog_name"]
+    ) -> InstalledModule:
+        repo_name = cast(str, data["repo_name"])
+        cog_name = cast(str, data["module_name"])
+        commit = cast(str, data.get("commit", ""))
+        pinned = cast(bool, data.get("pinned", False))
 
+        # TypedDict, where are you :/
         repo = repo_mgr.get_repo(repo_name)
         if repo is not None:
             repo_folder = repo.folder_path
@@ -228,4 +281,12 @@ class Installable(RepoJSONMixin):
 
         location = repo_folder / cog_name
 
-        return cls(location=location)
+        return cls(
+            location=location, repo=repo, commit=commit, pinned=pinned, json_repo_name=repo_name
+        )
+
+    @classmethod
+    def from_installable(cls, module: Installable, *, pinned: bool = False) -> InstalledModule:
+        return cls(
+            location=module._location, repo=module.repo, commit=module.commit, pinned=pinned
+        )

redbot/cogs/downloader/json_mixins.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+from typing import Optional, Tuple, Dict, Any
 
 
 class RepoJSONMixin:
@@ -8,18 +9,18 @@ class RepoJSONMixin:
     def __init__(self, repo_folder: Path):
         self._repo_folder = repo_folder
 
-        self.author = None
+        self.author: Optional[Tuple[str, ...]] = None
-        self.install_msg = None
+        self.install_msg: Optional[str] = None
-        self.short = None
+        self.short: Optional[str] = None
-        self.description = None
+        self.description: Optional[str] = None
 
         self._info_file = repo_folder / self.INFO_FILE_NAME
         if self._info_file.exists():
             self._read_info_file()
 
-        self._info = {}
+        self._info: Dict[str, Any] = {}
 
-    def _read_info_file(self):
+    def _read_info_file(self) -> None:
         if not (self._info_file.exists() or self._info_file.is_file()):
             return
 

redbot/cogs/permissions/permissions.py

@@ -299,6 +299,14 @@ class Permissions(commands.Cog):
         if not who_or_what:
             await ctx.send_help()
             return
+        if isinstance(cog_or_command.obj, commands.commands._AlwaysAvailableCommand):
+            await ctx.send(
+                _(
+                    "This command is designated as being always available and "
+                    "cannot be modified by permission rules."
+                )
+            )
+            return
         for w in who_or_what:
             await self._add_rule(
                 rule=cast(bool, allow_or_deny),
@@ -334,6 +342,14 @@ class Permissions(commands.Cog):
         if not who_or_what:
             await ctx.send_help()
             return
+        if isinstance(cog_or_command.obj, commands.commands._AlwaysAvailableCommand):
+            await ctx.send(
+                _(
+                    "This command is designated as being always available and "
+                    "cannot be modified by permission rules."
+                )
+            )
+            return
         for w in who_or_what:
             await self._add_rule(
                 rule=cast(bool, allow_or_deny),
@@ -544,7 +560,7 @@ class Permissions(commands.Cog):
 
         Handles config.
         """
-        self.bot.clear_permission_rules(guild_id)
+        self.bot.clear_permission_rules(guild_id, preserve_default_rule=False)
         for category in (COG, COMMAND):
             async with self.config.custom(category).all() as all_rules:
                 for name, rules in all_rules.items():

redbot/core/bot.py

@@ -8,6 +8,7 @@ from enum import Enum
 from importlib.machinery import ModuleSpec
 from pathlib import Path
 from typing import Optional, Union, List, Dict, NoReturn
+from types import MappingProxyType
 
 import discord
 from discord.ext.commands import when_mentioned_or
@@ -132,7 +133,6 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
 
         self._main_dir = bot_dir
         self._cog_mgr = CogManager()
-
         super().__init__(*args, help_command=None, **kwargs)
         # Do not manually use the help formatter attribute here, see `send_help_for`,
         # for a documented API. The internals of this object are still subject to change.
@@ -325,6 +325,7 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
 
     get_embed_colour = get_embed_color
 
+    # start config migrations
     async def _maybe_update_config(self):
         """
         This should be run prior to loading cogs or connecting to discord.
@@ -375,6 +376,57 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
                 await self._config.guild(guild_obj).admin_role.set(admin_roles)
         log.info("Done updating guild configs to support multiple mod/admin roles")
 
+    # end Config migrations
+
+    async def pre_flight(self, cli_flags):
+        """
+        This should only be run once, prior to connecting to discord.
+        """
+        await self._maybe_update_config()
+
+        packages = []
+
+        if cli_flags.no_cogs is False:
+            packages.extend(await self._config.packages())
+
+        if cli_flags.load_cogs:
+            packages.extend(cli_flags.load_cogs)
+
+        if packages:
+            # Load permissions first, for security reasons
+            try:
+                packages.remove("permissions")
+            except ValueError:
+                pass
+            else:
+                packages.insert(0, "permissions")
+
+            to_remove = []
+            print("Loading packages...")
+            for package in packages:
+                try:
+                    spec = await self._cog_mgr.find_cog(package)
+                    await asyncio.wait_for(self.load_extension(spec), 30)
+                except asyncio.TimeoutError:
+                    log.exception("Failed to load package %s (timeout)", package)
+                    to_remove.append(package)
+                except Exception as e:
+                    log.exception("Failed to load package {}".format(package), exc_info=e)
+                    await self.remove_loaded_package(package)
+                    to_remove.append(package)
+            for package in to_remove:
+                packages.remove(package)
+            if packages:
+                print("Loaded packages: " + ", ".join(packages))
+
+        if self.rpc_enabled:
+            await self.rpc.initialize(self.rpc_port)
+
+    async def start(self, *args, **kwargs):
+        cli_flags = kwargs.pop("cli_flags")
+        await self.pre_flight(cli_flags=cli_flags)
+        return await super().start(*args, **kwargs)
+
     async def send_help_for(
         self, ctx: commands.Context, help_for: Union[commands.Command, commands.GroupMixin, str]
     ):
@@ -531,6 +583,7 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
 
         async with self._config.custom(SHARED_API_TOKENS, service_name).all() as group:
             group.update(tokens)
+        self.dispatch("red_api_tokens_update", service_name, MappingProxyType(group))
 
     async def remove_shared_api_tokens(self, service_name: str, *token_names: str):
         """
@@ -653,7 +706,7 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
             ``True`` if immune
 
         """
-        guild = to_check.guild
+        guild = getattr(to_check, "guild", None)
         if not guild:
             return False
 
@@ -666,7 +719,8 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
             except AttributeError:
                 # webhook messages are a user not member,
                 # cheaper than isinstance
-                return True  # webhooks require significant permissions to enable.
+                if author.bot and author.discriminator == "0000":
+                    return True  # webhooks require significant permissions to enable.
             else:
                 ids_to_check.append(author.id)
 
@@ -779,7 +833,7 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
             for subcommand in set(command.walk_commands()):
                 subcommand.requires.reset()
 
-    def clear_permission_rules(self, guild_id: Optional[int]) -> None:
+    def clear_permission_rules(self, guild_id: Optional[int], **kwargs) -> None:
         """Clear all permission overrides in a scope.
 
         Parameters
@@ -789,11 +843,15 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
             ``None``, this will clear all global rules and leave all
             guild rules untouched.
 
+        **kwargs
+            Keyword arguments to be passed to each required call of
+            ``commands.Requires.clear_all_rules``
+
         """
         for cog in self.cogs.values():
-            cog.requires.clear_all_rules(guild_id)
+            cog.requires.clear_all_rules(guild_id, **kwargs)
         for command in self.walk_commands():
-            command.requires.clear_all_rules(guild_id)
+            command.requires.clear_all_rules(guild_id, **kwargs)
 
     def add_permissions_hook(self, hook: commands.CheckPredicate) -> None:
         """Add a permissions hook.

redbot/core/cli.py

@@ -1,17 +1,42 @@
 import argparse
 import asyncio
 import logging
+import sys
+from typing import Optional
 
 
-def confirm(m=""):
+def confirm(text: str, default: Optional[bool] = None) -> bool:
-    return input(m).lower().strip() in ("y", "yes")
+    if default is None:
+        options = "y/n"
+    elif default is True:
+        options = "Y/n"
+    elif default is False:
+        options = "y/N"
+    else:
+        raise TypeError(f"expected bool, not {type(default)}")
+
+    while True:
+        try:
+            value = input(f"{text}: [{options}] ").lower().strip()
+        except (KeyboardInterrupt, EOFError):
+            print("\nAborted!")
+            sys.exit(1)
+        if value in ("y", "yes"):
+            return True
+        if value in ("n", "no"):
+            return False
+        if value == "":
+            if default is not None:
+                return default
+        print("Error: invalid input")
 
 
-def interactive_config(red, token_set, prefix_set):
+def interactive_config(red, token_set, prefix_set, *, print_header=True):
     loop = asyncio.get_event_loop()
     token = ""
 
-    print("Red - Discord Bot | Configuration process\n")
+    if print_header:
+        print("Red - Discord Bot | Configuration process\n")
 
     if not token_set:
         print("Please enter a valid token:")
@@ -35,8 +60,7 @@ def interactive_config(red, token_set, prefix_set):
         while not prefix:
             prefix = input("Prefix> ")
             if len(prefix) > 10:
-                print("Your prefix seems overly long. Are you sure that it's correct? (y/n)")
+                if not confirm("Your prefix seems overly long. Are you sure that it's correct?"):
-                if not confirm("> "):
                     prefix = ""
             if prefix:
                 loop.run_until_complete(red._config.prefix.set([prefix]))
@@ -54,6 +78,37 @@ def parse_cli_flags(args):
         action="store_true",
         help="List all instance names setup with 'redbot-setup'",
     )
+    parser.add_argument(
+        "--edit",
+        action="store_true",
+        help="Edit the instance. This can be done without console interaction "
+        "by passing --no-prompt and arguments that you want to change (available arguments: "
+        "--edit-instance-name, --edit-data-path, --copy-data, --owner, --token).",
+    )
+    parser.add_argument(
+        "--edit-instance-name",
+        type=str,
+        help="New name for the instance. This argument only works with --edit argument passed.",
+    )
+    parser.add_argument(
+        "--overwrite-existing-instance",
+        action="store_true",
+        help="Confirm overwriting of existing instance when changing name."
+        " This argument only works with --edit argument passed.",
+    )
+    parser.add_argument(
+        "--edit-data-path",
+        type=str,
+        help=(
+            "New data path for the instance. This argument only works with --edit argument passed."
+        ),
+    )
+    parser.add_argument(
+        "--copy-data",
+        action="store_true",
+        help="Copy data from old location. This argument only works "
+        "with --edit and --edit-data-path arguments passed.",
+    )
     parser.add_argument(
         "--owner",
         type=int,
@@ -65,7 +120,7 @@ def parse_cli_flags(args):
         "--co-owner",
         type=int,
         default=[],
-        nargs="*",
+        nargs="+",
         help="ID of a co-owner. Only people who have access "
         "to the system that is hosting Red should be  "
         "co-owners, as this gives them complete access "
@@ -87,7 +142,7 @@ def parse_cli_flags(args):
     parser.add_argument(
         "--load-cogs",
         type=str,
-        nargs="*",
+        nargs="+",
         help="Force loading specified cogs from the installed packages. "
         "Can be used with the --no-cogs flag to load these cogs exclusively.",
     )

redbot/core/commands/commands.py

@@ -699,3 +699,29 @@ def get_command_disabler(guild: discord.Guild) -> Callable[["Context"], Awaitabl
 
         __command_disablers[guild] = disabler
         return disabler
+
+
+# This is intentionally left out of `__all__` as it is not intended for general use
+class _AlwaysAvailableCommand(Command):
+    """
+    This should be used only for informational commands
+    which should not be disabled or removed
+
+    These commands cannot belong to a cog.
+
+    These commands do not respect most forms of checks, and
+    should only be used with that in mind.
+
+    This particular class is not supported for 3rd party use
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.cog is not None:
+            raise TypeError("This command may not be added to a cog")
+
+    async def can_run(self, ctx, *args, **kwargs) -> bool:
+        return not ctx.author.bot
+
+    async def _verify_checks(self, ctx) -> bool:
+        return not ctx.author.bot

redbot/core/commands/requires.py

@@ -398,11 +398,9 @@ class Requires:
         else:
             rules[model_id] = rule
 
-    def clear_all_rules(self, guild_id: int) -> None:
+    def clear_all_rules(self, guild_id: int, *, preserve_default_rule: bool = True) -> None:
         """Clear all rules of a particular scope.
 
-        This will preserve the default rule, if set.
-
         Parameters
         ----------
         guild_id : int
@@ -410,6 +408,12 @@ class Requires:
             `Requires.GLOBAL`, this will clear all global rules and
             leave all guild rules untouched.
 
+        Other Parameters
+        ----------------
+        preserve_default_rule : bool
+            Whether to preserve the default rule or not.
+            This defaults to being preserved
+
         """
         if guild_id:
             rules = self._guild_rules.setdefault(guild_id, _RulesDict())
@@ -417,7 +421,7 @@ class Requires:
             rules = self._global_rules
         default = rules.get(self.DEFAULT, None)
         rules.clear()
-        if default is not None:
+        if default is not None and preserve_default_rule:
             rules[self.DEFAULT] = default
 
     def reset(self) -> None:

redbot/core/core_commands.py

@@ -33,7 +33,14 @@ from . import (
 )
 from .utils import create_backup
 from .utils.predicates import MessagePredicate
-from .utils.chat_formatting import humanize_timedelta, pagify, box, inline, humanize_list
+from .utils.chat_formatting import (
+    box,
+    humanize_list,
+    humanize_number,
+    humanize_timedelta,
+    inline,
+    pagify,
+)
 from .commands.requires import PrivilegeLevel
 
 
@@ -293,7 +300,7 @@ class Core(commands.Cog, CoreLogic):
                 data = await r.json()
         outdated = VersionInfo.from_str(data["info"]["version"]) > red_version_info
         about = _(
-            "This is an instance of [Red, an open source Discord bot]({}) "
+            "This bot is an instance of [Red, an open source Discord bot]({}) "
             "created by [Twentysix]({}) and [improved by many]({}).\n\n"
             "Red is backed by a passionate community who contributes and "
             "creates content for everyone to enjoy. [Join us today]({}) "
@@ -1136,7 +1143,7 @@ class Core(commands.Cog, CoreLogic):
     @checks.is_owner()
     async def api(self, ctx: commands.Context, service: str, *, tokens: TokenConverter):
         """Set various external API tokens.
-        
+
         This setting will be asked for by some 3rd party cogs and some core cogs.
 
         To add the keys provide the service name and the tokens as a comma separated
@@ -1162,7 +1169,7 @@ class Core(commands.Cog, CoreLogic):
         Allows the help command to be sent as a paginated menu instead of seperate
         messages.
 
-        This defaults to False. 
+        This defaults to False.
         Using this without a setting will toggle.
         """
         if use_menus is None:
@@ -1316,8 +1323,9 @@ class Core(commands.Cog, CoreLogic):
     @commands.command()
     @checks.is_owner()
     async def backup(self, ctx: commands.Context, *, backup_dir: str = None):
-        """Creates a backup of all data for the instance.
+        """Creates a backup of all data for this bot instance.
 
+        This backs up the bot's data and settings.
         You may provide a path to a directory for the backup archive to
         be placed in. If the directory does not exist, the bot will
         attempt to create it.
@@ -1877,6 +1885,53 @@ class Core(commands.Cog, CoreLogic):
         """Manage the bot's commands."""
         pass
 
+    @command_manager.group(name="listdisabled", invoke_without_command=True)
+    async def list_disabled(self, ctx: commands.Context):
+        """
+        List disabled commands.
+
+        If you're the bot owner, this will show global disabled commands by default.
+        """
+        # Select the scope based on the author's privileges
+        if await ctx.bot.is_owner(ctx.author):
+            await ctx.invoke(self.list_disabled_global)
+        else:
+            await ctx.invoke(self.list_disabled_guild)
+
+    @list_disabled.command(name="global")
+    async def list_disabled_global(self, ctx: commands.Context):
+        """List disabled commands globally."""
+        disabled_list = await self.bot._config.disabled_commands()
+        if not disabled_list:
+            return await ctx.send(_("There aren't any globally disabled commands."))
+
+        if len(disabled_list) > 1:
+            header = _("{} commands are disabled globally.\n").format(
+                humanize_number(len(disabled_list))
+            )
+        else:
+            header = _("1 command is disabled globally.\n")
+        paged = [box(x) for x in pagify(humanize_list(disabled_list), page_length=1000)]
+        paged[0] = header + paged[0]
+        await ctx.send_interactive(paged)
+
+    @list_disabled.command(name="guild")
+    async def list_disabled_guild(self, ctx: commands.Context):
+        """List disabled commands in this server."""
+        disabled_list = await self.bot._config.guild(ctx.guild).disabled_commands()
+        if not disabled_list:
+            return await ctx.send(_("There aren't any disabled commands in {}.").format(ctx.guild))
+
+        if len(disabled_list) > 1:
+            header = _("{} commands are disabled in {}.\n").format(
+                humanize_number(len(disabled_list)), ctx.guild
+            )
+        else:
+            header = _("1 command is disabled in {}.\n").format(ctx.guild)
+        paged = [box(x) for x in pagify(humanize_list(disabled_list), page_length=1000)]
+        paged[0] = header + paged[0]
+        await ctx.send_interactive(paged)
+
     @command_manager.group(name="disable", invoke_without_command=True)
     async def command_disable(self, ctx: commands.Context, *, command: str):
         """Disable a command.
@@ -1907,6 +1962,12 @@ class Core(commands.Cog, CoreLogic):
             )
             return
 
+        if isinstance(command_obj, commands.commands._AlwaysAvailableCommand):
+            await ctx.send(
+                _("This command is designated as being always available and cannot be disabled.")
+            )
+            return
+
         async with ctx.bot._config.disabled_commands() as disabled_commands:
             if command not in disabled_commands:
                 disabled_commands.append(command_obj.qualified_name)
@@ -1935,6 +1996,12 @@ class Core(commands.Cog, CoreLogic):
             )
             return
 
+        if isinstance(command_obj, commands.commands._AlwaysAvailableCommand):
+            await ctx.send(
+                _("This command is designated as being always available and cannot be disabled.")
+            )
+            return
+
         if command_obj.requires.privilege_level > await PrivilegeLevel.from_ctx(ctx):
             await ctx.send(_("You are not allowed to disable that command."))
             return
@@ -2215,3 +2282,21 @@ class Core(commands.Cog, CoreLogic):
     async def rpc_reload(self, request):
         await self.rpc_unload(request)
         await self.rpc_load(request)
+
+
+# Removing this command from forks is a violation of the GPLv3 under which it is licensed.
+# Otherwise interfering with the ability for this command to be accessible is also a violation.
+@commands.command(cls=commands.commands._AlwaysAvailableCommand, name="licenseinfo", i18n=_)
+async def license_info_command(ctx):
+    """
+    Get info about Red's licenses
+    """
+
+    message = (
+        "This bot is an instance of Red-DiscordBot (hereafter refered to as Red)\n"
+        "Red is a free and open source application made available to the public and "
+        "licensed under the GNU GPLv3. The full text of this license is available to you at "
+        "<https://github.com/Cog-Creators/Red-DiscordBot/blob/V3/develop/LICENSE>"
+    )
+    await ctx.send(message)
+    # We need a link which contains a thank you to other projects which we use at some point.

redbot/core/drivers/__init__.py

@@ -4,7 +4,6 @@ from typing import Optional, Type
 from .. import data_manager
 from .base import IdentifierData, BaseDriver, ConfigCategory
 from .json import JsonDriver
-from .mongo import MongoDriver
 from .postgres import PostgresDriver
 
 __all__ = [
@@ -13,7 +12,6 @@ __all__ = [
     "IdentifierData",
     "BaseDriver",
     "JsonDriver",
-    "MongoDriver",
     "PostgresDriver",
     "BackendType",
 ]
@@ -21,16 +19,13 @@ __all__ = [
 
 class BackendType(enum.Enum):
     JSON = "JSON"
-    MONGO = "MongoDBV2"
-    MONGOV1 = "MongoDB"
     POSTGRES = "Postgres"
+    # Dead drivrs below retained for error handling.
+    MONGOV1 = "MongoDB"
+    MONGO = "MongoDBV2"
 
 
-_DRIVER_CLASSES = {
+_DRIVER_CLASSES = {BackendType.JSON: JsonDriver, BackendType.POSTGRES: PostgresDriver}
-    BackendType.JSON: JsonDriver,
-    BackendType.MONGO: MongoDriver,
-    BackendType.POSTGRES: PostgresDriver,
-}
 
 
 def get_driver_class(storage_type: Optional[BackendType] = None) -> Type[BaseDriver]:
@@ -86,7 +81,7 @@ def get_driver(
     Raises
     ------
     RuntimeError
-        If the storage type is MongoV1 or invalid.
+        If the storage type is MongoV1, Mongo, or invalid.
 
     """
     if storage_type is None:
@@ -98,12 +93,10 @@ def get_driver(
     try:
         driver_cls: Type[BaseDriver] = get_driver_class(storage_type)
     except ValueError:
-        if storage_type == BackendType.MONGOV1:
+        if storage_type in (BackendType.MONGOV1, BackendType.MONGO):
             raise RuntimeError(
                 "Please convert to JSON first to continue using the bot."
-                " This is a required conversion prior to using the new Mongo driver."
+                "Mongo support was removed in 3.2."
-                " This message will be updated with a link to the update docs once those"
-                " docs have been created."
             ) from None
         else:
             raise RuntimeError(f"Invalid driver type: '{storage_type}'") from None

redbot/core/drivers/json.py

@@ -221,7 +221,7 @@ def _save_json(path: Path, data: Dict[str, Any]) -> None:
 
     On windows, it is not available in entirety.
     If a windows user ends up with tons of temp files, they should consider hosting on
-    something POSIX compatible, or using the mongo backend instead.
+    something POSIX compatible, or using a different backend instead.
 
     Most users wont encounter this issue, but with high write volumes,
     without the fsync on both the temp file, and after the replace on the directory,