do better with loop cleanup (#3245)

* do better with loop cleanup

* changelog

* remove redundant line

* Do this a bit better than the initial pass

* Improve windows support

Make some other things coroutines to work with improved design

* Wish we'd have done this right from the start...

* Update deps surrounding this

 - see bpo-23057
 - neccessary for windows users
 - nice for consistent support channel info / feature availability

* dep issue

* Fix tests

* duplication plugin py version

* actually handle this

* Reconfigure some checks with codeclimate, disable pylint for now

* style

* Is my exasperation showing yet?

* handle some stupid stuff

* meh

* dep changelog

.bandit.yml

@@ -0,0 +1,396 @@
+
+### Bandit config file generated
+
+### This config may optionally select a subset of tests to run or skip by
+### filling out the 'tests' and 'skips' lists given below. If no tests are
+### specified for inclusion then it is assumed all tests are desired. The skips
+### set will remove specific tests from the include set. This can be controlled
+### using the -t/-s CLI options. Note that the same test ID should not appear
+### in both 'tests' and 'skips', this would be nonsensical and is detected by
+### Bandit at runtime.
+
+# Available tests:
+# B101 : assert_used
+# B102 : exec_used
+# B103 : set_bad_file_permissions
+# B104 : hardcoded_bind_all_interfaces
+# B105 : hardcoded_password_string
+# B106 : hardcoded_password_funcarg
+# B107 : hardcoded_password_default
+# B108 : hardcoded_tmp_directory
+# B110 : try_except_pass
+# B112 : try_except_continue
+# B201 : flask_debug_true
+# B301 : pickle
+# B302 : marshal
+# B303 : md5
+# B304 : ciphers
+# B305 : cipher_modes
+# B306 : mktemp_q
+# B307 : eval
+# B308 : mark_safe
+# B309 : httpsconnection
+# B310 : urllib_urlopen
+# B311 : random
+# B312 : telnetlib
+# B313 : xml_bad_cElementTree
+# B314 : xml_bad_ElementTree
+# B315 : xml_bad_expatreader
+# B316 : xml_bad_expatbuilder
+# B317 : xml_bad_sax
+# B318 : xml_bad_minidom
+# B319 : xml_bad_pulldom
+# B320 : xml_bad_etree
+# B321 : ftplib
+# B322 : input
+# B323 : unverified_context
+# B324 : hashlib_new_insecure_functions
+# B325 : tempnam
+# B401 : import_telnetlib
+# B402 : import_ftplib
+# B403 : import_pickle
+# B404 : import_subprocess
+# B405 : import_xml_etree
+# B406 : import_xml_sax
+# B407 : import_xml_expat
+# B408 : import_xml_minidom
+# B409 : import_xml_pulldom
+# B410 : import_lxml
+# B411 : import_xmlrpclib
+# B412 : import_httpoxy
+# B413 : import_pycrypto
+# B501 : request_with_no_cert_validation
+# B502 : ssl_with_bad_version
+# B503 : ssl_with_bad_defaults
+# B504 : ssl_with_no_version
+# B505 : weak_cryptographic_key
+# B506 : yaml_load
+# B507 : ssh_no_host_key_verification
+# B601 : paramiko_calls
+# B602 : subprocess_popen_with_shell_equals_true
+# B603 : subprocess_without_shell_equals_true
+# B604 : any_other_function_with_shell_equals_true
+# B605 : start_process_with_a_shell
+# B606 : start_process_with_no_shell
+# B607 : start_process_with_partial_path
+# B608 : hardcoded_sql_expressions
+# B609 : linux_commands_wildcard_injection
+# B610 : django_extra_used
+# B611 : django_rawsql_used
+# B701 : jinja2_autoescape_false
+# B702 : use_of_mako_templates
+# B703 : django_mark_safe
+
+# (optional) list included test IDs here, eg '[B101, B406]':
+tests:
+
+# (optional) list skipped test IDs here, eg '[B101, B406]':
+skips:  ['B322']
+
+### (optional) plugin settings - some test plugins require configuration data
+### that may be given here, per-plugin. All bandit test plugins have a built in
+### set of sensible defaults and these will be used if no configuration is
+### provided. It is not necessary to provide settings for every (or any) plugin
+### if the defaults are acceptable.
+
+any_other_function_with_shell_equals_true:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+hardcoded_tmp_directory:
+  tmp_dirs:
+  - /tmp
+  - /var/tmp
+  - /dev/shm
+linux_commands_wildcard_injection:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+ssl_with_bad_defaults:
+  bad_protocol_versions:
+  - PROTOCOL_SSLv2
+  - SSLv2_METHOD
+  - SSLv23_METHOD
+  - PROTOCOL_SSLv3
+  - PROTOCOL_TLSv1
+  - SSLv3_METHOD
+  - TLSv1_METHOD
+ssl_with_bad_version:
+  bad_protocol_versions:
+  - PROTOCOL_SSLv2
+  - SSLv2_METHOD
+  - SSLv23_METHOD
+  - PROTOCOL_SSLv3
+  - PROTOCOL_TLSv1
+  - SSLv3_METHOD
+  - TLSv1_METHOD
+start_process_with_a_shell:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+start_process_with_no_shell:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+start_process_with_partial_path:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+subprocess_popen_with_shell_equals_true:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+subprocess_without_shell_equals_true:
+  no_shell:
+  - os.execl
+  - os.execle
+  - os.execlp
+  - os.execlpe
+  - os.execv
+  - os.execve
+  - os.execvp
+  - os.execvpe
+  - os.spawnl
+  - os.spawnle
+  - os.spawnlp
+  - os.spawnlpe
+  - os.spawnv
+  - os.spawnve
+  - os.spawnvp
+  - os.spawnvpe
+  - os.startfile
+  shell:
+  - os.system
+  - os.popen
+  - os.popen2
+  - os.popen3
+  - os.popen4
+  - popen2.popen2
+  - popen2.popen3
+  - popen2.popen4
+  - popen2.Popen3
+  - popen2.Popen4
+  - commands.getoutput
+  - commands.getstatusoutput
+  subprocess:
+  - subprocess.Popen
+  - subprocess.call
+  - subprocess.check_call
+  - subprocess.check_output
+  - subprocess.run
+try_except_continue:
+  check_typed_exception: false
+try_except_pass:
+  check_typed_exception: false
+weak_cryptographic_key:
+  weak_key_size_dsa_high: 1024
+  weak_key_size_dsa_medium: 2048
+  weak_key_size_ec_high: 160
+  weak_key_size_ec_medium: 224
+  weak_key_size_rsa_high: 1024
+  weak_key_size_rsa_medium: 2048
+

.codeclimate.yml

@@ -2,14 +2,15 @@ version: "2"  # required to adjust maintainability checks
 checks:
   argument-count:
     config:
-      threshold: 6
+      threshold: 8  # work on this later
   complex-logic:
     enabled: false  # Disabled in favor of using Radon for this
     config:
       threshold: 4
   file-lines:
+    enabled: false  # enable after audio stuff...
     config:
-      threshold: 1000  # I would set this lower if not for cogs as command containers.
+      threshold: 2000  # I would set this lower if not for cogs as command containers.
   method-complexity:
     enabled: false  # Disabled in favor of using Radon for this
     config:
@@ -24,7 +25,7 @@ checks:
       threshold: 25  # I'm fine with long methods, cautious about the complexity of a single method.
   nested-control-flow:
     config:
-      threshold: 4
+      threshold: 6
   return-statements:
     config:
       threshold: 6
@@ -33,12 +34,19 @@ checks:
     config:
       threshold: # language-specific defaults. an override will affect all languages.
   identical-code:
+    enabled: false
     config:
       threshold: # language-specific defaults. an override will affect all languages.
 plugins:
   bandit:
-    enabled: true
+    enabled: false
   radon: 
-    enabled: true
+    enabled: false
     config: 
       threshold: "D"
+  duplication:
+    enabled: false
+    config:
+      languages:
+        python:
+          python_version: 3

.travis.yml

@@ -5,7 +5,7 @@ notifications:
   email: false
 
 python:
-- 3.7.3
+- 3.8.1
 env:
   global:
     - PIPENV_IGNORE_VIRTUALENVS=1
@@ -30,7 +30,7 @@ jobs:
     # These jobs only occur on tag creation if the prior ones succeed
     - stage: PyPi Deployment
       if: tag IS present
-      python: 3.7.2
+      python: 3.8.1
       env:
         - DEPLOYING=true
         - TOXENV=py36
@@ -46,7 +46,7 @@ jobs:
             tags: true
     - stage: Crowdin Deployment
       if: tag IS present
-      python: 3.7.2
+      python: 3.8.1
       env:
         - DEPLOYING=true
         - TOXENV=py36

Makefile

@@ -1,4 +1,4 @@
-PYTHON ?= python3.7
+PYTHON ?= python3.8
 
 # Python Code Style
 reformat:

changelog.d/3245.dep.rst

@@ -0,0 +1,4 @@
+Update python minimum requirement to 3.8.1
+
+Update JRE to Java 11
+

changelog.d/3245.misc.rst

@@ -0,0 +1 @@
+Do a little better with loop cleanup

docs/install_linux_mac.rst

@@ -17,10 +17,10 @@ Installing the pre-requirements
 Please install the pre-requirements using the commands listed for your operating system.
 
 The pre-requirements are:
- - Python 3.7.0 or greater
+ - Python 3.8.1 or greater
- - Pip 9.0 or greater
+ - Pip 18.1 or greater
  - Git
- - Java Runtime Environment 8 or later (for audio support)
+ - Java Runtime Environment 11 or later (for audio support)
 
 We also recommend installing some basic compiler tools, in case our dependencies don't provide
 pre-built "wheels" for your architecture.
@@ -47,7 +47,7 @@ CentOS and RHEL 7
     yum -y groupinstall development
     yum -y install https://centos7.iuscommunity.org/ius-release.rpm
     sudo yum install zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel \
-      openssl-devel xz xz-devel libffi-devel findutils git2u java-1.8.0-openjdk
+      openssl-devel xz xz-devel libffi-devel findutils git2u java-11-openjdk
 
 Complete the rest of the installation by `installing Python 3.7 with pyenv <install-python-pyenv>`.
 
@@ -67,7 +67,7 @@ them with apt:
 .. code-block:: none
 
     sudo apt update
-    sudo apt install python3 python3-dev python3-venv python3-pip git default-jre-headless \
+    sudo apt install python3 python3-dev python3-venv python3-pip git openjdk-11-jre \
       build-essential
 
 Debian and Raspbian Stretch
@@ -81,9 +81,9 @@ Debian/Raspbian Stretch. This guide will tell you how. First, run the following
     sudo apt update
     sudo apt install build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
       libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \
-      liblzma-dev python3-openssl git default-jre-headless
+      liblzma-dev python3-openssl git openjdk-11-jre
 
-Complete the rest of the installation by `installing Python 3.7 with pyenv <install-python-pyenv>`.
+Complete the rest of the installation by `installing Python 3.8 with pyenv <install-python-pyenv>`.
 
 .. _install-fedora:
 
@@ -119,10 +119,10 @@ one-by-one:
     brew install python --with-brewed-openssl
     brew install git
     brew tap caskroom/versions
-    brew cask install homebrew/cask-versions/adoptopenjdk8
+    brew cask install homebrew/cask-versions/adoptopenjdk11
 
 It's possible you will have network issues. If so, go in your Applications folder, inside it, go in
-the Python 3.7 folder then double click ``Install certificates.command``.
+the Python 3.8 folder then double click ``Install certificates.command``.
 
 .. _install-opensuse:
 
@@ -133,7 +133,7 @@ openSUSE
 openSUSE Leap
 *************
 
-We recommend installing a community package to get Python 3.7 on openSUSE Leap. This package will
+We recommend installing a community package to get Python 3.8 on openSUSE Leap. This package will
 be installed to the ``/opt`` directory.
 
 First, add the Opt-Python community repository:
@@ -147,7 +147,7 @@ Now install the pre-requirements with zypper:
 
 .. code-block:: none
 
-    sudo zypper install opt-python37 opt-python37-setuptools git-core java-11-openjdk-headless
+    sudo zypper install opt-python38 opt-python38-setuptools git-core java-11-openjdk-headless
     sudo zypper install -t pattern devel_basis
 
 Since Python is now installed to ``/opt/python``, we should add it to PATH. You can add a file in
@@ -162,7 +162,7 @@ Now, install pip with easy_install:
 
 .. code-block:: none
 
-    sudo /opt/python/bin/easy_install-3.7 pip
+    sudo /opt/python/bin/easy_install-3.8 pip
 
 openSUSE Tumbleweed
 *******************
@@ -181,10 +181,9 @@ with zypper:
 Ubuntu
 ~~~~~~
 
-.. note:: **Ubuntu 16.04 Users**
+.. note:: **Ubuntu Python Availability**
 
-    You must add a 3rd-party repository to install Python 3.7 on Ubuntu 16.04 with apt. We
+   We recommend using the deadsnakes ppa to ensure up to date python availability.
-    recommend the ``deadsnakes`` repository:
 
     .. code-block:: none
 
@@ -196,7 +195,7 @@ Install the pre-requirements with apt:
 .. code-block:: none
 
     sudo apt update
-    sudo apt install python3.7 python3.7-dev python3.7-venv python3-pip git default-jre-headless \
+    sudo apt install python3.8 python3.8-dev python3.8-venv python3-pip git default-jre-headless \
       build-essential
 
 .. _install-python-pyenv:
@@ -210,7 +209,7 @@ Installing Python with pyenv
     If you followed one of the sections above, and weren't linked here afterwards, you should skip
     this section.
 
-On distributions where Python 3.7 needs to be compiled from source, we recommend the use of pyenv.
+On distributions where Python 3.8 needs to be compiled from source, we recommend the use of pyenv.
 This simplifies the compilation process and has the added bonus of simplifying setting up Red in a
 virtual environment.
 
@@ -225,7 +224,7 @@ Then run the following command:
 
 .. code-block:: none
 
-    CONFIGURE_OPTS=--enable-optimizations pyenv install 3.7.4 -v
+    CONFIGURE_OPTS=--enable-optimizations pyenv install 3.8.1 -v
 
 This may take a long time to complete, depending on your hardware. For some machines (such as
 Raspberry Pis and micro-tier VPSes), it may take over an hour; in this case, you may wish to remove
@@ -237,9 +236,9 @@ After that is finished, run:
 
 .. code-block:: none
 
-    pyenv global 3.7.4
+    pyenv global 3.8.1
 
-Pyenv is now installed and your system should be configured to run Python 3.7.
+Pyenv is now installed and your system should be configured to run Python 3.8.
 
 ------------------------------
 Creating a Virtual Environment
@@ -259,23 +258,23 @@ Choose one of the following commands to install Red.
 .. note::
 
     If you're not inside an activated virtual environment, include the ``--user`` flag with all
-    ``python3.7 -m pip install`` commands, like this:
+    ``python3.8 -m pip install`` commands, like this:
 
     .. code-block:: none
 
-        python3.7 -m pip install --user -U Red-DiscordBot
+        python3.8 -m pip install --user -U Red-DiscordBot
 
 To install without additional config backend support:
 
 .. code-block:: none
 
-    python3.7 -m pip install -U Red-DiscordBot
+    python3.8 -m pip install -U Red-DiscordBot
 
 Or, to install with PostgreSQL support:
 
 .. code-block:: none
 
-    python3.7 -m pip install -U Red-DiscordBot[postgres]
+    python3.8 -m pip install -U Red-DiscordBot[postgres]
 
 .. note::
 

docs/install_windows.rst

@@ -28,7 +28,7 @@ Then run each of the following commands:
     iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
     choco install git --params "/GitOnlyOnPath /WindowsTerminal" -y
     choco install visualstudio2019-workload-vctools -y
-    choco install python3 --version=3.7.5 -y
+    choco install python3 --version=3.8.1 -y
 
 For Audio support, you should also run the following command before exiting:
 
@@ -50,7 +50,7 @@ Manually installing dependencies
 
 * `MSVC Build tools <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2019>`_
 
-* `Python <https://www.python.org/downloads/>`_ - Red needs Python 3.7.2 or greater
+* `Python <https://www.python.org/downloads/>`_ - Red needs Python 3.8.1 or greater
 
 .. attention:: Please make sure that the box to add Python to PATH is CHECKED, otherwise
                you may run into issues when trying to run Red.
@@ -77,12 +77,12 @@ Installing Red
 
   .. note::
 
-      If you're not inside an activated virtual environment, use ``py -3.7`` in place of
+      If you're not inside an activated virtual environment, use ``py -3.8`` in place of
       ``python``, and include the ``--user`` flag with all ``pip install`` commands, like this:
 
       .. code-block:: none
 
-          py -3.7 -m pip install --user -U Red-DiscordBot
+          py -3.8 -m pip install --user -U Red-DiscordBot
 
   * Normal installation:
 
@@ -94,7 +94,7 @@ Installing Red
 
     .. code-block:: none
 
-        python3.7 -m pip install -U Red-DiscordBot[postgres]
+        python3.8 -m pip install -U Red-DiscordBot[postgres]
 
   .. note::
 

make.bat

@@ -22,7 +22,7 @@ black -l 99 --check --target-version py37 !PYFILES!
 exit /B %ERRORLEVEL%
 
 :newenv
-py -3.7 -m venv --clear .venv
+py -3.8 -m venv --clear .venv
 .\.venv\Scripts\python -m pip install -U pip setuptools
 goto syncenv
 

redbot/__init__.py

@@ -14,7 +14,7 @@ from typing import (
 )
 
 
-MIN_PYTHON_VERSION = (3, 7, 0)
+MIN_PYTHON_VERSION = (3, 8, 1)
 
 __all__ = [
     "MIN_PYTHON_VERSION",

redbot/__main__.py

@@ -3,6 +3,7 @@
 # Discord Version check
 
 import asyncio
+import functools
 import getpass
 import json
 import logging
@@ -10,7 +11,9 @@ import os
 import pip
 import platform
 import shutil
+import signal
 import sys
+from argparse import Namespace
 from copy import deepcopy
 from pathlib import Path
 
@@ -24,17 +27,11 @@ from redbot import _update_event_loop_policy, __version__
 _update_event_loop_policy()
 
 import redbot.logging
-from redbot.core.bot import Red, ExitCodes
+from redbot.core.bot import Red
-from redbot.core.cog_manager import CogManagerUI
-from redbot.core.global_checks import init_global_checks
-from redbot.core.events import init_events
 from redbot.core.cli import interactive_config, confirm, parse_cli_flags
-from redbot.core.core_commands import Core, license_info_command
 from redbot.setup import get_data_dir, get_name, save_config
-from redbot.core.dev_commands import Dev
+from redbot.core import data_manager, drivers
-from redbot.core import __version__, modlog, bank, data_manager, drivers
 from redbot.core._sharedlibdeprecation import SharedLibImportWarner
-from signal import SIGTERM
 
 
 log = logging.getLogger("red.main")
@@ -46,16 +43,6 @@ log = logging.getLogger("red.main")
 #
 
 
-async def _get_prefix_and_token(red, indict):
-    """
-    Again, please blame <@269933075037814786> for this.
-    :param indict:
-    :return:
-    """
-    indict["token"] = await red._config.token()
-    indict["prefix"] = await red._config.prefix()
-
-
 def _get_instance_names():
     with data_manager.config_file.open(encoding="utf-8") as fs:
         data = json.load(fs)
@@ -115,7 +102,7 @@ def debug_info():
     sys.exit(0)
 
 
-def edit_instance(red, cli_flags):
+async def edit_instance(red, cli_flags):
     no_prompt = cli_flags.no_prompt
     token = cli_flags.token
     owner = cli_flags.owner
@@ -138,8 +125,8 @@ def edit_instance(red, cli_flags):
         )
         sys.exit(1)
 
-    _edit_token(red, token, no_prompt)
+    await _edit_token(red, token, no_prompt)
-    _edit_owner(red, owner, no_prompt)
+    await _edit_owner(red, owner, no_prompt)
 
     data = deepcopy(data_manager.basic_config)
     name = _edit_instance_name(old_name, new_name, confirm_overwrite, no_prompt)
@@ -150,7 +137,7 @@ def edit_instance(red, cli_flags):
         save_config(old_name, {}, remove=True)
 
 
-def _edit_token(red, token, no_prompt):
+async def _edit_token(red, token, no_prompt):
     if token:
         if not len(token) >= 50:
             print(
@@ -158,13 +145,13 @@ def _edit_token(red, token, no_prompt):
                 " Instance's token will remain unchanged.\n"
             )
             return
-        red.loop.run_until_complete(red._config.token.set(token))
+        await red._config.token.set(token)
     elif not no_prompt and confirm("Would you like to change instance's token?", default=False):
-        interactive_config(red, False, True, print_header=False)
+        await interactive_config(red, False, True, print_header=False)
         print("Token updated.\n")
 
 
-def _edit_owner(red, owner, no_prompt):
+async def _edit_owner(red, owner, no_prompt):
     if owner:
         if not (15 <= len(str(owner)) <= 21):
             print(
@@ -172,7 +159,7 @@ def _edit_owner(red, owner, no_prompt):
                 " Instance's owner will remain unchanged."
             )
             return
-        red.loop.run_until_complete(red._config.owner.set(owner))
+        await red._config.owner.set(owner)
     elif not no_prompt and confirm("Would you like to change instance's owner?", default=False):
         print(
             "Remember:\n"
@@ -188,7 +175,7 @@ def _edit_owner(red, owner, no_prompt):
                     print("That doesn't look like a valid Discord user id.")
                     continue
                 owner_id = int(owner_id)
-                red.loop.run_until_complete(red._config.owner.set(owner_id))
+                await red._config.owner.set(owner_id)
                 print("Owner updated.")
                 break
         else:
@@ -259,14 +246,72 @@ def _copy_data(data):
     return True
 
 
-async def sigterm_handler(red, log):
+async def run_bot(red: Red, cli_flags: Namespace):
-    log.info("SIGTERM received. Quitting...")
+
-    await red.shutdown(restart=False)
+    driver_cls = drivers.get_driver_class()
+
+    await driver_cls.initialize(**data_manager.storage_details())
+
+    redbot.logging.init_logging(
+        level=cli_flags.logging_level, location=data_manager.core_data_path() / "logs"
+    )
+
+    log.debug("====Basic Config====")
+    log.debug("Data Path: %s", data_manager._base_data_path())
+    log.debug("Storage Type: %s", data_manager.storage_type())
+
+    if cli_flags.edit:
+        try:
+            edit_instance(red, cli_flags)
+        except (KeyboardInterrupt, EOFError):
+            print("Aborted!")
+        finally:
+            await driver_cls.teardown()
+        sys.exit(0)
+
+    # lib folder has to be in sys.path before trying to load any 3rd-party cog (GH-3061)
+    # We might want to change handling of requirements in Downloader at later date
+    LIB_PATH = data_manager.cog_data_path(raw_name="Downloader") / "lib"
+    LIB_PATH.mkdir(parents=True, exist_ok=True)
+    if str(LIB_PATH) not in sys.path:
+        sys.path.append(str(LIB_PATH))
+    sys.meta_path.insert(0, SharedLibImportWarner())
+
+    if cli_flags.token:
+        token = cli_flags.token
+    else:
+        token = os.environ.get("RED_TOKEN", None)
+        if not token:
+            token = await red._config.token()
+
+    prefix = cli_flags.prefix or await red._config.prefix()
+
+    if not (token and prefix):
+        if cli_flags.no_prompt is False:
+            new_token = await interactive_config(
+                red, token_set=bool(token), prefix_set=bool(prefix)
+            )
+            if new_token:
+                token = new_token
+        else:
+            log.critical("Token and prefix must be set in order to login.")
+            sys.exit(1)
+
+    if cli_flags.dry_run:
+        await red.http.close()
+        sys.exit(0)
+    try:
+        await red.start(token, bot=True, cli_flags=cli_flags)
+    except discord.LoginFailure:
+        log.critical("This token doesn't seem to be valid.")
+        db_token = await red._config.token()
+        if db_token and not cli_flags.no_prompt:
+            if confirm("\nDo you want to reset the token?"):
+                await red._config.token.set("")
+                print("Token has been reset.")
 
 
-def main():
+def handle_early_exit_flags(cli_flags: Namespace):
-    description = "Red V3"
-    cli_flags = parse_cli_flags(sys.argv[1:])
     if cli_flags.list_instances:
         list_instances()
     elif cli_flags.version:
@@ -278,111 +323,75 @@ def main():
     elif not cli_flags.instance_name and (not cli_flags.no_instance or cli_flags.edit):
         print("Error: No instance name was provided!")
         sys.exit(1)
-    if cli_flags.no_instance:
-        print(
-            "\033[1m"
-            "Warning: The data will be placed in a temporary folder and removed on next system "
-            "reboot."
-            "\033[0m"
-        )
-        cli_flags.instance_name = "temporary_red"
-        data_manager.create_temp_config()
-    loop = asyncio.get_event_loop()
 
-    data_manager.load_basic_configuration(cli_flags.instance_name)
-    driver_cls = drivers.get_driver_class()
-    loop.run_until_complete(driver_cls.initialize(**data_manager.storage_details()))
-    redbot.logging.init_logging(
-        level=cli_flags.logging_level, location=data_manager.core_data_path() / "logs"
-    )
 
-    log.debug("====Basic Config====")
+async def shutdown_handler(red, signal_type=None):
-    log.debug("Data Path: %s", data_manager._base_data_path())
+    if signal_type:
-    log.debug("Storage Type: %s", data_manager.storage_type())
+        log.info("%s received. Quitting...", signal_type)
+        exit_code = 0
+    else:
+        log.info("Shutting down from unhandled exception")
+        exit_code = 1
+    await red.logout()
+    await red.loop.shutdown_asyncgens()
+    pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+    [task.cancel() for task in pending]
+    await asyncio.gather(*pending, loop=red.loop, return_exceptions=True)
+    sys.exit(exit_code)
 
-    red = Red(
-        cli_flags=cli_flags, description=description, dm_help=None, fetch_offline_members=True
-    )
-    loop.run_until_complete(red._maybe_update_config())
 
-    if cli_flags.edit:
+def exception_handler(red, loop, context):
-        try:
+    msg = context.get("exception", context["message"])
-            edit_instance(red, cli_flags)
+    if isinstance(msg, KeyboardInterrupt):
-        except (KeyboardInterrupt, EOFError):
+        # Windows support is ugly, I'm sorry
-            print("Aborted!")
+        logging.error("Received KeyboardInterrupt, treating as interrupt")
-        finally:
+        signal_type = signal.SIGINT
-            loop.run_until_complete(driver_cls.teardown())
+    else:
-        sys.exit(0)
+        logging.critical("Caught fatal exception: %s", msg)
+        signal_type = None
+    loop.create_task(shutdown_handler(red, signal_type))
 
-    init_global_checks(red)
-    init_events(red, cli_flags)
 
-    # lib folder has to be in sys.path before trying to load any 3rd-party cog (GH-3061)
+def main():
-    # We might want to change handling of requirements in Downloader at later date
+    cli_flags = parse_cli_flags(sys.argv[1:])
-    LIB_PATH = data_manager.cog_data_path(raw_name="Downloader") / "lib"
+    handle_early_exit_flags(cli_flags)
-    LIB_PATH.mkdir(parents=True, exist_ok=True)
-    if str(LIB_PATH) not in sys.path:
-        sys.path.append(str(LIB_PATH))
-    sys.meta_path.insert(0, SharedLibImportWarner())
-
-    red.add_cog(Core(red))
-    red.add_cog(CogManagerUI())
-    red.add_command(license_info_command)
-    if cli_flags.dev:
-        red.add_cog(Dev())
-    # noinspection PyProtectedMember
-    loop.run_until_complete(modlog._init(red))
-    # noinspection PyProtectedMember
-    bank._init()
-
-    if os.name == "posix":
-        loop.add_signal_handler(SIGTERM, lambda: asyncio.ensure_future(sigterm_handler(red, log)))
-    tmp_data = {}
-    loop.run_until_complete(_get_prefix_and_token(red, tmp_data))
-    token = os.environ.get("RED_TOKEN", tmp_data["token"])
-    if cli_flags.token:
-        token = cli_flags.token
-    prefix = cli_flags.prefix or tmp_data["prefix"]
-    if not (token and prefix):
-        if cli_flags.no_prompt is False:
-            new_token = interactive_config(red, token_set=bool(token), prefix_set=bool(prefix))
-            if new_token:
-                token = new_token
-        else:
-            log.critical("Token and prefix must be set in order to login.")
-            sys.exit(1)
-    loop.run_until_complete(_get_prefix_and_token(red, tmp_data))
-
-    if cli_flags.dry_run:
-        loop.run_until_complete(red.http.close())
-        sys.exit(0)
     try:
-        loop.run_until_complete(red.start(token, bot=True, cli_flags=cli_flags))
+        loop = asyncio.get_event_loop()
-    except discord.LoginFailure:
-        log.critical("This token doesn't seem to be valid.")
-        db_token = loop.run_until_complete(red._config.token())
-        if db_token and not cli_flags.no_prompt:
-            if confirm("\nDo you want to reset the token?"):
-                loop.run_until_complete(red._config.token.set(""))
-                print("Token has been reset.")
-    except KeyboardInterrupt:
-        log.info("Keyboard interrupt detected. Quitting...")
-        loop.run_until_complete(red.logout())
-        red._shutdown_mode = ExitCodes.SHUTDOWN
-    except Exception as e:
-        log.critical("Fatal exception", exc_info=e)
-        loop.run_until_complete(red.logout())
-    finally:
-        pending = asyncio.Task.all_tasks(loop=red.loop)
-        gathered = asyncio.gather(*pending, loop=red.loop, return_exceptions=True)
-        gathered.cancel()
-        try:
-            loop.run_until_complete(red.rpc.close())
-        except AttributeError:
-            pass
 
-        sys.exit(red._shutdown_mode.value)
+        if cli_flags.no_instance:
+            print(
+                "\033[1m"
+                "Warning: The data will be placed in a temporary folder and removed on next system "
+                "reboot."
+                "\033[0m"
+            )
+            cli_flags.instance_name = "temporary_red"
+            data_manager.create_temp_config()
+
+        data_manager.load_basic_configuration(cli_flags.instance_name)
+
+        red = Red(
+            cli_flags=cli_flags, description=description, dm_help=None, fetch_offline_members=True
+        )
+
+        if os.name != "nt":
+            # None of this works on windows, and we have to catch KeyboardInterrupt in a global handler!
+            # At least it's not a redundant handler...
+            signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
+            for s in signals:
+                loop.add_signal_handler(
+                    s, lambda s=s: asyncio.create_task(shutdown_handler(red, s))
+                )
+
+        exc_handler = functools.partial(exception_handler, red)
+        loop.set_exception_handler(exc_handler)
+        # We actually can't use asyncio.run and have graceful cleanup on Windows...
+        loop.create_task(run_bot(red, cli_flags))
+        loop.run_forever()
+    finally:
+        loop.close()
 
 
 if __name__ == "__main__":
+    description = "Red V3"
     main()

redbot/core/bank.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 import asyncio
 import datetime
-from typing import Union, List, Optional
+from typing import Union, List, Optional, TYPE_CHECKING
 from functools import wraps
 
 import discord
@@ -8,9 +10,12 @@ import discord
 from redbot.core.utils.chat_formatting import humanize_number
 from . import Config, errors, commands
 from .i18n import Translator
-from .bot import Red
+
 from .errors import BankPruneError
 
+if TYPE_CHECKING:
+    from .bot import Red
+
 _ = Translator("Bank API", __file__)
 
 __all__ = [

redbot/core/bot.py

@@ -2,6 +2,7 @@ import asyncio
 import inspect
 import logging
 import os
+import sys
 from collections import namedtuple
 from datetime import datetime
 from enum import Enum
@@ -13,8 +14,12 @@ from types import MappingProxyType
 import discord
 from discord.ext.commands import when_mentioned_or
 
-from . import Config, i18n, commands, errors, drivers
+from . import Config, i18n, commands, errors, drivers, modlog, bank
-from .cog_manager import CogManager
+from .cog_manager import CogManager, CogManagerUI
+from .core_commands import license_info_command, Core
+from .dev_commands import Dev
+from .events import init_events
+from .global_checks import init_global_checks
 
 from .rpc import RPCMixin
 from .utils import common_filters
@@ -43,6 +48,7 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
 
     def __init__(self, *args, cli_flags=None, bot_dir: Path = Path.cwd(), **kwargs):
         self._shutdown_mode = ExitCodes.CRITICAL
+        self._cli_flags = cli_flags
         self._config = Config.get_core_conf(force_registration=False)
         self._co_owners = cli_flags.co_owner
         self.rpc_enabled = cli_flags.rpc
@@ -392,6 +398,18 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin):  # pylint: d
         """
         await self._maybe_update_config()
 
+        init_global_checks(self)
+        init_events(self, cli_flags)
+
+        self.add_cog(Core(self))
+        self.add_cog(CogManagerUI())
+        self.add_command(license_info_command)
+        if cli_flags.dev:
+            self.add_cog(Dev())
+
+        await modlog._init(self)
+        bank._init()
+
         packages = []
 
         if cli_flags.no_cogs is False:
@@ -971,6 +989,10 @@ class Red(RedBase, discord.AutoShardedClient):
         """Logs out of Discord and closes all connections."""
         await super().logout()
         await drivers.get_driver_class().teardown()
+        try:
+            await self.rpc.close()
+        except AttributeError:
+            pass
 
     async def shutdown(self, *, restart: bool = False):
         """Gracefully quit Red.
@@ -990,6 +1012,7 @@ class Red(RedBase, discord.AutoShardedClient):
             self._shutdown_mode = ExitCodes.RESTART
 
         await self.logout()
+        sys.exit(self._shutdown_mode)
 
 
 class ExitCodes(Enum):

redbot/core/cli.py

@@ -33,9 +33,8 @@ def confirm(text: str, default: Optional[bool] = None) -> bool:
         print("Error: invalid input")
 
 
-def interactive_config(red, token_set, prefix_set, *, print_header=True):
+async def interactive_config(red, token_set, prefix_set, *, print_header=True):
-    loop = asyncio.get_event_loop()
+    token = None
-    token = ""
 
     if print_header:
         print("Red - Discord Bot | Configuration process\n")
@@ -51,9 +50,9 @@ def interactive_config(red, token_set, prefix_set, *, print_header=True):
             token = input("> ")
             if not len(token) >= 50:
                 print("That doesn't look like a valid token.")
-                token = ""
+                token = None
             if token:
-                loop.run_until_complete(red._config.token.set(token))
+                await red._config.token.set(token)
 
     if not prefix_set:
         prefix = ""
@@ -70,7 +69,7 @@ def interactive_config(red, token_set, prefix_set, *, print_header=True):
                 if not confirm("Your prefix seems overly long. Are you sure that it's correct?"):
                     prefix = ""
             if prefix:
-                loop.run_until_complete(red._config.prefix.set([prefix]))
+                await red._config.prefix.set([prefix])
 
     return token
 

redbot/core/events.py

@@ -4,7 +4,6 @@ import codecs
 import datetime
 import logging
 import traceback
-import asyncio
 from datetime import timedelta
 
 import aiohttp
@@ -17,7 +16,6 @@ from redbot.core.commands import RedHelpFormatter
 from .. import __version__ as red_version, version_info as red_version_info, VersionInfo
 from . import commands
 from .config import get_latest_confs
-from .data_manager import storage_type
 from .utils.chat_formatting import inline, bordered, format_perms_list, humanize_timedelta
 from .utils import fuzzy_command_search, format_fuzzy_results
 

redbot/core/modlog.py

@@ -1,11 +1,12 @@
+from __future__ import annotations
+
 import asyncio
 from datetime import datetime, timedelta
-from typing import List, Union, Optional, cast
+from typing import List, Union, Optional, cast, TYPE_CHECKING
 
 import discord
 
 from redbot.core import Config
-from redbot.core.bot import Red
 
 from .utils.common_filters import (
     filter_invites,
@@ -17,6 +18,9 @@ from .i18n import Translator
 
 from .generic_casetypes import all_generics
 
+if TYPE_CHECKING:
+    from redbot.core.bot import Red
+
 __all__ = [
     "Case",
     "CaseType",

setup.cfg

@@ -18,13 +18,13 @@ classifiers =
     License :: OSI Approved :: GNU General Public License v3 (GPLv3)
     Natural Language :: English
     Operating System :: OS Independent
-    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
     Topic :: Communications :: Chat
     Topic :: Documentation :: Sphinx
 
 [options]
 packages = find_namespace:
-python_requires = >=3.7
+python_requires = >=3.8.1
 install_requires =
     aiohttp==3.5.4
     aiohttp-json-rpc==0.12.1
@@ -87,9 +87,8 @@ style =
     black==19.3b0
     toml==0.10.0
 test =
-    astroid==2.2.5
+    astroid==2.3.3
     atomicwrites==1.3.0
-    importlib-metadata==0.19
     isort==4.3.21
     lazy-object-proxy==1.4.2
     mccabe==0.6.1
@@ -99,9 +98,9 @@ test =
     py==1.8.0
     pylint==2.3.1
     pyparsing==2.4.2
-    pytest==5.1.2
+    pytest==5.3.2
     pytest-asyncio==0.10.0
-    pytest-mock==1.11.2
+    pytest-mock==1.13.0
     six==1.12.0
     typed-ast==1.4.0
     wcwidth==0.1.7

tests/cogs/downloader/test_downloader.py

@@ -25,18 +25,11 @@ class FakeCompletedProcess(NamedTuple):
     stderr: bytes = b""
 
 
-async def async_return(ret: Any):
-    return ret
-
-
 def _mock_run(
     mocker: MockFixture, repo: Repo, returncode: int, stdout: bytes = b"", stderr: bytes = b""
 ):
     return mocker.patch.object(
-        repo,
+        repo, "_run", autospec=True, return_value=FakeCompletedProcess(returncode, stdout, stderr)
-        "_run",
-        autospec=True,
-        return_value=async_return(FakeCompletedProcess(returncode, stdout, stderr)),
     )
 
 
@@ -46,11 +39,7 @@ def _mock_setup_repo(mocker: MockFixture, repo: Repo, commit: str):
         return mocker.DEFAULT
 
     return mocker.patch.object(
-        repo,
+        repo, "_setup_repo", autospec=True, side_effect=update_commit, return_value=None
-        "_setup_repo",
-        autospec=True,
-        side_effect=update_commit,
-        return_value=async_return(None),
     )
 
 
@@ -153,15 +142,13 @@ async def test_is_module_modified(mocker, repo):
         repo,
         "_get_file_update_statuses",
         autospec=True,
-        return_value=async_return(
+        return_value={
-            {
+            "added_file.txt": "A",
-                "added_file.txt": "A",
+            "mycog/__init__.py": "M",
-                "mycog/__init__.py": "M",
+            "sample_file1.txt": "D",
-                "sample_file1.txt": "D",
+            "sample_file2.txt": "D",
-                "sample_file2.txt": "D",
+            "sample_file3.txt": "A",
-                "sample_file3.txt": "A",
+        },
-            }
-        ),
     )
     ret = await repo._is_module_modified(module, old_rev)
     m.assert_called_once_with(old_rev, new_rev)
@@ -249,11 +236,11 @@ async def test_checkout(mocker, repo):
 @pytest.mark.asyncio
 async def test_checkout_ctx_manager(mocker, repo):
     commit = "c950fc05a540dd76b944719c2a3302da2e2f3090"
-    m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=async_return(None))
+    m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=None)
     old_commit = repo.commit
     async with repo.checkout(commit):
         m.assert_called_with(commit, force_checkout=False)
-        m.return_value = async_return(None)
+        m.return_value = None
 
     m.assert_called_with(old_commit, force_checkout=False)
 
@@ -261,7 +248,7 @@ async def test_checkout_ctx_manager(mocker, repo):
 @pytest.mark.asyncio
 async def test_checkout_await(mocker, repo):
     commit = "c950fc05a540dd76b944719c2a3302da2e2f3090"
-    m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=async_return(None))
+    m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=None)
     await repo.checkout(commit)
 
     m.assert_called_once_with(commit, force_checkout=False)
@@ -293,7 +280,7 @@ async def test_clone_without_branch(mocker, repo):
     repo.commit = ""
     m = _mock_run(mocker, repo, 0)
     _mock_setup_repo(mocker, repo, commit)
-    mocker.patch.object(repo, "current_branch", autospec=True, return_value=async_return(branch))
+    mocker.patch.object(repo, "current_branch", autospec=True, return_value=branch)
 
     await repo.clone()
 
@@ -309,10 +296,8 @@ async def test_update(mocker, repo):
     new_commit = "a0ccc2390883c85a361f5a90c72e1b07958939fa"
     m = _mock_run(mocker, repo, 0)
     _mock_setup_repo(mocker, repo, new_commit)
-    mocker.patch.object(
+    mocker.patch.object(repo, "latest_commit", autospec=True, return_value=old_commit)
-        repo, "latest_commit", autospec=True, return_value=async_return(old_commit)
+    mocker.patch.object(repo, "hard_reset", autospec=True, return_value=None)
-    )
-    mocker.patch.object(repo, "hard_reset", autospec=True, return_value=async_return(None))
     ret = await repo.update()
 
     assert ret == (old_commit, new_commit)

tools/bumpdeps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.7
+#!/usr/bin/env python3.8
 """Script to bump pinned dependencies in setup.cfg.
 
 This script aims to help update our list of pinned primary and

tools/edit_testrepo.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.7
+#!/usr/bin/env python3.8
 """Script to edit test repo used by Downloader git integration tests.
 
 This script aims to help update the human-readable version of repo

tox.ini

@@ -5,7 +5,7 @@
 
 [tox]
 envlist =
-    py37
+    py38
     docs
     style
 skip_missing_interpreters = True
@@ -19,7 +19,7 @@ extras = voice, test
 commands =
     python -m compileall ./redbot/cogs
     pytest
-    pylint ./redbot
+#    pylint ./redbot
 
 [testenv:postgres]
 description = Run pytest with PostgreSQL backend
@@ -48,7 +48,7 @@ setenv =
     # This is just for Windows
     # Prioritise make.bat over any make.exe which might be on PATH
     PATHEXT=.BAT;.EXE
-basepython = python3.7
+basepython = python3.8
 extras = docs
 commands =
     sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out/html" -W -bhtml
@@ -64,7 +64,7 @@ setenv =
     # This is just for Windows
     # Prioritise make.bat over any make.exe which might be on PATH
     PATHEXT=.BAT;.EXE
-basepython = python3.7
+basepython = python3.8
 extras = style
 commands =
     make stylecheck