From a80e20067ccbcf7e3ad8f7662451b03c3e2249e6 Mon Sep 17 00:00:00 2001 From: Michael H Date: Wed, 1 Jan 2020 19:26:32 -0500 Subject: [PATCH] do better with loop cleanup (#3245) * do better with loop cleanup * changelog * remove redundant line * Do this a bit better than the initial pass * Improve windows support Make some other things coroutines to work with improved design * Wish we'd have done this right from the start... * Update deps surrounding this - see bpo-23057 - neccessary for windows users - nice for consistent support channel info / feature availability * dep issue * Fix tests * duplication plugin py version * actually handle this * Reconfigure some checks with codeclimate, disable pylint for now * style * Is my exasperation showing yet? * handle some stupid stuff * meh * dep changelog --- .bandit.yml | 396 +++++++++++++++++++++++ .codeclimate.yml | 18 +- .travis.yml | 6 +- Makefile | 2 +- changelog.d/3245.dep.rst | 4 + changelog.d/3245.misc.rst | 1 + docs/install_linux_mac.rst | 47 ++- docs/install_windows.rst | 10 +- make.bat | 2 +- redbot/__init__.py | 2 +- redbot/__main__.py | 267 +++++++-------- redbot/core/bank.py | 9 +- redbot/core/bot.py | 27 +- redbot/core/cli.py | 11 +- redbot/core/events.py | 2 - redbot/core/modlog.py | 8 +- setup.cfg | 11 +- tests/cogs/downloader/test_downloader.py | 45 +-- tools/bumpdeps.py | 2 +- tools/edit_testrepo.py | 2 +- tox.ini | 8 +- 21 files changed, 655 insertions(+), 225 deletions(-) create mode 100644 .bandit.yml create mode 100644 changelog.d/3245.dep.rst create mode 100644 changelog.d/3245.misc.rst diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 000000000..4b100f568 --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,396 @@ + +### Bandit config file generated + +### This config may optionally select a subset of tests to run or skip by +### filling out the 'tests' and 'skips' lists given below. If no tests are +### specified for inclusion then it is assumed all tests are desired. The skips +### set will remove specific tests from the include set. This can be controlled +### using the -t/-s CLI options. Note that the same test ID should not appear +### in both 'tests' and 'skips', this would be nonsensical and is detected by +### Bandit at runtime. + +# Available tests: +# B101 : assert_used +# B102 : exec_used +# B103 : set_bad_file_permissions +# B104 : hardcoded_bind_all_interfaces +# B105 : hardcoded_password_string +# B106 : hardcoded_password_funcarg +# B107 : hardcoded_password_default +# B108 : hardcoded_tmp_directory +# B110 : try_except_pass +# B112 : try_except_continue +# B201 : flask_debug_true +# B301 : pickle +# B302 : marshal +# B303 : md5 +# B304 : ciphers +# B305 : cipher_modes +# B306 : mktemp_q +# B307 : eval +# B308 : mark_safe +# B309 : httpsconnection +# B310 : urllib_urlopen +# B311 : random +# B312 : telnetlib +# B313 : xml_bad_cElementTree +# B314 : xml_bad_ElementTree +# B315 : xml_bad_expatreader +# B316 : xml_bad_expatbuilder +# B317 : xml_bad_sax +# B318 : xml_bad_minidom +# B319 : xml_bad_pulldom +# B320 : xml_bad_etree +# B321 : ftplib +# B322 : input +# B323 : unverified_context +# B324 : hashlib_new_insecure_functions +# B325 : tempnam +# B401 : import_telnetlib +# B402 : import_ftplib +# B403 : import_pickle +# B404 : import_subprocess +# B405 : import_xml_etree +# B406 : import_xml_sax +# B407 : import_xml_expat +# B408 : import_xml_minidom +# B409 : import_xml_pulldom +# B410 : import_lxml +# B411 : import_xmlrpclib +# B412 : import_httpoxy +# B413 : import_pycrypto +# B501 : request_with_no_cert_validation +# B502 : ssl_with_bad_version +# B503 : ssl_with_bad_defaults +# B504 : ssl_with_no_version +# B505 : weak_cryptographic_key +# B506 : yaml_load +# B507 : ssh_no_host_key_verification +# B601 : paramiko_calls +# B602 : subprocess_popen_with_shell_equals_true +# B603 : subprocess_without_shell_equals_true +# B604 : any_other_function_with_shell_equals_true +# B605 : start_process_with_a_shell +# B606 : start_process_with_no_shell +# B607 : start_process_with_partial_path +# B608 : hardcoded_sql_expressions +# B609 : linux_commands_wildcard_injection +# B610 : django_extra_used +# B611 : django_rawsql_used +# B701 : jinja2_autoescape_false +# B702 : use_of_mako_templates +# B703 : django_mark_safe + +# (optional) list included test IDs here, eg '[B101, B406]': +tests: + +# (optional) list skipped test IDs here, eg '[B101, B406]': +skips: ['B322'] + +### (optional) plugin settings - some test plugins require configuration data +### that may be given here, per-plugin. All bandit test plugins have a built in +### set of sensible defaults and these will be used if no configuration is +### provided. It is not necessary to provide settings for every (or any) plugin +### if the defaults are acceptable. + +any_other_function_with_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +hardcoded_tmp_directory: + tmp_dirs: + - /tmp + - /var/tmp + - /dev/shm +linux_commands_wildcard_injection: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +ssl_with_bad_defaults: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD +ssl_with_bad_version: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD +start_process_with_a_shell: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +start_process_with_no_shell: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +start_process_with_partial_path: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +subprocess_popen_with_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +subprocess_without_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +try_except_continue: + check_typed_exception: false +try_except_pass: + check_typed_exception: false +weak_cryptographic_key: + weak_key_size_dsa_high: 1024 + weak_key_size_dsa_medium: 2048 + weak_key_size_ec_high: 160 + weak_key_size_ec_medium: 224 + weak_key_size_rsa_high: 1024 + weak_key_size_rsa_medium: 2048 + diff --git a/.codeclimate.yml b/.codeclimate.yml index fba462070..3f29a0496 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -2,14 +2,15 @@ version: "2" # required to adjust maintainability checks checks: argument-count: config: - threshold: 6 + threshold: 8 # work on this later complex-logic: enabled: false # Disabled in favor of using Radon for this config: threshold: 4 file-lines: + enabled: false # enable after audio stuff... config: - threshold: 1000 # I would set this lower if not for cogs as command containers. + threshold: 2000 # I would set this lower if not for cogs as command containers. method-complexity: enabled: false # Disabled in favor of using Radon for this config: @@ -24,7 +25,7 @@ checks: threshold: 25 # I'm fine with long methods, cautious about the complexity of a single method. nested-control-flow: config: - threshold: 4 + threshold: 6 return-statements: config: threshold: 6 @@ -33,12 +34,19 @@ checks: config: threshold: # language-specific defaults. an override will affect all languages. identical-code: + enabled: false config: threshold: # language-specific defaults. an override will affect all languages. plugins: bandit: - enabled: true + enabled: false radon: - enabled: true + enabled: false config: threshold: "D" + duplication: + enabled: false + config: + languages: + python: + python_version: 3 diff --git a/.travis.yml b/.travis.yml index 282bef8df..ec1b3e346 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ notifications: email: false python: -- 3.7.3 +- 3.8.1 env: global: - PIPENV_IGNORE_VIRTUALENVS=1 @@ -30,7 +30,7 @@ jobs: # These jobs only occur on tag creation if the prior ones succeed - stage: PyPi Deployment if: tag IS present - python: 3.7.2 + python: 3.8.1 env: - DEPLOYING=true - TOXENV=py36 @@ -46,7 +46,7 @@ jobs: tags: true - stage: Crowdin Deployment if: tag IS present - python: 3.7.2 + python: 3.8.1 env: - DEPLOYING=true - TOXENV=py36 diff --git a/Makefile b/Makefile index 7ae79fedd..640f8d460 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON ?= python3.7 +PYTHON ?= python3.8 # Python Code Style reformat: diff --git a/changelog.d/3245.dep.rst b/changelog.d/3245.dep.rst new file mode 100644 index 000000000..7796f8e42 --- /dev/null +++ b/changelog.d/3245.dep.rst @@ -0,0 +1,4 @@ +Update python minimum requirement to 3.8.1 + +Update JRE to Java 11 + diff --git a/changelog.d/3245.misc.rst b/changelog.d/3245.misc.rst new file mode 100644 index 000000000..017fa2a28 --- /dev/null +++ b/changelog.d/3245.misc.rst @@ -0,0 +1 @@ +Do a little better with loop cleanup diff --git a/docs/install_linux_mac.rst b/docs/install_linux_mac.rst index af99cf783..9966962a3 100644 --- a/docs/install_linux_mac.rst +++ b/docs/install_linux_mac.rst @@ -17,10 +17,10 @@ Installing the pre-requirements Please install the pre-requirements using the commands listed for your operating system. The pre-requirements are: - - Python 3.7.0 or greater - - Pip 9.0 or greater + - Python 3.8.1 or greater + - Pip 18.1 or greater - Git - - Java Runtime Environment 8 or later (for audio support) + - Java Runtime Environment 11 or later (for audio support) We also recommend installing some basic compiler tools, in case our dependencies don't provide pre-built "wheels" for your architecture. @@ -47,7 +47,7 @@ CentOS and RHEL 7 yum -y groupinstall development yum -y install https://centos7.iuscommunity.org/ius-release.rpm sudo yum install zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel \ - openssl-devel xz xz-devel libffi-devel findutils git2u java-1.8.0-openjdk + openssl-devel xz xz-devel libffi-devel findutils git2u java-11-openjdk Complete the rest of the installation by `installing Python 3.7 with pyenv `. @@ -67,7 +67,7 @@ them with apt: .. code-block:: none sudo apt update - sudo apt install python3 python3-dev python3-venv python3-pip git default-jre-headless \ + sudo apt install python3 python3-dev python3-venv python3-pip git openjdk-11-jre \ build-essential Debian and Raspbian Stretch @@ -81,9 +81,9 @@ Debian/Raspbian Stretch. This guide will tell you how. First, run the following sudo apt update sudo apt install build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ - liblzma-dev python3-openssl git default-jre-headless + liblzma-dev python3-openssl git openjdk-11-jre -Complete the rest of the installation by `installing Python 3.7 with pyenv `. +Complete the rest of the installation by `installing Python 3.8 with pyenv `. .. _install-fedora: @@ -119,10 +119,10 @@ one-by-one: brew install python --with-brewed-openssl brew install git brew tap caskroom/versions - brew cask install homebrew/cask-versions/adoptopenjdk8 + brew cask install homebrew/cask-versions/adoptopenjdk11 It's possible you will have network issues. If so, go in your Applications folder, inside it, go in -the Python 3.7 folder then double click ``Install certificates.command``. +the Python 3.8 folder then double click ``Install certificates.command``. .. _install-opensuse: @@ -133,7 +133,7 @@ openSUSE openSUSE Leap ************* -We recommend installing a community package to get Python 3.7 on openSUSE Leap. This package will +We recommend installing a community package to get Python 3.8 on openSUSE Leap. This package will be installed to the ``/opt`` directory. First, add the Opt-Python community repository: @@ -147,7 +147,7 @@ Now install the pre-requirements with zypper: .. code-block:: none - sudo zypper install opt-python37 opt-python37-setuptools git-core java-11-openjdk-headless + sudo zypper install opt-python38 opt-python38-setuptools git-core java-11-openjdk-headless sudo zypper install -t pattern devel_basis Since Python is now installed to ``/opt/python``, we should add it to PATH. You can add a file in @@ -162,7 +162,7 @@ Now, install pip with easy_install: .. code-block:: none - sudo /opt/python/bin/easy_install-3.7 pip + sudo /opt/python/bin/easy_install-3.8 pip openSUSE Tumbleweed ******************* @@ -181,10 +181,9 @@ with zypper: Ubuntu ~~~~~~ -.. note:: **Ubuntu 16.04 Users** +.. note:: **Ubuntu Python Availability** - You must add a 3rd-party repository to install Python 3.7 on Ubuntu 16.04 with apt. We - recommend the ``deadsnakes`` repository: + We recommend using the deadsnakes ppa to ensure up to date python availability. .. code-block:: none @@ -196,7 +195,7 @@ Install the pre-requirements with apt: .. code-block:: none sudo apt update - sudo apt install python3.7 python3.7-dev python3.7-venv python3-pip git default-jre-headless \ + sudo apt install python3.8 python3.8-dev python3.8-venv python3-pip git default-jre-headless \ build-essential .. _install-python-pyenv: @@ -210,7 +209,7 @@ Installing Python with pyenv If you followed one of the sections above, and weren't linked here afterwards, you should skip this section. -On distributions where Python 3.7 needs to be compiled from source, we recommend the use of pyenv. +On distributions where Python 3.8 needs to be compiled from source, we recommend the use of pyenv. This simplifies the compilation process and has the added bonus of simplifying setting up Red in a virtual environment. @@ -225,7 +224,7 @@ Then run the following command: .. code-block:: none - CONFIGURE_OPTS=--enable-optimizations pyenv install 3.7.4 -v + CONFIGURE_OPTS=--enable-optimizations pyenv install 3.8.1 -v This may take a long time to complete, depending on your hardware. For some machines (such as Raspberry Pis and micro-tier VPSes), it may take over an hour; in this case, you may wish to remove @@ -237,9 +236,9 @@ After that is finished, run: .. code-block:: none - pyenv global 3.7.4 + pyenv global 3.8.1 -Pyenv is now installed and your system should be configured to run Python 3.7. +Pyenv is now installed and your system should be configured to run Python 3.8. ------------------------------ Creating a Virtual Environment @@ -259,23 +258,23 @@ Choose one of the following commands to install Red. .. note:: If you're not inside an activated virtual environment, include the ``--user`` flag with all - ``python3.7 -m pip install`` commands, like this: + ``python3.8 -m pip install`` commands, like this: .. code-block:: none - python3.7 -m pip install --user -U Red-DiscordBot + python3.8 -m pip install --user -U Red-DiscordBot To install without additional config backend support: .. code-block:: none - python3.7 -m pip install -U Red-DiscordBot + python3.8 -m pip install -U Red-DiscordBot Or, to install with PostgreSQL support: .. code-block:: none - python3.7 -m pip install -U Red-DiscordBot[postgres] + python3.8 -m pip install -U Red-DiscordBot[postgres] .. note:: diff --git a/docs/install_windows.rst b/docs/install_windows.rst index f5dfd66c5..7332873de 100644 --- a/docs/install_windows.rst +++ b/docs/install_windows.rst @@ -28,7 +28,7 @@ Then run each of the following commands: iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) choco install git --params "/GitOnlyOnPath /WindowsTerminal" -y choco install visualstudio2019-workload-vctools -y - choco install python3 --version=3.7.5 -y + choco install python3 --version=3.8.1 -y For Audio support, you should also run the following command before exiting: @@ -50,7 +50,7 @@ Manually installing dependencies * `MSVC Build tools `_ -* `Python `_ - Red needs Python 3.7.2 or greater +* `Python `_ - Red needs Python 3.8.1 or greater .. attention:: Please make sure that the box to add Python to PATH is CHECKED, otherwise you may run into issues when trying to run Red. @@ -77,12 +77,12 @@ Installing Red .. note:: - If you're not inside an activated virtual environment, use ``py -3.7`` in place of + If you're not inside an activated virtual environment, use ``py -3.8`` in place of ``python``, and include the ``--user`` flag with all ``pip install`` commands, like this: .. code-block:: none - py -3.7 -m pip install --user -U Red-DiscordBot + py -3.8 -m pip install --user -U Red-DiscordBot * Normal installation: @@ -94,7 +94,7 @@ Installing Red .. code-block:: none - python3.7 -m pip install -U Red-DiscordBot[postgres] + python3.8 -m pip install -U Red-DiscordBot[postgres] .. note:: diff --git a/make.bat b/make.bat index 1e38b16e7..ba2d7d504 100644 --- a/make.bat +++ b/make.bat @@ -22,7 +22,7 @@ black -l 99 --check --target-version py37 !PYFILES! exit /B %ERRORLEVEL% :newenv -py -3.7 -m venv --clear .venv +py -3.8 -m venv --clear .venv .\.venv\Scripts\python -m pip install -U pip setuptools goto syncenv diff --git a/redbot/__init__.py b/redbot/__init__.py index 2a5c0a7e7..03497bebe 100644 --- a/redbot/__init__.py +++ b/redbot/__init__.py @@ -14,7 +14,7 @@ from typing import ( ) -MIN_PYTHON_VERSION = (3, 7, 0) +MIN_PYTHON_VERSION = (3, 8, 1) __all__ = [ "MIN_PYTHON_VERSION", diff --git a/redbot/__main__.py b/redbot/__main__.py index ef0afd4da..bd0ee2511 100644 --- a/redbot/__main__.py +++ b/redbot/__main__.py @@ -3,6 +3,7 @@ # Discord Version check import asyncio +import functools import getpass import json import logging @@ -10,7 +11,9 @@ import os import pip import platform import shutil +import signal import sys +from argparse import Namespace from copy import deepcopy from pathlib import Path @@ -24,17 +27,11 @@ from redbot import _update_event_loop_policy, __version__ _update_event_loop_policy() import redbot.logging -from redbot.core.bot import Red, ExitCodes -from redbot.core.cog_manager import CogManagerUI -from redbot.core.global_checks import init_global_checks -from redbot.core.events import init_events +from redbot.core.bot import Red from redbot.core.cli import interactive_config, confirm, parse_cli_flags -from redbot.core.core_commands import Core, license_info_command from redbot.setup import get_data_dir, get_name, save_config -from redbot.core.dev_commands import Dev -from redbot.core import __version__, modlog, bank, data_manager, drivers +from redbot.core import data_manager, drivers from redbot.core._sharedlibdeprecation import SharedLibImportWarner -from signal import SIGTERM log = logging.getLogger("red.main") @@ -46,16 +43,6 @@ log = logging.getLogger("red.main") # -async def _get_prefix_and_token(red, indict): - """ - Again, please blame <@269933075037814786> for this. - :param indict: - :return: - """ - indict["token"] = await red._config.token() - indict["prefix"] = await red._config.prefix() - - def _get_instance_names(): with data_manager.config_file.open(encoding="utf-8") as fs: data = json.load(fs) @@ -115,7 +102,7 @@ def debug_info(): sys.exit(0) -def edit_instance(red, cli_flags): +async def edit_instance(red, cli_flags): no_prompt = cli_flags.no_prompt token = cli_flags.token owner = cli_flags.owner @@ -138,8 +125,8 @@ def edit_instance(red, cli_flags): ) sys.exit(1) - _edit_token(red, token, no_prompt) - _edit_owner(red, owner, no_prompt) + await _edit_token(red, token, no_prompt) + await _edit_owner(red, owner, no_prompt) data = deepcopy(data_manager.basic_config) name = _edit_instance_name(old_name, new_name, confirm_overwrite, no_prompt) @@ -150,7 +137,7 @@ def edit_instance(red, cli_flags): save_config(old_name, {}, remove=True) -def _edit_token(red, token, no_prompt): +async def _edit_token(red, token, no_prompt): if token: if not len(token) >= 50: print( @@ -158,13 +145,13 @@ def _edit_token(red, token, no_prompt): " Instance's token will remain unchanged.\n" ) return - red.loop.run_until_complete(red._config.token.set(token)) + await red._config.token.set(token) elif not no_prompt and confirm("Would you like to change instance's token?", default=False): - interactive_config(red, False, True, print_header=False) + await interactive_config(red, False, True, print_header=False) print("Token updated.\n") -def _edit_owner(red, owner, no_prompt): +async def _edit_owner(red, owner, no_prompt): if owner: if not (15 <= len(str(owner)) <= 21): print( @@ -172,7 +159,7 @@ def _edit_owner(red, owner, no_prompt): " Instance's owner will remain unchanged." ) return - red.loop.run_until_complete(red._config.owner.set(owner)) + await red._config.owner.set(owner) elif not no_prompt and confirm("Would you like to change instance's owner?", default=False): print( "Remember:\n" @@ -188,7 +175,7 @@ def _edit_owner(red, owner, no_prompt): print("That doesn't look like a valid Discord user id.") continue owner_id = int(owner_id) - red.loop.run_until_complete(red._config.owner.set(owner_id)) + await red._config.owner.set(owner_id) print("Owner updated.") break else: @@ -259,14 +246,72 @@ def _copy_data(data): return True -async def sigterm_handler(red, log): - log.info("SIGTERM received. Quitting...") - await red.shutdown(restart=False) +async def run_bot(red: Red, cli_flags: Namespace): + + driver_cls = drivers.get_driver_class() + + await driver_cls.initialize(**data_manager.storage_details()) + + redbot.logging.init_logging( + level=cli_flags.logging_level, location=data_manager.core_data_path() / "logs" + ) + + log.debug("====Basic Config====") + log.debug("Data Path: %s", data_manager._base_data_path()) + log.debug("Storage Type: %s", data_manager.storage_type()) + + if cli_flags.edit: + try: + edit_instance(red, cli_flags) + except (KeyboardInterrupt, EOFError): + print("Aborted!") + finally: + await driver_cls.teardown() + sys.exit(0) + + # lib folder has to be in sys.path before trying to load any 3rd-party cog (GH-3061) + # We might want to change handling of requirements in Downloader at later date + LIB_PATH = data_manager.cog_data_path(raw_name="Downloader") / "lib" + LIB_PATH.mkdir(parents=True, exist_ok=True) + if str(LIB_PATH) not in sys.path: + sys.path.append(str(LIB_PATH)) + sys.meta_path.insert(0, SharedLibImportWarner()) + + if cli_flags.token: + token = cli_flags.token + else: + token = os.environ.get("RED_TOKEN", None) + if not token: + token = await red._config.token() + + prefix = cli_flags.prefix or await red._config.prefix() + + if not (token and prefix): + if cli_flags.no_prompt is False: + new_token = await interactive_config( + red, token_set=bool(token), prefix_set=bool(prefix) + ) + if new_token: + token = new_token + else: + log.critical("Token and prefix must be set in order to login.") + sys.exit(1) + + if cli_flags.dry_run: + await red.http.close() + sys.exit(0) + try: + await red.start(token, bot=True, cli_flags=cli_flags) + except discord.LoginFailure: + log.critical("This token doesn't seem to be valid.") + db_token = await red._config.token() + if db_token and not cli_flags.no_prompt: + if confirm("\nDo you want to reset the token?"): + await red._config.token.set("") + print("Token has been reset.") -def main(): - description = "Red V3" - cli_flags = parse_cli_flags(sys.argv[1:]) +def handle_early_exit_flags(cli_flags: Namespace): if cli_flags.list_instances: list_instances() elif cli_flags.version: @@ -278,111 +323,75 @@ def main(): elif not cli_flags.instance_name and (not cli_flags.no_instance or cli_flags.edit): print("Error: No instance name was provided!") sys.exit(1) - if cli_flags.no_instance: - print( - "\033[1m" - "Warning: The data will be placed in a temporary folder and removed on next system " - "reboot." - "\033[0m" - ) - cli_flags.instance_name = "temporary_red" - data_manager.create_temp_config() - loop = asyncio.get_event_loop() - data_manager.load_basic_configuration(cli_flags.instance_name) - driver_cls = drivers.get_driver_class() - loop.run_until_complete(driver_cls.initialize(**data_manager.storage_details())) - redbot.logging.init_logging( - level=cli_flags.logging_level, location=data_manager.core_data_path() / "logs" - ) - log.debug("====Basic Config====") - log.debug("Data Path: %s", data_manager._base_data_path()) - log.debug("Storage Type: %s", data_manager.storage_type()) +async def shutdown_handler(red, signal_type=None): + if signal_type: + log.info("%s received. Quitting...", signal_type) + exit_code = 0 + else: + log.info("Shutting down from unhandled exception") + exit_code = 1 + await red.logout() + await red.loop.shutdown_asyncgens() + pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + [task.cancel() for task in pending] + await asyncio.gather(*pending, loop=red.loop, return_exceptions=True) + sys.exit(exit_code) - red = Red( - cli_flags=cli_flags, description=description, dm_help=None, fetch_offline_members=True - ) - loop.run_until_complete(red._maybe_update_config()) - if cli_flags.edit: - try: - edit_instance(red, cli_flags) - except (KeyboardInterrupt, EOFError): - print("Aborted!") - finally: - loop.run_until_complete(driver_cls.teardown()) - sys.exit(0) +def exception_handler(red, loop, context): + msg = context.get("exception", context["message"]) + if isinstance(msg, KeyboardInterrupt): + # Windows support is ugly, I'm sorry + logging.error("Received KeyboardInterrupt, treating as interrupt") + signal_type = signal.SIGINT + else: + logging.critical("Caught fatal exception: %s", msg) + signal_type = None + loop.create_task(shutdown_handler(red, signal_type)) - init_global_checks(red) - init_events(red, cli_flags) - # lib folder has to be in sys.path before trying to load any 3rd-party cog (GH-3061) - # We might want to change handling of requirements in Downloader at later date - LIB_PATH = data_manager.cog_data_path(raw_name="Downloader") / "lib" - LIB_PATH.mkdir(parents=True, exist_ok=True) - if str(LIB_PATH) not in sys.path: - sys.path.append(str(LIB_PATH)) - sys.meta_path.insert(0, SharedLibImportWarner()) - - red.add_cog(Core(red)) - red.add_cog(CogManagerUI()) - red.add_command(license_info_command) - if cli_flags.dev: - red.add_cog(Dev()) - # noinspection PyProtectedMember - loop.run_until_complete(modlog._init(red)) - # noinspection PyProtectedMember - bank._init() - - if os.name == "posix": - loop.add_signal_handler(SIGTERM, lambda: asyncio.ensure_future(sigterm_handler(red, log))) - tmp_data = {} - loop.run_until_complete(_get_prefix_and_token(red, tmp_data)) - token = os.environ.get("RED_TOKEN", tmp_data["token"]) - if cli_flags.token: - token = cli_flags.token - prefix = cli_flags.prefix or tmp_data["prefix"] - if not (token and prefix): - if cli_flags.no_prompt is False: - new_token = interactive_config(red, token_set=bool(token), prefix_set=bool(prefix)) - if new_token: - token = new_token - else: - log.critical("Token and prefix must be set in order to login.") - sys.exit(1) - loop.run_until_complete(_get_prefix_and_token(red, tmp_data)) - - if cli_flags.dry_run: - loop.run_until_complete(red.http.close()) - sys.exit(0) +def main(): + cli_flags = parse_cli_flags(sys.argv[1:]) + handle_early_exit_flags(cli_flags) try: - loop.run_until_complete(red.start(token, bot=True, cli_flags=cli_flags)) - except discord.LoginFailure: - log.critical("This token doesn't seem to be valid.") - db_token = loop.run_until_complete(red._config.token()) - if db_token and not cli_flags.no_prompt: - if confirm("\nDo you want to reset the token?"): - loop.run_until_complete(red._config.token.set("")) - print("Token has been reset.") - except KeyboardInterrupt: - log.info("Keyboard interrupt detected. Quitting...") - loop.run_until_complete(red.logout()) - red._shutdown_mode = ExitCodes.SHUTDOWN - except Exception as e: - log.critical("Fatal exception", exc_info=e) - loop.run_until_complete(red.logout()) - finally: - pending = asyncio.Task.all_tasks(loop=red.loop) - gathered = asyncio.gather(*pending, loop=red.loop, return_exceptions=True) - gathered.cancel() - try: - loop.run_until_complete(red.rpc.close()) - except AttributeError: - pass + loop = asyncio.get_event_loop() - sys.exit(red._shutdown_mode.value) + if cli_flags.no_instance: + print( + "\033[1m" + "Warning: The data will be placed in a temporary folder and removed on next system " + "reboot." + "\033[0m" + ) + cli_flags.instance_name = "temporary_red" + data_manager.create_temp_config() + + data_manager.load_basic_configuration(cli_flags.instance_name) + + red = Red( + cli_flags=cli_flags, description=description, dm_help=None, fetch_offline_members=True + ) + + if os.name != "nt": + # None of this works on windows, and we have to catch KeyboardInterrupt in a global handler! + # At least it's not a redundant handler... + signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) + for s in signals: + loop.add_signal_handler( + s, lambda s=s: asyncio.create_task(shutdown_handler(red, s)) + ) + + exc_handler = functools.partial(exception_handler, red) + loop.set_exception_handler(exc_handler) + # We actually can't use asyncio.run and have graceful cleanup on Windows... + loop.create_task(run_bot(red, cli_flags)) + loop.run_forever() + finally: + loop.close() if __name__ == "__main__": + description = "Red V3" main() diff --git a/redbot/core/bank.py b/redbot/core/bank.py index a51e5dc50..4379199d8 100644 --- a/redbot/core/bank.py +++ b/redbot/core/bank.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import asyncio import datetime -from typing import Union, List, Optional +from typing import Union, List, Optional, TYPE_CHECKING from functools import wraps import discord @@ -8,9 +10,12 @@ import discord from redbot.core.utils.chat_formatting import humanize_number from . import Config, errors, commands from .i18n import Translator -from .bot import Red + from .errors import BankPruneError +if TYPE_CHECKING: + from .bot import Red + _ = Translator("Bank API", __file__) __all__ = [ diff --git a/redbot/core/bot.py b/redbot/core/bot.py index 858273a94..893723500 100644 --- a/redbot/core/bot.py +++ b/redbot/core/bot.py @@ -2,6 +2,7 @@ import asyncio import inspect import logging import os +import sys from collections import namedtuple from datetime import datetime from enum import Enum @@ -13,8 +14,12 @@ from types import MappingProxyType import discord from discord.ext.commands import when_mentioned_or -from . import Config, i18n, commands, errors, drivers -from .cog_manager import CogManager +from . import Config, i18n, commands, errors, drivers, modlog, bank +from .cog_manager import CogManager, CogManagerUI +from .core_commands import license_info_command, Core +from .dev_commands import Dev +from .events import init_events +from .global_checks import init_global_checks from .rpc import RPCMixin from .utils import common_filters @@ -43,6 +48,7 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin): # pylint: d def __init__(self, *args, cli_flags=None, bot_dir: Path = Path.cwd(), **kwargs): self._shutdown_mode = ExitCodes.CRITICAL + self._cli_flags = cli_flags self._config = Config.get_core_conf(force_registration=False) self._co_owners = cli_flags.co_owner self.rpc_enabled = cli_flags.rpc @@ -392,6 +398,18 @@ class RedBase(commands.GroupMixin, commands.bot.BotBase, RPCMixin): # pylint: d """ await self._maybe_update_config() + init_global_checks(self) + init_events(self, cli_flags) + + self.add_cog(Core(self)) + self.add_cog(CogManagerUI()) + self.add_command(license_info_command) + if cli_flags.dev: + self.add_cog(Dev()) + + await modlog._init(self) + bank._init() + packages = [] if cli_flags.no_cogs is False: @@ -971,6 +989,10 @@ class Red(RedBase, discord.AutoShardedClient): """Logs out of Discord and closes all connections.""" await super().logout() await drivers.get_driver_class().teardown() + try: + await self.rpc.close() + except AttributeError: + pass async def shutdown(self, *, restart: bool = False): """Gracefully quit Red. @@ -990,6 +1012,7 @@ class Red(RedBase, discord.AutoShardedClient): self._shutdown_mode = ExitCodes.RESTART await self.logout() + sys.exit(self._shutdown_mode) class ExitCodes(Enum): diff --git a/redbot/core/cli.py b/redbot/core/cli.py index 74ce9da89..4778bb642 100644 --- a/redbot/core/cli.py +++ b/redbot/core/cli.py @@ -33,9 +33,8 @@ def confirm(text: str, default: Optional[bool] = None) -> bool: print("Error: invalid input") -def interactive_config(red, token_set, prefix_set, *, print_header=True): - loop = asyncio.get_event_loop() - token = "" +async def interactive_config(red, token_set, prefix_set, *, print_header=True): + token = None if print_header: print("Red - Discord Bot | Configuration process\n") @@ -51,9 +50,9 @@ def interactive_config(red, token_set, prefix_set, *, print_header=True): token = input("> ") if not len(token) >= 50: print("That doesn't look like a valid token.") - token = "" + token = None if token: - loop.run_until_complete(red._config.token.set(token)) + await red._config.token.set(token) if not prefix_set: prefix = "" @@ -70,7 +69,7 @@ def interactive_config(red, token_set, prefix_set, *, print_header=True): if not confirm("Your prefix seems overly long. Are you sure that it's correct?"): prefix = "" if prefix: - loop.run_until_complete(red._config.prefix.set([prefix])) + await red._config.prefix.set([prefix]) return token diff --git a/redbot/core/events.py b/redbot/core/events.py index 5fd418d36..ec48f26c0 100644 --- a/redbot/core/events.py +++ b/redbot/core/events.py @@ -4,7 +4,6 @@ import codecs import datetime import logging import traceback -import asyncio from datetime import timedelta import aiohttp @@ -17,7 +16,6 @@ from redbot.core.commands import RedHelpFormatter from .. import __version__ as red_version, version_info as red_version_info, VersionInfo from . import commands from .config import get_latest_confs -from .data_manager import storage_type from .utils.chat_formatting import inline, bordered, format_perms_list, humanize_timedelta from .utils import fuzzy_command_search, format_fuzzy_results diff --git a/redbot/core/modlog.py b/redbot/core/modlog.py index 7ad4c786a..6fb8ba782 100644 --- a/redbot/core/modlog.py +++ b/redbot/core/modlog.py @@ -1,11 +1,12 @@ +from __future__ import annotations + import asyncio from datetime import datetime, timedelta -from typing import List, Union, Optional, cast +from typing import List, Union, Optional, cast, TYPE_CHECKING import discord from redbot.core import Config -from redbot.core.bot import Red from .utils.common_filters import ( filter_invites, @@ -17,6 +18,9 @@ from .i18n import Translator from .generic_casetypes import all_generics +if TYPE_CHECKING: + from redbot.core.bot import Red + __all__ = [ "Case", "CaseType", diff --git a/setup.cfg b/setup.cfg index 80be6282c..2ecf6d75a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,13 +18,13 @@ classifiers = License :: OSI Approved :: GNU General Public License v3 (GPLv3) Natural Language :: English Operating System :: OS Independent - Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Topic :: Communications :: Chat Topic :: Documentation :: Sphinx [options] packages = find_namespace: -python_requires = >=3.7 +python_requires = >=3.8.1 install_requires = aiohttp==3.5.4 aiohttp-json-rpc==0.12.1 @@ -87,9 +87,8 @@ style = black==19.3b0 toml==0.10.0 test = - astroid==2.2.5 + astroid==2.3.3 atomicwrites==1.3.0 - importlib-metadata==0.19 isort==4.3.21 lazy-object-proxy==1.4.2 mccabe==0.6.1 @@ -99,9 +98,9 @@ test = py==1.8.0 pylint==2.3.1 pyparsing==2.4.2 - pytest==5.1.2 + pytest==5.3.2 pytest-asyncio==0.10.0 - pytest-mock==1.11.2 + pytest-mock==1.13.0 six==1.12.0 typed-ast==1.4.0 wcwidth==0.1.7 diff --git a/tests/cogs/downloader/test_downloader.py b/tests/cogs/downloader/test_downloader.py index 29d49a7e7..eebff2bb3 100644 --- a/tests/cogs/downloader/test_downloader.py +++ b/tests/cogs/downloader/test_downloader.py @@ -25,18 +25,11 @@ class FakeCompletedProcess(NamedTuple): stderr: bytes = b"" -async def async_return(ret: Any): - return ret - - def _mock_run( mocker: MockFixture, repo: Repo, returncode: int, stdout: bytes = b"", stderr: bytes = b"" ): return mocker.patch.object( - repo, - "_run", - autospec=True, - return_value=async_return(FakeCompletedProcess(returncode, stdout, stderr)), + repo, "_run", autospec=True, return_value=FakeCompletedProcess(returncode, stdout, stderr) ) @@ -46,11 +39,7 @@ def _mock_setup_repo(mocker: MockFixture, repo: Repo, commit: str): return mocker.DEFAULT return mocker.patch.object( - repo, - "_setup_repo", - autospec=True, - side_effect=update_commit, - return_value=async_return(None), + repo, "_setup_repo", autospec=True, side_effect=update_commit, return_value=None ) @@ -153,15 +142,13 @@ async def test_is_module_modified(mocker, repo): repo, "_get_file_update_statuses", autospec=True, - return_value=async_return( - { - "added_file.txt": "A", - "mycog/__init__.py": "M", - "sample_file1.txt": "D", - "sample_file2.txt": "D", - "sample_file3.txt": "A", - } - ), + return_value={ + "added_file.txt": "A", + "mycog/__init__.py": "M", + "sample_file1.txt": "D", + "sample_file2.txt": "D", + "sample_file3.txt": "A", + }, ) ret = await repo._is_module_modified(module, old_rev) m.assert_called_once_with(old_rev, new_rev) @@ -249,11 +236,11 @@ async def test_checkout(mocker, repo): @pytest.mark.asyncio async def test_checkout_ctx_manager(mocker, repo): commit = "c950fc05a540dd76b944719c2a3302da2e2f3090" - m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=async_return(None)) + m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=None) old_commit = repo.commit async with repo.checkout(commit): m.assert_called_with(commit, force_checkout=False) - m.return_value = async_return(None) + m.return_value = None m.assert_called_with(old_commit, force_checkout=False) @@ -261,7 +248,7 @@ async def test_checkout_ctx_manager(mocker, repo): @pytest.mark.asyncio async def test_checkout_await(mocker, repo): commit = "c950fc05a540dd76b944719c2a3302da2e2f3090" - m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=async_return(None)) + m = mocker.patch.object(repo, "_checkout", autospec=True, return_value=None) await repo.checkout(commit) m.assert_called_once_with(commit, force_checkout=False) @@ -293,7 +280,7 @@ async def test_clone_without_branch(mocker, repo): repo.commit = "" m = _mock_run(mocker, repo, 0) _mock_setup_repo(mocker, repo, commit) - mocker.patch.object(repo, "current_branch", autospec=True, return_value=async_return(branch)) + mocker.patch.object(repo, "current_branch", autospec=True, return_value=branch) await repo.clone() @@ -309,10 +296,8 @@ async def test_update(mocker, repo): new_commit = "a0ccc2390883c85a361f5a90c72e1b07958939fa" m = _mock_run(mocker, repo, 0) _mock_setup_repo(mocker, repo, new_commit) - mocker.patch.object( - repo, "latest_commit", autospec=True, return_value=async_return(old_commit) - ) - mocker.patch.object(repo, "hard_reset", autospec=True, return_value=async_return(None)) + mocker.patch.object(repo, "latest_commit", autospec=True, return_value=old_commit) + mocker.patch.object(repo, "hard_reset", autospec=True, return_value=None) ret = await repo.update() assert ret == (old_commit, new_commit) diff --git a/tools/bumpdeps.py b/tools/bumpdeps.py index 82df1da0c..b6b38d0d0 100755 --- a/tools/bumpdeps.py +++ b/tools/bumpdeps.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.7 +#!/usr/bin/env python3.8 """Script to bump pinned dependencies in setup.cfg. This script aims to help update our list of pinned primary and diff --git a/tools/edit_testrepo.py b/tools/edit_testrepo.py index 1befef09f..93a4b29db 100644 --- a/tools/edit_testrepo.py +++ b/tools/edit_testrepo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.7 +#!/usr/bin/env python3.8 """Script to edit test repo used by Downloader git integration tests. This script aims to help update the human-readable version of repo diff --git a/tox.ini b/tox.ini index 1c079ae58..e780f0512 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] envlist = - py37 + py38 docs style skip_missing_interpreters = True @@ -19,7 +19,7 @@ extras = voice, test commands = python -m compileall ./redbot/cogs pytest - pylint ./redbot +# pylint ./redbot [testenv:postgres] description = Run pytest with PostgreSQL backend @@ -48,7 +48,7 @@ setenv = # This is just for Windows # Prioritise make.bat over any make.exe which might be on PATH PATHEXT=.BAT;.EXE -basepython = python3.7 +basepython = python3.8 extras = docs commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out/html" -W -bhtml @@ -64,7 +64,7 @@ setenv = # This is just for Windows # Prioritise make.bat over any make.exe which might be on PATH PATHEXT=.BAT;.EXE -basepython = python3.7 +basepython = python3.8 extras = style commands = make stylecheck