Logging enhancements and cleanup (#2502)

* Logging enhancements and cleanup

- Removed debug log messages every time `Config.get_conf` is used or a JSON file is read/saved. The basic configuration is now logged once with DEBUG when the bot starts up instead.
- Changed logging output format to reverse date order, include seconds, and use the logger's name instead of the module, function and line number.
- Log files are now kept in the `DATAPATH/core/logs` directory. Each time Red is restarted, a new log is created, and the old ones renamed in a rotating fashion. There can be a maximum of 9 logs in total.
- Each log file now has a smaller max size of 500KB before it will be split into multiple parts. There are also a maximum of 9 parts of each log.
- Discord.py logger now uses the same output formatter as red's loggers
- Moved logging setup code into `redbot.logging` module.

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Reformat

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Implement discussed changes

- We maintain a red.log over multiple runtimes, alongside a latest.log and previous.log for individual runtimes.
- Naming convention changed a bit. E.g. when latest.log is just one part, it will be named latest.log. When it becomes two parts, they will both be named latest-part1.log and latest-part2.log.
- Rotation direction is reversed. This means as the files end up being named in chronological order.

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

redbot/__main__.py

@@ -2,29 +2,31 @@
 
 # Discord Version check
 
+import asyncio
+import logging
+import os
 import sys
+
 import discord
+
+import redbot.logging
 from redbot.core.bot import Red, ExitCodes
 from redbot.core.cog_manager import CogManagerUI
-from redbot.core.data_manager import create_temp_config, load_basic_configuration, config_file
 from redbot.core.json_io import JsonIO
 from redbot.core.global_checks import init_global_checks
 from redbot.core.events import init_events
 from redbot.core.cli import interactive_config, confirm, parse_cli_flags
 from redbot.core.core_commands import Core
 from redbot.core.dev_commands import Dev
-from redbot.core import modlog, bank
+from redbot.core import modlog, bank, data_manager
 from signal import SIGTERM
-import asyncio
-import logging.handlers
-import logging
-import os
 
 # Let's not force this dependency, uvloop is much faster on cpython
 if sys.implementation.name == "cpython":
     try:
         import uvloop
     except ImportError:
+        uvloop = None
         pass
     else:
         asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
@@ -32,6 +34,7 @@ if sys.implementation.name == "cpython":
 if sys.platform == "win32":
     asyncio.set_event_loop(asyncio.ProactorEventLoop())
 
+log = logging.getLogger("red.main")
 
 #
 #               Red - Discord Bot v3
@@ -40,46 +43,6 @@ if sys.platform == "win32":
 #
 
 
-def init_loggers(cli_flags):
-    # d.py stuff
-    dpy_logger = logging.getLogger("discord")
-    dpy_logger.setLevel(logging.WARNING)
-    console = logging.StreamHandler()
-    console.setLevel(logging.WARNING)
-    dpy_logger.addHandler(console)
-
-    # Red stuff
-
-    logger = logging.getLogger("red")
-
-    red_format = logging.Formatter(
-        "%(asctime)s %(levelname)s %(module)s %(funcName)s %(lineno)d: %(message)s",
-        datefmt="[%d/%m/%Y %H:%M]",
-    )
-
-    stdout_handler = logging.StreamHandler(sys.stdout)
-    stdout_handler.setFormatter(red_format)
-
-    if cli_flags.debug:
-        os.environ["PYTHONASYNCIODEBUG"] = "1"
-        logger.setLevel(logging.DEBUG)
-    else:
-        logger.setLevel(logging.INFO)
-
-    from redbot.core.data_manager import core_data_path
-
-    logfile_path = core_data_path() / "red.log"
-    fhandler = logging.handlers.RotatingFileHandler(
-        filename=str(logfile_path), encoding="utf-8", mode="a", maxBytes=10 ** 7, backupCount=5
-    )
-    fhandler.setFormatter(red_format)
-
-    logger.addHandler(fhandler)
-    logger.addHandler(stdout_handler)
-
-    return logger
-
-
 async def _get_prefix_and_token(red, indict):
     """
     Again, please blame <@269933075037814786> for this.
@@ -91,14 +54,14 @@ async def _get_prefix_and_token(red, indict):
 
 
 def list_instances():
-    if not config_file.exists():
+    if not data_manager.config_file.exists():
         print(
             "No instances have been configured! Configure one "
             "using `redbot-setup` before trying to run the bot!"
         )
         sys.exit(1)
     else:
-        data = JsonIO(config_file)._load_json()
+        data = JsonIO(data_manager.config_file)._load_json()
         text = "Configured Instances:\n\n"
         for instance_name in sorted(data.keys()):
             text += "{}\n".format(instance_name)
@@ -125,13 +88,21 @@ def main():
     if cli_flags.no_instance:
         print(
             "\033[1m"
-            "Warning: The data will be placed in a temporary folder and removed on next system reboot."
+            "Warning: The data will be placed in a temporary folder and removed on next system "
+            "reboot."
             "\033[0m"
         )
         cli_flags.instance_name = "temporary_red"
-        create_temp_config()
-    load_basic_configuration(cli_flags.instance_name)
-    log = init_loggers(cli_flags)
+        data_manager.create_temp_config()
+    data_manager.load_basic_configuration(cli_flags.instance_name)
+    redbot.logging.init_logging(
+        level=cli_flags.logging_level, location=data_manager.core_data_path() / "logs"
+    )
+
+    log.debug("====Basic Config====")
+    log.debug("Data Path: %s", data_manager._base_data_path())
+    log.debug("Storage Type: %s", data_manager.storage_type())
+
     red = Red(cli_flags=cli_flags, description=description, pm_help=None)
     init_global_checks(red)
     init_events(red, cli_flags)