[V3 Audio] Harden Lavalink boot sequence (#1498)

* Do a bit of hardening

* Loop not asyncio

* Don't use new asyncio coolness

* I hate you all

* Muck up everything

* Fix version comparisons

docs/conf.py

@@ -62,10 +62,11 @@ author = 'Cog Creators'
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
+from redbot.core import __version__
 # The short X.Y version.
-version = '3.0.0b11'
+version = __version__
 # The full version, including alpha/beta/rc tags.
-release = '3.0.0b11'
+release = __version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

redbot/cogs/audio/__init__.py

@@ -1,18 +1,17 @@
 from pathlib import Path
 from aiohttp import ClientSession
 import shutil
-import asyncio
 
 from .audio import Audio
 from .manager import start_lavalink_server
 from discord.ext import commands
 from redbot.core.data_manager import cog_data_path
+import redbot.core
 
-LAVALINK_BUILD = 3112
+LAVALINK_DOWNLOAD_URL = (
-LAVALINK_BUILD_URL = (
+    "https://github.com/Cog-Creators/Red-DiscordBot/"
-    "https://ci.fredboat.com/repository/download/"
+    "releases/download/{}/Lavalink.jar"
-    "Lavalink_Build/{}:id/Lavalink.jar?guest=1"
+).format(redbot.core.__version__)
-).format(LAVALINK_BUILD)
 
 LAVALINK_DOWNLOAD_DIR = cog_data_path(raw_name="Audio")
 LAVALINK_JAR_FILE = LAVALINK_DOWNLOAD_DIR / "Lavalink.jar"
@@ -23,7 +22,7 @@ BUNDLED_APP_YML_FILE = Path(__file__).parent / "application.yml"
 
 async def download_lavalink(session):
     with LAVALINK_JAR_FILE.open(mode='wb') as f:
-        async with session.get(LAVALINK_BUILD_URL) as resp:
+        async with session.get(LAVALINK_DOWNLOAD_URL) as resp:
             while True:
                 chunk = await resp.content.read(512)
                 if not chunk:
@@ -33,24 +32,25 @@ async def download_lavalink(session):
 
 async def maybe_download_lavalink(loop, cog):
     jar_exists = LAVALINK_JAR_FILE.exists()
-    current_build = await cog.config.current_build()
+    current_build = redbot.core.VersionInfo(*await cog.config.current_build())
 
-    if not jar_exists or current_build < LAVALINK_BUILD:
+    session = ClientSession(loop=loop)
+
+    if not jar_exists or current_build < redbot.core.version_info:
         LAVALINK_DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)
-        with ClientSession(loop=loop) as session:
+        await download_lavalink(session)
-            await download_lavalink(session)
+        await cog.config.current_build.set(redbot.core.version_info.to_json())
-        await cog.config.current_build.set(LAVALINK_BUILD)
+
+    session.close()
 
     shutil.copyfile(str(BUNDLED_APP_YML_FILE), str(APP_YML_FILE))
 
 
 async def setup(bot: commands.Bot):
     cog = Audio(bot)
-    await maybe_download_lavalink(bot.loop, cog)
+    if not await cog.config.use_external_lavalink():
-    await start_lavalink_server(bot.loop)
+        await maybe_download_lavalink(bot.loop, cog)
+        await start_lavalink_server(bot.loop)
 
-    async def _finish():
+    bot.add_cog(cog)
-        await cog.init_config()
+    bot.loop.create_task(cog.init_config())
-        bot.add_cog(cog)
-
-    bot.loop.create_task(_finish())

redbot/cogs/audio/audio.py

@@ -16,7 +16,7 @@ __author__ = ["aikaterna", "billy/bollo/ati"]
 class Audio:
     def __init__(self, bot):
         self.bot = bot
-        self.config = Config.get_conf(self, 2711759128, force_registration=True)
+        self.config = Config.get_conf(self, 2711759130, force_registration=True)
 
         default_global = {
             "host": 'localhost',
@@ -24,7 +24,8 @@ class Audio:
             "ws_port": '2332',
             "password": 'youshallnotpass',
             "status": False,
-            "current_build": 0
+            "current_build": [3, 0, 0, 'alpha', 0],
+            "use_external_lavalink": False
         }
 
         default_guild = {

redbot/cogs/audio/manager.py

@@ -1,12 +1,22 @@
 import shlex
+import shutil
 import asyncio
-from subprocess import Popen, DEVNULL
+from subprocess import Popen, DEVNULL, PIPE
 import os
+import logging
+
+log = logging.getLogger('red.audio.manager')
 
 proc = None
 SHUTDOWN = asyncio.Event()
 
 
+def has_java_error(pid):
+    from . import LAVALINK_DOWNLOAD_DIR
+    poss_error_file = LAVALINK_DOWNLOAD_DIR / 'hs_err_pid{}.log'.format(pid)
+    return poss_error_file.exists()
+
+
 async def monitor_lavalink_server(loop):
     while not SHUTDOWN.is_set():
         if proc.poll() is not None:
@@ -14,13 +24,55 @@ async def monitor_lavalink_server(loop):
         await asyncio.sleep(0.5)
 
     if not SHUTDOWN.is_set():
-        print("Lavalink jar shutdown, restarting.")
+        log.info("Lavalink jar shutdown.")
-        await start_lavalink_server(loop)
+        if not has_java_error(proc.pid):
+            log.info("Restarting Lavalink jar.")
+            await start_lavalink_server(loop)
+        else:
+            log.error("Your Java is borked. Please find the hs_err_pid{}.log file"
+                      " in the Audio data folder and report this issue.".format(
+                proc.pid
+            ))
 
 
+async def has_java(loop):
+    java_available = shutil.which('java') is not None
+    if not java_available:
+        return False
+
+    version = await get_java_version(loop)
+    return version >= (1, 8), version
+
+
+async def get_java_version(loop):
+    """
+    This assumes we've already checked that java exists.
+    """
+    proc = Popen(
+        shlex.split("java -version", posix=os.name == 'posix'),
+        stdout=PIPE, stderr=PIPE
+    )
+    _, err = proc.communicate()
+
+    version_info = str(err, encoding='utf-8')
+
+    version_line = version_info.split('\n')[0]
+    version_start = version_line.find('"')
+    version_string = version_line[version_start + 1:-1]
+    major, minor = version_string.split('.')[:2]
+    return int(major), int(minor)
+
 async def start_lavalink_server(loop):
+    java_available, java_version = await has_java(loop)
+    if not java_available:
+        raise RuntimeError("You must install Java 1.8+ for Lavalink to run.")
+
+    extra_flags = ""
+    if java_version == (1, 8):
+        extra_flags = "-Dsun.zip.disableMemoryMapping=true"
+
     from . import LAVALINK_DOWNLOAD_DIR, LAVALINK_JAR_FILE
-    start_cmd = "java -jar {}".format(LAVALINK_JAR_FILE.resolve())
+    start_cmd = "java {} -jar {}".format(extra_flags, LAVALINK_JAR_FILE.resolve())
 
     global proc
     proc = Popen(
@@ -29,13 +81,15 @@ async def start_lavalink_server(loop):
         stdout=DEVNULL, stderr=DEVNULL
     )
 
-    print("Lavalink jar started. PID: {}".format(proc.pid))
+    log.info("Lavalink jar started. PID: {}".format(proc.pid))
 
     loop.create_task(monitor_lavalink_server(loop))
 
 
 def shutdown_lavalink_server():
-    print("Shutting down lavalink server.")
+    log.info("Shutting down lavalink server.")
     SHUTDOWN.set()
+    global proc
     if proc is not None:
         proc.terminate()
+        proc = None

redbot/core/__init__.py

@@ -1,11 +1,37 @@
-import pkg_resources
-
 from .config import Config
 from .context import RedContext
 
 __all__ = ["Config", "RedContext", "__version__"]
 
-try:
+
-    __version__ = pkg_resources.get_distribution("Red-DiscordBot").version
+class VersionInfo:
-except pkg_resources.DistributionNotFound:
+    def __init__(self, major, minor, micro, releaselevel, serial):
-    __version__ = "3.0.0"
+        self._levels = ['alpha', 'beta', 'final']
+        self.major = major
+        self.minor = minor
+        self.micro = micro
+
+        if releaselevel not in self._levels:
+            raise TypeError("'releaselevel' must be one of: {}".format(
+                ', '.join(self._levels)
+            ))
+
+        self.releaselevel = releaselevel
+        self.serial = serial
+
+    def __lt__(self, other):
+        my_index = self._levels.index(self.releaselevel)
+        other_index = self._levels.index(other.releaselevel)
+        return (self.major, self.minor, self.micro, my_index, self.serial) < \
+               (other.major, other.minor, other.micro, other_index, other.serial)
+
+    def __repr__(self):
+        return "VersionInfo(major={}, minor={}, micro={}, releaselevel={}, serial={})".format(
+            self.major, self.minor, self.micro, self.releaselevel, self.serial
+        )
+
+    def to_json(self):
+        return [self.major, self.minor, self.micro, self.releaselevel, self.serial]
+
+__version__ = "3.0.0b11"
+version_info = VersionInfo(3, 0, 0, 'beta', 11)

setup.py

@@ -1,6 +1,6 @@
 from distutils.core import setup
 from pathlib import Path
-from subprocess import run, PIPE
+import re
 
 import os
 import sys
@@ -37,29 +37,9 @@ def get_requirements():
 
 
 def get_version():
-    try:
+    with open('redbot/core/__init__.py') as f:
-        p = run(
+        version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', f.read(), re.MULTILINE).group(1)
-            "git describe --abbrev=0 --tags".split(),
+    return version
-            stdout=PIPE
-        )
-    except FileNotFoundError:
-        # No git
-        return 3, 0, 0
-
-    if p.returncode != 0:
-        return 3, 0, 0
-
-    stdout = p.stdout.strip().decode()
-    if stdout.startswith("v"):
-        numbers = stdout[1:].split('.')
-        args = [0, 0, 0]
-        for i in range(3):
-            try:
-                args[i] = int(numbers[i])
-            except (IndexError, ValueError):
-                args[i] = 0
-        return args
-    return 3, 0, 0
 
 
 def find_locale_folders():
@@ -94,7 +74,7 @@ def find_locale_folders():
 
 setup(
     name='Red-DiscordBot',
-    version="{}.{}.{}b11".format(*get_version()),
+    version=get_version(),
     packages=get_package_list(),
     package_data=find_locale_folders(),
     include_package_data=True,