I promise I'm not doing this on purpose (#4565)

* Prep for 0.7.2

* So What di i do here? I done Magic, magic only found in the tales of old.

* turns out formatting is something important

* fixes

* improved Error handling when Global API is enabled

* further improve resuming logic

* more of dat dark voodoo blood magic

* major tweaks to auto restore when auto play is enabled 👀

* fix duplicated "Auto play stated." message + Auto play restart :feelsgoodman:

* missed these

* fix the new duplicated fucking message bitch.

* Let discord handle player reconnects

* eh

* `Requires force install`, utilize new Exponential Backoff object on player and safer reconnect logic, emulating d.py and WL.

* hmmmmm gotta monitor

* mother fucking brackets

* Why didnt i consider this the first time?????????????

* new error code to handle?

* soooooooooooooooo these are import so why arent we ensuring they are set.

* improved logging

* improved logging

* aaaaaaaaaaaaaaa

* We need our own error and special handling to not conflict with dpy

* (Last Known Bug) Fix the infinite loop of 4006 that sometimes could happen as an edge case after a successful resume.

* This will require a force reinstall to install `RLL 0.8.0`, this properly fixes the bug mentioned on the previous commit.

* address "Localtrack names/paths need to be escaped." comment

* address Fixators crash mentioned in #AT

* style

* fix preda's crash mentioned in PR

* add a thing here add a thing there add a thing everywhere

* style

* fixes here, fixes there, and backbone for curated playlist.

* bypass aiohttp and githubs and cloudflare and yo mammas cache

* I propose the new style is no style.

* allow curated playlist to be updated it `[p]playlist update` and show the diff

* fix `[p]summon` not resuming playback until next track.

* Hopefully handle predas rate limits.

* what else did i break now

* Update Lavalink.jar build

* lets try this

* reset the queue

* Bring Edge commits over fix a bunch of shiz again

* Bring Edge commits over fix a bunch of shiz again

* Handle 4014 OPs, Change `skip_votes` key to be an int rather than guild object

* aaaaaaaaaaaaaaa im dumb

* ...

* Simplify some shiz + use a set instead of a list for votes.

Co-authored-by: aikaterna <20862007+aikaterna@users.noreply.github.com>

redbot/cogs/audio/core/events/cog.py

@@ -32,8 +32,10 @@ class AudioEvents(MixinMeta, metaclass=CompositeMetaClass):
 
         if await self.bot.cog_disabled_in_guild(self, guild):
             player = lavalink.get_player(guild.id)
+            player.store("autoplay_notified", False)
             await player.stop()
             await player.disconnect()
+            await self.config.guild_from_id(guild_id=guild.id).currently_auto_playing_in.set([])
             return
 
         track_identifier = track.track_identifier
@@ -157,7 +159,9 @@ class AudioEvents(MixinMeta, metaclass=CompositeMetaClass):
             await self.api_interface.persistent_queue_api.delete_scheduled()
 
     @commands.Cog.listener()
-    async def on_red_audio_track_enqueue(self, guild: discord.Guild, track, requester):
+    async def on_red_audio_track_enqueue(
+        self, guild: discord.Guild, track: lavalink.Track, requester: discord.Member
+    ):
         if not (track and guild):
             return
         persist_cache = self._persist_queue_cache.setdefault(
@@ -181,3 +185,28 @@ class AudioEvents(MixinMeta, metaclass=CompositeMetaClass):
             await self.api_interface.persistent_queue_api.drop(guild.id)
             await asyncio.sleep(5)
             await self.api_interface.persistent_queue_api.delete_scheduled()
+
+    @commands.Cog.listener()
+    async def on_red_audio_track_auto_play(
+        self,
+        guild: discord.Guild,
+        track: lavalink.Track,
+        requester: discord.Member,
+        player: lavalink.Player,
+    ):
+        notify_channel = self.bot.get_channel(player.fetch("channel"))
+        tries = 0
+        while not player._is_playing:
+            await asyncio.sleep(0.1)
+            if tries > 1000:
+                return
+
+        if notify_channel and not player.fetch("autoplay_notified", False):
+            if (
+                len(player.manager.players) < 10
+                or not player._last_resume
+                and player._last_resume + datetime.timedelta(seconds=60)
+                > datetime.datetime.now(tz=datetime.timezone.utc)
+            ):
+                await self.send_embed_msg(notify_channel, title=_("Auto Play started."))
+            player.store("autoplay_notified", True)

redbot/cogs/audio/core/events/dpy.py

@@ -244,7 +244,7 @@ class DpyEvents(MixinMeta, metaclass=CompositeMetaClass):
 
             lavalink.unregister_event_listener(self.lavalink_event_handler)
             lavalink.unregister_update_listener(self.lavalink_update_handler)
-            self.bot.loop.create_task(lavalink.close())
+            self.bot.loop.create_task(lavalink.close(self.bot))
             if self.player_manager is not None:
                 self.bot.loop.create_task(self.player_manager.shutdown())
 
@@ -259,12 +259,31 @@ class DpyEvents(MixinMeta, metaclass=CompositeMetaClass):
         await self.cog_ready_event.wait()
         if after.channel != before.channel:
             try:
-                self.skip_votes[before.channel.guild.id].remove(member.id)
+                self.skip_votes[before.channel.guild.id].discard(member.id)
             except (ValueError, KeyError, AttributeError):
                 pass
+
+        # if (
+        #     member == member.guild.me
+        #     and before.channel
+        #     and after.channel
+        #     and after.channel.id != before.channel.id
+        # ):
+        #     try:
+        #         player = lavalink.get_player(member.guild.id)
+        #         if player.is_playing:
+        #             await player.resume(player.current, start=player.position, replace=False)
+        #             log.debug("Bot changed channel - Resume playback")
+        #     except:
+        #         log.debug("Bot changed channel - Unable to resume playback")
+
         channel = self.rgetattr(member, "voice.channel", None)
         bot_voice_state = self.rgetattr(member, "guild.me.voice.self_deaf", None)
-        if channel and bot_voice_state is False:
+        if (
+            channel
+            and bot_voice_state is False
+            and await self.config.guild(member.guild).auto_deafen()
+        ):
             try:
                 player = lavalink.get_player(channel.guild.id)
             except (KeyError, AttributeError):
@@ -272,3 +291,15 @@ class DpyEvents(MixinMeta, metaclass=CompositeMetaClass):
             else:
                 if player.channel.id == channel.id:
                     await self.self_deafen(player)
+
+    @commands.Cog.listener()
+    async def on_shard_disconnect(self, shard_id):
+        self._diconnected_shard.add(shard_id)
+
+    @commands.Cog.listener()
+    async def on_shard_ready(self, shard_id):
+        self._diconnected_shard.discard(shard_id)
+
+    @commands.Cog.listener()
+    async def on_shard_resumed(self, shard_id):
+        self._diconnected_shard.discard(shard_id)

redbot/cogs/audio/core/events/lavalink.py

@@ -3,9 +3,12 @@ import contextlib
 import datetime
 import logging
 from pathlib import Path
+from typing import Dict
 
 import discord
 import lavalink
+from discord.backoff import ExponentialBackoff
+from discord.gateway import DiscordWebSocket
 
 from redbot.core.i18n import Translator, set_contextual_locales_from_guild
 from ...errors import DatabaseError, TrackEnqueueError
@@ -13,6 +16,9 @@ from ..abc import MixinMeta
 from ..cog_utils import CompositeMetaClass
 
 log = logging.getLogger("red.cogs.Audio.cog.Events.lavalink")
+ws_audio_log = logging.getLogger("red.Audio.WS.Audio")
+ws_audio_log.setLevel(logging.WARNING)
+
 _ = Translator("Audio", Path(__file__))
 
 
@@ -28,27 +34,62 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
     ) -> None:
         current_track = player.current
         current_channel = player.channel
-        if not current_channel:
-            return
         guild = self.rgetattr(current_channel, "guild", None)
+        if not (current_channel and guild):
+            player.store("autoplay_notified", False)
+            await player.stop()
+            await player.disconnect()
+            return
         if await self.bot.cog_disabled_in_guild(self, guild):
             await player.stop()
             await player.disconnect()
+            if guild:
+                await self.config.guild_from_id(guild_id=guild.id).currently_auto_playing_in.set(
+                    []
+                )
             return
         guild_id = self.rgetattr(guild, "id", None)
         if not guild:
             return
+        guild_data = await self.config.guild(guild).all()
+        disconnect = guild_data["disconnect"]
+        if event_type == lavalink.LavalinkEvents.FORCED_DISCONNECT:
+            self.bot.dispatch("red_audio_audio_disconnect", guild)
+            await self.config.guild_from_id(guild_id=guild_id).currently_auto_playing_in.set([])
+            self._ll_guild_updates.discard(guild.id)
+            return
+
+        if event_type == lavalink.LavalinkEvents.WEBSOCKET_CLOSED:
+            deafen = guild_data["auto_deafen"]
+            event_channel_id = extra.get("channelID")
+            _error_code = extra.get("code")
+            if _error_code in [1000] or not guild:
+                if _error_code == 1000:
+                    await player.resume(player.current, start=player.position, replace=False)
+                return
+            await self._ws_op_codes[guild_id].put((event_channel_id, _error_code))
+            try:
+                if guild_id not in self._ws_resume:
+                    self._ws_resume[guild_id].set()
+
+                await self._websocket_closed_handler(
+                    guild=guild, player=player, extra=extra, deafen=deafen, disconnect=disconnect
+                )
+            except Exception:
+                log.exception(
+                    f"Error in WEBSOCKET_CLOSED handling for guild: {player.channel.guild.id}"
+                )
+            return
+
         await set_contextual_locales_from_guild(self.bot, guild)
         current_requester = self.rgetattr(current_track, "requester", None)
         current_stream = self.rgetattr(current_track, "is_stream", None)
         current_length = self.rgetattr(current_track, "length", None)
         current_thumbnail = self.rgetattr(current_track, "thumbnail", None)
-        current_extras = self.rgetattr(current_track, "extras", {})
         current_id = self.rgetattr(current_track, "_info", {}).get("identifier")
-        guild_data = await self.config.guild(guild).all()
+
         repeat = guild_data["repeat"]
         notify = guild_data["notify"]
-        disconnect = guild_data["disconnect"]
         autoplay = guild_data["auto_play"]
         description = await self.get_track_description(
             current_track, self.local_folder_current_path
@@ -59,7 +100,7 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
         await self.maybe_reset_error_counter(player)
 
         if event_type == lavalink.LavalinkEvents.TRACK_START:
-            self.skip_votes[guild_id] = []
+            self.skip_votes[guild_id] = set()
             playing_song = player.fetch("playing_song")
             requester = player.fetch("requester")
             player.store("prev_song", playing_song)
@@ -71,25 +112,35 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
                 await self.api_interface.persistent_queue_api.played(
                     guild_id=guild_id, track_id=current_track.track_identifier
                 )
+            notify_channel = player.fetch("channel")
+            if notify_channel and autoplay:
+                await self.config.guild_from_id(guild_id=guild_id).currently_auto_playing_in.set(
+                    [notify_channel, player.channel.id]
+                )
+            else:
+                await self.config.guild_from_id(guild_id=guild_id).currently_auto_playing_in.set(
+                    []
+                )
         if event_type == lavalink.LavalinkEvents.TRACK_END:
             prev_requester = player.fetch("prev_requester")
             self.bot.dispatch("red_audio_track_end", guild, prev_song, prev_requester)
+            player.store("resume_attempts", 0)
         if event_type == lavalink.LavalinkEvents.QUEUE_END:
             prev_requester = player.fetch("prev_requester")
             self.bot.dispatch("red_audio_queue_end", guild, prev_song, prev_requester)
             if guild_id:
                 await self.api_interface.persistent_queue_api.drop(guild_id)
-            if (
+            if player.is_auto_playing or (
                 autoplay
                 and not player.queue
                 and player.fetch("playing_song") is not None
                 and self.playlist_api is not None
                 and self.api_interface is not None
             ):
+                notify_channel = player.fetch("channel")
                 try:
                     await self.api_interface.autoplay(player, self.playlist_api)
                 except DatabaseError:
-                    notify_channel = player.fetch("channel")
                     notify_channel = self.bot.get_channel(notify_channel)
                     if notify_channel:
                         await self.send_embed_msg(
@@ -97,7 +148,6 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
                         )
                     return
                 except TrackEnqueueError:
-                    notify_channel = player.fetch("channel")
                     notify_channel = self.bot.get_channel(notify_channel)
                     if notify_channel:
                         await self.send_embed_msg(
@@ -116,18 +166,7 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
                 if player.fetch("notify_message") is not None:
                     with contextlib.suppress(discord.HTTPException):
                         await player.fetch("notify_message").delete()
-
-                if (
-                    autoplay
-                    and current_extras.get("autoplay")
-                    and (
-                        prev_song is None
-                        or (hasattr(prev_song, "extras") and not prev_song.extras.get("autoplay"))
-                    )
-                ):
-                    await self.send_embed_msg(notify_channel, title=_("Auto Play started."))
-
-                if not description:
+                if not (description and notify_channel):
                     return
                 if current_stream:
                     dur = "LIVE"
@@ -166,6 +205,9 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
                     await self.send_embed_msg(notify_channel, title=_("Queue ended."))
                 if disconnect:
                     self.bot.dispatch("red_audio_audio_disconnect", guild)
+                    await self.config.guild_from_id(
+                        guild_id=guild_id
+                    ).currently_auto_playing_in.set([])
                     await player.disconnect()
                     self._ll_guild_updates.discard(guild.id)
             if status:
@@ -197,10 +239,14 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
                 eq = player.fetch("eq")
                 player.queue = []
                 player.store("playing_song", None)
+                player.store("autoplay_notified", False)
                 if eq:
                     await self.config.custom("EQUALIZER", guild_id).eq_bands.set(eq.bands)
                 await player.stop()
                 await player.disconnect()
+                await self.config.guild_from_id(guild_id=guild_id).currently_auto_playing_in.set(
+                    []
+                )
                 self._ll_guild_updates.discard(guild_id)
                 self.bot.dispatch("red_audio_audio_disconnect", guild)
             if message_channel:
@@ -240,3 +286,209 @@ class LavalinkEvents(MixinMeta, metaclass=CompositeMetaClass):
                             )
                     await message_channel.send(embed=embed)
             await player.skip()
+
+    async def _websocket_closed_handler(
+        self,
+        guild: discord.Guild,
+        player: lavalink.Player,
+        extra: Dict,
+        deafen: bool,
+        disconnect: bool,
+    ) -> None:
+        guild_id = guild.id
+        event_channel_id = extra.get("channelID")
+        try:
+            if not self._ws_resume[guild_id].is_set():
+                await self._ws_resume[guild_id].wait()
+            else:
+                self._ws_resume[guild_id].clear()
+            node = player.node
+            voice_ws: DiscordWebSocket = node.get_voice_ws(guild_id)
+            code = extra.get("code")
+            by_remote = extra.get("byRemote", "")
+            reason = extra.get("reason", "No Specified Reason").strip()
+            channel_id = player.channel.id
+            try:
+                event_channel_id, to_handle_code = await self._ws_op_codes[guild_id].get()
+            except asyncio.QueueEmpty:
+                log.debug("Empty queue - Resuming Processor - Early exit")
+                return
+
+            if code != to_handle_code:
+                code = to_handle_code
+                if player.channel.id != event_channel_id:
+                    code = 4014
+            if event_channel_id != channel_id:
+                ws_audio_log.info(
+                    f"Received an op code for a channel that is no longer valid; {event_channel_id} "
+                    f"in guild: {guild_id}  - Active channel {channel_id} | "
+                    f"Reason: Error code {code} & {reason}."
+                )
+                self._ws_op_codes[guild_id]._init(self._ws_op_codes[guild_id]._maxsize)
+                return
+            if player.channel:
+                current_perms = player.channel.permissions_for(player.channel.guild.me)
+                has_perm = current_perms.speak and current_perms.connect
+            else:
+                has_perm = False
+            if code in (1000,) and has_perm and player.current and player.is_playing:
+                player.store("resumes", player.fetch("resumes", 0) + 1)
+                await player.resume(player.current, start=player.position, replace=True)
+                ws_audio_log.info(
+                    f"Player resumed in channel {channel_id} in guild: {guild_id} | "
+                    f"Reason: Error code {code} & {reason}."
+                )
+                self._ws_op_codes[guild_id]._init(self._ws_op_codes[guild_id]._maxsize)
+                return
+
+            if voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open:
+                if player._con_delay:
+                    delay = player._con_delay.delay()
+                else:
+                    player._con_delay = ExponentialBackoff(base=1)
+                    delay = player._con_delay.delay()
+                ws_audio_log.warning(
+                    "YOU CAN IGNORE THIS UNLESS IT'S CONSISTENTLY REPEATING FOR THE SAME GUILD - "
+                    f"Voice websocket closed for guild {guild_id} -> "
+                    f"Socket Closed {voice_ws.socket._closing or voice_ws.socket.closed}.  "
+                    f"Code: {code} -- Remote: {by_remote} -- {reason}"
+                )
+                ws_audio_log.debug(
+                    f"Reconnecting to channel {channel_id} in guild: {guild_id} | {delay:.2f}s"
+                )
+                await asyncio.sleep(delay)
+                while voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open:
+                    voice_ws = node.get_voice_ws(guild_id)
+                    await asyncio.sleep(0.1)
+
+                if has_perm and player.current and player.is_playing:
+                    player.store("resumes", player.fetch("resumes", 0) + 1)
+                    await player.connect(deafen=deafen)
+                    await player.resume(player.current, start=player.position, replace=True)
+                    ws_audio_log.info(
+                        "Voice websocket reconnected "
+                        f"to channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Currently playing."
+                    )
+                elif has_perm and player.paused and player.current:
+                    player.store("resumes", player.fetch("resumes", 0) + 1)
+                    await player.connect(deafen=deafen)
+                    await player.resume(
+                        player.current, start=player.position, replace=True, pause=True
+                    )
+                    ws_audio_log.info(
+                        "Voice websocket reconnected "
+                        f"to channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Currently Paused."
+                    )
+                elif has_perm and (not disconnect) and (not player.is_playing):
+                    player.store("resumes", player.fetch("resumes", 0) + 1)
+                    await player.connect(deafen=deafen)
+                    ws_audio_log.info(
+                        "Voice websocket reconnected "
+                        f"to channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Not playing, but auto disconnect disabled."
+                    )
+                    self._ll_guild_updates.discard(guild_id)
+                elif not has_perm:
+                    self.bot.dispatch("red_audio_audio_disconnect", guild)
+                    ws_audio_log.info(
+                        "Voice websocket disconnected "
+                        f"from channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Missing permissions."
+                    )
+                    self._ll_guild_updates.discard(guild_id)
+                    player.store("autoplay_notified", False)
+                    await player.stop()
+                    await player.disconnect()
+                    await self.config.guild_from_id(
+                        guild_id=guild_id
+                    ).currently_auto_playing_in.set([])
+                else:
+                    self.bot.dispatch("red_audio_audio_disconnect", guild)
+                    ws_audio_log.info(
+                        "Voice websocket disconnected "
+                        f"from channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Unknown."
+                    )
+                    self._ll_guild_updates.discard(guild_id)
+                    player.store("autoplay_notified", False)
+                    await player.stop()
+                    await player.disconnect()
+                    await self.config.guild_from_id(
+                        guild_id=guild_id
+                    ).currently_auto_playing_in.set([])
+            elif code in (42069,) and has_perm and player.current and player.is_playing:
+                player.store("resumes", player.fetch("resumes", 0) + 1)
+                await player.connect(deafen=deafen)
+                await player.resume(player.current, start=player.position, replace=True)
+                ws_audio_log.info(
+                    f"Player resumed in channel {channel_id} in guild: {guild_id} | "
+                    f"Reason: Error code {code} & {reason}."
+                )
+            elif code in (4015, 4009, 4006, 4000, 1006):
+                if player._con_delay:
+                    delay = player._con_delay.delay()
+                else:
+                    player._con_delay = ExponentialBackoff(base=1)
+                    delay = player._con_delay.delay()
+                ws_audio_log.debug(
+                    f"Reconnecting to channel {channel_id} in guild: {guild_id} | {delay:.2f}s"
+                )
+                await asyncio.sleep(delay)
+                if has_perm and player.current and player.is_playing:
+                    await player.connect(deafen=deafen)
+                    await player.resume(player.current, start=player.position, replace=True)
+                    ws_audio_log.info(
+                        "Voice websocket reconnected "
+                        f"to channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Player is active."
+                    )
+                elif has_perm and player.paused and player.current:
+                    player.store("resumes", player.fetch("resumes", 0) + 1)
+                    await player.connect(deafen=deafen)
+                    await player.resume(
+                        player.current, start=player.position, replace=True, pause=True
+                    )
+                    ws_audio_log.info(
+                        "Voice websocket reconnected "
+                        f"to channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Player is paused."
+                    )
+                elif has_perm and (not disconnect) and (not player.is_playing):
+                    player.store("resumes", player.fetch("resumes", 0) + 1)
+                    await player.connect(deafen=deafen)
+                    ws_audio_log.info(
+                        "Voice websocket reconnected "
+                        f"to channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Not playing."
+                    )
+                    self._ll_guild_updates.discard(guild_id)
+                elif not has_perm:
+                    self.bot.dispatch("red_audio_audio_disconnect", guild)
+                    ws_audio_log.info(
+                        "Voice websocket disconnected "
+                        f"from channel {channel_id} in guild: {guild_id} | "
+                        f"Reason: Error code {code} & Missing permissions."
+                    )
+                    self._ll_guild_updates.discard(guild_id)
+                    player.store("autoplay_notified", False)
+                    await player.stop()
+                    await player.disconnect()
+                    await self.config.guild_from_id(
+                        guild_id=guild_id
+                    ).currently_auto_playing_in.set([])
+            else:
+                if not player.paused and player.current:
+                    player.store("resumes", player.fetch("resumes", 0) + 1)
+                    await player.resume(player.current, start=player.position, replace=False)
+                ws_audio_log.info(
+                    "WS EVENT - IGNORED (Healthy Socket) | "
+                    f"Voice websocket closed event for guild {guild_id} -> "
+                    f"Code: {code} -- Remote: {by_remote} -- {reason}"
+                )
+        except Exception:
+            log.exception("Error in task")
+        finally:
+            self._ws_op_codes[guild_id]._init(self._ws_op_codes[guild_id]._maxsize)
+            self._ws_resume[guild_id].set()