[3.2][Audio] Add a limiter of 500 to the Visible queue (#3279)

* Add a limited of 500 to the Visible queue and somce asyncio sleep every n tracks to assist with queue loading so it stops being blocking

* add a text for queue

Signed-off-by: Drapersniper <27962761+drapersniper@users.noreply.github.com>

* chore

Signed-off-by: Drapersniper <27962761+drapersniper@users.noreply.github.com>

* add asyncio sleep on playlist queue command

Signed-off-by: Drapersniper <27962761+drapersniper@users.noreply.github.com>

* Better error handling for large playlists

Signed-off-by: Drapersniper <27962761+drapersniper@users.noreply.github.com>

* remove files even if it errors

Signed-off-by: Drapersniper <27962761+drapersniper@users.noreply.github.com>

changelog.d/audio/3279.bugfix.1.rst

@@ -0,0 +1 @@
+Restrict the number of songs shown in the queue to first 500 to avoid heartbeats.

redbot/cogs/audio/audio.py

@@ -4,9 +4,11 @@ import datetime
 import heapq
 import json
 import logging
+import tarfile
 import math
 import random
 import re
+import os.path
 import time
 import traceback
 from collections import Counter, namedtuple
@@ -4484,6 +4486,7 @@ class Audio(commands.Cog):
     @checks.is_owner()
     @playlist.command(name="download", usage="<playlist_name_OR_id> [v2=False] [args]")
     @commands.bot_has_permissions(attach_files=True)
+    @commands.cooldown(1, 60, commands.BucketType.guild)
     async def _playlist_download(
         self,
         ctx: commands.Context,
@@ -4592,7 +4595,31 @@ class Audio(commands.Cog):
         to_write = BytesIO()
         to_write.write(playlist_data)
         to_write.seek(0)
-        await ctx.send(file=discord.File(to_write, filename=f"{file_name}.txt"))
+        if to_write.getbuffer().nbytes > ctx.guild.filesize_limit - 10000:
+            datapath = cog_data_path(raw_name="Audio")
+            temp_file = datapath / f"{file_name}.txt"
+            temp_tar = datapath / f"{file_name}.tar.gz"
+            with temp_file.open("wb") as playlist_file:
+                playlist_file.write(to_write.read())
+
+            with tarfile.open(str(temp_tar), "w:gz") as tar:
+                tar.add(
+                    str(temp_file), arcname=str(temp_file.relative_to(datapath)), recursive=False
+                )
+            try:
+                if os.path.getsize(str(temp_tar)) > ctx.guild.filesize_limit - 10000:
+                    await ctx.send(_("This playlist is too large to be send in this server."))
+                else:
+                    await ctx.send(
+                        content=_("Playlist is too large, here is the compressed version."),
+                        file=discord.File(str(temp_tar)),
+                    )
+            except Exception:
+                pass
+            temp_file.unlink()
+            temp_tar.unlink()
+        else:
+            await ctx.send(file=discord.File(to_write, filename=f"{file_name}.txt"))
         to_write.close()
 
     @playlist.command(name="info", usage="<playlist_name_OR_id> [args]")
@@ -4675,7 +4702,9 @@ class Audio(commands.Cog):
         track_idx = 0
         if track_len > 0:
             spaces = "\N{EN SPACE}" * (len(str(len(playlist.tracks))) + 2)
-            for track in playlist.tracks:
+            for i, track in enumerate(playlist.tracks, start=1):
+                if i % 500 == 0:  # TODO: Improve when Toby menu's are merged
+                    await asyncio.sleep(0.1)
                 track_idx = track_idx + 1
                 query = audio_dataclasses.Query.process_input(track["info"]["uri"])
                 if query.is_local:
@@ -4876,44 +4905,49 @@ class Audio(commands.Cog):
         ​ ​ ​ ​ [p]playlist queue MyGlobalPlaylist --scope Global
         ​ ​ ​ ​ [p]playlist queue MyPersonalPlaylist --scope User
         """
-        if scope_data is None:
+        async with ctx.typing():
-            scope_data = [PlaylistScope.GUILD.value, ctx.author, ctx.guild, False]
+            if scope_data is None:
-        scope, author, guild, specified_user = scope_data
+                scope_data = [PlaylistScope.GUILD.value, ctx.author, ctx.guild, False]
-        scope_name = humanize_scope(
+            scope, author, guild, specified_user = scope_data
-            scope, ctx=guild if scope == PlaylistScope.GUILD.value else author
+            scope_name = humanize_scope(
-        )
+                scope, ctx=guild if scope == PlaylistScope.GUILD.value else author
-        temp_playlist = FakePlaylist(author.id, scope)
-        if not await self.can_manage_playlist(scope, temp_playlist, ctx, author, guild):
-            ctx.command.reset_cooldown(ctx)
-            return
-        playlist_name = playlist_name.split(" ")[0].strip('"')[:32]
-        if playlist_name.isnumeric():
-            ctx.command.reset_cooldown(ctx)
-            return await self._embed_msg(
-                ctx,
-                title=_("Invalid Playlist Name"),
-                description=_(
-                    "Playlist names must be a single word "
-                    "(up to 32 characters) and not numbers only."
-                ),
             )
-        if not self._player_check(ctx):
+            temp_playlist = FakePlaylist(author.id, scope)
-            ctx.command.reset_cooldown(ctx)
+            if not await self.can_manage_playlist(scope, temp_playlist, ctx, author, guild):
-            return await self._embed_msg(ctx, title=_("Nothing playing."))
+                ctx.command.reset_cooldown(ctx)
+                return
+            playlist_name = playlist_name.split(" ")[0].strip('"')[:32]
+            if playlist_name.isnumeric():
+                ctx.command.reset_cooldown(ctx)
+                return await self._embed_msg(
+                    ctx,
+                    title=_("Invalid Playlist Name"),
+                    description=_(
+                        "Playlist names must be a single word "
+                        "(up to 32 characters) and not numbers only."
+                    ),
+                )
+            if not self._player_check(ctx):
+                ctx.command.reset_cooldown(ctx)
+                return await self._embed_msg(ctx, title=_("Nothing playing."))
 
-        player = lavalink.get_player(ctx.guild.id)
+            player = lavalink.get_player(ctx.guild.id)
-        if not player.queue:
+            if not player.queue:
-            ctx.command.reset_cooldown(ctx)
+                ctx.command.reset_cooldown(ctx)
-            return await self._embed_msg(ctx, title=_("There's nothing in the queue."))
+                return await self._embed_msg(ctx, title=_("There's nothing in the queue."))
-        tracklist = []
+            tracklist = []
-        np_song = track_creator(player, "np")
+            np_song = track_creator(player, "np")
-        tracklist.append(np_song)
+            tracklist.append(np_song)
-        for track in player.queue:
+            for i, track in enumerate(player.queue, start=1):
-            queue_idx = player.queue.index(track)
+                if i % 500 == 0:  # TODO: Improve when Toby menu's are merged
-            track_obj = track_creator(player, queue_idx)
+                    await asyncio.sleep(0.02)
-            tracklist.append(track_obj)
+                queue_idx = player.queue.index(track)
+                track_obj = track_creator(player, queue_idx)
+                tracklist.append(track_obj)
 
-        playlist = await create_playlist(ctx, scope, playlist_name, None, tracklist, author, guild)
+            playlist = await create_playlist(
+                ctx, scope, playlist_name, None, tracklist, author, guild
+            )
         await self._embed_msg(
             ctx,
             title=_("Playlist Created"),
@@ -5203,7 +5237,9 @@ class Audio(commands.Cog):
             player = lavalink.get_player(ctx.guild.id)
             tracks = playlist.tracks_obj
             empty_queue = not player.queue
-            for track in tracks:
+            for i, track in enumerate(tracks, start=1):
+                if i % 500 == 0:  # TODO: Improve when Toby menu's are merged
+                    await asyncio.sleep(0.02)
                 if not await is_allowed(
                     ctx.guild,
                     (
@@ -6124,26 +6160,31 @@ class Audio(commands.Cog):
             return await self._embed_msg(ctx, title=_("There's nothing in the queue."))
 
         async with ctx.typing():
-            len_queue_pages = math.ceil(len(player.queue) / 10)
+            limited_queue = player.queue[:500]  # TODO: Improve when Toby menu's are merged
+            len_queue_pages = math.ceil(len(limited_queue) / 10)
             queue_page_list = []
             for page_num in range(1, len_queue_pages + 1):
-                embed = await self._build_queue_page(ctx, player, page_num)
+                embed = await self._build_queue_page(ctx, limited_queue, player, page_num)
                 queue_page_list.append(embed)
             if page > len_queue_pages:
                 page = len_queue_pages
         return await menu(ctx, queue_page_list, queue_controls, page=(page - 1))
 
     async def _build_queue_page(
-        self, ctx: commands.Context, player: lavalink.player_manager.Player, page_num
+        self, ctx: commands.Context, queue: list, player: lavalink.player_manager.Player, page_num
     ):
         shuffle = await self.config.guild(ctx.guild).shuffle()
         repeat = await self.config.guild(ctx.guild).repeat()
         autoplay = await self.config.guild(ctx.guild).auto_play()
 
-        queue_num_pages = math.ceil(len(player.queue) / 10)
+        queue_num_pages = math.ceil(len(queue) / 10)
         queue_idx_start = (page_num - 1) * 10
         queue_idx_end = queue_idx_start + 10
-        queue_list = ""
+        if len(player.queue) > 500:
+            queue_list = "__Too many songs in the queue, only showing the first 500__.\n\n"
+        else:
+            queue_list = ""
+
         try:
             arrow = await draw_time(ctx)
         except AttributeError:
@@ -6189,9 +6230,10 @@ class Audio(commands.Cog):
             queue_list += _("Requested by: **{user}**").format(user=player.current.requester)
             queue_list += f"\n\n{arrow}`{pos}`/`{dur}`\n\n"
 
-        for i, track in enumerate(
+        for i, track in enumerate(queue[queue_idx_start:queue_idx_end], start=queue_idx_start):
-            player.queue[queue_idx_start:queue_idx_end], start=queue_idx_start
+            if i % 100 == 0:  # TODO: Improve when Toby menu's are merged
-        ):
+                await asyncio.sleep(0.1)
+
             if len(track.title) > 40:
                 track_title = str(track.title).replace("[", "")
                 track_title = "{}...".format((track_title[:40]).rstrip(" "))
@@ -6229,8 +6271,8 @@ class Audio(commands.Cog):
         text = _(
             "Page {page_num}/{total_pages} | {num_tracks} tracks, {num_remaining} remaining\n"
         ).format(
-            page_num=page_num,
+            page_num=humanize_number(page_num),
-            total_pages=queue_num_pages,
+            total_pages=humanize_number(queue_num_pages),
             num_tracks=len(player.queue) + 1,
             num_remaining=queue_total_duration,
         )
@@ -6258,7 +6300,9 @@ class Audio(commands.Cog):
     async def _build_queue_search_list(queue_list, search_words):
         track_list = []
         queue_idx = 0
-        for track in queue_list:
+        for i, track in enumerate(queue_list, start=1):
+            if i % 100 == 0:  # TODO: Improve when Toby menu's are merged
+                await asyncio.sleep(0.1)
             queue_idx = queue_idx + 1
             if not match_url(track.uri):
                 query = audio_dataclasses.Query.process_input(track)
@@ -6288,6 +6332,8 @@ class Audio(commands.Cog):
         for i, track in enumerate(
             search_list[search_idx_start:search_idx_end], start=search_idx_start
         ):
+            if i % 100 == 0:  # TODO: Improve when Toby menu's are merged
+                await asyncio.sleep(0.1)
             track_idx = i + 1
             if type(track) is str:
                 track_location = audio_dataclasses.LocalPath(track).to_string_user()
@@ -6299,7 +6345,9 @@ class Audio(commands.Cog):
         )
         embed.set_footer(
             text=(_("Page {page_num}/{total_pages}") + " | {num_tracks} tracks").format(
-                page_num=page_num, total_pages=search_num_pages, num_tracks=len(search_list)
+                page_num=humanize_number(page_num),
+                total_pages=humanize_number(search_num_pages),
+                num_tracks=len(search_list),
             )
         )
         return embed

redbot/cogs/audio/utils.py

@@ -2,8 +2,11 @@ import asyncio
 import contextlib
 import functools
 import re
+import tarfile
 import time
+import zipfile
 from enum import Enum, unique
+from io import BytesIO
 from typing import MutableMapping, Optional, TYPE_CHECKING
 from urllib.parse import urlparse