3rd party repos for downloader cog

cogs/downloader.py

@@ -1,21 +1,26 @@
-import discord
 from discord.ext import commands
-from .utils.dataIO import fileIO
+from cogs.utils.dataIO import fileIO
-from .utils import checks
+from cogs.utils import checks
-from .utils.chat_formatting import box
+from cogs.utils.chat_formatting import box
 from __main__ import send_cmd_help, set_cog
 import os
 from subprocess import call, Popen
-from distutils.dir_util import copy_tree
 import shutil
 import asyncio
 
+
 class Downloader:
     """Cog downloader/installer."""
 
     def __init__(self, bot):
         self.bot = bot
-        self.path = "data/downloader/cogs/"
+        self.path = "data/downloader/"
+        # {name:{url,cog1:{installed},cog1:{installed}}}
+        self.repos = fileIO("data/downloader/repos.json", "load")
+        self.update_repos()
+
+    def save_repos(self):
+        fileIO("data/downloader/repos.json", "save", self.repos)
 
     @commands.group(pass_context=True)
     @checks.is_owner()
@@ -24,124 +29,268 @@ class Downloader:
         if ctx.invoked_subcommand is None:
             await send_cmd_help(ctx)
 
+    @cog.group(pass_context=True)
+    async def repo(self, ctx):
+        """Repo management commands"""
+        if ctx.invoked_subcommand is None or \
+                isinstance(ctx.invoked_subcommand, commands.Group):
+            await send_cmd_help(ctx)
+            return
+
+    @repo.command(name="add", pass_context=True)
+    async def _repo_add(self, ctx, repo_name: str, repo_url: str):
+        """Adds repo to available repo lists
+
+        Warning: Adding 3RD Party Repositories is at your own
+        Risk."""
+        await self.bot.say("Type 'I agree' to confirm "
+                           "adding a 3rd party repo. This has the possibility"
+                           " of being harmful. You will not receive help "
+                           "in Red - Discord Bot #support for any cogs "
+                           "installed from this repo. If you do require "
+                           "support you should contact the owner of this "
+                           "repo.\n\nAgain, ANY repo you add is at YOUR"
+                           " discretion and the creator of Red has "
+                           "ABSOLUTELY ZERO responsibility to help if "
+                           "something goes wrong.")
+        answer = await self.bot.wait_for_message(timeout=15,
+                                                 author=ctx.message.author)
+        if answer is None:
+            await self.bot.say('Not adding repo.')
+            return
+        elif "i agree" not in answer.content.lower():
+            await self.bot.say('Not adding repo.')
+            return
+        self.repos[repo_name] = {}
+        self.repos[repo_name]['url'] = repo_url
+        self.update_repo(repo_name)
+        self.populate_list(repo_name)
+        self.save_repos()
+        await self.bot.say("Repo '{}' added.".format(repo_name))
+
+    @repo.command(name="remove")
+    async def _repo_del(self, repo_name: str):
+        """Removes repo from repo list. COGS ARE NOT REMOVED."""
+        if repo_name not in self.repos:
+            await self.bot.say("That repo doesn't exist.")
+            return
+        del self.repos[repo_name]
+        self.save_repos()
+        await self.bot.say("Repo '{}' removed.".format(repo_name))
+
     @cog.command(name="list")
-    async def _send_list(self):
+    async def _send_list(self, repo_name=None):
         """Lists installable cogs"""
-        index = await self.make_index()
+        retlist = []
-        msg = "Available cogs:\n\n"
+        if repo_name and repo_name in self.repos:
-        for cog in index.keys():
+            msg = "Available cogs:\n"
-            if not index[cog]["DISABLED"]:
+            for cog in sorted(self.repos[repo_name].keys()):
-                msg += cog + "\t" + index[cog]["NAME"] + "\n"
+                if 'url' == cog:
-        await self.bot.say(box(msg)) # Need to deal with over 2000 characters
+                    continue
+                data = self.get_info_data(repo_name, cog)
+                if data:
+                    retlist.append([cog, data['NAME']])
+                else:
+                    retlist.append([cog, ''])
+        else:
+            msg = "Available repos:\n"
+            retlist = sorted([[k, ''] for k in self.repos])
+
+        col_width = max(len(row[0]) for row in retlist) + 2
+        for row in retlist:
+            msg += "\t" + "".join(word.ljust(col_width) for word in row) + "\n"
+        await self.bot.say(box(msg))  # Need to deal with over 2000 characters
 
     @cog.command()
-    async def info(self, cog : str):
+    async def info(self, repo_name: str, cog: str):
         """Shows info about the specified cog"""
-        cogs = self.list_cogs()
+        cogs = self.list_cogs(repo_name)
-        info_file = self.path + cog + "/info.json"
         if cog in cogs:
-            if os.path.isfile(info_file):
+            data = self.get_info_data(repo_name, cog)
-                data = fileIO(info_file, "load")
+            if data:
                 msg = "{} by {}\n\n".format(cog, data["AUTHOR"])
                 msg += data["NAME"] + "\n\n" + data["DESCRIPTION"]
                 await self.bot.say(box(msg))
             else:
                 await self.bot.say("The specified cog has no info file.")
         else:
-            await self.bot.say("That cog doesn't exist. Use cog list to see the full list.")
+            await self.bot.say("That cog doesn't exist."
+                               " Use cog list to see the full list.")
 
     @cog.command(hidden=True)
-    async def search(self, *terms : str):
+    async def search(self, *terms: str):
         """Search installable cogs"""
-        pass #TO DO
+        pass  # TO DO
 
     @cog.command(pass_context=True)
     async def update(self, ctx):
         """Updates cogs"""
-        self.update_repo()
+        self.update_repos()
         await self.bot.say("Downloading updated cogs. Wait 10 seconds...")
-        await asyncio.sleep(10) # TO DO: Wait for the result instead, without being blocking.
+        # TO DO: Wait for the result instead, without being blocking.
-        downloadable_cogs = self.list_cogs()
+        await asyncio.sleep(10)
-        all_cogs = [f.replace(".py", "") for f in os.listdir("cogs/") if f.endswith(".py")]
+        installed_user_cogs = [(repo, cog) for repo in self.repos
-        installed_user_cogs = [f for f in all_cogs if f in downloadable_cogs]
+                               for cog in self.repos[repo]
+                               if cog != 'url' and
+                               self.repos[repo][cog]['INSTALLED'] is True]
         for cog in installed_user_cogs:
-            result = await self.install(cog)
+            await self.install(*cog)
         await self.bot.say("Cogs updated. Reload all installed cogs? (yes/no)")
-        answer = await self.bot.wait_for_message(timeout=15, author=ctx.message.author)
+        answer = await self.bot.wait_for_message(timeout=15,
+                                                 author=ctx.message.author)
         if answer is None:
-            await self.bot.say("Ok then, you can reload cogs with `{}reload <cog_name>`".format(ctx.prefix))
+            await self.bot.say("Ok then, you can reload cogs with"
-        elif answer.content.lower().strip() in ["yes", "y"]:
+                               " `{}reload <cog_name>`".format(ctx.prefix))
-            for cog in installed_user_cogs:
+        elif answer.content.lower().strip() == "yes":
+            for (repo, cog) in installed_user_cogs:
                 self.bot.unload_extension("cogs." + cog)
                 self.bot.load_extension("cogs." + cog)
             await self.bot.say("Done.")
         else:
-            await self.bot.say("Ok then, you can reload cogs with `{}reload <cog_name>`".format(ctx.prefix))
+            await self.bot.say("Ok then, you can reload cogs with"
+                               " `{}reload <cog_name>`".format(ctx.prefix))
 
     @cog.command(name="install", pass_context=True)
-    async def _install(self, ctx, cog : str):
+    async def _install(self, ctx, repo_name: str, cog: str):
         """Installs specified cog"""
-        install_cog = await self.install(cog)
+        if repo_name not in self.repos:
+            await self.bot.say("That repo doesn't exist.")
+            return
+        if cog not in self.repos[repo_name]:
+            await self.bot.say("That cog isn't available from that repo.")
+            return
+        install_cog = await self.install(repo_name, cog)
         if install_cog:
             await self.bot.say("Installation completed. Load it now? (yes/no)")
-            answer = await self.bot.wait_for_message(timeout=15, author=ctx.message.author)
+            answer = await self.bot.wait_for_message(timeout=15,
+                                                     author=ctx.message.author)
             if answer is None:
-                await self.bot.say("Ok then, you can load it with `{}load {}`".format(ctx.prefix, cog))
+                await self.bot.say("Ok then, you can load it with"
-            elif answer.content.lower().strip() in ["yes", "y"]:
+                                   " `{}load {}`".format(ctx.prefix, cog))
+            elif answer.content.lower().strip() == "yes":
                 set_cog("cogs." + cog, True)
                 self.bot.unload_extension("cogs." + cog)
                 self.bot.load_extension("cogs." + cog)
                 await self.bot.say("Done.")
             else:
-                await self.bot.say("Ok then, you can load it with `{}load {}`".format(ctx.prefix, cog))
+                await self.bot.say("Ok then, you can load it with"
-        elif install_cog == False:
+                                   " `{}load {}`".format(ctx.prefix, cog))
+        elif install_cog is False:
             await self.bot.say("Invalid cog. Installation aborted.")
         else:
-            await self.bot.say("That cog doesn't exist. Use cog list to see the full list.")
+            await self.bot.say("That cog doesn't exist. Use cog list to see"
+                               " the full list.")
 
-    async def make_index(self):
+    async def install(self, repo_name, cog):
-        cogs = self.list_cogs()
+        if cog.endswith('.py'):
-        index = {}
+            cog = cog[:-3]
-        if not cogs:
+
-            await self.bot.say("There are no cogs available for installation.")
+        path = self.repos[repo_name][cog]['file']
-            return
+        cog_folder_path = self.repos[repo_name][cog]['folder']
-        for cog in cogs:
+        cog_data_path = os.path.join(cog_folder_path, 'data')
-            if os.path.isfile(self.path + cog + "/info.json"):
+
-                info = fileIO(self.path + cog + "/info.json", "load")
+        to_path = os.path.join("cogs/", cog+".py")
-                index[cog] = info
+
-        # Sort by alphabetic order?
+        print("Copying {}...".format(cog))
-        return index
+        shutil.copy(path, to_path)
 
-    async def install(self, cog):
-        cogs = self.list_cogs()
-        cog = cog.lower()
-        if not cog in cogs:
-            return None
-        files = [f for f in os.listdir(self.path + cog) if os.path.isfile(self.path + cog + "/" + f)] # Listing all files (not dirs) in the cog directory
-        cog_file = [f for f in files if f.endswith(".py")] #Verifying the presence of a single py file
-        if len(cog_file) != 1:
-            return False
-        cog_file = cog_file[0]
-        print("Copying {}...".format(cog_file))
-        shutil.copy(self.path + cog + "/" + cog_file, "cogs/")
-        cog_data_path = self.path + cog + "/data"
         if os.path.exists(cog_data_path):
             print("Copying {}'s data folder...".format(cog))
-            copy_tree(cog_data_path, "data/" + cog)
+            if os.path.exists(os.path.join('data/', cog)):
+                shutil.rmtree(os.path.join('data/', cog))
+            shutil.copytree(cog_data_path, os.path.join('data/', cog))
+        self.repos[repo_name][cog]['INSTALLED'] = True
+        self.save_repos()
         return True
 
-    def list_cogs(self):
+    def get_info_data(self, repo_name, cog):
-        dirs = [d for d in os.listdir(self.path) if os.path.exists(self.path + d)]
+        cogs = self.list_cogs(repo_name)
-        return dirs
+        if cog in cogs:
+            info_file = os.path.join(cogs[cog].get('folder'), "info.json")
+            if os.path.isfile(info_file):
+                try:
+                    data = fileIO(info_file, "load")
+                except:
+                    return None
+                return data
+        return None
 
-    def update_repo(self):
+    def list_cogs(self, repo_name):
-            if not os.path.exists("data/downloader"):
+        valid_cogs = {}
-                print("Downloading cogs repo...")
+
-                call(["git", "clone", "https://github.com/Twentysix26/Red-Cogs.git", "data/downloader"]) # It's blocking but it shouldn't matter
+        repo_path = os.path.join(self.path, repo_name)
+        folders = [f for f in os.listdir(repo_path)
+                   if os.path.isdir(os.path.join(repo_path, f))]
+        legacy_path = os.path.join(repo_path, "cogs")
+        legacy_folders = []
+        if os.path.exists(legacy_path):
+            for f in os.listdir(legacy_path):
+                if os.path.isdir(os.path.join(legacy_path, f)):
+                    legacy_folders.append(os.path.join("cogs", f))
+
+        folders = folders + legacy_folders
+
+        for f in folders:
+            cog_folder_path = os.path.join(self.path, repo_name, f)
+            cog_folder = os.path.basename(cog_folder_path)
+            for cog in os.listdir(cog_folder_path):
+                cog_path = os.path.join(cog_folder_path, cog)
+                if os.path.isfile(cog_path) and cog_folder == cog[:-3]:
+                    valid_cogs[cog[:-3]] = {'folder': cog_folder_path,
+                                            'file': cog_path}
+        return valid_cogs
+
+    def get_dir_name(self, url):
+        splitted = url.split("/")
+        git_name = splitted[-1]
+        return git_name[:-4]
+
+    def populate_list(self, name):
+        valid_cogs = self.list_cogs(name)
+        for cog in valid_cogs:
+            if cog not in self.repos[name]:
+                self.repos[name][cog] = valid_cogs.get(cog, {})
+                self.repos[name][cog]['INSTALLED'] = False
             else:
-                Popen(["git", "-C", "data/downloader", "pull", "-q"])
+                self.repos[name][cog].update(valid_cogs[cog])
+
+    def update_repos(self):
+        for name in self.repos:
+            self.update_repo(name)
+            self.populate_list(name)
+        self.save_repos()
+
+    def update_repo(self, name):
+        if name not in self.repos:
+            return
+        if not os.path.exists("data/downloader/" + name):
+            print("Downloading cogs repo...")
+            url = self.repos[name]['url']
+            # It's blocking but it shouldn't matter
+            call(["git", "clone", url, "data/downloader/" + name])
+        else:
+            Popen(["git", "-C", "data/downloader/" + name, "pull", "-q"])
+
+
+def check_folders():
+    if not os.path.exists("data/downloader"):
+        print('Making repo downloads folder...')
+        os.mkdir('data/downloader')
+
+
+def check_files():
+    repos = \
+        {'community': {'url': "https://github.com/Twentysix26/Red-Cogs.git"}}
+
+    f = "data/downloader/repos.json"
+    if not fileIO(f, "check"):
+        print("Creating default data/downloader/repos.json")
+        fileIO(f, "save", repos)
+
+>>>>>>> f3b02f3539b7c2b9eac1ce4548b01700871c9fcd
 
 def setup(bot):
+    check_folders()
+    check_files()
     n = Downloader(bot)
-    n.update_repo()
     bot.add_cog(n)