[V3 Downloader] Make shared libraries work and make repo handling smarter (#1313)

* Make stuff stateless

* Update shared lib stuff

redbot/cogs/downloader/converters.py

@@ -15,7 +15,7 @@ class InstalledCog(commands.Converter):
         if downloader is None:
             raise commands.CommandError("Downloader not loaded.")
 
-        cog = discord.utils.get(downloader.installed_cogs, name=arg)
+        cog = discord.utils.get(await downloader.installed_cogs(), name=arg)
         if cog is None:
             raise commands.BadArgument(
                 "That cog is not installed"

redbot/cogs/downloader/downloader.py

@@ -7,6 +7,7 @@ from typing import Tuple, Union
 import discord
 from redbot.core import Config
 from redbot.core import checks
+from redbot.core.data_manager import cog_data_path
 from redbot.core.i18n import CogI18n
 from redbot.core.utils.chat_formatting import box, pagify
 from discord.ext import commands
@@ -30,13 +31,12 @@ class Downloader:
                                     force_registration=True)
 
         self.conf.register_global(
-            repos={},
             installed=[]
         )
 
         self.already_agreed = False
 
-        self.LIB_PATH = self.bot.main_dir / "lib"
+        self.LIB_PATH = cog_data_path(self) / "lib"
         self.SHAREDLIB_PATH = self.LIB_PATH / "cog_shared"
         self.SHAREDLIB_INIT = self.SHAREDLIB_PATH / "__init__.py"
 
@@ -73,7 +73,7 @@ class Downloader:
         """
         installed = await self.conf.installed()
         # noinspection PyTypeChecker
-        return tuple(Installable.from_json(v) for v in installed)
+        return tuple(Installable.from_json(v, self._repo_manager) for v in installed)
 
     async def _add_to_installed(self, cog: Installable):
         """Mark a cog as installed.
@@ -289,29 +289,37 @@ class Downloader:
             await ctx.send(_("`{}` was successfully removed.").format(real_name))
         else:
             await ctx.send(_("That cog was installed but can no longer"
-                           " be located. You may need to remove it's"
+                             " be located. You may need to remove it's"
-                           " files manually if it is still usable."))
+                             " files manually if it is still usable."))
 
     @cog.command(name="update")
     async def _cog_update(self, ctx, cog_name: InstalledCog=None):
         """
         Updates all cogs or one of your choosing.
         """
+        installed_cogs = set(await self.installed_cogs())
+
         if cog_name is None:
             updated = await self._repo_manager.update_all_repos()
-            installed_cogs = set(await self.installed_cogs())
-            updated_cogs = set(cog for repo in updated.keys() for cog in repo.available_cogs)
 
-            installed_and_updated = updated_cogs & installed_cogs
+        else:
+            try:
+                updated = await self._repo_manager.update_repo(cog_name.repo_name)
+            except KeyError:
+                # Thrown if the repo no longer exists
+                updated = {}
 
-            # noinspection PyTypeChecker
+        updated_cogs = set(cog for repo in updated.keys() for cog in repo.available_cogs)
-            await self._reinstall_requirements(installed_and_updated)
+        installed_and_updated = updated_cogs & installed_cogs
 
-            # noinspection PyTypeChecker
+        # noinspection PyTypeChecker
-            await self._reinstall_cogs(installed_and_updated)
+        await self._reinstall_requirements(installed_and_updated)
 
-            # noinspection PyTypeChecker
+        # noinspection PyTypeChecker
-            await self._reinstall_libraries(installed_and_updated)
+        await self._reinstall_cogs(installed_and_updated)
+
+        # noinspection PyTypeChecker
+        await self._reinstall_libraries(installed_and_updated)
         await ctx.send(_("Cog update completed successfully."))
 
     @cog.command(name="list")

redbot/cogs/downloader/installable.py

@@ -3,12 +3,15 @@ import distutils.dir_util
 import shutil
 from enum import Enum
 from pathlib import Path
-from typing import Union, MutableMapping, Any
+from typing import MutableMapping, Any
 
-from redbot.core import data_manager
+from redbot.core.utils import TYPE_CHECKING
 from .log import log
 from .json_mixins import RepoJSONMixin
 
+if TYPE_CHECKING:
+    from .repo_manager import RepoManager
+
 
 class InstallableType(Enum):
     UNKNOWN = 0
@@ -50,11 +53,6 @@ class Installable(RepoJSONMixin):
         :class:`InstallationType`.
 
     """
-
-    INFO_FILE_DESCRIPTION = """
-    
-    """
-
     def __init__(self, location: Path):
         """Base installable initializer.
 
@@ -192,13 +190,22 @@ class Installable(RepoJSONMixin):
         return info
 
     def to_json(self):
-        data_path = data_manager.cog_data_path()
         return {
-            "location": self._location.relative_to(data_path).parts
+            "repo_name": self.repo_name,
+            "cog_name": self.name
         }
 
     @classmethod
-    def from_json(cls, data: dict):
+    def from_json(cls, data: dict, repo_mgr: "RepoManager"):
-        data_path = data_manager.cog_data_path()
+        repo_name = data['repo_name']
-        location = data_path / Path(*data["location"])
+        cog_name = data['cog_name']
+
+        repo = repo_mgr.get_repo(repo_name)
+        if repo is not None:
+            repo_folder = repo.folder_path
+        else:
+            repo_folder = repo_mgr.repos_folder / "MISSING_REPO"
+
+        location = repo_folder / cog_name
+
         return cls(location=location)

redbot/cogs/downloader/repo_manager.py

@@ -30,6 +30,7 @@ class Repo(RepoJSONMixin):
                             " {old_hash} {new_hash}")
     GIT_LOG = ("git -C {path} log --relative-date --reverse {old_hash}.."
                " {relative_file_path}")
+    GIT_DISCOVER_REMOTE_URL = "git -C {path} config --get remote.origin.url"
 
     PIP_INSTALL = "{python} -m pip install -U -t {target_dir} {reqs}"
 
@@ -266,6 +267,39 @@ class Repo(RepoJSONMixin):
 
         return p.stdout.decode().strip()
 
+    async def current_url(self, folder: Path=None) -> str:
+        """
+        Discovers the FETCH URL for a Git repo.
+
+        Parameters
+        ----------
+        folder : pathlib.Path
+            The folder to search for a URL.
+
+        Returns
+        -------
+        str
+            The FETCH URL.
+
+        Raises
+        ------
+        RuntimeError
+            When the folder does not contain a git repo with a FETCH URL.
+        """
+        if folder is None:
+            folder = self.folder_path
+
+        p = await self._run(
+            Repo.GIT_DISCOVER_REMOTE_URL.format(
+                path=folder
+            ).split()
+        )
+
+        if p.returncode != 0:
+            raise RuntimeError("Unable to discover a repo URL.")
+
+        return p.stdout.decode().strip()
+
     async def hard_reset(self, branch: str=None) -> None:
         """Perform a hard reset on the current repo.
 
@@ -373,14 +407,17 @@ class Repo(RepoJSONMixin):
             The success of the installation.
 
         """
-        if libraries:
+        if len(libraries) > 0:
             if not all([i in self.available_libraries for i in libraries]):
                 raise ValueError("Some given libraries are not available in this repo.")
         else:
             libraries = self.available_libraries
 
-        if libraries:
+        if len(libraries) > 0:
-            return all([lib.copy_to(target_dir=target_dir) for lib in libraries])
+            ret = True
+            for lib in libraries:
+                ret = ret and await lib.copy_to(target_dir=target_dir)
+            return ret
         return True
 
     async def install_requirements(self, cog: Installable, target_dir: Path) -> bool:
@@ -467,23 +504,13 @@ class Repo(RepoJSONMixin):
              if m.type == InstallableType.SHARED_LIBRARY]
         )
 
-    def to_json(self):
-        data_path = data_manager.cog_data_path()
-        return {
-            "url": self.url,
-            "name": self.name,
-            "branch": self.branch,
-            "folder_path": self.folder_path.relative_to(data_path).parts,
-            "available_modules": [m.to_json() for m in self.available_modules]
-        }
-
     @classmethod
-    def from_json(cls, data):
+    async def from_folder(cls, folder: Path):
-        data_path = data_manager.cog_data_path()
+        repo = cls(name=folder.stem, branch="", url="", folder_path=folder)
-        # noinspection PyTypeChecker
+        repo.branch = await repo.current_branch()
-        return Repo(data['name'], data['url'], data['branch'],
+        repo.url = await repo.current_url()
-                    data_path / Path(*data['folder_path']),
+        repo._update_available_modules()
-                    tuple([Installable.from_json(m) for m in data['available_modules']]))
+        return repo
 
 
 class RepoManager:
@@ -540,7 +567,6 @@ class RepoManager:
         await r.clone()
 
         self._repos[name] = r
-        await self._save_repos()
 
         return r
 
@@ -596,7 +622,10 @@ class RepoManager:
         except KeyError:
             pass
 
-        await self._save_repos()
+    async def update_repo(self, repo_name: str) -> MutableMapping[Repo, Tuple[str, str]]:
+        repo = self._repos[repo_name]
+        old, new = await repo.update()
+        return {repo: (old, new)}
 
     async def update_all_repos(self) -> MutableMapping[Repo, Tuple[str, str]]:
         """Call `Repo.update` on all repositories.
@@ -609,23 +638,23 @@ class RepoManager:
 
         """
         ret = {}
-        for _, repo in self._repos.items():
+        for repo_name, _ in self._repos.items():
-            old, new = await repo.update()
+            repo, (old, new) = await self.update_repo(repo_name)
             if old != new:
                 ret[repo] = (old, new)
-
-        await self._save_repos()
         return ret
 
     async def _load_repos(self, set=False) -> MutableMapping[str, Repo]:
-        ret = {
+        ret = {}
-            name: Repo.from_json(data) for name, data in
+        for folder in self.repos_folder.iterdir():
-            (await self.downloader_config.repos()).items()
+            if not folder.is_dir():
-        }
+                continue
+            try:
+                ret[folder.stem] = await Repo.from_folder(folder)
+            except RuntimeError:
+                # Thrown when there's no findable git remote URL
+                pass
+
         if set:
             self._repos = ret
         return ret
-
-    async def _save_repos(self):
-        repo_json_info = {name: r.to_json() for name, r in self._repos.items()}
-        await self.downloader_config.repos.set(repo_json_info)

tests/cogs/downloader/test_installable.py

@@ -66,6 +66,6 @@ def test_repo_name(installable):
 
 def test_serialization(installable):
     data = installable.to_json()
-    location = data["location"]
+    cog_name = data["cog_name"]
 
-    assert location[-1] == "test_cog"
+    assert cog_name == "test_cog"