CopyBot/Red-DiscordBot
1
0
Fork 0
mirror of https://github.com/Cog-Creators/Red-DiscordBot.git synced 2025-11-21 18:27:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

[Downloader] More robust repo loading (#2121)

Browse Source 
Previously, when downloader was loaded, the RepoManager would spawn a task to load available repos. If one repo failed loading for some reason, the function would raise and the remaining repos would never be loaded, however downloader would still appear to load correctly.

This change handles exceptions better during repo loading, but also, if an unhandled exception is raised, downloader will fail to load as it should.

Also included, as requested in #1968, is the --recurse-submodules flag in cloning/pulling repositories.

This change resolves #1950.

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>
This commit is contained in:
Toby Harradine
2018-09-22 15:05:42 +10:00
committed by GitHub
parent df922a0e3e
commit e27682abd3
5 changed files with 76 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
redbot/cogs/downloader/repo_manager.py
View File

@@ -2,6 +2,7 @@ import asyncio
import functools
import os
import pkgutil
import shutil
import re
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
@@ -11,19 +12,19 @@ from typing import Tuple, MutableMapping, Union, Optional
from redbot.core import data_manager, commands
from redbot.core.utils import safe_delete
from .errors import *
from . import errors
from .installable import Installable, InstallableType
from .json_mixins import RepoJSONMixin
from .log import log
class Repo(RepoJSONMixin):
GIT_CLONE = "git clone -b {branch} {url} {folder}"
GIT_CLONE_NO_BRANCH = "git clone {url} {folder}"
GIT_CLONE = "git clone --recurse-submodules -b {branch} {url} {folder}"
GIT_CLONE_NO_BRANCH = "git clone --recurse-submodules {url} {folder}"
GIT_CURRENT_BRANCH = "git -C {path} rev-parse --abbrev-ref HEAD"
GIT_LATEST_COMMIT = "git -C {path} rev-parse {branch}"
GIT_HARD_RESET = "git -C {path} reset --hard origin/{branch} -q"
GIT_PULL = "git -C {path} pull -q --ff-only"
GIT_PULL = "git -C {path} --recurse-submodules=yes -q --ff-only"
GIT_DIFF_FILE_STATUS = "git -C {path} diff --no-commit-id --name-status {old_hash} {new_hash}"
GIT_LOG = "git -C {path} log --relative-date --reverse {old_hash}.. {relative_file_path}"
GIT_DISCOVER_REMOTE_URL = "git -C {path} config --get remote.origin.url"
@@ -93,7 +94,9 @@ class Repo(RepoJSONMixin):
)
if p.returncode != 0:
raise GitDiffError("Git diff failed for repo at path: {}".format(self.folder_path))
raise errors.GitDiffError(
"Git diff failed for repo at path: {}".format(self.folder_path)
)
stdout = p.stdout.strip().decode().split("\n")
@@ -123,7 +126,7 @@ class Repo(RepoJSONMixin):
)
if p.returncode != 0:
raise GitException(
raise errors.GitException(
"An exception occurred while executing git log on"
" this repo: {}".format(self.folder_path)
)
@@ -177,7 +180,7 @@ class Repo(RepoJSONMixin):
"""
exists, path = self._existing_git_repo()
if exists:
raise ExistingGitRepo("A git repo already exists at path: {}".format(path))
raise errors.ExistingGitRepo("A git repo already exists at path: {}".format(path))
if self.branch is not None:
p = await self._run(
@@ -190,8 +193,10 @@ class Repo(RepoJSONMixin):
self.GIT_CLONE_NO_BRANCH.format(url=self.url, folder=self.folder_path).split()
)
if p.returncode != 0:
raise CloningError("Error when running git clone.")
if p.returncode:
# Try cleaning up folder
shutil.rmtree(str(self.folder_path), ignore_errors=True)
raise errors.CloningError("Error when running git clone.")
if self.branch is None:
self.branch = await self.current_branch()
@@ -211,12 +216,14 @@ class Repo(RepoJSONMixin):
"""
exists, _ = self._existing_git_repo()
if not exists:
raise MissingGitRepo("A git repo does not exist at path: {}".format(self.folder_path))
raise errors.MissingGitRepo(
"A git repo does not exist at path: {}".format(self.folder_path)
)
p = await self._run(self.GIT_CURRENT_BRANCH.format(path=self.folder_path).split())
if p.returncode != 0:
raise GitException(
raise errors.GitException(
"Could not determine current branch at path: {}".format(self.folder_path)
)
@@ -241,14 +248,16 @@ class Repo(RepoJSONMixin):
exists, _ = self._existing_git_repo()
if not exists:
raise MissingGitRepo("A git repo does not exist at path: {}".format(self.folder_path))
raise errors.MissingGitRepo(
"A git repo does not exist at path: {}".format(self.folder_path)
)
p = await self._run(
self.GIT_LATEST_COMMIT.format(path=self.folder_path, branch=branch).split()
)
if p.returncode != 0:
raise CurrentHashError("Unable to determine old commit hash.")
raise errors.CurrentHashError("Unable to determine old commit hash.")
return p.stdout.decode().strip()
@@ -268,8 +277,9 @@ class Repo(RepoJSONMixin):
Raises
------
RuntimeError
.NoRemoteURL
When the folder does not contain a git repo with a FETCH URL.
"""
if folder is None:
folder = self.folder_path
@@ -277,7 +287,7 @@ class Repo(RepoJSONMixin):
p = await self._run(Repo.GIT_DISCOVER_REMOTE_URL.format(path=folder).split())
if p.returncode != 0:
raise RuntimeError("Unable to discover a repo URL.")
raise errors.NoRemoteURL("Unable to discover a repo URL.")
return p.stdout.decode().strip()
@@ -295,14 +305,16 @@ class Repo(RepoJSONMixin):
exists, _ = self._existing_git_repo()
if not exists:
raise MissingGitRepo("A git repo does not exist at path: {}".format(self.folder_path))
raise errors.MissingGitRepo(
"A git repo does not exist at path: {}".format(self.folder_path)
)
p = await self._run(
self.GIT_HARD_RESET.format(path=self.folder_path, branch=branch).split()
)
if p.returncode != 0:
raise HardResetError(
raise errors.HardResetError(
"Some error occurred when trying to"
" execute a hard reset on the repo at"
" the following path: {}".format(self.folder_path)
@@ -325,7 +337,7 @@ class Repo(RepoJSONMixin):
p = await self._run(self.GIT_PULL.format(path=self.folder_path).split())
if p.returncode != 0:
raise UpdateError(
raise errors.UpdateError(
"Git pull returned a non zero exit code"
" for the repo located at path: {}".format(self.folder_path)
)
@@ -354,7 +366,7 @@ class Repo(RepoJSONMixin):
"""
if cog not in self.available_cogs:
raise DownloaderException("That cog does not exist in this repo")
raise errors.DownloaderException("That cog does not exist in this repo")
if not target_dir.is_dir():
raise ValueError("That target directory is not actually a directory.")
@@ -500,6 +512,9 @@ class RepoManager:
loop = asyncio.get_event_loop()
loop.create_task(self._load_repos(set=True)) # str_name: Repo
async def initialize(self):
await self._load_repos(set=True)
@property
def repos_folder(self) -> Path:
data_folder = data_manager.cog_data_path(self)
@@ -511,7 +526,7 @@ class RepoManager:
@staticmethod
def validate_and_normalize_repo_name(name: str) -> str:
if not name.isidentifier():
raise InvalidRepoName("Not a valid Python variable name.")
raise errors.InvalidRepoName("Not a valid Python variable name.")
return name.lower()
async def add_repo(self, url: str, name: str, branch: Optional[str] = None) -> Repo:
@@ -533,7 +548,7 @@ class RepoManager:
"""
if self.does_repo_exist(name):
raise ExistingGitRepo(
raise errors.ExistingGitRepo(
"That repo name you provided already exists. Please choose another."
)
@@ -584,13 +599,13 @@ class RepoManager:
Raises
------
MissingGitRepo
.MissingGitRepo
If the repo does not exist.
"""
repo = self.get_repo(name)
if repo is None:
raise MissingGitRepo("There is no repo with the name {}".format(name))
raise errors.MissingGitRepo("There is no repo with the name {}".format(name))
safe_delete(repo.folder_path)
@@ -629,9 +644,16 @@ class RepoManager:
continue
try:
ret[folder.stem] = await Repo.from_folder(folder)
except RuntimeError:
# Thrown when there's no findable git remote URL
pass
except errors.NoRemoteURL:
log.warning("A remote URL does not exist for repo %s", folder.stem)
except errors.DownloaderException as err:
log.error("Discarding repo %s due to error.", folder.stem, exc_info=err)
shutil.rmtree(
str(folder),
onerror=lambda func, path, exc: log.error(
"Failed to remove folder %s", path, exc_info=exc
),
)
if set:
self._repos = ret