CopyBot/Red-DiscordBot
1
0
Fork 0
mirror of https://github.com/Cog-Creators/Red-DiscordBot.git synced 2025-11-06 19:28:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

[Downloader] Hotfix, better git error handling, tweak edit timer (#471)

Browse Source 
Handles error exit codes from git calls
Tweaked edit timer to not ratelimit on very fast updates
Restored old behavior of initializing community repo on first run
This commit is contained in:
Caleb Johnson 2016-11-17 02:26:17 -06:00 committed by Twentysix
parent ce5e810e33
commit b62108c56a
1 changed files with 98 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
cogs/downloader.py
View File

@ -14,6 +14,13 @@ from concurrent.futures import ThreadPoolExecutor
from time import time from time import time
NUM_THREADS = 4 NUM_THREADS = 4
REPO_NONEX = 0x1
REPO_CLONE = 0x2
REPO_SAME = 0x4
class UpdateError(Exception):
pass
class Downloader: class Downloader:
@ -26,6 +33,7 @@ class Downloader:
# {name:{url,cog1:{installed},cog1:{installed}}} # {name:{url,cog1:{installed},cog1:{installed}}}
self.repos = dataIO.load_json(self.file_path) self.repos = dataIO.load_json(self.file_path)
self.executor = ThreadPoolExecutor(NUM_THREADS) self.executor = ThreadPoolExecutor(NUM_THREADS)
self._do_first_run()
def save_repos(self): def save_repos(self):
dataIO.save_json(self.file_path, self.repos) dataIO.save_json(self.file_path, self.repos)
@ -88,6 +96,7 @@ class Downloader:
await self.bot.say("That repo doesn't exist.") await self.bot.say("That repo doesn't exist.")
return return
del self.repos[repo_name] del self.repos[repo_name]
#shutil.rmtree(os.path.join(self.path, repo_name))
self.save_repos() self.save_repos()
await self.bot.say("Repo '{}' removed.".format(repo_name)) await self.bot.say("Repo '{}' removed.".format(repo_name))
@ -162,8 +171,8 @@ class Downloader:
tasknum = 0 tasknum = 0
num_repos = len(self.repos) num_repos = len(self.repos)
min_dt = 0.25 min_dt = 0.5
burst_inc = 2*min_dt/NUM_THREADS burst_inc = 0.1/(NUM_THREADS)
touch_n = tasknum touch_n = tasknum
touch_t = time() touch_t = time()
@ -188,20 +197,26 @@ class Downloader:
updated_cogs = [] updated_cogs = []
new_cogs = [] new_cogs = []
deleted_cogs = [] deleted_cogs = []
error_repos = {}
installed_updated_cogs = [] installed_updated_cogs = []
for f in as_completed(tasks): for f in as_completed(tasks):
tasknum += 1 tasknum += 1
name, updates, oldhash = await f try:
if updates: name, updates, oldhash = await f
for k, l in updates.items(): if updates:
tl = [(name, c, oldhash) for c in l] if type(updates) is dict:
if k == 'A': for k, l in updates.items():
new_cogs.extend(tl) tl = [(name, c, oldhash) for c in l]
elif k == 'D': if k == 'A':
deleted_cogs.extend(tl) new_cogs.extend(tl)
elif k == 'M': elif k == 'D':
updated_cogs.extend(tl) deleted_cogs.extend(tl)
elif k == 'M':
updated_cogs.extend(tl)
except UpdateError as e:
name, what = e.args
error_repos[name] = what
edit, touch_t, touch_n = regulate(touch_t, touch_n) edit, touch_t, touch_n = regulate(touch_t, touch_n)
if edit: if edit:
status = ' %d/%d repos updated' % (tasknum, num_repos) status = ' %d/%d repos updated' % (tasknum, num_repos)
@ -221,6 +236,10 @@ class Downloader:
if updated_cogs: if updated_cogs:
status += '\nUpdated cogs: ' \ status += '\nUpdated cogs: ' \
+ ', '.join('%s/%s' % c[:2] for c in updated_cogs) + '.' + ', '.join('%s/%s' % c[:2] for c in updated_cogs) + '.'
if error_repos:
status += '\nThe following repos failed to update: '
for n, what in error_repos.items():
status += '\n%s: %s' % (n, what)
msg = await self._robust_edit(msg, base_msg + status) msg = await self._robust_edit(msg, base_msg + status)
@ -409,6 +428,17 @@ class Downloader:
git_name = splitted[-1] git_name = splitted[-1]
return git_name[:-4] return git_name[:-4]
def _do_first_run(self):
save = False
for repo in self.repos:
broken = 'url' in self.repos[repo] and len(self.repos[repo]) == 1
if broken:
self.update_repo(repo)
self.populate_list(repo)
save = True
if save:
self.save_repos()
def populate_list(self, name): def populate_list(self, name):
valid_cogs = self.list_cogs(name) valid_cogs = self.list_cogs(name)
new = set(valid_cogs.keys()) new = set(valid_cogs.keys())
@ -423,40 +453,63 @@ class Downloader:
del self.repos[name][cog] del self.repos[name][cog]
def update_repo(self, name): def update_repo(self, name):
dd = self.path try:
if name not in self.repos: dd = self.path
return name, None, None if name not in self.repos:
if not os.path.exists(dd + name): raise UpdateError("Repo does not exist in data, wtf")
url = self.repos[name]['url'] folder = os.path.join(dd, name)
run(["git", "clone", url, dd + name]) # Make sure we don't git reset the Red folder on accident
else: if not os.path.exists(os.path.join(folder, '.git')):
rpcmd = ["git", "-C", dd + name, "rev-parse", "HEAD"] #if os.path.exists(folder):
run(["git", "-C", dd + name, "reset", "--hard", #shutil.rmtree(folder)
"origin/HEAD", "-q"]) url = self.repos[name].get('url')
p = run(rpcmd, stdout=PIPE) if not url:
oldhash = p.stdout.decode().strip() raise UpdateError("Need to clone but no URL set")
run(["git", "-C", dd + name, "pull", "-q"]) p = run(["git", "clone", url, dd + name])
p = run(rpcmd, stdout=PIPE) if p.returncode != 0:
newhash = p.stdout.decode().strip() raise UpdateError("Error cloning")
if oldhash == newhash:
return name, None, None
else:
self.populate_list(name) self.populate_list(name)
self.save_repos() return name, REPO_CLONE, None
ret = {} else:
cmd = ['git', '-C', dd + name, 'diff', '--no-commit-id', rpcmd = ["git", "-C", dd + name, "rev-parse", "HEAD"]
'--name-status', oldhash, newhash] p = run(["git", "-C", dd + name, "reset", "--hard",
p = run(cmd, stdout=PIPE) "origin/HEAD", "-q"])
changed = p.stdout.strip().decode().split('\n') if p.returncode != 0:
for f in changed: raise UpdateError("Error resetting to origin/HEAD")
if not f.endswith('.py'): p = run(rpcmd, stdout=PIPE)
continue if p.returncode != 0:
status, cogpath = f.split('\t') raise UpdateError("Unable to determine old commit hash")
cogname = os.path.split(cogpath)[-1][:-3] # strip .py oldhash = p.stdout.decode().strip()
if status not in ret: p = run(["git", "-C", dd + name, "pull", "-q"])
ret[status] = [] if p.returncode != 0:
ret[status].append(cogname) raise UpdateError("Error pulling updates")
return name, ret, oldhash p = run(rpcmd, stdout=PIPE)
if p.returncode != 0:
raise UpdateError("Unable to determine new commit hash")
newhash = p.stdout.decode().strip()
if oldhash == newhash:
return name, REPO_SAME, None
else:
self.populate_list(name)
self.save_repos()
ret = {}
cmd = ['git', '-C', dd + name, 'diff', '--no-commit-id',
'--name-status', oldhash, newhash]
p = run(cmd, stdout=PIPE)
if p.returncode != 0:
raise UpdateError("Error in git diff")
changed = p.stdout.strip().decode().split('\n')
for f in changed:
if not f.endswith('.py'):
continue
status, cogpath = f.split('\t')
cogname = os.path.split(cogpath)[-1][:-3] # strip .py
if status not in ret:
ret[status] = []
ret[status].append(cogname)
return name, ret, oldhash
except UpdateError as e:
raise UpdateError(name, *e.args) from None
async def _robust_edit(self, msg, text): async def _robust_edit(self, msg, text):
try: try: