Better user feedback during slow backend migrations (#2952)

* Better user feedback during slow backend migrations

This uses a tqdm progress bar to keep the user updated on the progress of the migrations.

I didn't realise this was necessary until I did a migration from Mongo to Postgres on CASE, and it took quite a long time to complete, I started to doubt that it was actually making progress.

Also includes a utility to help with tqdm in slow asynchronous for-loops.

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Make `async_tqdm` support async-for loops

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Reformat

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Remove unused method

* Remove quotes for the return type annotation

* Few style improvements

Co-authored-by: jack1142 <6032823+jack1142@users.noreply.github.com>
This commit is contained in:
Toby Harradine
2021-04-05 17:41:31 +10:00
committed by GitHub
parent 560e1dfc3d
commit 18a23813c8
2 changed files with 100 additions and 2 deletions

View File

@@ -2,6 +2,8 @@ import abc
import enum
from typing import Tuple, Dict, Any, Union, List, AsyncIterator, Type
from redbot.core.utils._internal_utils import async_tqdm
__all__ = ["BaseDriver", "IdentifierData", "ConfigCategory"]
@@ -280,12 +282,23 @@ class BaseDriver(abc.ABC):
"""
# Backend-agnostic method of migrating from one driver to another.
async for cog_name, cog_id in cls.aiter_cogs():
cogs_progress_bar = async_tqdm(
(tup async for tup in cls.aiter_cogs()),
desc="Migration progress",
unit=" cogs",
bar_format="{desc}: {n_fmt}{unit} [{elapsed},{rate_noinv_fmt}{postfix}]",
leave=False,
dynamic_ncols=True,
miniters=1,
)
async for cog_name, cog_id in cogs_progress_bar:
cogs_progress_bar.set_postfix_str(f"Working on {cog_name}...")
this_driver = cls(cog_name, cog_id)
other_driver = new_driver_cls(cog_name, cog_id)
custom_group_data = all_custom_group_data.get(cog_name, {}).get(cog_id, {})
exported_data = await this_driver.export_data(custom_group_data)
await other_driver.import_data(exported_data, custom_group_data)
print()
@classmethod
async def delete_all_data(cls, **kwargs) -> None: