PostgreSQL driver, tests against DB backends, and general drivers cleanup (#2723)

* PostgreSQL driver and general drivers cleanup

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Make tests pass

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Add black --target-version flag in make.bat

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Rewrite postgres driver

Most of the logic is now in PL/pgSQL.

This completely avoids the use of Python f-strings to format identifiers into queries. Although an SQL-injection attack would have been impossible anyway (only the owner would have ever had the ability to do that), using PostgreSQL's format() is more reliable for unusual identifiers. Performance-wise, I'm not sure whether this is an improvement, but I highly doubt that it's worse.

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Reformat

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Fix PostgresDriver.delete_all_data()

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Clean up PL/pgSQL code

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* More PL/pgSQL cleanup

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* PL/pgSQL function optimisations

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Ensure compatibility with PostgreSQL 10 and below

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* More/better docstrings for PG functions

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Fix typo in docstring

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Return correct value on toggle()

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Use composite type for PG function parameters

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Fix JSON driver's Config.clear_all()

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Correct description for Mongo tox recipe

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Fix linting errors

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Update dep specification after merging bumpdeps

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Add towncrier entries

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Update from merge

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Mention [postgres] extra in install docs

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Support more connection options and use better defaults

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Actually pass PG env vars in tox

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>

* Replace event trigger with manual DELETE queries

Signed-off-by: Toby Harradine <tobyharradine@gmail.com>
This commit is contained in:
Toby Harradine
2019-08-27 12:02:26 +10:00
committed by Michael H
parent 57fa29dd64
commit d1a46acc9a
34 changed files with 2282 additions and 843 deletions

View File

@@ -1,32 +1,21 @@
#!/usr/bin/env python3
import asyncio
import json
import logging
import os
import sys
import tarfile
from copy import deepcopy
from datetime import datetime as dt
from pathlib import Path
import logging
from typing import Dict, Any, Optional
import appdirs
import click
import redbot.logging
from redbot.core.cli import confirm
from redbot.core.data_manager import (
basic_config_default,
load_basic_configuration,
instance_name,
basic_config,
cog_data_path,
core_data_path,
storage_details,
)
from redbot.core.utils import safe_delete
from redbot.core import Config
from redbot.core.utils import safe_delete, create_backup as _create_backup
from redbot.core import config, data_manager, drivers
from redbot.core.drivers import BackendType, IdentifierData
from redbot.core.drivers.red_json import JSON
conversion_log = logging.getLogger("red.converter")
@@ -61,11 +50,11 @@ else:
def save_config(name, data, remove=False):
config = load_existing_config()
if remove and name in config:
config.pop(name)
_config = load_existing_config()
if remove and name in _config:
_config.pop(name)
else:
if name in config:
if name in _config:
print(
"WARNING: An instance already exists with this name. "
"Continuing will overwrite the existing instance config."
@@ -73,10 +62,10 @@ def save_config(name, data, remove=False):
if not confirm("Are you absolutely certain you want to continue (y/n)? "):
print("Not continuing")
sys.exit(0)
config[name] = data
_config[name] = data
with config_file.open("w", encoding="utf-8") as fs:
json.dump(config, fs, indent=4)
json.dump(_config, fs, indent=4)
def get_data_dir():
@@ -118,13 +107,14 @@ def get_data_dir():
def get_storage_type():
storage_dict = {1: "JSON", 2: "MongoDB"}
storage_dict = {1: "JSON", 2: "MongoDB", 3: "PostgreSQL"}
storage = None
while storage is None:
print()
print("Please choose your storage backend (if you're unsure, choose 1).")
print("1. JSON (file storage, requires no database).")
print("2. MongoDB")
print("3. PostgreSQL")
storage = input("> ")
try:
storage = int(storage)
@@ -158,21 +148,16 @@ def basic_setup():
default_data_dir = get_data_dir()
default_dirs = deepcopy(basic_config_default)
default_dirs = deepcopy(data_manager.basic_config_default)
default_dirs["DATA_PATH"] = str(default_data_dir.resolve())
storage = get_storage_type()
storage_dict = {1: BackendType.JSON, 2: BackendType.MONGO}
storage_dict = {1: BackendType.JSON, 2: BackendType.MONGO, 3: BackendType.POSTGRES}
storage_type: BackendType = storage_dict.get(storage, BackendType.JSON)
default_dirs["STORAGE_TYPE"] = storage_type.value
if storage_type == BackendType.MONGO:
from redbot.core.drivers.red_mongo import get_config_details
default_dirs["STORAGE_DETAILS"] = get_config_details()
else:
default_dirs["STORAGE_DETAILS"] = {}
driver_cls = drivers.get_driver_class(storage_type)
default_dirs["STORAGE_DETAILS"] = driver_cls.get_config_details()
name = get_name()
save_config(name, default_dirs)
@@ -193,130 +178,38 @@ def get_target_backend(backend) -> BackendType:
return BackendType.JSON
elif backend == "mongo":
return BackendType.MONGO
elif backend == "postgres":
return BackendType.POSTGRES
async def json_to_mongov2(instance):
instance_vals = instance_data[instance]
current_data_dir = Path(instance_vals["DATA_PATH"])
async def do_migration(
current_backend: BackendType, target_backend: BackendType
) -> Dict[str, Any]:
cur_driver_cls = drivers.get_driver_class(current_backend)
new_driver_cls = drivers.get_driver_class(target_backend)
cur_storage_details = data_manager.storage_details()
new_storage_details = new_driver_cls.get_config_details()
load_basic_configuration(instance)
await cur_driver_cls.initialize(**cur_storage_details)
await new_driver_cls.initialize(**new_storage_details)
from redbot.core.drivers import red_mongo
await config.migrate(cur_driver_cls, new_driver_cls)
storage_details = red_mongo.get_config_details()
await cur_driver_cls.teardown()
await new_driver_cls.teardown()
core_conf = Config.get_core_conf()
new_driver = red_mongo.Mongo(cog_name="Core", identifier="0", **storage_details)
core_conf.init_custom("CUSTOM_GROUPS", 2)
custom_group_data = await core_conf.custom("CUSTOM_GROUPS").all()
curr_custom_data = custom_group_data.get("Core", {}).get("0", {})
exported_data = await core_conf.driver.export_data(curr_custom_data)
conversion_log.info("Starting Core conversion...")
await new_driver.import_data(exported_data, curr_custom_data)
conversion_log.info("Core conversion complete.")
for p in current_data_dir.glob("cogs/**/settings.json"):
cog_name = p.parent.stem
if "." in cog_name:
# Garbage handler
continue
with p.open(mode="r", encoding="utf-8") as f:
cog_data = json.load(f)
for identifier, all_data in cog_data.items():
try:
conf = Config.get_conf(None, int(identifier), cog_name=cog_name)
except ValueError:
continue
new_driver = red_mongo.Mongo(
cog_name=cog_name, identifier=conf.driver.unique_cog_identifier, **storage_details
)
curr_custom_data = custom_group_data.get(cog_name, {}).get(identifier, {})
exported_data = await conf.driver.export_data(curr_custom_data)
conversion_log.info(f"Converting {cog_name} with identifier {identifier}...")
await new_driver.import_data(exported_data, curr_custom_data)
conversion_log.info("Cog conversion complete.")
return storage_details
return new_storage_details
async def mongov2_to_json(instance):
load_basic_configuration(instance)
core_path = core_data_path()
from redbot.core.drivers import red_json
core_conf = Config.get_core_conf()
new_driver = red_json.JSON(cog_name="Core", identifier="0", data_path_override=core_path)
core_conf.init_custom("CUSTOM_GROUPS", 2)
custom_group_data = await core_conf.custom("CUSTOM_GROUPS").all()
curr_custom_data = custom_group_data.get("Core", {}).get("0", {})
exported_data = await core_conf.driver.export_data(curr_custom_data)
conversion_log.info("Starting Core conversion...")
await new_driver.import_data(exported_data, curr_custom_data)
conversion_log.info("Core conversion complete.")
collection_names = await core_conf.driver.db.list_collection_names()
splitted_names = list(
filter(
lambda elem: elem[1] != "" and elem[0] != "Core",
[n.split(".") for n in collection_names],
)
)
ident_map = {} # Cogname: idents list
for cog_name, category in splitted_names:
if cog_name not in ident_map:
ident_map[cog_name] = set()
idents = await core_conf.driver.db[cog_name][category].distinct("_id.RED_uuid")
ident_map[cog_name].update(set(idents))
for cog_name, idents in ident_map.items():
for identifier in idents:
curr_custom_data = custom_group_data.get(cog_name, {}).get(identifier, {})
try:
conf = Config.get_conf(None, int(identifier), cog_name=cog_name)
except ValueError:
continue
exported_data = await conf.driver.export_data(curr_custom_data)
new_path = cog_data_path(raw_name=cog_name)
new_driver = red_json.JSON(cog_name, identifier, data_path_override=new_path)
conversion_log.info(f"Converting {cog_name} with identifier {identifier}...")
await new_driver.import_data(exported_data, curr_custom_data)
# cog_data_path(raw_name=cog_name)
conversion_log.info("Cog conversion complete.")
return {}
async def mongo_to_json(instance):
load_basic_configuration(instance)
from redbot.core.drivers.red_mongo import Mongo
m = Mongo("Core", "0", **storage_details())
async def mongov1_to_json() -> Dict[str, Any]:
await drivers.MongoDriver.initialize(**data_manager.storage_details())
m = drivers.MongoDriver("Core", "0")
db = m.db
collection_names = await db.list_collection_names()
for collection_name in collection_names:
if "." in collection_name:
# Fix for one of Zeph's problems
continue
elif collection_name == "Core":
c_data_path = core_data_path()
else:
c_data_path = cog_data_path(raw_name=collection_name)
c_data_path.mkdir(parents=True, exist_ok=True)
# Every cog name has its own collection
collection = db[collection_name]
async for document in collection.find():
@@ -329,16 +222,22 @@ async def mongo_to_json(instance):
continue
elif not str(cog_id).isdigit():
continue
driver = JSON(collection_name, cog_id, data_path_override=c_data_path)
driver = drivers.JsonDriver(collection_name, cog_id)
for category, value in document.items():
ident_data = IdentifierData(str(cog_id), category, (), (), {})
ident_data = IdentifierData(
str(collection_name), str(cog_id), category, tuple(), tuple(), 0
)
await driver.set(ident_data, value=value)
conversion_log.info("Cog conversion complete.")
await drivers.MongoDriver.teardown()
return {}
async def edit_instance():
instance_list = load_existing_config()
if not instance_list:
_instance_list = load_existing_config()
if not _instance_list:
print("No instances have been set up!")
return
@@ -346,18 +245,18 @@ async def edit_instance():
"You have chosen to edit an instance. The following "
"is a list of instances that currently exist:\n"
)
for instance in instance_list.keys():
for instance in _instance_list.keys():
print("{}\n".format(instance))
print("Please select one of the above by entering its name")
selected = input("> ")
if selected not in instance_list.keys():
if selected not in _instance_list.keys():
print("That isn't a valid instance!")
return
instance_data = instance_list[selected]
default_dirs = deepcopy(basic_config_default)
_instance_data = _instance_list[selected]
default_dirs = deepcopy(data_manager.basic_config_default)
current_data_dir = Path(instance_data["DATA_PATH"])
current_data_dir = Path(_instance_data["DATA_PATH"])
print("You have selected '{}' as the instance to modify.".format(selected))
if not confirm("Please confirm (y/n):"):
print("Ok, we will not continue then.")
@@ -383,68 +282,47 @@ async def edit_instance():
print("Your basic configuration has been edited")
async def create_backup(instance):
instance_vals = instance_data[instance]
if confirm("Would you like to make a backup of the data for this instance? (y/n)"):
load_basic_configuration(instance)
if instance_vals["STORAGE_TYPE"] == "MongoDB":
await mongo_to_json(instance)
print("Backing up the instance's data...")
backup_filename = "redv3-{}-{}.tar.gz".format(
instance, dt.utcnow().strftime("%Y-%m-%d %H-%M-%S")
)
pth = Path(instance_vals["DATA_PATH"])
if pth.exists():
backup_pth = pth.home()
backup_file = backup_pth / backup_filename
to_backup = []
exclusions = [
"__pycache__",
"Lavalink.jar",
os.path.join("Downloader", "lib"),
os.path.join("CogManager", "cogs"),
os.path.join("RepoManager", "repos"),
]
from redbot.cogs.downloader.repo_manager import RepoManager
repo_mgr = RepoManager()
await repo_mgr.initialize()
repo_output = []
for repo in repo_mgr._repos.values():
repo_output.append({"url": repo.url, "name": repo.name, "branch": repo.branch})
repo_filename = pth / "cogs" / "RepoManager" / "repos.json"
with open(str(repo_filename), "w") as f:
f.write(json.dumps(repo_output, indent=4))
instance_vals = {instance_name: basic_config}
instance_file = pth / "instance.json"
with open(str(instance_file), "w") as instance_out:
instance_out.write(json.dumps(instance_vals, indent=4))
for f in pth.glob("**/*"):
if not any(ex in str(f) for ex in exclusions):
to_backup.append(f)
with tarfile.open(str(backup_file), "w:gz") as tar:
for f in to_backup:
tar.add(str(f), recursive=False)
print("A backup of {} has been made. It is at {}".format(instance, backup_file))
async def remove_instance(instance):
await create_backup(instance)
instance_vals = instance_data[instance]
if instance_vals["STORAGE_TYPE"] == "MongoDB":
from redbot.core.drivers.red_mongo import Mongo
m = Mongo("Core", **instance_vals["STORAGE_DETAILS"])
db = m.db
collections = await db.collection_names(include_system_collections=False)
for name in collections:
collection = await db.get_collection(name)
await collection.drop()
async def create_backup(instance: str) -> None:
data_manager.load_basic_configuration(instance)
backend_type = get_current_backend(instance)
if backend_type == BackendType.MONGOV1:
await mongov1_to_json()
elif backend_type != BackendType.JSON:
await do_migration(backend_type, BackendType.JSON)
print("Backing up the instance's data...")
backup_fpath = await _create_backup()
if backup_fpath is not None:
print(f"A backup of {instance} has been made. It is at {backup_fpath}")
else:
pth = Path(instance_vals["DATA_PATH"])
safe_delete(pth)
print("Creating the backup failed.")
async def remove_instance(
instance,
interactive: bool = False,
drop_db: Optional[bool] = None,
remove_datapath: Optional[bool] = None,
):
data_manager.load_basic_configuration(instance)
if confirm("Would you like to make a backup of the data for this instance? (y/n)"):
await create_backup(instance)
backend = get_current_backend(instance)
if backend == BackendType.MONGOV1:
driver_cls = drivers.MongoDriver
else:
driver_cls = drivers.get_driver_class(backend)
await driver_cls.delete_all_data(interactive=interactive, drop_db=drop_db)
if interactive is True and remove_datapath is None:
remove_datapath = confirm("Would you like to delete the instance's entire datapath? (y/n)")
if remove_datapath is True:
data_path = data_manager.core_data_path().parent
safe_delete(data_path)
save_config(instance, {}, remove=True)
print("The instance {} has been removed\n".format(instance))
@@ -467,8 +345,7 @@ async def remove_instance_interaction():
print("That isn't a valid instance!")
return
await create_backup(selected)
await remove_instance(selected)
await remove_instance(selected, interactive=True)
@click.group(invoke_without_command=True)
@@ -483,38 +360,56 @@ def cli(ctx, debug):
@cli.command()
@click.argument("instance", type=click.Choice(instance_list))
def delete(instance):
@click.option("--no-prompt", default=False, help="Don't ask for user input during the process.")
@click.option(
"--drop-db",
type=bool,
default=None,
help=(
"Drop the entire database constaining this instance's data. Has no effect on JSON "
"instances. If this option and --no-prompt are omitted, you will be asked about this."
),
)
@click.option(
"--remove-datapath",
type=bool,
default=None,
help=(
"Remove this entire instance's datapath. If this option and --no-prompt are omitted, you "
"will be asked about this."
),
)
def delete(instance: str, no_prompt: Optional[bool], drop_db: Optional[bool]):
loop = asyncio.get_event_loop()
loop.run_until_complete(remove_instance(instance))
if no_prompt is None:
interactive = None
else:
interactive = not no_prompt
loop.run_until_complete(remove_instance(instance, interactive, drop_db))
@cli.command()
@click.argument("instance", type=click.Choice(instance_list))
@click.argument("backend", type=click.Choice(["json", "mongo"]))
@click.argument("backend", type=click.Choice(["json", "mongo", "postgres"]))
def convert(instance, backend):
current_backend = get_current_backend(instance)
target = get_target_backend(backend)
data_manager.load_basic_configuration(instance)
default_dirs = deepcopy(basic_config_default)
default_dirs = deepcopy(data_manager.basic_config_default)
default_dirs["DATA_PATH"] = str(Path(instance_data[instance]["DATA_PATH"]))
loop = asyncio.get_event_loop()
new_storage_details = None
if current_backend == BackendType.MONGOV1:
if target == BackendType.MONGO:
if target == BackendType.JSON:
new_storage_details = loop.run_until_complete(mongov1_to_json())
else:
raise RuntimeError(
"Please see conversion docs for updating to the latest mongo version."
)
elif target == BackendType.JSON:
new_storage_details = loop.run_until_complete(mongo_to_json(instance))
elif current_backend == BackendType.JSON:
if target == BackendType.MONGO:
new_storage_details = loop.run_until_complete(json_to_mongov2(instance))
elif current_backend == BackendType.MONGO:
if target == BackendType.JSON:
new_storage_details = loop.run_until_complete(mongov2_to_json(instance))
else:
new_storage_details = loop.run_until_complete(do_migration(current_backend, target))
if new_storage_details is not None:
default_dirs["STORAGE_TYPE"] = target.value
@@ -522,7 +417,9 @@ def convert(instance, backend):
save_config(instance, default_dirs)
conversion_log.info(f"Conversion to {target} complete.")
else:
conversion_log.info(f"Cannot convert {current_backend} to {target} at this time.")
conversion_log.info(
f"Cannot convert {current_backend.value} to {target.value} at this time."
)
if __name__ == "__main__":