Red-DiscordBot/tools/bumpdeps.py

#!/usr/bin/env python3.7
"""Script to bump pinned dependencies in setup.cfg.

This script aims to help update our list of pinned primary and
secondary dependencies in *setup.cfg*, using the unpinned primary
dependencies listed in *primary_deps.ini*.

This script will not work when run on Windows.

What this script does
---------------------
It prints to stdout all primary and secondary dependencies for Red,
pinned to the latest possible version, within the constraints specified
in ``primary_deps.ini``. The output should be suitable for copying and
pasting into ``setup.cfg``. PEP 508 markers are preserved.

How this script works
---------------------
Overview:
1. Primary dependencies are read from primary_deps.ini using
setuptools' config parser.
2. A clean virtual environment is created in a temporary directory.
3. Core primary dependencies are passed to the ``pip install`` command
for that virtual environment.
4. Pinned primary dependencies are obtained by reading the output of
``pip freeze`` in that virtual environment, and any PEP 508 markers
shown with the requirement in ``primary_deps.ini`` are preserved.
5. Steps 2-4 are repeated for each extra requirement, but care is taken
not to duplicate core dependencies (primary or secondary) in the final
pinned extra dependencies.

This script makes use of the *packaging* library to parse version
specifiers and environment markers.

Known Limitations
-----------------
These limitations don't stop this script from being helpful, but
hopefully help explain in which situations some dependencies may need
to be listed manually in ``setup.cfg``.

1. Whilst environment markers of any primary dependencies specified in
``primary_deps.ini`` are preserved in the output, they will not be
added to secondary dependencies. So for example, if some package
*dep1* has a dependency *dep2*, and *dep1* is listed as a primary
dependency in ``primary_deps.ini`` like follows::
    dep1; sys_platform == "linux"

Then the output will look like this::
    dep1==1.1.1; sys_platform == "linux"
    dep2==2.2.2

So even though ``dep1`` and its dependencies should only be installed on
Linux, in reality, its dependencies will be installed regardless. To
work around this, simply list the secondary dependencies in
``primary_deps.ini`` as well, with the environment markers.

2. If a core requirement and an extra requirement have a common
sub-dependency, there is a chance the sub-dependency will have a version
conflict unless it is manually held back. This script will issue a
warning to stderr when it thinks this might be happening.

3. Environment markers which exclude dependencies from the system
running this script will cause those dependencies to be excluded from
the output. So for example, if a dependency has the environment marker
``sys_platform == "darwin"``, and the script is being run on linux, then
this dependency will be ignored, and must be added to ``setup.cfg``
manually.
"""
import shlex
import sys
import subprocess as sp
import tempfile
import textwrap
import venv
from pathlib import Path
from typing import Sequence, Iterable, Dict

import packaging.requirements
import setuptools.config

THIS_DIRECTORY = Path(__file__).parent
REQUIREMENTS_INI_PTH: Path = THIS_DIRECTORY / "primary_deps.ini"

PIP_INSTALL_ARGS = ("install", "--upgrade")
PIP_FREEZE_ARGS = ("freeze", "--no-color")


def main() -> int:
    if not REQUIREMENTS_INI_PTH.is_file():
        print("No primary_deps.ini found in the same directory as bumpdeps.py", file=sys.stderr)
        return 1

    primary_reqs_cfg = setuptools.config.read_configuration(str(REQUIREMENTS_INI_PTH))

    print("[options]")
    print("install_requires =")
    core_primary_deps = primary_reqs_cfg["options"]["install_requires"]
    full_core_reqs = get_all_reqs(core_primary_deps)
    print(textwrap.indent("\n".join(map(str, full_core_reqs)), " " * 4))
    print()

    print("[options.extras_require]")
    for extra, extra_primary_deps in primary_reqs_cfg["options"]["extras_require"].items():
        print(extra, "=")
        full_extra_reqs = get_all_reqs(
            extra_primary_deps, all_core_deps={r.name.lower(): r for r in full_core_reqs}
        )
        print(textwrap.indent("\n".join(map(str, full_extra_reqs)), " " * 4))

    return 0


def get_all_reqs(
    primary_deps: Iterable[str], all_core_deps: Dict[str, packaging.requirements.Requirement] = ()
) -> Sequence[packaging.requirements.Requirement]:
    reqs_dict = {r.name.lower(): r for r in map(packaging.requirements.Requirement, primary_deps)}
    with tempfile.TemporaryDirectory() as tmpdir:
        venv.create(tmpdir, system_site_packages=False, clear=True, with_pip=True)
        tmpdir_pth = Path(tmpdir)

        pip_exe_pth = tmpdir_pth / "bin" / "pip"

        # Upgrade pip to latest version
        sp.run((pip_exe_pth, *PIP_INSTALL_ARGS, "pip"), stdout=sp.DEVNULL, check=True)

        # Install the primary dependencies
        sp.run(
            (pip_exe_pth, *PIP_INSTALL_ARGS, *map(str, reqs_dict.values())),
            stdout=sp.DEVNULL,
            check=True,
        )

        # Get pinned primary+secondary dependencies from pip freeze
        proc = sp.run(
            (pip_exe_pth, *PIP_FREEZE_ARGS), stdout=sp.PIPE, check=True, encoding="utf-8"
        )

        # Return Requirement objects
        ret = []
        for req_obj in map(packaging.requirements.Requirement, proc.stdout.strip().split("\n")):
            dep_name = req_obj.name.lower()
            # Don't include core dependencies if these are extra dependencies
            if dep_name in all_core_deps:
                if req_obj.specifier != all_core_deps[dep_name].specifier:
                    print(
                        f"[WARNING] {dep_name} is listed as both a core requirement and an extra "
                        f"requirement, and it's possible that their versions conflict!",
                        file=sys.stderr,
                    )
                continue

            # Preserve environment markers
            if dep_name in reqs_dict:
                req_obj.marker = reqs_dict[dep_name].marker

            ret.append(req_obj)

        return ret


if __name__ == "__main__":
    try:
        exit_code = main()
    except sp.CalledProcessError as exc:
        cmd = " ".join(map(lambda c: shlex.quote(str(c)), exc.cmd))
        print(
            f"The following command failed with code {exc.returncode}:\n    ", cmd, file=sys.stderr
        )
        exit_code = 1

    sys.exit(exit_code)