pyenv/plugins/python-build/scripts/add_miniconda.py

#!/usr/bin/env python3
"""Script to add non-"latest" miniconda releases.
Written for python 3.7.

Checks the miniconda download archives for new versions,
then writes a build script for any which do not exist locally,
saving it to plugins/python-build/share/python-build.

Ignores releases below 4.3.30.
Also ignores sub-patch releases if that major.minor.patch already exists,
but otherwise, takes the latest sub-patch release for given OS/arch.
Assumes all miniconda3 releases < 4.7 default to python 3.6, and anything else 3.7.
"""
import textwrap
from argparse import ArgumentParser
from collections import defaultdict
from enum import Enum
from functools import total_ordering
from pathlib import Path
from typing import NamedTuple, List, Optional, DefaultDict, Dict
import logging

import requests_html

logger = logging.getLogger(__name__)

CONDA_REPO = "https://repo.anaconda.com"
MINICONDA_REPO = CONDA_REPO + "/miniconda"
# ANACONDA_REPO = CONDA_REPO + "/archive"

install_script_fmt = """
case "$(anaconda_architecture 2>/dev/null || true)" in
{install_lines}
* )
  {{ echo
    colorize 1 "ERROR"
    echo ": The binary distribution of Miniconda is not available for $(anaconda_architecture 2>/dev/null || true)."
    echo
  }} >&2
  exit 1
  ;;
esac
""".lstrip()

install_line_fmt = """
"{os}-{arch}" )
  install_script "Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "miniconda" verify_{py_version}
  ;;
""".strip()

here = Path(__file__).resolve()
out_dir: Path = here.parent.parent / "share" / "python-build"


class StrEnum(str, Enum):
    """Enum subclass whose members are also instances of str
    and directly comparable to strings. str type is forced at declaration.

    Adapted from https://github.com/kissgyorgy/enum34-custom/blob/dbc89596761c970398701d26c6a5bbcfcf70f548/enum_custom.py#L100
    (MIT license)
    """

    def __new__(cls, *args):
        for arg in args:
            if not isinstance(arg, str):
                raise TypeError("Not text %s:" % arg)

        return super(StrEnum, cls).__new__(cls, *args)

    def __str__(self):
        return str(self.value)


class SupportedOS(StrEnum):
    LINUX = "Linux"
    MACOSX = "MacOSX"


class SupportedArch(StrEnum):
    PPC64LE = "ppc64le"
    X86_64 = "x86_64"
    X86 = "x86"


class Suffix(StrEnum):
    TWO = "2"
    THREE = "3"


class PyVersion(StrEnum):
    PY27 = "py27"
    PY36 = "py36"
    PY37 = "py37"
    PY38 = "py38"
    PY39 = "py39"

    def version(self):
        first, *others = self.value[2:]
        return f"{first}.{''.join(others)}"

    def version_info(self):
        return tuple(int(n) for n in self.version().split("."))


@total_ordering
class VersionStr(str):
    def info(self):
        return tuple(int(n) for n in self.split("."))

    def __eq__(self, other):
        return str(self) == str(other)

    def __lt__(self, other):
        if isinstance(other, VersionStr):
            return self.info() < other.info()
        raise ValueError("VersionStr can only be compared to other VersionStr")

    @classmethod
    def from_info(cls, version_info):
        return VersionStr(".".join(str(n) for n in version_info))

    def __hash__(self):
        return hash(str(self))


class MinicondaVersion(NamedTuple):
    suffix: Suffix
    version_str: VersionStr
    py_version: Optional[PyVersion]

    @classmethod
    def from_str(cls, s):
        components = s.split("-")
        if len(components) == 3:
            miniconda_n, py_ver, ver = components
            py_ver = PyVersion(f"py{py_ver.replace('.', '')}")
        else:
            miniconda_n, ver = components
            py_ver = None
        return MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver)

    def to_filename(self):
        if self.py_version:
            return f"miniconda{self.suffix}-{self.py_version.version()}-{self.version_str}"
        else:
            return f"miniconda{self.suffix}-{self.version_str}"

    def default_py_version(self):
        if self.py_version:
            return self.py_version
        elif self.suffix == Suffix.TWO:
            return PyVersion.PY27
        elif self.version_str.info() < (4, 7):
            # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html
            return PyVersion.PY36
        else:
            return PyVersion.PY37

    def with_version_triple(self):
        return MinicondaVersion(
            self.suffix, VersionStr.from_info(self.version_str.info()[:3]), self.py_version
        )


class MinicondaSpec(NamedTuple):
    version: MinicondaVersion
    os: SupportedOS
    arch: SupportedArch
    md5: str
    py_version: Optional[PyVersion] = None

    @classmethod
    def from_filestem(cls, stem, md5, py_version=None):
        miniconda_n, ver, os, arch = stem.split("-")
        if ver.startswith("py"):
            py_ver, ver = ver.split("_", maxsplit=1)
            py_ver = PyVersion(py_ver)
        else:
            py_ver = None
        spec = MinicondaSpec(
            MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver),
            SupportedOS(os),
            SupportedArch(arch),
            md5,
        )
        if py_version is None:
            spec = spec.with_py_version(spec.version.default_py_version())
        return spec

    def to_install_lines(self):
        return install_line_fmt.format(
            repo=MINICONDA_REPO,
            suffix=self.version.suffix,
            version_str=self.version.version_str,
            version_py_version=f"{self.version.py_version}_" if self.version.py_version else "",
            os=self.os,
            arch=self.arch,
            md5=self.md5,
            py_version=self.py_version,
        )

    def with_py_version(self, py_version: PyVersion):
        return MinicondaSpec(*self[:-1], py_version=py_version)

    def with_version_triple(self):
        version, *others = self
        return MinicondaSpec(version.with_version_triple(), *others)


def make_script(specs: List[MinicondaSpec]):
    install_lines = [s.to_install_lines() for s in specs]
    return install_script_fmt.format(install_lines="\n".join(install_lines))


def get_existing_minicondas():
    logger.info("Getting known miniconda versions")
    for p in out_dir.iterdir():
        name = p.name
        if not p.is_file() or not name.startswith("miniconda"):
            continue
        try:
            v = MinicondaVersion.from_str(name)
            if v.version_str != "latest":
                logger.debug("Found existing miniconda version %s", v)
                yield v
        except ValueError:
            pass


def get_available_minicondas():
    logger.info("Fetching remote miniconda versions")
    session = requests_html.HTMLSession()
    response = session.get(MINICONDA_REPO)
    page: requests_html.HTML = response.html
    table = page.find("table", first=True)
    rows = table.find("tr")[1:]
    for row in rows:
        f, size, date, md5 = row.find("td")
        fname = f.text
        md5 = md5.text

        if not fname.endswith(".sh"):
            continue
        stem = fname[:-3]

        try:
            s = MinicondaSpec.from_filestem(stem, md5)
            if s.version.version_str != "latest":
                logger.debug("Found remote miniconda version %s", s)
                yield s
        except ValueError:
            pass


def key_fn(spec: MinicondaSpec):
    return (
        spec.version.version_str.info(),
        spec.version.suffix.value,
        spec.os.value,
        spec.arch.value,
    )


if __name__ == "__main__":
    parser = ArgumentParser(description=__doc__)
    parser.add_argument(
        "-d", "--dry-run", action="store_true",
        help="Do not write scripts, just report them to stdout",
    )
    parser.add_argument(
        "-v", "--verbose", action="count", default=0,
        help="Increase verbosity of logging",
    )
    parsed = parser.parse_args()

    log_level = {
        0: logging.WARNING,
        1: logging.INFO,
        2: logging.DEBUG,
    }.get(parsed.verbose, logging.DEBUG)
    logging.basicConfig(level=log_level)
    if parsed.verbose < 3:
        logging.getLogger("requests").setLevel(logging.WARNING)

    existing_versions = set(get_existing_minicondas())
    available_specs = set(get_available_minicondas())

    # version triple to triple-ified spec to raw spec
    to_add: DefaultDict[
        MinicondaVersion, Dict[MinicondaSpec, MinicondaSpec]
    ] = defaultdict(dict)

    logger.info("Checking for new versions")
    for s in sorted(available_specs, key=key_fn):
        key = s.version.with_version_triple()
        if key in existing_versions or key.version_str.info() <= (4, 3, 30):
            logger.debug("Ignoring version %s (too old or already exists)", s)
            continue

        to_add[key][s.with_version_triple()] = s

    logger.info("Writing %s scripts", len(to_add))
    for ver, d in to_add.items():
        specs = list(d.values())
        fpath = out_dir / ver.to_filename()
        script_str = make_script(specs)
        logger.debug("Writing script for %s", ver)
        if parsed.dry_run:
            print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, "  "))
        else:
            with open(fpath, "w") as f:
                f.write(script_str)