From 1fb6e795b621dafdf4c5fd6705c64b753c3f520c Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Tue, 25 Jun 2019 13:39:19 -0400 Subject: [PATCH] Script for new miniconda builds Scrapes available miniconda builds from anaconda repo --- plugins/python-build/scripts/add_miniconda.py | 262 ++++++++++++++++++ plugins/python-build/scripts/requirements.txt | 1 + 2 files changed, 263 insertions(+) create mode 100755 plugins/python-build/scripts/add_miniconda.py create mode 100644 plugins/python-build/scripts/requirements.txt diff --git a/plugins/python-build/scripts/add_miniconda.py b/plugins/python-build/scripts/add_miniconda.py new file mode 100755 index 00000000..04ae58b2 --- /dev/null +++ b/plugins/python-build/scripts/add_miniconda.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3.7 +"""Script to add non-"latest" miniconda releases. + +- Ignores releases below 4.3.30. +- Ignores sub-patch releases if that major.minor.patch already exists + - But otherwise, takes the latest sub-patch release for given OS/arch +- Assumes all miniconda3 releases default to python 3.6 (correct at time of writing) + +Use -d for dry run. +""" +import textwrap +from argparse import ArgumentParser +from collections import defaultdict +from enum import Enum +from functools import total_ordering +from pathlib import Path +from typing import NamedTuple, List, Optional, DefaultDict, Dict + +import requests_html + +CONDA_REPO = "https://repo.anaconda.com" +MINICONDA_REPO = CONDA_REPO + "/miniconda" +# ANACONDA_REPO = CONDA_REPO + "/archive" + +install_script_fmt = """ +case "$(anaconda_architecture 2>/dev/null || true)" in +{install_lines} +* ) + {{ echo + colorize 1 "ERROR" + echo ": The binary distribution of Miniconda is not available for $(anaconda_architecture 2>/dev/null || true)." + echo + }} >&2 + exit 1 + ;; +esac +""".lstrip() + +install_line_fmt = """ +"{os}-{arch}" ) + install_script "Miniconda{suffix}-{version_str}-{os}-{arch}" "{repo}/Miniconda{suffix}-{version_str}-{os}-{arch}.sh#{md5}" "miniconda" verify_{py_version} + ;; +""".strip() + +here = Path(__file__).resolve() +out_dir: Path = here.parent.parent / "share" / "python-build" + + +class StrEnum(str, Enum): + """Enum subclass which members are also instances of str + and directly comparable to strings. str type is forced at declaration. + + Adapted from https://github.com/kissgyorgy/enum34-custom/blob/dbc89596761c970398701d26c6a5bbcfcf70f548/enum_custom.py#L100 + (MIT license) + """ + def __new__(cls, *args): + for arg in args: + if not isinstance(arg, str): + raise TypeError('Not text %s:' % arg) + + return super(StrEnum, cls).__new__(cls, *args) + + def __str__(self): + return str(self.value) + + +class SupportedOS(StrEnum): + LINUX = "Linux" + MACOSX = "MacOSX" + + +class SupportedArch(StrEnum): + PPC64LE = "ppc64le" + X86_64 = "x86_64" + X86 = "x86" + + +class Suffix(StrEnum): + TWO = "2" + THREE = "3" + + +class PyVersion(StrEnum): + PY27 = "py27" + PY36 = "py36" + PY37 = "py37" + PY38 = "py38" + PY39 = "py39" + + def version(self): + first, *others = self.value[2:] + return f"{first}.{''.join(others)}" + + def version_info(self): + return tuple(int(n) for n in self.version().split('.')) + + +@total_ordering +class VersionStr(str): + def info(self): + return tuple(int(n) for n in self.split('.')) + + def __eq__(self, other): + return str(self) == str(other) + + def __lt__(self, other): + if isinstance(other, VersionStr): + return self.info() < other.info() + raise ValueError("VersionStr can only be compared to other VersionStr") + + @classmethod + def from_info(cls, version_info): + return VersionStr('.'.join(str(n) for n in version_info)) + + def __hash__(self): + return hash(str(self)) + + +class MinicondaVersion(NamedTuple): + suffix: Suffix + version_str: VersionStr + + @classmethod + def from_str(cls, s): + miniconda_n, ver = s.split('-') + return MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver)) + + def to_filename(self): + return f"miniconda{self.suffix}-{self.version_str}" + + def default_py_version(self): + if self.suffix == Suffix.TWO: + return PyVersion.PY27 + else: + # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html + return PyVersion.PY36 + + def with_version_triple(self): + return MinicondaVersion(self.suffix, VersionStr.from_info(self.version_str.info()[:3])) + + +class MinicondaSpec(NamedTuple): + version: MinicondaVersion + os: SupportedOS + arch: SupportedArch + md5: str + py_version: Optional[PyVersion] = None + + @classmethod + def from_filestem(cls, stem, md5, py_version=None): + miniconda_n, ver, os, arch = stem.split('-') + spec = MinicondaSpec( + MinicondaVersion( + Suffix(miniconda_n[-1]), + VersionStr(ver), + ), + SupportedOS(os), + SupportedArch(arch), + md5, + ) + if py_version is None: + spec = spec.with_py_version(spec.version.default_py_version()) + return spec + + def to_install_lines(self): + return install_line_fmt.format( + repo=MINICONDA_REPO, + suffix=self.version.suffix, + version_str=self.version.version_str, + os=self.os, + arch=self.arch, + md5=self.md5, + py_version=self.py_version, + ) + + def with_py_version(self, py_version: PyVersion): + return MinicondaSpec(*self[:-1], py_version=py_version) + + def with_version_triple(self): + version, *others = self + return MinicondaSpec(version.with_version_triple(), *others) + + +def make_script(specs: List[MinicondaSpec]): + install_lines = [s.to_install_lines() for s in specs] + return install_script_fmt.format(install_lines='\n'.join(install_lines)) + + +def get_existing_minicondas(): + for p in out_dir.iterdir(): + name = p.name + if not p.is_file() or not name.startswith("miniconda"): + continue + try: + v = MinicondaVersion.from_str(name) + if v.version_str != "latest": + yield v + except ValueError: + pass + + +def get_available_minicondas(): + session = requests_html.HTMLSession() + response = session.get(MINICONDA_REPO) + page: requests_html.HTML = response.html + table = page.find('table', first=True) + rows = table.find("tr")[1:] + for row in rows: + f, size, date, md5 = row.find('td') + fname = f.text + md5 = md5.text + + if not fname.endswith(".sh"): + continue + stem = fname[:-3] + + try: + s = MinicondaSpec.from_filestem(stem, md5) + if s.version.version_str != "latest": + yield s + except ValueError: + pass + + +def key_fn(spec: MinicondaSpec): + return ( + spec.version.version_str.info(), + spec.version.suffix.value, + spec.os.value, + spec.arch.value, + ) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("-d", "--dry_run", action="store_true") + parsed = parser.parse_args() + + existing_versions = set(get_existing_minicondas()) + available_specs = set(get_available_minicondas()) + + # version triple to triple-ified spec to raw spec + to_add: DefaultDict[MinicondaVersion, Dict[MinicondaSpec, MinicondaSpec]] = defaultdict(dict) + + for s in sorted(available_specs, key=key_fn): + key = s.version.with_version_triple() + if key in existing_versions or key.version_str.info() <= (4, 3, 30): + continue + + to_add[key][s.with_version_triple()] = s + + for ver, d in to_add.items(): + specs = list(d.values()) + fpath = out_dir / ver.to_filename() + script_str = make_script(specs) + + if parsed.dry_run: + print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, ' ')) + else: + with open(fpath, 'w') as f: + f.write(script_str) + diff --git a/plugins/python-build/scripts/requirements.txt b/plugins/python-build/scripts/requirements.txt new file mode 100644 index 00000000..a8fdb314 --- /dev/null +++ b/plugins/python-build/scripts/requirements.txt @@ -0,0 +1 @@ +requests-html