Script for new miniconda builds

Scrapes available miniconda builds from anaconda repo

plugins/python-build/scripts/add_miniconda.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3.7
+"""Script to add non-"latest" miniconda releases.
+
+- Ignores releases below 4.3.30.
+- Ignores sub-patch releases if that major.minor.patch already exists
+  - But otherwise, takes the latest sub-patch release for given OS/arch
+- Assumes all miniconda3 releases default to python 3.6 (correct at time of writing)
+
+Use -d for dry run.
+"""
+import textwrap
+from argparse import ArgumentParser
+from collections import defaultdict
+from enum import Enum
+from functools import total_ordering
+from pathlib import Path
+from typing import NamedTuple, List, Optional, DefaultDict, Dict
+
+import requests_html
+
+CONDA_REPO = "https://repo.anaconda.com"
+MINICONDA_REPO = CONDA_REPO + "/miniconda"
+# ANACONDA_REPO = CONDA_REPO + "/archive"
+
+install_script_fmt = """
+case "$(anaconda_architecture 2>/dev/null || true)" in
+{install_lines}
+* )
+  {{ echo
+    colorize 1 "ERROR"
+    echo ": The binary distribution of Miniconda is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo
+  }} >&2
+  exit 1
+  ;;
+esac
+""".lstrip()
+
+install_line_fmt = """
+"{os}-{arch}" )
+  install_script "Miniconda{suffix}-{version_str}-{os}-{arch}" "{repo}/Miniconda{suffix}-{version_str}-{os}-{arch}.sh#{md5}" "miniconda" verify_{py_version}
+  ;;
+""".strip()
+
+here = Path(__file__).resolve()
+out_dir: Path = here.parent.parent / "share" / "python-build"
+
+
+class StrEnum(str, Enum):
+    """Enum subclass which members are also instances of str
+    and directly comparable to strings. str type is forced at declaration.
+
+    Adapted from https://github.com/kissgyorgy/enum34-custom/blob/dbc89596761c970398701d26c6a5bbcfcf70f548/enum_custom.py#L100
+    (MIT license)
+    """
+    def __new__(cls, *args):
+        for arg in args:
+            if not isinstance(arg, str):
+                raise TypeError('Not text %s:' % arg)
+
+        return super(StrEnum, cls).__new__(cls, *args)
+
+    def __str__(self):
+        return str(self.value)
+
+
+class SupportedOS(StrEnum):
+    LINUX = "Linux"
+    MACOSX = "MacOSX"
+
+
+class SupportedArch(StrEnum):
+    PPC64LE = "ppc64le"
+    X86_64 = "x86_64"
+    X86 = "x86"
+
+
+class Suffix(StrEnum):
+    TWO = "2"
+    THREE = "3"
+
+
+class PyVersion(StrEnum):
+    PY27 = "py27"
+    PY36 = "py36"
+    PY37 = "py37"
+    PY38 = "py38"
+    PY39 = "py39"
+
+    def version(self):
+        first, *others = self.value[2:]
+        return f"{first}.{''.join(others)}"
+
+    def version_info(self):
+        return tuple(int(n) for n in self.version().split('.'))
+
+
+@total_ordering
+class VersionStr(str):
+    def info(self):
+        return tuple(int(n) for n in self.split('.'))
+
+    def __eq__(self, other):
+        return str(self) == str(other)
+
+    def __lt__(self, other):
+        if isinstance(other, VersionStr):
+            return self.info() < other.info()
+        raise ValueError("VersionStr can only be compared to other VersionStr")
+
+    @classmethod
+    def from_info(cls, version_info):
+        return VersionStr('.'.join(str(n) for n in version_info))
+
+    def __hash__(self):
+        return hash(str(self))
+
+
+class MinicondaVersion(NamedTuple):
+    suffix: Suffix
+    version_str: VersionStr
+
+    @classmethod
+    def from_str(cls, s):
+        miniconda_n, ver = s.split('-')
+        return MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver))
+
+    def to_filename(self):
+        return f"miniconda{self.suffix}-{self.version_str}"
+
+    def default_py_version(self):
+        if self.suffix == Suffix.TWO:
+            return PyVersion.PY27
+        else:
+            # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html
+            return PyVersion.PY36
+
+    def with_version_triple(self):
+        return MinicondaVersion(self.suffix, VersionStr.from_info(self.version_str.info()[:3]))
+
+
+class MinicondaSpec(NamedTuple):
+    version: MinicondaVersion
+    os: SupportedOS
+    arch: SupportedArch
+    md5: str
+    py_version: Optional[PyVersion] = None
+
+    @classmethod
+    def from_filestem(cls, stem, md5, py_version=None):
+        miniconda_n, ver, os, arch = stem.split('-')
+        spec = MinicondaSpec(
+            MinicondaVersion(
+                Suffix(miniconda_n[-1]),
+                VersionStr(ver),
+            ),
+            SupportedOS(os),
+            SupportedArch(arch),
+            md5,
+        )
+        if py_version is None:
+            spec = spec.with_py_version(spec.version.default_py_version())
+        return spec
+
+    def to_install_lines(self):
+        return install_line_fmt.format(
+            repo=MINICONDA_REPO,
+            suffix=self.version.suffix,
+            version_str=self.version.version_str,
+            os=self.os,
+            arch=self.arch,
+            md5=self.md5,
+            py_version=self.py_version,
+        )
+
+    def with_py_version(self, py_version: PyVersion):
+        return MinicondaSpec(*self[:-1], py_version=py_version)
+
+    def with_version_triple(self):
+        version, *others = self
+        return MinicondaSpec(version.with_version_triple(), *others)
+
+
+def make_script(specs: List[MinicondaSpec]):
+    install_lines = [s.to_install_lines() for s in specs]
+    return install_script_fmt.format(install_lines='\n'.join(install_lines))
+
+
+def get_existing_minicondas():
+    for p in out_dir.iterdir():
+        name = p.name
+        if not p.is_file() or not name.startswith("miniconda"):
+            continue
+        try:
+            v = MinicondaVersion.from_str(name)
+            if v.version_str != "latest":
+                yield v
+        except ValueError:
+            pass
+
+
+def get_available_minicondas():
+    session = requests_html.HTMLSession()
+    response = session.get(MINICONDA_REPO)
+    page: requests_html.HTML = response.html
+    table = page.find('table', first=True)
+    rows = table.find("tr")[1:]
+    for row in rows:
+        f, size, date, md5 = row.find('td')
+        fname = f.text
+        md5 = md5.text
+
+        if not fname.endswith(".sh"):
+            continue
+        stem = fname[:-3]
+
+        try:
+            s = MinicondaSpec.from_filestem(stem, md5)
+            if s.version.version_str != "latest":
+                yield s
+        except ValueError:
+            pass
+
+
+def key_fn(spec: MinicondaSpec):
+    return (
+        spec.version.version_str.info(),
+        spec.version.suffix.value,
+        spec.os.value,
+        spec.arch.value,
+    )
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("-d", "--dry_run", action="store_true")
+    parsed = parser.parse_args()
+
+    existing_versions = set(get_existing_minicondas())
+    available_specs = set(get_available_minicondas())
+
+    # version triple to triple-ified spec to raw spec
+    to_add: DefaultDict[MinicondaVersion, Dict[MinicondaSpec, MinicondaSpec]] = defaultdict(dict)
+
+    for s in sorted(available_specs, key=key_fn):
+        key = s.version.with_version_triple()
+        if key in existing_versions or key.version_str.info() <= (4, 3, 30):
+            continue
+
+        to_add[key][s.with_version_triple()] = s
+
+    for ver, d in to_add.items():
+        specs = list(d.values())
+        fpath = out_dir / ver.to_filename()
+        script_str = make_script(specs)
+
+        if parsed.dry_run:
+            print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, '  '))
+        else:
+            with open(fpath, 'w') as f:
+                f.write(script_str)
+

plugins/python-build/scripts/requirements.txt

@@ -0,0 +1 @@
+requests-html