mirror of
https://github.com/pyenv/pyenv.git
synced 2024-12-23 16:21:40 +00:00
312 lines
9.2 KiB
Python
Executable file
312 lines
9.2 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Script to add non-"latest" miniconda releases.
|
|
Written for python 3.7.
|
|
|
|
Checks the miniconda download archives for new versions,
|
|
then writes a build script for any which do not exist locally,
|
|
saving it to plugins/python-build/share/python-build.
|
|
|
|
Ignores releases below 4.3.30.
|
|
Also ignores sub-patch releases if that major.minor.patch already exists,
|
|
but otherwise, takes the latest sub-patch release for given OS/arch.
|
|
Assumes all miniconda3 releases < 4.7 default to python 3.6, and anything else 3.7.
|
|
"""
|
|
import textwrap
|
|
from argparse import ArgumentParser
|
|
from collections import defaultdict
|
|
from enum import Enum
|
|
from functools import total_ordering
|
|
from pathlib import Path
|
|
from typing import NamedTuple, List, Optional, DefaultDict, Dict
|
|
import logging
|
|
|
|
import requests_html
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
CONDA_REPO = "https://repo.anaconda.com"
|
|
MINICONDA_REPO = CONDA_REPO + "/miniconda"
|
|
# ANACONDA_REPO = CONDA_REPO + "/archive"
|
|
|
|
install_script_fmt = """
|
|
case "$(anaconda_architecture 2>/dev/null || true)" in
|
|
{install_lines}
|
|
* )
|
|
{{ echo
|
|
colorize 1 "ERROR"
|
|
echo ": The binary distribution of Miniconda is not available for $(anaconda_architecture 2>/dev/null || true)."
|
|
echo
|
|
}} >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
""".lstrip()
|
|
|
|
install_line_fmt = """
|
|
"{os}-{arch}" )
|
|
install_script "Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "miniconda" verify_{py_version}
|
|
;;
|
|
""".strip()
|
|
|
|
here = Path(__file__).resolve()
|
|
out_dir: Path = here.parent.parent / "share" / "python-build"
|
|
|
|
|
|
class StrEnum(str, Enum):
|
|
"""Enum subclass whose members are also instances of str
|
|
and directly comparable to strings. str type is forced at declaration.
|
|
|
|
Adapted from https://github.com/kissgyorgy/enum34-custom/blob/dbc89596761c970398701d26c6a5bbcfcf70f548/enum_custom.py#L100
|
|
(MIT license)
|
|
"""
|
|
|
|
def __new__(cls, *args):
|
|
for arg in args:
|
|
if not isinstance(arg, str):
|
|
raise TypeError("Not text %s:" % arg)
|
|
|
|
return super(StrEnum, cls).__new__(cls, *args)
|
|
|
|
def __str__(self):
|
|
return str(self.value)
|
|
|
|
|
|
class SupportedOS(StrEnum):
|
|
LINUX = "Linux"
|
|
MACOSX = "MacOSX"
|
|
|
|
|
|
class SupportedArch(StrEnum):
|
|
PPC64LE = "ppc64le"
|
|
X86_64 = "x86_64"
|
|
X86 = "x86"
|
|
|
|
|
|
class Suffix(StrEnum):
|
|
TWO = "2"
|
|
THREE = "3"
|
|
|
|
|
|
class PyVersion(StrEnum):
|
|
PY27 = "py27"
|
|
PY36 = "py36"
|
|
PY37 = "py37"
|
|
PY38 = "py38"
|
|
PY39 = "py39"
|
|
|
|
def version(self):
|
|
first, *others = self.value[2:]
|
|
return f"{first}.{''.join(others)}"
|
|
|
|
def version_info(self):
|
|
return tuple(int(n) for n in self.version().split("."))
|
|
|
|
|
|
@total_ordering
|
|
class VersionStr(str):
|
|
def info(self):
|
|
return tuple(int(n) for n in self.split("."))
|
|
|
|
def __eq__(self, other):
|
|
return str(self) == str(other)
|
|
|
|
def __lt__(self, other):
|
|
if isinstance(other, VersionStr):
|
|
return self.info() < other.info()
|
|
raise ValueError("VersionStr can only be compared to other VersionStr")
|
|
|
|
@classmethod
|
|
def from_info(cls, version_info):
|
|
return VersionStr(".".join(str(n) for n in version_info))
|
|
|
|
def __hash__(self):
|
|
return hash(str(self))
|
|
|
|
|
|
class MinicondaVersion(NamedTuple):
|
|
suffix: Suffix
|
|
version_str: VersionStr
|
|
py_version: Optional[PyVersion]
|
|
|
|
@classmethod
|
|
def from_str(cls, s):
|
|
components = s.split("-")
|
|
if len(components) == 3:
|
|
miniconda_n, py_ver, ver = components
|
|
py_ver = PyVersion(f"py{py_ver.replace('.', '')}")
|
|
else:
|
|
miniconda_n, ver = components
|
|
py_ver = None
|
|
return MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver)
|
|
|
|
def to_filename(self):
|
|
if self.py_version:
|
|
return f"miniconda{self.suffix}-{self.py_version.version()}-{self.version_str}"
|
|
else:
|
|
return f"miniconda{self.suffix}-{self.version_str}"
|
|
|
|
def default_py_version(self):
|
|
if self.py_version:
|
|
return self.py_version
|
|
elif self.suffix == Suffix.TWO:
|
|
return PyVersion.PY27
|
|
elif self.version_str.info() < (4, 7):
|
|
# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html
|
|
return PyVersion.PY36
|
|
else:
|
|
return PyVersion.PY37
|
|
|
|
def with_version_triple(self):
|
|
return MinicondaVersion(
|
|
self.suffix, VersionStr.from_info(self.version_str.info()[:3]), self.py_version
|
|
)
|
|
|
|
|
|
class MinicondaSpec(NamedTuple):
|
|
version: MinicondaVersion
|
|
os: SupportedOS
|
|
arch: SupportedArch
|
|
md5: str
|
|
py_version: Optional[PyVersion] = None
|
|
|
|
@classmethod
|
|
def from_filestem(cls, stem, md5, py_version=None):
|
|
miniconda_n, ver, os, arch = stem.split("-")
|
|
if ver.startswith("py"):
|
|
py_ver, ver = ver.split("_", maxsplit=1)
|
|
py_ver = PyVersion(py_ver)
|
|
else:
|
|
py_ver = None
|
|
spec = MinicondaSpec(
|
|
MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver),
|
|
SupportedOS(os),
|
|
SupportedArch(arch),
|
|
md5,
|
|
)
|
|
if py_version is None:
|
|
spec = spec.with_py_version(spec.version.default_py_version())
|
|
return spec
|
|
|
|
def to_install_lines(self):
|
|
return install_line_fmt.format(
|
|
repo=MINICONDA_REPO,
|
|
suffix=self.version.suffix,
|
|
version_str=self.version.version_str,
|
|
version_py_version=f"{self.version.py_version}_" if self.version.py_version else "",
|
|
os=self.os,
|
|
arch=self.arch,
|
|
md5=self.md5,
|
|
py_version=self.py_version,
|
|
)
|
|
|
|
def with_py_version(self, py_version: PyVersion):
|
|
return MinicondaSpec(*self[:-1], py_version=py_version)
|
|
|
|
def with_version_triple(self):
|
|
version, *others = self
|
|
return MinicondaSpec(version.with_version_triple(), *others)
|
|
|
|
|
|
def make_script(specs: List[MinicondaSpec]):
|
|
install_lines = [s.to_install_lines() for s in specs]
|
|
return install_script_fmt.format(install_lines="\n".join(install_lines))
|
|
|
|
|
|
def get_existing_minicondas():
|
|
logger.info("Getting known miniconda versions")
|
|
for p in out_dir.iterdir():
|
|
name = p.name
|
|
if not p.is_file() or not name.startswith("miniconda"):
|
|
continue
|
|
try:
|
|
v = MinicondaVersion.from_str(name)
|
|
if v.version_str != "latest":
|
|
logger.debug("Found existing miniconda version %s", v)
|
|
yield v
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
def get_available_minicondas():
|
|
logger.info("Fetching remote miniconda versions")
|
|
session = requests_html.HTMLSession()
|
|
response = session.get(MINICONDA_REPO)
|
|
page: requests_html.HTML = response.html
|
|
table = page.find("table", first=True)
|
|
rows = table.find("tr")[1:]
|
|
for row in rows:
|
|
f, size, date, md5 = row.find("td")
|
|
fname = f.text
|
|
md5 = md5.text
|
|
|
|
if not fname.endswith(".sh"):
|
|
continue
|
|
stem = fname[:-3]
|
|
|
|
try:
|
|
s = MinicondaSpec.from_filestem(stem, md5)
|
|
if s.version.version_str != "latest":
|
|
logger.debug("Found remote miniconda version %s", s)
|
|
yield s
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
def key_fn(spec: MinicondaSpec):
|
|
return (
|
|
spec.version.version_str.info(),
|
|
spec.version.suffix.value,
|
|
spec.os.value,
|
|
spec.arch.value,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"-d", "--dry-run", action="store_true",
|
|
help="Do not write scripts, just report them to stdout",
|
|
)
|
|
parser.add_argument(
|
|
"-v", "--verbose", action="count", default=0,
|
|
help="Increase verbosity of logging",
|
|
)
|
|
parsed = parser.parse_args()
|
|
|
|
log_level = {
|
|
0: logging.WARNING,
|
|
1: logging.INFO,
|
|
2: logging.DEBUG,
|
|
}.get(parsed.verbose, logging.DEBUG)
|
|
logging.basicConfig(level=log_level)
|
|
if parsed.verbose < 3:
|
|
logging.getLogger("requests").setLevel(logging.WARNING)
|
|
|
|
existing_versions = set(get_existing_minicondas())
|
|
available_specs = set(get_available_minicondas())
|
|
|
|
# version triple to triple-ified spec to raw spec
|
|
to_add: DefaultDict[
|
|
MinicondaVersion, Dict[MinicondaSpec, MinicondaSpec]
|
|
] = defaultdict(dict)
|
|
|
|
logger.info("Checking for new versions")
|
|
for s in sorted(available_specs, key=key_fn):
|
|
key = s.version.with_version_triple()
|
|
if key in existing_versions or key.version_str.info() <= (4, 3, 30):
|
|
logger.debug("Ignoring version %s (too old or already exists)", s)
|
|
continue
|
|
|
|
to_add[key][s.with_version_triple()] = s
|
|
|
|
logger.info("Writing %s scripts", len(to_add))
|
|
for ver, d in to_add.items():
|
|
specs = list(d.values())
|
|
fpath = out_dir / ver.to_filename()
|
|
script_str = make_script(specs)
|
|
logger.debug("Writing script for %s", ver)
|
|
if parsed.dry_run:
|
|
print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, " "))
|
|
else:
|
|
with open(fpath, "w") as f:
|
|
f.write(script_str)
|