From 64aacd5cfdcbf1d9b37f9fa1838a1d8d682b4f82 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Mon, 6 Jun 2022 01:07:46 +0300 Subject: [PATCH 1/3] add_miniconda: support Anaconda, support non-CPython scripts in scripts check --- .github/workflows/modified_scripts_build.yml | 46 +++-- plugins/python-build/scripts/add_miniconda.py | 158 ++++++++++++------ 2 files changed, 136 insertions(+), 68 deletions(-) diff --git a/.github/workflows/modified_scripts_build.yml b/.github/workflows/modified_scripts_build.yml index 3003f4fb..9750fc93 100644 --- a/.github/workflows/modified_scripts_build.yml +++ b/.github/workflows/modified_scripts_build.yml @@ -33,22 +33,27 @@ jobs: - uses: actions/checkout@v2 - run: | brew install openssl openssl@1.1 readline sqlite3 xz zlib - # https://github.com/pyenv/pyenv#installation - - run: pwd - - env: - PYENV_ROOT: /Users/runner/work/pyenv/pyenv - run: | + - run: | + echo "PYENV_ROOT=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - run: | echo $PYENV_ROOT echo "$PYENV_ROOT/shims:$PYENV_ROOT/bin" >> $GITHUB_PATH - bin/pyenv install ${{ matrix.python-version }} - bin/pyenv global ${{ matrix.python-version }} - bin/pyenv rehash + - run: | + pyenv install ${{ matrix.python-version }} + pyenv global ${{ matrix.python-version }} - run: python --version - run: python -m pip --version - shell: python # Prove that actual Python == expected Python env: EXPECTED_PYTHON: ${{ matrix.python-version }} - run: import os, sys ; assert sys.version.startswith(os.getenv("EXPECTED_PYTHON")) + run: | + import os, sys, os.path + correct_dir = os.path.join( + os.environ['PYENV_ROOT'], + 'versions', + os.environ['EXPECTED_PYTHON'], + 'bin') + assert os.path.dirname(sys.executable) == correct_dir ubuntu_build: needs: discover_modified_scripts @@ -66,19 +71,24 @@ jobs: libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \ wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git - # https://github.com/pyenv/pyenv#installation - - run: pwd - - env: - PYENV_ROOT: /home/runner/work/pyenv/pyenv - run: | + - run: | + echo "PYENV_ROOT=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - run: | echo $PYENV_ROOT echo "$PYENV_ROOT/shims:$PYENV_ROOT/bin" >> $GITHUB_PATH - bin/pyenv install ${{ matrix.python-version }} - bin/pyenv global ${{ matrix.python-version }} - bin/pyenv rehash + - run: | + pyenv install ${{ matrix.python-version }} + pyenv global ${{ matrix.python-version }} - run: python --version - run: python -m pip --version - shell: python # Prove that actual Python == expected Python env: EXPECTED_PYTHON: ${{ matrix.python-version }} - run: import os, sys ; assert sys.version.startswith(os.getenv("EXPECTED_PYTHON")) + run: | + import os, sys, os.path + correct_dir = os.path.join( + os.environ['PYENV_ROOT'], + 'versions', + os.environ['EXPECTED_PYTHON'], + 'bin') + assert os.path.dirname(sys.executable) == correct_dir diff --git a/plugins/python-build/scripts/add_miniconda.py b/plugins/python-build/scripts/add_miniconda.py index 195c9ecc..b53618bc 100755 --- a/plugins/python-build/scripts/add_miniconda.py +++ b/plugins/python-build/scripts/add_miniconda.py @@ -19,6 +19,7 @@ from functools import total_ordering from pathlib import Path from typing import NamedTuple, List, Optional, DefaultDict, Dict import logging +import string import requests_html @@ -26,7 +27,7 @@ logger = logging.getLogger(__name__) CONDA_REPO = "https://repo.anaconda.com" MINICONDA_REPO = CONDA_REPO + "/miniconda" -# ANACONDA_REPO = CONDA_REPO + "/archive" +ANACONDA_REPO = CONDA_REPO + "/archive" install_script_fmt = """ case "$(anaconda_architecture 2>/dev/null || true)" in @@ -34,7 +35,7 @@ case "$(anaconda_architecture 2>/dev/null || true)" in * ) {{ echo colorize 1 "ERROR" - echo ": The binary distribution of Miniconda is not available for $(anaconda_architecture 2>/dev/null || true)." + echo ": The binary distribution of {tflavor} is not available for $(anaconda_architecture 2>/dev/null || true)." echo }} >&2 exit 1 @@ -44,7 +45,7 @@ esac install_line_fmt = """ "{os}-{arch}" ) - install_script "Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "miniconda" verify_{py_version} + install_script "{tflavor}{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/{tflavor}{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "{flavor}" verify_{py_version} ;; """.strip() @@ -85,9 +86,20 @@ class SupportedArch(StrEnum): X86 = "x86" +class Flavor(StrEnum): + ANACONDA = "anaconda" + MINICONDA = "miniconda" + + +class TFlavor(StrEnum): + ANACONDA = "Anaconda" + MINICONDA = "Miniconda" + + class Suffix(StrEnum): TWO = "2" THREE = "3" + NONE = "" class PyVersion(StrEnum): @@ -126,7 +138,8 @@ class VersionStr(str): return hash(str(self)) -class MinicondaVersion(NamedTuple): +class CondaVersion(NamedTuple): + flavor: Flavor suffix: Suffix version_str: VersionStr py_version: Optional[PyVersion] @@ -134,7 +147,7 @@ class MinicondaVersion(NamedTuple): @classmethod def from_str(cls, s): """ - Convert a string of the form "miniconda_n-ver" or "miniconda_n-py_ver-ver" to a :class:`MinicondaVersion` object. + Convert a string of the form "miniconda_n-ver" or "miniconda_n-py_ver-ver" to a :class:`CondaVersion` object. """ components = s.split("-") if len(components) == 3: @@ -143,13 +156,20 @@ class MinicondaVersion(NamedTuple): else: miniconda_n, ver = components py_ver = None - return MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver) + + suffix = miniconda_n[-1] + if suffix in string.digits: + flavor = miniconda_n[:-1] + else: + flavor = miniconda_n + suffix = "" + return CondaVersion(Flavor(flavor), Suffix(suffix), VersionStr(ver), py_ver) def to_filename(self): if self.py_version: - return f"miniconda{self.suffix}-{self.py_version.version()}-{self.version_str}" + return f"{self.flavor}{self.suffix}-{self.py_version.version()}-{self.version_str}" else: - return f"miniconda{self.suffix}-{self.version_str}" + return f"{self.flavor}{self.suffix}-{self.version_str}" def default_py_version(self): """ @@ -159,38 +179,61 @@ class MinicondaVersion(NamedTuple): return self.py_version elif self.suffix == Suffix.TWO: return PyVersion.PY27 - elif self.version_str.info() < (4, 7): + + v = self.version_str.info() + if self.flavor == "miniconda": # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html + if v < (4, 7): + return PyVersion.PY36 + else: + return PyVersion.PY37 + if self.flavor == "anaconda": + # https://docs.anaconda.com/anaconda/reference/release-notes/ + if v >= (2021,11): + return PyVersion.PY39 + if v >= (2020,7): + return PyVersion.PY38 + if v >= (2020,2): + return PyVersion.PY37 + if v >= (5,3,0): + return PyVersion.PY37 return PyVersion.PY36 - else: - return PyVersion.PY37 - def with_version_triple(self): - return MinicondaVersion( - self.suffix, VersionStr.from_info(self.version_str.info()[:3]), self.py_version - ) + raise ValueError(flavor) -class MinicondaSpec(NamedTuple): - version: MinicondaVersion +class CondaSpec(NamedTuple): + tflavor: TFlavor + version: CondaVersion os: SupportedOS arch: SupportedArch md5: str + repo: str py_version: Optional[PyVersion] = None @classmethod - def from_filestem(cls, stem, md5, py_version=None): + def from_filestem(cls, stem, md5, repo, py_version=None): miniconda_n, ver, os, arch = stem.split("-") + suffix = miniconda_n[-1] + if suffix in string.digits: + tflavor = miniconda_n[:-1] + else: + tflavor = miniconda_n + suffix = "" + flavor = tflavor.lower() + if ver.startswith("py"): py_ver, ver = ver.split("_", maxsplit=1) py_ver = PyVersion(py_ver) else: py_ver = None - spec = MinicondaSpec( - MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver), + spec = CondaSpec( + TFlavor(tflavor), + CondaVersion(Flavor(flavor), Suffix(suffix), VersionStr(ver), py_ver), SupportedOS(os), SupportedArch(arch), md5, + repo, ) if py_version is None: spec = spec.with_py_version(spec.version.default_py_version()) @@ -201,7 +244,9 @@ class MinicondaSpec(NamedTuple): Installation command for this version of Miniconda for use in a Pyenv installation script """ return install_line_fmt.format( - repo=MINICONDA_REPO, + tflavor=self.tflavor, + flavor=self.version.flavor, + repo=self.repo, suffix=self.version.suffix, version_str=self.version.version_str, version_py_version=f"{self.version.py_version}_" if self.version.py_version else "", @@ -212,48 +257,47 @@ class MinicondaSpec(NamedTuple): ) def with_py_version(self, py_version: PyVersion): - return MinicondaSpec(*self[:-1], py_version=py_version) - - def with_version_triple(self): - version, *others = self - return MinicondaSpec(version.with_version_triple(), *others) + return CondaSpec(*self[:-1], py_version=py_version) -def make_script(specs: List[MinicondaSpec]): +def make_script(specs: List[CondaSpec]): install_lines = [s.to_install_lines() for s in specs] - return install_script_fmt.format(install_lines="\n".join(install_lines)) + return install_script_fmt.format( + install_lines="\n".join(install_lines), + tflavor=specs[0].tflavor, + ) -def get_existing_minicondas(): +def get_existing_condas(name): """ Enumerate existing Miniconda installation scripts in share/python-build/ except rolling releases. - :returns: A generator of :class:`MinicondaVersion` objects. + :returns: A generator of :class:`CondaVersion` objects. """ - logger.info("Getting known miniconda versions") + logger.info("Getting known %(name)s versions",locals()) for p in out_dir.iterdir(): - name = p.name - if not p.is_file() or not name.startswith("miniconda"): + entry_name = p.name + if not p.is_file() or not entry_name.startswith(name): continue try: - v = MinicondaVersion.from_str(name) + v = CondaVersion.from_str(entry_name) if v.version_str != "latest": - logger.debug("Found existing miniconda version %s", v) + logger.debug("Found existing %(name)s version %(v)s", locals()) yield v except ValueError: pass -def get_available_minicondas(): +def get_available_condas(name, repo): """ Fetch remote miniconda versions. - :returns: A generator of :class:`MinicondaSpec` objects for each release available for download + :returns: A generator of :class:`CondaSpec` objects for each release available for download except rolling releases. """ - logger.info("Fetching remote miniconda versions") + logger.info("Fetching remote %(name)s versions",locals()) session = requests_html.HTMLSession() - response = session.get(MINICONDA_REPO) + response = session.get(repo) page: requests_html.HTML = response.html table = page.find("table", first=True) rows = table.find("tr")[1:] @@ -267,16 +311,17 @@ def get_available_minicondas(): stem = fname[:-3] try: - s = MinicondaSpec.from_filestem(stem, md5) + s = CondaSpec.from_filestem(stem, md5, repo) if s.version.version_str != "latest": - logger.debug("Found remote miniconda version %s", s) + logger.debug("Found remote %(name)s version %(s)s", locals()) yield s except ValueError: pass -def key_fn(spec: MinicondaSpec): +def key_fn(spec: CondaSpec): return ( + spec.tflavor, spec.version.version_str.info(), spec.version.suffix.value, spec.os.value, @@ -305,29 +350,42 @@ if __name__ == "__main__": if parsed.verbose < 3: logging.getLogger("requests").setLevel(logging.WARNING) - existing_versions = set(get_existing_minicondas()) - available_specs = set(get_available_minicondas()) + existing_versions = set() + available_specs = set() + for name,repo in ("miniconda",MINICONDA_REPO),("anaconda",ANACONDA_REPO): + existing_versions |= set(get_existing_condas(name)) + available_specs |= set(get_available_condas(name, repo)) # version triple to triple-ified spec to raw spec to_add: DefaultDict[ - MinicondaVersion, Dict[MinicondaSpec, MinicondaSpec] + CondaVersion, Dict[CondaSpec, CondaSpec] ] = defaultdict(dict) logger.info("Checking for new versions") for s in sorted(available_specs, key=key_fn): - key = s.version.with_version_triple() - if key in existing_versions or key.version_str.info() <= (4, 3, 30): - logger.debug("Ignoring version %s (too old or already exists)", s) + key = s.version + vv = key.version_str.info() + + reason = None + if key in existing_versions: + reason = "already exists" + elif key.version_str.info() <= (4, 3, 30): + reason = "too old" + elif len(key.version_str.info()) >= 4: + reason = "ignoring hotfix releases" + + if reason: + logger.debug("Ignoring version %(s)s (%(reason)s)", locals()) continue - to_add[key][s.with_version_triple()] = s + to_add[key][s] = s logger.info("Writing %s scripts", len(to_add)) for ver, d in to_add.items(): specs = list(d.values()) fpath = out_dir / ver.to_filename() script_str = make_script(specs) - logger.debug("Writing script for %s", ver) + logger.info("Writing script for %s", ver) if parsed.dry_run: print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, " ")) else: From e9f95065adc96d4b25543ac08a406da1c02ebfed Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Mon, 6 Jun 2022 03:09:56 +0300 Subject: [PATCH 2/3] Add Anaconda 2019.10, 2021.04, 2022.05 --- .../share/python-build/anaconda2-2019.10 | 19 +++++++++++++ .../share/python-build/anaconda3-2021.04 | 25 +++++++++++++++++ .../share/python-build/anaconda3-2022.05 | 28 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 plugins/python-build/share/python-build/anaconda2-2019.10 create mode 100644 plugins/python-build/share/python-build/anaconda3-2021.04 create mode 100644 plugins/python-build/share/python-build/anaconda3-2022.05 diff --git a/plugins/python-build/share/python-build/anaconda2-2019.10 b/plugins/python-build/share/python-build/anaconda2-2019.10 new file mode 100644 index 00000000..96c0c824 --- /dev/null +++ b/plugins/python-build/share/python-build/anaconda2-2019.10 @@ -0,0 +1,19 @@ +case "$(anaconda_architecture 2>/dev/null || true)" in +"Linux-ppc64le" ) + install_script "Anaconda2-2019.10-Linux-ppc64le" "https://repo.anaconda.com/archive/Anaconda2-2019.10-Linux-ppc64le.sh#6b9809bf5d36782bfa1e35b791d983a0" "anaconda" verify_py27 + ;; +"Linux-x86_64" ) + install_script "Anaconda2-2019.10-Linux-x86_64" "https://repo.anaconda.com/archive/Anaconda2-2019.10-Linux-x86_64.sh#69c64167b8cf3a8fc6b50d12d8476337" "anaconda" verify_py27 + ;; +"MacOSX-x86_64" ) + install_script "Anaconda2-2019.10-MacOSX-x86_64" "https://repo.anaconda.com/archive/Anaconda2-2019.10-MacOSX-x86_64.sh#311aeb49cbe6d296f499efcd01a73f5e" "anaconda" verify_py27 + ;; +* ) + { echo + colorize 1 "ERROR" + echo ": The binary distribution of Anaconda is not available for $(anaconda_architecture 2>/dev/null || true)." + echo + } >&2 + exit 1 + ;; +esac diff --git a/plugins/python-build/share/python-build/anaconda3-2021.04 b/plugins/python-build/share/python-build/anaconda3-2021.04 new file mode 100644 index 00000000..5a59a848 --- /dev/null +++ b/plugins/python-build/share/python-build/anaconda3-2021.04 @@ -0,0 +1,25 @@ +case "$(anaconda_architecture 2>/dev/null || true)" in +"Linux-aarch64" ) + install_script "Anaconda3-2021.04-Linux-aarch64" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-aarch64.sh#14f48f5d1310478b11940a3b96eec7b6" "anaconda" verify_py38 + ;; +"Linux-ppc64le" ) + install_script "Anaconda3-2021.04-Linux-ppc64le" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-ppc64le.sh#e5c8220526b95293e669734f91194acc" "anaconda" verify_py38 + ;; +"Linux-s390x" ) + install_script "Anaconda3-2021.04-Linux-s390x" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-s390x.sh#e61fac26bf61bc5c3e3c1a93abc4d8e2" "anaconda" verify_py38 + ;; +"Linux-x86_64" ) + install_script "Anaconda3-2021.04-Linux-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-x86_64.sh#230f2c3c343ee58073bf41bd896dd76c" "anaconda" verify_py38 + ;; +"MacOSX-x86_64" ) + install_script "Anaconda3-2021.04-MacOSX-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2021.04-MacOSX-x86_64.sh#3caed29ad5564b3567676504669342dc" "anaconda" verify_py38 + ;; +* ) + { echo + colorize 1 "ERROR" + echo ": The binary distribution of Anaconda is not available for $(anaconda_architecture 2>/dev/null || true)." + echo + } >&2 + exit 1 + ;; +esac diff --git a/plugins/python-build/share/python-build/anaconda3-2022.05 b/plugins/python-build/share/python-build/anaconda3-2022.05 new file mode 100644 index 00000000..3fd8b100 --- /dev/null +++ b/plugins/python-build/share/python-build/anaconda3-2022.05 @@ -0,0 +1,28 @@ +case "$(anaconda_architecture 2>/dev/null || true)" in +"Linux-aarch64" ) + install_script "Anaconda3-2022.05-Linux-aarch64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-aarch64.sh#7e822f5622fa306c0aa42430ba884454" "anaconda" verify_py39 + ;; +"Linux-ppc64le" ) + install_script "Anaconda3-2022.05-Linux-ppc64le" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-ppc64le.sh#166b576c7e9d438b0a80840f94b44827" "anaconda" verify_py39 + ;; +"Linux-s390x" ) + install_script "Anaconda3-2022.05-Linux-s390x" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-s390x.sh#00ba3bf29ac51db5e0954b6f217fa468" "anaconda" verify_py39 + ;; +"Linux-x86_64" ) + install_script "Anaconda3-2022.05-Linux-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh#a01150aff48fcb6fcd6472381652de04" "anaconda" verify_py39 + ;; +"MacOSX-arm64" ) + install_script "Anaconda3-2022.05-MacOSX-arm64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-MacOSX-arm64.sh#c35c8bdbeeda5e5ffa5b79d1f5ee8082" "anaconda" verify_py39 + ;; +"MacOSX-x86_64" ) + install_script "Anaconda3-2022.05-MacOSX-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-MacOSX-x86_64.sh#5319de6536212892dd2da8b70d602ee1" "anaconda" verify_py39 + ;; +* ) + { echo + colorize 1 "ERROR" + echo ": The binary distribution of Anaconda is not available for $(anaconda_architecture 2>/dev/null || true)." + echo + } >&2 + exit 1 + ;; +esac From 42cace010b8c7684bafdc5a58d4456d59c816a8d Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Wed, 8 Jun 2022 20:36:10 +0300 Subject: [PATCH 3/3] CI: workaround MacOS jobs hanging for some Anaconda releases Shims from executables bundled with some older Anaconda releases cause MacOS script check jobs to hang at the end --- .github/workflows/modified_scripts_build.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/modified_scripts_build.yml b/.github/workflows/modified_scripts_build.yml index 9750fc93..15ca6497 100644 --- a/.github/workflows/modified_scripts_build.yml +++ b/.github/workflows/modified_scripts_build.yml @@ -34,15 +34,15 @@ jobs: - run: | brew install openssl openssl@1.1 readline sqlite3 xz zlib - run: | - echo "PYENV_ROOT=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - run: | - echo $PYENV_ROOT + export PYENV_ROOT="$GITHUB_WORKSPACE" + echo "PYENV_ROOT=$PYENV_ROOT" >> $GITHUB_ENV echo "$PYENV_ROOT/shims:$PYENV_ROOT/bin" >> $GITHUB_PATH - run: | pyenv install ${{ matrix.python-version }} pyenv global ${{ matrix.python-version }} - - run: python --version - - run: python -m pip --version + - run: | + python --version + python -m pip --version - shell: python # Prove that actual Python == expected Python env: EXPECTED_PYTHON: ${{ matrix.python-version }} @@ -54,6 +54,10 @@ jobs: os.environ['EXPECTED_PYTHON'], 'bin') assert os.path.dirname(sys.executable) == correct_dir + # bundled executables in some Anaconda releases cause the post-run step to hang in MacOS + - run: | + pyenv global system + rm -f "$(pyenv root)"/shims/* ubuntu_build: needs: discover_modified_scripts @@ -72,9 +76,8 @@ jobs: wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git - run: | - echo "PYENV_ROOT=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - run: | - echo $PYENV_ROOT + export PYENV_ROOT="$GITHUB_WORKSPACE" + echo "PYENV_ROOT=$PYENV_ROOT" >> $GITHUB_ENV echo "$PYENV_ROOT/shims:$PYENV_ROOT/bin" >> $GITHUB_PATH - run: | pyenv install ${{ matrix.python-version }}