Merge pull request #2385 from native-api/condas

Add Anaconda 2019.10, 2021.04, 2022.05; support Anaconda in add_miniconda.py

.github/workflows/modified_scripts_build.yml

@@ -33,22 +33,31 @@ jobs:
       - uses: actions/checkout@v2
       - run: |
           brew install openssl openssl@1.1 readline sqlite3 xz zlib
-      # https://github.com/pyenv/pyenv#installation
+      - run: |
-      - run: pwd
+          export PYENV_ROOT="$GITHUB_WORKSPACE"
-      - env:
+          echo "PYENV_ROOT=$PYENV_ROOT" >> $GITHUB_ENV
-          PYENV_ROOT: /Users/runner/work/pyenv/pyenv
-        run: |
-          echo $PYENV_ROOT
           echo "$PYENV_ROOT/shims:$PYENV_ROOT/bin" >> $GITHUB_PATH
-          bin/pyenv install ${{ matrix.python-version }}
+      - run: |
-          bin/pyenv global ${{ matrix.python-version }}
+          pyenv install ${{ matrix.python-version }}
-          bin/pyenv rehash
+          pyenv global ${{ matrix.python-version }}
-      - run: python --version
+      - run: |
-      - run: python -m pip --version
+          python --version
+          python -m pip --version
       - shell: python  # Prove that actual Python == expected Python
         env:
           EXPECTED_PYTHON: ${{ matrix.python-version }}
-        run: import os, sys ; assert sys.version.startswith(os.getenv("EXPECTED_PYTHON"))
+        run: |
+          import os, sys, os.path
+          correct_dir = os.path.join(
+              os.environ['PYENV_ROOT'],
+              'versions',
+              os.environ['EXPECTED_PYTHON'],
+              'bin')
+          assert os.path.dirname(sys.executable) == correct_dir
+      # bundled executables in some Anaconda releases cause the post-run step to hang in MacOS
+      - run: |
+          pyenv global system
+          rm -f "$(pyenv root)"/shims/*
 
   ubuntu_build:
     needs: discover_modified_scripts
@@ -66,19 +75,23 @@ jobs:
             libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
             wget curl llvm libncurses5-dev libncursesw5-dev \
             xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
-      # https://github.com/pyenv/pyenv#installation
+      - run: |
-      - run: pwd
+          export PYENV_ROOT="$GITHUB_WORKSPACE"
-      - env:
+          echo "PYENV_ROOT=$PYENV_ROOT" >> $GITHUB_ENV
-          PYENV_ROOT: /home/runner/work/pyenv/pyenv
-        run: |
-          echo $PYENV_ROOT
           echo "$PYENV_ROOT/shims:$PYENV_ROOT/bin" >> $GITHUB_PATH
-          bin/pyenv install ${{ matrix.python-version }}
+      - run: |
-          bin/pyenv global ${{ matrix.python-version }}
+          pyenv install ${{ matrix.python-version }}
-          bin/pyenv rehash
+          pyenv global ${{ matrix.python-version }}
       - run: python --version
       - run: python -m pip --version
       - shell: python  # Prove that actual Python == expected Python
         env:
           EXPECTED_PYTHON: ${{ matrix.python-version }}
-        run: import os, sys ; assert sys.version.startswith(os.getenv("EXPECTED_PYTHON"))
+        run: |
+          import os, sys, os.path
+          correct_dir = os.path.join(
+              os.environ['PYENV_ROOT'],
+              'versions',
+              os.environ['EXPECTED_PYTHON'],
+              'bin')
+          assert os.path.dirname(sys.executable) == correct_dir

plugins/python-build/scripts/add_miniconda.py

@@ -19,6 +19,7 @@ from functools import total_ordering
 from pathlib import Path
 from typing import NamedTuple, List, Optional, DefaultDict, Dict
 import logging
+import string
 
 import requests_html
 
@@ -26,7 +27,7 @@ logger = logging.getLogger(__name__)
 
 CONDA_REPO = "https://repo.anaconda.com"
 MINICONDA_REPO = CONDA_REPO + "/miniconda"
-# ANACONDA_REPO = CONDA_REPO + "/archive"
+ANACONDA_REPO = CONDA_REPO + "/archive"
 
 install_script_fmt = """
 case "$(anaconda_architecture 2>/dev/null || true)" in
@@ -34,7 +35,7 @@ case "$(anaconda_architecture 2>/dev/null || true)" in
 * )
   {{ echo
     colorize 1 "ERROR"
-    echo ": The binary distribution of Miniconda is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo ": The binary distribution of {tflavor} is not available for $(anaconda_architecture 2>/dev/null || true)."
     echo
   }} >&2
   exit 1
@@ -44,7 +45,7 @@ esac
 
 install_line_fmt = """
 "{os}-{arch}" )
-  install_script "Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/Miniconda{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "miniconda" verify_{py_version}
+  install_script "{tflavor}{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/{tflavor}{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "{flavor}" verify_{py_version}
   ;;
 """.strip()
 
@@ -85,9 +86,20 @@ class SupportedArch(StrEnum):
     X86 = "x86"
 
 
+class Flavor(StrEnum):
+    ANACONDA = "anaconda"
+    MINICONDA = "miniconda"
+
+    
+class TFlavor(StrEnum):
+    ANACONDA = "Anaconda"
+    MINICONDA = "Miniconda"
+
+
 class Suffix(StrEnum):
     TWO = "2"
     THREE = "3"
+    NONE = ""
 
 
 class PyVersion(StrEnum):
@@ -126,7 +138,8 @@ class VersionStr(str):
         return hash(str(self))
 
 
-class MinicondaVersion(NamedTuple):
+class CondaVersion(NamedTuple):
+    flavor: Flavor
     suffix: Suffix
     version_str: VersionStr
     py_version: Optional[PyVersion]
@@ -134,7 +147,7 @@ class MinicondaVersion(NamedTuple):
     @classmethod
     def from_str(cls, s):
         """
-        Convert a string of the form "miniconda_n-ver" or "miniconda_n-py_ver-ver" to a :class:`MinicondaVersion` object.
+        Convert a string of the form "miniconda_n-ver" or "miniconda_n-py_ver-ver" to a :class:`CondaVersion` object.
         """
         components = s.split("-")
         if len(components) == 3:
@@ -143,13 +156,20 @@ class MinicondaVersion(NamedTuple):
         else:
             miniconda_n, ver = components
             py_ver = None
-        return MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver)
+
+        suffix = miniconda_n[-1]
+        if suffix in string.digits:
+            flavor = miniconda_n[:-1]
+        else:
+            flavor = miniconda_n
+            suffix = ""
+        return CondaVersion(Flavor(flavor), Suffix(suffix), VersionStr(ver), py_ver)
 
     def to_filename(self):
         if self.py_version:
-            return f"miniconda{self.suffix}-{self.py_version.version()}-{self.version_str}"
+            return f"{self.flavor}{self.suffix}-{self.py_version.version()}-{self.version_str}"
         else:
-            return f"miniconda{self.suffix}-{self.version_str}"
+            return f"{self.flavor}{self.suffix}-{self.version_str}"
 
     def default_py_version(self):
         """
@@ -159,38 +179,61 @@ class MinicondaVersion(NamedTuple):
             return self.py_version
         elif self.suffix == Suffix.TWO:
             return PyVersion.PY27
-        elif self.version_str.info() < (4, 7):
+
+        v = self.version_str.info()
+        if self.flavor == "miniconda":
             # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html
+            if v < (4, 7):
                 return PyVersion.PY36
             else:
                 return PyVersion.PY37
+        if self.flavor == "anaconda":
+            # https://docs.anaconda.com/anaconda/reference/release-notes/
+            if v >= (2021,11):
+                return PyVersion.PY39
+            if v >= (2020,7):
+                return PyVersion.PY38
+            if v >= (2020,2):
+                return PyVersion.PY37
+            if v >= (5,3,0):
+                return PyVersion.PY37
+            return PyVersion.PY36
 
-    def with_version_triple(self):
+        raise ValueError(flavor)
-        return MinicondaVersion(
-            self.suffix, VersionStr.from_info(self.version_str.info()[:3]), self.py_version
-        )
 
 
-class MinicondaSpec(NamedTuple):
+class CondaSpec(NamedTuple):
-    version: MinicondaVersion
+    tflavor: TFlavor
+    version: CondaVersion
     os: SupportedOS
     arch: SupportedArch
     md5: str
+    repo: str
     py_version: Optional[PyVersion] = None
 
     @classmethod
-    def from_filestem(cls, stem, md5, py_version=None):
+    def from_filestem(cls, stem, md5, repo, py_version=None):
         miniconda_n, ver, os, arch = stem.split("-")
+        suffix = miniconda_n[-1]
+        if suffix in string.digits:
+            tflavor = miniconda_n[:-1]
+        else:
+            tflavor = miniconda_n
+            suffix = ""
+        flavor = tflavor.lower()
+
         if ver.startswith("py"):
             py_ver, ver = ver.split("_", maxsplit=1)
             py_ver = PyVersion(py_ver)
         else:
             py_ver = None
-        spec = MinicondaSpec(
+        spec = CondaSpec(
-            MinicondaVersion(Suffix(miniconda_n[-1]), VersionStr(ver), py_ver),
+            TFlavor(tflavor),
+            CondaVersion(Flavor(flavor), Suffix(suffix), VersionStr(ver), py_ver),
             SupportedOS(os),
             SupportedArch(arch),
             md5,
+            repo,
         )
         if py_version is None:
             spec = spec.with_py_version(spec.version.default_py_version())
@@ -201,7 +244,9 @@ class MinicondaSpec(NamedTuple):
         Installation command for this version of Miniconda for use in a Pyenv installation script
         """
         return install_line_fmt.format(
-            repo=MINICONDA_REPO,
+            tflavor=self.tflavor,
+            flavor=self.version.flavor,
+            repo=self.repo,
             suffix=self.version.suffix,
             version_str=self.version.version_str,
             version_py_version=f"{self.version.py_version}_" if self.version.py_version else "",
@@ -212,48 +257,47 @@ class MinicondaSpec(NamedTuple):
         )
 
     def with_py_version(self, py_version: PyVersion):
-        return MinicondaSpec(*self[:-1], py_version=py_version)
+        return CondaSpec(*self[:-1], py_version=py_version)
-
-    def with_version_triple(self):
-        version, *others = self
-        return MinicondaSpec(version.with_version_triple(), *others)
 
 
-def make_script(specs: List[MinicondaSpec]):
+def make_script(specs: List[CondaSpec]):
     install_lines = [s.to_install_lines() for s in specs]
-    return install_script_fmt.format(install_lines="\n".join(install_lines))
+    return install_script_fmt.format(
+        install_lines="\n".join(install_lines),
+        tflavor=specs[0].tflavor,
+    )
 
 
-def get_existing_minicondas():
+def get_existing_condas(name):
     """
     Enumerate existing Miniconda installation scripts in share/python-build/ except rolling releases.
 
-    :returns: A generator of :class:`MinicondaVersion` objects.
+    :returns: A generator of :class:`CondaVersion` objects.
     """
-    logger.info("Getting known miniconda versions")
+    logger.info("Getting known %(name)s versions",locals())
     for p in out_dir.iterdir():
-        name = p.name
+        entry_name = p.name
-        if not p.is_file() or not name.startswith("miniconda"):
+        if not p.is_file() or not entry_name.startswith(name):
             continue
         try:
-            v = MinicondaVersion.from_str(name)
+            v = CondaVersion.from_str(entry_name)
             if v.version_str != "latest":
-                logger.debug("Found existing miniconda version %s", v)
+                logger.debug("Found existing %(name)s version %(v)s", locals())
                 yield v
         except ValueError:
             pass
 
 
-def get_available_minicondas():
+def get_available_condas(name, repo):
     """
     Fetch remote miniconda versions.
 
-    :returns: A generator of :class:`MinicondaSpec` objects for each release available for download
+    :returns: A generator of :class:`CondaSpec` objects for each release available for download
     except rolling releases.
     """
-    logger.info("Fetching remote miniconda versions")
+    logger.info("Fetching remote %(name)s versions",locals())
     session = requests_html.HTMLSession()
-    response = session.get(MINICONDA_REPO)
+    response = session.get(repo)
     page: requests_html.HTML = response.html
     table = page.find("table", first=True)
     rows = table.find("tr")[1:]
@@ -267,16 +311,17 @@ def get_available_minicondas():
         stem = fname[:-3]
 
         try:
-            s = MinicondaSpec.from_filestem(stem, md5)
+            s = CondaSpec.from_filestem(stem, md5, repo)
             if s.version.version_str != "latest":
-                logger.debug("Found remote miniconda version %s", s)
+                logger.debug("Found remote %(name)s version %(s)s", locals())
                 yield s
         except ValueError:
             pass
 
 
-def key_fn(spec: MinicondaSpec):
+def key_fn(spec: CondaSpec):
     return (
+        spec.tflavor,
         spec.version.version_str.info(),
         spec.version.suffix.value,
         spec.os.value,
@@ -305,29 +350,42 @@ if __name__ == "__main__":
     if parsed.verbose < 3:
         logging.getLogger("requests").setLevel(logging.WARNING)
 
-    existing_versions = set(get_existing_minicondas())
+    existing_versions = set()
-    available_specs = set(get_available_minicondas())
+    available_specs = set()
+    for name,repo in ("miniconda",MINICONDA_REPO),("anaconda",ANACONDA_REPO):
+        existing_versions |= set(get_existing_condas(name))
+        available_specs |= set(get_available_condas(name, repo))
 
     # version triple to triple-ified spec to raw spec
     to_add: DefaultDict[
-        MinicondaVersion, Dict[MinicondaSpec, MinicondaSpec]
+        CondaVersion, Dict[CondaSpec, CondaSpec]
     ] = defaultdict(dict)
 
     logger.info("Checking for new versions")
     for s in sorted(available_specs, key=key_fn):
-        key = s.version.with_version_triple()
+        key = s.version
-        if key in existing_versions or key.version_str.info() <= (4, 3, 30):
+        vv = key.version_str.info()
-            logger.debug("Ignoring version %s (too old or already exists)", s)
+        
+        reason = None
+        if key in existing_versions:
+            reason = "already exists"
+        elif key.version_str.info() <= (4, 3, 30):
+            reason = "too old"
+        elif len(key.version_str.info()) >= 4:
+            reason = "ignoring hotfix releases"
+        
+        if reason:
+            logger.debug("Ignoring version %(s)s (%(reason)s)", locals())
             continue
 
-        to_add[key][s.with_version_triple()] = s
+        to_add[key][s] = s
 
     logger.info("Writing %s scripts", len(to_add))
     for ver, d in to_add.items():
         specs = list(d.values())
         fpath = out_dir / ver.to_filename()
         script_str = make_script(specs)
-        logger.debug("Writing script for %s", ver)
+        logger.info("Writing script for %s", ver)
         if parsed.dry_run:
             print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, "  "))
         else:

plugins/python-build/share/python-build/anaconda2-2019.10

@@ -0,0 +1,19 @@
+case "$(anaconda_architecture 2>/dev/null || true)" in
+"Linux-ppc64le" )
+  install_script "Anaconda2-2019.10-Linux-ppc64le" "https://repo.anaconda.com/archive/Anaconda2-2019.10-Linux-ppc64le.sh#6b9809bf5d36782bfa1e35b791d983a0" "anaconda" verify_py27
+  ;;
+"Linux-x86_64" )
+  install_script "Anaconda2-2019.10-Linux-x86_64" "https://repo.anaconda.com/archive/Anaconda2-2019.10-Linux-x86_64.sh#69c64167b8cf3a8fc6b50d12d8476337" "anaconda" verify_py27
+  ;;
+"MacOSX-x86_64" )
+  install_script "Anaconda2-2019.10-MacOSX-x86_64" "https://repo.anaconda.com/archive/Anaconda2-2019.10-MacOSX-x86_64.sh#311aeb49cbe6d296f499efcd01a73f5e" "anaconda" verify_py27
+  ;;
+* )
+  { echo
+    colorize 1 "ERROR"
+    echo ": The binary distribution of Anaconda is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo
+  } >&2
+  exit 1
+  ;;
+esac

plugins/python-build/share/python-build/anaconda3-2021.04

@@ -0,0 +1,25 @@
+case "$(anaconda_architecture 2>/dev/null || true)" in
+"Linux-aarch64" )
+  install_script "Anaconda3-2021.04-Linux-aarch64" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-aarch64.sh#14f48f5d1310478b11940a3b96eec7b6" "anaconda" verify_py38
+  ;;
+"Linux-ppc64le" )
+  install_script "Anaconda3-2021.04-Linux-ppc64le" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-ppc64le.sh#e5c8220526b95293e669734f91194acc" "anaconda" verify_py38
+  ;;
+"Linux-s390x" )
+  install_script "Anaconda3-2021.04-Linux-s390x" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-s390x.sh#e61fac26bf61bc5c3e3c1a93abc4d8e2" "anaconda" verify_py38
+  ;;
+"Linux-x86_64" )
+  install_script "Anaconda3-2021.04-Linux-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-x86_64.sh#230f2c3c343ee58073bf41bd896dd76c" "anaconda" verify_py38
+  ;;
+"MacOSX-x86_64" )
+  install_script "Anaconda3-2021.04-MacOSX-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2021.04-MacOSX-x86_64.sh#3caed29ad5564b3567676504669342dc" "anaconda" verify_py38
+  ;;
+* )
+  { echo
+    colorize 1 "ERROR"
+    echo ": The binary distribution of Anaconda is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo
+  } >&2
+  exit 1
+  ;;
+esac

plugins/python-build/share/python-build/anaconda3-2022.05

@@ -0,0 +1,28 @@
+case "$(anaconda_architecture 2>/dev/null || true)" in
+"Linux-aarch64" )
+  install_script "Anaconda3-2022.05-Linux-aarch64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-aarch64.sh#7e822f5622fa306c0aa42430ba884454" "anaconda" verify_py39
+  ;;
+"Linux-ppc64le" )
+  install_script "Anaconda3-2022.05-Linux-ppc64le" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-ppc64le.sh#166b576c7e9d438b0a80840f94b44827" "anaconda" verify_py39
+  ;;
+"Linux-s390x" )
+  install_script "Anaconda3-2022.05-Linux-s390x" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-s390x.sh#00ba3bf29ac51db5e0954b6f217fa468" "anaconda" verify_py39
+  ;;
+"Linux-x86_64" )
+  install_script "Anaconda3-2022.05-Linux-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh#a01150aff48fcb6fcd6472381652de04" "anaconda" verify_py39
+  ;;
+"MacOSX-arm64" )
+  install_script "Anaconda3-2022.05-MacOSX-arm64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-MacOSX-arm64.sh#c35c8bdbeeda5e5ffa5b79d1f5ee8082" "anaconda" verify_py39
+  ;;
+"MacOSX-x86_64" )
+  install_script "Anaconda3-2022.05-MacOSX-x86_64" "https://repo.anaconda.com/archive/Anaconda3-2022.05-MacOSX-x86_64.sh#5319de6536212892dd2da8b70d602ee1" "anaconda" verify_py39
+  ;;
+* )
+  { echo
+    colorize 1 "ERROR"
+    echo ": The binary distribution of Anaconda is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo
+  } >&2
+  exit 1
+  ;;
+esac