Merge pull request #640 from getsentry/indygreg-again-again

use indygreg pythons again, again

.github/workflows/build.yml

@@ -27,10 +27,15 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     steps:
     - uses: actions/checkout@v3
+    - run: |
+        # work around https://github.com/indygreg/python-build-standalone/issues/208
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install gnu-tar
+        echo "$(brew --prefix gnu-tar)/libexec/gnubin" >> "$GITHUB_PATH"
     - run: python3 -u docker/install-pythons --dest pythons
     - run: |
         echo "$PWD/pythons/cp310-cp310/bin" >> "$GITHUB_PATH"
         echo "$PWD/pythons/cp311-cp311/bin" >> "$GITHUB_PATH"
+        echo "$PWD/pythons/cp312-cp312/bin" >> "$GITHUB_PATH"
         echo "$PWD/venv/bin" >> "$GITHUB_PATH"
     - run: python3 -um venv venv && pip install -r docker/requirements.txt
     - run: python3 -um build --pypi-url https://pypi.devinfra.sentry.io

docker/Dockerfile

@@ -27,6 +27,7 @@ RUN : \
         uuid-dev \
         xz-utils \
         zlib1g-dev \
+        zstd \
     && rm -rf /var/lib/apt/lists/*
 
 # https://github.com/pypa/auditwheel/issues/229
@@ -45,7 +46,7 @@ RUN : \
 
 ENV \
     BUILD_IN_CONTAINER=1 \
-    PATH=/venv/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:$PATH \
+    PATH=/venv/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:/opt/python/cp312-cp312/bin:$PATH \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_NO_CACHE_DIR=1 \
     PIP_NO_WARN_ABOUT_ROOT_USER=0

docker/install-pythons

@@ -11,32 +11,46 @@ import subprocess
 import sys
 import tempfile
 
-# gsutil cp gs://sentry-dev-infra-assets/prebuilt-pythons/python-{3.10.8+0,3.11.0+0}.sha256sums -
+RELEASE = (
+    "https://github.com/indygreg/python-build-standalone/releases/download/20240107/"
+)
+# curl --silent --location https://github.com/indygreg/python-build-standalone/releases/download/20240107/SHA256SUMS | grep -E '(aarch64-apple-darwin-pgo\+lto-full|x86_64-apple-darwin-pgo\+lto-full|aarch64-unknown-linux-gnu-lto-full|x86_64-unknown-linux-gnu-pgo\+lto-full)' | grep -v 'cpython-3\.[89]'
 CHECKSUMS = """\
-503f2b82d52647c322af626052995cded0a23899cca94600aa80eb193e4b926a  python-3.10.9+0-macosx_12_0_arm64.tgz
-30c919dd9be7767488d7953946c6dd39d058571b85c280c7e33df2fb771149cc  python-3.10.9+0-macosx_12_0_x86_64.tgz
-2ca822fa3c09e515dcbdcf7c8f4b091cc47b5c935b44083f69fd572580f9ceca  python-3.10.9+0-manylinux_2_28_aarch64.tgz
-639fb9945dc8bccc821aff98595f877e0a50e273a1b352886c31908ce95e1098  python-3.10.9+0-manylinux_2_28_x86_64.tgz
-ffdf9ade5d7023921a175294248003ac95f4c4ee6babb5c3d7f9f7650146a96e  python-3.11.1+0-macosx_12_0_arm64.tgz
-951f381c4e286717b89a03bd8e217cb07f745727f101de21d4a048a08139ae3e  python-3.11.1+0-macosx_12_0_x86_64.tgz
-42fba60f08fffc41e2a2eed0763eb8df57c7e79a56809d883c73efdc3dbba0e2  python-3.11.1+0-manylinux_2_28_aarch64.tgz
-be999ab895e67ebf7b199108090c0c09a0fad6cd9f1034dd9bd15ba61f8ef05f  python-3.11.1+0-manylinux_2_28_x86_64.tgz
+d1a777a0688bafd2a62050c680508769d9b6c14779f64fee591f4e135c11e711  cpython-3.10.13+20240107-aarch64-apple-darwin-pgo+lto-full.tar.zst
+4e9fcb141a0c9af986f0819ab7a64c62ceb7b68f33df75753e669fc3d23a3412  cpython-3.10.13+20240107-aarch64-unknown-linux-gnu-lto-full.tar.zst
+b61f6f9cf0c35fd6df90b424e757a3bc1b483e8f8d8fadfa6c1ddd1a0c39c003  cpython-3.10.13+20240107-x86_64-apple-darwin-pgo+lto-full.tar.zst
+60e7ca89d37dd8a630a5525bda6143a66a3949c4f03c8319295ddb1d1023b425  cpython-3.10.13+20240107-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst
+c1f3dd13825906a5eae23ed8de9b653edb620568b2e0226eef3784eb1cce7eed  cpython-3.11.7+20240107-aarch64-apple-darwin-pgo+lto-full.tar.zst
+e066d3fb69162e401d2bb1f3c20798fde7c2fffcba0912d792e46d569b591ab3  cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-lto-full.tar.zst
+3f8caf73f2bfe22efa9666974c119727e163716e88af8ed3caa1e0ae5493de61  cpython-3.11.7+20240107-x86_64-apple-darwin-pgo+lto-full.tar.zst
+b7e19b262c19dfb82107e092ba3959b2da9b8bc53aafeb86727996afdb577221  cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst
+61e51e3490537b800fcefad718157cf775de41044e95aa538b63ab599f66f3a9  cpython-3.12.1+20240107-aarch64-apple-darwin-pgo+lto-full.tar.zst
+3621be2cd8b5686e10a022f04869911cad9197a3ef77b30879fe25e792d7c249  cpython-3.12.1+20240107-aarch64-unknown-linux-gnu-lto-full.tar.zst
+bf2b176b0426d7b4d4909c1b19bbb25b4893f9ebdc61e32df144df2b10dcc800  cpython-3.12.1+20240107-x86_64-apple-darwin-pgo+lto-full.tar.zst
+f267489a041daf4e523c03d32639de04ee59ca925dff49a8c3ce2f28a9f70a3b  cpython-3.12.1+20240107-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst
 """
-VERSIONS = ("3.10.9+0", "3.11.1+0")
-PLAT_TAG = {"linux": "manylinux", "darwin": "macosx"}
-PLAT_RE = re.compile(f"{PLAT_TAG[sys.platform]}_.+{platform.machine()}$")
+VERSIONS = ("3.10.13", "3.11.7", "3.12.1")
+ARCH_MAP = {"arm64": "aarch64"}
+ARCH = ARCH_MAP.get(platform.machine(), platform.machine())
+
+CLANG_PP = re.compile(r"\bclang\+\+")
+CLANG = re.compile(r"\bclang\b")
+
+
+def _must_sub(reg: re.Pattern[str], new: str, s: str) -> str:
+    after = reg.sub(new, s)
+    if after == s:
+        raise AssertionError(f"expected replacement by {reg} => {new}!")
+    return after
 
 
 def _checksum_url(version: str) -> tuple[str, str]:
     for line in CHECKSUMS.splitlines():
         sha256, filename = line.split()
-        base, _ = os.path.splitext(filename)
-        _, f_version, plat = base.split("-")
-        if version == f_version and PLAT_RE.match(plat):
-            return (
-                sha256,
-                f"https://storage.googleapis.com/sentry-dev-infra-assets/prebuilt-pythons/{filename}",
-            )
+        _, f_version_release, arch, _, plat, *_ = filename.split("-")
+        f_version, _ = f_version_release.split("+")
+        if version == f_version and sys.platform == plat and ARCH == arch:
+            return (sha256, f"{RELEASE}/{filename}")
     else:
         raise NotImplementedError(version, sys.platform, platform.machine())
 
@@ -65,9 +79,31 @@ def main() -> int:
                 raise AssertionError(f"checksum mismatch {sha256=} {expected=}")
 
             os.makedirs(dest, exist_ok=True)
-            tar_cmd = ("tar", "-C", dest, "--strip-components=1", "-xf", tgz_dest)
+            tar_cmd = (
+                "tar",
+                "-C",
+                dest,
+                "--strip-components=2",
+                "-xf",
+                tgz_dest,
+                "python/install",
+            )
             subprocess.check_call(tar_cmd)
 
+            # https://github.com/indygreg/python-build-standalone/issues/209
+            if sys.platform == "linux" and ARCH == "x86_64":
+                for fname in (
+                    f"{dest}/lib/python{major}.{minor}/config-{major}.{minor}-x86_64-linux-gnu/Makefile",
+                    f"{dest}/lib/python{major}.{minor}/_sysconfigdata__linux_x86_64-linux-gnu.py",
+                ):
+                    print(f"XXX: fixing up build metadata in {fname}")
+                    with open(fname) as f:
+                        contents = f.read()
+                    contents = _must_sub(CLANG_PP, "c++", contents)
+                    contents = _must_sub(CLANG, "cc", contents)
+                    with open(fname, "w") as f:
+                        f.write(contents)
+
             py = os.path.join(dest, "bin", "python3")
             subprocess.check_call((py, "-mensurepip"))
             subprocess.check_call(