unix: support building Linux x86-64 from macOS aarch64

cpython-unix/build-main.py

@@ -142,7 +142,14 @@ def main():
     effective_host_platform = host_platform
     if building_linux_from_macos:
         if host_platform == "macos_arm64":
-            effective_host_platform = "linux_aarch64"
+            if target_triple.startswith("aarch64"):
+                effective_host_platform = "linux_aarch64"
+            elif target_triple.startswith("x86_64"):
+                effective_host_platform = "linux_x86_64"
+            else:
+                raise Exception(
+                    f"unsupported macOS cross-compile: {host_platform} -> {target_triple}"
+                )
         else:
             raise Exception(f"Unhandled macOS platform: {host_platform}")
         print(

cpython-unix/build.py

@@ -25,14 +25,20 @@
     meets_python_minimum_version,
     parse_setup_line,
 )
-from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
+from pythonbuild.docker import (
+    build_docker_image,
+    docker_platform_from_host_platform,
+    get_image,
+    write_dockerfiles,
+)
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.logging import log, set_logger
 from pythonbuild.utils import (
     add_env_common,
     add_licenses_to_extension_entry,
     clang_toolchain,
     create_tar_from_directory,
+    current_host_platform,
     download_entry,
     get_target_settings,
     get_targets,
@@ -98,21 +104,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options)
         extra_target_ldflags.append("--rtlib=compiler-rt")
 
     if build_platform.startswith("linux_"):
-        machine = platform.machine()
+        # autoconf is not aware of microarch triples. Normalize those out:
+        # we force targeting via -march CFLAG.
+        env["TARGET_TRIPLE"] = (
+            target_triple.replace("x86_64_v2-", "x86_64-")
+            .replace("x86_64_v3-", "x86_64-")
+            .replace("x86_64_v4-", "x86_64-")
+        )
 
-        # arm64 allows building for Linux on a macOS host using Docker
-        if machine == "aarch64" or machine == "arm64":
-            env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = target_triple
-        elif machine == "x86_64":
-            env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = (
-                target_triple.replace("x86_64_v2-", "x86_64-")
-                .replace("x86_64_v3-", "x86_64-")
-                .replace("x86_64_v4-", "x86_64-")
-            )
+        # On macOS, we support building Linux in a virtualized container that
+        # always matches the target platform. Set build/host triple to whatever
+        # we're building.
+        #
+        # Note: we always use the *-gnu triple otherwise autoconf can have
+        # trouble reasoning about cross-compiling since its detected triple from
+        # our build environment is always GNU based.
+        if current_host_platform().startswith("macos_"):
+            env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu")
         else:
-            raise Exception("unhandled Linux machine value: %s" % machine)
+            # Otherwise assume the container environment matches the machine
+            # type of the current process.
+            host_machine = platform.machine()
+            if host_machine == "aarch64" or host_machine == "arm64":
+                env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
+            elif host_machine == "x86_64":
+                env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
+            else:
+                raise Exception("unhandled Linux machine value: %s" % host_machine)
 
         # This will make x86_64_v2, etc count as cross-compiling. This is
         # semantically correct, since the current machine may not support
@@ -969,16 +987,6 @@ def main():
     DOWNLOADS_PATH.mkdir(exist_ok=True)
     (BUILD / "logs").mkdir(exist_ok=True)
 
-    if os.environ.get("PYBUILD_NO_DOCKER"):
-        client = None
-    else:
-        try:
-            client = docker.from_env(timeout=600)
-            client.ping()
-        except Exception as e:
-            print("unable to connect to Docker: %s" % e, file=sys.stderr)
-            return 1
-
     # Note these arguments must be synced with `build-main.py`
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -1040,6 +1048,17 @@ def main():
 
     settings = get_target_settings(TARGETS_CONFIG, target_triple)
 
+    if os.environ.get("PYBUILD_NO_DOCKER"):
+        client = None
+    else:
+        try:
+            client = docker.from_env(timeout=600)
+            client.ping()
+            client._pbs_platform = docker_platform_from_host_platform(host_platform)
+        except Exception as e:
+            print("unable to connect to Docker: %s" % e, file=sys.stderr)
+            return 1
+
     if args.action == "dockerfiles":
         log_name = "dockerfiles"
     elif args.action == "makefiles":

cpython-unix/targets.yml

@@ -671,6 +671,7 @@ x86_64-unknown-linux-gnu:
   linux_uapi_include_arch: x86_64-linux-gnu
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -732,6 +733,7 @@ x86_64_v2-unknown-linux-gnu:
   linux_uapi_include_arch: x86_64-linux-gnu
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -794,6 +796,7 @@ x86_64_v3-unknown-linux-gnu:
   linux_uapi_include_arch: x86_64-linux-gnu
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -856,6 +859,9 @@ x86_64_v4-unknown-linux-gnu:
   linux_uapi_include_arch: x86_64-linux-gnu
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -917,6 +923,7 @@ x86_64_v4-unknown-linux-gnu:
 x86_64-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -972,6 +979,7 @@ x86_64-unknown-linux-musl:
 x86_64_v2-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -1028,6 +1036,7 @@ x86_64_v2-unknown-linux-musl:
 x86_64_v3-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -1084,6 +1093,9 @@ x86_64_v3-unknown-linux-musl:
 x86_64_v4-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'

docs/building.rst

@@ -37,7 +37,8 @@ Linux
 The host system must be x86-64 or aarch64. 
 The execution environment must have access to a Docker
 daemon (all build operations are performed in Docker containers for
-isolation from the host system).
+isolation from the host system). Docker Buildx must be installed and available
+as ``docker buildx``.
 
 ``build.py`` accepts a ``--target-triple`` argument to support building
 for non-native targets (i.e. cross-compiling). 
@@ -76,9 +77,11 @@ The ``--target-triple`` argument can be used to build for an Intel Mac on an arm
 
     $ ./build.py --target-triple x86_64-apple-darwin
 
-Additionally, an arm64 macOS host can be used to build Linux aarch64 targets using Docker::
+Additionally, an arm64 macOS host can be used to build Linux aarch64 and x86-64
+targets using Docker::
 
     $ ./build.py --target-triple aarch64-unknown-linux-gnu
+    $ ./build.py --target-triple x86_64-unknown-linux-gnu
 
 The ``APPLE_SDK_PATH`` environment variable is recognized as the path
 to the Apple SDK to use. If not defined, the build will attempt to find
@@ -104,4 +107,4 @@ You will need to specify the path to ``sh.exe`` from cygwin::
 
    $ uv run --no-dev build.py --sh c:\cygwin\bin\sh.exe
 
-To build a 32-bit x86 binary, simply use an ``x86 Native Tools Command Prompt`` instead of ``x64``.
+To build a 32-bit x86 binary, simply use an ``x86 Native Tools Command Prompt`` instead of ``x64``.

pythonbuild/buildenv.py

@@ -266,7 +266,10 @@ def find_output_files(self, base_path, pattern):
 def build_environment(client, image):
     if client is not None:
         container = client.containers.run(
-            image, command=["/bin/sleep", "86400"], detach=True
+            image,
+            command=["/bin/sleep", "86400"],
+            detach=True,
+            platform=client._pbs_platform,
         )
         td = None
         context = ContainerContext(container)

pythonbuild/docker.py

@@ -2,12 +2,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import contextlib
 import io
 import operator
 import os
 import pathlib
+import subprocess
+import sys
 import tarfile
+import typing
 
 import docker  # type: ignore
 import jinja2
@@ -29,41 +31,79 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path):
         write_if_different(dest_dir / f, data.encode("utf-8"))
 
 
+def docker_platform_from_host_platform(host_platform: str) -> typing.Optional[str]:
+    """Convert a PBS host platform to a Docker platform.
+
+    This function assumes we're using a Docker daemon on the local machine.
+    """
+    # On macOS, we only use Docker to build Linux distros. We also support
+    # using Rosetta for x86-64 emulation when building from aarch64. So it
+    # is allowed to specify a non-native platform here.
+    if sys.platform == "darwin":
+        if host_platform == "linux_x86_64":
+            return "linux/amd64"
+        elif host_platform == "linux_aarch64":
+            return "linux/arm64"
+        else:
+            return None
+    else:
+        # On Linux, we currently only support running native architecture
+        # containers. Cross-compiling is achieved by running a native arch
+        # container and cross-compiling within it. We don't specify a platform
+        # here and let the Docker runtime use the defaults.
+        return None
+
+
 def build_docker_image(
     client, image_data: bytes, image_dir: pathlib.Path, name, host_platform
 ):
     image_path = image_dir / f"image-{name}.{host_platform}"
 
-    return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path)
-
-
-def ensure_docker_image(client, fh, image_path=None):
-    res = client.api.build(fileobj=fh, decode=True)
-
-    image = None
-
-    for s in res:
-        if "stream" in s:
-            for l in s["stream"].strip().splitlines():
-                log(l)
-
-        if "aux" in s and "ID" in s["aux"]:
-            image = s["aux"]["ID"]
-
-        if "error" in s:
-            log(s["error"])
-
-    if not image:
-        raise Exception("unable to determine built Docker image")
+    return ensure_docker_image(
+        client,
+        io.BytesIO(image_data),
+        image_path=image_path,
+        platform=docker_platform_from_host_platform(host_platform),
+    )
 
-    if image_path:
-        tar_path = pathlib.Path(str(image_path) + ".tar")
-        with tar_path.open("wb") as fh:
-            for chunk in client.images.get(image).save():
-                fh.write(chunk)
 
-        with image_path.open("w") as fh:
-            fh.write(image + "\n")
+def ensure_docker_image(client, fh, image_path, platform=None):
+    args = [
+        "docker",
+        "buildx",
+        "build",
+        "--progress=plain",
+        "--load",
+        f"--iidfile={image_path}",
+    ]
+    if platform:
+        args.append(f"--platform={platform}")
+    args.append("-")
+
+    with subprocess.Popen(
+        args,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ) as process:
+        assert process.stdin is not None
+        assert process.stdout is not None
+        process.stdin.write(fh.read())
+        process.stdin.close()
+
+        for line in process.stdout:
+            log(line.rstrip(b"\r\n"))
+
+        returncode = process.wait()
+
+    if returncode:
+        raise subprocess.CalledProcessError(returncode, args)
+
+    image = image_path.read_text().strip()
+    tar_path = pathlib.Path(str(image_path) + ".tar")
+    with tar_path.open("wb") as fh:
+        for chunk in client.images.get(image).save():
+            fh.write(chunk)
 
     return image
 
@@ -111,18 +151,6 @@ def copy_file_to_container(path, container, container_path, archive_path=None):
     container.put_archive(container_path, buf.getvalue())
 
 
-@contextlib.contextmanager
-def run_container(client, image):
-    container = client.containers.run(
-        image, command=["/bin/sleep", "86400"], detach=True
-    )
-    try:
-        yield container
-    finally:
-        container.stop(timeout=0)
-        container.remove()
-
-
 def container_exec(container, command, user="build", environment=None):
     # docker-py's exec_run() won't return the exit code. So we reinvent the
     # wheel.