diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 0e1f6a7cf..e4d9fe9a2 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -142,7 +142,14 @@ def main(): effective_host_platform = host_platform if building_linux_from_macos: if host_platform == "macos_arm64": - effective_host_platform = "linux_aarch64" + if target_triple.startswith("aarch64"): + effective_host_platform = "linux_aarch64" + elif target_triple.startswith("x86_64"): + effective_host_platform = "linux_x86_64" + else: + raise Exception( + f"unsupported macOS cross-compile: {host_platform} -> {target_triple}" + ) else: raise Exception(f"Unhandled macOS platform: {host_platform}") print( diff --git a/cpython-unix/build.py b/cpython-unix/build.py index c3bc0bbfa..7fd4a0298 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -25,7 +25,12 @@ meets_python_minimum_version, parse_setup_line, ) -from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles +from pythonbuild.docker import ( + build_docker_image, + docker_platform_from_host_platform, + get_image, + write_dockerfiles, +) from pythonbuild.downloads import DOWNLOADS from pythonbuild.logging import log, set_logger from pythonbuild.utils import ( @@ -33,6 +38,7 @@ add_licenses_to_extension_entry, clang_toolchain, create_tar_from_directory, + current_host_platform, download_entry, get_target_settings, get_targets, @@ -98,21 +104,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options) extra_target_ldflags.append("--rtlib=compiler-rt") if build_platform.startswith("linux_"): - machine = platform.machine() + # autoconf is not aware of microarch triples. Normalize those out: + # we force targeting via -march CFLAG. + env["TARGET_TRIPLE"] = ( + target_triple.replace("x86_64_v2-", "x86_64-") + .replace("x86_64_v3-", "x86_64-") + .replace("x86_64_v4-", "x86_64-") + ) - # arm64 allows building for Linux on a macOS host using Docker - if machine == "aarch64" or machine == "arm64": - env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" - env["TARGET_TRIPLE"] = target_triple - elif machine == "x86_64": - env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" - env["TARGET_TRIPLE"] = ( - target_triple.replace("x86_64_v2-", "x86_64-") - .replace("x86_64_v3-", "x86_64-") - .replace("x86_64_v4-", "x86_64-") - ) + # On macOS, we support building Linux in a virtualized container that + # always matches the target platform. Set build/host triple to whatever + # we're building. + # + # Note: we always use the *-gnu triple otherwise autoconf can have + # trouble reasoning about cross-compiling since its detected triple from + # our build environment is always GNU based. + if current_host_platform().startswith("macos_"): + env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu") else: - raise Exception("unhandled Linux machine value: %s" % machine) + # Otherwise assume the container environment matches the machine + # type of the current process. + host_machine = platform.machine() + if host_machine == "aarch64" or host_machine == "arm64": + env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" + elif host_machine == "x86_64": + env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + else: + raise Exception("unhandled Linux machine value: %s" % host_machine) # This will make x86_64_v2, etc count as cross-compiling. This is # semantically correct, since the current machine may not support @@ -969,16 +987,6 @@ def main(): DOWNLOADS_PATH.mkdir(exist_ok=True) (BUILD / "logs").mkdir(exist_ok=True) - if os.environ.get("PYBUILD_NO_DOCKER"): - client = None - else: - try: - client = docker.from_env(timeout=600) - client.ping() - except Exception as e: - print("unable to connect to Docker: %s" % e, file=sys.stderr) - return 1 - # Note these arguments must be synced with `build-main.py` parser = argparse.ArgumentParser() parser.add_argument( @@ -1040,6 +1048,17 @@ def main(): settings = get_target_settings(TARGETS_CONFIG, target_triple) + if os.environ.get("PYBUILD_NO_DOCKER"): + client = None + else: + try: + client = docker.from_env(timeout=600) + client.ping() + client._pbs_platform = docker_platform_from_host_platform(host_platform) + except Exception as e: + print("unable to connect to Docker: %s" % e, file=sys.stderr) + return 1 + if args.action == "dockerfiles": log_name = "dockerfiles" elif args.action == "makefiles": diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index c39f20baf..51fdbcd80 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -671,6 +671,7 @@ x86_64-unknown-linux-gnu: linux_uapi_include_arch: x86_64-linux-gnu host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -732,6 +733,7 @@ x86_64_v2-unknown-linux-gnu: linux_uapi_include_arch: x86_64-linux-gnu host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -794,6 +796,7 @@ x86_64_v3-unknown-linux-gnu: linux_uapi_include_arch: x86_64-linux-gnu host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -856,6 +859,9 @@ x86_64_v4-unknown-linux-gnu: linux_uapi_include_arch: x86_64-linux-gnu host_platforms: - linux_x86_64 + # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries + # under Rosetta. But they can build correctly. + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -917,6 +923,7 @@ x86_64_v4-unknown-linux-gnu: x86_64-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -972,6 +979,7 @@ x86_64-unknown-linux-musl: x86_64_v2-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -1028,6 +1036,7 @@ x86_64_v2-unknown-linux-musl: x86_64_v3-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -1084,6 +1093,9 @@ x86_64_v3-unknown-linux-musl: x86_64_v4-unknown-linux-musl: host_platforms: - linux_x86_64 + # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries + # under Rosetta. But they can build correctly. + - macos_arm64 pythons_supported: - '3.10' - '3.11' diff --git a/docs/building.rst b/docs/building.rst index 947871a83..ce49d2034 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -37,7 +37,8 @@ Linux The host system must be x86-64 or aarch64. The execution environment must have access to a Docker daemon (all build operations are performed in Docker containers for -isolation from the host system). +isolation from the host system). Docker Buildx must be installed and available +as ``docker buildx``. ``build.py`` accepts a ``--target-triple`` argument to support building for non-native targets (i.e. cross-compiling). @@ -76,9 +77,11 @@ The ``--target-triple`` argument can be used to build for an Intel Mac on an arm $ ./build.py --target-triple x86_64-apple-darwin -Additionally, an arm64 macOS host can be used to build Linux aarch64 targets using Docker:: +Additionally, an arm64 macOS host can be used to build Linux aarch64 and x86-64 +targets using Docker:: $ ./build.py --target-triple aarch64-unknown-linux-gnu + $ ./build.py --target-triple x86_64-unknown-linux-gnu The ``APPLE_SDK_PATH`` environment variable is recognized as the path to the Apple SDK to use. If not defined, the build will attempt to find @@ -104,4 +107,4 @@ You will need to specify the path to ``sh.exe`` from cygwin:: $ uv run --no-dev build.py --sh c:\cygwin\bin\sh.exe -To build a 32-bit x86 binary, simply use an ``x86 Native Tools Command Prompt`` instead of ``x64``. \ No newline at end of file +To build a 32-bit x86 binary, simply use an ``x86 Native Tools Command Prompt`` instead of ``x64``. diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index a946813ed..37019ca26 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -266,7 +266,10 @@ def find_output_files(self, base_path, pattern): def build_environment(client, image): if client is not None: container = client.containers.run( - image, command=["/bin/sleep", "86400"], detach=True + image, + command=["/bin/sleep", "86400"], + detach=True, + platform=client._pbs_platform, ) td = None context = ContainerContext(container) diff --git a/pythonbuild/docker.py b/pythonbuild/docker.py index 0be78e4f8..931c81f63 100644 --- a/pythonbuild/docker.py +++ b/pythonbuild/docker.py @@ -2,12 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -import contextlib import io import operator import os import pathlib +import subprocess +import sys import tarfile +import typing import docker # type: ignore import jinja2 @@ -29,41 +31,79 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path): write_if_different(dest_dir / f, data.encode("utf-8")) +def docker_platform_from_host_platform(host_platform: str) -> typing.Optional[str]: + """Convert a PBS host platform to a Docker platform. + + This function assumes we're using a Docker daemon on the local machine. + """ + # On macOS, we only use Docker to build Linux distros. We also support + # using Rosetta for x86-64 emulation when building from aarch64. So it + # is allowed to specify a non-native platform here. + if sys.platform == "darwin": + if host_platform == "linux_x86_64": + return "linux/amd64" + elif host_platform == "linux_aarch64": + return "linux/arm64" + else: + return None + else: + # On Linux, we currently only support running native architecture + # containers. Cross-compiling is achieved by running a native arch + # container and cross-compiling within it. We don't specify a platform + # here and let the Docker runtime use the defaults. + return None + + def build_docker_image( client, image_data: bytes, image_dir: pathlib.Path, name, host_platform ): image_path = image_dir / f"image-{name}.{host_platform}" - return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path) - - -def ensure_docker_image(client, fh, image_path=None): - res = client.api.build(fileobj=fh, decode=True) - - image = None - - for s in res: - if "stream" in s: - for l in s["stream"].strip().splitlines(): - log(l) - - if "aux" in s and "ID" in s["aux"]: - image = s["aux"]["ID"] - - if "error" in s: - log(s["error"]) - - if not image: - raise Exception("unable to determine built Docker image") + return ensure_docker_image( + client, + io.BytesIO(image_data), + image_path=image_path, + platform=docker_platform_from_host_platform(host_platform), + ) - if image_path: - tar_path = pathlib.Path(str(image_path) + ".tar") - with tar_path.open("wb") as fh: - for chunk in client.images.get(image).save(): - fh.write(chunk) - with image_path.open("w") as fh: - fh.write(image + "\n") +def ensure_docker_image(client, fh, image_path, platform=None): + args = [ + "docker", + "buildx", + "build", + "--progress=plain", + "--load", + f"--iidfile={image_path}", + ] + if platform: + args.append(f"--platform={platform}") + args.append("-") + + with subprocess.Popen( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + assert process.stdin is not None + assert process.stdout is not None + process.stdin.write(fh.read()) + process.stdin.close() + + for line in process.stdout: + log(line.rstrip(b"\r\n")) + + returncode = process.wait() + + if returncode: + raise subprocess.CalledProcessError(returncode, args) + + image = image_path.read_text().strip() + tar_path = pathlib.Path(str(image_path) + ".tar") + with tar_path.open("wb") as fh: + for chunk in client.images.get(image).save(): + fh.write(chunk) return image @@ -111,18 +151,6 @@ def copy_file_to_container(path, container, container_path, archive_path=None): container.put_archive(container_path, buf.getvalue()) -@contextlib.contextmanager -def run_container(client, image): - container = client.containers.run( - image, command=["/bin/sleep", "86400"], detach=True - ) - try: - yield container - finally: - container.stop(timeout=0) - container.remove() - - def container_exec(container, command, user="build", environment=None): # docker-py's exec_run() won't return the exit code. So we reinvent the # wheel.