Fix Halide/HalideHelpers CMake package dependency inversion

.github/workflows/testing-ecosystem.yml

@@ -6,8 +6,9 @@ on:
       - '**.h'
       - '**.c'
       - '**.cpp'
-      - 'CMakeLists.txt'
+      - '**/CMakeLists.txt'
       - '**.cmake'
+      - 'test/integration/**'
       - '.github/workflows/testing-ecosystem.yml'
 
 concurrency:
@@ -40,6 +41,8 @@ jobs:
             "clang-${LLVM_VERSION}" \
             curl \
             flatbuffers-compiler \
+            g++-aarch64-linux-gnu \
+            gcc-aarch64-linux-gnu \
             "libclang-${LLVM_VERSION}-dev" \
             libedit-dev \
             libflatbuffers-dev \
@@ -48,7 +51,48 @@ jobs:
             "liblld-${LLVM_VERSION}-dev" \
             "lld-${LLVM_VERSION}" \
             "llvm-${LLVM_VERSION}-dev" \
-            pybind11-dev
+            ninja-build \
+            pybind11-dev \
+            qemu-user
 
-      - name: Check CMake configures
-        run: cmake --preset ubuntu-release -DHalide_LLVM_ROOT="/usr/lib/llvm-${LLVM_VERSION}"
+      - name: Build and install Halide (shared)
+        run: |
+          cmake --preset ubuntu-release -G Ninja -B build-shared \
+            -DHalide_LLVM_ROOT="/usr/lib/llvm-${LLVM_VERSION}" \
+            -DWITH_TESTS=NO -DWITH_TUTORIALS=NO -DWITH_UTILS=NO
+          cmake --build build-shared -j "$(nproc)"
+          cmake --install build-shared --prefix "${{ github.workspace }}/pkg-install"
+
+      # A second, statically-linked build with serialization enabled (the
+      # default) is what actually exercises the FlatBuffers- and
+      # LLVM-dependency resolution paths in the installed HalideCompiler
+      # package: those dependencies only show up as PUBLIC/exported link
+      # requirements for a static Halide::Halide, not a shared one.
+      - name: Build and install Halide (static)
+        run: |
+          cmake --preset ubuntu-release -G Ninja -B build-static \
+            -DHalide_LLVM_ROOT="/usr/lib/llvm-${LLVM_VERSION}" \
+            -DWITH_TESTS=NO -DWITH_TUTORIALS=NO -DWITH_UTILS=NO \
+            -DBUILD_SHARED_LIBS=NO
+          cmake --build build-static -j "$(nproc)"
+          cmake --install build-static --prefix "${{ github.workspace }}/pkg-install"
+
+      # A prefix containing only the Halide (helpers) package, with no
+      # HalideCompiler at all, used to verify that forcing the JIT component
+      # while cross-compiling fails cleanly when no matching compiler package
+      # is available -- see cross_compile_aarch64_force_jit_fails_cleanly in
+      # test/integration/CMakeLists.txt.
+      - name: Prepare a Halide-helpers-only prefix
+        run: |
+          cp -R "${{ github.workspace }}/pkg-install" "${{ github.workspace }}/pkg-install-helpers-only"
+          rm -rf "${{ github.workspace }}/pkg-install-helpers-only/lib/cmake/HalideCompiler"
+
+      - name: Test the installed CMake package (test/integration)
+        env:
+          CMAKE_PREFIX_PATH: ${{ github.workspace }}/pkg-install
+          Halide_LLVM_ROOT: /usr/lib/llvm-${{ env.LLVM_VERSION }}
+        run: |
+          cmake -G Ninja -S test/integration -B build-integration \
+            -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
+            "-Dxc_HELPERS_ONLY_PREFIX=${{ github.workspace }}/pkg-install-helpers-only"
+          ctest --test-dir build-integration --output-on-failure -j "$(nproc)"

.gitignore

@@ -272,4 +272,4 @@ TAGS
 
 # TODO: these should become .cmake.in
 !packaging/common/HalideConfig.cmake
-!packaging/common/HalideHelpersConfig.cmake
+!packaging/common/HalideCompilerConfig.cmake

CMakeLists.txt

@@ -19,8 +19,8 @@ file(CONFIGURE
 )
 
 file(CONFIGURE
-    OUTPUT "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/HalideHelpersConfig.cmake"
-    CONTENT "set(HalideHelpers_FOUND 1)\n"
+    OUTPUT "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/HalideCompilerConfig.cmake"
+    CONTENT "set(HalideCompiler_FOUND 1)\n"
 )
 
 ##

apps/HelloBaremetal/cmake-external_project/CMakeLists.txt

@@ -30,8 +30,7 @@ ExternalProject_Add(
 # Build application with cross compiler,
 # where the generator executable built in Step 1 is imported and called
 
-# Import Halide. Instead of the package Halide, HalideHelpers is enough as we don't use JIT mode.
-find_package(HalideHelpers REQUIRED)
+find_package(Halide REQUIRED)
 
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..)
 

apps/HelloBaremetal/cmake-external_project/generator/CMakeLists.txt

@@ -13,7 +13,7 @@ set(CMAKE_CXX_EXTENSIONS NO)
 
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 
-find_package(Halide REQUIRED)  # We need Halide::Generator
+find_package(Halide REQUIRED)
 
 # Build generator
 add_halide_generator(${GEN_EXE} SOURCES ${SRC_DIR}/add_generator.cpp)

apps/HelloBaremetal/cmake-super_build/app/CMakeLists.txt

@@ -12,8 +12,7 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED YES)
 set(CMAKE_CXX_EXTENSIONS NO)
 
-# Import Halide. Instead of the package Halide, HalideHelpers is enough as we don't use JIT mode.
-find_package(HalideHelpers REQUIRED)
+find_package(Halide REQUIRED)
 
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 

apps/HelloBaremetal/cmake-super_build/generator/CMakeLists.txt

@@ -13,7 +13,7 @@ set(CMAKE_CXX_EXTENSIONS NO)
 
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 
-find_package(Halide REQUIRED)  # We need Halide::Generator
+find_package(Halide REQUIRED)
 
 # Build generator
 add_halide_generator(

apps/HelloBaremetal/cmake-twice/CMakeLists.txt

@@ -12,8 +12,7 @@ if (NOT DEFINED GEN_PACKAGE)
     set(GEN_PACKAGE "HelloBaremetal-add_generator")
 endif ()
 
-# Find Halide. Instead of the package Halide, HalideHelpers is enough as we don't use JIT mode.
-find_package(HalideHelpers REQUIRED)
+find_package(Halide REQUIRED)
 
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..)
 

apps/hannk/CMakeLists.txt

@@ -42,8 +42,8 @@ add_compile_definitions(HANNK_BUILD_TFLITE=$<BOOL:${HANNK_BUILD_TFLITE}>)  # nol
 
 # ----------------------------
 
-# Find HalideHelpers -- this is just the Runtime headers and CMake functions, but no libraries
-find_package(HalideHelpers REQUIRED)
+find_package(Halide REQUIRED)
+
 if (HANNK_BUILD_TFLITE)
     find_package(tensorflow-lite CONFIG REQUIRED)
 endif ()

apps/hannk/configure_cmake.sh

@@ -23,8 +23,7 @@ HANNK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 SOURCE_DIR="${HANNK_DIR}"
 CMAKE_DEFS=(
     -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
-    -DHalide_DIR="${HALIDE_INSTALL_PATH}/lib/cmake/Halide"
-    -DHalideHelpers_DIR="${HALIDE_INSTALL_PATH}/lib/cmake/HalideHelpers"
+    -DHalide_ROOT="${HALIDE_INSTALL_PATH}"
     -DHalide_TARGET="${HL_TARGET}"
 )
 
@@ -38,8 +37,7 @@ if [[ ! ${HL_TARGET} =~ ^host*|${HL_HOST_TARGET}* ]]; then
     SOURCE_DIR="${HANNK_DIR}/cmake/superbuild"
     CMAKE_DEFS=(
         "${CMAKE_DEFS[@]}"
-        -DHANNK_HOST_Halide_DIR="${HALIDE_INSTALL_PATH}/lib/cmake/Halide"
-        -DHANNK_HOST_HalideHelpers_DIR="${HALIDE_INSTALL_PATH}/lib/cmake/HalideHelpers"
+        -DHANNK_HOST_Halide_ROOT="${HALIDE_INSTALL_PATH}"
     )
 
     # Special settings for cross-compiling targets with known quirks

cmake/HalideGeneratorHelpers.cmake

@@ -119,7 +119,7 @@ function(add_halide_generator TARGET)
             add_executable(${gen} ALIAS ${TARGET})
 
             if (NOT TARGET Halide::Generator)
-                find_package(Halide REQUIRED)
+                find_package(HalideCompiler REQUIRED)
             endif ()
             target_link_libraries(${TARGET} PRIVATE Halide::Generator ${ARG_LINK_LIBRARIES})
 
@@ -186,6 +186,10 @@ function(add_halide_generator TARGET)
             VISIBILITY_INLINES_HIDDEN ON
             POSITION_INDEPENDENT_CODE ON
         )
+
+        if (NOT TARGET Halide::PyStubs)
+            find_package(HalideCompiler REQUIRED COMPONENTS Python)
+        endif ()
         target_link_libraries(
             ${TARGET}_pystub
             PRIVATE Halide::PyStubs Halide::Halide ${ARG_LINK_LIBRARIES}
@@ -790,6 +794,11 @@ function(add_halide_library TARGET)
 
     if (ARG_AUTOSCHEDULER)
         if ("${ARG_AUTOSCHEDULER}" MATCHES "::")
+            # Autoscheduler plugins are compiled targets that live in HalideCompiler;
+            # lazily load it if it hasn't already been (e.g. while cross-compiling).
+            if (NOT TARGET "${ARG_AUTOSCHEDULER}")
+                find_package(HalideCompiler REQUIRED)
+            endif ()
             if (NOT TARGET "${ARG_AUTOSCHEDULER}")
                 message(FATAL_ERROR "Autoscheduler ${ARG_AUTOSCHEDULER} does not exist.")
             endif ()

cmake/HalidePackageConfigHelpers.cmake

@@ -35,7 +35,16 @@
 # Helper for registering package dependencies
 
 function(_Halide_pkgdep PKG)
-    cmake_parse_arguments(PARSE_ARGV 1 ARG "" "" "PACKAGE_VARS")
+    # NAMESPACE lets the target-namespace substring match (below, in
+    # _Halide_install_pkgdeps) differ from the name used to find_package()
+    # this dependency -- some packages export targets under a namespace with
+    # different casing than their own package name (e.g. FlatBuffers exports
+    # flatbuffers::flatbuffers).
+    cmake_parse_arguments(PARSE_ARGV 1 ARG "" "NAMESPACE" "PACKAGE_VARS")
+
+    if (NOT ARG_NAMESPACE)
+        set(ARG_NAMESPACE "${PKG}")
+    endif ()
 
     set(code "")
     foreach (var IN LISTS ARG_PACKAGE_VARS)
@@ -53,8 +62,11 @@ function(_Halide_pkgdep PKG)
         )
     endif ()
 
+    set(key "pkgdeps_namespace[${PKG}]")
+
     set_property(DIRECTORY "${PROJECT_SOURCE_DIR}" APPEND PROPERTY pkgdeps "${PKG}")  # nolint
     set_property(DIRECTORY "${PROJECT_SOURCE_DIR}" PROPERTY "pkgdeps[${PKG}]" "${code}")  # nolint
+    set_property(DIRECTORY "${PROJECT_SOURCE_DIR}" PROPERTY "${key}" "${ARG_NAMESPACE}")  # nolint
 endfunction()
 
 ##
@@ -87,8 +99,9 @@ function(_Halide_install_pkgdeps)
     get_property(pkgdeps DIRECTORY "${PROJECT_SOURCE_DIR}" PROPERTY pkgdeps)
     foreach (dep IN LISTS pkgdeps)
         get_property(pkgcode DIRECTORY "${PROJECT_SOURCE_DIR}" PROPERTY "pkgdeps[${dep}]")
+        get_property(pkgns DIRECTORY "${PROJECT_SOURCE_DIR}" PROPERTY "pkgdeps_namespace[${dep}]")
         _Halide_install_code(
-            "if (target_cmake MATCHES \"${dep}::\")"
+            "if (target_cmake MATCHES \"${pkgns}::\")"
             "  file(APPEND \"${depFile}.in\""
             "       [===[${pkgcode}]===] \"\\n\")"
             "endif ()"

doc/HalideCMakePackage.md

@@ -253,6 +253,39 @@ try the static libs first then fall back to the shared libs.
 To ensure that the Python bindings are available, include the `Python`
 component.
 
+Finally, when [cross compiling](#cross-compiling),
+`find_package(Halide REQUIRED)` never pulls in the compiled compiler/JIT library
+(`Halide::Halide`, `Halide::Generator`, etc.) unless you explicitly ask for it.
+Add the `JIT` component (or `Python`) to force it to be loaded even while
+cross-compiling:
+
+```cmake
+find_package(Halide REQUIRED COMPONENTS JIT)
+```
+
+This looks for a `HalideCompiler` package matching your current (target)
+platform and fails with a normal `find_package` error if none is found.
+`HalideCompiler` is the name of the underlying platform-specific package that
+actually contains the compiled libraries; you generally don't need to
+`find_package(HalideCompiler)` directly, but its name is useful for
+`HalideCompiler_ROOT`/`-DHalideCompiler_DIR=...` when pointing CMake at a
+specific installed build.
+
+Note that `static`/`shared`, unlike `JIT`/`Python`, never force this load by
+themselves -- requesting one merely records your preference for whichever
+package eventually loads the compiled compiler (whether that's this same
+`find_package(Halide ...)` call, because you're not cross-compiling or also
+requested `JIT`/`Python`, or a later, unrelated one, such as the internal lookup
+`add_halide_generator` performs when it needs to build a generator). This
+preference is scoped to the current directory (and any subdirectories added
+after it), so independent parts of a project -- so long as neither is a
+subdirectory of the other -- can request different linkage without conflicting
+with each other. If one directory's `find_package` call ends up loading the
+compiled compiler before a subdirectory requests the other flavor, that's a real
+conflict (CMake can only load one flavor of `Halide::Halide` per directory
+scope) and fails cleanly with a descriptive error rather than silently keeping
+whichever flavor loaded first.
+
 ## Variables
 
 Variables that control package loading:
@@ -579,6 +612,11 @@ If you are writing new programs that use Halide, you might wish to use
 `add_halide_generator`. When using this helper, you are expected to build your
 project twice: once for your build host and again for your intended target.
 
+On the target-side build, a plain `find_package(Halide REQUIRED)` is all you
+need (no separate package name): it never pulls in the compiled compiler, and
+`add_halide_generator` will lazily load it under the hood only if it can't find
+a prebuilt host generators package to import instead.
+
 When building the host build, you can use the `<package-name>` (see the
 documentation above) target to build _just_ the generators. Then, in the target
 build, set `<package-name>_ROOT` to the host build directory.