From 676ef93a6d4056b1de988742e74b23a26274adcf Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Wed, 4 Nov 2026 16:48:51 -0800 Subject: [PATCH] Make the ESP-IDF kernel bundle at build time when Python has executorch When a project declares models but no kernel bundle, the component's configure step now checks whether EXECUTORCH_PYTHON (set with -D or in the environment; by default the Python ESP-IDF builds with) can import executorch. If it can, it runs gen_kernels.py on the declared models into the build directory and uses that bundle, so a project that exports and builds in one environment no longer has to run gen_kernels.py or keep a bundle in its tree. The result is byte-identical to a bundle made by hand, which CI checks by building the ESP32-S3 smoke consumer this way. Without such a Python, or if generation fails, the configure error names the Python, shows why, and prints the gen_kernels.py command. Authored with Claude Code. --- .ci/scripts/test_espressif_idf_component.sh | 24 ++++--- .../idf_component_smoke/CMakeLists.txt | 8 +-- .../idf_component_template/README.md | 8 +++ .../project_include.cmake | 64 +++++++++++++++++-- 4 files changed, 86 insertions(+), 18 deletions(-) diff --git a/.ci/scripts/test_espressif_idf_component.sh b/.ci/scripts/test_espressif_idf_component.sh index 9accef8151e..3074f2e9fa7 100755 --- a/.ci/scripts/test_espressif_idf_component.sh +++ b/.ci/scripts/test_espressif_idf_component.sh @@ -6,11 +6,11 @@ # LICENSE file in the root directory of this source tree. # Generates the ExecuTorch ESP-IDF component and a kernel bundle for the smoke -# model, builds the idf_component_smoke consumer for ESP32-S3 and, with the -# Cadence kernels too, for ESP32-C6, and runs the ESP32-S3 build on the -# emulator. Also checks that stale or mismatched bundles fail at configure time, -# and that an app declaring nothing links to the same size as one without the -# component. +# model, builds the idf_component_smoke consumer for ESP32-C6 with that bundle, +# plus the Cadence kernels, and for ESP32-S3 with one the build generates +# itself, and runs the ESP32-S3 build on the emulator. Also checks that stale or +# mismatched bundles fail at configure time, and that an app declaring nothing +# links to the same size as one without the component. set -euo pipefail @@ -126,7 +126,12 @@ expect_configure_error commit-mismatch component_commit \ -DET_SMOKE_PTE="${build_dir}/smoke.pte" \ -DET_SMOKE_KERNELS="${build_dir}/mismatch_kernels" -expect_configure_error no-bundle "no kernel bundle is" -DET_SMOKE_PTE="${build_dir}/smoke.pte" +# Without an ExecuTorch Python to generate one, a missing bundle is an error. +python -m venv --without-pip --clear "${build_dir}/bare_python" +EXECUTORCH_PYTHON="${build_dir}/bare_python/bin/python" expect_configure_error \ + no-bundle "cannot import executorch" -DET_SMOKE_PTE="${build_dir}/smoke.pte" +expect_configure_error relative-python "must be the absolute path" \ + -DET_SMOKE_PTE="${build_dir}/smoke.pte" -DEXECUTORCH_PYTHON=python # A model the bundle was not made from, declared instead of the one it was. mkdir -p "${build_dir}/other" @@ -211,11 +216,14 @@ if ! tr -s ' \n' ' ' < "${build_dir}/firmware-nano.log" \ exit 1 fi -consumer smoke-esp32s3 esp32s3 -DET_COMPONENT_DIR="${component_dir}" \ - -DET_SMOKE_PTE="${build_dir}/smoke.pte" -DET_SMOKE_KERNELS="${kernels_dir}" build consumer smoke-esp32c6 esp32c6 -DET_COMPONENT_DIR="${component_dir}" \ "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/large_registry.defaults" \ -DET_SMOKE_PTE="${build_dir}/smoke.pte" -DET_SMOKE_KERNELS="${build_dir}/wakeword_kernels" build +# No bundle declared: the build makes one with its own Python, which has +# executorch here, and it must match the one gen_kernels.py made above. +consumer smoke-esp32s3 esp32s3 -DET_COMPONENT_DIR="${component_dir}" \ + -DET_SMOKE_PTE="${build_dir}/smoke.pte" build +diff -r "${build_dir}/firmware-smoke-esp32s3/executorch_kernels" "${kernels_dir}" firmware_dir="${build_dir}/firmware-smoke-esp32s3" idf.py -C "${project}" -B "${firmware_dir}" merge-bin \ diff --git a/examples/espressif/idf_component_smoke/CMakeLists.txt b/examples/espressif/idf_component_smoke/CMakeLists.txt index 1303c49643c..47497007845 100644 --- a/examples/espressif/idf_component_smoke/CMakeLists.txt +++ b/examples/espressif/idf_component_smoke/CMakeLists.txt @@ -7,10 +7,10 @@ # Minimal ESP-IDF consumer of the generated ExecuTorch component, used by # .ci/scripts/test_espressif_idf_component.sh. Pass the generated component # directory as ET_COMPONENT_DIR, the plain smoke .pte from export_smoke_model.py -# as ET_SMOKE_PTE, and its kernel bundle from the component's -# tools/gen_kernels.py as ET_SMOKE_KERNELS. ET_SMOKE_UNDECLARED runs the model -# without declaring it. Without ET_SMOKE_PTE it builds an app that does not use -# ExecuTorch, for size comparisons. +# as ET_SMOKE_PTE, and optionally its kernel bundle from the component's +# tools/gen_kernels.py as ET_SMOKE_KERNELS; without one the build makes it. +# ET_SMOKE_UNDECLARED runs the model without declaring it. Without ET_SMOKE_PTE +# it builds an app that does not use ExecuTorch, for size comparisons. cmake_minimum_required(VERSION 3.16) diff --git a/examples/espressif/idf_component_template/README.md b/examples/espressif/idf_component_template/README.md index 4699567cc5a..8240cc7a241 100644 --- a/examples/espressif/idf_component_template/README.md +++ b/examples/espressif/idf_component_template/README.md @@ -44,6 +44,14 @@ executorch_add_model(PTE "${CMAKE_CURRENT_SOURCE_DIR}/model.pte") executorch_add_kernels(DIR "${CMAKE_CURRENT_SOURCE_DIR}/et_kernels") ``` +If the Python ESP-IDF builds with can import `executorch` (or you +point `EXECUTORCH_PYTHON` at one that can, with `-D` or in the +environment, for example +`idf.py -DEXECUTORCH_PYTHON=$HOME/venvs/et/bin/python build`), you can +leave out `executorch_add_kernels`: the build then makes the bundle in +its build directory from the declared models, with the same result as +running `gen_kernels.py` yourself. + The build fails at configure time if the bundle doesn't match the declared models (a model was re-exported, added or removed) or was made for another version of this component, and prints the diff --git a/examples/espressif/idf_component_template/project_include.cmake b/examples/espressif/idf_component_template/project_include.cmake index be75b6603ec..b99245ef991 100644 --- a/examples/espressif/idf_component_template/project_include.cmake +++ b/examples/espressif/idf_component_template/project_include.cmake @@ -11,7 +11,9 @@ # executorch_add_model(PTE [PTE ...]) # models this firmware runs # executorch_add_kernels(DIR ) # from tools/gen_kernels.py # -# Calls are unioned across the project; a project has one kernel bundle. +# Calls are unioned across the project; a project has one kernel bundle. With +# models but no bundle, the build makes one if EXECUTORCH_PYTHON (-D or the +# environment; default: the build's Python) can import executorch. function(executorch_add_model) cmake_parse_arguments(_arg "" "" "PTE" ${ARGN}) @@ -71,20 +73,70 @@ function(__executorch_finalize) list(REMOVE_DUPLICATES _models) set(_command "python \"${_dir}/tools/gen_kernels.py\"") set(_pte_args "") + set(_gen_args "") foreach(_pte IN LISTS _models) if(NOT EXISTS "${_pte}") message(FATAL_ERROR "ExecuTorch: declared model ${_pte} does not exist") endif() string(APPEND _pte_args " --pte \"${_pte}\"") + list(APPEND _gen_args --pte "${_pte}") endforeach() if(NOT _bundle) + if(NOT EXECUTORCH_PYTHON) + set(EXECUTORCH_PYTHON "$ENV{EXECUTORCH_PYTHON}") + endif() + if(NOT EXECUTORCH_PYTHON) + idf_build_get_property(EXECUTORCH_PYTHON PYTHON) + elseif(NOT IS_ABSOLUTE "${EXECUTORCH_PYTHON}" OR NOT EXISTS + "${EXECUTORCH_PYTHON}" + ) + message( + FATAL_ERROR + "ExecuTorch: EXECUTORCH_PYTHON must be the absolute path of a " + "python, not ${EXECUTORCH_PYTHON}" + ) + endif() + string( + CONCAT + _manual + "Make the bundle with the Python environment the models were exported " + "with:\n ${_command}${_pte_args} --out \nthen call " + "executorch_add_kernels(DIR ), or set EXECUTORCH_PYTHON to " + "that environment's python." + ) + execute_process( + COMMAND "${EXECUTORCH_PYTHON}" -c + "import executorch.codegen.tools.selective_build" + RESULT_VARIABLE _no_executorch + OUTPUT_QUIET + ERROR_VARIABLE _probe_error + ) + if(_no_executorch) + message( + FATAL_ERROR + "ExecuTorch: models are declared but no kernel bundle is, and " + "${EXECUTORCH_PYTHON} cannot import executorch to make one:\n" + "${_probe_error}${_manual}" + ) + endif() + set(_bundle "${CMAKE_BINARY_DIR}/executorch_kernels") message( - FATAL_ERROR - "ExecuTorch: models are declared but no kernel bundle is. Make one " - "with the Python environment the models were exported with:\n" - " ${_command}${_pte_args} --out \n" - "then call executorch_add_kernels(DIR )." + STATUS "ExecuTorch: making the kernel bundle with ${EXECUTORCH_PYTHON}" + ) + execute_process( + COMMAND "${EXECUTORCH_PYTHON}" "${_dir}/tools/gen_kernels.py" ${_gen_args} + --out "${_bundle}" + RESULT_VARIABLE _failed + OUTPUT_VARIABLE _output + ERROR_VARIABLE _output ) + if(_failed) + message( + FATAL_ERROR + "ExecuTorch: ${EXECUTORCH_PYTHON} failed to make the kernel bundle:\n" + "${_output}${_manual}" + ) + endif() endif() set(_manifest "${_bundle}/kernels.json")