Skip to content

[ExecuTorch][WebGPU] Add Gemma 4 MTP and speculative-decode source-closure tests #21678

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JCNTH wants to merge 1 commit into
base: gh/JCNTH/218/base
Choose a base branch
from
Open

[ExecuTorch][WebGPU] Add Gemma 4 MTP and speculative-decode source-closure tests #21678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions backends/webgpu/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,12 @@ if(EXECUTORCH_BUILD_WEBGPU_TEST)
target_link_libraries(webgpu_dynamic_shape_test PRIVATE GTest::gtest)
add_webgpu_native_test(webgpu_index_test test/native/test_index.cpp)
target_link_libraries(webgpu_index_test PRIVATE GTest::gtest)
add_webgpu_native_test(webgpu_topk_test test/native/test_topk.cpp)
target_link_libraries(webgpu_topk_test PRIVATE GTest::gtest)
add_webgpu_native_test(webgpu_scatter_test test/native/test_scatter.cpp)
target_link_libraries(webgpu_scatter_test PRIVATE GTest::gtest)
add_webgpu_native_test(webgpu_q4gsw_m3_test test/native/test_q4gsw_m3.cpp)
target_link_libraries(webgpu_q4gsw_m3_test PRIVATE GTest::gtest)

# Device-free fold unit test (gtest_main provides main; no device needed).
add_webgpu_native_test(
Expand Down
63 changes: 60 additions & 3 deletions backends/webgpu/scripts/test_webgpu_native_ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,39 @@ fi

cd "${EXECUTORCH_ROOT}"

bash "${SCRIPT_DIR}/test_gemma4_wasm_factory_contract.sh" --validate-names
buck2 test fbcode//executorch/backends/webgpu/test:test_wgsl_codegen

# ── Exports for the model-driven executables ─────────────────────────────────
if ! "${PYTHON_EXECUTABLE}" -c "import executorch" 2>/dev/null; then
echo "ERROR: executorch wheel unavailable; required fixture exports cannot run" >&2
exit 1
fi

# ── Source contracts: no Buck target, they read across packages ──────────────
$PYTHON_EXECUTABLE -m unittest \
executorch.backends.webgpu.test.test_native_ci_contract
$PYTHON_EXECUTABLE -m unittest \
executorch.examples.models.gemma4.tests.test_oss_source_closure

require_file() {
if [[ ! -f "$1" ]]; then
echo "ERROR: required WebGPU fixture missing: $1" >&2
exit 1
fi
}

recreate_exact_directory() {
local target="$1"
local expected="$2"
if [[ "${target}" != "${expected}" ]]; then
echo "ERROR: refusing to recreate unexpected directory: ${target}" >&2
return 1
fi
rm -rf -- "${target}"
mkdir -p -- "${target}"
}

run_with_required_device() {
local output
if ! output="$("$@" 2>&1)"; then
Expand Down Expand Up @@ -84,7 +104,7 @@ run_required_gtests() {
echo "ERROR: required WebGPU run did not pass exactly three tests" >&2
return 1
fi
if grep -Eq '^\\[ SKIPPED \\]' <<<"${output}"; then
if grep -Eq '^\[ SKIPPED \]' <<<"${output}"; then
echo "ERROR: required WebGPU run skipped a test" >&2
return 1
fi
Expand All @@ -93,6 +113,9 @@ run_required_gtests() {
DISPATCH_ORDER_DIR="/tmp/dispatch_order"
UPDATE_CACHE_DIR="/tmp/update_cache"
INDEX_DIR="/tmp/index"
TOPK_DIR="/tmp/topk"
TOPK_AUTHORITY="/tmp/topk_eager_authority.json"
SCATTER_DIR="/tmp/scatter"
DYNAMIC_SHAPE_DIR="/tmp/dynamic_shape"
ROPE_HF_DIR="/tmp/webgpu_rope_hf"
SYMINT_BLOB="/tmp/sdpa_dyn_small.pte"
Expand All @@ -116,6 +139,8 @@ PREPACK2_GOLDEN="/tmp/webgpu_prepack_two_const_golden.bin"
PREPACK_TIED_MODEL="/tmp/webgpu_prepack_tied_const.pte"
PREPACK_TIED_GOLDEN="/tmp/webgpu_prepack_tied_const_golden.bin"

recreate_exact_directory "${UPDATE_CACHE_DIR}" "/tmp/update_cache"

$PYTHON_EXECUTABLE -c "
from executorch.backends.webgpu.test.ops.test_quantized_linear import export_all_quantized_linear_models, export_output_suppression_models
export_all_quantized_linear_models('/tmp')
Expand Down Expand Up @@ -153,25 +178,47 @@ export_dispatch_order_cases('${DISPATCH_ORDER_DIR}')

$PYTHON_EXECUTABLE -c "
from executorch.backends.webgpu.test.ops.test_update_cache import (
export_dynamic_update_cache,
export_intermediate_dynamic_update_cache,
export_update_cache_cases,
export_update_cache_replay,
export_update_cache_negative,
)
export_update_cache_cases('${UPDATE_CACHE_DIR}')
export_update_cache_replay('${UPDATE_CACHE_DIR}')
export_update_cache_negative('${UPDATE_CACHE_DIR}')
export_dynamic_update_cache('${UPDATE_CACHE_DIR}/dynamic.pte')
export_intermediate_dynamic_update_cache('${UPDATE_CACHE_DIR}/dynamic_intermediate.pte')
"
require_file "${UPDATE_CACHE_DIR}/dynamic.pte"
require_file "${UPDATE_CACHE_DIR}/dynamic_intermediate.pte"

$PYTHON_EXECUTABLE -c "
from executorch.backends.webgpu.test.ops.index.test_index import export_all_index_models
export_all_index_models('${INDEX_DIR}')
"

# The exporter validates this receipt against the committed authority digest.
EAGLE_TOPK_EAGER_RECEIPT="${TOPK_AUTHORITY}" $PYTHON_EXECUTABLE -m unittest \
executorch.backends.webgpu.test.ops.topk.test_topk.TestEagleTopKCpu.test_eager_reference_is_repeatable

$PYTHON_EXECUTABLE -m executorch.backends.webgpu.test.ops.topk.export_topk_artifacts \
"${TOPK_DIR}" "${TOPK_AUTHORITY}"

$PYTHON_EXECUTABLE -m executorch.backends.webgpu.test.ops.scatter.export_scatter_artifacts \
"${SCATTER_DIR}"

recreate_exact_directory "${DYNAMIC_SHAPE_DIR}" "/tmp/dynamic_shape"
WEBGPU_TEST_HEAVY=1 $PYTHON_EXECUTABLE -c "
from executorch.backends.webgpu.test.ops.dynamic_shape.test_dynamic_shape_export import export_dynamic_shape_cases
export_dynamic_shape_cases('${DYNAMIC_SHAPE_DIR}')
"
require_file "${DYNAMIC_SHAPE_DIR}/dyn_cat_2d.pte"
require_file "${DYNAMIC_SHAPE_DIR}/dyn_slice_2d.pte"
require_file "${DYNAMIC_SHAPE_DIR}/slice_dual_store.pte"
require_file "${DYNAMIC_SHAPE_DIR}/slice_dual_store.input.bin"
require_file "${DYNAMIC_SHAPE_DIR}/slice_dual_store.out0.golden.bin"
require_file "${DYNAMIC_SHAPE_DIR}/slice_dual_store.out1.golden.bin"

$PYTHON_EXECUTABLE -c "
from executorch.backends.webgpu.test.ops.test_sdpa import (
Expand All @@ -189,6 +236,10 @@ export_incache_decode('/tmp')
require_file "${ROPE_HF_DIR}/rope_hf_dynamic.pte"
require_file "${SYMINT_BLOB}"
require_file "${OUTPUT_SUPPRESSION_DIR}/input.bin"
require_file "${TOPK_AUTHORITY}"
require_file "${TOPK_DIR}/cases.txt"
require_file "${SCATTER_DIR}/cases.txt"
require_file "${SCATTER_DIR}/base.bin"

# ── Configure (Dawn-only: no -DWEBGPU_IMPL; Dawn is the sole backend) ─────────
echo "=== Configure WebGPU native tests on Dawn ==="
Expand All @@ -209,7 +260,7 @@ cmake \
"${EXECUTORCH_ROOT}"

# ── Build + run every fixed native test target in this tree ──────────────────
REQUIRED_TARGETS=(webgpu_native_test webgpu_dispatch_order_test webgpu_scratch_buffer_test webgpu_update_cache_test webgpu_index_test webgpu_dynamic_shape_test webgpu_dispatch_2d_test webgpu_compute_dispatch_test webgpu_execution_options_test webgpu_output_suppression_test webgpu_op_test_util_test)
REQUIRED_TARGETS=(webgpu_native_test webgpu_dispatch_order_test webgpu_scratch_buffer_test webgpu_update_cache_test webgpu_update_cache_state_test webgpu_index_test webgpu_dynamic_shape_test webgpu_dispatch_2d_test webgpu_compute_dispatch_test webgpu_execution_options_test webgpu_output_suppression_test webgpu_op_test_util_test webgpu_topk_test webgpu_scatter_test webgpu_q4gsw_m3_test)
BIN_DIR="${BUILD_DIR}/backends/webgpu"

DEFINED_TARGETS="$(cmake --build "${BUILD_DIR}" --target help 2>/dev/null || true)"
Expand All @@ -224,6 +275,7 @@ for t in "${REQUIRED_TARGETS[@]}"; do
done

echo "=== Run native tests on Dawn + SwiftShader ==="
"${BIN_DIR}/webgpu_update_cache_state_test"
run_with_required_device env WEBGPU_TEST_SDPA_DIR=/tmp/ \
WEBGPU_TEST_QUANTIZED_LINEAR_DIR=/tmp/ \
WEBGPU_TEST_EMBEDDING_Q4GSW_MODEL="${EMBEDDING_MODEL}" \
Expand All @@ -247,9 +299,14 @@ run_with_required_device env WEBGPU_TEST_SDPA_DIR=/tmp/ \
WEBGPU_TEST_PREPACK_TIED_MODEL="${PREPACK_TIED_MODEL}" \
WEBGPU_TEST_PREPACK_TIED_GOLDEN="${PREPACK_TIED_GOLDEN}" \
"${BIN_DIR}/webgpu_native_test"
"${BIN_DIR}/webgpu_update_cache_test" "${UPDATE_CACHE_DIR}"
run_with_required_device env WEBGPU_REQUIRE_DEVICE=1 \
WEBGPU_UPDATE_CACHE_DIR="${UPDATE_CACHE_DIR}" \
"${BIN_DIR}/webgpu_update_cache_test" "${UPDATE_CACHE_DIR}"
"${BIN_DIR}/webgpu_dispatch_order_test" "${DISPATCH_ORDER_DIR}"
"${BIN_DIR}/webgpu_index_test" "${INDEX_DIR}"
"${BIN_DIR}/webgpu_topk_test" "${TOPK_DIR}"
"${BIN_DIR}/webgpu_scatter_test" "${SCATTER_DIR}"
"${BIN_DIR}/webgpu_q4gsw_m3_test"
"${BIN_DIR}/webgpu_dynamic_shape_test" "${DYNAMIC_SHAPE_DIR}"
run_required_gtests env WEBGPU_REQUIRE_DEVICE=1 WEBGPU_TEST_HEAVY=1 \
"${BIN_DIR}/webgpu_dynamic_shape_test" "${DYNAMIC_SHAPE_DIR}" \
Expand Down
84 changes: 84 additions & 0 deletions backends/webgpu/test/BUCK
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,90 @@ fbcode_target(
],
)

fbcode_target(
_kind = python_unittest,
name = "test_to_copy",
srcs = [
"ops/test_to_copy.py",
],
deps = [
"//caffe2:torch",
"//executorch/backends/vulkan/partitioner:vulkan_partitioner",
"//executorch/backends/vulkan:vulkan_preprocess",
"//executorch/exir:lib",
],
)

fbcode_target(
_kind = python_unittest,
name = "test_gather",
srcs = [
"ops/test_gather.py",
],
deps = [
"//caffe2:torch",
"//executorch/backends/vulkan/partitioner:vulkan_partitioner",
"//executorch/backends/vulkan:vulkan_preprocess",
"//executorch/exir:lib",
],
)

fbcode_target(
_kind = python_unittest,
name = "test_where",
srcs = [
"ops/test_where.py",
],
deps = [
"//caffe2:torch",
"//executorch/backends/vulkan/partitioner:vulkan_partitioner",
"//executorch/backends/vulkan:vulkan_preprocess",
"//executorch/exir:lib",
],
)

# Pure-stdlib CPU authority for the top-k route; no torch, no GPU.
fbcode_target(
_kind = python_unittest,
name = "test_topk_cpu",
srcs = [
"ops/topk/test_topk.py",
],
)

# Pure-stdlib CPU authority for the scatter route; no torch, no GPU.
fbcode_target(
_kind = python_unittest,
name = "test_scatter_cpu",
srcs = [
"ops/scatter/test_scatter.py",
],
)

# Source contract over the op-test driver; reads its inputs, so they ship as resources.
fbcode_target(
_kind = python_unittest,
name = "test_typed_input_contract",
srcs = [
"op_tests/test_typed_input_contract.py",
],
resources = [
"op_tests/cases.py",
"op_tests/generate_op_tests.py",
"op_tests/op_test_driver.cpp",
"ops/test_to_copy.py",
"ops/test_where.py",
],
)

# The native CI contract reads four Buck packages; test_webgpu_native_ci.sh runs it.
fbcode_target(
_kind = runtime.python_library,
name = "test_native_ci_contract",
srcs = ["test_native_ci_contract.py"],
typing = True,
)

fbcode_target(
_kind = python_unittest,
name = "test_rope_hf_single",
Expand Down
Loading
Loading