diff --git a/backends/xnnpack/runtime/XNNCompiler.cpp b/backends/xnnpack/runtime/XNNCompiler.cpp index 74a5eec4a91..ebac7fb418e 100644 --- a/backends/xnnpack/runtime/XNNCompiler.cpp +++ b/backends/xnnpack/runtime/XNNCompiler.cpp @@ -2131,7 +2131,7 @@ ET_NODISCARD Error XNNCompiler::compileModel( XNNWeightsCache* weights_cache, xnn_workspace_t workspace, const NamedDataMap* named_data_map, - bool use_weight_cache) { + const XNNCompilerOptions& options) { Result header = XNNHeader::Parse(buffer_pointer, num_bytes); const uint8_t* flatbuffer_data = nullptr; const uint8_t* constant_data = nullptr; @@ -2239,7 +2239,7 @@ ET_NODISCARD Error XNNCompiler::compileModel( // With the weights cache the buffers land in the cache rather than in // unpacked_buffers, so track its list too. size_t prev_cached_buffers = - use_weight_cache ? weights_cache->get_num_unpacked_data() : 0; + options.use_weight_cache ? weights_cache->get_num_unpacked_data() : 0; err = defineTensor( subgraph.get(), remapped_ids, @@ -2253,7 +2253,7 @@ ET_NODISCARD Error XNNCompiler::compileModel( named_data_map, unpacked_buffers, weights_cache, - use_weight_cache); + options.use_weight_cache); if (err != Error::Ok) { return err; @@ -2268,7 +2268,7 @@ ET_NODISCARD Error XNNCompiler::compileModel( executor->unpacked_buffers_.push_back(std::move(unpacked_buffers[i])); } unpacked_buffers.resize(prev_buffers); - if (use_weight_cache) { + if (options.use_weight_cache) { weights_cache->take_unpacked_data_from( prev_cached_buffers, executor->unpacked_buffers_); } @@ -2288,10 +2288,20 @@ ET_NODISCARD Error XNNCompiler::compileModel( runtime_flags |= XNN_FLAG_BASIC_PROFILING; #endif + if (options.disable_packed_lhs) { +#if defined(XNN_FLAG_NO_PACKED_LHS) + runtime_flags |= XNN_FLAG_NO_PACKED_LHS; +#else + ET_LOG( + Error, + "Ignoring disable_packed_lhs: this XNNPACK build does not support it"); +#endif + } + xnn_runtime_t runtime_ptr = nullptr; xnn_weights_cache_t weights_cache_ptr = nullptr; - if (use_weight_cache) { + if (options.use_weight_cache) { ET_CHECK_OR_RETURN_ERROR( unpacked_buffers.size() == 0, Internal, @@ -2319,7 +2329,7 @@ ET_NODISCARD Error XNNCompiler::compileModel( xnn_status_to_string(status)); std::vector packed_weights_names; - if (use_weight_cache) { + if (options.use_weight_cache) { // Constants XNNPACK does not pack were already moved to the executor in // the value loop above, so everything left here is safe to free. auto packed_weights_names_result = weights_cache->finalize_for_runtime(); diff --git a/backends/xnnpack/runtime/XNNCompiler.h b/backends/xnnpack/runtime/XNNCompiler.h index 639df0438cb..4d615d71ab3 100644 --- a/backends/xnnpack/runtime/XNNCompiler.h +++ b/backends/xnnpack/runtime/XNNCompiler.h @@ -18,6 +18,11 @@ namespace backends { namespace xnnpack { namespace delegate { +struct XNNCompilerOptions { + bool use_weight_cache{false}; + bool disable_packed_lhs{false}; +}; + class XNNCompiler { public: // Takes Flatbuffer Serialized XNNPACK Model and rebuilds the xnn-subgraph @@ -30,7 +35,7 @@ class XNNCompiler { XNNWeightsCache* weights_cache, xnn_workspace_t workspace, const NamedDataMap* named_data_map, - bool use_weight_cache); + const XNNCompilerOptions& options); }; } // namespace delegate diff --git a/backends/xnnpack/runtime/XNNExecutor.h b/backends/xnnpack/runtime/XNNExecutor.h index ad70a18e5e7..dca236ce700 100644 --- a/backends/xnnpack/runtime/XNNExecutor.h +++ b/backends/xnnpack/runtime/XNNExecutor.h @@ -146,6 +146,7 @@ class XNNExecutor { executorch::runtime::Span args) const; friend class XNNCompiler; + friend class XNNPackedLhsTest; }; } // namespace delegate diff --git a/backends/xnnpack/runtime/XNNPACKBackend.cpp b/backends/xnnpack/runtime/XNNPACKBackend.cpp index a76a0832def..1f7237b8521 100644 --- a/backends/xnnpack/runtime/XNNPACKBackend.cpp +++ b/backends/xnnpack/runtime/XNNPACKBackend.cpp @@ -90,12 +90,15 @@ class XnnpackBackend final } auto workspace = workspace_result.get(); - bool use_weight_cache = options_.resolve_weight_cache(context); + const xnnpack::delegate::XNNCompilerOptions compiler_options{ + .use_weight_cache = options_.resolve_weight_cache(context), + .disable_packed_lhs = options_.resolve_disable_packed_lhs(context), + }; // Per-path weights cache: same-path PTEs share an instance and // serialize on its mutex; different paths run in parallel. std::shared_ptr weights_cache; std::unique_lock lock_weights_cache; - if (use_weight_cache) { + if (compiler_options.use_weight_cache) { // Only honor a path coming through runtime_spec (per-PTE opt-in). // Reading the backend-singleton global would let a non-opt-in PTE // inherit another model's cache file. @@ -131,7 +134,7 @@ class XnnpackBackend final weights_cache.get(), workspace_ptr, named_data_map, - use_weight_cache); + compiler_options); // This backend does not need its processed data after compiling the model. processed->Free(); @@ -147,7 +150,7 @@ class XnnpackBackend final // Hand the cache to the executor (held by shared_ptr so it // outlives any sibling executors that share it). - if (use_weight_cache) { + if (compiler_options.use_weight_cache) { executor->set_weights_cache(std::move(weights_cache)); } diff --git a/backends/xnnpack/runtime/XNNPACKBackend.h b/backends/xnnpack/runtime/XNNPACKBackend.h index df83a500489..8e9a8577d4b 100644 --- a/backends/xnnpack/runtime/XNNPACKBackend.h +++ b/backends/xnnpack/runtime/XNNPACKBackend.h @@ -19,6 +19,13 @@ const char workspace_sharing_mode_option_key[] = "workspace_sharing_mode"; // across delegate instances. Changes only affect subsequently loaded models. const char weight_cache_option_key[] = "weight_cache_enabled"; +/// The key for disabling packed-LHS graph rewrites. When enabled, XNNPACK +/// preserves explicit dynamic-quantization nodes instead of folding them into +/// packed-LHS consumers. +// Must remain a C array for the BackendOptions template overloads. +// @lint-ignore CLANGTIDY facebook-hte-CArray +const char disable_packed_lhs_option_key[] = "disable_packed_lhs"; + /// Path for the packed weight file. When set, reserve_space() allocates from /// a MAP_SHARED file instead of heap; msync makes pages clean on iOS. // Must remain a C array (not const char*) so it can bind to the diff --git a/backends/xnnpack/runtime/XnnpackBackendOptions.cpp b/backends/xnnpack/runtime/XnnpackBackendOptions.cpp index 9820e7a8eef..b8fbd667c4f 100644 --- a/backends/xnnpack/runtime/XnnpackBackendOptions.cpp +++ b/backends/xnnpack/runtime/XnnpackBackendOptions.cpp @@ -37,6 +37,8 @@ Error XnnpackBackendOptions::get_option(BackendOption& option) const { option.value = static_cast(sharing_mode_.load()); } else if (strcmp(option.key, weight_cache_option_key) == 0) { option.value = weight_cache_enabled_.load(); + } else if (strcmp(option.key, disable_packed_lhs_option_key) == 0) { + option.value = disable_packed_lhs_.load(); } else if (strcmp(option.key, packed_cache_path_option_key) == 0) { std::array arr{}; const std::lock_guard lock(path_mutex_); @@ -73,6 +75,14 @@ Error XnnpackBackendOptions::set_option(const BackendOption& option) { } ET_LOG(Debug, "Setting XNNPACK weight cache enabled to %d.", *val); weight_cache_enabled_.store(*val); + } else if (strcmp(option.key, disable_packed_lhs_option_key) == 0) { + auto* val = std::get_if(&option.value); + if (!val) { + ET_LOG(Error, "XNNPACK disable packed LHS must be a bool."); + return Error::InvalidArgument; + } + ET_LOG(Debug, "Setting XNNPACK disable packed LHS to %d.", *val); + disable_packed_lhs_.store(*val); } else if (strcmp(option.key, packed_cache_path_option_key) == 0) { auto* val = std::get_if>( &option.value); @@ -126,6 +136,12 @@ bool XnnpackBackendOptions::resolve_weight_cache( context, weight_cache_option_key, weight_cache_enabled_.load()); } +bool XnnpackBackendOptions::resolve_disable_packed_lhs( + const ET_RUNTIME_NAMESPACE::BackendInitContext& context) const { + return resolve_option( + context, disable_packed_lhs_option_key, disable_packed_lhs_.load()); +} + runtime::Result XnnpackBackendOptions::resolve_sharing_mode( const ET_RUNTIME_NAMESPACE::BackendInitContext& context) const { diff --git a/backends/xnnpack/runtime/XnnpackBackendOptions.h b/backends/xnnpack/runtime/XnnpackBackendOptions.h index 52ba71b99db..dc922b71620 100644 --- a/backends/xnnpack/runtime/XnnpackBackendOptions.h +++ b/backends/xnnpack/runtime/XnnpackBackendOptions.h @@ -37,6 +37,11 @@ class XnnpackBackendOptions { bool resolve_weight_cache( const ET_RUNTIME_NAMESPACE::BackendInitContext& context) const; + // Resolve whether packed-LHS graph rewrites are disabled for a delegate, + // applying any runtime spec override. + bool resolve_disable_packed_lhs( + const ET_RUNTIME_NAMESPACE::BackendInitContext& context) const; + // Resolve the effective workspace sharing mode for a delegate, applying // any runtime spec override. Returns InvalidArgument for out-of-range values. runtime::Result resolve_sharing_mode( @@ -95,6 +100,8 @@ class XnnpackBackendOptions { std::atomic weight_cache_enabled_{false}; #endif + std::atomic disable_packed_lhs_{false}; + // The most-recently-set packed cache path. Today's callers (Cria // runner) set this via set_option(packed_cache_path_option_key) per // PTE, then invoke init() which reads it and calls diff --git a/backends/xnnpack/test/CMakeLists.txt b/backends/xnnpack/test/CMakeLists.txt index 3d9c77d6ad6..090503da590 100644 --- a/backends/xnnpack/test/CMakeLists.txt +++ b/backends/xnnpack/test/CMakeLists.txt @@ -36,6 +36,7 @@ et_cxx_test( target_include_directories( backends_xnnpack_test PRIVATE ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/XNNPACK/include + ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/XNNPACK ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/XNNPACK/src ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/cpuinfo/include ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/pthreadpool/include diff --git a/backends/xnnpack/test/runtime/test_weight_cache.cpp b/backends/xnnpack/test/runtime/test_weight_cache.cpp index 384edbaec8c..1c35431208e 100644 --- a/backends/xnnpack/test/runtime/test_weight_cache.cpp +++ b/backends/xnnpack/test/runtime/test_weight_cache.cpp @@ -20,6 +20,7 @@ using namespace ::testing; +using executorch::backends::xnnpack::disable_packed_lhs_option_key; using executorch::backends::xnnpack::get_packed_cache_report; using executorch::backends::xnnpack::packed_cache_path_option_key; using executorch::backends::xnnpack::PackedCacheHeapReason; @@ -53,6 +54,24 @@ static void set_and_check_weight_cache_enabled(bool enabled) { ASSERT_EQ(std::get(read_option.value), enabled); } +static void set_and_check_disable_packed_lhs(bool disabled) { + executorch::runtime::runtime_init(); + + BackendOptions<1> backend_options; + backend_options.set_option(disable_packed_lhs_option_key, disabled); + + auto status = executorch::runtime::set_option( + xnnpack_backend_key, backend_options.view()); + ASSERT_EQ(status, Error::Ok); + + BackendOption read_option; + strcpy(read_option.key, disable_packed_lhs_option_key); + read_option.value = !disabled; + status = get_option(xnnpack_backend_key, read_option); + + ASSERT_EQ(std::get(read_option.value), disabled); +} + static TensorPtr create_input_tensor(float val) { std::vector data(1000, val); return executorch::extension::make_tensor_ptr({10, 10, 10}, std::move(data)); @@ -101,6 +120,22 @@ TEST(WeightCache, SetInvalidType) { ASSERT_EQ(status, Error::InvalidArgument); } +TEST(PackedLhs, SetDisabled) { + set_and_check_disable_packed_lhs(true); + set_and_check_disable_packed_lhs(false); +} + +TEST(PackedLhs, SetInvalidType) { + executorch::runtime::runtime_init(); + + BackendOptions<1> backend_options; + backend_options.set_option(disable_packed_lhs_option_key, 1); + + auto status = executorch::runtime::set_option( + xnnpack_backend_key, backend_options.view()); + ASSERT_EQ(status, Error::InvalidArgument); +} + TEST(WeightCache, SetMultipleOptions) { executorch::runtime::runtime_init(); diff --git a/backends/xnnpack/test/runtime/test_xnnexecutor.cpp b/backends/xnnpack/test/runtime/test_xnnexecutor.cpp index cd7eadacdea..5754da31e39 100644 --- a/backends/xnnpack/test/runtime/test_xnnexecutor.cpp +++ b/backends/xnnpack/test/runtime/test_xnnexecutor.cpp @@ -7,10 +7,18 @@ */ #include +#include +#include #include +#include +#include #include +#include #include +#include +#include + using executorch::aten::Tensor; using executorch::backends::xnnpack::delegate::XNNExecutor; using executorch::runtime::Error; @@ -18,6 +26,137 @@ using executorch::runtime::EValue; using executorch::runtime::Span; using executorch::runtime::testing::TensorFactory; +namespace executorch::backends::xnnpack::delegate { + +class XNNPackedLhsTest : public ::testing::Test { + protected: + void SetUp() override { + runtime::runtime_init(); + backend_ = runtime::get_backend_class(xnnpack_backend_key); + ASSERT_NE(backend_, nullptr); + ASSERT_EQ( + original_options_.set_option(disable_packed_lhs_option_key, false), + Error::Ok); + ASSERT_EQ( + runtime::get_option(xnnpack_backend_key, original_options_.view()), + Error::Ok); + + std::vector> values; + for (uint32_t id = 0; id < 3; ++id) { + const auto tensor = fb_xnnpack::CreateXNNTensorValue( + builder_, + fb_xnnpack::XNNDatatype::xnn_datatype_fp32, + 1, + builder_.CreateVector({1}), + 0, + id, + id == 2 ? XNN_VALUE_FLAG_EXTERNAL_OUTPUT + : XNN_VALUE_FLAG_EXTERNAL_INPUT, + id); + values.push_back(fb_xnnpack::CreateXValue( + builder_, fb_xnnpack::XValueUnion::XNNTensorValue, tensor.Union())); + } + const auto add = fb_xnnpack::Create_XNNNode2x1(builder_, 0, 1, 2); + const auto node = fb_xnnpack::CreateXNode( + builder_, fb_xnnpack::XNodeUnion::XNNAdd, add.Union()); + const auto graph = fb_xnnpack::CreateXNNGraph( + builder_, + builder_.CreateString("0"), + builder_.CreateVector(&node, 1), + builder_.CreateVector(values), + 3, + builder_.CreateVector({0, 1}), + builder_.CreateVector({2})); + fb_xnnpack::FinishXNNGraphBuffer(builder_, graph); + } + + void TearDown() override { + EXPECT_EQ( + runtime::set_option(xnnpack_backend_key, original_options_.view()), + Error::Ok); + } + + runtime::Result create_runtime_flags( + Span specs) { + std::array memory{}; + runtime::MemoryAllocator allocator(memory.size(), memory.data()); + runtime::BackendInitContext context( + &allocator, nullptr, nullptr, nullptr, {specs.data(), specs.size()}); + runtime::FreeableBuffer processed( + builder_.GetBufferPointer(), builder_.GetSize(), nullptr); + auto handle = backend_->init(context, &processed, {}); + if (!handle.ok()) { + return handle.error(); + } + const auto* executor = static_cast(handle.get()); + const uint32_t flags = executor->runtime_->flags; + backend_->destroy(handle.get()); + return flags; + } + + runtime::BackendInterface* backend_{nullptr}; + runtime::BackendOptions<1> original_options_; + flatbuffers::FlatBufferBuilder builder_; +}; + +TEST_F(XNNPackedLhsTest, RuntimeSpecOverridesGlobalDefault) { + runtime::BackendOptions<1> global_options; + ASSERT_EQ( + global_options.set_option(disable_packed_lhs_option_key, false), + Error::Ok); + ASSERT_EQ( + runtime::set_option(xnnpack_backend_key, global_options.view()), + Error::Ok); + runtime::BackendOptions<2> specs; + ASSERT_EQ(specs.set_option(weight_cache_option_key, false), Error::Ok); + const auto baseline = create_runtime_flags(specs.view()); + ASSERT_TRUE(baseline.ok()); +#if defined(XNN_FLAG_NO_PACKED_LHS) + ASSERT_EQ(baseline.get() & XNN_FLAG_NO_PACKED_LHS, 0); +#endif + + for (bool global_default : {false, true}) { + SCOPED_TRACE(global_default); + ASSERT_EQ( + global_options.set_option( + disable_packed_lhs_option_key, global_default), + Error::Ok); + ASSERT_EQ( + runtime::set_option(xnnpack_backend_key, global_options.view()), + Error::Ok); + for (auto override_value : + {std::optional{}, + std::optional{false}, + std::optional{true}}) { + SCOPED_TRACE(::testing::PrintToString(override_value)); + runtime::BackendOptions<2> model_options; + ASSERT_EQ( + model_options.set_option(weight_cache_option_key, false), Error::Ok); + if (override_value.has_value()) { + ASSERT_EQ( + model_options.set_option( + disable_packed_lhs_option_key, *override_value), + Error::Ok); + } + const auto flags = create_runtime_flags(model_options.view()); + ASSERT_TRUE(flags.ok()); + uint32_t expected_flags = baseline.get(); +#if defined(XNN_FLAG_NO_PACKED_LHS) + if (override_value.value_or(global_default)) { + expected_flags |= XNN_FLAG_NO_PACKED_LHS; + } +#endif + EXPECT_EQ(flags.get(), expected_flags); + ASSERT_EQ( + runtime::get_option(xnnpack_backend_key, global_options.view()), + Error::Ok); + EXPECT_EQ(std::get(global_options.view()[0].value), global_default); + } + } +} + +} // namespace executorch::backends::xnnpack::delegate + TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) { XNNExecutor executor({}); xnn_subgraph_t subgraph = nullptr; diff --git a/backends/xnnpack/test/targets.bzl b/backends/xnnpack/test/targets.bzl index a291e63245c..0d4cb15ca23 100644 --- a/backends/xnnpack/test/targets.bzl +++ b/backends/xnnpack/test/targets.bzl @@ -22,12 +22,16 @@ def define_common_targets(): runtime.cxx_test( name = "xnnexecutor_test", srcs = ["runtime/test_xnnexecutor.cpp"], + fbcode_deps = [ + third_party_dep("XNNPACK").replace(":XNNPACK", ":public-headers"), + ], deps = [ third_party_dep("XNNPACK"), third_party_dep("FP16"), "//executorch/runtime/core/exec_aten/testing_util:tensor_util", "//executorch/runtime/core/exec_aten/util:scalar_type_util", "//executorch/backends/xnnpack:xnnpack_backend", + "//executorch/backends/xnnpack/serialization:xnnpack_flatbuffer_header", ], )