From b00e38ebd8f3f6fad5b49c4192b6863136deb31d Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Tue, 25 Aug 2026 12:01:59 -0400
Subject: [PATCH 01/22] hoist_invariants(): return one Func per accumulator,
 not a Tuple

hoist_invariants() looks for a single loop-invariant factor of the whole
increment, so a sum of terms with different factors -- a*g(r) + b*h(r) --
had nothing it could do: flattening the multiply chain sees the sum as one
opaque leaf, and the directive threw. Give each term its own accumulator
instead, all advanced by one loop over the same reduction domain, with the
write-back applying each factor once. Terms sharing a factor stay in one
accumulator, so an increment that was already hoistable is unaffected.
This is the TODO that was sitting on extract_factor().

Each term's accumulator is its own single-valued Func, not a value of one
Tuple-valued intermediate, so it can be scheduled -- or severed by
Pipeline::compute_offline() -- independently of the others. An update
definition that did not split keeps the shape it had: a single intermediate
carrying every value of the original reduction, whose values may reference
each other's. hoist_invariants() therefore returns a std::vector<Func>: one
entry per accumulator, or a single entry for a definition that did not split.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 test/correctness/hoist_invariants.cpp        | 67 ++++++++++++++++++++
 test/correctness/struct_type_dot_product.cpp |  2 +-
 test/performance/tiled_matmul_arm_neon.cpp   |  2 +-
 3 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/test/correctness/hoist_invariants.cpp b/test/correctness/hoist_invariants.cpp
index 7f61562ca580..87740882d4b1 100644
--- a/test/correctness/hoist_invariants.cpp
+++ b/test/correctness/hoist_invariants.cpp
@@ -588,6 +588,72 @@ int hoist_invariants_nothing_to_hoist_rejected_test() {
     return 0;
 }
 
+// An increment written as a sum of terms with *different* invariant factors has
+// no single factor to hoist. Each term gets its own accumulator instead, all
+// advanced by one loop, with the write-back applying each factor once.
+int hoist_invariants_terms_test() {
+    const int K = 64;
+    ImageParam G{Int(8), 1, "G"};
+    ImageParam H{Int(8), 1, "H"};
+    ImageParam A{Float(32), 1, "A"};
+    ImageParam B{Float(32), 1, "B"};
+
+    Var i{"i"};
+    RDom r(0, K, "r");
+
+    Func Acc{"Acc"};
+    Acc(i) = 0.0f;
+    Acc(i) += A(i) * cast<float>(G(r)) + B(i) * cast<float>(H(r));
+
+    std::vector<Func> Acc_intm = Acc.update().hoist_invariants();
+    internal_assert(Acc_intm.size() == 2)
+        << "hoist_invariants terms: expected one accumulator per term, got "
+        << Acc_intm.size() << "\n";
+    for (Func &f : Acc_intm) {
+        f.compute_root();
+    }
+
+    Buffer<int8_t> g_buf(K), h_buf(K);
+    Buffer<float> a_buf(1), b_buf(1);
+    for (int k = 0; k < K; k++) {
+        g_buf(k) = 3;
+        h_buf(k) = 5;
+    }
+    a_buf(0) = 2.0f;
+    b_buf(0) = 7.0f;
+    G.set(g_buf);
+    H.set(h_buf);
+    A.set(a_buf);
+    B.set(b_buf);
+
+    Buffer<float> result = Acc.realize({1});
+    const float expected = (2.0f * 3.0f + 7.0f * 5.0f) * (float)K;
+    internal_assert(result(0) == expected)
+        << "hoist_invariants terms: got " << result(0) << ", expected " << expected << "\n";
+
+    // Terms still get one accumulator each even when they share an identical
+    // factor: hoist_invariants() doesn't try to detect and merge such terms,
+    // so the accumulator count only ever depends on the number of terms.
+    Func Shared{"Shared"};
+    Shared(i) = 0.0f;
+    Shared(i) += A(i) * cast<float>(G(r)) + A(i) * cast<float>(H(r));
+    std::vector<Func> Shared_intm = Shared.update().hoist_invariants();
+    internal_assert(Shared_intm.size() == 2)
+        << "hoist_invariants terms: expected one accumulator per term even "
+        << "when terms share a factor, got " << Shared_intm.size() << "\n";
+    for (Func &f : Shared_intm) {
+        f.compute_root();
+    }
+
+    Buffer<float> shared = Shared.realize({1});
+    const float shared_expected = 2.0f * (3.0f + 5.0f) * (float)K;
+    internal_assert(shared(0) == shared_expected)
+        << "hoist_invariants shared factor: got " << shared(0) << ", expected "
+        << shared_expected << "\n";
+
+    return 0;
+}
+
 }  // namespace
 
 int main(int argc, char **argv) {
@@ -610,6 +676,7 @@ int main(int argc, char **argv) {
         {"hoist_invariants test (after rfactor)", hoist_invariants_after_rfactor_test},
         {"hoist_invariants test (invalid law rejected)", hoist_invariants_invalid_law_rejected_test},
         {"hoist_invariants test (nothing to hoist rejected)", hoist_invariants_nothing_to_hoist_rejected_test},
+        {"hoist_invariants test (one accumulator per term)", hoist_invariants_terms_test},
     };
 
     using Sharder = Halide::Internal::Test::Sharder;
diff --git a/test/correctness/struct_type_dot_product.cpp b/test/correctness/struct_type_dot_product.cpp
index 63067a80bb3d..186a74a5e9ab 100644
--- a/test/correctness/struct_type_dot_product.cpp
+++ b/test/correctness/struct_type_dot_product.cpp
@@ -107,7 +107,7 @@ struct Pipeline {
         RVar ko("ko"), ki("ki");
         result.update().split(r[1], ko, ki, 16);
         Func partial = result.update().rfactor(r[0], u);
-        Func dot = partial.update().eager_inline(x_wt, y_wt).hoist_invariants();
+        Func dot = partial.update().eager_inline(x_wt, y_wt).hoist_invariants()[0];
         Func dot_i32 = dot.change_type(Int(32));
 
         // One block's scaled contribution per outer step; its quants reduce
diff --git a/test/performance/tiled_matmul_arm_neon.cpp b/test/performance/tiled_matmul_arm_neon.cpp
index 807049e1e7b3..975b715f62fd 100644
--- a/test/performance/tiled_matmul_arm_neon.cpp
+++ b/test/performance/tiled_matmul_arm_neon.cpp
@@ -107,7 +107,7 @@ int main(int argc, char **argv) {
             // and Acc's own update collapses to one multiply per row,
             // Acc(i) = WtScale(i) * VecScale * Acc_wb(i). Acc_wb still accumulates
             // at Float(32) at this point.
-            Func Acc_wb = Acc.update().eager_inline(Wt, VecDq).hoist_invariants();
+            Func Acc_wb = Acc.update().eager_inline(Wt, VecDq).hoist_invariants()[0];
 
             // Second, factor Acc_wb's own (now scale-free) reduction by block.
             // Preserving ro turns it into a new dimension u of Acc_dot, so

From bc98aa475552bd9027d9ced14f238e89e05b2bc6 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Tue, 25 Aug 2026 12:02:56 -0400
Subject: [PATCH 02/22] Add Stage::distribute()

Reaching the terms of a *product* of sums needs the increment multiplied
out first, and whether that pays is not something to guess at: it depends
on what the terms turn out to contain. Splitting s * (r + 1) into two
accumulators is a pessimization, while splitting (d*q(r) + m) * (e*p(r))
into d*e*sum(q*p) + m*e*sum(p) turns one float reduction into two integer
ones. So multiplying out is its own schedule directive, Stage::distribute(),
rather than a heuristic inside hoist_invariants().

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/Func.cpp                          | 88 ++++++++++++++++++++++++++
 src/Func.h                            | 26 ++++++++
 test/correctness/hoist_invariants.cpp | 89 +++++++++++++++++++++++++++
 3 files changed, 203 insertions(+)

diff --git a/src/Func.cpp b/src/Func.cpp
index c99ca7ea6f8d..445da0323c65 100644
--- a/src/Func.cpp
+++ b/src/Func.cpp
@@ -1266,6 +1266,94 @@ Func Stage::rfactor(const vector<pair<RVar, Var>> &preserved) {
     return intm;
 }
 
+// How many nodes distribute_products() may rewrite before giving up. Multiplying
+// out is exponential in the nesting depth of sums, and a reduction body deep
+// enough to hit this has no business being split into that many accumulators.
+constexpr int distribute_budget = 1024;
+
+// Multiply out products over sums, so that a sum reduction's increment becomes
+// a flat sum of products: (a + b) * c -> a * c + b * c. This exposes terms with
+// *different* invariant factors, which a single flattening of the multiply chain
+// cannot see -- in (a + b) * c the sum is one opaque leaf.
+//
+// Subtraction is deliberately left alone (treated as a leaf): distributing it
+// would have to push the sign into one side, which does not group and is not
+// meaningful for unsigned wraparound.
+Expr distribute_products(const Expr &e, int &budget) {
+    if (--budget < 0) {
+        return e;
+    }
+    if (e.node_type() == IRNodeType::Add) {
+        auto [a, b] = *as_binary_operands(e);
+        return Add::make(distribute_products(a, budget), distribute_products(b, budget));
+    }
+    if (const Mul *mul = e.as<Mul>()) {
+        Expr a = distribute_products(mul->a, budget);
+        Expr b = distribute_products(mul->b, budget);
+        // Recurse on the rewritten form: either side may itself be a product of
+        // sums that only became visible after this step.
+        if (a.node_type() == IRNodeType::Add) {
+            auto [a0, a1] = *as_binary_operands(a);
+            return distribute_products(Add::make(Mul::make(a0, b), Mul::make(a1, b)), budget);
+        }
+        if (b.node_type() == IRNodeType::Add) {
+            auto [b0, b1] = *as_binary_operands(b);
+            return distribute_products(Add::make(Mul::make(a, b0), Mul::make(a, b1)), budget);
+        }
+        return Mul::make(a, b);
+    }
+    return e;
+}
+
+Stage &Stage::distribute() {
+    user_assert(!definition.is_init()) << "distribute() must be called on an update definition\n";
+
+    definition.schedule().touched() = true;
+
+    const auto &prover_result = prove_associativity(function.name(), definition.args(), definition.values());
+    user_assert(prover_result.associative())
+        << "distribute() requires an associative update definition, but the update "
+        << "definition of " << function.name() << " is not associative.\n";
+
+    auto is_self_ref = [&](const Expr &e) {
+        const Call *c = e.as<Call>();
+        return c && c->name == function.name() && c->call_type == Call::Halide;
+    };
+
+    vector<Expr> values = definition.values();
+    bool changed = false;
+    for (size_t i = 0; i < values.size(); ++i) {
+        optional<DistributiveLaw> law = distributive_law_for(prover_result.pattern.ops[i]);
+        if (!law || law->outer_op != IRNodeType::Add || law->inner_op != IRNodeType::Mul) {
+            continue;
+        }
+        // Lets may hide the combiner (e.g. an rfactor of this same update
+        // introduced promise_clamped bindings), so inline them first.
+        Expr value = substitute_in_all_lets(values[i]);
+        optional<pair<Expr, Expr>> split = select_binary_operand(value, law->outer_op, is_self_ref);
+        if (!split) {
+            continue;
+        }
+        int budget = distribute_budget;
+        Expr expanded = distribute_products(split->second, budget);
+        user_assert(budget >= 0)
+            << "distribute() gave up multiplying out the update definition of "
+            << function.name() << ": it expands to more than " << distribute_budget
+            << " nodes.\n";
+        if (!equal(expanded, split->second)) {
+            values[i] = Add::make(split->first, expanded);
+            changed = true;
+        }
+    }
+
+    user_assert(changed)
+        << "distribute() found no product over a sum to multiply out in the update "
+        << "definition of " << function.name() << ".\n";
+
+    definition.values() = values;
+    return *this;
+}
+
 FuncVec Stage::hoist_invariants() {
     user_assert(!definition.is_init()) << "hoist_invariants() must be called on an update definition\n";
 
diff --git a/src/Func.h b/src/Func.h
index f33d3f2d4af4..f08f2c88cda3 100644
--- a/src/Func.h
+++ b/src/Func.h
@@ -258,6 +258,32 @@ class Stage {
      */
     FuncVec hoist_invariants();
 
+    /** Multiply out products over sums in this update definition's increment,
+     * so that hoist_invariants() sees a flat sum of terms. Like rfactor(), this
+     * must be called on an update definition, and it rewrites that definition in
+     * place. Returns this Stage, so it can be chained.
+     *
+     * (a + b) * c becomes a * c + b * c, recursively. Subtraction is left alone.
+     *
+     * This is a schedule decision, not a normalization: whether it pays depends
+     * on what the terms turn out to contain. It pays when the sum hides operands
+     * with different loop-invariant factors, since each then reduces to a
+     * factor-free body of its own:
+     * \code
+     * f() += (d*q(r) + m) * (e*p(r));
+     * \endcode
+     * multiplies out to d*e * q(r)*p(r) + m*e * p(r), which hoist_invariants()
+     * turns into two factor-free accumulators. Left alone, the best it could do
+     * is hoist e and reduce over the sum.
+     *
+     * It costs an accumulator per distinct factor, so it is a pessimization when
+     * the terms share a factor that was already hoistable -- s * (r + 1) is
+     * better left as one accumulator with factor s than split into two.
+     *
+     * It is an error if there is no product over a sum to multiply out.
+     */
+    Stage &distribute();
+
     /** Schedule the iteration over this stage to be fused with another
      * stage 's' from outermost loop to a given LoopLevel. 'this' stage will
      * be computed AFTER 's' in the innermost fused dimension. There should not
diff --git a/test/correctness/hoist_invariants.cpp b/test/correctness/hoist_invariants.cpp
index 87740882d4b1..46ba3552e5bb 100644
--- a/test/correctness/hoist_invariants.cpp
+++ b/test/correctness/hoist_invariants.cpp
@@ -654,6 +654,93 @@ int hoist_invariants_terms_test() {
     return 0;
 }
 
+// distribute() multiplies a product of sums out so that hoist_invariants() can
+// see terms that were not written as terms. This is the affine-quantized dot
+// product: sum_k (d*q_k + m) * (e*p_k), whose expansion
+// d*e*sum_k(q_k*p_k) + m*e*sum_k(p_k) has two integer-bodied accumulators where
+// the unexpanded form has one float one.
+int hoist_invariants_distribute_test() {
+    const int K = 32;
+    ImageParam Q{Int(8), 1, "Q"};
+    ImageParam P{Int(8), 1, "P"};
+    ImageParam D{Float(32), 1, "D"};
+    ImageParam M{Float(32), 1, "M"};
+    ImageParam E{Float(32), 1, "E"};
+
+    Var i{"i"};
+    RDom r(0, K, "r");
+
+    Func Acc{"Acc"};
+    Acc(i) = 0.0f;
+    Acc(i) += (cast<float>(Q(r)) * D(i) + M(i)) * (cast<float>(P(r)) * E(i));
+
+    std::vector<Func> Acc_intm = Acc.update().distribute().hoist_invariants();
+    internal_assert(Acc_intm.size() == 2)
+        << "distribute: expected the multiplied-out increment to yield two "
+        << "accumulators, got " << Acc_intm.size() << "\n";
+
+    // Both bodies are integer sums of int8 products, so both retype -- and each
+    // accumulator is its own Func, so each may take its own target type.
+    Func qp = Acc_intm[0].change_type(Int(32));
+    Func p_sum = Acc_intm[1].change_type(Int(16));
+    internal_assert(qp.types()[0] == Int(32) && p_sum.types()[0] == Int(16))
+        << "distribute: retyping the accumulators separately gave "
+        << qp.types()[0] << " and " << p_sum.types()[0] << "\n";
+    qp.compute_root();
+    p_sum.compute_root();
+
+    Buffer<int8_t> q_buf(K), p_buf(K);
+    Buffer<float> d_buf(1), m_buf(1), e_buf(1);
+    int64_t sum_qp = 0, sum_p = 0;
+    for (int k = 0; k < K; k++) {
+        q_buf(k) = (int8_t)(k % 15);
+        p_buf(k) = (int8_t)((k * 5) % 127 - 63);
+        sum_qp += (int64_t)q_buf(k) * p_buf(k);
+        sum_p += p_buf(k);
+    }
+    d_buf(0) = 0.25f;
+    m_buf(0) = -0.5f;
+    e_buf(0) = 2.0f;
+    Q.set(q_buf);
+    P.set(p_buf);
+    D.set(d_buf);
+    M.set(m_buf);
+    E.set(e_buf);
+
+    Buffer<float> result = Acc.realize({1});
+    const float expected = 0.25f * 2.0f * (float)sum_qp + -0.5f * 2.0f * (float)sum_p;
+    internal_assert(std::abs(result(0) - expected) < 1e-3f)
+        << "distribute: got " << result(0) << ", expected " << expected << "\n";
+
+    return 0;
+}
+
+// distribute() is a schedule decision, so it says so when there is nothing to
+// multiply out rather than quietly leaving the reduction alone.
+int distribute_nothing_to_do_rejected_test() {
+    if (!Halide::exceptions_enabled()) {
+        return 0;
+    }
+    ImageParam A{Float(32), 1, "A"};
+    Var i{"i"};
+    RDom r(0, 8, "r");
+
+    Func f{"f"};
+    f(i) = 0.0f;
+    f(i) += A(i) * cast<float>(r);
+
+    try {
+        f.update().distribute();
+    } catch (const Halide::CompileError &e) {
+        const std::string msg = e.what();
+        internal_assert(msg.find("no product over a sum") != std::string::npos)
+            << "distribute() rejected the update for the wrong reason: " << msg << "\n";
+        return 0;
+    }
+    internal_assert(false) << "distribute() accepted an update with nothing to multiply out\n";
+    return 0;
+}
+
 }  // namespace
 
 int main(int argc, char **argv) {
@@ -677,6 +764,8 @@ int main(int argc, char **argv) {
         {"hoist_invariants test (invalid law rejected)", hoist_invariants_invalid_law_rejected_test},
         {"hoist_invariants test (nothing to hoist rejected)", hoist_invariants_nothing_to_hoist_rejected_test},
         {"hoist_invariants test (one accumulator per term)", hoist_invariants_terms_test},
+        {"distribute test (affine dot product)", hoist_invariants_distribute_test},
+        {"distribute test (nothing to distribute rejected)", distribute_nothing_to_do_rejected_test},
     };
 
     using Sharder = Halide::Internal::Test::Sharder;

From e671fd8a590128a17ed315d3408c8f7df740f358 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sat, 1 Aug 2026 19:43:28 -0400
Subject: [PATCH 03/22] Add Pipeline::compute_offline() directive

compute_offline() severs a set of Funcs from a Pipeline's computation,
rewriting every call to them (and anything only they depended on) into
calls to fresh or caller-supplied ImageParams of matching type and
dimensionality. This turns work that would otherwise be recomputed on
every realize() -- e.g. a static weight quantizer's encode() step -- into
ordinary input data supplied once, from wherever the returned `offline`
Pipeline was realized or compiled.

The rewrite is eager and destructive, like rfactor()/approximate_by():
by the time the call returns, none of the Pipeline's outputs depend on
the severed Funcs for their computation, only for the shape the returned
ImageParams must satisfy. Two overloads: one mints fresh ImageParams, one
binds each severed Func to a caller-supplied ImageParam (e.g. a
Generator's own Input). v1 requires each severed Func to be single-valued.

Verified by test/correctness/compute_offline.cpp.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/Pipeline.cpp                     |  76 +++++++++++
 src/Pipeline.h                       |  48 +++++++
 test/correctness/CMakeLists.txt      |   1 +
 test/correctness/compute_offline.cpp | 191 +++++++++++++++++++++++++++
 4 files changed, 316 insertions(+)
 create mode 100644 test/correctness/compute_offline.cpp

diff --git a/src/Pipeline.cpp b/src/Pipeline.cpp
index e8faf1561e3c..08c0708f639e 100644
--- a/src/Pipeline.cpp
+++ b/src/Pipeline.cpp
@@ -8,6 +8,7 @@
 #include "FindCalls.h"
 #include "Func.h"
 #include "IRVisitor.h"
+#include "ImageParam.h"
 #include "InferArguments.h"
 #include "LLVM_Output.h"
 #include "Lower.h"
@@ -231,6 +232,81 @@ std::vector<Internal::Stmt> Pipeline::requirements() const {
     return contents->requirements;
 }
 
+namespace {
+
+ComputeOfflineResult compute_offline_impl(const Pipeline &pipeline, const vector<Func> &to_sever,
+                                          const vector<ImageParam> *bind_to) {
+    vector<Function> output_funcs;
+    for (const Func &f : pipeline.outputs()) {
+        output_funcs.push_back(f.function());
+    }
+    std::map<string, Function> env = build_environment(output_funcs);
+
+    // Everything transitively reachable from to_sever (to_sever itself, plus
+    // anything only *it* depends on, e.g. a per-block reduction Func or a
+    // sibling encode() output one of to_sever's own definitions reads) is
+    // part of the offline half and must keep computing its true values --
+    // none of it should have its calls redirected to an online stand-in,
+    // even if it's also present in `env` because it used to be reachable
+    // from *this's outputs before severing.
+    vector<Function> to_sever_funcs;
+    to_sever_funcs.reserve(to_sever.size());
+    for (const Func &f : to_sever) {
+        to_sever_funcs.push_back(f.function());
+    }
+    std::map<string, Function> offline_env = build_environment(to_sever_funcs);
+
+    if (bind_to) {
+        user_assert(bind_to->size() == to_sever.size())
+            << "Pipeline::compute_offline(): bind_to has " << bind_to->size()
+            << " ImageParams, but to_sever has " << to_sever.size() << " Funcs\n";
+    }
+
+    std::map<FunctionPtr, FunctionPtr> substitutions;
+    vector<ImageParam> online_inputs;
+    online_inputs.reserve(to_sever.size());
+    for (size_t i = 0; i < to_sever.size(); i++) {
+        const Func &f = to_sever[i];
+        user_assert(f.types().size() == 1)
+            << "Pipeline::compute_offline() requires single-valued Funcs, but "
+            << f.name() << " has " << f.types().size() << " values\n";
+
+        ImageParam im = bind_to ? (*bind_to)[i] : ImageParam(f.types()[0], f.dimensions(), f.name() + "_im");
+        if (bind_to) {
+            user_assert(im.type() == f.types()[0] && im.dimensions() == f.dimensions())
+                << "Pipeline::compute_offline(): bind_to[" << i << "] (" << im.name()
+                << ") has type/dimensionality mismatched with " << f.name() << "\n";
+        }
+
+        Func stand_in(f.name() + "_offline_input");
+        vector<Var> args = f.args();
+        vector<Expr> args_as_exprs(args.begin(), args.end());
+        stand_in(args) = im(args_as_exprs);
+
+        substitutions[f.function().get_contents()] = stand_in.function().get_contents();
+        online_inputs.push_back(im);
+    }
+
+    for (auto &entry : env) {
+        if (offline_env.count(entry.first)) {
+            continue;
+        }
+        entry.second.substitute_calls(substitutions);
+    }
+
+    return {Pipeline(to_sever), std::move(online_inputs)};
+}
+
+}  // namespace
+
+ComputeOfflineResult Pipeline::compute_offline(const vector<Func> &to_sever) {
+    return compute_offline_impl(*this, to_sever, nullptr);
+}
+
+ComputeOfflineResult Pipeline::compute_offline(const vector<Func> &to_sever, const vector<ImageParam> &bind_to) {
+    return compute_offline_impl(*this, to_sever, &bind_to);
+}
+
 /* static */
 std::map<std::string, AutoSchedulerFn> &Pipeline::get_autoscheduler_map() {
     static std::map<std::string, AutoSchedulerFn> autoschedulers = {};
diff --git a/src/Pipeline.h b/src/Pipeline.h
index fac90b99a8dd..981e81b754e9 100644
--- a/src/Pipeline.h
+++ b/src/Pipeline.h
@@ -26,7 +26,9 @@ struct Argument;
 class Callable;
 class Func;
 class FuncVec;
+class ImageParam;
 struct PipelineContents;
+struct ComputeOfflineResult;
 
 /** Special the Autoscheduler to be used (if any), along with arbitrary
  * additional arguments specific to the given Autoscheduler.
@@ -211,6 +213,41 @@ class Pipeline {
     /** Get the requirements of this pipeline. */
     std::vector<Internal::Stmt> requirements() const;
 
+    /** Rewrite every call to each Func in `to_sever`, anywhere in this
+     * Pipeline's transitive call graph, to instead call a fresh ImageParam
+     * of matching type and dimensionality -- severing this Pipeline's
+     * *computation* of `to_sever` (and anything only `to_sever` depended on)
+     * while preserving the shape contract those Funcs stood in for. This is
+     * eager and destructive, like rfactor()/approximate_by(): by the time
+     * this call returns, none of this Pipeline's outputs depend on
+     * `to_sever` for their computation, only for the shape the returned
+     * ImageParams must satisfy.
+     *
+     * This is the seam doc/ApproximationDesign.md calls "compute_offline":
+     * splicing it after a static weight quantizer's encode() step turns
+     * that quantization from something recomputed on every realize() into
+     * ordinary input data, supplied once from wherever `offline` was
+     * realized or compiled.
+     *
+     * v1 restriction: each Func in `to_sever` must be single-valued (no
+     * Tuples). */
+    ComputeOfflineResult compute_offline(const std::vector<Func> &to_sever);
+
+    /** Like compute_offline(const std::vector<Func> &) above, but binds each
+     * severed Func to a caller-supplied ImageParam instead of minting a
+     * fresh one -- e.g. a Generator's own Input<Buffer<>> (which converts to
+     * ImageParam implicitly), so the online half's severed input is exactly
+     * the port `configure()`/`add_input()` already declared, rather than an
+     * unrelated ImageParam nothing else in the compiled artifact knows
+     * about. `bind_to` must have the same length as `to_sever`, and each
+     * `bind_to[i]` must have the same type and dimensionality as
+     * `to_sever[i]`. `online_inputs` in the result is `bind_to` itself,
+     * returned for symmetry with the other overload -- callers that already
+     * have `bind_to` don't need it, but code generic over both overloads
+     * still gets a uniform result shape. */
+    ComputeOfflineResult compute_offline(const std::vector<Func> &to_sever,
+                                         const std::vector<ImageParam> &bind_to);
+
     /** Generate a schedule for the pipeline using the specified autoscheduler. */
     AutoSchedulerResults apply_autoscheduler(const Target &target,
                                              const AutoschedulerParams &autoscheduler_params) const;
@@ -519,6 +556,17 @@ class Pipeline {
     std::string generate_function_name() const;
 };
 
+/** The result of Pipeline::compute_offline(): `offline` computes exactly
+ * `to_sever`'s original values, unmodified -- realize it once (JIT) or
+ * compile it as its own artifact (AOT), then feed the result to
+ * `online_inputs` before realizing the pipeline compute_offline() was called
+ * on again. `online_inputs` has one ImageParam per element of `to_sever`, in
+ * the same order, with matching type and dimensionality. */
+struct ComputeOfflineResult {
+    Pipeline offline;
+    std::vector<ImageParam> online_inputs;
+};
+
 struct ExternSignature {
 private:
     Type ret_type_;  // Only meaningful if is_void_return is false; must be default value otherwise
diff --git a/test/correctness/CMakeLists.txt b/test/correctness/CMakeLists.txt
index 6794e958e5e1..bb6c5ded26b7 100644
--- a/test/correctness/CMakeLists.txt
+++ b/test/correctness/CMakeLists.txt
@@ -60,6 +60,7 @@ tests(
     compute_at_reordered_update_stage.cpp
     compute_at_split_rvar.cpp
     compute_inside_guard.cpp
+    compute_offline.cpp
     compute_with_in.cpp
     compute_with_inlined.cpp
     computed_index.cpp
diff --git a/test/correctness/compute_offline.cpp b/test/correctness/compute_offline.cpp
new file mode 100644
index 000000000000..14e81266cba4
--- /dev/null
+++ b/test/correctness/compute_offline.cpp
@@ -0,0 +1,191 @@
+#include "Halide.h"
+#include <cmath>
+#include <cstdio>
+#include <functional>
+#include <vector>
+
+using namespace Halide;
+
+// Exercises Pipeline::compute_offline: rewriting calls to a Func into calls to
+// a fresh ImageParam, severing a pipeline's *computation* of that Func while
+// preserving the shape contract it stood in for.
+
+namespace {
+
+// Symmetric int8 quantization for a 1-D Func, same shape as the one in
+// approximation_composition.cpp: encode() returns q(k) and scale(); decode()
+// dequantizes q(k) * scale().
+class SymmetricQuantize : public Approximation {
+public:
+    explicit SymmetricQuantize(int k)
+        : k_(k) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        Func v = inputs[0];
+        Var k("k");
+        RDom r(0, k_, "r");
+
+        Func amax("amax");
+        amax() = 0.0f;
+        amax() = max(amax(), abs(v(r)));
+
+        Func d("scale");
+        d() = amax() / 127.0f;
+
+        Func q("q");
+        Expr id = select(d() != 0.0f, 1.0f / d(), 0.0f);
+        q(k) = cast<int8_t>(clamp(round(v(k) * id), -127, 127));
+
+        return {{q, d}, {amax}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        Func q = encoded[0], d = encoded[1];
+        Var k("k");
+        Func dequantized("dequantized");
+        dequantized(k) = cast<float>(q(k)) * d();
+        return {{dequantized}, {}};
+    }
+
+private:
+    int k_;
+};
+
+void reference_symmetric_quantize(int k, const std::function<float(int)> &values,
+                                  std::vector<int8_t> &q, float &scale) {
+    float amax = 0.0f;
+    for (int kk = 0; kk < k; kk++) {
+        amax = std::max(amax, std::fabs(values(kk)));
+    }
+    scale = amax / 127.0f;
+    float id = scale != 0.0f ? 1.0f / scale : 0.0f;
+    q.resize(k);
+    for (int kk = 0; kk < k; kk++) {
+        int v = (int)std::round(values(kk) * id);
+        q[kk] = (int8_t)std::max(-127, std::min(127, v));
+    }
+}
+
+// A minimal, non-Approximation check that compute_offline() actually severs
+// the call graph, rather than being a no-op that happens to still produce the
+// right answer once. f(x) = x*2, g(x) = f(x) + 1: after
+// Pipeline({g}).compute_offline({f}), g must stop depending on f's own
+// computation -- setting a buffer on the returned ImageParam that disagrees
+// with f's true values must change g's output accordingly.
+int minimal_severance_test() {
+    Var x("x");
+    Func f("f"), g("g");
+    f(x) = x * 2;
+    g(x) = f(x) + 1;
+
+    ComputeOfflineResult split = Pipeline({g}).compute_offline({f});
+
+    Buffer<int> f_values = split.offline.realize({10});
+    for (int x = 0; x < 10; x++) {
+        if (f_values(x) != x * 2) {
+            printf("minimal_severance_test: offline f(%d) = %d, expected %d\n", x, f_values(x), x * 2);
+            return 1;
+        }
+    }
+
+    // Feed f's true values through the ImageParam: g should compute as if
+    // nothing changed.
+    split.online_inputs[0].set(f_values);
+    Buffer<int> g_true = g.realize({10});
+    for (int x = 0; x < 10; x++) {
+        if (g_true(x) != x * 2 + 1) {
+            printf("minimal_severance_test: g(%d) = %d with true f, expected %d\n", x, g_true(x), x * 2 + 1);
+            return 1;
+        }
+    }
+
+    // Now feed different values through the same ImageParam. If g still
+    // depended on f's own computation, this would have no effect.
+    Buffer<int> f_fake(10);
+    for (int x = 0; x < 10; x++) {
+        f_fake(x) = 1000 + x;
+    }
+    split.online_inputs[0].set(f_fake);
+    Buffer<int> g_fake = g.realize({10});
+    for (int x = 0; x < 10; x++) {
+        int expected = f_fake(x) + 1;
+        if (g_fake(x) != expected) {
+            printf("minimal_severance_test: g(%d) = %d with fake f, expected %d "
+                   "(compute_offline() did not actually sever the call graph)\n",
+                   x, g_fake(x), expected);
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+// A realistic case: sever a quantized vector's encode() output from a
+// consumer built via approximate_by(), and check the final result still
+// matches the plain-C++ reference round trip.
+int quantized_offline_test() {
+    const int K = 64;
+    Var k("k");
+
+    Func Vec("Vec");
+    Vec(k) = cos(cast<float>(k) * 0.05f) * 3.0f;
+
+    SymmetricQuantize quantize(K);
+
+    Func Result("Result");
+    Result(k) = Vec(k) * 2.0f;
+
+    ApproximationResult result = Vec.approximate_by(quantize, {Result});
+    Result.eager_inline({result.replacement});
+
+    // result.handles is [q, d, amax]: encode()'s two signature-contract
+    // outputs, then its own scheduling-only handle. q and d are the actual
+    // Funcs Result's call graph depends on (approximate_by() calls encode()
+    // internally; a separately-called quantize.encode({Vec}) here would
+    // build an unrelated, unconnected copy of the same graph shape).
+    std::vector<Func> encoded = {result.handles[0], result.handles[1]};
+    for (size_t i = 2; i < result.handles.size(); i++) {
+        result.handles[i].compute_root();
+    }
+
+    ComputeOfflineResult split = Pipeline({Result}).compute_offline(encoded);
+
+    // q(k) and scale() have different dimensionality (1-D vs scalar), so they
+    // can't share a single realize({sizes}) call -- realize into
+    // pre-allocated buffers of the right shape instead.
+    Buffer<int8_t> q_buf(K);
+    Buffer<float> scale_buf = Buffer<float>::make_scalar();
+    split.offline.realize({q_buf, scale_buf});
+    split.online_inputs[0].set(q_buf);
+    split.online_inputs[1].set(scale_buf);
+
+    Buffer<float> out = Result.realize({K});
+
+    std::vector<int8_t> ref_q;
+    float ref_scale;
+    reference_symmetric_quantize(K, [](int kk) { return cosf(kk * 0.05f) * 3.0f; }, ref_q, ref_scale);
+    for (int kk = 0; kk < K; kk++) {
+        float expected = (ref_q[kk] * ref_scale) * 2.0f;
+        if (std::fabs(out(kk) - expected) > 1e-3f * std::fabs(expected)) {
+            printf("quantized_offline_test: Result(%d) = %f, expected %f\n", kk, out(kk), expected);
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    if (minimal_severance_test()) {
+        return 1;
+    }
+    if (quantized_offline_test()) {
+        return 1;
+    }
+
+    printf("Success!\n");
+    return 0;
+}

From 9f4a678c97b5a611085a6017babca6caf0afd146 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sat, 1 Aug 2026 19:43:59 -0400
Subject: [PATCH 04/22] Add Func::approximate_by() directive and Approximation
 combinators

approximate_by() eagerly and destructively replaces every call to a Func
inside a set of consumers with a call to the round trip
decode(encode(f)) -- a lossy, quantified Func-to-Func transform where
decode(encode(f)) reproduces f's signature. The substitution happens
immediately, like rfactor(), rather than being deferred to lowering the
way Func::in() is.

The transform is expressed as an Approximation: a bidirectional
encode()/decode() pair (src/Approximation.{h,cpp}), with combinators --
Compose, Apply, Permute, Choose, Identity, and the TrustedInverse escape
hatch -- for building up a codec from smaller pieces. See
doc/ApproximationDesign.md for the design rationale.

Also adds the Generator plumbing that lets a Generator wire the two
halves of an approximate_by()/compute_offline() split into real ports:
GeneratorBase::add_input(const ImageParam &) and add_output(const Func &),
backed by adopt() helpers on GeneratorInputBase/GeneratorOutputBase.

Verified by test/correctness/approximate_by.cpp.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 doc/ApproximationDesign.md          | 424 ++++++++++++++++++++++++++++
 src/Approximation.cpp               |  77 +++++
 src/Approximation.h                 | 333 ++++++++++++++++++++++
 src/CMakeLists.txt                  |   2 +
 src/Func.cpp                        |  36 +++
 src/Func.h                          |  14 +
 src/Generator.cpp                   |  53 ++++
 src/Generator.h                     |  38 +++
 test/correctness/CMakeLists.txt     |   1 +
 test/correctness/approximate_by.cpp | 115 ++++++++
 10 files changed, 1093 insertions(+)
 create mode 100644 doc/ApproximationDesign.md
 create mode 100644 src/Approximation.cpp
 create mode 100644 src/Approximation.h
 create mode 100644 test/correctness/approximate_by.cpp

diff --git a/doc/ApproximationDesign.md b/doc/ApproximationDesign.md
new file mode 100644
index 000000000000..0d87bea2df2d
--- /dev/null
+++ b/doc/ApproximationDesign.md
@@ -0,0 +1,424 @@
+# Approximation: a core concept for lossy, quantified Func substitution
+
+Status: design draft, no implementation yet.
+
+## Motivation
+
+Two pieces of prior work motivate this:
+
+1. `apps/ggml/halide/*_generators.cpp` hand-implements ~24 GGML quantized weight
+   formats as pairs of Halide Generators (quantize, dequantize), each file
+   independently encoding its own byte layout, scale/bias math, and (for
+   K-quants and a few others) a scaffolded extern-stage call to GGML's own
+   reference quantizer pending a native port. This works, but every type
+   duplicates the same shape of logic (block layout, scale search, bit-packing)
+   by hand, and the "quantize happens once offline, dequantize happens on every
+   inference call" relationship between the two directions is enforced by
+   nothing but convention and comments.
+
+2. A private Python research prototype explores building the same set of formats
+   compositionally instead: a small `Approximation` ABC (`encode`/`decode`, each
+   operating on Halide `Func`s) with a handful of primitives (block layout
+   reshaping, a linear integer quantizer, a shift-by-min helper for affine
+   schemes, bit-packers) that compose to reconstruct all of Q2_K..Q8_K. It's
+   Python-only, JIT-only, and only covers quantize/dequantize round trips (no
+   vec_dot, no repack, no scheduling story).
+
+This document proposes promoting the core of that prototype (`Approximation`)
+into a first-class Halide C++ concept, plus two small pieces of surrounding API
+(`approximate_by`, `compute_offline`) needed to wire an `Approximation` into a
+real pipeline's call graph and, optionally, split it across a compile-time
+boundary.
+
+## Why this can't just be ordinary scheduling
+
+Halide's algorithm/schedule separation depends on schedule directives being
+meaning-preserving: `.compute_root()` vs `.compute_at()` never changes what a
+pipeline computes, only how. An `Approximation` is the opposite by design — it
+deliberately changes the *value* computed (a real weight becomes a
+quantized-then-dequantized approximation of itself), in a bounded, quantified
+way. Wiring that in by disguising it as an ordinary Func substitution (e.g. a
+custom `.in()` wrapper with no other marking) would make a semantics-changing
+operation look, to any future reader, like a semantics-preserving one. It needs
+its own footing in the API.
+
+## Core concept: `Approximation`
+
+```cpp
+// encode()/decode() each return (funcs, handles): the "public" Func(s) that
+// participate in the signature contract, plus extra intermediate Funcs
+// (reduction accumulators, per-block stats, etc.) that have no meaning
+// outside scheduling but still need someone to schedule them -- see
+// "approximate_by" below for why silently dropping handles is a real bug,
+// not a simplification.
+struct EncodeResult {
+    std::vector<Func> encoded;  // the signature-contract output(s)
+    std::vector<Func> handles;  // scheduling-only, no semantic meaning
+};
+struct DecodeResult {
+    std::vector<Func> decoded;  // decoded[0] is the round-trip replacement
+    std::vector<Func> handles;
+};
+
+class Approximation {
+public:
+    virtual ~Approximation() = default;
+
+    // Both operate purely on Funcs -- no opinion about placement (compute
+    // root vs fused, offline vs online). See "Scope: placement is not
+    // semantics" below for why that split matters.
+    virtual EncodeResult encode(Func f) = 0;
+    virtual DecodeResult decode(std::vector<Func> encoded) = 0;
+};
+```
+
+Virtual dispatch is used specifically because composed Approximations
+(`Compose`/`Apply`, below) need to hold a runtime-heterogeneous list of
+`Approximation` references and call `encode`/`decode` on each polymorphically.
+(A templated/CRTP alternative was considered and rejected for this reason: it
+would make composed, heterogeneous chains require type erasure some other way,
+for no benefit here.)
+
+**Signature contract.** `decode(encode(f).encoded).decoded[0]` must reproduce
+`f`'s arg list and value type exactly — that's what makes it valid to splice
+back into a call graph in place of `f`. This is not proposed to be enforced
+generically by the base class in v1; each concrete `Approximation` is
+responsible for it, treating round-trip error and convergence as a testable
+property rather than a type-level guarantee. `approximate_by`'s substitution
+step (see below) does do a shape/type check at the point of substitution, which
+catches violations, just not at `Approximation`-definition time.
+
+**`encode`'s output arity is scheme-dependent, and that's fine.** This follows
+directly from a memory-layout choice each `Approximation` makes:
+
+- *Packed*: one opaque `Buffer<uint8_t>`, fields recovered via
+  `reinterpret<T>()` at fixed byte offsets inside `decode`. This is what every
+  type in `apps/ggml/halide` already does today (`Buffer<uint8_t, 2>` dim 0 =
+  byte-within-block), because Halide has no struct type to give a K-quant's
+  `(d, dmin, scales[12], qs[128])` a real typed signature.
+- *Planar*: multiple typed Funcs (e.g. a separate `float16_t` delta Func, a
+  separate `int8_t` quants Func), more Halide-native and type-safe, but the
+  Generator's public signature grows with the scheme's field count.
+
+Both are legitimate; `Approximation` doesn't pick one. A Generator wrapping an
+`Approximation` (see below) ends up with a scheme-dependent public signature
+either way — accepted as a consequence of this choice, not something the
+framework tries to paper over.
+
+## `approximate_by`: wiring an `Approximation` into a call graph
+
+### Rejected first approach: building on `Func::in`
+
+`Func::in(g)` (`src/Func.cpp:2456`) looked like a natural fit initially: it
+registers a wrapper Function in `f`'s `FuncSchedule::wrappers()` map, which
+`WrapCalls.cpp::wrap_func_calls` later rewrites `g`'s `Call` nodes against. The
+problem is *later*: that rewrite only happens during `lower()`, well after
+`configure()`/`generate()`/`schedule()` have all already run. Anything that
+needs to reason about "what does `g` actually call" before that point — in
+particular, `compute_offline`'s `configure()`-time branching — would be looking
+at stale, pre-substitution state. Using `.in()` here would make `approximate_by`
+and `compute_offline` fundamentally unable to compose in the same pipeline.
+
+### The actual mechanism: eager, destructive, like `rfactor`
+
+`Stage::rfactor` (`src/Func.cpp:1001`) is the right precedent instead. It never
+defers to a lowering pass: it builds a new `Func intm(...)`, calls
+`intm.function().define_update(...)` immediately, and rewrites the *original*
+Function's own definition via `substitute_self_reference` — all synchronously,
+as part of the `.rfactor()` call itself. By the time `.rfactor()` returns, the
+graph already reflects the change, which is why a caller can immediately turn
+around and schedule `intm` in the same breath.
+
+`approximate_by` should behave the same way, using the same class of primitive
+`WrapCalls.cpp` already relies on internally —
+`Function::substitute_calls(SubstitutionMap)` — but invoked immediately,
+directly on an explicitly-given set of consumers, instead of registered for a
+later pass:
+
+`approximate_by` is a member of `Func` (`f.approximate_by(p, consumers)`), not a
+free function — it's a graph-editing operation on `f` in exactly the same sense
+`f.in(...)`/`f.rfactor(...)` are, so it should read like the rest of that family
+instead of standing apart as a free function taking `f` as its first argument:
+
+```cpp
+struct ApproximationResult {
+    Func replacement;           // decode's round-trip output; already
+                                 // spliced into every Func in `consumers`
+    std::vector<Func> handles;  // encode's output(s) + encode's handles +
+                                 // decode's handles -- all need scheduling,
+                                 // none are part of the signature contract
+};
+
+// Func.h: ApproximationResult approximate_by(Approximation &p, const std::vector<Func> &consumers);
+ApproximationResult Func::approximate_by(Approximation &p, const std::vector<Func> &consumers) {
+    EncodeResult enc = p.encode(*this);
+    DecodeResult dec = p.decode(enc.encoded);
+    Func round_trip = dec.decoded[0];  // signature contract: matches *this exactly
+
+    for (const Func &g : consumers) {
+        // eager, destructive -- happens now, not at lowering time
+        g.function().substitute_calls(func, round_trip.function());
+    }
+
+    std::vector<Func> handles = enc.encoded;
+    handles.insert(handles.end(), enc.handles.begin(), enc.handles.end());
+    handles.insert(handles.end(), dec.handles.begin(), dec.handles.end());
+    return {round_trip, handles};
+}
+```
+
+(`func` is `Func`'s own private `Internal::Function` member; `g.function()` is
+the public accessor for a different `Func`'s. `Function::substitute_calls`
+already has a single-pair overload taking `(orig, substitute)` directly, so no
+manual `SubstitutionMap` construction is needed.)
+
+This needs **no new public primitive at all**, which is a smaller change than
+either of the two things earlier drafts of this design proposed (a `Func::in`
+overload, or a public wrapper around `substitute_calls`):
+`Function::substitute_calls` is already an ordinary method on the internal
+`Function` class, and `approximate_by` is implemented inside libHalide itself
+(`Approximation.h`/`Func.cpp`), so it can call it directly — the "new public
+entry point" concern only would have applied if this were being built as
+external, non-core code.
+
+**Returning `handles` is not optional.** Both `encode` and `decode` can
+introduce intermediate Funcs with update definitions (per-block reductions, a
+shift-by-min helper's own min-reduction, etc.). Left unscheduled, Halide doesn't
+error on these — it computes them at the innermost valid loop level by default
+(a Func can only fail to compile this way if something explicitly forces
+`.compute_inline()` on it, which is illegal for a Func with an update
+definition). But that default placement is exactly that, a default: the caller
+has no way to override it, or to apply the fusion patterns from "Scope:
+placement is not semantics" below (e.g. `compute_at`-ing `enc.encoded` into a
+producer for dynamic activation requantization). The struct above bundles every
+encode/decode handle together for exactly this reason — so the caller can
+schedule all of them, not just the primary output.
+
+### Consequence: consumers must already exist
+
+Because the substitution is eager, `approximate_by` can only rewrite Funcs that
+are already built at the point of the call — there is no equivalent of `.in()`'s
+global mode (redirect *every* current and future consumer, including ones not
+yet written). This is a real capability loss relative to `.in()`, but it's the
+same scoping `rfactor` already lives with (it never retroactively touches
+definitions that don't exist yet either), and it matches how Generator code is
+actually written: within `generate()`, `f` and its consumer(s) are typically
+built in the same breath, so passing `consumers` explicitly costs nothing there.
+It only forecloses a genuinely different use case — transparently intercepting
+calls inside some large, opaque, externally-authored algorithm without being
+able to enumerate its call sites — which is out of scope for this design.
+
+## Scope: placement is not semantics
+
+An `Approximation`'s `encode`/`decode` never make any claim about *where* or
+*when* they're computed relative to the rest of the pipeline. This was initially
+proposed otherwise — an early draft of this design suggested that `encode`'s
+output could just always be treated as "the offline half" — and was rejected on
+a concrete counterexample: **dynamic activation requantization**.
+
+Contrast:
+
+- **Static weight quantization**: `encode` (e.g. Q4_0 quantize) runs exactly
+  once, ever, fully decoupled from any inference call — a genuine
+  compile-time/compilation-unit boundary. `decode` gets fused inline into the
+  consumer's inner loop — already exactly what every `*VecDotGenerator` in
+  `apps/ggml/halide` does (e.g. `ggml_halide::q8_0_value(y_blocks_, r)` called
+  inline inside `sum()`, never materialized as its own Func).
+- **Dynamic activation requantization**: `encode` (quantize a just-computed
+  activation tile) needs to be fused into the *producer's* schedule — same
+  granularity, same loop nest, no separate storage, recomputed every call.
+  `decode` is fused into the consumer's tiles exactly as before.
+
+Same `Approximation`, opposite treatment of where `encode` is computed. If
+"encode ⇒ offline" were baked into the interface, the activation case would need
+an escape hatch to override it — at which point the shortcut has bought nothing,
+and worse, it would invite tooling to assume every quantize step is safe to
+hoist to conversion time, which is a real correctness trap for anything computed
+at inference time.
+
+**Consequence, and a scope reduction**: the "fuse encode into producer" / "fuse
+decode into consumer tiles" cases need *no new Halide feature at all*. Ordinary
+`.compute_at()` / `.compute_inline()` on `ApproximationResult`'s `replacement`
+and `handles` (in particular `enc.encoded`, the piece that needs to be fused
+into the producer for the activation case) already achieves this, since they're
+just regular Funcs sitting in the call graph. `compute_offline` (below) is
+needed only for the strictly narrower case of actually severing the graph into
+two separately-compiled artifacts — the static-weight case.
+
+## `compute_offline`: v1 scope
+
+**Decision: "seam exposure," not automatic pipeline splitting.** Given one Func
+`f` that should become a compile-time boundary, `compute_offline`-style support
+means:
+
+- `f`'s computation exists in one compile, as a normal `Output` of a Generator
+  (the "offline" artifact).
+- A same-shaped `Input` (e.g. an opaque `Buffer<uint8_t>` for a packed layout)
+  exists in another compile, standing in for `f` (the "online" pipeline).
+- Both are ordinary, statically-declared Generator I/O — no dynamic discovery of
+  new ports mid-`generate()`.
+
+The alternative (true automatic splitting: one Generator definition, two
+artifacts emitted automatically, no extra static I/O declared by the author) was
+considered and rejected for v1: it would require a Generator to discover an
+extra Input/Output *during* `generate()`, based on the structure of a Func graph
+that doesn't exist yet when `configure()` runs and declares I/O — a
+phase-ordering problem, not just an ergonomics one.
+
+### Why this doesn't need new Generator machinery
+
+Generators already support dynamic I/O declared *before* `generate()` runs:
+`configure()` (`src/Generator.h:192`) exists specifically so `add_input<>()`/
+`add_output<>()` (`src/Generator.h:3236` ff.) can be called based on
+`GeneratorParam` values decided earlier. The only in-repo precedent
+(`apps/hannk/halide/conv_generator.cpp:75-81`) uses `configure()` narrowly (to
+pick a `Target`-dependent filter type), but the mechanism generalizes directly:
+`compute_offline`'s "seam" is just another `GeneratorParam`-driven decision
+about which ports to declare.
+
+### Composability with `approximate_by`
+
+Now that `approximate_by`'s substitution is eager (see above), this composes
+cleanly: whichever order `compute_offline`'s `configure()`-time branching and an
+`approximate_by` call run in, within the same `configure()`/`generate()`, the
+graph state at every point *is* the true state — no later lowering pass can
+silently change what a Func calls out from under code that already ran. In
+practice, a Generator authoring an op from scratch (the shape below) usually
+doesn't need `approximate_by` at all — it can just build `enc`/`dec` and
+reference `dec.decoded[0]` directly wherever the op's math needs the value,
+since it's writing the consumer fresh anyway. `approximate_by` earns its keep
+when the consumer already exists as written code the author doesn't want to edit
+by hand (e.g. a shared reduction reused across several ops) — and because it's
+now eager, nothing stops using it *inside* the same `generate()` that also does
+`compute_offline`-style branching, in either order.
+
+### Non-goal for v1: provenance checking
+
+Nothing proposed here guarantees that the `Approximation` used to produce the
+offline artifact in one compile is *actually* the same one the online compile
+expects when decoding it — correctness rests entirely on both sides referencing
+the same registry name (see the Generator shape below). Building real guarantees
+on top of that (e.g. embedding a scheme fingerprint in the artifact, checked at
+load time) is explicitly deferred to a future version; v1 accepts this as a user
+obligation, the same way `apps/ggml/halide`'s quantize/dequantize split already
+relies on convention today.
+
+## Generator shape
+
+There's very little in-repo precedent for a Generator whose `configure()` does
+real work beyond a `Target`/`GeneratorParam` type tweak (`conv_generator.cpp` is
+the only example). The proposed shape leans on the existing three-phase
+lifecycle (`call_configure()` → `call_generate()` → `call_schedule()`,
+`src/Generator.h:3826-3829`) much harder than any existing Generator does:
+
+- **`configure()`** is the real "compiler driver": it looks up `Approximation`
+  objects by name from an independent, non-Generator registry
+  (`make_approximation(name)`, a lookup table of named schemes), decides which
+  Inputs/Outputs to declare based on `GeneratorParam`s (per-argument scheme
+  choice, a `quantize_only` mode, a `weight_precomputed` flag selecting between
+  the split and combined-artifact shapes), and stores the selected
+  `Approximation`s as members.
+- **`generate()`** must exist (`static_assert(has_generate_method_v<T>)`,
+  `src/Generator.h:3901`) but can be a thin pass-through: it builds the actual
+  op math using the `Approximation`s selected in `configure()` (weight value via
+  `decode`, activation `encode`d and fused at whatever granularity, the op's own
+  reduction) and assigns the result to the `Output<>` members declared in
+  `configure()`.
+- **`schedule()`** is a thin dispatch reading further `GeneratorParam`s to pick
+  a named tiling/fusion strategy, rather than scheduling being written inline in
+  `generate()` (the style every existing Generator in this repo uses).
+
+```cpp
+class MatMulOp : public Generator<MatMulOp> {
+public:
+    GeneratorParam<std::string> weight_scheme_{"weight_scheme", "q4_0"};
+    GeneratorParam<std::string> activation_scheme_{"activation_scheme", "q8_0"};
+    GeneratorParam<std::string> quantize_only_{"quantize_only", ""};  // "" | "weight" | "activation"
+    GeneratorParam<bool> weight_precomputed_{"weight_precomputed", true};
+
+    void configure() {
+        weight_approx_ = make_approximation(weight_scheme_);
+        activation_approx_ = make_approximation(activation_scheme_);
+
+        if (!quantize_only_.value().empty()) {
+            x_ = add_input<Buffer<float>>("x", 1);
+            EncodeResult enc = (quantize_only_ == "weight" ? weight_approx_ : activation_approx_)->encode(Func(*x_));
+            for (size_t i = 0; i < enc.encoded.size(); i++) {
+                *add_output<Buffer<void>>("packed_" + std::to_string(i), enc.encoded[i].dimensions()) = enc.encoded[i];
+            }
+            // enc.handles (e.g. per-block reduction Funcs) still need scheduling here too.
+            return;
+        }
+
+        if (weight_precomputed_) {
+            weight_packed_ = add_input<Buffer<void>>("weight_packed", 2);
+        } else {
+            weight_fp32_ = add_input<Buffer<float>>("weight_fp32", 2);
+        }
+        activation_ = add_input<Buffer<float>>("activation", 1);
+        result_ = add_output<Buffer<float>>("result", 1);
+    }
+
+    void generate() {
+        Func weight_value = weight_precomputed_
+            ? weight_approx_->decode({Func(*weight_packed_)}).decoded[0]
+            : weight_approx_->decode(weight_approx_->encode(Func(*weight_fp32_)).encoded).decoded[0];
+        EncodeResult activation_enc = activation_approx_->encode(Func(*activation_));
+        // activation_enc.encoded / .handles get fused at whatever granularity
+        // schedule() picks -- see "Scope: placement is not semantics" above.
+        *result_ = /* the actual matmul reduction, calling weight_value / activation_enc.encoded inline */;
+    }
+
+    void schedule() {
+        // e.g. pick a named tiling strategy based on another GeneratorParam.
+    }
+
+private:
+    std::unique_ptr<Approximation> weight_approx_, activation_approx_;
+    Input<Buffer<float>> *x_ = nullptr, *weight_fp32_ = nullptr, *activation_ = nullptr;
+    Input<Buffer<void>> *weight_packed_ = nullptr;
+    Output<Buffer<float>> *result_ = nullptr;
+};
+```
+
+Known rough edge, accepted for now: `add_input`/`add_output` return raw
+pointers, needing member-pointer bookkeeping (`Input<Buffer<float>> *x_`) that
+the static `Input<>`/`Output<>` member style used elsewhere in this repo doesn't
+need. Revisiting this ergonomics gap is out of scope until the rest of this
+design is validated.
+
+## Summary of decisions and open items
+
+| Item                                                                                        | Status                                                                                                       |
+| ------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+| `Approximation`: virtual-dispatch abstract class, operates on `Func`s only                  | Decided                                                                                                      |
+| `Approximation` makes no placement claims (offline vs fused)                                | Decided                                                                                                      |
+| `encode`/`decode` return `(funcs, handles)`, not bare `vector<Func>`                        | Decided; handles are scheduling-only intermediates, kept separate from the signature-contract output         |
+| `decode(encode(f).encoded).decoded[0]` signature contract                                   | Decided; enforced only at `approximate_by`'s substitution time in v1, not at `Approximation`-definition time |
+| `encode`'s output arity/layout (packed vs planar)                                           | Left to each `Approximation`; not decided by the framework                                                   |
+| `approximate_by`: eager, destructive `substitute_calls`, not `Func::in`                     | Decided; needed so it composes with `compute_offline` — see rejected-`Func::in` writeup above                |
+| `approximate_by` is a `Func` member (`f.approximate_by(p, consumers)`), not a free function | Decided; reads like the rest of the `in()`/`rfactor()`/`clone_in()` graph-editing family                     |
+| `approximate_by` requires explicit, already-existing `consumers`                            | Decided trade-off; no more "global" wrapper covering not-yet-written Funcs, same scoping as `rfactor`        |
+| `compute_offline`: seam exposure via static, `GeneratorParam`-driven `configure()`-time I/O | Decided for v1                                                                                               |
+| `compute_offline`: true automatic pipeline splitting                                        | Rejected for v1 (phase-ordering conflict with `configure()`/`generate()`)                                    |
+| `compute_offline`: cross-compile provenance checking                                        | Explicitly deferred to v2; v1 relies on "same registry name"                                                 |
+| `compute_offline` + `approximate_by` used together in one pipeline                          | Supported — both are eager/destructive, sequenced by ordinary program order, like `rfactor` + scheduling     |
+| Fusing `encode`/`decode` into neighboring stages (activation requantization)                | No new mechanism needed — ordinary `.compute_at()`/`.compute_inline()`                                       |
+| Generator I/O ergonomics (`add_input`/`add_output` pointer bookkeeping)                     | Accepted rough edge, deferred                                                                                |
+
+## Prior art referenced
+
+- `apps/ggml/halide/*_generators.cpp` — the manually-duplicated
+  quantize/dequantize/vec_dot implementations this design generalizes.
+- A private Python research prototype exploring the same compositional
+  `Approximation` idea, including its `(funcs, handles)` return convention — not
+  a public artifact, referenced here only for context.
+- `apps/hannk/halide/conv_generator.cpp` — sole in-repo precedent for a
+  non-trivial `configure()`.
+- `src/Func.cpp: Stage::rfactor` — the eager, destructive graph-editing
+  precedent `approximate_by` follows instead of `.in()`.
+- `src/Func.cpp`, `src/Function.cpp`, `src/WrapCalls.cpp` — origin of
+  `Function::substitute_calls`, the primitive `approximate_by` calls directly
+  and eagerly instead of through the deferred wrapper map.
+- `src/Generator.h` — the `configure()`/`generate()`/`schedule()` lifecycle
+  `compute_offline` and the Generator shape build on.
diff --git a/src/Approximation.cpp b/src/Approximation.cpp
new file mode 100644
index 000000000000..dc7777b236fe
--- /dev/null
+++ b/src/Approximation.cpp
@@ -0,0 +1,77 @@
+#include "Approximation.h"
+
+#include "Error.h"
+
+namespace Halide {
+
+EncodeResult Compose::encode(std::vector<Func> inputs) {
+    user_assert(!stages_.empty()) << "Compose::encode: no stages\n";
+
+    std::vector<Func> handles;
+    std::vector<Func> current = std::move(inputs);
+    for (int i = (int)stages_.size() - 1; i >= 0; i--) {
+        EncodeResult r = stages_[i]->encode(std::move(current));
+        if (i > 0) {
+            // Not the final (outermost) stage -- its encoded output is an
+            // intermediate between stages, so it needs scheduling like any
+            // other handle, but isn't part of the signature contract this
+            // Compose itself returns.
+            handles.insert(handles.end(), r.encoded.begin(), r.encoded.end());
+        }
+        handles.insert(handles.end(), r.handles.begin(), r.handles.end());
+        current = std::move(r.encoded);
+    }
+    return {current, handles};
+}
+
+DecodeResult Compose::decode(std::vector<Func> encoded) {
+    user_assert(!stages_.empty()) << "Compose::decode: no stages\n";
+
+    std::vector<Func> handles;
+    std::vector<Func> current = std::move(encoded);
+    for (int i = 0; i < (int)stages_.size(); i++) {
+        DecodeResult r = stages_[i]->decode(std::move(current));
+        if (i + 1 < (int)stages_.size()) {
+            handles.insert(handles.end(), r.decoded.begin(), r.decoded.end());
+        }
+        handles.insert(handles.end(), r.handles.begin(), r.handles.end());
+        current = std::move(r.decoded);
+    }
+    return {current, handles};
+}
+
+EncodeResult Apply::encode(std::vector<Func> inputs) {
+    user_assert(idx_ + encode_arity_ <= (int)inputs.size())
+        << "Apply::encode: idx (" << idx_ << ") + encode_arity (" << encode_arity_
+        << ") exceeds the input count (" << inputs.size() << ")\n";
+    std::vector<Func> target(inputs.begin() + idx_, inputs.begin() + idx_ + encode_arity_);
+    EncodeResult inner_result = inner_->encode(std::move(target));
+
+    std::vector<Func> encoded(inputs.begin(), inputs.begin() + idx_);
+    encoded.insert(encoded.end(), inner_result.encoded.begin(), inner_result.encoded.end());
+    encoded.insert(encoded.end(), inputs.begin() + idx_ + encode_arity_, inputs.end());
+    return {encoded, inner_result.handles};
+}
+
+DecodeResult Apply::decode(std::vector<Func> encoded) {
+    user_assert(idx_ + decode_arity_ <= (int)encoded.size())
+        << "Apply::decode: idx (" << idx_ << ") + decode_arity (" << decode_arity_
+        << ") exceeds the input count (" << encoded.size() << ")\n";
+    std::vector<Func> target(encoded.begin() + idx_, encoded.begin() + idx_ + decode_arity_);
+    DecodeResult inner_result = inner_->decode(std::move(target));
+
+    std::vector<Func> decoded(encoded.begin(), encoded.begin() + idx_);
+    decoded.insert(decoded.end(), inner_result.decoded.begin(), inner_result.decoded.end());
+    decoded.insert(decoded.end(), encoded.begin() + idx_ + decode_arity_, encoded.end());
+    return {decoded, inner_result.handles};
+}
+
+EncodeResult TrustedInverse::encode(std::vector<Func> inputs) {
+    return encoder_->encode(std::move(inputs));
+}
+
+DecodeResult TrustedInverse::decode(std::vector<Func> encoded) {
+    return decoder_->decode(std::move(encoded));
+}
+
+}  // namespace Halide
diff --git a/src/Approximation.h b/src/Approximation.h
new file mode 100644
index 000000000000..d11ba0b65b47
--- /dev/null
+++ b/src/Approximation.h
@@ -0,0 +1,333 @@
+#ifndef HALIDE_APPROXIMATION_H
+#define HALIDE_APPROXIMATION_H
+
+/** \file
+ * Defines Approximation, a core interface for lossy, quantified
+ * Func-to-Func transformations (e.g. a quantize/dequantize round trip), and
+ * Compose/Apply, which build larger Approximations out of smaller ones. See
+ * Func::approximate_by(), which splices such a round trip into an existing
+ * call graph, and doc/ApproximationDesign.md for the design rationale.
+ */
+
+#include <memory>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "Func.h"
+
+namespace Halide {
+
+/** The result of Approximation::encode(): the Func(s) that make up the
+ * signature contract other code is expected to consume, plus any extra
+ * intermediate Funcs ("handles") that have no meaning outside scheduling
+ * (e.g. per-block reduction Funcs) but must still be scheduled by whoever
+ * calls encode(). */
+struct EncodeResult {
+    std::vector<Func> encoded;
+    std::vector<Func> handles;
+};
+
+/** The result of Approximation::decode(): decoded is the round-trip
+ * replacement for whatever Func(s) were originally encoded, plus any
+ * additional scheduling-only handles. When an Approximation is used
+ * directly with Func::approximate_by(), decoded must contain exactly one
+ * Func; when it's used as one stage of a larger Compose/Apply chain,
+ * decoded may contain however many Funcs the next stage down expects. */
+struct DecodeResult {
+    std::vector<Func> decoded;
+    std::vector<Func> handles;
+};
+
+/** Approximation is the base class for a lossy, quantified transformation
+ * of one or more Funcs' values -- e.g. quantize-then-dequantize. Unlike an
+ * ordinary schedule directive, an Approximation deliberately changes the
+ * *value* computed, not just how or where it's computed: decode(encode(f))
+ * is expected to approximately reproduce f, not exactly reproduce it.
+ *
+ * encode()/decode() take and return a *vector* of Funcs, not a single Func,
+ * even though the common case (a leaf Approximation like a plain quantizer)
+ * only ever uses one. This is what makes Compose and Apply below possible:
+ * a composed Approximation's inner stage can produce multiple Funcs (e.g. a
+ * quantized-values Func plus a separate scale Func), and the next stage
+ * needs to be able to consume all of them, or select just one to act on.
+ *
+ * An Approximation makes no claim about *where* or *when* encode/decode are
+ * computed relative to the rest of a pipeline (offline vs fused inline,
+ * compute_root vs compute_at) -- that is a scheduling decision, orthogonal
+ * to the semantics defined here. Concretely: the same Approximation can be
+ * used with encode() computed once, offline, ahead of any other stage (a
+ * static weight quantizer) or fused into a producer's inner loop and
+ * recomputed on every call (dynamic activation requantization) -- nothing
+ * about the interface favors one over the other. See Func::approximate_by()
+ * for splicing an Approximation into an existing call graph. */
+class Approximation {
+public:
+    virtual ~Approximation() = default;
+
+    /** Produce the encoded form of `inputs`. EncodeResult::encoded's
+     * elements are not required to have the same type, dimensionality, or
+     * count as `inputs` -- an Approximation is free to choose a packed
+     * representation (a single opaque byte buffer, fields recovered via
+     * reinterpret<>() inside decode) or a planar one (multiple typed Funcs,
+     * one per field). Either is legitimate; the framework does not
+     * decide. */
+    virtual EncodeResult encode(std::vector<Func> inputs) = 0;
+
+    /** Reconstruct an approximation of the original Func(s) from their
+     * encoded form. See DecodeResult for the constraint on `decoded`'s
+     * size, which depends on how this Approximation is used. */
+    virtual DecodeResult decode(std::vector<Func> encoded) = 0;
+};
+
+namespace Internal {
+
+/** Not for direct use. Type-erases an Approximation-derived value (or an
+ * already-type-erased std::unique_ptr<Approximation>, for the rare case
+ * where the concrete type is only known at runtime, e.g. chosen by an
+ * if/else) into an owned std::unique_ptr<Approximation> -- what lets
+ * Compose/Apply's constructors accept a plain mix of concrete Approximation
+ * values while still handling runtime-chosen ones, without exposing that
+ * distinction as something a caller has to think about. */
+// @{
+template<typename T>
+std::unique_ptr<Approximation> approximation_ptr(T &&value) {
+    return std::make_unique<std::decay_t<T>>(std::forward<T>(value));
+}
+inline std::unique_ptr<Approximation> approximation_ptr(std::unique_ptr<Approximation> value) {
+    return value;
+}
+// @}
+
+}  // namespace Internal
+
+/** The result of Func::approximate_by(): the primary replacement Func
+ * (already spliced into every Func in `consumers`), plus every
+ * intermediate Func produced by encode()/decode() along the way that needs
+ * scheduling (compute_root, compute_at, etc.) -- none of `handles` are
+ * part of the Approximation's signature contract, but Halide still
+ * requires Funcs with update definitions to be scheduled, and the fusion
+ * patterns described on Approximation above (e.g. compute_at-ing the
+ * encoded Func into a producer) are only possible if the caller has a
+ * handle to schedule. */
+struct ApproximationResult {
+    Func replacement;
+    /** The Func(s) produced by encode() -- the signature-contract boundary
+     * between the original values and their approximated form (e.g. a
+     * quantizer's packed byte buffer). This is a subset of `handles` (kept
+     * there too, so existing code that schedules everything in `handles`
+     * doesn't need to change), broken out separately so callers can act on
+     * exactly this boundary -- e.g. Pipeline::compute_offline(result.encoded)
+     * -- without calling Approximation::encode() themselves. */
+    std::vector<Func> encoded;
+    std::vector<Func> handles;
+};
+
+/** Sequentially composes any number of Approximations into a pipeline:
+ * encode() runs `stages` back-to-front (the last stage first, on the
+ * original inputs), feeding each stage's encoded output to the one before
+ * it; decode() runs the mirror image, front-to-back. So `stages[0]` is the
+ * "outermost" stage -- the one whose encode() output is this Compose's own
+ * encoded result, and whose decode() input is this Compose's own encoded
+ * argument -- and `stages.back()` is "innermost", closest to the original
+ * values. This generalizes what used to be a fixed two-stage
+ * `Compose(outer, inner)`; that's just the two-element case.
+ *
+ * Compose owns every stage: each constructor argument is moved into (or, if
+ * already a std::unique_ptr<Approximation> -- e.g. because the concrete
+ * type was only known at runtime -- taken as) internal storage, so callers
+ * don't need to keep named locals alive alongside the Compose itself:
+ *
+ * \code
+ * Compose scheme{
+ *     StructPack{...},
+ *     Apply{1, 1, 1, Fp16Pack{}},
+ *     SymmetricAffineQuantize{block_size, qmax, rounding, anchor},
+ * };
+ * \endcode
+ */
+class Compose : public Approximation {
+public:
+    explicit Compose(std::vector<std::unique_ptr<Approximation>> stages)
+        : stages_(std::move(stages)) {
+    }
+
+    template<typename... Stages>
+    explicit Compose(Stages &&...stages) {
+        stages_.reserve(sizeof...(Stages));
+        (stages_.push_back(Internal::approximation_ptr(std::forward<Stages>(stages))), ...);
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override;
+    DecodeResult decode(std::vector<Func> encoded) override;
+
+private:
+    std::vector<std::unique_ptr<Approximation>> stages_;
+};
+
+class ComposeBuilder {
+public:
+    template<typename Stage>
+    ComposeBuilder &add(Stage &&stage) {
+        stages_.emplace_back(Internal::approximation_ptr(std::forward<Stage>(stage)));
+        return *this;
+    }
+
+    [[nodiscard]] std::unique_ptr<Approximation> build() {
+        return std::make_unique<Compose>(std::move(stages_));
+    }
+
+private:
+    std::vector<std::unique_ptr<Approximation>> stages_;
+};
+
+/** Applies `inner` to just the sub-range `[idx, idx + arity)` of a Func
+ * vector, passing every other element through unchanged -- e.g. applying a
+ * quantizer to just the "shifted" component of an affine (shift + scale)
+ * scheme's encoded output while leaving the shift amount itself untouched.
+ * `encode_arity`/`decode_arity` (how many Funcs `inner` consumes at that
+ * position for each direction) must be given explicitly, since C++ has no
+ * way to infer them generically from `inner` itself. Apply owns `inner` --
+ * moved in (or taken directly, if already a std::unique_ptr<Approximation>)
+ * -- the same way Compose owns its stages. */
+class Apply : public Approximation {
+public:
+    template<typename Inner>
+    Apply(int idx, int encode_arity, int decode_arity, Inner &&inner)
+        : idx_(idx), encode_arity_(encode_arity), decode_arity_(decode_arity),
+          inner_(Internal::approximation_ptr(std::forward<Inner>(inner))) {
+    }
+
+    template<typename Inner>
+    Apply(int idx, Inner &&inner)
+        : Apply(idx, 1, 1, std::forward<Inner>(inner)) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override;
+    DecodeResult decode(std::vector<Func> encoded) override;
+
+private:
+    int idx_, encode_arity_, decode_arity_;
+    std::unique_ptr<Approximation> inner_;
+};
+
+/** Routes encode() to one Approximation and decode() to another, taking each
+ * direction from a *different* source. This is the deliberate backdoor out of
+ * the structural guarantee Compose provides.
+ *
+ * Every Approximation is meant to be an approximate identity, factored into a
+ * decode-after-encode pair (decode(encode(f)) ~= f). Compose preserves that by
+ * construction: it interleaves its stages' encode()s and decode()s in mirror
+ * order, so the composed round trip (d1 . d2) . (e2 . e1) is *guaranteed* to be
+ * an approximate identity for the same structural reason each stage is -- the
+ * two halves provably come from one stage list. TrustedInverse pairs an encode
+ * and a decode from unrelated Approximations, so nothing structural guarantees
+ * they compose to an identity: the caller is *trusted* to have supplied a true
+ * inverse pair. Hence the name -- "trusted" as in "taken on trust", not "known
+ * safe".
+ *
+ * The motivating case: a scheme whose forward map (quantize) is an opaque
+ * offline black box -- a per-block codeword search, a transcendental scale fit,
+ * typically an extern call -- that no composition of Halide Funcs reproduces
+ * bit-for-bit, but whose reverse map (dequantize) *is* an ordinary Compose of
+ * invertible primitives. Compose can't express that pairing; TrustedInverse
+ * can, keeping the decode side a clean composition while the encode side is
+ * whatever opaque Approximation actually produces the encoded form:
+ *
+ * \code
+ * TrustedInverse{
+ *     ExternQuantize{"q4_k_quantize_via_ggml"},   // encode(): values -> bytes
+ *     Compose{                                      // decode(): bytes -> values
+ *         StructPack{...}, Apply{...}, ..., BlockReshape{block_size},
+ *     },
+ * };
+ * \endcode
+ *
+ * The unused half of each side is never called (here, the ExternQuantize's
+ * decode() and the Compose's encode()); supplying an Approximation whose
+ * relevant half is a stub is expected. TrustedInverse owns both sides the same
+ * way Compose/Apply own their stages -- moved in, or taken directly if already
+ * a std::unique_ptr<Approximation>. */
+class TrustedInverse : public Approximation {
+public:
+    template<typename Enc, typename Dec>
+    TrustedInverse(Enc &&encoder, Dec &&decoder)
+        : encoder_(Internal::approximation_ptr(std::forward<Enc>(encoder))),
+          decoder_(Internal::approximation_ptr(std::forward<Dec>(decoder))) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override;
+    DecodeResult decode(std::vector<Func> encoded) override;
+
+private:
+    std::unique_ptr<Approximation> encoder_, decoder_;
+};
+
+/** Picks one of two Approximations at construction time based on `cond` */
+class Choose : public Approximation {
+public:
+    template<typename True, typename False>
+    Choose(bool cond, True &&if_true, False &&if_false)
+        : chosen_(cond ? Internal::approximation_ptr(std::forward<True>(if_true)) :
+                         Internal::approximation_ptr(std::forward<False>(if_false))) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) {
+        return chosen_->encode(std::move(inputs));
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) {
+        return chosen_->decode(std::move(encoded));
+    }
+
+private:
+    std::unique_ptr<Approximation> chosen_;
+};
+
+class Identity : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        return {inputs, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        return {encoded, {}};
+    }
+};
+
+class Permute : public Approximation {
+public:
+    explicit Permute(std::vector<int> permutation) : forward_(std::move(permutation)) {
+        backward_.resize(forward_.size());
+        for (int i = 0; i < (int)forward_.size(); i++) {
+            backward_[forward_[i]] = i;
+        }
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == forward_.size()) << "Permutation size does not match input size";
+        std::vector<Func> result;
+        result.reserve(inputs.size());
+        for (int i = 0; i < (int)inputs.size(); i++) {
+            result.push_back(inputs[forward_[i]]);
+        }
+        return {result, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == forward_.size()) << "Permutation size does not match encoded size";
+        std::vector<Func> result;
+        result.reserve(encoded.size());
+        for (int i = 0; i < (int)encoded.size(); i++) {
+            result.push_back(encoded[backward_[i]]);
+        }
+        return {result, {}};
+    }
+
+private:
+    std::vector<int> forward_, backward_;
+};
+
+}  // namespace Halide
+
+#endif
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 511917c9f6b7..7ea25905b3a9 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -61,6 +61,7 @@ target_sources(
     AlignLoads.h
     AllocationBoundsInference.h
     ApplySplit.h
+    Approximation.h
     Argument.h
     AssociativeOpsTable.h
     Associativity.h
@@ -246,6 +247,7 @@ target_sources(
     AlignLoads.cpp
     AllocationBoundsInference.cpp
     ApplySplit.cpp
+    Approximation.cpp
     Argument.cpp
     AssociativeOpsTable.cpp
     Associativity.cpp
diff --git a/src/Func.cpp b/src/Func.cpp
index 445da0323c65..c895ea799bc8 100644
--- a/src/Func.cpp
+++ b/src/Func.cpp
@@ -13,6 +13,7 @@
 #endif
 
 #include "ApplySplit.h"
+#include "Approximation.h"
 #include "Argument.h"
 #include "Associativity.h"
 #include "Bounds.h"
@@ -2852,6 +2853,41 @@ Func Func::clone_in(const vector<Func> &fs) {
     return get_wrapper(func, name() + "_clone", fs, true);
 }
 
+ApproximationResult Func::approximate_by(Approximation &p, const vector<Func> &consumers) {
+    EncodeResult enc = p.encode({*this});
+    user_assert(!enc.encoded.empty())
+        << "approximate_by: Approximation::encode(" << name() << ") returned no Funcs\n";
+
+    DecodeResult dec = p.decode(enc.encoded);
+    user_assert(dec.decoded.size() == 1)
+        << "approximate_by: Approximation::decode() must return exactly one Func (the "
+        << "round-trip replacement), but returned " << dec.decoded.size() << "\n";
+
+    Func round_trip = dec.decoded[0];
+    user_assert(round_trip.dimensions() == dimensions())
+        << "approximate_by: decode(encode(" << name() << "))'s result (" << round_trip.name()
+        << ") has " << round_trip.dimensions() << " dimensions, but " << name() << " has "
+        << dimensions() << " -- Approximation implementations must reproduce the original "
+        << "Func's signature exactly\n";
+    user_assert(round_trip.types() == types())
+        << "approximate_by: decode(encode(" << name() << "))'s result (" << round_trip.name()
+        << ") has a different type than " << name() << " -- Approximation implementations "
+        << "must reproduce the original Func's signature exactly\n";
+
+    for (const Func &g : consumers) {
+        user_assert(g.name() != name())
+            << "approximate_by: " << name() << " cannot be its own consumer\n";
+        // Eager and destructive, like Func::rfactor() -- not deferred to
+        // lowering the way Func::in() is. See Approximation.h for why.
+        g.function().substitute_calls(func, round_trip.function());
+    }
+
+    vector<Func> handles = enc.encoded;
+    handles.insert(handles.end(), enc.handles.begin(), enc.handles.end());
+    handles.insert(handles.end(), dec.handles.begin(), dec.handles.end());
+    return {round_trip, enc.encoded, handles};
+}
+
 Func Func::copy_to_device(DeviceAPI d) {
     user_assert(defined())
         << "copy_to_device on Func " << name() << " with no definition\n";
diff --git a/src/Func.h b/src/Func.h
index f08f2c88cda3..56d746d3f6cd 100644
--- a/src/Func.h
+++ b/src/Func.h
@@ -60,6 +60,8 @@ struct VarOrRVar {
 
 class ImageParam;
 class FuncVec;
+class Approximation;
+struct ApproximationResult;
 
 namespace Internal {
 struct AssociativeOp;
@@ -1557,6 +1559,18 @@ class Func {
     Func clone_in(const std::vector<Func> &fs);
     //@}
 
+    /** Eagerly and destructively replace every call to this Func inside
+     * each Func in 'consumers' with a call to the round trip
+     * p.decode(p.encode(*this)) instead. The substitution happens
+     * immediately, the same way Func::rfactor() immediately rewrites a
+     * Func's definition, rather than being deferred to lowering the way
+     * Func::in() is -- see Approximation.h and doc/ApproximationDesign.md
+     * for the rationale. Because the substitution is eager, it can only
+     * rewrite Funcs that are already fully defined at the point of the
+     * call -- there is no "global" mode that also covers Funcs written
+     * later, unlike Func::in(). */
+    ApproximationResult approximate_by(Approximation &p, const std::vector<Func> &consumers);
+
     /** Declare that this function should be implemented by a call to
      * halide_buffer_copy with the given target device API. Asserts
      * that the Func has a pure definition which is a simple call to a
diff --git a/src/Generator.cpp b/src/Generator.cpp
index 24e0226cb1ec..0ec7988c923a 100644
--- a/src/Generator.cpp
+++ b/src/Generator.cpp
@@ -1665,6 +1665,28 @@ void GeneratorBase::pre_schedule() {
 void GeneratorBase::post_schedule() {
 }
 
+GeneratorInput<Buffer<>> *GeneratorBase::add_input(const ImageParam &existing) {
+    check_exact_phase(GeneratorBase::ConfigureCalled);
+    claim_name(existing.name(), "input");
+    auto *p = new GeneratorInput<Buffer<>>(existing.name(), existing.type(), existing.dimensions());
+    p->generator = this;
+    p->adopt(existing.parameter());
+    param_info_ptr->owned_extras.push_back(std::unique_ptr<Internal::GIOBase>(p));
+    param_info_ptr->filter_inputs.push_back(p);
+    return p;
+}
+
+GeneratorOutput<Buffer<>> *GeneratorBase::add_output(const Func &existing) {
+    check_exact_phase(GeneratorBase::ConfigureCalled);
+    claim_name(existing.name(), "output");
+    auto *p = new GeneratorOutput<Buffer<>>(existing.name(), existing.types(), existing.dimensions());
+    p->generator = this;
+    p->adopt(existing);
+    param_info_ptr->owned_extras.push_back(std::unique_ptr<Internal::GIOBase>(p));
+    param_info_ptr->filter_outputs.push_back(p);
+    return p;
+}
+
 void GeneratorBase::add_requirement(const Expr &condition, const std::vector<Expr> &error_args) {
     internal_assert(!pipeline.defined());
     requirements.push_back({condition, error_args});
@@ -2218,6 +2240,23 @@ void GeneratorInputBase::set_estimates_impl(const Region &estimates) {
     }
 }
 
+void GeneratorInputBase::adopt(const Parameter &p) {
+    user_assert(p.defined()) << "GeneratorBase::add_input(const ImageParam &): "
+                             << name() << " is not defined.\n";
+    user_assert(p.is_buffer()) << "GeneratorBase::add_input(const ImageParam &): "
+                               << name() << " is not a Buffer Parameter.\n";
+    check_matching_types({p.type()});
+    check_matching_dims(p.dimensions());
+    parameters_.clear();
+    exprs_.clear();
+    funcs_.clear();
+    funcs_.push_back(make_param_func(p, name()));
+    parameters_.push_back(p);
+    set_def_min_max();
+    verify_internals();
+    inputs_set = true;
+}
+
 GeneratorOutputBase::GeneratorOutputBase(size_t array_size, const std::string &name, ArgInfoKind kind, const std::vector<Type> &t, int d)
     : GIOBase(array_size, name, kind, t, d) {
     internal_assert(kind != ArgInfoKind::Scalar);
@@ -2239,6 +2278,9 @@ void GeneratorOutputBase::check_value_writable() const {
 }
 
 void GeneratorOutputBase::init_internals() {
+    if (adopted_) {
+        return;
+    }
     exprs_.clear();
     funcs_.clear();
     if (array_size_defined()) {
@@ -2250,6 +2292,17 @@ void GeneratorOutputBase::init_internals() {
     }
 }
 
+void GeneratorOutputBase::adopt(const Func &f) {
+    user_assert(f.defined()) << "GeneratorBase::add_output(const Func &): "
+                             << name() << " is not defined.\n";
+    check_matching_types(f.types());
+    check_matching_dims(f.dimensions());
+    exprs_.clear();
+    funcs_.clear();
+    funcs_.push_back(f);
+    adopted_ = true;
+}
+
 void GeneratorOutputBase::resize(size_t size) {
     internal_assert(is_array());
     internal_assert(!array_size_defined()) << "You may only call " << name()
diff --git a/src/Generator.h b/src/Generator.h
index d88bff8bea3f..bd39c7d1a640 100644
--- a/src/Generator.h
+++ b/src/Generator.h
@@ -1578,6 +1578,14 @@ class GeneratorInputBase : public GIOBase {
     void set_inputs(const std::vector<StubInput> &inputs);
     bool inputs_set = false;
 
+    // Adopts `p` (which must be a defined Buffer Parameter) as this Input's
+    // backing Parameter directly, immediately (not waiting for the
+    // ordinary InputsSet phase set_inputs() above is used for) -- sets
+    // inputs_set so init_internals() leaves it alone. Only used by
+    // GeneratorBase::add_input(const ImageParam &); see there for why this
+    // exists.
+    void adopt(const Parameter &p);
+
     virtual void set_def_min_max();
 
     void verify_internals() override;
@@ -2396,6 +2404,15 @@ class GeneratorOutputBase : public GIOBase {
         return "Output";
     }
 
+    // Adopts `f` (which must already be defined) as this Output's value
+    // directly, bypassing the usual "generate() assigns via operator()="
+    // path -- so init_internals() must leave `funcs_` alone once this has
+    // been called, the same way GeneratorInputBase::init_internals() skips
+    // its own rebuild when inputs_set is true. Only used by
+    // GeneratorBase::add_output(Func); see there for why this exists.
+    void adopt(const Func &f);
+    bool adopted_ = false;
+
 public:
     ~GeneratorOutputBase() override;
 };
@@ -3433,6 +3450,27 @@ class GeneratorBase : public NamesInterface, public AbstractGenerator {
         return p;
     }
 
+    /** Declares an Input<Buffer<>> backed directly by `existing` -- e.g. an
+     * ImageParam a Pipeline::compute_offline() call minted, or a
+     * Func::approximate_by() round trip severed into -- instead of a fresh
+     * one for generate() to leave for the caller to Buffer::set(). Useful
+     * when a Generator's whole pipeline is assembled in configure() (see
+     * doc/ApproximationDesign.md's "Usage shape" section for the motivating
+     * case: quantize/dequantize sharing a single approximate_by()/
+     * compute_offline() call, each adopting whichever half applies to it),
+     * leaving generate() an empty stub. `existing` must already be defined
+     * (have a concrete type/dimensionality); its own name is used as the
+     * port's name. */
+    GeneratorInput<Buffer<>> *add_input(const ImageParam &existing);
+
+    /** Declares an Output<Buffer<>> whose value is `existing` directly --
+     * e.g. the encoded/decoded Func half of an approximate_by()/
+     * compute_offline() split -- instead of a fresh, undefined Func for
+     * generate() to assign via operator(). See add_input(const ImageParam&)
+     * above for the motivating use. `existing` must already be defined;
+     * its own name is used as the port's name. */
+    GeneratorOutput<Buffer<>> *add_output(const Func &existing);
+
     void add_requirement(const Expr &condition, const std::vector<Expr> &error_args);
 
     template<typename... Args,
diff --git a/test/correctness/CMakeLists.txt b/test/correctness/CMakeLists.txt
index bb6c5ded26b7..b72b4f159e0c 100644
--- a/test/correctness/CMakeLists.txt
+++ b/test/correctness/CMakeLists.txt
@@ -5,6 +5,7 @@ tests(
     SOURCES
     # keep-sorted start case=no
     align_bounds.cpp
+    approximate_by.cpp
     argmax.cpp
     arm_cpu_detect.cpp
     associativity.cpp
diff --git a/test/correctness/approximate_by.cpp b/test/correctness/approximate_by.cpp
new file mode 100644
index 000000000000..00a79833d07b
--- /dev/null
+++ b/test/correctness/approximate_by.cpp
@@ -0,0 +1,115 @@
+#include "Halide.h"
+#include <cmath>
+#include <cstdio>
+
+using namespace Halide;
+
+namespace {
+
+constexpr int kBlockSize = 8;
+
+// A minimal symmetric integer quantizer -- self-contained (no relation to
+// any specific real-world format), just enough to exercise: encode()
+// returning multiple Funcs plus a genuine scheduling-only handle (the
+// per-block amax reduction), decode() combining them back into a single
+// Func matching the original's signature, and approximate_by()'s eager
+// substitution.
+class SymmetricQuantizer : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        Func f = inputs[0];
+        Var x("x"), i("i");
+        RDom r(0, kBlockSize, "r");
+
+        Func amax("amax");
+        amax(i) = 0.0f;
+        amax(i) = max(amax(i), abs(f(i * kBlockSize + r)));
+
+        Func d("d");
+        d(i) = amax(i) / 127.0f;
+
+        Func q("q");
+        Expr id = select(d(x / kBlockSize) != 0.0f, 1.0f / d(x / kBlockSize), 0.0f);
+        q(x) = cast<int8_t>(clamp(round(f(x) * id), -127, 127));
+
+        return {{q, d}, {amax}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        Func q = encoded[0], d = encoded[1];
+        Var x("x");
+        Func dequantized("dequantized");
+        dequantized(x) = cast<float>(q(x)) * d(x / kBlockSize);
+        return {{dequantized}, {}};
+    }
+};
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    Var x("x");
+
+    Func f("f");
+    f(x) = sin(cast<float>(x) * 0.1f) * 100.0f;
+
+    // g is rewired by approximate_by() below; h is not, and must keep
+    // seeing the exact, unquantized f.
+    Func g("g");
+    g(x) = f(x) * 2.0f + 1.0f;
+
+    Func h("h");
+    h(x) = f(x) * 3.0f;
+
+    SymmetricQuantizer quant;
+    ApproximationResult result = f.approximate_by(quant, {g});
+
+    if (result.handles.empty()) {
+        printf("Expected approximate_by() to return scheduling handles\n");
+        return 1;
+    }
+    result.replacement.compute_root();
+    for (Func handle : result.handles) {
+        handle.compute_root();
+    }
+
+    const int kSize = 64;
+    Buffer<float> g_out = g.realize({kSize});
+    Buffer<float> h_out = h.realize({kSize});
+
+    for (int i = 0; i < kSize; i++) {
+        const float fx = sinf(i * 0.1f) * 100.0f;
+
+        // Independently recompute the same per-block quantization encode()
+        // performs, to build a bit-exact reference for what g should see.
+        const int block = i / kBlockSize;
+        float amax = 0.0f;
+        for (int j = 0; j < kBlockSize; j++) {
+            const float v = sinf((block * kBlockSize + j) * 0.1f) * 100.0f;
+            amax = std::max(amax, std::fabs(v));
+        }
+        const float d = amax / 127.0f;
+        const float id = d != 0.0f ? 1.0f / d : 0.0f;
+        float q = std::round(fx * id);
+        q = std::max(-127.0f, std::min(127.0f, q));
+        const float dequantized = q * d;
+
+        const float expected_g = dequantized * 2.0f + 1.0f;
+        if (std::fabs(g_out(i) - expected_g) > 1e-4f) {
+            printf("g(%d) = %f, expected %f -- approximate_by's substitution did not take effect\n",
+                   i, g_out(i), expected_g);
+            return 1;
+        }
+
+        // h was never passed as a consumer to approximate_by(): it must
+        // see the real f, not the quantized round trip.
+        const float expected_h = fx * 3.0f;
+        if (std::fabs(h_out(i) - expected_h) > 1e-4f) {
+            printf("h(%d) = %f, expected %f -- approximate_by affected a Func not in `consumers`\n",
+                   i, h_out(i), expected_h);
+            return 1;
+        }
+    }
+
+    printf("Success!\n");
+    return 0;
+}

From a38f662908199bb9bebe5654e7ad4a15052baef6 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sat, 1 Aug 2026 20:09:27 -0400
Subject: [PATCH 05/22] Add apps/ggml: Halide reimplementation of GGML quant
 kernels

Ports apps/ggml onto the mature quantized-kernels substrate: a from-scratch
Halide reimplementation of GGML's quantize / dequantize / vec-dot / repack
kernels for the full format catalog (Q4_0..Q8_K, K-quants, IQ*, TQ*, FP4,
f16/bf16), benchmarked against GGML's own kernels via kernel-bench. The
codecs are built as composable Approximation combinators (quant_components.h)
and spliced into pipelines with Func::approximate_by() /
Pipeline::compute_offline().

Wiring: add_app(ggml) + a "ggml" dependency and apps/vcpkg/ports/ggml overlay
port (ggml v0.15.3, CPU static). A minimal apps/ggml/vcpkg.json +
vcpkg-configuration.json make the standalone build the app's CMakeLists
already advertises actually resolve ggml (the integrated apps/ build still
uses the root manifest).

The app was written against the alpha rfactor-hoisting API, which was renamed
and split on the mature branch; the generators are migrated accordingly:
  - inline_calls({...}) -> .update().eager_inline({...})
  - rfactor(..., RFactorOptions::HoistInvariantFactor) ->
    rfactor(...) + eager_inline(...).hoist_invariants() + change_type(...)

Known deviation: the SDOT vec-dot schedules (q4_0/q8_0/q5_0/iq4_nl/mxfp4) are
temporarily set to the correct Float schedule. The mature hoist_invariants()
cannot lift the per-block scale out of the reduction when the dequant is built
through approximate_by()'s round-trip replacement (a single eager_inline()
leaves the scale behind decode-chain Func boundaries), unlike
test/correctness/struct_type_dot_product.cpp which inlines direct dequantizer
Funcs. See the TODOs in symmetric_vec_dot_generator.cpp.

Verified: all 28 codec round-trip tests pass, and kernel-bench --all reports
zero correctness mismatches vs GGML across quantize/dequantize/vec_dot/repack.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/CMakeLists.txt                           |    1 +
 apps/ggml/CMakeLists.txt                      |   81 +
 apps/ggml/README.md                           |   66 +
 apps/ggml/halide/CMakeLists.txt               |  768 +++
 apps/ggml/halide/bf16_generators.cpp          |   58 +
 apps/ggml/halide/codec_generator_base.h       |  114 +
 apps/ggml/halide/f16_generators.cpp           |   55 +
 apps/ggml/halide/ggml_extern_quantize.cpp     |  116 +
 apps/ggml/halide/ggml_quants.cpp              | 1732 ++++++
 apps/ggml/halide/ggml_quants.h                |  276 +
 apps/ggml/halide/iq_grids_data.h              | 4770 +++++++++++++++++
 apps/ggml/halide/k_quant_generators.cpp       |   80 +
 .../ggml/halide/k_quant_vec_dot_generator.cpp |   63 +
 .../halide/lookup_table_quant_generators.cpp  |  116 +
 .../halide/lookup_table_vec_dot_generator.cpp |  114 +
 apps/ggml/halide/quant_components.h           | 3011 +++++++++++
 apps/ggml/halide/repack_matmul_generator.cpp  |  263 +
 .../halide/repack_quantize_mat_generators.cpp |  121 +
 .../halide/symmetric_quant_generators.cpp     |  120 +
 .../halide/symmetric_vec_dot_generator.cpp    |  148 +
 apps/ggml/halide/test_bf16.cpp                |   45 +
 apps/ggml/halide/test_f16.cpp                 |   45 +
 apps/ggml/halide/test_iq1_m.cpp               |   48 +
 apps/ggml/halide/test_iq1_s.cpp               |   47 +
 apps/ggml/halide/test_iq2_s.cpp               |   48 +
 apps/ggml/halide/test_iq2_xs.cpp              |   47 +
 apps/ggml/halide/test_iq2_xxs.cpp             |   52 +
 apps/ggml/halide/test_iq3_s.cpp               |   48 +
 apps/ggml/halide/test_iq3_xxs.cpp             |   48 +
 apps/ggml/halide/test_iq4_nl.cpp              |   46 +
 apps/ggml/halide/test_iq4_xs.cpp              |   46 +
 apps/ggml/halide/test_mxfp4.cpp               |   46 +
 apps/ggml/halide/test_nvfp4.cpp               |   46 +
 apps/ggml/halide/test_q1_0.cpp                |   45 +
 apps/ggml/halide/test_q2_k.cpp                |   49 +
 apps/ggml/halide/test_q3_k.cpp                |   49 +
 apps/ggml/halide/test_q4_0.cpp                |   46 +
 apps/ggml/halide/test_q4_1.cpp                |   46 +
 apps/ggml/halide/test_q4_k.cpp                |   49 +
 apps/ggml/halide/test_q5_0.cpp                |   46 +
 apps/ggml/halide/test_q5_1.cpp                |   46 +
 apps/ggml/halide/test_q5_k.cpp                |   49 +
 apps/ggml/halide/test_q6_k.cpp                |   49 +
 apps/ggml/halide/test_q8_0.cpp                |   46 +
 apps/ggml/halide/test_q8_1.cpp                |   37 +
 apps/ggml/halide/test_q8_k.cpp                |   40 +
 apps/ggml/halide/test_tq1_0.cpp               |   46 +
 apps/ggml/halide/test_tq2_0.cpp               |   46 +
 apps/ggml/halide/vec_dot_generator_base.h     |  151 +
 apps/ggml/include/kernel_registry.h           |  156 +
 apps/ggml/providers/README.md                 |   60 +
 apps/ggml/providers/ggml_internal_abi.h       |  295 +
 apps/ggml/providers/ggml_provider.cpp         |  293 +
 apps/ggml/providers/ggml_provider.h           |   21 +
 apps/ggml/providers/halide_provider.cpp       |  236 +
 apps/ggml/providers/halide_provider.h         |    8 +
 apps/ggml/src/bench_dequantize.cpp            |   67 +
 apps/ggml/src/bench_quantize.cpp              |   65 +
 apps/ggml/src/bench_repack.cpp                |  269 +
 apps/ggml/src/bench_vecdot.cpp                |   83 +
 apps/ggml/src/benchmarks.h                    |   13 +
 apps/ggml/src/compare.h                       |   20 +
 apps/ggml/src/data_gen.h                      |   60 +
 apps/ggml/src/main.cpp                        |  122 +
 apps/ggml/src/report.cpp                      |   50 +
 apps/ggml/src/report.h                        |   42 +
 apps/ggml/src/timing.h                        |   73 +
 apps/ggml/vcpkg-configuration.json            |    5 +
 apps/ggml/vcpkg.json                          |    9 +
 apps/vcpkg.json                               |    1 +
 apps/vcpkg/ports/ggml/portfile.cmake          |   26 +
 apps/vcpkg/ports/ggml/vcpkg.json              |   17 +
 72 files changed, 15516 insertions(+)
 create mode 100644 apps/ggml/CMakeLists.txt
 create mode 100644 apps/ggml/README.md
 create mode 100644 apps/ggml/halide/CMakeLists.txt
 create mode 100644 apps/ggml/halide/bf16_generators.cpp
 create mode 100644 apps/ggml/halide/codec_generator_base.h
 create mode 100644 apps/ggml/halide/f16_generators.cpp
 create mode 100644 apps/ggml/halide/ggml_extern_quantize.cpp
 create mode 100644 apps/ggml/halide/ggml_quants.cpp
 create mode 100644 apps/ggml/halide/ggml_quants.h
 create mode 100644 apps/ggml/halide/iq_grids_data.h
 create mode 100644 apps/ggml/halide/k_quant_generators.cpp
 create mode 100644 apps/ggml/halide/k_quant_vec_dot_generator.cpp
 create mode 100644 apps/ggml/halide/lookup_table_quant_generators.cpp
 create mode 100644 apps/ggml/halide/lookup_table_vec_dot_generator.cpp
 create mode 100644 apps/ggml/halide/quant_components.h
 create mode 100644 apps/ggml/halide/repack_matmul_generator.cpp
 create mode 100644 apps/ggml/halide/repack_quantize_mat_generators.cpp
 create mode 100644 apps/ggml/halide/symmetric_quant_generators.cpp
 create mode 100644 apps/ggml/halide/symmetric_vec_dot_generator.cpp
 create mode 100644 apps/ggml/halide/test_bf16.cpp
 create mode 100644 apps/ggml/halide/test_f16.cpp
 create mode 100644 apps/ggml/halide/test_iq1_m.cpp
 create mode 100644 apps/ggml/halide/test_iq1_s.cpp
 create mode 100644 apps/ggml/halide/test_iq2_s.cpp
 create mode 100644 apps/ggml/halide/test_iq2_xs.cpp
 create mode 100644 apps/ggml/halide/test_iq2_xxs.cpp
 create mode 100644 apps/ggml/halide/test_iq3_s.cpp
 create mode 100644 apps/ggml/halide/test_iq3_xxs.cpp
 create mode 100644 apps/ggml/halide/test_iq4_nl.cpp
 create mode 100644 apps/ggml/halide/test_iq4_xs.cpp
 create mode 100644 apps/ggml/halide/test_mxfp4.cpp
 create mode 100644 apps/ggml/halide/test_nvfp4.cpp
 create mode 100644 apps/ggml/halide/test_q1_0.cpp
 create mode 100644 apps/ggml/halide/test_q2_k.cpp
 create mode 100644 apps/ggml/halide/test_q3_k.cpp
 create mode 100644 apps/ggml/halide/test_q4_0.cpp
 create mode 100644 apps/ggml/halide/test_q4_1.cpp
 create mode 100644 apps/ggml/halide/test_q4_k.cpp
 create mode 100644 apps/ggml/halide/test_q5_0.cpp
 create mode 100644 apps/ggml/halide/test_q5_1.cpp
 create mode 100644 apps/ggml/halide/test_q5_k.cpp
 create mode 100644 apps/ggml/halide/test_q6_k.cpp
 create mode 100644 apps/ggml/halide/test_q8_0.cpp
 create mode 100644 apps/ggml/halide/test_q8_1.cpp
 create mode 100644 apps/ggml/halide/test_q8_k.cpp
 create mode 100644 apps/ggml/halide/test_tq1_0.cpp
 create mode 100644 apps/ggml/halide/test_tq2_0.cpp
 create mode 100644 apps/ggml/halide/vec_dot_generator_base.h
 create mode 100644 apps/ggml/include/kernel_registry.h
 create mode 100644 apps/ggml/providers/README.md
 create mode 100644 apps/ggml/providers/ggml_internal_abi.h
 create mode 100644 apps/ggml/providers/ggml_provider.cpp
 create mode 100644 apps/ggml/providers/ggml_provider.h
 create mode 100644 apps/ggml/providers/halide_provider.cpp
 create mode 100644 apps/ggml/providers/halide_provider.h
 create mode 100644 apps/ggml/src/bench_dequantize.cpp
 create mode 100644 apps/ggml/src/bench_quantize.cpp
 create mode 100644 apps/ggml/src/bench_repack.cpp
 create mode 100644 apps/ggml/src/bench_vecdot.cpp
 create mode 100644 apps/ggml/src/benchmarks.h
 create mode 100644 apps/ggml/src/compare.h
 create mode 100644 apps/ggml/src/data_gen.h
 create mode 100644 apps/ggml/src/main.cpp
 create mode 100644 apps/ggml/src/report.cpp
 create mode 100644 apps/ggml/src/report.h
 create mode 100644 apps/ggml/src/timing.h
 create mode 100644 apps/ggml/vcpkg-configuration.json
 create mode 100644 apps/ggml/vcpkg.json
 create mode 100644 apps/vcpkg/ports/ggml/portfile.cmake
 create mode 100644 apps/vcpkg/ports/ggml/vcpkg.json

diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt
index afc90b173081..70976a979c6f 100644
--- a/apps/CMakeLists.txt
+++ b/apps/CMakeLists.txt
@@ -53,6 +53,7 @@ add_app(cuda_mat_mul)
 add_app(depthwise_separable_conv)
 add_app(fft)
 add_app(gaussian_blur)
+add_app(ggml)
 add_app(hannk)
 add_app(harris)
 # add_app(HelloAndroid)  # don't build HelloAndroid here because it is driven by gradle
diff --git a/apps/ggml/CMakeLists.txt b/apps/ggml/CMakeLists.txt
new file mode 100644
index 000000000000..a06682271249
--- /dev/null
+++ b/apps/ggml/CMakeLists.txt
@@ -0,0 +1,81 @@
+cmake_minimum_required(VERSION 3.28)
+project(ggml)
+
+enable_testing()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+set(CMAKE_CXX_EXTENSIONS NO)
+
+# GGML is expected to be provided externally -- in this tree, via the
+# apps/vcpkg/ports/ggml overlay port (see its portfile.cmake). GGML ships its
+# own CMake package config (installed to <prefix>/share/ggml/ by vcpkg's
+# vcpkg_cmake_config_fixup), so no Find module is needed here.
+find_package(ggml CONFIG REQUIRED)
+
+# Halide is already found once by the parent apps/CMakeLists.txt when this app
+# is built as part of the full apps/ tree (a no-op re-find in that case); this
+# call makes apps/ggml also independently configurable/buildable on its own,
+# matching every other app's convention (see e.g. apps/blur/CMakeLists.txt).
+find_package(Halide REQUIRED)
+
+# halide/ contains a from-scratch Halide reimplementation of GGML's Q4_0
+# quantize/dequantize kernels (ggml_quants_halide), benchmarked against
+# GGML's own reference by providers/halide_provider.cpp below.
+add_subdirectory(halide)
+
+add_executable(
+    kernel-bench
+    src/main.cpp
+    src/report.cpp
+    src/bench_quantize.cpp
+    src/bench_dequantize.cpp
+    src/bench_vecdot.cpp
+    src/bench_repack.cpp
+    providers/ggml_provider.cpp
+    providers/halide_provider.cpp
+)
+
+target_include_directories(kernel-bench PRIVATE include providers src halide)
+
+target_link_libraries(kernel-bench PRIVATE ggml::ggml ggml_quants_halide)
+
+# GGML_VERSION isn't exposed through any runtime API, but ggml-config.cmake
+# sets it as a plain CMake variable (baked in from the exporting build's own
+# GGML_VERSION), so this is exactly the version of the library we just linked
+# against -- reliable without touching GGML internals.
+if (DEFINED GGML_VERSION)
+    target_compile_definitions(kernel-bench PRIVATE KERNEL_BENCH_GGML_VERSION="${GGML_VERSION}")
+endif ()
+
+# ggml-config.cmake.in only creates ggml::<backend> import targets
+# (including ggml::ggml-cpu) when GGML was built with GGML_BACKEND_DL=OFF
+# (the default) -- see its `if (NOT GGML_BACKEND_DL)` guard. In DL mode the
+# CPU backend is a runtime-loaded module with no link-time target at all, and
+# the private ABI symbols this tool depends on (see
+# providers/ggml_internal_abi.h) are then unreachable through the CMake
+# package. Fall back to locating the library file directly by name; fail
+# loudly if that isn't possible either, rather than producing a mysterious
+# link error.
+if (NOT TARGET ggml::ggml-cpu)
+    find_library(
+        GGML_CPU_DL_LIB
+        NAMES ggml-cpu
+        HINTS "${ggml_LIB_DIR}" "${ggml_LIB_DIR}/ggml"
+        PATH_SUFFIXES lib lib/ggml bin
+    )
+    if (GGML_CPU_DL_LIB)
+        message(
+            STATUS "kernel-bench: linking ggml-cpu directly (GGML_BACKEND_DL build): ${GGML_CPU_DL_LIB}"
+        )
+        target_link_libraries(kernel-bench PRIVATE "${GGML_CPU_DL_LIB}")
+    else ()
+        message(
+            FATAL_ERROR "kernel-bench: could not find ggml::ggml-cpu or a standalone ggml-cpu library. "
+            "This GGML install appears to have been built with -DGGML_BACKEND_DL=ON, which "
+            "loads the CPU backend as a runtime module instead of linking it -- kernel-bench "
+            "needs to link directly against its internal symbols. Rebuild GGML with "
+            "-DGGML_BACKEND_DL=OFF (the default) and reinstall."
+        )
+    endif ()
+endif ()
diff --git a/apps/ggml/README.md b/apps/ggml/README.md
new file mode 100644
index 000000000000..1d5adc675b90
--- /dev/null
+++ b/apps/ggml/README.md
@@ -0,0 +1,66 @@
+# kernel-bench
+
+Benchmarks GGML's CPU quantize / dequantize / vec_dot / repack kernels against a
+designated correctness reference, and is structured so that a from-scratch
+implementation of any of those kernels can be dropped in and compared too. See
+`providers/README.md` for how to add one.
+
+This is a **standalone** CMake project. It is not built as part of GGML itself
+and consumes an already-built-and-installed GGML purely as an external
+dependency via `find_package`.
+
+## Build
+
+```sh
+# 1. Build and install GGML somewhere (skip if you already have an install).
+#    GGML_BACKEND_DL=OFF (the default) is required -- see the "private ABI" note below.
+cmake -S /path/to/ggml -B /path/to/ggml/build -DCMAKE_BUILD_TYPE=Release
+cmake --build /path/to/ggml/build -j
+cmake --install /path/to/ggml/build --prefix /path/to/ggml/install
+
+# 2. Build kernel-bench against that install.
+cmake -S . -B build -DCMAKE_PREFIX_PATH=/path/to/ggml/install
+cmake --build build -j
+
+./build/kernel-bench --all
+```
+
+## What the report means
+
+Each row shows a `ggml_type` (or, for repack, a specific interleave layout like
+`q4_0_4x4_q8_0`), GGML's designated **reference** implementation and its
+timing/throughput, and one column per **candidate** implementation registered
+for that kernel: its timing/throughput, speedup relative to the reference, and
+whether its output matched the reference (quantize/repack packing is checked
+byte-for-byte; dequantize/vec_dot/gemv/gemm results are checked within a
+relative-error tolerance, since those involve floating point accumulation that
+different implementations may order differently).
+
+A candidate flagged "identical to reference" has the exact same function address
+as the reference -- this happens whenever the current CPU architecture has no
+separate optimized kernel for that type (GGML's `src/ggml-cpu/arch-fallback.h`
+collapses the two names onto one symbol in that case), so timing it separately
+would only measure noise.
+
+The dequantize table currently shows only a reference column with no candidates:
+GGML has exactly one dequantize implementation per type (arch-independent, in
+`src/ggml-quants.c`), so there's nothing to compare it against yet -- this is
+intentionally the first place to plug in a new provider (see
+`providers/README.md`).
+
+## Why this needs a private ABI header
+
+GGML's public API (`ggml_get_type_traits` / `ggml_get_type_traits_cpu` in
+`include/ggml.h` / `include/ggml-cpu.h`) exposes exactly one reference and one
+CPU-dispatched implementation per type, which is sufficient for the quantize and
+dequantize benchmarks without touching anything private. It does **not** expose
+the always-available pure-C fallback for `vec_dot`, nor anything for the repack
+`quantize_mat`/`gemv`/`gemm` kernels. Those are only reachable because
+`ggml-cpu` is built without `-fvisibility=hidden`, so its internal (but
+non-`static`) C symbols end up with default/exported linker visibility by
+accident of the build configuration rather than by design.
+`providers/ggml_internal_abi.h` redeclares exactly the symbols needed, copied
+from GGML's uninstalled `src/ggml-cpu/quants.h` / `repack.h` as of the commit
+this tool was written against. If a future GGML release renames or changes the
+signature of one of these functions, that header (and
+`providers/ggml_provider.cpp`) are the only places that need updating.
diff --git a/apps/ggml/halide/CMakeLists.txt b/apps/ggml/halide/CMakeLists.txt
new file mode 100644
index 000000000000..e73cc8cdf175
--- /dev/null
+++ b/apps/ggml/halide/CMakeLists.txt
@@ -0,0 +1,768 @@
+add_halide_generator(
+    quants.generator
+    SOURCES
+    f16_generators.cpp
+    bf16_generators.cpp
+    repack_quantize_mat_generators.cpp
+    repack_matmul_generator.cpp
+    symmetric_quant_generators.cpp
+    symmetric_vec_dot_generator.cpp
+    lookup_table_quant_generators.cpp
+    lookup_table_vec_dot_generator.cpp
+    k_quant_vec_dot_generator.cpp
+    k_quant_generators.cpp
+)
+
+# Q4_0's and Q8_0's quantize/dequantize/vec_dot kernels are GENERATOR_ARGS
+# instantiations of the generic, reusable Approximation-based
+# symmetric_quantize/symmetric_dequantize/symmetric_vec_dot generators (see
+# symmetric_quant_generators.cpp/symmetric_vec_dot_generator.cpp and
+# quant_components.h) -- not their own per-format C++ Generator classes. See
+# quant_components.h's RoundingMode/ScaleAnchor for what these params mean.
+add_halide_library(
+    q4_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS
+    block_size=32 qmax=8 code_bits=4 rounding=truncate_half_up_with_offset anchor=extreme_signed
+)
+add_halide_library(
+    q4_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS
+    block_size=32 qmax=8 code_bits=4 rounding=truncate_half_up_with_offset anchor=extreme_signed
+)
+add_halide_library(
+    q4_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+    w_kind=symmetric block_size=32 w_qmax=8 w_code_bits=4 w_rounding=truncate_half_up_with_offset
+    w_anchor=extreme_signed a_kind=q8_0 a_qmax=127
+)
+# Q4_1 is affine (min+scale, not symmetric): quant_components.h's
+# AffineQuantize + NibblePack, matching block_q4_1's {fp16 d; fp16 m; qs[16];}.
+add_halide_library(
+    q4_1_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=affine block_size=32 levels=15 code_bits=4 affine_rounding=clamped_int8
+)
+add_halide_library(
+    q4_1_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS kind=affine block_size=32 levels=15 code_bits=4 affine_rounding=clamped_int8
+)
+add_halide_library(
+    q4_1_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+    w_kind=affine block_size=32 w_levels=15 w_code_bits=4 w_affine_rounding=clamped_int8 a_kind=q8_1
+    a_qmax=127
+)
+# Q5_0 is symmetric like Q4_0, but 5-bit: quant_components.h's
+# SymmetricAffineQuantize + FiveBitPack, matching block_q5_0's
+# {fp16 d; qh[4]; qs[16];}.
+add_halide_library(
+    q5_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=symmetric_5bit block_size=32 qmax=16
+)
+add_halide_library(
+    q5_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS kind=symmetric_5bit block_size=32 qmax=16
+)
+add_halide_library(
+    q5_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS w_kind=symmetric_5bit block_size=32 w_qmax=16 a_kind=q8_0 a_qmax=127
+)
+# Q5_1 is affine like Q4_1, but 5-bit: quant_components.h's AffineQuantize +
+# FiveBitPack, matching block_q5_1's {fp16 d; fp16 m; qh[4]; qs[16];}.
+add_halide_library(
+    q5_1_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=affine_5bit block_size=32 levels=31 affine_rounding=unclamped_uint8
+)
+add_halide_library(
+    q5_1_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS kind=affine_5bit block_size=32 levels=31 affine_rounding=unclamped_uint8
+)
+add_halide_library(
+    q5_1_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+    w_kind=affine_5bit block_size=32 w_levels=31 w_affine_rounding=unclamped_uint8 a_kind=q8_1
+    a_qmax=127
+)
+add_halide_library(
+    q8_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS block_size=32 qmax=127 code_bits=8 rounding=nearest anchor=abs_max
+)
+add_halide_library(
+    q8_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS block_size=32 qmax=127 code_bits=8 rounding=nearest anchor=abs_max
+)
+add_halide_library(
+    q8_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+    w_kind=symmetric block_size=32 w_qmax=127 w_code_bits=8 w_rounding=nearest w_anchor=abs_max
+    a_kind=q8_0 a_qmax=127
+)
+# Q8_1 is symmetric byte-packed like Q8_0, plus AppendCodeSum's derived 's'
+# field, matching block_q8_1's {fp16 d; fp16 s; qs[32];}. Activation-only
+# (no public to_float, see q8_1_generators.cpp's header comment -- it's
+# gone now, but this scheme's decode() is still used by any vec_dot pairing
+# against Q8_1), so there's no q8_1_dequantize library.
+add_halide_library(
+    q8_1_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=symmetric_byte_sum block_size=32 qmax=127
+)
+# Q8_K's quantize kernel (activation-only, no dequantize -- see
+# q8_k_generators.cpp) is a GENERATOR_ARGS instantiation of the generic,
+# reusable Approximation-based symmetric_quantize generator (see
+# symmetric_quant_generators.cpp and quant_components.h's
+# AppendGroupSumsInt16/F32Pack/RoundingMode::NearestEvenClampedHigh/
+# ScaleAnchor::ExtremeSignedValueTwoStep).
+add_halide_library(
+    q8_k_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=q8k block_size=256 qmax=127
+)
+# Q2_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/NibblePairPack/PlanarBitPack).
+add_halide_library(
+    q2_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q2_k
+)
+add_halide_library(
+    q2_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q2_k
+)
+add_halide_library(q2_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q2_k)
+# Q6_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/CombinedBitsCode/BytePack).
+add_halide_library(
+    q6_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q6_k
+)
+add_halide_library(
+    q6_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q6_k
+)
+add_halide_library(q6_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q6_k)
+# Q4_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/K4ScaleMinPack/PlanarBitPack).
+add_halide_library(
+    q4_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q4_k
+)
+add_halide_library(
+    q4_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q4_k
+)
+add_halide_library(q4_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q4_k)
+# Q5_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/K4ScaleMinPack/CombinedBitsCode).
+add_halide_library(
+    q5_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q5_k
+)
+add_halide_library(
+    q5_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q5_k
+)
+add_halide_library(q5_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q5_k)
+# Q3_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/Q3KScalePack/CombinedBitsCode).
+add_halide_library(
+    q3_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q3_k
+)
+add_halide_library(
+    q3_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q3_k
+)
+add_halide_library(q3_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q3_k)
+# Q1_0 is symmetric with a mean-abs scale and sign-only (1-bit) codes:
+# quant_components.h's SymmetricAffineQuantize (ScaleAnchor::MeanAbs,
+# RoundingMode::SignOnly) + BitPack, matching block_q1_0's {fp16 d; qs[16];}.
+add_halide_library(
+    q1_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS block_size=128 qmax=1 code_bits=1 rounding=sign_only anchor=mean_abs
+)
+add_halide_library(
+    q1_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS block_size=128 qmax=1 code_bits=1 rounding=sign_only anchor=mean_abs
+)
+add_halide_library(
+    q1_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+    w_kind=symmetric block_size=128 w_qmax=1 w_code_bits=1 w_rounding=sign_only w_anchor=mean_abs
+    a_kind=q8_0 a_qmax=127
+)
+# MXFP4/IQ4_NL's quantize/dequantize kernels are GENERATOR_ARGS
+# instantiations of the generic, reusable Approximation-based
+# lookup_table_quantize/lookup_table_dequantize generators (see
+# lookup_table_quant_generators.cpp and quant_components.h's
+# LookupTableQuantize/E8M0Pack) -- not their own per-format C++ Generator
+# classes. vec_dot is still its own hand-rolled, unscheduled Generator (see
+# mxfp4_generators.cpp/iq4_nl_generators.cpp), matching Q4_0/Q8_0's own
+# treatment before they got the symmetric_vec_dot generic generator.
+add_halide_library(
+    mxfp4_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=mxfp4
+)
+add_halide_library(
+    mxfp4_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=mxfp4
+)
+add_halide_library(
+    mxfp4_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=mxfp4
+)
+# NVFP4's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's LookupTableQuantize's num_scales/UE4M3Pack).
+add_halide_library(
+    nvfp4_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=nvfp4
+)
+add_halide_library(
+    nvfp4_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=nvfp4
+)
+add_halide_library(
+    nvfp4_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=nvfp4
+)
+add_halide_library(
+    iq4_nl_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq4_nl
+)
+add_halide_library(
+    iq4_nl_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq4_nl
+)
+add_halide_library(
+    iq4_nl_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq4_nl
+)
+# IQ4_XS's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's IQ4XSDequantize).
+add_halide_library(
+    iq4_xs_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq4_xs
+)
+add_halide_library(
+    iq4_xs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq4_xs
+)
+add_halide_library(
+    iq4_xs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq4_xs
+)
+# TQ1_0's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's LookupTableQuantize/TritPack).
+add_halide_library(
+    tq1_0_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=tq1_0
+)
+add_halide_library(
+    tq1_0_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=tq1_0
+)
+add_halide_library(
+    tq1_0_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=tq1_0
+)
+# TQ2_0's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's LookupTableQuantize/PlanarBitPack).
+add_halide_library(
+    tq2_0_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=tq2_0
+)
+add_halide_library(
+    tq2_0_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=tq2_0
+)
+add_halide_library(
+    tq2_0_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=tq2_0
+)
+add_halide_library(
+    iq2_xxs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq2_xxs
+)
+add_halide_library(
+    iq2_xxs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq2_xxs
+)
+add_halide_library(
+    iq2_xs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq2_xs
+)
+add_halide_library(
+    iq2_xs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq2_xs
+)
+# IQ2_S/IQ3_XXS/IQ3_S's quantize/dequantize kernels are GENERATOR_ARGS
+# instantiations of the generic, reusable Approximation-based
+# lookup_table_quantize/lookup_table_dequantize generators (see
+# lookup_table_quant_generators.cpp and quant_components.h's
+# IQ2SGridDequantize/IQ3XXSGridDequantize/IQ3SGridDequantize).
+add_halide_library(
+    iq2_s_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq2_s
+)
+add_halide_library(
+    iq2_s_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq2_s
+)
+add_halide_library(
+    iq2_s_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq2_s
+)
+add_halide_library(
+    iq3_xxs_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq3_xxs
+)
+add_halide_library(
+    iq3_xxs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq3_xxs
+)
+add_halide_library(
+    iq3_xxs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq3_xxs
+)
+add_halide_library(
+    iq3_s_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq3_s
+)
+add_halide_library(
+    iq3_s_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq3_s
+)
+add_halide_library(
+    iq3_s_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq3_s
+)
+add_halide_library(
+    iq1_s_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq1_s
+)
+add_halide_library(
+    iq1_s_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq1_s
+)
+add_halide_library(
+    iq1_m_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq1_m
+)
+add_halide_library(
+    iq1_m_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq1_m
+)
+add_halide_library(f16_quantize FROM quants.generator GENERATOR f16_quantize)
+add_halide_library(f16_dequantize FROM quants.generator GENERATOR f16_dequantize)
+add_halide_library(bf16_quantize FROM quants.generator GENERATOR bf16_quantize)
+add_halide_library(bf16_dequantize FROM quants.generator GENERATOR bf16_dequantize)
+add_halide_library(q8_0_4x4_quantize_mat FROM quants.generator GENERATOR q8_0_4x4_quantize_mat)
+add_halide_library(q8_0_4x8_quantize_mat FROM quants.generator GENERATOR q8_0_4x8_quantize_mat)
+add_halide_library(q8_k_4x4_quantize_mat FROM quants.generator GENERATOR q8_k_4x4_quantize_mat)
+add_halide_library(q8_k_4x8_quantize_mat FROM quants.generator GENERATOR q8_k_4x8_quantize_mat)
+
+add_halide_library(
+    q4_0_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q4_0_4x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    q4_0_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_0 n_cols=8 blocklen=8
+)
+add_halide_library(
+    q8_0_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q8_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q8_0_4x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q8_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    q4_0_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q4_0_4x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    q4_0_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_0 n_cols=8 blocklen=8
+)
+add_halide_library(
+    q8_0_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q8_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q8_0_4x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q8_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    iq4_nl_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=iq4_nl n_cols=4 blocklen=4
+)
+add_halide_library(
+    iq4_nl_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=iq4_nl n_cols=8 blocklen=8
+)
+add_halide_library(
+    mxfp4_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=mxfp4 n_cols=4 blocklen=4
+)
+add_halide_library(
+    mxfp4_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=mxfp4 n_cols=8 blocklen=8
+)
+add_halide_library(
+    iq4_nl_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=iq4_nl n_cols=4 blocklen=4
+)
+add_halide_library(
+    iq4_nl_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=iq4_nl n_cols=8 blocklen=8
+)
+add_halide_library(
+    mxfp4_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=mxfp4 n_cols=4 blocklen=4
+)
+add_halide_library(
+    mxfp4_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=mxfp4 n_cols=8 blocklen=8
+)
+add_halide_library(
+    q4_k_8x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q4_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q4_k_8x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q4_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q5_k_8x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q5_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q5_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q5_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q5_k_8x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q5_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q5_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q5_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q6_k_8x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q6_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q6_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q6_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q6_k_8x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q6_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q6_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q6_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q2_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q2_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q2_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q2_k n_cols=8 blocklen=8
+)
+
+# ggml_extern_quantize.cpp implements the extern-stage C functions that
+# several of the quantize pipelines above call out to -- see that file for
+# why. It's the one source in this library that depends on GGML, hence the
+# extra ggml::ggml link below (every other source here is GGML-independent).
+add_library(ggml_quants_halide ggml_quants.cpp ggml_extern_quantize.cpp)
+target_link_libraries(
+    ggml_quants_halide
+    PUBLIC
+    q4_0_quantize q4_0_dequantize q4_0_vec_dot q4_1_quantize q4_1_dequantize q4_1_vec_dot
+    q5_0_quantize q5_0_dequantize q5_0_vec_dot q5_1_quantize q5_1_dequantize q5_1_vec_dot
+    q8_0_quantize q8_0_dequantize q8_0_vec_dot q8_1_quantize q8_k_quantize q2_k_quantize
+    q2_k_dequantize q2_k_vec_dot q6_k_quantize q6_k_dequantize q6_k_vec_dot q4_k_quantize
+    q4_k_dequantize q4_k_vec_dot q5_k_quantize q5_k_dequantize q5_k_vec_dot q3_k_quantize
+    q3_k_dequantize q3_k_vec_dot q1_0_quantize q1_0_dequantize q1_0_vec_dot mxfp4_quantize
+    mxfp4_dequantize mxfp4_vec_dot nvfp4_quantize nvfp4_dequantize nvfp4_vec_dot iq4_nl_quantize
+    iq4_nl_dequantize iq4_nl_vec_dot iq4_xs_quantize iq4_xs_dequantize iq4_xs_vec_dot tq1_0_quantize
+    tq1_0_dequantize tq1_0_vec_dot tq2_0_quantize tq2_0_dequantize tq2_0_vec_dot iq2_xxs_dequantize
+    iq2_xxs_vec_dot iq2_xs_dequantize iq2_xs_vec_dot iq2_s_quantize iq2_s_dequantize iq2_s_vec_dot
+    iq3_xxs_quantize iq3_xxs_dequantize iq3_xxs_vec_dot iq3_s_quantize iq3_s_dequantize
+    iq3_s_vec_dot iq1_s_dequantize iq1_s_vec_dot iq1_m_dequantize iq1_m_vec_dot f16_quantize
+    f16_dequantize bf16_quantize bf16_dequantize q8_0_4x4_quantize_mat q8_0_4x8_quantize_mat
+    q8_k_4x4_quantize_mat q8_k_4x8_quantize_mat q4_0_4x4_gemv q4_0_4x8_gemv q4_0_8x8_gemv
+    q8_0_4x4_gemv q8_0_4x8_gemv q4_0_4x4_gemm q4_0_4x8_gemm q4_0_8x8_gemm q8_0_4x4_gemm
+    q8_0_4x8_gemm iq4_nl_4x4_gemv iq4_nl_8x8_gemv mxfp4_4x4_gemv mxfp4_8x8_gemv iq4_nl_4x4_gemm
+    iq4_nl_8x8_gemm mxfp4_4x4_gemm mxfp4_8x8_gemm q4_k_8x4_gemv q4_k_8x8_gemv q4_k_8x4_gemm
+    q4_k_8x8_gemm q5_k_8x4_gemv q5_k_8x8_gemv q5_k_8x4_gemm q5_k_8x8_gemm q6_k_8x4_gemv
+    q6_k_8x8_gemv q6_k_8x4_gemm q6_k_8x8_gemm q2_k_8x8_gemv q2_k_8x8_gemm ggml::ggml
+)
+target_include_directories(ggml_quants_halide PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+# Standalone correctness checks against GGML's own reference, run before (and
+# independently of) wiring these into kernel-bench as providers.
+foreach (t IN ITEMS
+    q4_0
+    q4_1
+    q5_0
+    q5_1
+    q8_0
+    q8_1
+    q8_k
+    q2_k
+    q6_k
+    q4_k
+    q5_k
+    q3_k
+    q1_0
+    mxfp4
+    nvfp4
+    iq4_nl
+    iq4_xs
+    tq1_0
+    tq2_0
+    iq2_xxs
+    iq2_xs
+    iq2_s
+    iq3_xxs
+    iq3_s
+    iq1_s
+    iq1_m
+    f16
+    bf16
+)
+    add_executable(test_${t} test_${t}.cpp)
+    target_link_libraries(test_${t} PRIVATE ggml_quants_halide ggml::ggml)
+    target_include_directories(test_${t} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src)
+    add_test(NAME ${t}_roundtrip COMMAND test_${t})
+    set_tests_properties(${t}_roundtrip PROPERTIES PASS_REGULAR_EXPRESSION "Success!")
+endforeach ()
diff --git a/apps/ggml/halide/bf16_generators.cpp b/apps/ggml/halide/bf16_generators.cpp
new file mode 100644
index 000000000000..bab89572cb5f
--- /dev/null
+++ b/apps/ggml/halide/bf16_generators.cpp
@@ -0,0 +1,58 @@
+// From-scratch Halide reimplementation of GGML's BF16 quantize/dequantize
+// "kernels" (see src/ggml-impl.h: ggml_compute_fp32_to_bf16 /
+// ggml_compute_bf16_to_fp32 upstream, as of GGML v0.15.3). Like F16, BF16
+// isn't really a quantized format -- it's a 1-element/block cast, no
+// header/payload split -- so this is just Halide's native bfloat16_t cast
+// in both directions.
+//
+// GGML's bf16 encode is round-to-nearest-even truncation of the top 16 bits
+// of the IEEE binary32 representation (with NaNs forced quiet); decode is
+// a plain `bits << 16` reinterpretation. Halide's cast<bfloat16_t>/cast<float>
+// compile to the same IEEE-mandated round-to-nearest-even truncation and
+// zero-extension, so this matches bit-for-bit for all finite inputs (the
+// only divergence possible is NaN payload/quieting, which the synthetic
+// benchmark/test data never produces).
+//
+// This is intentionally unscheduled -- scheduling for performance is a
+// later step.
+
+#include "Halide.h"
+
+using namespace Halide;
+
+namespace {
+
+class BF16DequantizeGenerator : public Generator<BF16DequantizeGenerator> {
+public:
+    // Raw bf16 bit patterns, one uint16 per element (block size 1).
+    Input<Buffer<uint16_t, 1>> x_{"x"};
+    Output<Buffer<float, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = cast<float>(reinterpret<bfloat16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+class BF16QuantizeGenerator : public Generator<BF16QuantizeGenerator> {
+public:
+    Input<Buffer<float, 1>> x_{"x"};
+    // Raw bf16 bit patterns, one uint16 per element (block size 1).
+    Output<Buffer<uint16_t, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = reinterpret<uint16_t>(cast<bfloat16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(BF16DequantizeGenerator, bf16_dequantize)
+HALIDE_REGISTER_GENERATOR(BF16QuantizeGenerator, bf16_quantize)
diff --git a/apps/ggml/halide/codec_generator_base.h b/apps/ggml/halide/codec_generator_base.h
new file mode 100644
index 000000000000..0145349690be
--- /dev/null
+++ b/apps/ggml/halide/codec_generator_base.h
@@ -0,0 +1,114 @@
+#pragma once
+
+// Shared configure()/generate() scaffolding for every *_quant_generators.cpp
+// file's Direction-templated Generator (SymmetricCodecGenerator,
+// LookupTableCodecGenerator, KQuantCodecGenerator): all three build the
+// exact same "real ImageParam -> approximate_by -> compute_offline -> adopt
+// one half as a port" pipeline in configure(), differing only in how their
+// SchemeAndBytes gets built. This factors that shared body out via CRTP
+// (Derived::build_scheme()) -- the same static-polymorphism idiom
+// Halide::Generator<T> itself already uses (see its own `T` template
+// parameter), not a virtual method: the concrete type is always known at
+// compile time, so there's no reason to pay for a vtable. Confirmed safe to
+// insert as a base class between a leaf Generator and Halide::Generator<T>:
+// GeneratorParam/Input/Output discovery is address-range-based (see
+// Generator.cpp's ObjectInstanceRegistry::register_instance/
+// instances_in_range), not declaration-order or hierarchy-position based,
+// so it doesn't matter which class in the chain declares them.
+//
+// Usage:
+//   class FooCodecGenerator : public CodecGeneratorBase<FooCodecGenerator<dir>, dir> {
+//   public:
+//       GeneratorParam<...> whatever{...};
+//       SchemeAndBytes build_scheme() const { return ::build_scheme(whatever); }
+//   };
+
+#include "Halide.h"
+
+#include "quant_components.h"
+
+namespace ggml_halide {
+
+enum class Direction { Quantize,
+                       Dequantize };
+
+// SchemeAndBytes itself now lives in quant_components.h (its `scheme` is
+// held as a polymorphic owning handle -- a single leaf, a Compose, or a
+// TrustedInverse, whichever the format is; see the make_*() factories
+// there) -- moved there so those factories can return it directly instead
+// of every Generator switch hand-summing a byte count alongside a bare
+// scheme.
+
+template<typename Derived, Direction dir>
+class CodecGeneratorBase : public Halide::Generator<Derived> {
+public:
+    void configure() {
+        using namespace Halide;
+        SchemeAndBytes sb = static_cast<Derived *>(this)->build_scheme();
+
+        // The "obvious" identity: a real ImageParam (never a placeholder --
+        // that's what lets *both* directions share this one call below)
+        // flowing through unchanged.
+        Var x("x");
+        ImageParam input(Float(32), 1, "x");
+        Func identity("y");
+        identity(x) = input(x);
+
+        ApproximationResult r = Func(input).approximate_by(*sb.scheme, {identity});
+        for (Func h : r.handles) {
+            h.compute_root();
+        }
+
+        // Every scheme here produces a single 2-D uint8 packed byte buffer
+        // as its encoded form -- bind compute_offline() to a properly-named
+        // ImageParam of that shape up front, instead of letting it mint one
+        // named after whatever internal Func happened to produce
+        // r.encoded[0] (e.g. "struct_pack_packed"). Only Dequantize below
+        // adopts it as a port (named "blocks_in" rather than reusing
+        // Quantize's output name "blocks_out" below, so the two don't
+        // collide and get uniquified within this same configure() call --
+        // they're never both real ports at once, but both objects always
+        // exist).
+        ImageParam blocks_in(UInt(8), 2, "blocks_in");
+
+        // Severs `identity` from `input`/encode() entirely: `q.offline`
+        // recomputes r.encoded (quantize) from `input`, while `identity`
+        // (post-severance) instead reads from `blocks_in` (dequantize).
+        // Each direction below adopts exactly one of these two independent
+        // halves; the other is simply never registered as a port and so
+        // never gets compiled in.
+        ComputeOfflineResult q = Pipeline({identity}).compute_offline(r.encoded, {blocks_in});
+
+        if constexpr (dir == Direction::Quantize) {
+            input.dim(0).set_min(0);
+
+            // q.offline.outputs()[0] is r.encoded[0] itself (an internally-
+            // named Func) -- a thin renamed passthrough is the only way to
+            // give the compiled Output a clean name, the same way
+            // `blocks_in` above did for the Input side; Halide inlines it
+            // away, so this costs nothing.
+            Func blocks_out("blocks_out");
+            Var byte("byte"), blk("blk");
+            blocks_out(byte, blk) = q.offline.outputs()[0](byte, blk);
+            blocks_out.output_buffer().dim(0).set_bounds(0, sb.block_bytes);
+            blocks_out.output_buffer().dim(1).set_min(0);
+
+            this->add_input(input);
+            this->add_output(blocks_out);
+        } else {
+            blocks_in.dim(0).set_bounds(0, sb.block_bytes);
+            blocks_in.dim(1).set_min(0);
+            identity.output_buffer().dim(0).set_min(0);
+
+            this->add_input(blocks_in);
+            this->add_output(identity);
+        }
+    }
+
+    void generate() {
+        // Nothing left to do: configure() already built (and, via
+        // add_input/add_output, wired up) the whole pipeline.
+    }
+};
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/halide/f16_generators.cpp b/apps/ggml/halide/f16_generators.cpp
new file mode 100644
index 000000000000..9e5431df20f6
--- /dev/null
+++ b/apps/ggml/halide/f16_generators.cpp
@@ -0,0 +1,55 @@
+// From-scratch Halide reimplementation of GGML's F16 quantize/dequantize
+// "kernels" (see src/ggml.c: ggml_fp32_to_fp16_row / ggml_fp16_to_fp32_row
+// upstream, as of GGML v0.15.3). F16 isn't really a quantized format -- it's
+// a 1-element/block plain IEEE-754 binary16 cast, no header/payload split at
+// all -- so unlike every other type here, this is just Halide's native
+// float16_t cast in both directions.
+//
+// GGML's own conversion (GGML_COMPUTE_FP32_TO_FP16 / _FP16_TO_FP32 in
+// src/ggml-impl.h) is a correctly-rounded (round-to-nearest-even) software
+// IEEE binary16 <-> binary32 conversion; Halide's cast<float16_t>/cast<float>
+// compiles to the same IEEE-mandated conversion, so this matches bit-for-bit.
+//
+// This is intentionally unscheduled -- scheduling for performance is a
+// later step.
+
+#include "Halide.h"
+
+using namespace Halide;
+
+namespace {
+
+class F16DequantizeGenerator : public Generator<F16DequantizeGenerator> {
+public:
+    // Raw fp16 bit patterns, one uint16 per element (block size 1).
+    Input<Buffer<uint16_t, 1>> x_{"x"};
+    Output<Buffer<float, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = cast<float>(reinterpret<float16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+class F16QuantizeGenerator : public Generator<F16QuantizeGenerator> {
+public:
+    Input<Buffer<float, 1>> x_{"x"};
+    // Raw fp16 bit patterns, one uint16 per element (block size 1).
+    Output<Buffer<uint16_t, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = reinterpret<uint16_t>(cast<float16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(F16DequantizeGenerator, f16_dequantize)
+HALIDE_REGISTER_GENERATOR(F16QuantizeGenerator, f16_quantize)
diff --git a/apps/ggml/halide/ggml_extern_quantize.cpp b/apps/ggml/halide/ggml_extern_quantize.cpp
new file mode 100644
index 000000000000..519b8fd29878
--- /dev/null
+++ b/apps/ggml/halide/ggml_extern_quantize.cpp
@@ -0,0 +1,116 @@
+// Extern-stage scaffolding for K-quant quantize kernels.
+//
+// GGML's reference quantizer for the K-quant super-block formats (Q2_K,
+// Q3_K, Q4_K, Q5_K, Q6_K) isn't a closed-form scale computation like every
+// other type in this directory -- it runs an iterative, per-sub-block
+// error-minimizing search over ~19 candidate scale factors (see
+// src/ggml-quants.c: make_qx_quants / make_qkx1_quants / make_qkx2_quants /
+// make_q3_quants). Porting that search to Halide is deferred; per the
+// project's current phase, this sets up the Halide extern-stage plumbing
+// now (a Func whose realization is computed by an external C function) and
+// simply calls out to GGML's own public from_float_ref for the actual
+// computation. Dequantize (a pure unpacking operation, no search) is
+// implemented natively in Halide for these types -- see qX_k_generators.cpp.
+//
+// This is the one file in halide/ that depends on GGML's public API --
+// every generator's *body* stays GGML-independent, but this scaffold
+// deliberately borrows GGML's own reference computation for now, to be
+// replaced with a from-scratch Halide search later.
+//
+// Extern-stage ABI: a plain C function taking one halide_buffer_t* per
+// Func argument/output, returning 0 on success. Halide calls it twice per
+// realization: once in "bounds query" mode (host pointers null, dimensions
+// need to be filled in based on the output's already-concrete request) and
+// once for real (host pointers valid, actually compute the data). See
+// test/correctness/extern_bounds_inference.cpp for the reference pattern.
+
+#include <HalideRuntime.h>
+
+#include <ggml.h>
+
+namespace {
+
+int quantize_via_ggml_reference(ggml_type type, halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    if (x_buf->is_bounds_query()) {
+        // out_buf already carries the concrete requested region (its dim[1]
+        // extent is the number of blocks); the input row needed is exactly
+        // that many blocks' worth of elements.
+        const int64_t nb = out_buf->dim[1].extent;
+        x_buf->dim[0].min = 0;
+        x_buf->dim[0].extent = static_cast<int32_t>(nb * ggml_blck_size(type));
+        return 0;
+    }
+
+    const float *x = reinterpret_cast<const float *>(x_buf->host);
+    void *y = reinterpret_cast<void *>(out_buf->host);
+    const int64_t k = x_buf->dim[0].extent;
+
+    ggml_get_type_traits(type)->from_float_ref(x, y, k);
+    return 0;
+}
+
+}  // namespace
+
+extern "C" int q2_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q2_K, x_buf, out_buf);
+}
+
+extern "C" int q3_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q3_K, x_buf, out_buf);
+}
+
+extern "C" int q4_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q4_K, x_buf, out_buf);
+}
+
+extern "C" int q5_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q5_K, x_buf, out_buf);
+}
+
+extern "C" int q6_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q6_K, x_buf, out_buf);
+}
+
+// The remaining types below use this same scaffolding for different
+// reasons than the K-quants: MXFP4/NVFP4 derive their scale via a
+// transcendental (log2) or rounding-sensitive fixed-point float format not
+// guaranteed to be bit-reproducible from scratch; IQ4_NL/IQ4_XS run a
+// per-block/sub-block nearest-codeword search with scale refinement; TQ1_0/
+// TQ2_0's byte-packing, while closed-form, is fiddly to unroll in Halide's
+// functional style. All are deferred the same way, for now.
+
+extern "C" int mxfp4_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_MXFP4, x_buf, out_buf);
+}
+
+extern "C" int nvfp4_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_NVFP4, x_buf, out_buf);
+}
+
+extern "C" int iq4_nl_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ4_NL, x_buf, out_buf);
+}
+
+extern "C" int iq4_xs_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ4_XS, x_buf, out_buf);
+}
+
+extern "C" int tq1_0_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_TQ1_0, x_buf, out_buf);
+}
+
+extern "C" int tq2_0_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_TQ2_0, x_buf, out_buf);
+}
+
+extern "C" int iq3_xxs_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ3_XXS, x_buf, out_buf);
+}
+
+extern "C" int iq3_s_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ3_S, x_buf, out_buf);
+}
+
+extern "C" int iq2_s_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ2_S, x_buf, out_buf);
+}
diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
new file mode 100644
index 000000000000..2fe2ec3a1bcb
--- /dev/null
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -0,0 +1,1732 @@
+#include "ggml_quants.h"
+
+#include <cstdio>
+
+#include "HalideBuffer.h"
+#include "bf16_dequantize.h"
+#include "bf16_quantize.h"
+#include "f16_dequantize.h"
+#include "f16_quantize.h"
+#include "iq1_m_dequantize.h"
+#include "iq1_m_vec_dot.h"
+#include "iq1_s_dequantize.h"
+#include "iq1_s_vec_dot.h"
+#include "iq2_s_dequantize.h"
+#include "iq2_s_quantize.h"
+#include "iq2_s_vec_dot.h"
+#include "iq2_xs_dequantize.h"
+#include "iq2_xs_vec_dot.h"
+#include "iq2_xxs_dequantize.h"
+#include "iq2_xxs_vec_dot.h"
+#include "iq3_s_dequantize.h"
+#include "iq3_s_quantize.h"
+#include "iq3_s_vec_dot.h"
+#include "iq3_xxs_dequantize.h"
+#include "iq3_xxs_quantize.h"
+#include "iq3_xxs_vec_dot.h"
+#include "iq4_nl_4x4_gemm.h"
+#include "iq4_nl_4x4_gemv.h"
+#include "iq4_nl_8x8_gemm.h"
+#include "iq4_nl_8x8_gemv.h"
+#include "iq4_nl_dequantize.h"
+#include "iq4_nl_quantize.h"
+#include "iq4_nl_vec_dot.h"
+#include "iq4_xs_dequantize.h"
+#include "iq4_xs_quantize.h"
+#include "iq4_xs_vec_dot.h"
+#include "mxfp4_4x4_gemm.h"
+#include "mxfp4_4x4_gemv.h"
+#include "mxfp4_8x8_gemm.h"
+#include "mxfp4_8x8_gemv.h"
+#include "mxfp4_dequantize.h"
+#include "mxfp4_quantize.h"
+#include "mxfp4_vec_dot.h"
+#include "nvfp4_dequantize.h"
+#include "nvfp4_quantize.h"
+#include "nvfp4_vec_dot.h"
+#include "q1_0_dequantize.h"
+#include "q1_0_quantize.h"
+#include "q1_0_vec_dot.h"
+#include "q2_k_8x8_gemm.h"
+#include "q2_k_8x8_gemv.h"
+#include "q2_k_dequantize.h"
+#include "q2_k_quantize.h"
+#include "q2_k_vec_dot.h"
+#include "q3_k_dequantize.h"
+#include "q3_k_quantize.h"
+#include "q3_k_vec_dot.h"
+#include "q4_0_4x4_gemm.h"
+#include "q4_0_4x4_gemv.h"
+#include "q4_0_4x8_gemm.h"
+#include "q4_0_4x8_gemv.h"
+#include "q4_0_8x8_gemm.h"
+#include "q4_0_8x8_gemv.h"
+#include "q4_0_dequantize.h"
+#include "q4_0_quantize.h"
+#include "q4_0_vec_dot.h"
+#include "q4_1_dequantize.h"
+#include "q4_1_quantize.h"
+#include "q4_1_vec_dot.h"
+#include "q4_k_8x4_gemm.h"
+#include "q4_k_8x4_gemv.h"
+#include "q4_k_8x8_gemm.h"
+#include "q4_k_8x8_gemv.h"
+#include "q4_k_dequantize.h"
+#include "q4_k_quantize.h"
+#include "q4_k_vec_dot.h"
+#include "q5_0_dequantize.h"
+#include "q5_0_quantize.h"
+#include "q5_0_vec_dot.h"
+#include "q5_1_dequantize.h"
+#include "q5_1_quantize.h"
+#include "q5_1_vec_dot.h"
+#include "q5_k_8x4_gemm.h"
+#include "q5_k_8x4_gemv.h"
+#include "q5_k_8x8_gemm.h"
+#include "q5_k_8x8_gemv.h"
+#include "q5_k_dequantize.h"
+#include "q5_k_quantize.h"
+#include "q5_k_vec_dot.h"
+#include "q6_k_8x4_gemm.h"
+#include "q6_k_8x4_gemv.h"
+#include "q6_k_8x8_gemm.h"
+#include "q6_k_8x8_gemv.h"
+#include "q6_k_dequantize.h"
+#include "q6_k_quantize.h"
+#include "q6_k_vec_dot.h"
+#include "q8_0_4x4_gemm.h"
+#include "q8_0_4x4_gemv.h"
+#include "q8_0_4x4_quantize_mat.h"
+#include "q8_0_4x8_gemm.h"
+#include "q8_0_4x8_gemv.h"
+#include "q8_0_4x8_quantize_mat.h"
+#include "q8_0_dequantize.h"
+#include "q8_0_quantize.h"
+#include "q8_0_vec_dot.h"
+#include "q8_1_quantize.h"
+#include "q8_k_4x4_quantize_mat.h"
+#include "q8_k_4x8_quantize_mat.h"
+#include "q8_k_quantize.h"
+#include "tq1_0_dequantize.h"
+#include "tq1_0_quantize.h"
+#include "tq1_0_vec_dot.h"
+#include "tq2_0_dequantize.h"
+#include "tq2_0_quantize.h"
+#include "tq2_0_vec_dot.h"
+
+using Halide::Runtime::Buffer;
+
+namespace {
+
+void check(int result, const char *what) {
+    if (result != 0) {
+        std::fprintf(stderr, "ggml_quants_halide: %s failed (%d)\n", what, result);
+    }
+}
+
+}  // namespace
+
+extern "C" {
+
+//
+// Q4_0 -- block size 32, 18 bytes/block (2 delta + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q4_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q4_0_quantize(xb, blocks), "q4_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q4_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q4_0_dequantize(blocks, yb), "q4_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q4_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 2 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q4_0_vec_dot(xb, yb, result), "q4_0_vec_dot");
+}
+
+//
+// Q4_1 -- block size 32, 20 bytes/block (2 delta + 2 min + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q4_1(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q4_1_quantize(xb, blocks), "q4_1_quantize");
+}
+
+void ggml_quants_halide_dequantize_q4_1(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q4_1_dequantize(blocks, yb), "q4_1_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q4_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 4 + kQK / 2, kBlockBytesY = 4 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q4_1_vec_dot(xb, yb, result), "q4_1_vec_dot");
+}
+
+//
+// Q5_0 -- block size 32, 22 bytes/block (2 delta + 4 qh + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q5_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q5_0_quantize(xb, blocks), "q5_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q5_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q5_0_dequantize(blocks, yb), "q5_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q5_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 2 + 4 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q5_0_vec_dot(xb, yb, result), "q5_0_vec_dot");
+}
+
+//
+// Q5_1 -- block size 32, 24 bytes/block (2 delta + 2 min + 4 qh + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q5_1(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q5_1_quantize(xb, blocks), "q5_1_quantize");
+}
+
+void ggml_quants_halide_dequantize_q5_1(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q5_1_dequantize(blocks, yb), "q5_1_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q5_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 4 + 4 + kQK / 2, kBlockBytesY = 4 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q5_1_vec_dot(xb, yb, result), "q5_1_vec_dot");
+}
+
+//
+// Q8_0 -- block size 32, 34 bytes/block (2 delta + 32 int8 values).
+//
+
+void ggml_quants_halide_quantize_q8_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q8_0_quantize(xb, blocks), "q8_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q8_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q8_0_dequantize(blocks, yb), "q8_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q8_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q8_0_vec_dot(xb, yb, result), "q8_0_vec_dot");
+}
+
+//
+// Q8_1 -- block size 32, 36 bytes/block (2 delta + 2 sum + 32 int8 values).
+// Quantize only -- GGML has no public dequantize for this activation-only format.
+//
+
+void ggml_quants_halide_quantize_q8_1(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + kQK;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q8_1_quantize(xb, blocks), "q8_1_quantize");
+}
+
+//
+// Q8_K -- superblock size 256, 292 bytes/block (4 float32 delta + 256 int8
+// values + 16 int16 bsums). Quantize only -- GGML has no public dequantize
+// for this activation-only format.
+//
+
+void ggml_quants_halide_quantize_q8_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + kQK + (kQK / 16) * 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q8_k_quantize(xb, blocks), "q8_k_quantize");
+}
+
+//
+// Q2_K -- superblock size 256, 84 bytes/block (16 scale/min nibble bytes +
+// 64 packed-2-bit bytes + 2 delta + 2 dmin). Dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q2_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 16 + kQK / 4 + 4;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q2_k_quantize(xb, blocks), "q2_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q2_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 16 + kQK / 4 + 4;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q2_k_dequantize(blocks, yb), "q2_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q2_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 16 + kQK / 4 + 4, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q2_k_vec_dot(xb, yb, result), "q2_k_vec_dot");
+}
+
+//
+// Q6_K -- superblock size 256, 210 bytes/block (128 ql + 64 qh + 16 signed
+// int8 scales + 2 delta). Dequantize is native Halide; quantize calls out
+// to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q6_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 2 + kQK / 4 + kQK / 16 + 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q6_k_quantize(xb, blocks), "q6_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q6_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 2 + kQK / 4 + kQK / 16 + 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q6_k_dequantize(blocks, yb), "q6_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q6_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 2 + kQK / 4 + kQK / 16 + 2,
+                  kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q6_k_vec_dot(xb, yb, result), "q6_k_vec_dot");
+}
+
+//
+// Q4_K -- superblock size 256, 144 bytes/block (2 delta + 2 dmin + 12 packed
+// scale/min bytes + 128 packed-4-bit bytes). Dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q4_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q4_k_quantize(xb, blocks), "q4_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q4_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q4_k_dequantize(blocks, yb), "q4_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q4_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 4 + 12 + kQK / 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q4_k_vec_dot(xb, yb, result), "q4_k_vec_dot");
+}
+
+//
+// Q5_K -- superblock size 256, 176 bytes/block (2 delta + 2 dmin + 12 packed
+// scale/min bytes + 32 high-bit bytes + 128 packed-4-bit bytes). Dequantize
+// is native Halide; quantize calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q5_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 8 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q5_k_quantize(xb, blocks), "q5_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q5_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 8 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q5_k_dequantize(blocks, yb), "q5_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q5_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 4 + 12 + kQK / 8 + kQK / 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q5_k_vec_dot(xb, yb, result), "q5_k_vec_dot");
+}
+
+//
+// Q3_K -- superblock size 256, 110 bytes/block (32 hmask + 64 packed-2-bit
+// bytes + 12 packed scale bytes + 2 delta). Dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q3_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 8 + kQK / 4 + 12 + 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q3_k_quantize(xb, blocks), "q3_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q3_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 8 + kQK / 4 + 12 + 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q3_k_dequantize(blocks, yb), "q3_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q3_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 8 + kQK / 4 + 12 + 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q3_k_vec_dot(xb, yb, result), "q3_k_vec_dot");
+}
+
+//
+// Q1_0 -- block size 128, 18 bytes/block (2 delta + 16 sign-bit bytes).
+// Closed-form both directions, fully native.
+//
+
+void ggml_quants_halide_quantize_q1_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 128, kBlockBytes = 2 + kQK / 8;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q1_0_quantize(xb, blocks), "q1_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q1_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 128, kBlockBytes = 2 + kQK / 8;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q1_0_dequantize(blocks, yb), "q1_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q1_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    // Q1_0's own block size (128) differs from Q8_0's (32) -- unlike the
+    // same-granularity types above, nb must be computed separately per side.
+    constexpr int kQKX = 128, kBlockBytesX = 2 + kQKX / 8;
+    constexpr int kQKY = 32, kBlockBytesY = 2 + kQKY;
+    const int32_t nbx = static_cast<int32_t>(n / kQKX);
+    const int32_t nby = static_cast<int32_t>(n / kQKY);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nbx, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nby, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q1_0_vec_dot(xb, yb, result), "q1_0_vec_dot");
+}
+
+//
+// MXFP4 -- block size 32, 17 bytes/block (1 E8M0 exponent + 16 packed-4-bit
+// codebook-index bytes). Dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_mxfp4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 1 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(mxfp4_quantize(xb, blocks), "mxfp4_quantize");
+}
+
+void ggml_quants_halide_dequantize_mxfp4(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 1 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(mxfp4_dequantize(blocks, yb), "mxfp4_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_mxfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 1 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(mxfp4_vec_dot(xb, yb, result), "mxfp4_vec_dot");
+}
+
+//
+// NVFP4 -- block size 64, 36 bytes/block (4 UE4M3 scales + 32 packed-4-bit
+// codebook-index bytes). Dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_nvfp4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 64, kSub = 16, kBlockBytes = kQK / kSub + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(nvfp4_quantize(xb, blocks), "nvfp4_quantize");
+}
+
+void ggml_quants_halide_dequantize_nvfp4(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 64, kSub = 16, kBlockBytes = kQK / kSub + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(nvfp4_dequantize(blocks, yb), "nvfp4_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_nvfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    // NVFP4's own block size (64) differs from Q8_0's (32) -- see Q1_0's
+    // vec_dot wrapper above for why nb must be computed separately per side.
+    constexpr int kQKX = 64, kSub = 16, kBlockBytesX = kQKX / kSub + kQKX / 2;
+    constexpr int kQKY = 32, kBlockBytesY = 2 + kQKY;
+    const int32_t nbx = static_cast<int32_t>(n / kQKX);
+    const int32_t nby = static_cast<int32_t>(n / kQKY);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nbx, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nby, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(nvfp4_vec_dot(xb, yb, result), "nvfp4_vec_dot");
+}
+
+//
+// IQ4_NL -- block size 32, 18 bytes/block (2 delta + 16 packed-4-bit
+// codebook-index bytes). Dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_iq4_nl(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq4_nl_quantize(xb, blocks), "iq4_nl_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq4_nl(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq4_nl_dequantize(blocks, yb), "iq4_nl_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq4_nl_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 2 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq4_nl_vec_dot(xb, yb, result), "iq4_nl_vec_dot");
+}
+
+//
+// IQ4_XS -- superblock size 256, 136 bytes/block (2 delta + 2 scales_h + 4
+// scales_l + 128 packed-4-bit codebook-index bytes). Dequantize is native
+// Halide; quantize calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_iq4_xs(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 2 + 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq4_xs_quantize(xb, blocks), "iq4_xs_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq4_xs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 2 + 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq4_xs_dequantize(blocks, yb), "iq4_xs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq4_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + 2 + 4 + kQK / 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq4_xs_vec_dot(xb, yb, result), "iq4_xs_vec_dot");
+}
+
+//
+// TQ1_0 -- superblock size 256, 54 bytes/block (48 base-3-packed qs + 4
+// base-3-packed qh + 2 delta). Dequantize is native Halide; quantize calls
+// out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_tq1_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 54;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(tq1_0_quantize(xb, blocks), "tq1_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_tq1_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 54;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(tq1_0_dequantize(blocks, yb), "tq1_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_tq1_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 54, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(tq1_0_vec_dot(xb, yb, result), "tq1_0_vec_dot");
+}
+
+//
+// TQ2_0 -- superblock size 256, 66 bytes/block (64 packed-2-bit qs + 2
+// delta -- qs before d, unlike every other type here). Dequantize is native
+// Halide; quantize calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_tq2_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 4 + 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(tq2_0_quantize(xb, blocks), "tq2_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_tq2_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 4 + 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(tq2_0_dequantize(blocks, yb), "tq2_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_tq2_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 4 + 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(tq2_0_vec_dot(xb, yb, result), "tq2_0_vec_dot");
+}
+
+//
+// IQ2_XXS -- superblock size 256, 66 bytes/block (2 delta + 64 packed qs).
+// Dequantize only -- see ggml_quants.h for why there's no quantize here.
+//
+
+void ggml_quants_halide_dequantize_iq2_xxs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 * 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq2_xxs_dequantize(blocks, yb), "iq2_xxs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq2_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 * 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq2_xxs_vec_dot(xb, yb, result), "iq2_xxs_vec_dot");
+}
+
+//
+// IQ2_XS -- superblock size 256, 74 bytes/block. Dequantize only.
+//
+
+void ggml_quants_halide_dequantize_iq2_xs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 * 2 + kQK / 32;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq2_xs_dequantize(blocks, yb), "iq2_xs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq2_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 * 2 + kQK / 32, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq2_xs_vec_dot(xb, yb, result), "iq2_xs_vec_dot");
+}
+
+//
+// IQ2_S -- superblock size 256, 82 bytes/block. Dequantize is native
+// Halide; quantize calls out to GGML's own reference.
+//
+
+void ggml_quants_halide_quantize_iq2_s(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 + kQK / 8 + kQK / 32 + kQK / 32;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq2_s_quantize(xb, blocks), "iq2_s_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq2_s(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 + kQK / 8 + kQK / 32 + kQK / 32;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq2_s_dequantize(blocks, yb), "iq2_s_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq2_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 + kQK / 8 + kQK / 32 + kQK / 32,
+                  kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq2_s_vec_dot(xb, yb, result), "iq2_s_vec_dot");
+}
+
+//
+// IQ3_XXS -- superblock size 256, 98 bytes/block. Dequantize is native
+// Halide; quantize calls out to GGML's own reference.
+//
+
+void ggml_quants_halide_quantize_iq3_xxs(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 3 * kQK / 8;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq3_xxs_quantize(xb, blocks), "iq3_xxs_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq3_xxs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 3 * kQK / 8;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq3_xxs_dequantize(blocks, yb), "iq3_xxs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq3_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + 3 * kQK / 8, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq3_xxs_vec_dot(xb, yb, result), "iq3_xxs_vec_dot");
+}
+
+//
+// IQ3_S -- superblock size 256, 110 bytes/block. Dequantize is native
+// Halide; quantize calls out to GGML's own reference.
+//
+
+void ggml_quants_halide_quantize_iq3_s(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 4 + kQK / 32 + kQK / 8 + kQK / 64;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq3_s_quantize(xb, blocks), "iq3_s_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq3_s(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 4 + kQK / 32 + kQK / 8 + kQK / 64;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq3_s_dequantize(blocks, yb), "iq3_s_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq3_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 4 + kQK / 32 + kQK / 8 + kQK / 64,
+                  kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq3_s_vec_dot(xb, yb, result), "iq3_s_vec_dot");
+}
+
+//
+// IQ1_S -- superblock size 256, 50 bytes/block. Dequantize only.
+//
+
+void ggml_quants_halide_dequantize_iq1_s(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 + kQK / 16;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq1_s_dequantize(blocks, yb), "iq1_s_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq1_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 + kQK / 16, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq1_s_vec_dot(xb, yb, result), "iq1_s_vec_dot");
+}
+
+//
+// IQ1_M -- superblock size 256, 56 bytes/block. Dequantize only.
+//
+
+void ggml_quants_halide_dequantize_iq1_m(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 8 + kQK / 16 + kQK / 32;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq1_m_dequantize(blocks, yb), "iq1_m_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq1_m_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 8 + kQK / 16 + kQK / 32, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq1_m_vec_dot(xb, yb, result), "iq1_m_vec_dot");
+}
+
+//
+// F16 -- block size 1, 2 bytes/element (plain IEEE binary16 cast, no header).
+//
+
+void ggml_quants_halide_quantize_f16(const float *x, void *y, int64_t k) {
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    Buffer<uint16_t, 1> yb(static_cast<uint16_t *>(y), static_cast<int>(k));
+    check(f16_quantize(xb, yb), "f16_quantize");
+}
+
+void ggml_quants_halide_dequantize_f16(const void *x, float *y, int64_t k) {
+    Buffer<uint16_t, 1> xb(const_cast<uint16_t *>(static_cast<const uint16_t *>(x)), static_cast<int>(k));
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(f16_dequantize(xb, yb), "f16_dequantize");
+}
+
+//
+// BF16 -- block size 1, 2 bytes/element (plain bfloat16 cast, no header).
+//
+
+void ggml_quants_halide_quantize_bf16(const float *x, void *y, int64_t k) {
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    Buffer<uint16_t, 1> yb(static_cast<uint16_t *>(y), static_cast<int>(k));
+    check(bf16_quantize(xb, yb), "bf16_quantize");
+}
+
+void ggml_quants_halide_dequantize_bf16(const void *x, float *y, int64_t k) {
+    Buffer<uint16_t, 1> xb(const_cast<uint16_t *>(static_cast<const uint16_t *>(x)), static_cast<int>(k));
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(bf16_dequantize(xb, yb), "bf16_dequantize");
+}
+
+//
+// Repack quantize_mat -- interleaves 4 contiguous rows of `k` floats (row r
+// at x[r*k .. r*k+k)) into one packed activation-format block per chunk.
+// `x` is wrapped as a 2-D buffer (dim 0: column-within-row, extent k; dim 1:
+// row, extent 4, stride k) so the generator can address x_(col, row)
+// directly instead of hand-computing `row*k + col`.
+//
+
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 * 2 + kQK * 4;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_0_4x4_quantize_mat(xb, blocks), "q8_0_4x4_quantize_mat");
+}
+
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x8(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 * 2 + kQK * 4;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_0_4x8_quantize_mat(xb, blocks), "q8_0_4x8_quantize_mat");
+}
+
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 * 4 + kQK * 4 + (kQK / 16) * 4 * 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_k_4x4_quantize_mat(xb, blocks), "q8_k_4x4_quantize_mat");
+}
+
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x8(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 * 4 + kQK * 4 + (kQK / 16) * 4 * 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_k_4x8_quantize_mat(xb, blocks), "q8_k_4x8_quantize_mat");
+}
+
+//
+// Repack gemv/gemm: dot a repack-interleaved weight matrix against a Q8_0
+// activation row (gemv) or `nr` rows packed 4 at a time (gemm). See
+// repack_gemv_generators.cpp/repack_gemm_generators.cpp for the packed
+// buffer layouts these shapes describe; `bs` (gemm's output row stride) is
+// unused by gemv, matching gemx_fn_t's own nr == 1 convention.
+//
+
+void ggml_quants_halide_repack_gemv_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_0_4x4_gemv(wb, ab, sb), "q4_0_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_0_4x8_gemv(wb, ab, sb), "q4_0_4x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_0_8x8_gemv(wb, ab, sb), "q4_0_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q8_0_4x4_gemv(wb, ab, sb), "q8_0_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q8_0_4x8_gemv(wb, ab, sb), "q8_0_4x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_0_4x4_gemm(wb, ab, sb), "q4_0_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_0_4x8_gemm(wb, ab, sb), "q4_0_4x8_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_0_8x8_gemm(wb, ab, sb), "q4_0_8x8_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q8_0_4x4_gemm(wb, ab, sb), "q8_0_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q8_0_4x8_gemm(wb, ab, sb), "q8_0_4x8_gemm");
+}
+
+//
+// IQ4_NL/MXFP4 repack gemv/gemm: same 3-D packed-buffer shapes as Q4_0's
+// above, but MXFP4's weight header is N bytes (1 E8M0 exponent per column)
+// instead of 2*N (an fp16 delta per column) -- see repack_gemv_generators.cpp.
+//
+
+void ggml_quants_halide_repack_gemv_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(iq4_nl_4x4_gemv(wb, ab, sb), "iq4_nl_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(iq4_nl_8x8_gemv(wb, ab, sb), "iq4_nl_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(mxfp4_4x4_gemv(wb, ab, sb), "mxfp4_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(mxfp4_8x8_gemv(wb, ab, sb), "mxfp4_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(iq4_nl_4x4_gemm(wb, ab, sb), "iq4_nl_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(iq4_nl_8x8_gemm(wb, ab, sb), "iq4_nl_8x8_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(mxfp4_4x4_gemm(wb, ab, sb), "mxfp4_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(mxfp4_8x8_gemm(wb, ab, sb), "mxfp4_8x8_gemm");
+}
+
+//
+// Q4_K repack gemv/gemm: 256-element superblocks, always 8 interleaved
+// columns, paired with Q8_K activations. See repack_gemv_generators.cpp/
+// repack_gemm_generators.cpp for the packed buffer layouts.
+//
+
+void ggml_quants_halide_repack_gemv_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_k_8x4_gemv(wb, ab, sb), "q4_k_8x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_k_8x8_gemv(wb, ab, sb), "q4_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    // block_q8_Kx4's real per-block size is 1168 bytes (16 header + 1024 qs +
+    // 128 bsums) -- that's the actual stride between consecutive blocks in
+    // memory, even though the Halide kernel only ever reads the first 1040
+    // bytes (header + qs; bsums are unused, see repack_gemm_generators.cpp).
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_k_8x4_gemm(wb, ab, sb), "q4_k_8x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_k_8x8_gemm(wb, ab, sb), "q4_k_8x8_gemm");
+}
+
+//
+// Q5_K repack gemv/gemm: same 256-element superblock/8-column structure as
+// Q4_K, plus a 256-byte qh (5th bit) array between scales and qs.
+//
+
+void ggml_quants_halide_repack_gemv_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q5_k_8x4_gemv(wb, ab, sb), "q5_k_8x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q5_k_8x8_gemv(wb, ab, sb), "q5_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q5_k_8x4_gemm(wb, ab, sb), "q5_k_8x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q5_k_8x8_gemm(wb, ab, sb), "q5_k_8x8_gemm");
+}
+
+//
+// Q6_K repack gemv/gemm: plain signed-int8-per-sub-group scales, no compact
+// bit packing -- see repack_gemv_generators.cpp/repack_gemm_generators.cpp.
+//
+
+void ggml_quants_halide_repack_gemv_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q6_k_8x4_gemv(wb, ab, sb), "q6_k_8x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q6_k_8x8_gemv(wb, ab, sb), "q6_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q6_k_8x4_gemm(wb, ab, sb), "q6_k_8x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q6_k_8x8_gemm(wb, ab, sb), "q6_k_8x8_gemm");
+}
+
+//
+// Q2_K repack gemv/gemm: only one registered variant (8x8). See
+// repack_gemv_generators.cpp/repack_gemm_generators.cpp.
+//
+
+void ggml_quants_halide_repack_gemv_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 128 + (kQK * kNCols * 2) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q2_k_8x8_gemv(wb, ab, sb), "q2_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 128 + (kQK * kNCols * 2) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q2_k_8x8_gemm(wb, ab, sb), "q2_k_8x8_gemm");
+}
+
+}  // extern "C"
diff --git a/apps/ggml/halide/ggml_quants.h b/apps/ggml/halide/ggml_quants.h
new file mode 100644
index 000000000000..84704cbda070
--- /dev/null
+++ b/apps/ggml/halide/ggml_quants.h
@@ -0,0 +1,276 @@
+#pragma once
+
+// Plain C ABI for the Halide-generated quantize/dequantize/vec_dot kernels,
+// matching apps/ggml/include/kernel_registry.h's quantize_fn_t/
+// dequantize_fn_t/vec_dot_fn_t signatures exactly (by signature
+// compatibility alone -- no shared header needed between this library and
+// the benchmark harness).
+//
+// Q8_1 has no dequantize entry: it's an activation-only format (GGML itself
+// has no public to_float for it), so there is nothing to implement.
+//
+// Every vec_dot_<x>_<y> function computes the dot product between a row of
+// weight-type x and a row of activation-type y (y is x's GGML vec_dot_type).
+// Both operands flow through the Approximation framework: the generic
+// symmetric/lookup_table/k_quant vec_dot generators splice weight and
+// activation codecs from quant_components.h via approximate_by/compute_offline
+// (see vec_dot_generator_base.h).
+
+#include <cstddef>
+#include <cstdint>
+
+extern "C" {
+
+void ggml_quants_halide_quantize_q4_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q4_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q4_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q4_1(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q4_1(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q4_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q5_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q5_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q5_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q5_1(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q5_1(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q5_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q8_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q8_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q8_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q8_1(const float *x, void *y, int64_t k);
+
+void ggml_quants_halide_quantize_q8_k(const float *x, void *y, int64_t k);
+
+// Q2_K, Q6_K: dequantize is a from-scratch Halide implementation; quantize
+// is scaffolding that calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp) pending a from-scratch port of GGML's iterative
+// scale search. vec_dot is from-scratch (against Q8_K activations).
+void ggml_quants_halide_quantize_q2_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q2_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q2_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q6_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q6_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q6_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q4_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q4_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q4_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q5_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q5_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q5_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q3_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q3_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q3_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+// Q1_0: closed-form both directions, fully native.
+void ggml_quants_halide_quantize_q1_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q1_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q1_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+// MXFP4, NVFP4, IQ4_NL, IQ4_XS, TQ1_0, TQ2_0: dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+void ggml_quants_halide_quantize_mxfp4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_mxfp4(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_mxfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_nvfp4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_nvfp4(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_nvfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq4_nl(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq4_nl(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq4_nl_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq4_xs(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq4_xs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq4_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc);
+
+void ggml_quants_halide_quantize_tq1_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_tq1_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_tq1_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_tq2_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_tq2_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_tq2_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+// IQ2_XXS: dequantize only. GGML has no public from_float_ref for this
+// importance-matrix-only codebook type (only a private whole-matrix
+// quantizer -- see providers/ggml_internal_abi.h), so there is no
+// from-scratch quantizer to write against it. vec_dot is still implemented
+// (GGML's own reference quantizer is used to produce test/benchmark input).
+void ggml_quants_halide_dequantize_iq2_xxs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq2_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc);
+
+// IQ2_XS: dequantize only (same reason as IQ2_XXS above).
+void ggml_quants_halide_dequantize_iq2_xs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq2_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc);
+
+// IQ2_S, IQ3_XXS, IQ3_S: dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp) -- these three do
+// have a public from_float_ref, unlike IQ2_XXS/IQ2_XS/IQ1_S/IQ1_M.
+void ggml_quants_halide_quantize_iq2_s(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq2_s(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq2_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq3_xxs(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq3_xxs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq3_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq3_s(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq3_s(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq3_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+// IQ1_S, IQ1_M: dequantize only (same reason as IQ2_XXS above).
+void ggml_quants_halide_dequantize_iq1_s(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq1_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_dequantize_iq1_m(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq1_m_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+// F16, BF16: not really "quantized" types -- block size 1, a plain per-
+// element float cast. Both directions are fully native Halide (Halide's
+// built-in float16_t/bfloat16_t casts implement the same IEEE round-to-
+// nearest-even conversions GGML's own reference uses). No vec_dot: not part
+// of the quantized-format vec_dot sweep this directory otherwise covers.
+void ggml_quants_halide_quantize_f16(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_f16(const void *x, float *y, int64_t k);
+
+void ggml_quants_halide_quantize_bf16(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_bf16(const void *x, float *y, int64_t k);
+
+// Repack quantize_mat kernels: interleave 4 contiguous rows of `k` floats
+// into one packed activation-format block per `k`-sized chunk (see
+// repack_quantize_mat_generators.cpp). Signature-compatible with
+// quantize_fn_t (same as every other quantize_* above) -- these are just
+// keyed by activation format + interleave width rather than weight type,
+// since GGML itself only has 4 distinct quantize_mat implementations shared
+// across every repack weight type (see k_repack_entries in
+// providers/ggml_provider.cpp and this library's registration in
+// providers/halide_provider.cpp).
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x8(const float *x, void *y, int64_t k);
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x8(const float *x, void *y, int64_t k);
+
+// Repack gemv/gemm: dot a repack-interleaved weight matrix (see
+// repack_quantize_mat_generators.cpp/repack_gemv_generators.cpp for the
+// packed layout) against, respectively, one plain-Q8_0 activation row
+// (gemv, matching gemx_fn_t's nr == 1 case) or `nr` activation rows packed 4
+// at a time by the matching repack_quantize_mat_* kernel above (gemm).
+// Q4_0's own two variants (4x4, 4x8) share one packed-weight byte layout
+// (see repack_gemv_generators.cpp); 8x8 uses a wider one. All 3 use
+// activations quantized by the matching-blocklen quantize_mat kernel, same
+// as ggml_provider.cpp's own k_repack_entries table.
+void ggml_quants_halide_repack_gemv_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+void ggml_quants_halide_repack_gemm_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// IQ4_NL/MXFP4: same repack-interleave scheme as Q4_0 (nibble/halves split),
+// but a plain codebook lookup (no XOR trick) for the weight value, and only
+// two interleave widths each (4x4, 8x8 -- no 4x8), matching GGML's own
+// k_repack_entries.
+void ggml_quants_halide_repack_gemv_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemv_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemv_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+void ggml_quants_halide_repack_gemv_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+
+void ggml_quants_halide_repack_gemm_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemm_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemm_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+void ggml_quants_halide_repack_gemm_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+
+// Q4_K: 256-element superblocks, always 8 interleaved columns, paired with
+// Q8_K activations.
+void ggml_quants_halide_repack_gemv_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// Q5_K: same 256-element superblock/8-column structure as Q4_K, plus a 5th
+// (high) bit array.
+void ggml_quants_halide_repack_gemv_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// Q6_K: plain signed-int8-per-sub-group scales (no compact bit packing).
+void ggml_quants_halide_repack_gemv_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// Q2_K: only one registered variant (8x8 -- GGML has no ARM 8x4 path).
+void ggml_quants_halide_repack_gemv_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+}
diff --git a/apps/ggml/halide/iq_grids_data.h b/apps/ggml/halide/iq_grids_data.h
new file mode 100644
index 000000000000..9076fe4ec162
--- /dev/null
+++ b/apps/ggml/halide/iq_grids_data.h
@@ -0,0 +1,4770 @@
+// Constant lookup tables copied verbatim from GGML's src/ggml-common.h
+// (commit eced84c86f8b012c752c016f7fe789adea168e1e / v0.15.3), used by the
+// codebook-based IQ2/IQ3/IQ1 dequantize generators. These are fixed,
+// published codebook constants, not derived logic -- transcribed exactly,
+// not "included" from GGML (no GGML headers are used elsewhere in this
+// directory's generators).
+#pragma once
+#include <cstdint>
+namespace iq_grids {
+constexpr uint8_t kmask_iq2xs[8] = {
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+};
+constexpr uint8_t ksigns_iq2xs[128] = {
+    0,
+    129,
+    130,
+    3,
+    132,
+    5,
+    6,
+    135,
+    136,
+    9,
+    10,
+    139,
+    12,
+    141,
+    142,
+    15,
+    144,
+    17,
+    18,
+    147,
+    20,
+    149,
+    150,
+    23,
+    24,
+    153,
+    154,
+    27,
+    156,
+    29,
+    30,
+    159,
+    160,
+    33,
+    34,
+    163,
+    36,
+    165,
+    166,
+    39,
+    40,
+    169,
+    170,
+    43,
+    172,
+    45,
+    46,
+    175,
+    48,
+    177,
+    178,
+    51,
+    180,
+    53,
+    54,
+    183,
+    184,
+    57,
+    58,
+    187,
+    60,
+    189,
+    190,
+    63,
+    192,
+    65,
+    66,
+    195,
+    68,
+    197,
+    198,
+    71,
+    72,
+    201,
+    202,
+    75,
+    204,
+    77,
+    78,
+    207,
+    80,
+    209,
+    210,
+    83,
+    212,
+    85,
+    86,
+    215,
+    216,
+    89,
+    90,
+    219,
+    92,
+    221,
+    222,
+    95,
+    96,
+    225,
+    226,
+    99,
+    228,
+    101,
+    102,
+    231,
+    232,
+    105,
+    106,
+    235,
+    108,
+    237,
+    238,
+    111,
+    240,
+    113,
+    114,
+    243,
+    116,
+    245,
+    246,
+    119,
+    120,
+    249,
+    250,
+    123,
+    252,
+    125,
+    126,
+    255,
+};
+constexpr uint64_t iq2xxs_grid[256] = {
+    0x0808080808080808,
+    0x080808080808082b,
+    0x0808080808081919,
+    0x0808080808082b08,
+    0x0808080808082b2b,
+    0x0808080808190819,
+    0x0808080808191908,
+    0x08080808082b0808,
+    0x08080808082b082b,
+    0x08080808082b2b08,
+    0x08080808082b2b2b,
+    0x0808080819080819,
+    0x0808080819081908,
+    0x0808080819190808,
+    0x0808080819192b08,
+    0x08080808192b0819,
+    0x08080808192b1908,
+    0x080808082b080808,
+    0x080808082b08082b,
+    0x080808082b082b2b,
+    0x080808082b2b082b,
+    0x0808081908080819,
+    0x0808081908081908,
+    0x0808081908190808,
+    0x0808081908191919,
+    0x0808081919080808,
+    0x080808192b081908,
+    0x080808192b192b08,
+    0x0808082b08080808,
+    0x0808082b0808082b,
+    0x0808082b082b082b,
+    0x0808082b2b08082b,
+    0x0808190808080819,
+    0x0808190808081908,
+    0x0808190808190808,
+    0x08081908082b0819,
+    0x08081908082b1908,
+    0x0808190819080808,
+    0x080819081908082b,
+    0x0808190819082b08,
+    0x08081908192b0808,
+    0x080819082b080819,
+    0x080819082b081908,
+    0x080819082b190808,
+    0x080819082b2b1908,
+    0x0808191908080808,
+    0x080819190808082b,
+    0x0808191908082b08,
+    0x08081919082b0808,
+    0x080819191908192b,
+    0x08081919192b2b19,
+    0x080819192b080808,
+    0x080819192b190819,
+    0x0808192b08082b19,
+    0x0808192b08190808,
+    0x0808192b19080808,
+    0x0808192b2b081908,
+    0x0808192b2b2b1908,
+    0x08082b0808080808,
+    0x08082b0808081919,
+    0x08082b0808082b08,
+    0x08082b0808191908,
+    0x08082b08082b2b08,
+    0x08082b0819080819,
+    0x08082b0819081908,
+    0x08082b0819190808,
+    0x08082b081919082b,
+    0x08082b082b082b08,
+    0x08082b1908081908,
+    0x08082b1919080808,
+    0x08082b2b0808082b,
+    0x08082b2b08191908,
+    0x0819080808080819,
+    0x0819080808081908,
+    0x0819080808190808,
+    0x08190808082b0819,
+    0x0819080819080808,
+    0x08190808192b0808,
+    0x081908082b081908,
+    0x081908082b190808,
+    0x081908082b191919,
+    0x0819081908080808,
+    0x0819081908082b08,
+    0x08190819082b0808,
+    0x0819081919190808,
+    0x0819081919192b2b,
+    0x081908192b080808,
+    0x0819082b082b1908,
+    0x0819082b19081919,
+    0x0819190808080808,
+    0x0819190808082b08,
+    0x08191908082b0808,
+    0x08191908082b1919,
+    0x0819190819082b19,
+    0x081919082b080808,
+    0x0819191908192b08,
+    0x08191919192b082b,
+    0x0819192b08080808,
+    0x0819192b0819192b,
+    0x08192b0808080819,
+    0x08192b0808081908,
+    0x08192b0808190808,
+    0x08192b0819080808,
+    0x08192b082b080819,
+    0x08192b1908080808,
+    0x08192b1908081919,
+    0x08192b192b2b0808,
+    0x08192b2b19190819,
+    0x082b080808080808,
+    0x082b08080808082b,
+    0x082b080808082b2b,
+    0x082b080819081908,
+    0x082b0808192b0819,
+    0x082b08082b080808,
+    0x082b08082b08082b,
+    0x082b0819082b2b19,
+    0x082b081919082b08,
+    0x082b082b08080808,
+    0x082b082b0808082b,
+    0x082b190808080819,
+    0x082b190808081908,
+    0x082b190808190808,
+    0x082b190819080808,
+    0x082b19081919192b,
+    0x082b191908080808,
+    0x082b191919080819,
+    0x082b1919192b1908,
+    0x082b192b2b190808,
+    0x082b2b0808082b08,
+    0x082b2b08082b0808,
+    0x082b2b082b191908,
+    0x082b2b2b19081908,
+    0x1908080808080819,
+    0x1908080808081908,
+    0x1908080808190808,
+    0x1908080808192b08,
+    0x19080808082b0819,
+    0x19080808082b1908,
+    0x1908080819080808,
+    0x1908080819082b08,
+    0x190808081919192b,
+    0x19080808192b0808,
+    0x190808082b080819,
+    0x190808082b081908,
+    0x190808082b190808,
+    0x1908081908080808,
+    0x19080819082b0808,
+    0x19080819192b0819,
+    0x190808192b080808,
+    0x190808192b081919,
+    0x1908082b08080819,
+    0x1908082b08190808,
+    0x1908082b19082b08,
+    0x1908082b1919192b,
+    0x1908082b192b2b08,
+    0x1908190808080808,
+    0x1908190808082b08,
+    0x19081908082b0808,
+    0x190819082b080808,
+    0x190819082b192b19,
+    0x190819190819082b,
+    0x19081919082b1908,
+    0x1908192b08080808,
+    0x19082b0808080819,
+    0x19082b0808081908,
+    0x19082b0808190808,
+    0x19082b0819080808,
+    0x19082b0819081919,
+    0x19082b1908080808,
+    0x19082b1919192b08,
+    0x19082b19192b0819,
+    0x19082b192b08082b,
+    0x19082b2b19081919,
+    0x19082b2b2b190808,
+    0x1919080808080808,
+    0x1919080808082b08,
+    0x1919080808190819,
+    0x1919080808192b19,
+    0x19190808082b0808,
+    0x191908082b080808,
+    0x191908082b082b08,
+    0x1919081908081908,
+    0x191908191908082b,
+    0x191908192b2b1908,
+    0x1919082b2b190819,
+    0x191919082b190808,
+    0x191919082b19082b,
+    0x1919191908082b2b,
+    0x1919192b08080819,
+    0x1919192b19191908,
+    0x19192b0808080808,
+    0x19192b0808190819,
+    0x19192b0808192b19,
+    0x19192b08192b1908,
+    0x19192b1919080808,
+    0x19192b2b08082b08,
+    0x192b080808081908,
+    0x192b080808190808,
+    0x192b080819080808,
+    0x192b0808192b2b08,
+    0x192b081908080808,
+    0x192b081919191919,
+    0x192b082b08192b08,
+    0x192b082b192b0808,
+    0x192b190808080808,
+    0x192b190808081919,
+    0x192b191908190808,
+    0x192b19190819082b,
+    0x192b19192b081908,
+    0x192b2b081908082b,
+    0x2b08080808080808,
+    0x2b0808080808082b,
+    0x2b08080808082b2b,
+    0x2b08080819080819,
+    0x2b0808082b08082b,
+    0x2b08081908081908,
+    0x2b08081908192b08,
+    0x2b08081919080808,
+    0x2b08082b08190819,
+    0x2b08190808080819,
+    0x2b08190808081908,
+    0x2b08190808190808,
+    0x2b08190808191919,
+    0x2b08190819080808,
+    0x2b081908192b0808,
+    0x2b08191908080808,
+    0x2b0819191908192b,
+    0x2b0819192b191908,
+    0x2b08192b08082b19,
+    0x2b08192b19080808,
+    0x2b08192b192b0808,
+    0x2b082b080808082b,
+    0x2b082b1908081908,
+    0x2b082b2b08190819,
+    0x2b19080808081908,
+    0x2b19080808190808,
+    0x2b190808082b1908,
+    0x2b19080819080808,
+    0x2b1908082b2b0819,
+    0x2b1908190819192b,
+    0x2b1908192b080808,
+    0x2b19082b19081919,
+    0x2b19190808080808,
+    0x2b191908082b082b,
+    0x2b19190819081908,
+    0x2b19191919190819,
+    0x2b192b082b080819,
+    0x2b192b19082b0808,
+    0x2b2b08080808082b,
+    0x2b2b080819190808,
+    0x2b2b08082b081919,
+    0x2b2b081908082b19,
+    0x2b2b082b08080808,
+    0x2b2b190808192b08,
+    0x2b2b2b0819190808,
+    0x2b2b2b1908081908,
+};
+constexpr uint64_t iq2xs_grid[512] = {
+    0x0808080808080808,
+    0x080808080808082b,
+    0x0808080808081919,
+    0x0808080808082b08,
+    0x0808080808082b2b,
+    0x0808080808190819,
+    0x0808080808191908,
+    0x080808080819192b,
+    0x0808080808192b19,
+    0x08080808082b0808,
+    0x08080808082b082b,
+    0x08080808082b1919,
+    0x08080808082b2b08,
+    0x0808080819080819,
+    0x0808080819081908,
+    0x080808081908192b,
+    0x0808080819082b19,
+    0x0808080819190808,
+    0x080808081919082b,
+    0x0808080819191919,
+    0x0808080819192b08,
+    0x08080808192b0819,
+    0x08080808192b1908,
+    0x080808082b080808,
+    0x080808082b08082b,
+    0x080808082b081919,
+    0x080808082b082b08,
+    0x080808082b190819,
+    0x080808082b191908,
+    0x080808082b192b19,
+    0x080808082b2b0808,
+    0x0808081908080819,
+    0x0808081908081908,
+    0x080808190808192b,
+    0x0808081908082b19,
+    0x0808081908190808,
+    0x080808190819082b,
+    0x0808081908191919,
+    0x0808081908192b08,
+    0x0808081908192b2b,
+    0x08080819082b0819,
+    0x08080819082b1908,
+    0x0808081919080808,
+    0x080808191908082b,
+    0x0808081919081919,
+    0x0808081919082b08,
+    0x0808081919190819,
+    0x0808081919191908,
+    0x08080819192b0808,
+    0x08080819192b2b08,
+    0x080808192b080819,
+    0x080808192b081908,
+    0x080808192b190808,
+    0x0808082b08080808,
+    0x0808082b0808082b,
+    0x0808082b08081919,
+    0x0808082b08082b08,
+    0x0808082b08190819,
+    0x0808082b08191908,
+    0x0808082b082b0808,
+    0x0808082b19080819,
+    0x0808082b19081908,
+    0x0808082b19190808,
+    0x0808082b19191919,
+    0x0808082b2b080808,
+    0x0808082b2b082b2b,
+    0x0808190808080819,
+    0x0808190808081908,
+    0x080819080808192b,
+    0x0808190808082b19,
+    0x0808190808190808,
+    0x080819080819082b,
+    0x0808190808191919,
+    0x0808190808192b08,
+    0x08081908082b0819,
+    0x08081908082b1908,
+    0x0808190819080808,
+    0x080819081908082b,
+    0x0808190819081919,
+    0x0808190819082b08,
+    0x0808190819190819,
+    0x0808190819191908,
+    0x080819081919192b,
+    0x08081908192b0808,
+    0x080819082b080819,
+    0x080819082b081908,
+    0x080819082b190808,
+    0x0808191908080808,
+    0x080819190808082b,
+    0x0808191908081919,
+    0x0808191908082b08,
+    0x0808191908190819,
+    0x0808191908191908,
+    0x08081919082b0808,
+    0x0808191919080819,
+    0x0808191919081908,
+    0x0808191919190808,
+    0x08081919192b0819,
+    0x080819192b080808,
+    0x0808192b08080819,
+    0x0808192b08081908,
+    0x0808192b08190808,
+    0x0808192b082b192b,
+    0x0808192b19080808,
+    0x0808192b1908082b,
+    0x0808192b2b081908,
+    0x08082b0808080808,
+    0x08082b080808082b,
+    0x08082b0808081919,
+    0x08082b0808082b08,
+    0x08082b0808082b2b,
+    0x08082b0808190819,
+    0x08082b0808191908,
+    0x08082b08082b0808,
+    0x08082b08082b1919,
+    0x08082b0819080819,
+    0x08082b0819081908,
+    0x08082b0819190808,
+    0x08082b0819192b08,
+    0x08082b082b080808,
+    0x08082b082b2b0808,
+    0x08082b082b2b2b2b,
+    0x08082b1908080819,
+    0x08082b1908081908,
+    0x08082b1908190808,
+    0x08082b1919080808,
+    0x08082b192b080819,
+    0x08082b192b082b19,
+    0x08082b2b08080808,
+    0x08082b2b082b0808,
+    0x08082b2b082b2b08,
+    0x08082b2b2b19192b,
+    0x08082b2b2b2b0808,
+    0x0819080808080819,
+    0x0819080808081908,
+    0x081908080808192b,
+    0x0819080808082b19,
+    0x0819080808190808,
+    0x081908080819082b,
+    0x0819080808191919,
+    0x0819080808192b08,
+    0x08190808082b0819,
+    0x08190808082b1908,
+    0x0819080819080808,
+    0x081908081908082b,
+    0x0819080819081919,
+    0x0819080819082b08,
+    0x0819080819190819,
+    0x0819080819191908,
+    0x08190808192b0808,
+    0x08190808192b2b2b,
+    0x081908082b080819,
+    0x081908082b081908,
+    0x081908082b190808,
+    0x0819081908080808,
+    0x081908190808082b,
+    0x0819081908081919,
+    0x0819081908082b08,
+    0x0819081908190819,
+    0x0819081908191908,
+    0x08190819082b0808,
+    0x0819081919080819,
+    0x0819081919081908,
+    0x0819081919190808,
+    0x081908192b080808,
+    0x081908192b191908,
+    0x081908192b19192b,
+    0x0819082b08080819,
+    0x0819082b08081908,
+    0x0819082b0808192b,
+    0x0819082b08190808,
+    0x0819082b19080808,
+    0x0819082b192b0808,
+    0x0819190808080808,
+    0x081919080808082b,
+    0x0819190808081919,
+    0x0819190808082b08,
+    0x0819190808190819,
+    0x0819190808191908,
+    0x08191908082b0808,
+    0x0819190819080819,
+    0x0819190819081908,
+    0x0819190819082b19,
+    0x0819190819190808,
+    0x08191908192b1908,
+    0x081919082b080808,
+    0x0819191908080819,
+    0x0819191908081908,
+    0x0819191908190808,
+    0x0819191919080808,
+    0x0819192b08080808,
+    0x0819192b08191908,
+    0x0819192b19082b19,
+    0x08192b0808080819,
+    0x08192b0808081908,
+    0x08192b0808190808,
+    0x08192b080819082b,
+    0x08192b0819080808,
+    0x08192b0819191908,
+    0x08192b082b08192b,
+    0x08192b1908080808,
+    0x08192b1908081919,
+    0x08192b19192b192b,
+    0x08192b2b19190819,
+    0x08192b2b2b2b2b19,
+    0x082b080808080808,
+    0x082b08080808082b,
+    0x082b080808081919,
+    0x082b080808082b08,
+    0x082b080808082b2b,
+    0x082b080808190819,
+    0x082b080808191908,
+    0x082b0808082b0808,
+    0x082b080819080819,
+    0x082b080819081908,
+    0x082b080819190808,
+    0x082b08082b080808,
+    0x082b08082b2b0808,
+    0x082b081908080819,
+    0x082b081908081908,
+    0x082b081908190808,
+    0x082b081919080808,
+    0x082b081919082b08,
+    0x082b0819192b1919,
+    0x082b082b08080808,
+    0x082b082b082b082b,
+    0x082b082b2b080808,
+    0x082b082b2b2b2b08,
+    0x082b190808080819,
+    0x082b190808081908,
+    0x082b190808190808,
+    0x082b1908082b2b19,
+    0x082b190819080808,
+    0x082b191908080808,
+    0x082b191919080819,
+    0x082b19191919082b,
+    0x082b19192b192b19,
+    0x082b192b08080819,
+    0x082b192b08192b2b,
+    0x082b192b2b2b192b,
+    0x082b2b0808080808,
+    0x082b2b0808082b08,
+    0x082b2b0808082b2b,
+    0x082b2b08082b0808,
+    0x082b2b0819191919,
+    0x082b2b082b082b08,
+    0x082b2b082b2b082b,
+    0x082b2b19192b2b08,
+    0x082b2b192b190808,
+    0x082b2b2b08082b08,
+    0x082b2b2b082b0808,
+    0x082b2b2b2b08082b,
+    0x082b2b2b2b082b08,
+    0x082b2b2b2b082b2b,
+    0x1908080808080819,
+    0x1908080808081908,
+    0x190808080808192b,
+    0x1908080808082b19,
+    0x1908080808190808,
+    0x190808080819082b,
+    0x1908080808191919,
+    0x1908080808192b08,
+    0x19080808082b0819,
+    0x19080808082b1908,
+    0x1908080819080808,
+    0x190808081908082b,
+    0x1908080819081919,
+    0x1908080819082b08,
+    0x1908080819082b2b,
+    0x1908080819190819,
+    0x1908080819191908,
+    0x19080808192b0808,
+    0x19080808192b1919,
+    0x190808082b080819,
+    0x190808082b081908,
+    0x190808082b190808,
+    0x1908081908080808,
+    0x190808190808082b,
+    0x1908081908081919,
+    0x1908081908082b08,
+    0x1908081908190819,
+    0x1908081908191908,
+    0x19080819082b0808,
+    0x1908081919080819,
+    0x1908081919081908,
+    0x1908081919190808,
+    0x190808192b080808,
+    0x190808192b081919,
+    0x190808192b2b082b,
+    0x1908082b08080819,
+    0x1908082b08081908,
+    0x1908082b08190808,
+    0x1908082b0819082b,
+    0x1908082b082b2b19,
+    0x1908082b19080808,
+    0x1908190808080808,
+    0x190819080808082b,
+    0x1908190808081919,
+    0x1908190808082b08,
+    0x1908190808190819,
+    0x1908190808191908,
+    0x1908190808192b19,
+    0x19081908082b0808,
+    0x1908190819080819,
+    0x1908190819081908,
+    0x1908190819190808,
+    0x190819082b080808,
+    0x190819082b191908,
+    0x1908191908080819,
+    0x1908191908081908,
+    0x1908191908190808,
+    0x19081919082b1908,
+    0x1908191919080808,
+    0x190819192b192b2b,
+    0x1908192b08080808,
+    0x1908192b08082b2b,
+    0x1908192b19081908,
+    0x1908192b19190808,
+    0x19082b0808080819,
+    0x19082b0808081908,
+    0x19082b0808190808,
+    0x19082b0819080808,
+    0x19082b0819081919,
+    0x19082b0819191908,
+    0x19082b08192b082b,
+    0x19082b1908080808,
+    0x19082b1908190819,
+    0x19082b1919081908,
+    0x19082b1919190808,
+    0x19082b19192b2b19,
+    0x19082b2b08081908,
+    0x1919080808080808,
+    0x191908080808082b,
+    0x1919080808081919,
+    0x1919080808082b08,
+    0x1919080808190819,
+    0x1919080808191908,
+    0x19190808082b0808,
+    0x19190808082b2b08,
+    0x1919080819080819,
+    0x1919080819081908,
+    0x1919080819190808,
+    0x191908082b080808,
+    0x1919081908080819,
+    0x1919081908081908,
+    0x1919081908190808,
+    0x1919081908191919,
+    0x1919081919080808,
+    0x191908191908082b,
+    0x1919082b08080808,
+    0x1919082b19081908,
+    0x1919082b2b2b2b2b,
+    0x1919190808080819,
+    0x1919190808081908,
+    0x1919190808190808,
+    0x19191908082b0819,
+    0x1919190819080808,
+    0x19191908192b0808,
+    0x191919082b080819,
+    0x191919082b2b0819,
+    0x1919191908080808,
+    0x1919191908082b08,
+    0x191919192b080808,
+    0x191919192b082b08,
+    0x1919192b082b0819,
+    0x1919192b192b2b08,
+    0x1919192b2b2b0819,
+    0x19192b0808080808,
+    0x19192b0808191908,
+    0x19192b0819080819,
+    0x19192b0819190808,
+    0x19192b082b192b19,
+    0x19192b1908192b2b,
+    0x19192b1919080808,
+    0x19192b191908082b,
+    0x19192b2b2b081919,
+    0x192b080808080819,
+    0x192b080808081908,
+    0x192b080808190808,
+    0x192b080819080808,
+    0x192b080819191908,
+    0x192b0808192b082b,
+    0x192b08082b08192b,
+    0x192b08082b2b2b19,
+    0x192b081908080808,
+    0x192b082b082b1908,
+    0x192b082b19082b2b,
+    0x192b082b2b19082b,
+    0x192b190808080808,
+    0x192b19080819192b,
+    0x192b191908190808,
+    0x192b191919080808,
+    0x192b191919081919,
+    0x192b19192b2b1908,
+    0x192b2b0808080819,
+    0x192b2b08192b2b2b,
+    0x192b2b19082b1919,
+    0x192b2b2b0808192b,
+    0x192b2b2b19191908,
+    0x192b2b2b192b082b,
+    0x2b08080808080808,
+    0x2b0808080808082b,
+    0x2b08080808081919,
+    0x2b08080808082b08,
+    0x2b08080808190819,
+    0x2b08080808191908,
+    0x2b080808082b0808,
+    0x2b080808082b2b2b,
+    0x2b08080819080819,
+    0x2b08080819081908,
+    0x2b08080819190808,
+    0x2b0808082b080808,
+    0x2b0808082b08082b,
+    0x2b0808082b2b2b08,
+    0x2b0808082b2b2b2b,
+    0x2b08081908080819,
+    0x2b08081908081908,
+    0x2b0808190808192b,
+    0x2b08081908190808,
+    0x2b08081919080808,
+    0x2b08081919190819,
+    0x2b08081919192b19,
+    0x2b08082b08080808,
+    0x2b08082b082b0808,
+    0x2b08082b2b080808,
+    0x2b08082b2b08082b,
+    0x2b08082b2b2b0808,
+    0x2b08082b2b2b2b08,
+    0x2b08190808080819,
+    0x2b08190808081908,
+    0x2b08190808190808,
+    0x2b0819080819082b,
+    0x2b08190808191919,
+    0x2b08190819080808,
+    0x2b081908192b0808,
+    0x2b0819082b082b19,
+    0x2b08191908080808,
+    0x2b08191919081908,
+    0x2b0819192b2b1919,
+    0x2b08192b08192b08,
+    0x2b08192b192b2b2b,
+    0x2b082b0808080808,
+    0x2b082b0808082b08,
+    0x2b082b08082b1919,
+    0x2b082b0819192b2b,
+    0x2b082b082b080808,
+    0x2b082b082b08082b,
+    0x2b082b082b2b2b08,
+    0x2b082b190808192b,
+    0x2b082b2b082b082b,
+    0x2b082b2b2b080808,
+    0x2b082b2b2b082b08,
+    0x2b082b2b2b19192b,
+    0x2b082b2b2b2b2b08,
+    0x2b19080808080819,
+    0x2b19080808081908,
+    0x2b19080808190808,
+    0x2b19080819080808,
+    0x2b1908081919192b,
+    0x2b1908082b081908,
+    0x2b19081908080808,
+    0x2b190819082b082b,
+    0x2b190819192b1908,
+    0x2b19082b1919192b,
+    0x2b19082b2b082b19,
+    0x2b19190808080808,
+    0x2b19190808081919,
+    0x2b19190819081908,
+    0x2b19190819190808,
+    0x2b19190819192b08,
+    0x2b191919082b2b19,
+    0x2b1919192b190808,
+    0x2b1919192b19082b,
+    0x2b19192b19080819,
+    0x2b192b0819190819,
+    0x2b192b082b2b192b,
+    0x2b192b1919082b19,
+    0x2b192b2b08191919,
+    0x2b192b2b192b0808,
+    0x2b2b080808080808,
+    0x2b2b08080808082b,
+    0x2b2b080808082b08,
+    0x2b2b080808082b2b,
+    0x2b2b0808082b0808,
+    0x2b2b0808082b2b2b,
+    0x2b2b08082b2b0808,
+    0x2b2b081919190819,
+    0x2b2b081919192b19,
+    0x2b2b08192b2b192b,
+    0x2b2b082b08080808,
+    0x2b2b082b0808082b,
+    0x2b2b082b08082b08,
+    0x2b2b082b082b2b2b,
+    0x2b2b082b2b080808,
+    0x2b2b082b2b2b0808,
+    0x2b2b190819080808,
+    0x2b2b19082b191919,
+    0x2b2b192b192b1919,
+    0x2b2b192b2b192b08,
+    0x2b2b2b0808082b2b,
+    0x2b2b2b08082b0808,
+    0x2b2b2b08082b082b,
+    0x2b2b2b08082b2b08,
+    0x2b2b2b082b2b0808,
+    0x2b2b2b082b2b2b08,
+    0x2b2b2b1908081908,
+    0x2b2b2b192b081908,
+    0x2b2b2b192b08192b,
+    0x2b2b2b2b082b2b08,
+    0x2b2b2b2b082b2b2b,
+    0x2b2b2b2b2b190819,
+    0x2b2b2b2b2b2b2b2b,
+};
+constexpr uint64_t iq2s_grid[1024] = {
+    0x0808080808080808,
+    0x080808080808082b,
+    0x0808080808081919,
+    0x0808080808082b08,
+    0x0808080808082b2b,
+    0x0808080808190819,
+    0x0808080808191908,
+    0x080808080819192b,
+    0x0808080808192b19,
+    0x08080808082b0808,
+    0x08080808082b082b,
+    0x08080808082b1919,
+    0x08080808082b2b08,
+    0x0808080819080819,
+    0x0808080819081908,
+    0x080808081908192b,
+    0x0808080819082b19,
+    0x0808080819190808,
+    0x080808081919082b,
+    0x0808080819191919,
+    0x0808080819192b08,
+    0x08080808192b0819,
+    0x08080808192b1908,
+    0x08080808192b192b,
+    0x08080808192b2b19,
+    0x080808082b080808,
+    0x080808082b08082b,
+    0x080808082b081919,
+    0x080808082b082b08,
+    0x080808082b190819,
+    0x080808082b191908,
+    0x080808082b2b0808,
+    0x080808082b2b1919,
+    0x080808082b2b2b2b,
+    0x0808081908080819,
+    0x0808081908081908,
+    0x080808190808192b,
+    0x0808081908082b19,
+    0x0808081908190808,
+    0x080808190819082b,
+    0x0808081908191919,
+    0x0808081908192b08,
+    0x08080819082b0819,
+    0x08080819082b1908,
+    0x0808081919080808,
+    0x080808191908082b,
+    0x0808081919081919,
+    0x0808081919082b08,
+    0x0808081919190819,
+    0x0808081919191908,
+    0x080808191919192b,
+    0x0808081919192b19,
+    0x08080819192b0808,
+    0x08080819192b1919,
+    0x08080819192b2b08,
+    0x080808192b080819,
+    0x080808192b081908,
+    0x080808192b190808,
+    0x080808192b19082b,
+    0x080808192b191919,
+    0x080808192b2b0819,
+    0x080808192b2b1908,
+    0x0808082b08080808,
+    0x0808082b0808082b,
+    0x0808082b08081919,
+    0x0808082b08082b08,
+    0x0808082b08190819,
+    0x0808082b08191908,
+    0x0808082b082b0808,
+    0x0808082b082b2b2b,
+    0x0808082b19080819,
+    0x0808082b19081908,
+    0x0808082b1908192b,
+    0x0808082b19082b19,
+    0x0808082b19190808,
+    0x0808082b19191919,
+    0x0808082b2b080808,
+    0x0808082b2b081919,
+    0x0808082b2b082b2b,
+    0x0808082b2b191908,
+    0x0808082b2b2b082b,
+    0x0808190808080819,
+    0x0808190808081908,
+    0x080819080808192b,
+    0x0808190808082b19,
+    0x0808190808190808,
+    0x080819080819082b,
+    0x0808190808191919,
+    0x0808190808192b08,
+    0x08081908082b0819,
+    0x08081908082b1908,
+    0x08081908082b192b,
+    0x08081908082b2b19,
+    0x0808190819080808,
+    0x080819081908082b,
+    0x0808190819081919,
+    0x0808190819082b08,
+    0x0808190819082b2b,
+    0x0808190819190819,
+    0x0808190819191908,
+    0x080819081919192b,
+    0x0808190819192b19,
+    0x08081908192b0808,
+    0x08081908192b082b,
+    0x08081908192b1919,
+    0x080819082b080819,
+    0x080819082b081908,
+    0x080819082b08192b,
+    0x080819082b082b19,
+    0x080819082b190808,
+    0x080819082b191919,
+    0x080819082b192b08,
+    0x080819082b2b0819,
+    0x080819082b2b1908,
+    0x0808191908080808,
+    0x080819190808082b,
+    0x0808191908081919,
+    0x0808191908082b08,
+    0x0808191908082b2b,
+    0x0808191908190819,
+    0x0808191908191908,
+    0x080819190819192b,
+    0x0808191908192b19,
+    0x08081919082b0808,
+    0x08081919082b1919,
+    0x08081919082b2b08,
+    0x0808191919080819,
+    0x0808191919081908,
+    0x080819191908192b,
+    0x0808191919082b19,
+    0x0808191919190808,
+    0x080819191919082b,
+    0x0808191919191919,
+    0x0808191919192b08,
+    0x08081919192b0819,
+    0x08081919192b1908,
+    0x080819192b080808,
+    0x080819192b08082b,
+    0x080819192b081919,
+    0x080819192b082b08,
+    0x080819192b190819,
+    0x080819192b191908,
+    0x080819192b2b0808,
+    0x0808192b08080819,
+    0x0808192b08081908,
+    0x0808192b0808192b,
+    0x0808192b08082b19,
+    0x0808192b08190808,
+    0x0808192b08191919,
+    0x0808192b19080808,
+    0x0808192b19081919,
+    0x0808192b19082b08,
+    0x0808192b19190819,
+    0x0808192b19191908,
+    0x0808192b192b0808,
+    0x0808192b2b080819,
+    0x0808192b2b081908,
+    0x0808192b2b190808,
+    0x08082b0808080808,
+    0x08082b080808082b,
+    0x08082b0808081919,
+    0x08082b0808082b08,
+    0x08082b0808190819,
+    0x08082b0808191908,
+    0x08082b080819192b,
+    0x08082b0808192b19,
+    0x08082b08082b0808,
+    0x08082b08082b1919,
+    0x08082b08082b2b2b,
+    0x08082b0819080819,
+    0x08082b0819081908,
+    0x08082b081908192b,
+    0x08082b0819082b19,
+    0x08082b0819190808,
+    0x08082b081919082b,
+    0x08082b0819191919,
+    0x08082b0819192b08,
+    0x08082b08192b0819,
+    0x08082b08192b1908,
+    0x08082b082b080808,
+    0x08082b082b081919,
+    0x08082b082b191908,
+    0x08082b082b2b2b2b,
+    0x08082b1908080819,
+    0x08082b1908081908,
+    0x08082b1908190808,
+    0x08082b190819082b,
+    0x08082b1908191919,
+    0x08082b1908192b08,
+    0x08082b19082b0819,
+    0x08082b1919080808,
+    0x08082b1919081919,
+    0x08082b1919082b08,
+    0x08082b1919190819,
+    0x08082b1919191908,
+    0x08082b19192b0808,
+    0x08082b192b080819,
+    0x08082b192b190808,
+    0x08082b2b08080808,
+    0x08082b2b08190819,
+    0x08082b2b08191908,
+    0x08082b2b082b082b,
+    0x08082b2b082b2b08,
+    0x08082b2b082b2b2b,
+    0x08082b2b19190808,
+    0x08082b2b2b192b19,
+    0x0819080808080819,
+    0x0819080808081908,
+    0x081908080808192b,
+    0x0819080808082b19,
+    0x0819080808190808,
+    0x081908080819082b,
+    0x0819080808191919,
+    0x0819080808192b08,
+    0x08190808082b0819,
+    0x08190808082b1908,
+    0x08190808082b192b,
+    0x0819080819080808,
+    0x081908081908082b,
+    0x0819080819081919,
+    0x0819080819082b08,
+    0x0819080819190819,
+    0x0819080819191908,
+    0x081908081919192b,
+    0x0819080819192b19,
+    0x08190808192b0808,
+    0x08190808192b082b,
+    0x08190808192b1919,
+    0x08190808192b2b08,
+    0x081908082b080819,
+    0x081908082b081908,
+    0x081908082b08192b,
+    0x081908082b190808,
+    0x081908082b191919,
+    0x081908082b192b08,
+    0x081908082b2b0819,
+    0x081908082b2b1908,
+    0x0819081908080808,
+    0x081908190808082b,
+    0x0819081908081919,
+    0x0819081908082b08,
+    0x0819081908082b2b,
+    0x0819081908190819,
+    0x0819081908191908,
+    0x081908190819192b,
+    0x0819081908192b19,
+    0x08190819082b0808,
+    0x08190819082b082b,
+    0x08190819082b1919,
+    0x08190819082b2b08,
+    0x0819081919080819,
+    0x0819081919081908,
+    0x081908191908192b,
+    0x0819081919082b19,
+    0x0819081919190808,
+    0x081908191919082b,
+    0x0819081919191919,
+    0x0819081919192b08,
+    0x08190819192b0819,
+    0x08190819192b1908,
+    0x081908192b080808,
+    0x081908192b08082b,
+    0x081908192b081919,
+    0x081908192b082b08,
+    0x081908192b190819,
+    0x081908192b191908,
+    0x0819082b08080819,
+    0x0819082b08081908,
+    0x0819082b08082b19,
+    0x0819082b08190808,
+    0x0819082b08191919,
+    0x0819082b082b0819,
+    0x0819082b082b1908,
+    0x0819082b19080808,
+    0x0819082b19081919,
+    0x0819082b19190819,
+    0x0819082b19191908,
+    0x0819082b2b080819,
+    0x0819082b2b081908,
+    0x0819082b2b190808,
+    0x0819190808080808,
+    0x081919080808082b,
+    0x0819190808081919,
+    0x0819190808082b08,
+    0x0819190808190819,
+    0x0819190808191908,
+    0x081919080819192b,
+    0x0819190808192b19,
+    0x08191908082b0808,
+    0x08191908082b1919,
+    0x08191908082b2b08,
+    0x0819190819080819,
+    0x0819190819081908,
+    0x081919081908192b,
+    0x0819190819082b19,
+    0x0819190819190808,
+    0x081919081919082b,
+    0x0819190819191919,
+    0x0819190819192b08,
+    0x08191908192b0819,
+    0x08191908192b1908,
+    0x081919082b080808,
+    0x081919082b08082b,
+    0x081919082b081919,
+    0x081919082b082b08,
+    0x081919082b190819,
+    0x081919082b191908,
+    0x081919082b2b0808,
+    0x0819191908080819,
+    0x0819191908081908,
+    0x081919190808192b,
+    0x0819191908082b19,
+    0x0819191908190808,
+    0x081919190819082b,
+    0x0819191908191919,
+    0x0819191908192b08,
+    0x08191919082b0819,
+    0x08191919082b1908,
+    0x0819191919080808,
+    0x081919191908082b,
+    0x0819191919081919,
+    0x0819191919082b08,
+    0x0819191919190819,
+    0x0819191919191908,
+    0x08191919192b0808,
+    0x081919192b080819,
+    0x081919192b081908,
+    0x081919192b190808,
+    0x0819192b08080808,
+    0x0819192b08081919,
+    0x0819192b08082b08,
+    0x0819192b08190819,
+    0x0819192b08191908,
+    0x0819192b082b0808,
+    0x0819192b19080819,
+    0x0819192b19081908,
+    0x0819192b19190808,
+    0x0819192b2b080808,
+    0x0819192b2b2b2b2b,
+    0x08192b0808080819,
+    0x08192b0808081908,
+    0x08192b080808192b,
+    0x08192b0808082b19,
+    0x08192b0808190808,
+    0x08192b0808191919,
+    0x08192b0808192b08,
+    0x08192b08082b0819,
+    0x08192b0819080808,
+    0x08192b081908082b,
+    0x08192b0819081919,
+    0x08192b0819082b08,
+    0x08192b0819190819,
+    0x08192b0819191908,
+    0x08192b08192b0808,
+    0x08192b082b080819,
+    0x08192b082b081908,
+    0x08192b1908080808,
+    0x08192b190808082b,
+    0x08192b1908081919,
+    0x08192b1908082b08,
+    0x08192b1908190819,
+    0x08192b1908191908,
+    0x08192b19082b0808,
+    0x08192b1919080819,
+    0x08192b1919081908,
+    0x08192b1919190808,
+    0x08192b19192b2b19,
+    0x08192b192b2b082b,
+    0x08192b2b08081908,
+    0x08192b2b08190808,
+    0x08192b2b19080808,
+    0x08192b2b1919192b,
+    0x082b080808080808,
+    0x082b08080808082b,
+    0x082b080808081919,
+    0x082b080808082b08,
+    0x082b080808190819,
+    0x082b080808191908,
+    0x082b08080819192b,
+    0x082b080808192b19,
+    0x082b0808082b0808,
+    0x082b0808082b1919,
+    0x082b0808082b2b2b,
+    0x082b080819080819,
+    0x082b080819081908,
+    0x082b080819190808,
+    0x082b08081919082b,
+    0x082b080819191919,
+    0x082b0808192b1908,
+    0x082b08082b080808,
+    0x082b08082b082b2b,
+    0x082b08082b191908,
+    0x082b08082b2b2b2b,
+    0x082b081908080819,
+    0x082b081908081908,
+    0x082b081908190808,
+    0x082b08190819082b,
+    0x082b081908191919,
+    0x082b0819082b0819,
+    0x082b081919080808,
+    0x082b08191908082b,
+    0x082b081919081919,
+    0x082b081919190819,
+    0x082b081919191908,
+    0x082b0819192b0808,
+    0x082b08192b080819,
+    0x082b08192b081908,
+    0x082b08192b190808,
+    0x082b082b08080808,
+    0x082b082b08082b2b,
+    0x082b082b082b082b,
+    0x082b082b082b2b08,
+    0x082b082b082b2b2b,
+    0x082b082b19081908,
+    0x082b082b19190808,
+    0x082b082b2b082b08,
+    0x082b082b2b082b2b,
+    0x082b082b2b2b2b08,
+    0x082b190808080819,
+    0x082b190808081908,
+    0x082b19080808192b,
+    0x082b190808082b19,
+    0x082b190808190808,
+    0x082b190808191919,
+    0x082b190808192b08,
+    0x082b1908082b0819,
+    0x082b1908082b1908,
+    0x082b190819080808,
+    0x082b19081908082b,
+    0x082b190819081919,
+    0x082b190819082b08,
+    0x082b190819190819,
+    0x082b190819191908,
+    0x082b1908192b0808,
+    0x082b19082b080819,
+    0x082b19082b081908,
+    0x082b19082b190808,
+    0x082b191908080808,
+    0x082b191908081919,
+    0x082b191908082b08,
+    0x082b191908190819,
+    0x082b191908191908,
+    0x082b1919082b0808,
+    0x082b191919080819,
+    0x082b191919081908,
+    0x082b191919190808,
+    0x082b1919192b192b,
+    0x082b19192b080808,
+    0x082b192b08080819,
+    0x082b192b08081908,
+    0x082b192b08190808,
+    0x082b192b19080808,
+    0x082b192b19192b19,
+    0x082b2b0808080808,
+    0x082b2b0808081919,
+    0x082b2b0808190819,
+    0x082b2b0808191908,
+    0x082b2b0819080819,
+    0x082b2b0819081908,
+    0x082b2b0819190808,
+    0x082b2b082b082b2b,
+    0x082b2b082b2b2b2b,
+    0x082b2b1908080819,
+    0x082b2b1908081908,
+    0x082b2b1908190808,
+    0x082b2b192b191919,
+    0x082b2b2b08082b2b,
+    0x082b2b2b082b082b,
+    0x082b2b2b192b1908,
+    0x082b2b2b2b082b08,
+    0x082b2b2b2b082b2b,
+    0x1908080808080819,
+    0x1908080808081908,
+    0x190808080808192b,
+    0x1908080808082b19,
+    0x1908080808190808,
+    0x190808080819082b,
+    0x1908080808191919,
+    0x1908080808192b08,
+    0x1908080808192b2b,
+    0x19080808082b0819,
+    0x19080808082b1908,
+    0x19080808082b192b,
+    0x1908080819080808,
+    0x190808081908082b,
+    0x1908080819081919,
+    0x1908080819082b08,
+    0x1908080819082b2b,
+    0x1908080819190819,
+    0x1908080819191908,
+    0x190808081919192b,
+    0x1908080819192b19,
+    0x19080808192b0808,
+    0x19080808192b082b,
+    0x19080808192b1919,
+    0x190808082b080819,
+    0x190808082b081908,
+    0x190808082b190808,
+    0x190808082b191919,
+    0x190808082b192b08,
+    0x190808082b2b0819,
+    0x190808082b2b1908,
+    0x1908081908080808,
+    0x190808190808082b,
+    0x1908081908081919,
+    0x1908081908082b08,
+    0x1908081908190819,
+    0x1908081908191908,
+    0x190808190819192b,
+    0x1908081908192b19,
+    0x19080819082b0808,
+    0x19080819082b082b,
+    0x19080819082b1919,
+    0x1908081919080819,
+    0x1908081919081908,
+    0x190808191908192b,
+    0x1908081919082b19,
+    0x1908081919190808,
+    0x190808191919082b,
+    0x1908081919191919,
+    0x1908081919192b08,
+    0x19080819192b0819,
+    0x19080819192b1908,
+    0x190808192b080808,
+    0x190808192b08082b,
+    0x190808192b081919,
+    0x190808192b082b08,
+    0x190808192b190819,
+    0x190808192b191908,
+    0x190808192b2b0808,
+    0x1908082b08080819,
+    0x1908082b08081908,
+    0x1908082b08190808,
+    0x1908082b0819082b,
+    0x1908082b08191919,
+    0x1908082b08192b08,
+    0x1908082b082b1908,
+    0x1908082b19080808,
+    0x1908082b19081919,
+    0x1908082b19082b08,
+    0x1908082b19190819,
+    0x1908082b19191908,
+    0x1908082b192b0808,
+    0x1908082b2b080819,
+    0x1908082b2b081908,
+    0x1908190808080808,
+    0x190819080808082b,
+    0x1908190808081919,
+    0x1908190808082b08,
+    0x1908190808082b2b,
+    0x1908190808190819,
+    0x1908190808191908,
+    0x190819080819192b,
+    0x1908190808192b19,
+    0x19081908082b0808,
+    0x19081908082b082b,
+    0x19081908082b1919,
+    0x19081908082b2b08,
+    0x1908190819080819,
+    0x1908190819081908,
+    0x190819081908192b,
+    0x1908190819082b19,
+    0x1908190819190808,
+    0x190819081919082b,
+    0x1908190819191919,
+    0x1908190819192b08,
+    0x19081908192b0819,
+    0x19081908192b1908,
+    0x190819082b080808,
+    0x190819082b08082b,
+    0x190819082b081919,
+    0x190819082b082b08,
+    0x190819082b190819,
+    0x190819082b191908,
+    0x190819082b2b0808,
+    0x1908191908080819,
+    0x1908191908081908,
+    0x190819190808192b,
+    0x1908191908082b19,
+    0x1908191908190808,
+    0x190819190819082b,
+    0x1908191908191919,
+    0x1908191908192b08,
+    0x19081919082b0819,
+    0x19081919082b1908,
+    0x1908191919080808,
+    0x190819191908082b,
+    0x1908191919081919,
+    0x1908191919082b08,
+    0x1908191919190819,
+    0x1908191919191908,
+    0x19081919192b0808,
+    0x19081919192b2b2b,
+    0x190819192b080819,
+    0x190819192b081908,
+    0x190819192b190808,
+    0x1908192b08080808,
+    0x1908192b0808082b,
+    0x1908192b08081919,
+    0x1908192b08082b08,
+    0x1908192b08190819,
+    0x1908192b08191908,
+    0x1908192b082b0808,
+    0x1908192b19080819,
+    0x1908192b19081908,
+    0x1908192b19190808,
+    0x1908192b2b080808,
+    0x1908192b2b2b1919,
+    0x19082b0808080819,
+    0x19082b0808081908,
+    0x19082b0808082b19,
+    0x19082b0808190808,
+    0x19082b080819082b,
+    0x19082b0808191919,
+    0x19082b0808192b08,
+    0x19082b08082b0819,
+    0x19082b08082b1908,
+    0x19082b0819080808,
+    0x19082b081908082b,
+    0x19082b0819081919,
+    0x19082b0819082b08,
+    0x19082b0819190819,
+    0x19082b0819191908,
+    0x19082b08192b0808,
+    0x19082b082b081908,
+    0x19082b082b190808,
+    0x19082b1908080808,
+    0x19082b190808082b,
+    0x19082b1908081919,
+    0x19082b1908082b08,
+    0x19082b1908190819,
+    0x19082b1908191908,
+    0x19082b19082b0808,
+    0x19082b1919080819,
+    0x19082b1919081908,
+    0x19082b1919190808,
+    0x19082b192b080808,
+    0x19082b192b19192b,
+    0x19082b2b08080819,
+    0x19082b2b08081908,
+    0x19082b2b08190808,
+    0x19082b2b19080808,
+    0x1919080808080808,
+    0x191908080808082b,
+    0x1919080808081919,
+    0x1919080808082b08,
+    0x1919080808190819,
+    0x1919080808191908,
+    0x191908080819192b,
+    0x1919080808192b19,
+    0x19190808082b0808,
+    0x19190808082b082b,
+    0x19190808082b1919,
+    0x19190808082b2b08,
+    0x1919080819080819,
+    0x1919080819081908,
+    0x191908081908192b,
+    0x1919080819082b19,
+    0x1919080819190808,
+    0x191908081919082b,
+    0x1919080819191919,
+    0x1919080819192b08,
+    0x19190808192b0819,
+    0x19190808192b1908,
+    0x191908082b080808,
+    0x191908082b08082b,
+    0x191908082b081919,
+    0x191908082b082b08,
+    0x191908082b190819,
+    0x191908082b191908,
+    0x1919081908080819,
+    0x1919081908081908,
+    0x191908190808192b,
+    0x1919081908082b19,
+    0x1919081908190808,
+    0x191908190819082b,
+    0x1919081908191919,
+    0x1919081908192b08,
+    0x19190819082b0819,
+    0x19190819082b1908,
+    0x1919081919080808,
+    0x191908191908082b,
+    0x1919081919081919,
+    0x1919081919082b08,
+    0x1919081919190819,
+    0x1919081919191908,
+    0x19190819192b0808,
+    0x191908192b080819,
+    0x191908192b081908,
+    0x191908192b190808,
+    0x1919082b08080808,
+    0x1919082b08081919,
+    0x1919082b08082b08,
+    0x1919082b08190819,
+    0x1919082b08191908,
+    0x1919082b082b0808,
+    0x1919082b19080819,
+    0x1919082b19081908,
+    0x1919082b19190808,
+    0x1919082b192b2b19,
+    0x1919082b2b080808,
+    0x1919190808080819,
+    0x1919190808081908,
+    0x191919080808192b,
+    0x1919190808082b19,
+    0x1919190808190808,
+    0x191919080819082b,
+    0x1919190808191919,
+    0x1919190808192b08,
+    0x19191908082b0819,
+    0x19191908082b1908,
+    0x1919190819080808,
+    0x191919081908082b,
+    0x1919190819081919,
+    0x1919190819082b08,
+    0x1919190819190819,
+    0x1919190819191908,
+    0x19191908192b0808,
+    0x191919082b080819,
+    0x191919082b081908,
+    0x191919082b190808,
+    0x1919191908080808,
+    0x191919190808082b,
+    0x1919191908081919,
+    0x1919191908082b08,
+    0x1919191908190819,
+    0x1919191908191908,
+    0x19191919082b0808,
+    0x1919191919080819,
+    0x1919191919081908,
+    0x1919191919190808,
+    0x191919192b080808,
+    0x1919192b08080819,
+    0x1919192b08081908,
+    0x1919192b08190808,
+    0x1919192b082b192b,
+    0x1919192b19080808,
+    0x19192b0808080808,
+    0x19192b080808082b,
+    0x19192b0808081919,
+    0x19192b0808082b08,
+    0x19192b0808190819,
+    0x19192b0808191908,
+    0x19192b08082b0808,
+    0x19192b0819080819,
+    0x19192b0819081908,
+    0x19192b0819190808,
+    0x19192b0819192b2b,
+    0x19192b082b080808,
+    0x19192b1908080819,
+    0x19192b1908081908,
+    0x19192b1908190808,
+    0x19192b1919080808,
+    0x19192b2b08080808,
+    0x19192b2b08192b19,
+    0x19192b2b2b081919,
+    0x19192b2b2b2b2b08,
+    0x192b080808080819,
+    0x192b080808081908,
+    0x192b08080808192b,
+    0x192b080808190808,
+    0x192b08080819082b,
+    0x192b080808191919,
+    0x192b080808192b08,
+    0x192b0808082b0819,
+    0x192b0808082b1908,
+    0x192b080819080808,
+    0x192b080819081919,
+    0x192b080819082b08,
+    0x192b080819190819,
+    0x192b080819191908,
+    0x192b0808192b0808,
+    0x192b08082b081908,
+    0x192b08082b190808,
+    0x192b081908080808,
+    0x192b08190808082b,
+    0x192b081908081919,
+    0x192b081908082b08,
+    0x192b081908190819,
+    0x192b081908191908,
+    0x192b0819082b0808,
+    0x192b081919080819,
+    0x192b081919081908,
+    0x192b081919190808,
+    0x192b08192b080808,
+    0x192b08192b192b19,
+    0x192b082b08081908,
+    0x192b082b08190808,
+    0x192b082b19080808,
+    0x192b082b1919192b,
+    0x192b082b2b2b0819,
+    0x192b190808080808,
+    0x192b190808081919,
+    0x192b190808082b08,
+    0x192b190808190819,
+    0x192b190808191908,
+    0x192b1908082b0808,
+    0x192b190819080819,
+    0x192b190819081908,
+    0x192b190819190808,
+    0x192b19082b080808,
+    0x192b191908080819,
+    0x192b191908081908,
+    0x192b191908190808,
+    0x192b191919080808,
+    0x192b191919082b2b,
+    0x192b1919192b2b08,
+    0x192b19192b19082b,
+    0x192b192b08080808,
+    0x192b192b2b191908,
+    0x192b2b0808080819,
+    0x192b2b0808081908,
+    0x192b2b0808190808,
+    0x192b2b08192b1919,
+    0x192b2b082b192b08,
+    0x192b2b1908080808,
+    0x192b2b19082b2b2b,
+    0x192b2b2b1908082b,
+    0x192b2b2b2b2b0819,
+    0x2b08080808080808,
+    0x2b0808080808082b,
+    0x2b08080808081919,
+    0x2b08080808082b08,
+    0x2b08080808190819,
+    0x2b08080808191908,
+    0x2b08080808192b19,
+    0x2b080808082b0808,
+    0x2b080808082b1919,
+    0x2b08080819080819,
+    0x2b08080819081908,
+    0x2b08080819190808,
+    0x2b0808081919082b,
+    0x2b08080819191919,
+    0x2b08080819192b08,
+    0x2b080808192b0819,
+    0x2b0808082b080808,
+    0x2b0808082b081919,
+    0x2b0808082b190819,
+    0x2b0808082b191908,
+    0x2b08081908080819,
+    0x2b08081908081908,
+    0x2b08081908082b19,
+    0x2b08081908190808,
+    0x2b0808190819082b,
+    0x2b08081908191919,
+    0x2b08081908192b08,
+    0x2b080819082b0819,
+    0x2b080819082b1908,
+    0x2b08081919080808,
+    0x2b0808191908082b,
+    0x2b08081919081919,
+    0x2b08081919082b08,
+    0x2b08081919190819,
+    0x2b08081919191908,
+    0x2b0808192b080819,
+    0x2b0808192b081908,
+    0x2b0808192b190808,
+    0x2b0808192b2b2b19,
+    0x2b08082b08080808,
+    0x2b08082b08081919,
+    0x2b08082b08082b2b,
+    0x2b08082b08190819,
+    0x2b08082b08191908,
+    0x2b08082b19080819,
+    0x2b08082b19081908,
+    0x2b08082b19190808,
+    0x2b08190808080819,
+    0x2b08190808081908,
+    0x2b0819080808192b,
+    0x2b08190808082b19,
+    0x2b08190808190808,
+    0x2b0819080819082b,
+    0x2b08190808191919,
+    0x2b08190808192b08,
+    0x2b081908082b0819,
+    0x2b08190819080808,
+    0x2b0819081908082b,
+    0x2b08190819081919,
+    0x2b08190819082b08,
+    0x2b08190819190819,
+    0x2b08190819191908,
+    0x2b081908192b0808,
+    0x2b0819082b080819,
+    0x2b0819082b081908,
+    0x2b0819082b190808,
+    0x2b08191908080808,
+    0x2b0819190808082b,
+    0x2b08191908081919,
+    0x2b08191908082b08,
+    0x2b08191908190819,
+    0x2b08191908191908,
+    0x2b081919082b0808,
+    0x2b08191919080819,
+    0x2b08191919081908,
+    0x2b08191919190808,
+    0x2b0819192b080808,
+    0x2b0819192b082b2b,
+    0x2b08192b08080819,
+    0x2b08192b08081908,
+    0x2b08192b08190808,
+    0x2b08192b082b2b19,
+    0x2b08192b19080808,
+    0x2b082b0808080808,
+    0x2b082b0808081919,
+    0x2b082b0808190819,
+    0x2b082b0808191908,
+    0x2b082b0819080819,
+    0x2b082b0819081908,
+    0x2b082b0819190808,
+    0x2b082b082b2b082b,
+    0x2b082b1908080819,
+    0x2b082b1908081908,
+    0x2b082b1919080808,
+    0x2b082b19192b1919,
+    0x2b082b2b082b082b,
+    0x2b082b2b19192b08,
+    0x2b082b2b19192b2b,
+    0x2b082b2b2b08082b,
+    0x2b082b2b2b2b082b,
+    0x2b19080808080819,
+    0x2b19080808081908,
+    0x2b19080808082b19,
+    0x2b19080808190808,
+    0x2b1908080819082b,
+    0x2b19080808191919,
+    0x2b19080808192b08,
+    0x2b190808082b1908,
+    0x2b19080819080808,
+    0x2b1908081908082b,
+    0x2b19080819081919,
+    0x2b19080819082b08,
+    0x2b19080819190819,
+    0x2b19080819191908,
+    0x2b190808192b0808,
+    0x2b1908082b080819,
+    0x2b1908082b081908,
+    0x2b1908082b190808,
+    0x2b19081908080808,
+    0x2b19081908081919,
+    0x2b19081908190819,
+    0x2b19081908191908,
+    0x2b19081919080819,
+    0x2b19081919081908,
+    0x2b19081919190808,
+    0x2b19081919192b2b,
+    0x2b19082b08080819,
+    0x2b19082b08081908,
+    0x2b19082b08190808,
+    0x2b19082b19080808,
+    0x2b19082b2b2b192b,
+    0x2b19190808080808,
+    0x2b1919080808082b,
+    0x2b19190808081919,
+    0x2b19190808082b08,
+    0x2b19190808190819,
+    0x2b19190808191908,
+    0x2b191908082b0808,
+    0x2b19190819080819,
+    0x2b19190819081908,
+    0x2b19190819190808,
+    0x2b1919082b080808,
+    0x2b1919082b19192b,
+    0x2b19191908080819,
+    0x2b19191908081908,
+    0x2b19191908190808,
+    0x2b19191919080808,
+    0x2b1919192b192b08,
+    0x2b1919192b2b0819,
+    0x2b19192b08080808,
+    0x2b19192b1908192b,
+    0x2b19192b192b1908,
+    0x2b192b0808080819,
+    0x2b192b0808081908,
+    0x2b192b0808190808,
+    0x2b192b08082b192b,
+    0x2b192b0819080808,
+    0x2b192b082b2b2b19,
+    0x2b192b1908080808,
+    0x2b192b1919082b19,
+    0x2b192b191919082b,
+    0x2b192b2b2b190808,
+    0x2b2b080808080808,
+    0x2b2b080808081919,
+    0x2b2b080808082b2b,
+    0x2b2b080808191908,
+    0x2b2b0808082b082b,
+    0x2b2b0808082b2b2b,
+    0x2b2b080819080819,
+    0x2b2b080819081908,
+    0x2b2b080819190808,
+    0x2b2b08082b2b082b,
+    0x2b2b08082b2b2b2b,
+    0x2b2b081919080808,
+    0x2b2b0819192b1919,
+    0x2b2b082b0808082b,
+    0x2b2b082b08082b2b,
+    0x2b2b082b082b082b,
+    0x2b2b082b082b2b08,
+    0x2b2b082b082b2b2b,
+    0x2b2b082b2b08082b,
+    0x2b2b082b2b082b08,
+    0x2b2b082b2b082b2b,
+    0x2b2b082b2b2b2b08,
+    0x2b2b190808080819,
+    0x2b2b190808081908,
+    0x2b2b190808190808,
+    0x2b2b190819080808,
+    0x2b2b19082b082b19,
+    0x2b2b19082b2b1908,
+    0x2b2b191908080808,
+    0x2b2b191908192b19,
+    0x2b2b192b19190819,
+    0x2b2b2b0808082b2b,
+    0x2b2b2b08082b2b08,
+    0x2b2b2b082b2b082b,
+    0x2b2b2b1919191908,
+    0x2b2b2b192b08192b,
+    0x2b2b2b2b08082b08,
+    0x2b2b2b2b08082b2b,
+    0x2b2b2b2b082b0808,
+    0x2b2b2b2b082b082b,
+    0x2b2b2b2b082b2b08,
+    0x2b2b2b2b2b082b08,
+    0x2b2b2b2b2b2b2b2b,
+};
+constexpr uint32_t iq3xxs_grid[256] = {
+    0x04040404,
+    0x04040414,
+    0x04040424,
+    0x04040c0c,
+    0x04040c1c,
+    0x04040c3e,
+    0x04041404,
+    0x04041414,
+    0x04041c0c,
+    0x04042414,
+    0x04043e1c,
+    0x04043e2c,
+    0x040c040c,
+    0x040c041c,
+    0x040c0c04,
+    0x040c0c14,
+    0x040c140c,
+    0x040c142c,
+    0x040c1c04,
+    0x040c1c14,
+    0x040c240c,
+    0x040c2c24,
+    0x040c3e04,
+    0x04140404,
+    0x04140414,
+    0x04140424,
+    0x04140c0c,
+    0x04141404,
+    0x04141414,
+    0x04141c0c,
+    0x04141c1c,
+    0x04141c3e,
+    0x04142c0c,
+    0x04142c3e,
+    0x04143e2c,
+    0x041c040c,
+    0x041c043e,
+    0x041c0c04,
+    0x041c0c14,
+    0x041c142c,
+    0x041c3e04,
+    0x04240c1c,
+    0x04241c3e,
+    0x04242424,
+    0x04242c3e,
+    0x04243e1c,
+    0x04243e2c,
+    0x042c040c,
+    0x042c043e,
+    0x042c1c14,
+    0x042c2c14,
+    0x04341c2c,
+    0x04343424,
+    0x043e0c04,
+    0x043e0c24,
+    0x043e0c34,
+    0x043e241c,
+    0x043e340c,
+    0x0c04040c,
+    0x0c04041c,
+    0x0c040c04,
+    0x0c040c14,
+    0x0c04140c,
+    0x0c04141c,
+    0x0c041c04,
+    0x0c041c14,
+    0x0c041c24,
+    0x0c04243e,
+    0x0c042c04,
+    0x0c0c0404,
+    0x0c0c0414,
+    0x0c0c0c0c,
+    0x0c0c1404,
+    0x0c0c1414,
+    0x0c14040c,
+    0x0c14041c,
+    0x0c140c04,
+    0x0c140c14,
+    0x0c14140c,
+    0x0c141c04,
+    0x0c143e14,
+    0x0c1c0404,
+    0x0c1c0414,
+    0x0c1c1404,
+    0x0c1c1c0c,
+    0x0c1c2434,
+    0x0c1c3434,
+    0x0c24040c,
+    0x0c24042c,
+    0x0c242c04,
+    0x0c2c1404,
+    0x0c2c1424,
+    0x0c2c2434,
+    0x0c2c3e0c,
+    0x0c34042c,
+    0x0c3e1414,
+    0x0c3e2404,
+    0x14040404,
+    0x14040414,
+    0x14040c0c,
+    0x14040c1c,
+    0x14041404,
+    0x14041414,
+    0x14041434,
+    0x14041c0c,
+    0x14042414,
+    0x140c040c,
+    0x140c041c,
+    0x140c042c,
+    0x140c0c04,
+    0x140c0c14,
+    0x140c140c,
+    0x140c1c04,
+    0x140c341c,
+    0x140c343e,
+    0x140c3e04,
+    0x14140404,
+    0x14140414,
+    0x14140c0c,
+    0x14140c3e,
+    0x14141404,
+    0x14141414,
+    0x14141c3e,
+    0x14142404,
+    0x14142c2c,
+    0x141c040c,
+    0x141c0c04,
+    0x141c0c24,
+    0x141c3e04,
+    0x141c3e24,
+    0x14241c2c,
+    0x14242c1c,
+    0x142c041c,
+    0x142c143e,
+    0x142c240c,
+    0x142c3e24,
+    0x143e040c,
+    0x143e041c,
+    0x143e0c34,
+    0x143e242c,
+    0x1c04040c,
+    0x1c040c04,
+    0x1c040c14,
+    0x1c04140c,
+    0x1c04141c,
+    0x1c042c04,
+    0x1c04342c,
+    0x1c043e14,
+    0x1c0c0404,
+    0x1c0c0414,
+    0x1c0c1404,
+    0x1c0c1c0c,
+    0x1c0c2424,
+    0x1c0c2434,
+    0x1c14040c,
+    0x1c14041c,
+    0x1c140c04,
+    0x1c14142c,
+    0x1c142c14,
+    0x1c143e14,
+    0x1c1c0c0c,
+    0x1c1c1c1c,
+    0x1c241c04,
+    0x1c24243e,
+    0x1c243e14,
+    0x1c2c0404,
+    0x1c2c0434,
+    0x1c2c1414,
+    0x1c2c2c2c,
+    0x1c340c24,
+    0x1c341c34,
+    0x1c34341c,
+    0x1c3e1c1c,
+    0x1c3e3404,
+    0x24040424,
+    0x24040c3e,
+    0x24041c2c,
+    0x24041c3e,
+    0x24042c1c,
+    0x24042c3e,
+    0x240c3e24,
+    0x24141404,
+    0x24141c3e,
+    0x24142404,
+    0x24143404,
+    0x24143434,
+    0x241c043e,
+    0x241c242c,
+    0x24240424,
+    0x24242c0c,
+    0x24243424,
+    0x242c142c,
+    0x242c241c,
+    0x242c3e04,
+    0x243e042c,
+    0x243e0c04,
+    0x243e0c14,
+    0x243e1c04,
+    0x2c040c14,
+    0x2c04240c,
+    0x2c043e04,
+    0x2c0c0404,
+    0x2c0c0434,
+    0x2c0c1434,
+    0x2c0c2c2c,
+    0x2c140c24,
+    0x2c141c14,
+    0x2c143e14,
+    0x2c1c0414,
+    0x2c1c2c1c,
+    0x2c240c04,
+    0x2c24141c,
+    0x2c24143e,
+    0x2c243e14,
+    0x2c2c0414,
+    0x2c2c1c0c,
+    0x2c342c04,
+    0x2c3e1424,
+    0x2c3e2414,
+    0x34041424,
+    0x34042424,
+    0x34042434,
+    0x34043424,
+    0x340c140c,
+    0x340c340c,
+    0x34140c3e,
+    0x34143424,
+    0x341c1c04,
+    0x341c1c34,
+    0x34242424,
+    0x342c042c,
+    0x342c2c14,
+    0x34341c1c,
+    0x343e041c,
+    0x343e140c,
+    0x3e04041c,
+    0x3e04042c,
+    0x3e04043e,
+    0x3e040c04,
+    0x3e041c14,
+    0x3e042c14,
+    0x3e0c1434,
+    0x3e0c2404,
+    0x3e140c14,
+    0x3e14242c,
+    0x3e142c14,
+    0x3e1c0404,
+    0x3e1c0c2c,
+    0x3e1c1c1c,
+    0x3e1c3404,
+    0x3e24140c,
+    0x3e24240c,
+    0x3e2c0404,
+    0x3e2c0414,
+    0x3e2c1424,
+    0x3e341c04,
+};
+constexpr uint32_t iq3s_grid[512] = {
+    0x01010101,
+    0x01010103,
+    0x01010105,
+    0x0101010b,
+    0x0101010f,
+    0x01010301,
+    0x01010303,
+    0x01010305,
+    0x01010309,
+    0x0101030d,
+    0x01010501,
+    0x01010503,
+    0x0101050b,
+    0x01010707,
+    0x01010901,
+    0x01010905,
+    0x0101090b,
+    0x0101090f,
+    0x01010b03,
+    0x01010b07,
+    0x01010d01,
+    0x01010d05,
+    0x01010f03,
+    0x01010f09,
+    0x01010f0f,
+    0x01030101,
+    0x01030103,
+    0x01030105,
+    0x01030109,
+    0x01030301,
+    0x01030303,
+    0x0103030b,
+    0x01030501,
+    0x01030507,
+    0x0103050f,
+    0x01030703,
+    0x0103070b,
+    0x01030909,
+    0x01030d03,
+    0x01030d0b,
+    0x01030f05,
+    0x01050101,
+    0x01050103,
+    0x0105010b,
+    0x0105010f,
+    0x01050301,
+    0x01050307,
+    0x0105030d,
+    0x01050503,
+    0x0105050b,
+    0x01050701,
+    0x01050709,
+    0x01050905,
+    0x0105090b,
+    0x0105090f,
+    0x01050b03,
+    0x01050b07,
+    0x01050f01,
+    0x01050f07,
+    0x01070107,
+    0x01070303,
+    0x0107030b,
+    0x01070501,
+    0x01070505,
+    0x01070703,
+    0x01070707,
+    0x0107070d,
+    0x01070909,
+    0x01070b01,
+    0x01070b05,
+    0x01070d0f,
+    0x01070f03,
+    0x01070f0b,
+    0x01090101,
+    0x01090307,
+    0x0109030f,
+    0x01090503,
+    0x01090509,
+    0x01090705,
+    0x01090901,
+    0x01090907,
+    0x01090b03,
+    0x01090f01,
+    0x010b0105,
+    0x010b0109,
+    0x010b0501,
+    0x010b0505,
+    0x010b050d,
+    0x010b0707,
+    0x010b0903,
+    0x010b090b,
+    0x010b090f,
+    0x010b0d0d,
+    0x010b0f07,
+    0x010d010d,
+    0x010d0303,
+    0x010d0307,
+    0x010d0703,
+    0x010d0b05,
+    0x010d0f03,
+    0x010f0101,
+    0x010f0105,
+    0x010f0109,
+    0x010f0501,
+    0x010f0505,
+    0x010f050d,
+    0x010f0707,
+    0x010f0b01,
+    0x010f0b09,
+    0x03010101,
+    0x03010103,
+    0x03010105,
+    0x03010109,
+    0x03010301,
+    0x03010303,
+    0x03010307,
+    0x0301030b,
+    0x0301030f,
+    0x03010501,
+    0x03010505,
+    0x03010703,
+    0x03010709,
+    0x0301070d,
+    0x03010b09,
+    0x03010b0d,
+    0x03010d03,
+    0x03010f05,
+    0x03030101,
+    0x03030103,
+    0x03030107,
+    0x0303010d,
+    0x03030301,
+    0x03030309,
+    0x03030503,
+    0x03030701,
+    0x03030707,
+    0x03030903,
+    0x03030b01,
+    0x03030b05,
+    0x03030f01,
+    0x03030f0d,
+    0x03050101,
+    0x03050305,
+    0x0305030b,
+    0x0305030f,
+    0x03050501,
+    0x03050509,
+    0x03050705,
+    0x03050901,
+    0x03050907,
+    0x03050b0b,
+    0x03050d01,
+    0x03050f05,
+    0x03070103,
+    0x03070109,
+    0x0307010f,
+    0x03070301,
+    0x03070307,
+    0x03070503,
+    0x0307050f,
+    0x03070701,
+    0x03070709,
+    0x03070903,
+    0x03070d05,
+    0x03070f01,
+    0x03090107,
+    0x0309010b,
+    0x03090305,
+    0x03090309,
+    0x03090703,
+    0x03090707,
+    0x03090905,
+    0x0309090d,
+    0x03090b01,
+    0x03090b09,
+    0x030b0103,
+    0x030b0301,
+    0x030b0307,
+    0x030b0503,
+    0x030b0701,
+    0x030b0705,
+    0x030b0b03,
+    0x030d0501,
+    0x030d0509,
+    0x030d050f,
+    0x030d0909,
+    0x030d090d,
+    0x030f0103,
+    0x030f0107,
+    0x030f0301,
+    0x030f0305,
+    0x030f0503,
+    0x030f070b,
+    0x030f0903,
+    0x030f0d05,
+    0x030f0f01,
+    0x05010101,
+    0x05010103,
+    0x05010107,
+    0x0501010b,
+    0x0501010f,
+    0x05010301,
+    0x05010305,
+    0x05010309,
+    0x0501030d,
+    0x05010503,
+    0x05010507,
+    0x0501050f,
+    0x05010701,
+    0x05010705,
+    0x05010903,
+    0x05010907,
+    0x0501090b,
+    0x05010b01,
+    0x05010b05,
+    0x05010d0f,
+    0x05010f01,
+    0x05010f07,
+    0x05010f0b,
+    0x05030101,
+    0x05030105,
+    0x05030301,
+    0x05030307,
+    0x0503030f,
+    0x05030505,
+    0x0503050b,
+    0x05030703,
+    0x05030709,
+    0x05030905,
+    0x05030b03,
+    0x05050103,
+    0x05050109,
+    0x0505010f,
+    0x05050503,
+    0x05050507,
+    0x05050701,
+    0x0505070f,
+    0x05050903,
+    0x05050b07,
+    0x05050b0f,
+    0x05050f03,
+    0x05050f09,
+    0x05070101,
+    0x05070105,
+    0x0507010b,
+    0x05070303,
+    0x05070505,
+    0x05070509,
+    0x05070703,
+    0x05070707,
+    0x05070905,
+    0x05070b01,
+    0x05070d0d,
+    0x05090103,
+    0x0509010f,
+    0x05090501,
+    0x05090507,
+    0x05090705,
+    0x0509070b,
+    0x05090903,
+    0x05090f05,
+    0x05090f0b,
+    0x050b0109,
+    0x050b0303,
+    0x050b0505,
+    0x050b070f,
+    0x050b0901,
+    0x050b0b07,
+    0x050b0f01,
+    0x050d0101,
+    0x050d0105,
+    0x050d010f,
+    0x050d0503,
+    0x050d0b0b,
+    0x050d0d03,
+    0x050f010b,
+    0x050f0303,
+    0x050f050d,
+    0x050f0701,
+    0x050f0907,
+    0x050f0b01,
+    0x07010105,
+    0x07010303,
+    0x07010307,
+    0x0701030b,
+    0x0701030f,
+    0x07010505,
+    0x07010703,
+    0x07010707,
+    0x0701070b,
+    0x07010905,
+    0x07010909,
+    0x0701090f,
+    0x07010b03,
+    0x07010d07,
+    0x07010f03,
+    0x07030103,
+    0x07030107,
+    0x0703010b,
+    0x07030309,
+    0x07030503,
+    0x07030507,
+    0x07030901,
+    0x07030d01,
+    0x07030f05,
+    0x07030f0d,
+    0x07050101,
+    0x07050305,
+    0x07050501,
+    0x07050705,
+    0x07050709,
+    0x07050b01,
+    0x07070103,
+    0x07070301,
+    0x07070309,
+    0x07070503,
+    0x07070507,
+    0x0707050f,
+    0x07070701,
+    0x07070903,
+    0x07070907,
+    0x0707090f,
+    0x07070b0b,
+    0x07070f07,
+    0x07090107,
+    0x07090303,
+    0x0709030d,
+    0x07090505,
+    0x07090703,
+    0x07090b05,
+    0x07090d01,
+    0x07090d09,
+    0x070b0103,
+    0x070b0301,
+    0x070b0305,
+    0x070b050b,
+    0x070b0705,
+    0x070b0909,
+    0x070b0b0d,
+    0x070b0f07,
+    0x070d030d,
+    0x070d0903,
+    0x070f0103,
+    0x070f0107,
+    0x070f0501,
+    0x070f0505,
+    0x070f070b,
+    0x09010101,
+    0x09010109,
+    0x09010305,
+    0x09010501,
+    0x09010509,
+    0x0901050f,
+    0x09010705,
+    0x09010903,
+    0x09010b01,
+    0x09010f01,
+    0x09030105,
+    0x0903010f,
+    0x09030303,
+    0x09030307,
+    0x09030505,
+    0x09030701,
+    0x0903070b,
+    0x09030907,
+    0x09030b03,
+    0x09030b0b,
+    0x09050103,
+    0x09050107,
+    0x09050301,
+    0x0905030b,
+    0x09050503,
+    0x09050707,
+    0x09050901,
+    0x09050b0f,
+    0x09050d05,
+    0x09050f01,
+    0x09070109,
+    0x09070303,
+    0x09070307,
+    0x09070501,
+    0x09070505,
+    0x09070703,
+    0x0907070b,
+    0x09090101,
+    0x09090105,
+    0x09090509,
+    0x0909070f,
+    0x09090901,
+    0x09090f03,
+    0x090b010b,
+    0x090b010f,
+    0x090b0503,
+    0x090b0d05,
+    0x090d0307,
+    0x090d0709,
+    0x090d0d01,
+    0x090f0301,
+    0x090f030b,
+    0x090f0701,
+    0x090f0907,
+    0x090f0b03,
+    0x0b010105,
+    0x0b010301,
+    0x0b010309,
+    0x0b010505,
+    0x0b010901,
+    0x0b010909,
+    0x0b01090f,
+    0x0b010b05,
+    0x0b010d0d,
+    0x0b010f09,
+    0x0b030103,
+    0x0b030107,
+    0x0b03010b,
+    0x0b030305,
+    0x0b030503,
+    0x0b030705,
+    0x0b030f05,
+    0x0b050101,
+    0x0b050303,
+    0x0b050507,
+    0x0b050701,
+    0x0b05070d,
+    0x0b050b07,
+    0x0b070105,
+    0x0b07010f,
+    0x0b070301,
+    0x0b07050f,
+    0x0b070909,
+    0x0b070b03,
+    0x0b070d0b,
+    0x0b070f07,
+    0x0b090103,
+    0x0b090109,
+    0x0b090501,
+    0x0b090705,
+    0x0b09090d,
+    0x0b0b0305,
+    0x0b0b050d,
+    0x0b0b0b03,
+    0x0b0b0b07,
+    0x0b0d0905,
+    0x0b0f0105,
+    0x0b0f0109,
+    0x0b0f0505,
+    0x0d010303,
+    0x0d010307,
+    0x0d01030b,
+    0x0d010703,
+    0x0d010707,
+    0x0d010d01,
+    0x0d030101,
+    0x0d030501,
+    0x0d03050f,
+    0x0d030d09,
+    0x0d050305,
+    0x0d050709,
+    0x0d050905,
+    0x0d050b0b,
+    0x0d050d05,
+    0x0d050f01,
+    0x0d070101,
+    0x0d070309,
+    0x0d070503,
+    0x0d070901,
+    0x0d09050b,
+    0x0d090907,
+    0x0d090d05,
+    0x0d0b0101,
+    0x0d0b0107,
+    0x0d0b0709,
+    0x0d0b0d01,
+    0x0d0d010b,
+    0x0d0d0901,
+    0x0d0f0303,
+    0x0d0f0307,
+    0x0f010101,
+    0x0f010109,
+    0x0f01010f,
+    0x0f010501,
+    0x0f010505,
+    0x0f01070d,
+    0x0f010901,
+    0x0f010b09,
+    0x0f010d05,
+    0x0f030105,
+    0x0f030303,
+    0x0f030509,
+    0x0f030907,
+    0x0f03090b,
+    0x0f050103,
+    0x0f050109,
+    0x0f050301,
+    0x0f05030d,
+    0x0f050503,
+    0x0f050701,
+    0x0f050b03,
+    0x0f070105,
+    0x0f070705,
+    0x0f07070b,
+    0x0f070b07,
+    0x0f090103,
+    0x0f09010b,
+    0x0f090307,
+    0x0f090501,
+    0x0f090b01,
+    0x0f0b0505,
+    0x0f0b0905,
+    0x0f0d0105,
+    0x0f0d0703,
+    0x0f0f0101,
+};
+constexpr uint64_t iq1s_grid[2048] = {
+    0xffffffffffffffff,
+    0xffffffffffffff01,
+    0xffffffffffff0000,
+    0xffffffffffff01ff,
+    0xffffffffffff0101,
+    0xffffffffff00ff00,
+    0xffffffffff000000,
+    0xffffffffff01ffff,
+    0xffffffffff01ff01,
+    0xffffffffff0101ff,
+    0xffffffffff010101,
+    0xffffffff00ff0000,
+    0xffffffff0000ff00,
+    0xffffffff000000ff,
+    0xffffffff00000001,
+    0xffffffff00010000,
+    0xffffffff01ffffff,
+    0xffffffff01ffff01,
+    0xffffffff01ff01ff,
+    0xffffffff01ff0101,
+    0xffffffff01000000,
+    0xffffffff0101ffff,
+    0xffffffff0101ff01,
+    0xffffffff010101ff,
+    0xffffffff01010101,
+    0xffffff00ffff00ff,
+    0xffffff00ffff0000,
+    0xffffff00ff00ff00,
+    0xffffff00ff0000ff,
+    0xffffff00ff000001,
+    0xffffff00ff000100,
+    0xffffff00ff000101,
+    0xffffff00ff010000,
+    0xffffff0000ffff00,
+    0xffffff0000ff0001,
+    0xffffff0000ff0100,
+    0xffffff000000ff01,
+    0xffffff0000000000,
+    0xffffff0000000101,
+    0xffffff000001ff00,
+    0xffffff00000100ff,
+    0xffffff0000010001,
+    0xffffff00000101ff,
+    0xffffff0001ff0000,
+    0xffffff000100ff00,
+    0xffffff00010000ff,
+    0xffffff0001000001,
+    0xffffff0001010000,
+    0xffffff01ffffffff,
+    0xffffff01ffffff01,
+    0xffffff01ffff01ff,
+    0xffffff01ffff0101,
+    0xffffff01ff000000,
+    0xffffff01ff01ffff,
+    0xffffff01ff01ff01,
+    0xffffff01ff0101ff,
+    0xffffff01ff010101,
+    0xffffff0100ff0000,
+    0xffffff010000ff00,
+    0xffffff0100000100,
+    0xffffff01000100ff,
+    0xffffff0100010100,
+    0xffffff0101ffffff,
+    0xffffff0101ffff01,
+    0xffffff0101ff01ff,
+    0xffffff0101ff0101,
+    0xffffff010100ff00,
+    0xffffff0101000000,
+    0xffffff0101000100,
+    0xffffff010101ffff,
+    0xffffff010101ff01,
+    0xffffff01010101ff,
+    0xffffff0101010101,
+    0xffff00ffff00ff00,
+    0xffff00ffff0000ff,
+    0xffff00ffff000001,
+    0xffff00ffff010000,
+    0xffff00ff00ffff00,
+    0xffff00ff00ff0100,
+    0xffff00ff00000000,
+    0xffff00ff00000101,
+    0xffff00ff000100ff,
+    0xffff00ff00010000,
+    0xffff00ff0100ff00,
+    0xffff00ff01000100,
+    0xffff00ff01010000,
+    0xffff0000ffffff00,
+    0xffff0000ffff00ff,
+    0xffff0000ffff0000,
+    0xffff0000ffff0001,
+    0xffff0000ff000000,
+    0xffff0000ff0001ff,
+    0xffff0000ff000101,
+    0xffff0000ff010100,
+    0xffff000000ffffff,
+    0xffff000000ff0000,
+    0xffff000000ff0101,
+    0xffff00000000ffff,
+    0xffff00000000ff00,
+    0xffff0000000000ff,
+    0xffff000000000000,
+    0xffff000000000001,
+    0xffff000000000100,
+    0xffff00000001ffff,
+    0xffff00000001ff01,
+    0xffff000000010000,
+    0xffff0000000101ff,
+    0xffff000000010101,
+    0xffff000001ffff00,
+    0xffff00000100ff00,
+    0xffff000001000000,
+    0xffff0000010001ff,
+    0xffff000001000101,
+    0xffff00000101ff00,
+    0xffff0000010100ff,
+    0xffff000001010000,
+    0xffff000001010001,
+    0xffff000001010100,
+    0xffff0001ff0000ff,
+    0xffff0001ff000100,
+    0xffff000100ffff00,
+    0xffff000100ff00ff,
+    0xffff00010000ffff,
+    0xffff00010000ff01,
+    0xffff000100000000,
+    0xffff0001000001ff,
+    0xffff00010001ffff,
+    0xffff00010001ff00,
+    0xffff000100010001,
+    0xffff000100010100,
+    0xffff000101ff0000,
+    0xffff00010100ff00,
+    0xffff0001010000ff,
+    0xffff000101000100,
+    0xffff01ffffffffff,
+    0xffff01ffffffff01,
+    0xffff01ffffff01ff,
+    0xffff01ffffff0101,
+    0xffff01ffff000000,
+    0xffff01ffff01ffff,
+    0xffff01ffff01ff01,
+    0xffff01ffff0101ff,
+    0xffff01ffff010101,
+    0xffff01ff00ff0000,
+    0xffff01ff0000ff00,
+    0xffff01ff00000001,
+    0xffff01ff00010000,
+    0xffff01ff01ffffff,
+    0xffff01ff01ffff01,
+    0xffff01ff01ff01ff,
+    0xffff01ff01ff0101,
+    0xffff01ff01000000,
+    0xffff01ff0101ffff,
+    0xffff01ff0101ff01,
+    0xffff01ff010101ff,
+    0xffff01ff01010101,
+    0xffff0100ffff0000,
+    0xffff0100ff00ff00,
+    0xffff0100ff0000ff,
+    0xffff0100ff000100,
+    0xffff0100ff0100ff,
+    0xffff0100ff010000,
+    0xffff010000ffff00,
+    0xffff01000000ffff,
+    0xffff01000000ff00,
+    0xffff010000000000,
+    0xffff01000001ff00,
+    0xffff0100000100ff,
+    0xffff010000010100,
+    0xffff01000100ff00,
+    0xffff0100010000ff,
+    0xffff010001000001,
+    0xffff010001000100,
+    0xffff010001010000,
+    0xffff0101ffffffff,
+    0xffff0101ffffff01,
+    0xffff0101ffff01ff,
+    0xffff0101ffff0101,
+    0xffff0101ff000000,
+    0xffff0101ff01ffff,
+    0xffff0101ff01ff01,
+    0xffff0101ff0101ff,
+    0xffff0101ff010101,
+    0xffff010100ff0000,
+    0xffff01010000ff00,
+    0xffff010100000100,
+    0xffff01010001ff00,
+    0xffff010100010000,
+    0xffff010101ffffff,
+    0xffff010101ffff01,
+    0xffff010101ff0000,
+    0xffff010101ff01ff,
+    0xffff010101ff0101,
+    0xffff010101000000,
+    0xffff01010101ffff,
+    0xffff01010101ff01,
+    0xffff0101010101ff,
+    0xffff010101010101,
+    0xff00ffffff00ffff,
+    0xff00ffffff00ff00,
+    0xff00ffffff0000ff,
+    0xff00ffffff000100,
+    0xff00ffffff0100ff,
+    0xff00ffffff010000,
+    0xff00ffff00ffff00,
+    0xff00ffff00ff00ff,
+    0xff00ffff0000ffff,
+    0xff00ffff00000000,
+    0xff00ffff000001ff,
+    0xff00ffff0001ff00,
+    0xff00ffff000100ff,
+    0xff00ffff00010000,
+    0xff00ffff00010100,
+    0xff00ffff0100ff00,
+    0xff00ffff010000ff,
+    0xff00ffff01000001,
+    0xff00ffff0101ff00,
+    0xff00ffff01010000,
+    0xff00ff00ffffff00,
+    0xff00ff00ffff00ff,
+    0xff00ff00ffff0001,
+    0xff00ff00ffff0100,
+    0xff00ff00ff00ffff,
+    0xff00ff00ff00ff01,
+    0xff00ff00ff000000,
+    0xff00ff00ff0001ff,
+    0xff00ff00ff01ff00,
+    0xff00ff00ff0100ff,
+    0xff00ff00ff010100,
+    0xff00ff0000ff0000,
+    0xff00ff0000ff0101,
+    0xff00ff000000ffff,
+    0xff00ff000000ff00,
+    0xff00ff000000ff01,
+    0xff00ff00000000ff,
+    0xff00ff0000000000,
+    0xff00ff0000000001,
+    0xff00ff0000000100,
+    0xff00ff000001ffff,
+    0xff00ff0000010000,
+    0xff00ff0001ff00ff,
+    0xff00ff000100ff01,
+    0xff00ff0001000000,
+    0xff00ff000101ff00,
+    0xff00ff00010100ff,
+    0xff00ff01ff00ff00,
+    0xff00ff01ff0000ff,
+    0xff00ff01ff000001,
+    0xff00ff01ff010000,
+    0xff00ff0100ffffff,
+    0xff00ff0100ff0001,
+    0xff00ff0100ff0100,
+    0xff00ff010000ff01,
+    0xff00ff0100000000,
+    0xff00ff01000001ff,
+    0xff00ff0100000101,
+    0xff00ff01000100ff,
+    0xff00ff0100010001,
+    0xff00ff0101ff0000,
+    0xff00ff010100ff00,
+    0xff00ff01010000ff,
+    0xff00ff0101000001,
+    0xff00ff0101010000,
+    0xff0000ffffffff00,
+    0xff0000ffffff0001,
+    0xff0000ffffff0100,
+    0xff0000ffff0000ff,
+    0xff0000ffff000000,
+    0xff0000ffff0001ff,
+    0xff0000ffff000100,
+    0xff0000ffff01ff00,
+    0xff0000ffff010001,
+    0xff0000ff00ffff00,
+    0xff0000ff00ff0000,
+    0xff0000ff00ff0001,
+    0xff0000ff00ff01ff,
+    0xff0000ff00ff0101,
+    0xff0000ff0000ff00,
+    0xff0000ff000000ff,
+    0xff0000ff00000000,
+    0xff0000ff00000001,
+    0xff0000ff00000100,
+    0xff0000ff0001ff01,
+    0xff0000ff00010000,
+    0xff0000ff000101ff,
+    0xff0000ff01ff00ff,
+    0xff0000ff01ff0100,
+    0xff0000ff0100ffff,
+    0xff0000ff010000ff,
+    0xff0000ff01000000,
+    0xff0000ff010001ff,
+    0xff0000ff01000100,
+    0xff0000ff01000101,
+    0xff0000ff0101ff00,
+    0xff0000ff010100ff,
+    0xff0000ff01010000,
+    0xff0000ff01010100,
+    0xff000000ffffff01,
+    0xff000000ffff0000,
+    0xff000000ffff0101,
+    0xff000000ff00ff00,
+    0xff000000ff0000ff,
+    0xff000000ff000000,
+    0xff000000ff000001,
+    0xff000000ff000100,
+    0xff000000ff01ffff,
+    0xff000000ff01ff01,
+    0xff000000ff010000,
+    0xff000000ff0101ff,
+    0xff000000ff010101,
+    0xff00000000ffff00,
+    0xff00000000ff00ff,
+    0xff00000000ff0000,
+    0xff00000000ff0001,
+    0xff0000000000ff00,
+    0xff0000000000ff01,
+    0xff000000000000ff,
+    0xff00000000000000,
+    0xff00000000000001,
+    0xff00000000000100,
+    0xff00000000000101,
+    0xff0000000001ff00,
+    0xff000000000100ff,
+    0xff00000000010000,
+    0xff00000000010001,
+    0xff00000000010100,
+    0xff00000001ffffff,
+    0xff00000001ffff01,
+    0xff00000001ff00ff,
+    0xff00000001ff0000,
+    0xff00000001ff01ff,
+    0xff00000001ff0101,
+    0xff0000000100ffff,
+    0xff0000000100ff00,
+    0xff000000010000ff,
+    0xff00000001000000,
+    0xff00000001000001,
+    0xff00000001000100,
+    0xff00000001000101,
+    0xff0000000101ffff,
+    0xff0000000101ff01,
+    0xff00000001010000,
+    0xff000001ffffff00,
+    0xff000001ffff00ff,
+    0xff000001ffff0000,
+    0xff000001ffff0001,
+    0xff000001ff000000,
+    0xff000001ff000001,
+    0xff000001ff0001ff,
+    0xff000001ff000101,
+    0xff000001ff01ff00,
+    0xff000001ff010001,
+    0xff00000100ffffff,
+    0xff00000100ffff01,
+    0xff00000100ff00ff,
+    0xff00000100ff0000,
+    0xff00000100ff01ff,
+    0xff00000100ff0101,
+    0xff0000010000ff00,
+    0xff00000100000000,
+    0xff00000100000001,
+    0xff000001000001ff,
+    0xff00000100000100,
+    0xff0000010001ff00,
+    0xff000001000100ff,
+    0xff00000100010000,
+    0xff000001000101ff,
+    0xff00000100010100,
+    0xff00000100010101,
+    0xff00000101ff0001,
+    0xff00000101ff0101,
+    0xff0000010100ff01,
+    0xff00000101000000,
+    0xff000001010100ff,
+    0xff00000101010100,
+    0xff0001ffff00ff00,
+    0xff0001ffff000001,
+    0xff0001ffff010000,
+    0xff0001ff00ffff00,
+    0xff0001ff00ff00ff,
+    0xff0001ff00ff0001,
+    0xff0001ff00ff0100,
+    0xff0001ff0000ffff,
+    0xff0001ff00000000,
+    0xff0001ff000001ff,
+    0xff0001ff00000101,
+    0xff0001ff0001ffff,
+    0xff0001ff0001ff00,
+    0xff0001ff000100ff,
+    0xff0001ff00010001,
+    0xff0001ff00010100,
+    0xff0001ff01ff0000,
+    0xff0001ff0100ff00,
+    0xff0001ff010000ff,
+    0xff0001ff01010000,
+    0xff000100ff00ffff,
+    0xff000100ff00ff01,
+    0xff000100ff000000,
+    0xff000100ff000101,
+    0xff000100ff01ff00,
+    0xff000100ff010000,
+    0xff00010000ffff01,
+    0xff00010000ff00ff,
+    0xff00010000ff0000,
+    0xff00010000ff01ff,
+    0xff0001000000ff00,
+    0xff000100000000ff,
+    0xff00010000000000,
+    0xff00010000000001,
+    0xff00010000000100,
+    0xff00010000000101,
+    0xff0001000001ffff,
+    0xff00010000010000,
+    0xff00010000010101,
+    0xff00010001ff0100,
+    0xff0001000100ff00,
+    0xff0001000100ff01,
+    0xff00010001000000,
+    0xff000100010001ff,
+    0xff0001000101ff00,
+    0xff00010001010001,
+    0xff00010001010100,
+    0xff000101ffff0100,
+    0xff000101ff000001,
+    0xff000101ff0100ff,
+    0xff000101ff010001,
+    0xff00010100ff00ff,
+    0xff00010100ff0001,
+    0xff00010100ff0100,
+    0xff0001010000ffff,
+    0xff0001010000ff01,
+    0xff00010100000000,
+    0xff000101000001ff,
+    0xff0001010001ff00,
+    0xff00010100010001,
+    0xff00010100010100,
+    0xff00010101ff0000,
+    0xff0001010100ff00,
+    0xff00010101000001,
+    0xff00010101000101,
+    0xff01ffffffffffff,
+    0xff01ffffffffff01,
+    0xff01ffffffff01ff,
+    0xff01ffffffff0101,
+    0xff01ffffff000000,
+    0xff01ffffff01ffff,
+    0xff01ffffff01ff01,
+    0xff01ffffff010000,
+    0xff01ffffff0101ff,
+    0xff01ffffff010101,
+    0xff01ffff00ff0000,
+    0xff01ffff0000ff00,
+    0xff01ffff00000100,
+    0xff01ffff0001ff00,
+    0xff01ffff00010000,
+    0xff01ffff01ffffff,
+    0xff01ffff01ffff01,
+    0xff01ffff01ff01ff,
+    0xff01ffff01ff0101,
+    0xff01ffff01000000,
+    0xff01ffff0101ffff,
+    0xff01ffff0101ff01,
+    0xff01ffff01010000,
+    0xff01ffff010101ff,
+    0xff01ffff01010101,
+    0xff01ff00ffff0000,
+    0xff01ff00ff00ff00,
+    0xff01ff00ff0000ff,
+    0xff01ff00ff000100,
+    0xff01ff00ff010000,
+    0xff01ff0000ffff01,
+    0xff01ff0000ff00ff,
+    0xff01ff0000ff0100,
+    0xff01ff0000000000,
+    0xff01ff00000001ff,
+    0xff01ff0000000101,
+    0xff01ff000001ff00,
+    0xff01ff00000100ff,
+    0xff01ff0000010000,
+    0xff01ff0000010001,
+    0xff01ff0001ff0000,
+    0xff01ff000100ffff,
+    0xff01ff0001000001,
+    0xff01ff0001000100,
+    0xff01ff0001010000,
+    0xff01ff01ffffff00,
+    0xff01ff01ffff01ff,
+    0xff01ff01ffff0101,
+    0xff01ff01ff00ff00,
+    0xff01ff01ff000000,
+    0xff01ff01ff01ffff,
+    0xff01ff01ff01ff01,
+    0xff01ff01ff0101ff,
+    0xff01ff01ff010101,
+    0xff01ff0100ff0000,
+    0xff01ff010000ff00,
+    0xff01ff0100000001,
+    0xff01ff0100000100,
+    0xff01ff0100010000,
+    0xff01ff0101ffff00,
+    0xff01ff0101ff01ff,
+    0xff01ff0101ff0101,
+    0xff01ff010100ff00,
+    0xff01ff0101000000,
+    0xff01ff010101ffff,
+    0xff01ff010101ff01,
+    0xff01ff01010101ff,
+    0xff01ff0101010101,
+    0xff0100ffffff0000,
+    0xff0100ffff0000ff,
+    0xff0100ffff000001,
+    0xff0100ffff000100,
+    0xff0100ffff010000,
+    0xff0100ff00ff00ff,
+    0xff0100ff00ff0000,
+    0xff0100ff00ff0001,
+    0xff0100ff00ff0100,
+    0xff0100ff0000ff01,
+    0xff0100ff00000000,
+    0xff0100ff000001ff,
+    0xff0100ff00000101,
+    0xff0100ff00010001,
+    0xff0100ff01ff0000,
+    0xff0100ff0100ff00,
+    0xff0100ff010000ff,
+    0xff0100ff01000100,
+    0xff0100ff0101ff00,
+    0xff0100ff01010000,
+    0xff010000ffff0100,
+    0xff010000ff000000,
+    0xff010000ff01ff00,
+    0xff010000ff010100,
+    0xff01000000ffffff,
+    0xff01000000ff0000,
+    0xff01000000ff01ff,
+    0xff0100000000ff00,
+    0xff010000000000ff,
+    0xff01000000000000,
+    0xff01000000000100,
+    0xff0100000001ff01,
+    0xff01000000010000,
+    0xff010000000101ff,
+    0xff01000001ff0100,
+    0xff0100000100ffff,
+    0xff010000010000ff,
+    0xff01000001000000,
+    0xff010000010001ff,
+    0xff01000001000101,
+    0xff0100000101ff00,
+    0xff010000010100ff,
+    0xff01000001010001,
+    0xff01000001010100,
+    0xff010001ffff0000,
+    0xff010001ff00ffff,
+    0xff010001ff00ff01,
+    0xff010001ff000100,
+    0xff010001ff010000,
+    0xff01000100ffff00,
+    0xff01000100ff0100,
+    0xff01000100000000,
+    0xff0100010001ffff,
+    0xff0100010001ff00,
+    0xff01000100010100,
+    0xff01000101ff00ff,
+    0xff01000101ff0001,
+    0xff0100010100ffff,
+    0xff01000101000101,
+    0xff0101ffffffffff,
+    0xff0101ffffffff01,
+    0xff0101ffffff01ff,
+    0xff0101ffffff0101,
+    0xff0101ffff000000,
+    0xff0101ffff01ffff,
+    0xff0101ffff01ff01,
+    0xff0101ffff0101ff,
+    0xff0101ffff010101,
+    0xff0101ff00ff0000,
+    0xff0101ff0000ff00,
+    0xff0101ff000000ff,
+    0xff0101ff00010000,
+    0xff0101ff01ffffff,
+    0xff0101ff01ffff01,
+    0xff0101ff01ff01ff,
+    0xff0101ff01ff0101,
+    0xff0101ff0101ffff,
+    0xff0101ff0101ff01,
+    0xff0101ff010101ff,
+    0xff0101ff01010101,
+    0xff010100ffff0100,
+    0xff010100ff00ff00,
+    0xff010100ff0000ff,
+    0xff010100ff000100,
+    0xff010100ff010000,
+    0xff01010000ff0001,
+    0xff01010000ff0100,
+    0xff0101000000ff01,
+    0xff01010000000000,
+    0xff0101000001ff00,
+    0xff010100000100ff,
+    0xff01010000010001,
+    0xff01010000010100,
+    0xff01010001ff0000,
+    0xff0101000100ffff,
+    0xff01010001000001,
+    0xff01010001000100,
+    0xff010100010100ff,
+    0xff01010001010000,
+    0xff010101ffffffff,
+    0xff010101ffffff01,
+    0xff010101ffff01ff,
+    0xff010101ffff0101,
+    0xff010101ff01ffff,
+    0xff010101ff01ff01,
+    0xff010101ff0101ff,
+    0xff010101ff010101,
+    0xff01010100ff0000,
+    0xff0101010000ff00,
+    0xff01010100000001,
+    0xff01010100000100,
+    0xff01010100010000,
+    0xff01010101ffffff,
+    0xff01010101ffff01,
+    0xff01010101ff01ff,
+    0xff01010101ff0101,
+    0xff01010101000000,
+    0xff0101010101ffff,
+    0xff0101010101ff01,
+    0xff010101010101ff,
+    0xff01010101010101,
+    0x00ffffffffff0000,
+    0x00ffffffff00ff00,
+    0x00ffffffff000001,
+    0x00ffffffff010000,
+    0x00ffffff00ff0100,
+    0x00ffffff0000ff01,
+    0x00ffffff00000000,
+    0x00ffffff000001ff,
+    0x00ffffff00000101,
+    0x00ffffff0001ff00,
+    0x00ffffff000100ff,
+    0x00ffffff00010001,
+    0x00ffffff010000ff,
+    0x00ffffff01000100,
+    0x00ffffff0101ff00,
+    0x00ffffff01010001,
+    0x00ffff00ffffffff,
+    0x00ffff00ffffff00,
+    0x00ffff00ffff00ff,
+    0x00ffff00ffff0001,
+    0x00ffff00ffff0100,
+    0x00ffff00ff00ff01,
+    0x00ffff00ff000000,
+    0x00ffff00ff000001,
+    0x00ffff00ff0001ff,
+    0x00ffff00ff000101,
+    0x00ffff00ff01ff00,
+    0x00ffff00ff010001,
+    0x00ffff00ff010100,
+    0x00ffff0000ff0000,
+    0x00ffff0000ff01ff,
+    0x00ffff0000ff0101,
+    0x00ffff000000ff00,
+    0x00ffff00000000ff,
+    0x00ffff0000000000,
+    0x00ffff0000000001,
+    0x00ffff0000000100,
+    0x00ffff0000000101,
+    0x00ffff0000010000,
+    0x00ffff00000101ff,
+    0x00ffff0000010101,
+    0x00ffff0001ffff00,
+    0x00ffff0001ff00ff,
+    0x00ffff0001ff0001,
+    0x00ffff000100ffff,
+    0x00ffff000100ff01,
+    0x00ffff0001000000,
+    0x00ffff000101ffff,
+    0x00ffff000101ff00,
+    0x00ffff000101ff01,
+    0x00ffff01ffff0000,
+    0x00ffff01ff00ff00,
+    0x00ffff01ff0000ff,
+    0x00ffff01ff000001,
+    0x00ffff01ff010000,
+    0x00ffff0100ffff00,
+    0x00ffff010000ff01,
+    0x00ffff0100000000,
+    0x00ffff0100000101,
+    0x00ffff01000100ff,
+    0x00ffff0100010100,
+    0x00ffff0101ff0100,
+    0x00ffff01010000ff,
+    0x00ffff0101010000,
+    0x00ff00ffffffff00,
+    0x00ff00ffff000000,
+    0x00ff00ffff000100,
+    0x00ff00ffff010100,
+    0x00ff00ff00ff0000,
+    0x00ff00ff00ff01ff,
+    0x00ff00ff00ff0101,
+    0x00ff00ff0000ff00,
+    0x00ff00ff000000ff,
+    0x00ff00ff00000000,
+    0x00ff00ff00000001,
+    0x00ff00ff0001ff00,
+    0x00ff00ff0001ff01,
+    0x00ff00ff00010000,
+    0x00ff00ff000101ff,
+    0x00ff00ff00010101,
+    0x00ff00ff01ffff00,
+    0x00ff00ff01ff0001,
+    0x00ff00ff01ff0100,
+    0x00ff00ff0100ffff,
+    0x00ff00ff0100ff01,
+    0x00ff00ff01000000,
+    0x00ff00ff0101ffff,
+    0x00ff00ff0101ff00,
+    0x00ff00ff01010100,
+    0x00ff0000ffffff00,
+    0x00ff0000ffffff01,
+    0x00ff0000ffff0000,
+    0x00ff0000ffff0101,
+    0x00ff0000ff00ff00,
+    0x00ff0000ff0000ff,
+    0x00ff0000ff000000,
+    0x00ff0000ff000001,
+    0x00ff0000ff000100,
+    0x00ff0000ff01ffff,
+    0x00ff0000ff010000,
+    0x00ff0000ff010101,
+    0x00ff000000ffff00,
+    0x00ff000000ff00ff,
+    0x00ff000000ff0000,
+    0x00ff000000ff0001,
+    0x00ff000000ff0100,
+    0x00ff00000000ffff,
+    0x00ff00000000ff00,
+    0x00ff0000000000ff,
+    0x00ff000000000000,
+    0x00ff000000000001,
+    0x00ff0000000001ff,
+    0x00ff000000000100,
+    0x00ff00000001ff00,
+    0x00ff0000000100ff,
+    0x00ff000000010000,
+    0x00ff000000010001,
+    0x00ff000000010100,
+    0x00ff000001ffff01,
+    0x00ff000001ff00ff,
+    0x00ff000001ff0000,
+    0x00ff000001ff01ff,
+    0x00ff00000100ff00,
+    0x00ff0000010000ff,
+    0x00ff000001000000,
+    0x00ff000001000001,
+    0x00ff000001000100,
+    0x00ff000001000101,
+    0x00ff000001010000,
+    0x00ff0000010101ff,
+    0x00ff000001010101,
+    0x00ff0001ffffff00,
+    0x00ff0001ffff0000,
+    0x00ff0001ffff0100,
+    0x00ff0001ff0000ff,
+    0x00ff0001ff000000,
+    0x00ff0001ff0001ff,
+    0x00ff0001ff000101,
+    0x00ff0001ff01ff00,
+    0x00ff0001ff0100ff,
+    0x00ff0001ff010100,
+    0x00ff000100ffffff,
+    0x00ff000100ffff01,
+    0x00ff000100ff0000,
+    0x00ff000100ff01ff,
+    0x00ff00010000ffff,
+    0x00ff00010000ff00,
+    0x00ff00010000ff01,
+    0x00ff000100000000,
+    0x00ff000100000001,
+    0x00ff000100000100,
+    0x00ff00010001ff01,
+    0x00ff000100010000,
+    0x00ff0001000101ff,
+    0x00ff000101ffff00,
+    0x00ff000101ff0000,
+    0x00ff000101ff0101,
+    0x00ff0001010000ff,
+    0x00ff000101000000,
+    0x00ff00010101ff00,
+    0x00ff0001010100ff,
+    0x00ff000101010001,
+    0x00ff01ffffff0000,
+    0x00ff01ffff00ff00,
+    0x00ff01ffff000000,
+    0x00ff01ffff000101,
+    0x00ff01ffff010000,
+    0x00ff01ff00ffff01,
+    0x00ff01ff00ff0100,
+    0x00ff01ff0000ffff,
+    0x00ff01ff00000000,
+    0x00ff01ff000001ff,
+    0x00ff01ff0001ff00,
+    0x00ff01ff000100ff,
+    0x00ff01ff00010001,
+    0x00ff01ff00010100,
+    0x00ff01ff01ff0000,
+    0x00ff01ff0100ff00,
+    0x00ff01ff010000ff,
+    0x00ff01ff01000001,
+    0x00ff01ff01000100,
+    0x00ff01ff01010000,
+    0x00ff0100ffffff00,
+    0x00ff0100ffff0000,
+    0x00ff0100ffff0001,
+    0x00ff0100ffff0101,
+    0x00ff0100ff00ffff,
+    0x00ff0100ff0000ff,
+    0x00ff0100ff000000,
+    0x00ff0100ff0001ff,
+    0x00ff0100ff01ff00,
+    0x00ff0100ff0100ff,
+    0x00ff0100ff010001,
+    0x00ff010000ffffff,
+    0x00ff010000ff0000,
+    0x00ff010000ff0101,
+    0x00ff01000000ff00,
+    0x00ff01000000ff01,
+    0x00ff0100000000ff,
+    0x00ff010000000000,
+    0x00ff010000000001,
+    0x00ff010000000100,
+    0x00ff01000001ffff,
+    0x00ff01000001ff01,
+    0x00ff010000010000,
+    0x00ff010000010001,
+    0x00ff010000010101,
+    0x00ff010001ff0001,
+    0x00ff010001ff0100,
+    0x00ff01000100ff01,
+    0x00ff010001000000,
+    0x00ff010001000001,
+    0x00ff0100010001ff,
+    0x00ff01000101ff00,
+    0x00ff0100010100ff,
+    0x00ff010001010001,
+    0x00ff010001010100,
+    0x00ff0101ff000001,
+    0x00ff010100ff00ff,
+    0x00ff010100ff0001,
+    0x00ff010100ff0100,
+    0x00ff010100000000,
+    0x00ff0101000001ff,
+    0x00ff010100000101,
+    0x00ff0101000100ff,
+    0x00ff010100010100,
+    0x00ff0101010000ff,
+    0x00ff010101010000,
+    0x0000ffffffffff00,
+    0x0000ffffffff00ff,
+    0x0000ffffffff0000,
+    0x0000ffffffff0001,
+    0x0000ffffffff0100,
+    0x0000ffffff00ff01,
+    0x0000ffffff000000,
+    0x0000ffffff000101,
+    0x0000ffffff01ff00,
+    0x0000ffffff0100ff,
+    0x0000ffffff010100,
+    0x0000ffff00ffffff,
+    0x0000ffff00ff0000,
+    0x0000ffff00ff01ff,
+    0x0000ffff0000ff00,
+    0x0000ffff000000ff,
+    0x0000ffff00000000,
+    0x0000ffff00000001,
+    0x0000ffff00000100,
+    0x0000ffff00010000,
+    0x0000ffff000101ff,
+    0x0000ffff01ff0001,
+    0x0000ffff01ff0100,
+    0x0000ffff01000000,
+    0x0000ffff010001ff,
+    0x0000ffff0101ffff,
+    0x0000ffff0101ff00,
+    0x0000ffff01010001,
+    0x0000ffff01010100,
+    0x0000ff00ffff0000,
+    0x0000ff00ffff01ff,
+    0x0000ff00ffff0100,
+    0x0000ff00ffff0101,
+    0x0000ff00ff00ff00,
+    0x0000ff00ff0000ff,
+    0x0000ff00ff000000,
+    0x0000ff00ff000001,
+    0x0000ff00ff0001ff,
+    0x0000ff00ff000100,
+    0x0000ff00ff01ffff,
+    0x0000ff00ff010000,
+    0x0000ff00ff010001,
+    0x0000ff00ff0101ff,
+    0x0000ff00ff010101,
+    0x0000ff0000ffff00,
+    0x0000ff0000ff00ff,
+    0x0000ff0000ff0000,
+    0x0000ff0000ff0001,
+    0x0000ff0000ff0100,
+    0x0000ff000000ffff,
+    0x0000ff000000ff00,
+    0x0000ff000000ff01,
+    0x0000ff00000000ff,
+    0x0000ff0000000000,
+    0x0000ff0000000001,
+    0x0000ff00000001ff,
+    0x0000ff0000000100,
+    0x0000ff0000000101,
+    0x0000ff000001ff00,
+    0x0000ff00000100ff,
+    0x0000ff0000010000,
+    0x0000ff0000010001,
+    0x0000ff0000010100,
+    0x0000ff0001ffff01,
+    0x0000ff0001ff0000,
+    0x0000ff000100ff00,
+    0x0000ff00010000ff,
+    0x0000ff0001000000,
+    0x0000ff0001000001,
+    0x0000ff0001000100,
+    0x0000ff000101ffff,
+    0x0000ff0001010000,
+    0x0000ff0001010101,
+    0x0000ff01ffffff00,
+    0x0000ff01ffff0001,
+    0x0000ff01ff00ff01,
+    0x0000ff01ff000000,
+    0x0000ff01ff000101,
+    0x0000ff01ff01ff00,
+    0x0000ff01ff0100ff,
+    0x0000ff0100ffff01,
+    0x0000ff0100ff0000,
+    0x0000ff0100ff0101,
+    0x0000ff010000ff00,
+    0x0000ff01000000ff,
+    0x0000ff0100000000,
+    0x0000ff0100000001,
+    0x0000ff0100000100,
+    0x0000ff010001ff01,
+    0x0000ff0100010000,
+    0x0000ff0101ff0000,
+    0x0000ff010100ffff,
+    0x0000ff010100ff01,
+    0x0000ff0101000000,
+    0x0000ff0101000100,
+    0x0000ff0101000101,
+    0x0000ff01010100ff,
+    0x000000ffffff00ff,
+    0x000000ffffff0000,
+    0x000000ffff00ff00,
+    0x000000ffff0000ff,
+    0x000000ffff000000,
+    0x000000ffff000001,
+    0x000000ffff0001ff,
+    0x000000ffff000100,
+    0x000000ffff01ff00,
+    0x000000ffff010000,
+    0x000000ffff0101ff,
+    0x000000ffff010101,
+    0x000000ff00ffff00,
+    0x000000ff00ff00ff,
+    0x000000ff00ff0000,
+    0x000000ff00ff0001,
+    0x000000ff00ff0100,
+    0x000000ff00ff0101,
+    0x000000ff0000ffff,
+    0x000000ff0000ff00,
+    0x000000ff000000ff,
+    0x000000ff00000000,
+    0x000000ff00000001,
+    0x000000ff000001ff,
+    0x000000ff00000100,
+    0x000000ff00000101,
+    0x000000ff0001ff00,
+    0x000000ff0001ff01,
+    0x000000ff000100ff,
+    0x000000ff00010000,
+    0x000000ff00010001,
+    0x000000ff00010100,
+    0x000000ff01ffffff,
+    0x000000ff01ff01ff,
+    0x000000ff01ff0101,
+    0x000000ff0100ff00,
+    0x000000ff010000ff,
+    0x000000ff01000000,
+    0x000000ff01000001,
+    0x000000ff01000100,
+    0x000000ff0101ff00,
+    0x000000ff010100ff,
+    0x000000ff01010000,
+    0x000000ff01010101,
+    0x00000000ffffff00,
+    0x00000000ffffff01,
+    0x00000000ffff00ff,
+    0x00000000ffff0000,
+    0x00000000ffff0001,
+    0x00000000ffff0100,
+    0x00000000ff00ffff,
+    0x00000000ff00ff00,
+    0x00000000ff00ff01,
+    0x00000000ff0000ff,
+    0x00000000ff000000,
+    0x00000000ff000001,
+    0x00000000ff000100,
+    0x00000000ff000101,
+    0x00000000ff01ff00,
+    0x00000000ff0100ff,
+    0x00000000ff010000,
+    0x00000000ff010001,
+    0x00000000ff010100,
+    0x0000000000ffffff,
+    0x0000000000ffff00,
+    0x0000000000ffff01,
+    0x0000000000ff00ff,
+    0x0000000000ff0000,
+    0x0000000000ff0001,
+    0x0000000000ff01ff,
+    0x0000000000ff0100,
+    0x000000000000ffff,
+    0x000000000000ff00,
+    0x000000000000ff01,
+    0x00000000000000ff,
+    0x0000000000000000,
+    0x0000000000000001,
+    0x00000000000001ff,
+    0x0000000000000100,
+    0x0000000000000101,
+    0x000000000001ffff,
+    0x000000000001ff00,
+    0x00000000000100ff,
+    0x0000000000010000,
+    0x0000000000010001,
+    0x00000000000101ff,
+    0x0000000000010100,
+    0x0000000000010101,
+    0x0000000001ffff00,
+    0x0000000001ff00ff,
+    0x0000000001ff0000,
+    0x0000000001ff0100,
+    0x0000000001ff0101,
+    0x000000000100ffff,
+    0x000000000100ff00,
+    0x00000000010000ff,
+    0x0000000001000000,
+    0x0000000001000001,
+    0x00000000010001ff,
+    0x0000000001000100,
+    0x000000000101ff00,
+    0x00000000010100ff,
+    0x0000000001010000,
+    0x0000000001010001,
+    0x0000000001010100,
+    0x00000001ffffffff,
+    0x00000001ffffff00,
+    0x00000001ffffff01,
+    0x00000001ffff00ff,
+    0x00000001ffff0001,
+    0x00000001ffff01ff,
+    0x00000001ffff0100,
+    0x00000001ff00ff00,
+    0x00000001ff0000ff,
+    0x00000001ff000000,
+    0x00000001ff0001ff,
+    0x00000001ff000100,
+    0x00000001ff01ffff,
+    0x00000001ff01ff00,
+    0x00000001ff01ff01,
+    0x00000001ff0100ff,
+    0x00000001ff010000,
+    0x00000001ff010001,
+    0x00000001ff0101ff,
+    0x00000001ff010100,
+    0x0000000100ffff00,
+    0x0000000100ff0000,
+    0x0000000100ff0001,
+    0x0000000100ff01ff,
+    0x0000000100ff0100,
+    0x0000000100ff0101,
+    0x000000010000ffff,
+    0x000000010000ff00,
+    0x000000010000ff01,
+    0x00000001000000ff,
+    0x0000000100000000,
+    0x0000000100000001,
+    0x00000001000001ff,
+    0x0000000100000100,
+    0x0000000100000101,
+    0x000000010001ff00,
+    0x00000001000100ff,
+    0x0000000100010000,
+    0x0000000100010100,
+    0x0000000101ffff01,
+    0x0000000101ff0000,
+    0x0000000101ff0001,
+    0x0000000101ff01ff,
+    0x0000000101ff0100,
+    0x0000000101ff0101,
+    0x000000010100ff00,
+    0x0000000101000000,
+    0x0000000101000101,
+    0x000000010101ff01,
+    0x0000000101010000,
+    0x0000000101010001,
+    0x00000001010101ff,
+    0x0000000101010100,
+    0x000001ffffff00ff,
+    0x000001ffffff0000,
+    0x000001ffffff0001,
+    0x000001ffffff0100,
+    0x000001ffff00ffff,
+    0x000001ffff000000,
+    0x000001ffff0001ff,
+    0x000001ffff01ff00,
+    0x000001ffff010101,
+    0x000001ff00ff0000,
+    0x000001ff00ff01ff,
+    0x000001ff00ff0101,
+    0x000001ff0000ff00,
+    0x000001ff000000ff,
+    0x000001ff00000000,
+    0x000001ff00000001,
+    0x000001ff000001ff,
+    0x000001ff00000100,
+    0x000001ff0001ffff,
+    0x000001ff0001ff01,
+    0x000001ff000100ff,
+    0x000001ff00010000,
+    0x000001ff01ffff01,
+    0x000001ff01ff0100,
+    0x000001ff0100ffff,
+    0x000001ff0100ff01,
+    0x000001ff01000000,
+    0x000001ff010001ff,
+    0x000001ff0101ff00,
+    0x000001ff01010100,
+    0x00000100ffffff00,
+    0x00000100ffffff01,
+    0x00000100ffff0000,
+    0x00000100ffff0101,
+    0x00000100ff00ff00,
+    0x00000100ff0000ff,
+    0x00000100ff000000,
+    0x00000100ff000001,
+    0x00000100ff000100,
+    0x00000100ff010000,
+    0x0000010000ffff00,
+    0x0000010000ff00ff,
+    0x0000010000ff0000,
+    0x0000010000ff0001,
+    0x0000010000ff0100,
+    0x000001000000ffff,
+    0x000001000000ff00,
+    0x000001000000ff01,
+    0x00000100000000ff,
+    0x0000010000000000,
+    0x0000010000000001,
+    0x00000100000001ff,
+    0x0000010000000100,
+    0x0000010000000101,
+    0x000001000001ff00,
+    0x00000100000100ff,
+    0x0000010000010000,
+    0x0000010000010001,
+    0x0000010000010100,
+    0x0000010001ffff00,
+    0x0000010001ff0000,
+    0x0000010001ff0100,
+    0x000001000100ff00,
+    0x00000100010000ff,
+    0x0000010001000000,
+    0x0000010001000001,
+    0x00000100010001ff,
+    0x0000010001000100,
+    0x0000010001010000,
+    0x00000101ffff00ff,
+    0x00000101ffff01ff,
+    0x00000101ff000000,
+    0x00000101ff000101,
+    0x00000101ff01ffff,
+    0x00000101ff010000,
+    0x00000101ff010001,
+    0x00000101ff010100,
+    0x0000010100ff0000,
+    0x0000010100ff01ff,
+    0x0000010100ff0100,
+    0x000001010000ff00,
+    0x0000010100000000,
+    0x0000010100000001,
+    0x00000101000001ff,
+    0x0000010100000100,
+    0x000001010001ff01,
+    0x0000010100010000,
+    0x00000101000101ff,
+    0x0000010100010101,
+    0x0000010101ffff00,
+    0x0000010101ff0101,
+    0x000001010100ff01,
+    0x0000010101000000,
+    0x0000010101000001,
+    0x00000101010001ff,
+    0x0000010101000101,
+    0x000001010101ff00,
+    0x0001ffffffff0000,
+    0x0001ffffff0000ff,
+    0x0001ffffff000001,
+    0x0001ffffff000100,
+    0x0001ffffff010000,
+    0x0001ffff00ff00ff,
+    0x0001ffff0000ffff,
+    0x0001ffff00000000,
+    0x0001ffff00000001,
+    0x0001ffff000001ff,
+    0x0001ffff00000101,
+    0x0001ffff0001ff00,
+    0x0001ffff000100ff,
+    0x0001ffff00010001,
+    0x0001ffff00010100,
+    0x0001ffff01ffff00,
+    0x0001ffff01000001,
+    0x0001ffff01010000,
+    0x0001ff00ffffff00,
+    0x0001ff00ffff00ff,
+    0x0001ff00ffff0001,
+    0x0001ff00ffff0100,
+    0x0001ff00ff00ff01,
+    0x0001ff00ff000000,
+    0x0001ff00ff01ff00,
+    0x0001ff00ff01ff01,
+    0x0001ff00ff010001,
+    0x0001ff00ff010100,
+    0x0001ff0000ff0000,
+    0x0001ff0000ff0100,
+    0x0001ff000000ff00,
+    0x0001ff0000000000,
+    0x0001ff0000000001,
+    0x0001ff0000000100,
+    0x0001ff0000010000,
+    0x0001ff0000010001,
+    0x0001ff0000010101,
+    0x0001ff0001ff00ff,
+    0x0001ff0001ff0101,
+    0x0001ff000100ff01,
+    0x0001ff0001000000,
+    0x0001ff000101ff00,
+    0x0001ff0001010001,
+    0x0001ff0001010100,
+    0x0001ff01ff00ff00,
+    0x0001ff01ff000001,
+    0x0001ff01ff000100,
+    0x0001ff0100ffffff,
+    0x0001ff0100ffff00,
+    0x0001ff0100ff0001,
+    0x0001ff0100000000,
+    0x0001ff0100000001,
+    0x0001ff01000001ff,
+    0x0001ff010001ffff,
+    0x0001ff0101ff0000,
+    0x0001ff010100ff00,
+    0x0001ff0101000001,
+    0x0001ff0101010000,
+    0x000100ffff00ff00,
+    0x000100ffff00ff01,
+    0x000100ffff000000,
+    0x000100ffff000001,
+    0x000100ffff000101,
+    0x000100ffff01ff00,
+    0x000100ffff010001,
+    0x000100ffff010100,
+    0x000100ff00ffffff,
+    0x000100ff00ffff01,
+    0x000100ff00ff0000,
+    0x000100ff00ff01ff,
+    0x000100ff00ff0101,
+    0x000100ff0000ff00,
+    0x000100ff000000ff,
+    0x000100ff00000000,
+    0x000100ff00000001,
+    0x000100ff00000100,
+    0x000100ff00000101,
+    0x000100ff0001ffff,
+    0x000100ff0001ff01,
+    0x000100ff00010000,
+    0x000100ff01ff00ff,
+    0x000100ff01ff0000,
+    0x000100ff01ff0100,
+    0x000100ff0100ffff,
+    0x000100ff0100ff01,
+    0x000100ff010000ff,
+    0x000100ff01000000,
+    0x000100ff01000001,
+    0x000100ff010001ff,
+    0x000100ff01000101,
+    0x000100ff0101ff00,
+    0x000100ff010100ff,
+    0x000100ff01010100,
+    0x00010000ffff0000,
+    0x00010000ffff01ff,
+    0x00010000ffff0101,
+    0x00010000ff00ff00,
+    0x00010000ff000000,
+    0x00010000ff000001,
+    0x00010000ff000100,
+    0x0001000000ff00ff,
+    0x0001000000ff0000,
+    0x0001000000ff0001,
+    0x0001000000ff0100,
+    0x000100000000ffff,
+    0x000100000000ff00,
+    0x00010000000000ff,
+    0x0001000000000000,
+    0x0001000000000001,
+    0x0001000000000100,
+    0x000100000001ff00,
+    0x00010000000100ff,
+    0x0001000000010000,
+    0x0001000000010001,
+    0x0001000000010100,
+    0x0001000001ff0001,
+    0x0001000001ff0100,
+    0x0001000001ff0101,
+    0x000100000100ff00,
+    0x0001000001000000,
+    0x0001000001000001,
+    0x0001000001000100,
+    0x0001000001000101,
+    0x000100000101ff01,
+    0x0001000001010000,
+    0x0001000001010001,
+    0x00010000010101ff,
+    0x00010001ffffff01,
+    0x00010001ffff0100,
+    0x00010001ff000000,
+    0x00010001ff01ffff,
+    0x00010001ff010001,
+    0x00010001ff0101ff,
+    0x00010001ff010100,
+    0x0001000100ffffff,
+    0x0001000100ff0000,
+    0x0001000100ff01ff,
+    0x0001000100ff0101,
+    0x000100010000ff00,
+    0x00010001000000ff,
+    0x0001000100000000,
+    0x0001000100000001,
+    0x00010001000001ff,
+    0x0001000100000101,
+    0x000100010001ffff,
+    0x0001000100010000,
+    0x00010001000101ff,
+    0x0001000101ffffff,
+    0x0001000101ffff01,
+    0x0001000101ff0000,
+    0x0001000101ff0101,
+    0x00010001010000ff,
+    0x0001000101000001,
+    0x00010001010001ff,
+    0x0001000101000100,
+    0x000100010101ffff,
+    0x00010001010100ff,
+    0x0001000101010001,
+    0x0001000101010101,
+    0x000101ffff000001,
+    0x000101ffff000100,
+    0x000101ffff010000,
+    0x000101ff00ffff00,
+    0x000101ff0000ff01,
+    0x000101ff00000000,
+    0x000101ff00000101,
+    0x000101ff0001ff00,
+    0x000101ff00010100,
+    0x000101ff01ff0000,
+    0x000101ff0100ff00,
+    0x000101ff010001ff,
+    0x000101ff01010001,
+    0x00010100ffffff00,
+    0x00010100ffff00ff,
+    0x00010100ff00ffff,
+    0x00010100ff000000,
+    0x00010100ff01ff00,
+    0x00010100ff0100ff,
+    0x00010100ff010001,
+    0x00010100ff010100,
+    0x0001010000ffffff,
+    0x0001010000ffff00,
+    0x0001010000ff0000,
+    0x0001010000ff0001,
+    0x0001010000ff01ff,
+    0x000101000000ff00,
+    0x00010100000000ff,
+    0x0001010000000000,
+    0x0001010000000001,
+    0x0001010000000100,
+    0x000101000001ffff,
+    0x0001010000010000,
+    0x0001010000010101,
+    0x0001010001ffff01,
+    0x0001010001ff00ff,
+    0x0001010001ff0101,
+    0x0001010001000000,
+    0x000101000101ff00,
+    0x00010100010100ff,
+    0x0001010001010000,
+    0x0001010001010100,
+    0x00010101ff00ff00,
+    0x00010101ff000001,
+    0x00010101ff0001ff,
+    0x0001010100ffff00,
+    0x0001010100ff00ff,
+    0x0001010100ff0100,
+    0x000101010000ffff,
+    0x0001010100000000,
+    0x00010101000001ff,
+    0x0001010100000101,
+    0x00010101000100ff,
+    0x0001010100010000,
+    0x0001010100010100,
+    0x0001010101ff0001,
+    0x00010101010000ff,
+    0x00010101010001ff,
+    0x0001010101000101,
+    0x0001010101010001,
+    0x01ffffffffffffff,
+    0x01ffffffffffff01,
+    0x01ffffffffff01ff,
+    0x01ffffffffff0101,
+    0x01ffffffff01ffff,
+    0x01ffffffff01ff01,
+    0x01ffffffff0101ff,
+    0x01ffffffff010101,
+    0x01ffffff00ff0000,
+    0x01ffffff0000ffff,
+    0x01ffffff0000ff00,
+    0x01ffffff000000ff,
+    0x01ffffff00000001,
+    0x01ffffff00000100,
+    0x01ffffff00010000,
+    0x01ffffff01ffffff,
+    0x01ffffff01ffff01,
+    0x01ffffff01ff01ff,
+    0x01ffffff01ff0101,
+    0x01ffffff01000000,
+    0x01ffffff0101ffff,
+    0x01ffffff0101ff01,
+    0x01ffffff010101ff,
+    0x01ffffff01010101,
+    0x01ffff00ffff0000,
+    0x01ffff00ff00ff00,
+    0x01ffff00ff0000ff,
+    0x01ffff00ff000001,
+    0x01ffff00ff000100,
+    0x01ffff00ff010000,
+    0x01ffff0000ffff00,
+    0x01ffff0000ff00ff,
+    0x01ffff0000ff0100,
+    0x01ffff000000ffff,
+    0x01ffff000000ff01,
+    0x01ffff0000000000,
+    0x01ffff0000000001,
+    0x01ffff00000001ff,
+    0x01ffff0000000100,
+    0x01ffff00000100ff,
+    0x01ffff0000010001,
+    0x01ffff0000010100,
+    0x01ffff0001ff0000,
+    0x01ffff0001ff0100,
+    0x01ffff00010000ff,
+    0x01ffff0001000001,
+    0x01ffff0001000100,
+    0x01ffff0001010000,
+    0x01ffff01ffffffff,
+    0x01ffff01ffffff01,
+    0x01ffff01ffff01ff,
+    0x01ffff01ffff0101,
+    0x01ffff01ff000000,
+    0x01ffff01ff01ffff,
+    0x01ffff01ff01ff01,
+    0x01ffff01ff0101ff,
+    0x01ffff01ff010101,
+    0x01ffff010000ff00,
+    0x01ffff01000000ff,
+    0x01ffff0100000100,
+    0x01ffff0100010000,
+    0x01ffff0101ffffff,
+    0x01ffff0101ffff01,
+    0x01ffff0101ff01ff,
+    0x01ffff0101ff0101,
+    0x01ffff0101000000,
+    0x01ffff010101ffff,
+    0x01ffff010101ff01,
+    0x01ffff01010101ff,
+    0x01ffff0101010101,
+    0x01ff00ffff0000ff,
+    0x01ff00ffff000100,
+    0x01ff00ff00ffff00,
+    0x01ff00ff00ff00ff,
+    0x01ff00ff0000ff00,
+    0x01ff00ff00000000,
+    0x01ff00ff00000101,
+    0x01ff00ff0001ff00,
+    0x01ff00ff000100ff,
+    0x01ff00ff00010100,
+    0x01ff00ff010000ff,
+    0x01ff00ff01000100,
+    0x01ff0000ffffff00,
+    0x01ff0000ffff0100,
+    0x01ff0000ff00ff01,
+    0x01ff0000ff000000,
+    0x01ff0000ff000101,
+    0x01ff0000ff010001,
+    0x01ff0000ff010100,
+    0x01ff000000ffffff,
+    0x01ff000000ffff00,
+    0x01ff000000ff0000,
+    0x01ff000000ff01ff,
+    0x01ff00000000ff00,
+    0x01ff0000000000ff,
+    0x01ff000000000000,
+    0x01ff000000000001,
+    0x01ff000000000100,
+    0x01ff000000000101,
+    0x01ff000000010000,
+    0x01ff000000010001,
+    0x01ff0000000101ff,
+    0x01ff000000010101,
+    0x01ff000001ffff00,
+    0x01ff000001ff00ff,
+    0x01ff000001ff0001,
+    0x01ff000001ff0100,
+    0x01ff00000100ffff,
+    0x01ff00000100ff01,
+    0x01ff000001000000,
+    0x01ff0000010001ff,
+    0x01ff000001010001,
+    0x01ff0001ff00ff00,
+    0x01ff0001ff000001,
+    0x01ff0001ff000100,
+    0x01ff0001ff010000,
+    0x01ff000100ffff00,
+    0x01ff000100ff00ff,
+    0x01ff000100ff0100,
+    0x01ff000100ff0101,
+    0x01ff00010000ffff,
+    0x01ff000100000000,
+    0x01ff000100000100,
+    0x01ff000100000101,
+    0x01ff00010001ff00,
+    0x01ff000100010001,
+    0x01ff000100010101,
+    0x01ff000101ff0000,
+    0x01ff00010100ff00,
+    0x01ff000101000101,
+    0x01ff0001010100ff,
+    0x01ff01ffffffffff,
+    0x01ff01ffffffff01,
+    0x01ff01ffffff01ff,
+    0x01ff01ffffff0101,
+    0x01ff01ffff000000,
+    0x01ff01ffff01ffff,
+    0x01ff01ffff01ff01,
+    0x01ff01ffff0101ff,
+    0x01ff01ffff010101,
+    0x01ff01ff00ffff00,
+    0x01ff01ff00ff0000,
+    0x01ff01ff0000ff00,
+    0x01ff01ff000000ff,
+    0x01ff01ff00000100,
+    0x01ff01ff00010000,
+    0x01ff01ff00010100,
+    0x01ff01ff01ffffff,
+    0x01ff01ff01ffff01,
+    0x01ff01ff01ff01ff,
+    0x01ff01ff01ff0101,
+    0x01ff01ff01000000,
+    0x01ff01ff0101ffff,
+    0x01ff01ff0101ff01,
+    0x01ff01ff010101ff,
+    0x01ff01ff01010101,
+    0x01ff0100ffff0000,
+    0x01ff0100ffff0001,
+    0x01ff0100ff00ff00,
+    0x01ff0100ff0000ff,
+    0x01ff0100ff000001,
+    0x01ff0100ff010000,
+    0x01ff010000ffff00,
+    0x01ff010000ff00ff,
+    0x01ff010000ff0001,
+    0x01ff010000ff0100,
+    0x01ff01000000ffff,
+    0x01ff01000000ff01,
+    0x01ff010000000000,
+    0x01ff010000000101,
+    0x01ff01000001ff00,
+    0x01ff0100000100ff,
+    0x01ff010001ff0000,
+    0x01ff010001000001,
+    0x01ff010001000100,
+    0x01ff010001010000,
+    0x01ff0101ffffffff,
+    0x01ff0101ffffff01,
+    0x01ff0101ffff01ff,
+    0x01ff0101ffff0101,
+    0x01ff0101ff000000,
+    0x01ff0101ff01ffff,
+    0x01ff0101ff01ff01,
+    0x01ff0101ff0101ff,
+    0x01ff0101ff010101,
+    0x01ff010100ff0000,
+    0x01ff01010000ff00,
+    0x01ff0101000000ff,
+    0x01ff010100000001,
+    0x01ff010101ffffff,
+    0x01ff010101ffff01,
+    0x01ff010101ff01ff,
+    0x01ff010101ff0101,
+    0x01ff010101000000,
+    0x01ff01010101ffff,
+    0x01ff01010101ff01,
+    0x01ff0101010101ff,
+    0x01ff010101010101,
+    0x0100ffffffff0000,
+    0x0100ffffff00ff00,
+    0x0100ffffff000001,
+    0x0100ffffff0001ff,
+    0x0100ffffff000100,
+    0x0100ffffff010000,
+    0x0100ffff00ffff00,
+    0x0100ffff00ff0001,
+    0x0100ffff00ff0100,
+    0x0100ffff00000000,
+    0x0100ffff000001ff,
+    0x0100ffff00000101,
+    0x0100ffff00010100,
+    0x0100ffff00010101,
+    0x0100ffff01ff0000,
+    0x0100ffff0100ff00,
+    0x0100ffff010000ff,
+    0x0100ffff01000001,
+    0x0100ffff01000100,
+    0x0100ffff01010000,
+    0x0100ff00ffffff00,
+    0x0100ff00ffff00ff,
+    0x0100ff00ffff0001,
+    0x0100ff00ffff0100,
+    0x0100ff00ff00ffff,
+    0x0100ff00ff000000,
+    0x0100ff00ff0001ff,
+    0x0100ff00ff000101,
+    0x0100ff00ff01ff00,
+    0x0100ff00ff0100ff,
+    0x0100ff00ff010001,
+    0x0100ff00ff010100,
+    0x0100ff0000ffffff,
+    0x0100ff0000ff0000,
+    0x0100ff000000ffff,
+    0x0100ff000000ff00,
+    0x0100ff00000000ff,
+    0x0100ff0000000000,
+    0x0100ff0000000001,
+    0x0100ff0000000100,
+    0x0100ff000001ff01,
+    0x0100ff0000010000,
+    0x0100ff0001ff00ff,
+    0x0100ff0001ff0001,
+    0x0100ff000100ff01,
+    0x0100ff0001000000,
+    0x0100ff00010001ff,
+    0x0100ff000101ff00,
+    0x0100ff00010100ff,
+    0x0100ff0001010001,
+    0x0100ff0001010100,
+    0x0100ff01ffff0000,
+    0x0100ff01ff00ff00,
+    0x0100ff01ff0000ff,
+    0x0100ff01ff000100,
+    0x0100ff01ff010000,
+    0x0100ff0100ff00ff,
+    0x0100ff0100ff0001,
+    0x0100ff0100ff0100,
+    0x0100ff010000ffff,
+    0x0100ff010000ff01,
+    0x0100ff0100000000,
+    0x0100ff01000001ff,
+    0x0100ff0100010001,
+    0x0100ff0100010100,
+    0x0100ff0101ff0000,
+    0x0100ff01010000ff,
+    0x0100ff0101000001,
+    0x0100ff0101010100,
+    0x010000ffffffff00,
+    0x010000ffffff00ff,
+    0x010000ffffff0001,
+    0x010000ffff00ffff,
+    0x010000ffff000000,
+    0x010000ffff0001ff,
+    0x010000ffff010001,
+    0x010000ff00ffffff,
+    0x010000ff00ff0101,
+    0x010000ff0000ff00,
+    0x010000ff000000ff,
+    0x010000ff00000000,
+    0x010000ff00000001,
+    0x010000ff000001ff,
+    0x010000ff00000100,
+    0x010000ff0001ffff,
+    0x010000ff0001ff00,
+    0x010000ff0001ff01,
+    0x010000ff00010000,
+    0x010000ff01ff00ff,
+    0x010000ff01ff0001,
+    0x010000ff0100ff01,
+    0x010000ff010000ff,
+    0x010000ff01000000,
+    0x010000ff010001ff,
+    0x010000ff0101ff00,
+    0x010000ff01010100,
+    0x01000000ffffffff,
+    0x01000000ffff0000,
+    0x01000000ffff01ff,
+    0x01000000ffff0101,
+    0x01000000ff00ffff,
+    0x01000000ff00ff00,
+    0x01000000ff0000ff,
+    0x01000000ff000000,
+    0x01000000ff000001,
+    0x01000000ff000100,
+    0x01000000ff01ff00,
+    0x01000000ff010000,
+    0x01000000ff010100,
+    0x01000000ff010101,
+    0x0100000000ffff00,
+    0x0100000000ff00ff,
+    0x0100000000ff0000,
+    0x0100000000ff0001,
+    0x0100000000ff0100,
+    0x010000000000ffff,
+    0x010000000000ff00,
+    0x010000000000ff01,
+    0x01000000000000ff,
+    0x0100000000000000,
+    0x0100000000000001,
+    0x01000000000001ff,
+    0x0100000000000100,
+    0x0100000000000101,
+    0x010000000001ff00,
+    0x01000000000100ff,
+    0x0100000000010000,
+    0x0100000000010001,
+    0x0100000000010100,
+    0x0100000001ffff00,
+    0x0100000001ff0000,
+    0x0100000001ff01ff,
+    0x010000000100ff00,
+    0x010000000100ff01,
+    0x01000000010000ff,
+    0x0100000001000000,
+    0x0100000001000001,
+    0x0100000001000100,
+    0x0100000001000101,
+    0x010000000101ffff,
+    0x010000000101ff01,
+    0x0100000001010000,
+    0x01000000010101ff,
+    0x0100000001010101,
+    0x01000001ffffff00,
+    0x01000001ffff00ff,
+    0x01000001ff00ffff,
+    0x01000001ff000000,
+    0x01000001ff000100,
+    0x01000001ff01ffff,
+    0x01000001ff010001,
+    0x01000001ff010100,
+    0x0100000100ff0000,
+    0x0100000100ff01ff,
+    0x0100000100ff0100,
+    0x010000010000ff00,
+    0x010000010000ff01,
+    0x0100000100000000,
+    0x0100000100000001,
+    0x0100000100000100,
+    0x0100000100010000,
+    0x01000001000101ff,
+    0x0100000101ffff01,
+    0x0100000101ff00ff,
+    0x0100000101ff0100,
+    0x0100000101ff0101,
+    0x010000010100ff01,
+    0x01000001010000ff,
+    0x0100000101000000,
+    0x01000001010100ff,
+    0x0100000101010001,
+    0x0100000101010100,
+    0x010001ffffff0000,
+    0x010001ffff000001,
+    0x010001ffff000100,
+    0x010001ffff010000,
+    0x010001ff00ffff00,
+    0x010001ff00ff0001,
+    0x010001ff0000ffff,
+    0x010001ff0000ff01,
+    0x010001ff00000000,
+    0x010001ff00000001,
+    0x010001ff00000101,
+    0x010001ff000100ff,
+    0x010001ff00010000,
+    0x010001ff01ff0000,
+    0x010001ff0100ff00,
+    0x010001ff01000001,
+    0x010001ff01000100,
+    0x010001ff01010000,
+    0x01000100ffff00ff,
+    0x01000100ffff0001,
+    0x01000100ffff0100,
+    0x01000100ff00ffff,
+    0x01000100ff00ff01,
+    0x01000100ff000000,
+    0x01000100ff0001ff,
+    0x01000100ff000101,
+    0x01000100ff01ffff,
+    0x01000100ff01ff00,
+    0x01000100ff0100ff,
+    0x01000100ff010001,
+    0x0100010000ffffff,
+    0x0100010000ffff01,
+    0x0100010000ff0000,
+    0x0100010000ff01ff,
+    0x0100010000ff0101,
+    0x010001000000ff00,
+    0x01000100000000ff,
+    0x0100010000000000,
+    0x0100010000000001,
+    0x0100010000000100,
+    0x010001000001ff01,
+    0x0100010000010000,
+    0x0100010000010001,
+    0x0100010000010101,
+    0x0100010001ffff00,
+    0x0100010001ff00ff,
+    0x010001000100ffff,
+    0x010001000100ff01,
+    0x0100010001000000,
+    0x0100010001000101,
+    0x010001000101ff00,
+    0x0100010001010001,
+    0x01000101ffff0000,
+    0x01000101ff000000,
+    0x01000101ff010000,
+    0x0100010100ff00ff,
+    0x0100010100ff0001,
+    0x0100010100ff0100,
+    0x010001010000ffff,
+    0x0100010100000000,
+    0x01000101000001ff,
+    0x010001010001ff00,
+    0x0100010101ff0000,
+    0x010001010100ff00,
+    0x01000101010000ff,
+    0x0100010101000000,
+    0x0100010101000001,
+    0x0101ffffffffffff,
+    0x0101ffffffffff01,
+    0x0101ffffffff01ff,
+    0x0101ffffffff0101,
+    0x0101ffffff000000,
+    0x0101ffffff01ffff,
+    0x0101ffffff01ff01,
+    0x0101ffffff0101ff,
+    0x0101ffffff010101,
+    0x0101ffff00ff0000,
+    0x0101ffff0000ff00,
+    0x0101ffff000000ff,
+    0x0101ffff00000001,
+    0x0101ffff00000100,
+    0x0101ffff01ffffff,
+    0x0101ffff01ffff01,
+    0x0101ffff01ff01ff,
+    0x0101ffff01ff0101,
+    0x0101ffff01000000,
+    0x0101ffff0101ffff,
+    0x0101ffff0101ff01,
+    0x0101ffff010101ff,
+    0x0101ffff01010101,
+    0x0101ff00ffff0000,
+    0x0101ff00ffff0100,
+    0x0101ff00ff00ff00,
+    0x0101ff00ff0000ff,
+    0x0101ff00ff000001,
+    0x0101ff00ff000100,
+    0x0101ff00ff000101,
+    0x0101ff0000ff0001,
+    0x0101ff0000ff0100,
+    0x0101ff000000ff00,
+    0x0101ff0000000000,
+    0x0101ff00000001ff,
+    0x0101ff0000000101,
+    0x0101ff000001ff00,
+    0x0101ff00000100ff,
+    0x0101ff0001ff0000,
+    0x0101ff000100ffff,
+    0x0101ff000100ff01,
+    0x0101ff0001000001,
+    0x0101ff0001000100,
+    0x0101ff01ffffff01,
+    0x0101ff01ffff01ff,
+    0x0101ff01ffff0101,
+    0x0101ff01ff00ffff,
+    0x0101ff01ff000100,
+    0x0101ff01ff01ff01,
+    0x0101ff01ff0101ff,
+    0x0101ff01ff010101,
+    0x0101ff0100ff0000,
+    0x0101ff010000ff00,
+    0x0101ff0100000001,
+    0x0101ff0100000100,
+    0x0101ff0100010000,
+    0x0101ff0101ffffff,
+    0x0101ff0101ffff01,
+    0x0101ff0101ff01ff,
+    0x0101ff0101ff0101,
+    0x0101ff0101000000,
+    0x0101ff010101ffff,
+    0x0101ff010101ff01,
+    0x0101ff01010101ff,
+    0x0101ff0101010101,
+    0x010100ffff000100,
+    0x010100ffff010000,
+    0x010100ff00ffff00,
+    0x010100ff00ff00ff,
+    0x010100ff0000ffff,
+    0x010100ff000000ff,
+    0x010100ff00000000,
+    0x010100ff000001ff,
+    0x010100ff00000101,
+    0x010100ff0001ff00,
+    0x010100ff00010000,
+    0x010100ff00010001,
+    0x010100ff000101ff,
+    0x010100ff00010100,
+    0x010100ff01ff0000,
+    0x01010000ffff0001,
+    0x01010000ffff0100,
+    0x01010000ff00ffff,
+    0x01010000ff00ff01,
+    0x01010000ff000000,
+    0x01010000ff0001ff,
+    0x01010000ff010001,
+    0x01010000ff010100,
+    0x0101000000ffff01,
+    0x0101000000ff0000,
+    0x010100000000ff00,
+    0x01010000000000ff,
+    0x0101000000000000,
+    0x0101000000000001,
+    0x0101000000000100,
+    0x0101000000010000,
+    0x0101000000010101,
+    0x0101000001ffff00,
+    0x0101000001ff00ff,
+    0x0101000001ff0000,
+    0x0101000001ff0001,
+    0x0101000001ff0100,
+    0x010100000100ff01,
+    0x0101000001000000,
+    0x01010000010001ff,
+    0x01010001ffff0000,
+    0x01010001ff00ff00,
+    0x01010001ff000001,
+    0x01010001ff000101,
+    0x01010001ff01ff00,
+    0x01010001ff010000,
+    0x0101000100ff00ff,
+    0x0101000100ff0001,
+    0x0101000100ff0101,
+    0x010100010000ff01,
+    0x0101000100000000,
+    0x0101000100000001,
+    0x01010001000001ff,
+    0x010100010001ffff,
+    0x010100010001ff01,
+    0x0101000101ff0001,
+    0x010100010100ffff,
+    0x0101000101000000,
+    0x0101000101000001,
+    0x0101000101000100,
+    0x010100010101ff00,
+    0x01010001010100ff,
+    0x0101000101010001,
+    0x010101ffffffffff,
+    0x010101ffffffff01,
+    0x010101ffffff01ff,
+    0x010101ffffff0101,
+    0x010101ffff01ffff,
+    0x010101ffff01ff01,
+    0x010101ffff0101ff,
+    0x010101ffff010101,
+    0x010101ff0000ff00,
+    0x010101ff000000ff,
+    0x010101ff00000001,
+    0x010101ff00000100,
+    0x010101ff01ffffff,
+    0x010101ff01ffff01,
+    0x010101ff01ff01ff,
+    0x010101ff01ff0101,
+    0x010101ff01000000,
+    0x010101ff0101ffff,
+    0x010101ff0101ff01,
+    0x010101ff010101ff,
+    0x010101ff01010101,
+    0x01010100ffff0000,
+    0x01010100ff0000ff,
+    0x01010100ff000100,
+    0x01010100ff01ff00,
+    0x01010100ff010000,
+    0x0101010000ffff00,
+    0x010101000000ffff,
+    0x0101010000000000,
+    0x0101010000000101,
+    0x010101000001ff00,
+    0x0101010000010001,
+    0x0101010000010100,
+    0x010101000100ffff,
+    0x0101010001000001,
+    0x01010101ffffffff,
+    0x01010101ffffff01,
+    0x01010101ffff01ff,
+    0x01010101ffff0101,
+    0x01010101ff01ffff,
+    0x01010101ff01ff01,
+    0x01010101ff0101ff,
+    0x01010101ff010101,
+    0x010101010000ff00,
+    0x01010101000000ff,
+    0x0101010100000001,
+    0x0101010101ffffff,
+    0x0101010101ffff01,
+    0x0101010101ff01ff,
+    0x0101010101ff0101,
+    0x0101010101000000,
+    0x010101010101ffff,
+    0x010101010101ff01,
+    0x01010101010101ff,
+    0x0101010101010101,
+};
+}  // namespace iq_grids
diff --git a/apps/ggml/halide/k_quant_generators.cpp b/apps/ggml/halide/k_quant_generators.cpp
new file mode 100644
index 000000000000..72a089919960
--- /dev/null
+++ b/apps/ggml/halide/k_quant_generators.cpp
@@ -0,0 +1,80 @@
+// Generic, GeneratorParam-driven quantize/dequantize pair for GGML's
+// K-quant super-block formats (see quant_components.h's make_k_quant_scheme/
+// CombineBits/PlanarBitPack/K4ScaleMinPack/Q3KScalePack for the reusable
+// Approximation pieces this assembles). "Q2_K"/"Q3_K"/"Q4_K"/"Q5_K"/"Q6_K" are not
+// distinct C++ classes here -- they're just different GENERATOR_ARGS
+// instantiations of the same generator template, registered in
+// CMakeLists.txt as q4_k_quantize/q4_k_dequantize etc. -- following the
+// exact same shape as lookup_table_quant_generators.cpp's
+// LookupTableCodecGenerator<Direction> (see that file's header comment for
+// the full rationale: one shared configure() builds the whole pipeline once
+// from a real ImageParam, each direction just adopts whichever half applies
+// via add_input(const ImageParam&)/add_output(const Func&); generate() is
+// an empty stub).
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by() and Pipeline::compute_offline(). This
+// configure()/generate() body is identical across every *_quant_generators.cpp
+// file in this directory, so it lives in codec_generator_base.h's
+// CodecGeneratorBase<Derived, dir> instead of being repeated here -- this
+// class only needs to supply its own GeneratorParams and a build_scheme().
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+// Which of quant_components.h's make_*_scheme() factories to use, and the
+// on-disk block size that goes with it -- not derivable from a plain
+// GeneratorParam combination, since each K-quant format's field layout,
+// scale scheme, and code bit-width all differ.
+enum class Family { Q2_K,
+                    Q3_K,
+                    Q4_K,
+                    Q5_K,
+                    Q6_K };
+
+template<Direction dir>
+class KQuantCodecGenerator : public CodecGeneratorBase<KQuantCodecGenerator<dir>, dir> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::Q4_K,
+        {{"q2_k", Family::Q2_K},
+         {"q3_k", Family::Q3_K},
+         {"q4_k", Family::Q4_K},
+         {"q5_k", Family::Q5_K},
+         {"q6_k", Family::Q6_K}}};
+
+    SchemeAndBytes build_scheme() const {
+        // switch's controlling expression can't resolve GeneratorParam<T>'s
+        // implicit conversion operators unambiguously -- .value() sidesteps
+        // that by returning the plain Family directly. Each make_*_scheme()
+        // now returns its own block_bytes alongside the scheme (computed from
+        // the same field list it builds internally), so there's no byte
+        // arithmetic to duplicate here.
+        switch (family.value()) {
+        case Family::Q2_K:
+            return make_q2_k_scheme();  // {scales[16]; qs[64]; fp16 d; fp16 dmin;}
+        case Family::Q3_K:
+            return make_q3_k_scheme();  // {hmask[32]; qs[64]; scales[12]; fp16 d;}
+        case Family::Q4_K:
+            return make_q4_k_scheme();  // {fp16 d; fp16 dmin; scales[12]; qs[128];}
+        case Family::Q5_K:
+            return make_q5_k_scheme();  // {fp16 d; fp16 dmin; scales[12]; qh[32]; qs[128];}
+        case Family::Q6_K:
+            return make_q6_k_scheme();  // {ql[128]; qh[64]; scales[16]; fp16 d;}
+        }
+        _halide_internal_error << "unreachable Family\n";
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(KQuantCodecGenerator<Direction::Quantize>, k_quant_quantize)
+HALIDE_REGISTER_GENERATOR(KQuantCodecGenerator<Direction::Dequantize>, k_quant_dequantize)
diff --git a/apps/ggml/halide/k_quant_vec_dot_generator.cpp b/apps/ggml/halide/k_quant_vec_dot_generator.cpp
new file mode 100644
index 000000000000..852f94034c47
--- /dev/null
+++ b/apps/ggml/halide/k_quant_vec_dot_generator.cpp
@@ -0,0 +1,63 @@
+// Generic, family-driven vec_dot for the K-quant super-block formats, the
+// vec_dot counterpart of k_quant_generators.cpp's KQuantCodecGenerator (same
+// family set). "q4_k_vec_dot" is a PARAMS family=q4_k instantiation of this
+// one generator. Weight is a block-indexed K-quant codec, activation is the
+// block-indexed Q8_K codec; VecDotGeneratorBase splices both via
+// approximate_by/compute_offline. K-quant decode is a two-level (sub-block)
+// scale, so the per-block scale is not single-invariant -> Float schedule.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "vec_dot_generator_base.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class Family { Q2_K,
+                    Q3_K,
+                    Q4_K,
+                    Q5_K,
+                    Q6_K };
+
+class KQuantVecDotGenerator : public VecDotGeneratorBase<KQuantVecDotGenerator> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::Q4_K,
+        {{"q2_k", Family::Q2_K},
+         {"q3_k", Family::Q3_K},
+         {"q4_k", Family::Q4_K},
+         {"q5_k", Family::Q5_K},
+         {"q6_k", Family::Q6_K}}};
+
+    // Q8_K activation codec (block_q8_K = {float d; qs[256]; bsums[16]} = 292 bytes).
+    static std::unique_ptr<Halide::Approximation> q8_k_codec() {
+        return make_q8_k_scheme(256, 127, Layout::BlockIndexed).scheme;
+    }
+
+    VecDotSpec build_vec_dot() const {
+        // All K-quants: 256-element super-block, Q8_K activation, two-level
+        // scale -> Float schedule.
+        switch (family.value()) {
+        case Family::Q2_K:
+            return {make_q2_k_scheme(Layout::BlockIndexed).scheme, 84, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q3_K:
+            return {make_q3_k_scheme(Layout::BlockIndexed).scheme, 110, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q4_K:
+            return {make_q4_k_scheme(Layout::BlockIndexed).scheme, 144, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q5_K:
+            return {make_q5_k_scheme(Layout::BlockIndexed).scheme, 176, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q6_K:
+            return {make_q6_k_scheme(Layout::BlockIndexed).scheme, 210, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        }
+        _halide_internal_error << "KQuantVecDotGenerator: family not yet converted\n";
+        return {};
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(KQuantVecDotGenerator, k_quant_vec_dot)
diff --git a/apps/ggml/halide/lookup_table_quant_generators.cpp b/apps/ggml/halide/lookup_table_quant_generators.cpp
new file mode 100644
index 000000000000..3c3ca741e1de
--- /dev/null
+++ b/apps/ggml/halide/lookup_table_quant_generators.cpp
@@ -0,0 +1,116 @@
+// Generic, GeneratorParam-driven quantize/dequantize pair for GGML's
+// codebook-quantized (lookup-table) formats (see quant_components.h's
+// LookupTableQuantize/E8M0Pack for the reusable Approximation pieces this
+// assembles). "IQ4_NL"/"MXFP4" are not distinct C++ classes here -- they're
+// just different GENERATOR_ARGS instantiations of the same generator
+// template, registered in CMakeLists.txt as iq4_nl_quantize/mxfp4_quantize
+// etc. -- following the same shape as symmetric_quant_generators.cpp's
+// SymmetricCodecGenerator<Direction> (see that file's header comment for the
+// full rationale: one shared configure() builds the whole pipeline once from
+// a real ImageParam, each direction just adopts whichever half applies via
+// add_input(const ImageParam&)/add_output(const Func&); generate() is an
+// empty stub).
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by() and Pipeline::compute_offline(). This
+// configure()/generate() body is identical across every *_quant_generators.cpp
+// file in this directory, so it lives in codec_generator_base.h's
+// CodecGeneratorBase<Derived, dir> instead of being repeated here -- this
+// class only needs to supply its own GeneratorParams and a build_scheme().
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+// Which of quant_components.h's make_*_scheme() factories to use, and the
+// on-disk block size that goes with it -- not derivable from a plain
+// GeneratorParam combination the way the affine/symmetric family's
+// SchemeKind's block_bytes is, since each codebook has its own fixed layout.
+enum class Family { IQ4_NL,
+                    MXFP4,
+                    TQ2_0,
+                    TQ1_0,
+                    NVFP4,
+                    IQ2_S,
+                    IQ3_XXS,
+                    IQ3_S,
+                    IQ4_XS,
+                    IQ2_XS,
+                    IQ2_XXS,
+                    IQ1_S,
+                    IQ1_M };
+
+template<Direction dir>
+class LookupTableCodecGenerator : public CodecGeneratorBase<LookupTableCodecGenerator<dir>, dir> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::IQ4_NL,
+        {{"iq4_nl", Family::IQ4_NL},
+         {"mxfp4", Family::MXFP4},
+         {"tq2_0", Family::TQ2_0},
+         {"tq1_0", Family::TQ1_0},
+         {"nvfp4", Family::NVFP4},
+         {"iq2_s", Family::IQ2_S},
+         {"iq3_xxs", Family::IQ3_XXS},
+         {"iq3_s", Family::IQ3_S},
+         {"iq4_xs", Family::IQ4_XS},
+         {"iq2_xs", Family::IQ2_XS},
+         {"iq2_xxs", Family::IQ2_XXS},
+         {"iq1_s", Family::IQ1_S},
+         {"iq1_m", Family::IQ1_M}}};
+
+    SchemeAndBytes build_scheme() const {
+        // switch's controlling expression can't resolve GeneratorParam<T>'s
+        // implicit conversion operators unambiguously -- .value() sidesteps
+        // that by returning the plain Family directly. Every make_*_scheme()
+        // now returns its own block_bytes alongside the scheme (from its
+        // field table, or -- for the IQ grid leaves, which are deliberately
+        // NOT field-table-decomposed; see quant_components.h section 6's
+        // design note -- the hand-verified constant declared next to the
+        // leaf), so there's no literal byte count to keep in sync here.
+        switch (family.value()) {
+        case Family::IQ4_NL:
+            return make_iq4_nl_scheme();
+        case Family::MXFP4:
+            return make_mxfp4_scheme();
+        case Family::TQ2_0:
+            return make_tq2_0_scheme();
+        case Family::TQ1_0:
+            return make_tq1_0_scheme();
+        case Family::NVFP4:
+            return make_nvfp4_scheme();
+        case Family::IQ2_S:
+            return make_iq2_s_scheme();  // {fp16 d; qs[32]; signs[32]; qh[8]; scales[8];}
+        case Family::IQ3_XXS:
+            return make_iq3_xxs_scheme();  // {fp16 d; qs[64]; scales_and_signs[32];}
+        case Family::IQ3_S:
+            return make_iq3_s_scheme();  // {fp16 d; qs[64]; qh[8]; signs[32]; scales[4];}
+        case Family::IQ4_XS:
+            return make_iq4_xs_scheme();  // {fp16 d; scales_h[2]; scales_l[4]; qs[128];}
+        // Importance-matrix-only (dequantize direction only -- no quantize
+        // library is built for these; SeveredEncode stands in for the missing
+        // forward map).
+        case Family::IQ2_XS:
+            return make_iq2_xs_scheme();
+        case Family::IQ2_XXS:
+            return make_iq2_xxs_scheme();
+        case Family::IQ1_S:
+            return make_iq1_s_scheme();
+        case Family::IQ1_M:
+            return make_iq1_m_scheme();
+        }
+        _halide_internal_error << "unreachable Family\n";
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(LookupTableCodecGenerator<Direction::Quantize>, lookup_table_quantize)
+HALIDE_REGISTER_GENERATOR(LookupTableCodecGenerator<Direction::Dequantize>, lookup_table_dequantize)
diff --git a/apps/ggml/halide/lookup_table_vec_dot_generator.cpp b/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
new file mode 100644
index 000000000000..609b250e0ea4
--- /dev/null
+++ b/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
@@ -0,0 +1,114 @@
+// Generic, family-driven vec_dot for the codebook/grid formats, the vec_dot
+// counterpart of lookup_table_quant_generators.cpp's LookupTableCodecGenerator
+// (same family set). "iq4_nl_vec_dot" is not a distinct C++ class -- it's a
+// PARAMS family=iq4_nl instantiation of this one generator, registered in
+// CMakeLists.txt. Weight and activation are both block-indexed codecs from
+// quant_components.h; VecDotGeneratorBase splices them via approximate_by/
+// compute_offline (see vec_dot_generator_base.h).
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "vec_dot_generator_base.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class Family { IQ4_NL,
+                    MXFP4,
+                    TQ2_0,
+                    TQ1_0,
+                    NVFP4,
+                    IQ2_S,
+                    IQ3_XXS,
+                    IQ3_S,
+                    IQ4_XS,
+                    IQ2_XS,
+                    IQ2_XXS,
+                    IQ1_S,
+                    IQ1_M };
+
+class LookupTableVecDotGenerator : public VecDotGeneratorBase<LookupTableVecDotGenerator> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::IQ4_NL,
+        {{"iq4_nl", Family::IQ4_NL},
+         {"mxfp4", Family::MXFP4},
+         {"tq2_0", Family::TQ2_0},
+         {"tq1_0", Family::TQ1_0},
+         {"nvfp4", Family::NVFP4},
+         {"iq2_s", Family::IQ2_S},
+         {"iq3_xxs", Family::IQ3_XXS},
+         {"iq3_s", Family::IQ3_S},
+         {"iq4_xs", Family::IQ4_XS},
+         {"iq2_xs", Family::IQ2_XS},
+         {"iq2_xxs", Family::IQ2_XXS},
+         {"iq1_s", Family::IQ1_S},
+         {"iq1_m", Family::IQ1_M}}};
+
+    // Q8_0 activation codec (block_q8_0 = {fp16 d; qs[32]} = 34 bytes).
+    static std::unique_ptr<Halide::Approximation> q8_0_codec() {
+        return make_symmetric_block_scheme(32, 127, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
+    }
+    // Q8_K activation codec (block_q8_K = {float d; qs[256]; bsums[16]} = 292 bytes).
+    static std::unique_ptr<Halide::Approximation> q8_k_codec() {
+        return make_q8_k_scheme(256, 127, Layout::BlockIndexed).scheme;
+    }
+
+    VecDotSpec build_vec_dot() const {
+        switch (family.value()) {
+        case Family::IQ4_NL:
+            // 4-bit codebook, single fp16 scale x Q8_0: single per-block scale
+            // and int8 codebook values -> SDOT-eligible in principle. See the
+            // TODO in symmetric_vec_dot_generator.cpp: mature hoist_invariants()
+            // can't lift the scale through approximate_by()'s round-trip
+            // replacement, so use the correct Float schedule for now.
+            return {make_iq4_nl_scheme(Layout::BlockIndexed).scheme, 18, q8_0_codec(), 34, 32, ScheduleKind::Float};
+        case Family::MXFP4:
+            // Same single-scale codebook shape as IQ4_NL (E8M0 scale) x Q8_0;
+            // same SDOT/hoist_invariants limitation -> Float for now.
+            return {make_mxfp4_scheme(Layout::BlockIndexed).scheme, 17, q8_0_codec(), 34, 32, ScheduleKind::Float};
+        case Family::NVFP4:
+            // 64-element block (4 sub-scales) x Q8_0 (32-block): the activation
+            // is Reblocked 32 -> 64 so both share the weight's block. Sub-block
+            // scales -> Float.
+            return {make_nvfp4_scheme(Layout::BlockIndexed).scheme, 36, reblock_activation(q8_0_codec(), 32, 64), 34, 64, ScheduleKind::Float};
+        // TQ1_0/TQ2_0 x Q8_K, IQ4_XS x Q8_K: single fp16 scale (TQ) or two-level
+        // scale (IQ4_XS) -> Float schedule for now.
+        case Family::TQ2_0:
+            return {make_tq2_0_scheme(Layout::BlockIndexed).scheme, 66, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::TQ1_0:
+            return {make_tq1_0_scheme(Layout::BlockIndexed).scheme, 54, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ4_XS:
+            return {make_iq4_xs_scheme(Layout::BlockIndexed).scheme, 136, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        // Grid formats x Q8_K: per-group scale + sign -> Float schedule. The
+        // block-indexed codec collapses the leaf's {8,4,8} output to (kk, blk).
+        case Family::IQ2_S:
+            return {make_iq2_s_scheme(Layout::BlockIndexed).scheme, 82, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ3_XXS:
+            return {make_iq3_xxs_scheme(Layout::BlockIndexed).scheme, 98, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ3_S:
+            return {make_iq3_s_scheme(Layout::BlockIndexed).scheme, 110, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        // Importance-matrix-only formats (SeveredEncode weight scheme) x Q8_K.
+        case Family::IQ2_XS:
+            return {make_iq2_xs_scheme(Layout::BlockIndexed).scheme, 74, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ2_XXS:
+            return {make_iq2_xxs_scheme(Layout::BlockIndexed).scheme, 66, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ1_S:
+            return {make_iq1_s_scheme(Layout::BlockIndexed).scheme, 50, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ1_M:
+            return {make_iq1_m_scheme(Layout::BlockIndexed).scheme, 56, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        default:
+            break;
+        }
+        _halide_internal_error << "LookupTableVecDotGenerator: family not yet converted\n";
+        return {};
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(LookupTableVecDotGenerator, lookup_table_vec_dot)
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
new file mode 100644
index 000000000000..11517cfef869
--- /dev/null
+++ b/apps/ggml/halide/quant_components.h
@@ -0,0 +1,3011 @@
+#pragma once
+
+// Reusable Approximation components for GGML-style per-block quantized
+// weight formats -- see doc/ApproximationDesign.md and the plan this file
+// implements for the rationale. Every weight format is built by composing
+// these kinds of pieces via Halide::Compose/Halide::Apply (and, for the
+// extern-delegated formats, Halide::TrustedInverse) into a scheme (see the
+// make_*_scheme() factory functions below), which the
+// Generators then splice in via Func::approximate_by()/
+// Pipeline::compute_offline() -- never by calling Approximation::encode()/
+// decode() directly.
+//
+//   1. BlockReshape -- lossless relayout: flat values <-> (kk, blk).
+//   2. SymmetricAffineQuantize/AffineQuantize -- the actual lossy step:
+//      block values <-> (integer codes, one or two float(s) per block).
+//   3a. Fp16Pack/PlanarBitPack/BytePack -- per-field packing:
+//      a typed field (codes, scale, min) <-> its own on-disk byte encoding.
+//   3b. AppendSums -- a derived extra field, computed from other
+//      already-encoded fields rather than from the original values.
+//   3c. StructPack -- concatenates N already-packed fields into one
+//      byte-addressed buffer, matching a specific on-disk block layout
+//      (e.g. block_q4_0).
+//   4. Extern-delegated formats (codebook/K-quant/IQ grid/IQ4_XS): their
+//      quantize is an opaque GGML extern (ExternQuantize), paired with a
+//      compositional dequantize via Halide::TrustedInverse. The extra
+//      decode-only math leaves those need -- Codebook (codes -> table[codes]),
+//      LinearDequant (the scale multiply), CombineBits
+//      (K-quant split codes) -- live in section 4 alongside ExternQuantize.
+//
+// None of these know about any specific GGML type name -- "Q4_0"/"Q4_1"/etc.
+// are just particular parameter choices, assembled where the Generators live
+// (symmetric_quant_generators.cpp), not encoded here.
+//
+// None of these components call Func::bound() on their own intermediate
+// Funcs, even where a range is intrinsically known (e.g. codes(kk, blk) for
+// kk in [0, block_size)): with everything left at its default (inline)
+// schedule, as it is here, Halide already infers the true required range by
+// propagating backward from wherever the final packed buffer is actually
+// realized or scheduled (a Generator's Output dim bounds, or an explicit
+// realize() shape) -- an explicit bound() on an inlined Func is simply
+// ignored ("meaningless... because the function is scheduled inline", per
+// Halide's own warning). bound() only becomes useful once a component's Func
+// is deliberately scheduled non-inline (compute_root/compute_at), which is a
+// scheduling-time decision made where that happens, not decided here.
+//
+// DIMENSION / WILDCARD CONVENTION: a component's Func indices are laid out as
+// (field dims..., blk, lane dims...) -- the field's own within-block dims
+// first (kk, or byte, or (plane, sub), ...), then the block index blk, then
+// any trailing "lane" dims (e.g. a repack matmul weight's column-in-group j
+// and col-group x), carried by the Halide::_ placeholder. Decode-only stages
+// should be lane-general by default: write decode() with a trailing
+// Halide::_ on both sides (f(kk, blk, _) = g(..., blk, _)) so the same
+// component runs unchanged whether there are zero lane dims (the plain
+// (kk, blk) codecs) or several (the repack weight schemes). Encode stages
+// stay at fixed arity -- every encoder here runs on a plain (kk, blk) or
+// flat row, so there is nothing for a wildcard to carry. Currently
+// lane-general (decode side): Fp16Pack, F32Pack, E8M0Pack, BytePack,
+// PlanarBitPack (both normal and plane-axis modes), Codebook, LinearDequant,
+// K4ScaleMinPack, Q3KScalePack, IQ4XSScalePack. The remaining decoders
+// (BlockReshape/Reblock, the repack interleave/de-interleave leaves, the IQ
+// grid leaves, TritPack, BitPack, Int16Pack, UE4M3Pack) are fixed-arity by
+// design or have no lane-general consumer yet. The full dimension-general
+// BlockLayout sketched in the DESIGN NOTE by Reblock (splitting/permuting
+// arbitrary dims) remains deliberately deferred.
+
+// Only the aggregated Halide.h is installed for apps to consume (individual
+// per-class headers like Approximation.h are not) -- it already pulls in
+// Approximation/Compose/Apply/Pipeline::compute_offline.
+#include "Halide.h"
+
+#include <algorithm>
+
+#include "iq_grids_data.h"
+
+namespace ggml_halide {
+
+// The result of a make_*_scheme() factory: the scheme itself plus its
+// on-disk block byte count, computed once from the same field list
+// make_block_layout() (see FieldSpec below) already sums, rather than
+// hand-summed again at each Generator call site. Shared with
+// codec_generator_base.h's CodecGeneratorBase (which is what actually
+// consumes it -- see there).
+struct SchemeAndBytes {
+    std::unique_ptr<Halide::Approximation> scheme;
+    int block_bytes;
+};
+
+// Every make_*_scheme() factory below takes a Layout, selecting what its
+// outermost BlockReshape (or grid BlockReshape) does with the "flat" side:
+//   - FlatRow (the default): a fully-flat 1-D row -- the shape
+//     quantize_row/dequantize_row Generators want.
+//   - BlockIndexed: a passthrough (kk, blk) -- the shape a vec_dot/repack
+//     Generator wants (its own reduction already runs over (kk, blk); no
+//     flat<->block reshape is needed on top). This used to be a *separate*
+//     make_*_codec() function per scheme (build the codec, skip the
+//     reshape); now it's the same factory with BlockReshape's own
+//     block_indexed flag set true, which makes it a lossless identity
+//     passthrough (see BlockReshape's own comment) -- so there's no
+//     behavioral difference, just one fewer named entry point per scheme.
+enum class Layout { FlatRow,
+                    BlockIndexed };
+
+// ---------------------------------------------------------------------------
+// 1. Lossless relayout.
+// ---------------------------------------------------------------------------
+
+// A lossless flat <-> block reshape. In the common one-dimensional case
+// (BlockReshape(block_size)), packed(kk, blk) = flat(blk*block_size + kk) --
+// one within-block index kk in [0, block_size), one block index blk.
+//
+// The general case (BlockReshape({e0, e1, ...})) unflattens the within-block
+// index into *several* dimensions, innermost/fastest-varying first, so a
+// component whose values have nested block structure can index those
+// dimensions directly instead of re-deriving them from a flat kk via div/mod.
+// E.g. an IQ 256-element superblock structured as group(8) x l(4) x elem(8)
+// uses extents {8, 4, 8}: packed(elem, l, group, blk), where the flat
+// within-block index kk = elem + 8*l + 32*group (product of extents = block
+// size). The single-int constructor is exactly the one-extent case.
+//
+// `block_indexed` selects what the "flat" side looks like:
+//   - false (default): a fully-flat 1-D row f(k), k = blk*block_size + within
+//     -- the shape quantize_row/dequantize_row want.
+//   - true: a block-indexed 2-D f(kk, blk), within-block index kept separate
+//     from the block index -- the shape a per-block vec_dot reduction wants
+//     (so the block index stays a distinct RVar for the SDOT rfactor hoist).
+// In block-indexed mode a single-extent reshape is a (kk,blk) passthrough,
+// and a multi-extent one collapses the nested dims (elem,l,group,blk) into
+// (kk,blk) -- the only difference from flat mode is folding blk into k or not.
+class BlockReshape : public Halide::Approximation {
+public:
+    explicit BlockReshape(int block_size, bool block_indexed = false)
+        : extents_{block_size}, block_indexed_(block_indexed) {
+    }
+    explicit BlockReshape(std::vector<int> extents, bool block_indexed = false)
+        : extents_(std::move(extents)), block_indexed_(block_indexed) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func flat = inputs[0];  // f(k), or f(kk, blk) when block_indexed_
+        std::vector<Var> dims = block_vars();
+        Var blk("blk");
+
+        // packed(d0, d1, ..., blk) = flat(<within>), within = d0 + e0*d1 + ...
+        Expr within = cast<int>(0);
+        int stride = 1;
+        for (size_t i = 0; i < dims.size(); i++) {
+            within += dims[i] * stride;
+            stride *= extents_[i];
+        }
+        std::vector<Var> args = dims;
+        args.push_back(blk);
+        Func packed("block_reshape_packed");
+        packed(args) = block_indexed_ ? flat(within, blk) : flat(blk * block_size() + within);
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];
+        Var k("k"), kk("kk"), blk("blk");
+
+        // Read packed(within%e0, (within/e0)%e1, ..., block); the within-block
+        // index and block index come from either a flat k or a (kk, blk) pair.
+        Expr within = block_indexed_ ? (Expr)kk : k % block_size();
+        Expr block = block_indexed_ ? (Expr)blk : k / block_size();
+        std::vector<Expr> args;
+        Expr rem = within;
+        for (int e : extents_) {
+            args.push_back(rem % e);
+            rem = rem / e;
+        }
+        args.push_back(block);
+        Func out("block_reshape_unpacked");
+        if (block_indexed_) {
+            out(kk, blk) = packed(args);
+        } else {
+            out(k) = packed(args);
+        }
+        return {{out}, {}};
+    }
+
+private:
+    std::vector<int> extents_;
+    bool block_indexed_;
+
+    int block_size() const {
+        int p = 1;
+        for (int e : extents_) {
+            p *= e;
+        }
+        return p;
+    }
+    // One Var per within-block dimension; the familiar "kk" in the common
+    // one-dimensional case, "d0"/"d1"/... otherwise.
+    std::vector<Halide::Var> block_vars() const {
+        std::vector<Halide::Var> vs;
+        for (size_t i = 0; i < extents_.size(); i++) {
+            vs.push_back(extents_.size() == 1 ? Halide::Var("kk") : Halide::Var("d" + std::to_string(i)));
+        }
+        return vs;
+    }
+};
+
+// ---------------------------------------------------------------------------
+// Lossless block-layout relayouts (Reblock, and the repack Interleave below).
+//
+// DESIGN NOTE (intended library form, deferred): the clean, general shape for
+// these is a *dimension-general* block-relayout -- a component that splits and
+// permutes arbitrary index dimensions, carrying any trailing dims through
+// untouched (the way the Python research sketch's BlockLayout(splits=...)/
+// SplitStorage do, via Halide's `_` placeholder), quantizing/packing whatever
+// falls out. That would let a single `BlockLayout` utility live in the core
+// Approximation library and compose in front of *any* lossy quant, with the
+// repack interleave being just one instantiation. We deliberately do NOT do
+// that here: `Func`/`Var` `_` wildcard semantics were a source of trouble, and
+// pinning them down is its own rabbit hole. Instead these relayouts are
+// written at fixed, concrete arities (folding any extra "lane" -- e.g. repack's
+// 4 interleaved rows -- into the block index blk), reusing the existing
+// (kk, blk) quant/pack components unchanged. When the wildcard story is sorted,
+// these should graduate to the dimension-general form.
+// ---------------------------------------------------------------------------
+
+// Losslessly re-view a block-indexed Func at a different block size (the flat
+// element order is unchanged; only the (kk, blk) factorization differs).
+// decode(): (kk_from, blk_from) at `from_block` -> (kk_to, blk_to) at
+// `to_block`, reading the same global element g = blk_to*to_block + kk_to from
+// its source position (g % from_block, g / from_block). encode() is the
+// mirror. This is what lets a vec_dot present an activation stored in its own
+// (smaller) block size -- e.g. Q8_0's 32-element blocks -- at a weight's
+// (larger) block size (Q1_0's 128, NVFP4's 64), so both operands share one
+// (kk, blk) and the Generator's reduction stays uniform. The block-structure
+// reconciliation lives here, in an Approximation, not open-coded in the
+// reduction.
+class Reblock : public Halide::Approximation {
+public:
+    Reblock(int from_block, int to_block)
+        : from_(from_block), to_(to_block) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func in = inputs[0];  // (kk, blk) at to_block
+        Var kk("kk"), blk("blk");
+        Expr g = blk * from_ + kk;
+        Func out("reblock_encoded");
+        out(kk, blk) = in(g % to_, g / to_);
+        return {{out}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func in = encoded[0];  // (kk, blk) at from_block
+        Var kk("kk"), blk("blk");
+        Expr g = blk * to_ + kk;
+        Func out("reblock_decoded");
+        out(kk, blk) = in(g % from_, g / from_);
+        return {{out}, {}};
+    }
+
+private:
+    int from_, to_;
+};
+
+// An activation codec that decodes to (kk, blk) at `to_block`: `act_codec`
+// (block-indexed at the activation's own `from_block`) composed with a Reblock
+// when the two differ, else `act_codec` unchanged. Lets a vec_dot pair a weight
+// of one block size against an activation of another (Q1_0/NVFP4 x Q8_0).
+inline std::unique_ptr<Halide::Approximation> reblock_activation(
+    std::unique_ptr<Halide::Approximation> act_codec, int from_block, int to_block) {
+    if (from_block == to_block) {
+        return act_codec;
+    }
+    return std::make_unique<Halide::Compose>(std::move(act_codec), Reblock{from_block, to_block});
+}
+
+// ---------------------------------------------------------------------------
+// 2. The lossy step.
+// ---------------------------------------------------------------------------
+
+// GGML's own reference quantizers round differently depending on the target
+// bit width, not out of taste but because they use different formulas:
+//   - Nearest: plain round-half-away-from-zero (Q8_0's quantize_row_q8_0_ref
+//     uses roundf()). Halide's round() matches this exactly.
+//   - TruncateHalfUpWithOffset: a truncate-based "+qmax+0.5f then cast"
+//     trick used by nibble-packed formats (Q4_0's quantize_row_q4_0_ref).
+//     Verified by hand this is round-half-*up*, not round-half-away-from-
+//     zero: floor(x+8.5) at x=-0.5 gives 8 (rounds toward +inf), whereas
+//     round-half-away-from-zero would give 7.
+//   - SignOnly: code = sign(x0) in {-1, +1}, ignoring magnitude entirely --
+//     Q1_0's actual quantizer (1-bit codes, no rounding to speak of; paired
+//     with ScaleAnchor::MeanAbs below, not qmax-based like every other
+//     anchor).
+//   - NearestEvenClampedHigh: round-half-to-even (not round-half-away-from-
+//     zero like Nearest), then clamp only the high end to qmax -- Q8_K's
+//     actual quantizer. GGML computes this via a magic-number float trick
+//     (nearest_int(), reproduced by nearest_int() below) that exploits the
+//     default IEEE-754 round-to-nearest-even rounding of the addition
+//     itself; Halide has no round-to-even primitive exposed, so the same
+//     bit trick is used here to match bit-for-bit. Always paired with
+//     ScaleAnchor::ExtremeSignedValueTwoStep below.
+enum class RoundingMode { Nearest,
+                          TruncateHalfUpWithOffset,
+                          SignOnly,
+                          NearestEvenClampedHigh };
+
+// Same magic-number trick as GGML's static inline nearest_int() in
+// src/ggml-quants.c: adding 1.5*2^23 forces the CPU's default round-to-
+// nearest-even addition to round fval's fractional part, then the rounded
+// integer is recovered from the float's mantissa bits.
+inline Halide::Expr nearest_int(Halide::Expr fval) {
+    using namespace Halide;
+    Expr val = fval + 12582912.0f;  // 1.5 * 2^23
+    Expr bits = reinterpret<int32_t>(val);
+    return (bits & 0x007fffff) - 0x00400000;
+}
+
+// How a block's scale is derived from its values -- this is a second,
+// independent axis GGML varies per format, not just rounding:
+//   - AbsMax: scale = max(|v|) / qmax -- ordinary symmetric quantization
+//     (Q8_0).
+//   - ExtremeSignedValue: scale = -extreme / qmax, where `extreme` is the
+//     *signed* value with the largest magnitude in the block (ties keep the
+//     first-seen value, matching GGML's single left-to-right loop with a
+//     strict '<' comparison). This deliberately anchors the block's most
+//     extreme value at code -qmax, using the full negative side of an
+//     asymmetric signed range like [-8, 7] (Q4_0).
+//   - MeanAbs: scale = mean(|v|) over the block (a sum reduction divided by
+//     block_size, not a max reduction divided by qmax) -- Q1_0's anchor,
+//     always paired with RoundingMode::SignOnly.
+//   - ExtremeSignedValueTwoStep: mathematically the same value as
+//     ExtremeSignedValue (scale = -extreme/qmax), but computed as GGML's
+//     own two *separate* divisions -- `iscale = -qmax/extreme` first, then
+//     `scale = 1/iscale` -- rather than the algebraically-equivalent single
+//     multiply above. Floating point isn't associative, so these round
+//     differently in the last bit; `iscale` itself (not a fresh 1/scale
+//     recomputed afterward) is also what SymmetricAffineQuantize::encode()
+//     uses to derive codes for this anchor, to stay bit-exact with GGML's
+//     quantize_row_q8_K_ref -- see encode()'s `id`/`scale` computation.
+//     Always paired with RoundingMode::NearestEvenClampedHigh.
+enum class ScaleAnchor { AbsMax,
+                         ExtremeSignedValue,
+                         MeanAbs,
+                         ExtremeSignedValueTwoStep };
+
+// encode(): block(kk, blk) -> {codes(kk, blk) in [-qmax, qmax-1], scale(blk)}.
+// decode(): {codes, scale} -> cast<float>(codes) * scale -- this half is
+// exactly the same regardless of rounding/anchor (both Q4_0's and Q8_0's
+// existing hand-written dequantize math already reduce to this one formula).
+class SymmetricAffineQuantize : public Halide::Approximation {
+public:
+    SymmetricAffineQuantize(int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor)
+        : block_size_(block_size), qmax_(qmax), rounding_(rounding), anchor_(anchor) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func block = inputs[0];  // block(kk, blk)
+        Var kk("kk"), blk("blk");
+        RDom r(0, block_size_, "r");
+
+        Func stat("affine_quantize_stat");
+        Func scale("affine_quantize_scale");
+        Func id("affine_quantize_id");
+        auto define_extreme_signed_stat = [&]() {
+            stat(blk) = Tuple(0.0f, 0.0f);  // {amax, extreme_signed}
+            Expr v = block(r, blk);
+            Expr take = abs(v) > stat(blk)[0];
+            stat(blk) = Tuple(select(take, abs(v), stat(blk)[0]),
+                              select(take, v, stat(blk)[1]));
+        };
+        if (anchor_ == ScaleAnchor::AbsMax) {
+            stat(blk) = 0.0f;
+            stat(blk) = max(stat(blk), abs(block(r, blk)));
+            scale(blk) = stat(blk) / (float)qmax_;
+            id(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else if (anchor_ == ScaleAnchor::ExtremeSignedValue) {
+            define_extreme_signed_stat();
+            scale(blk) = stat(blk)[1] * (-1.0f / (float)qmax_);
+            id(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else if (anchor_ == ScaleAnchor::MeanAbs) {
+            stat(blk) = 0.0f;
+            stat(blk) += abs(block(r, blk));
+            scale(blk) = stat(blk) / (float)block_size_;
+            id(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else {  // ExtremeSignedValueTwoStep
+            define_extreme_signed_stat();
+            // `id` (== GGML's `iscale`) is computed FIRST here, and `scale`
+            // is derived from it -- the reverse order of every other
+            // anchor above -- because GGML's own reference computes
+            // `iscale = -qmax/extreme` then `d = 1/iscale` as two
+            // *separate* divisions, and quantizes using `iscale` directly.
+            // Re-deriving `id` as `1/scale` afterward (like every other
+            // anchor does) would round through an extra reciprocal
+            // (`1/(1/iscale)`) that isn't guaranteed to reproduce `iscale`
+            // bit-for-bit.
+            id(blk) = select(stat(blk)[0] == 0.0f, 0.0f, (-1.0f * (float)qmax_) / stat(blk)[1]);
+            scale(blk) = select(id(blk) != 0.0f, 1.0f / id(blk), 0.0f);
+        }
+        // stat has an update definition, so it must be scheduled somewhere
+        // (Halide can't inline it) -- like SymmetricRowQuantize's `amax` in
+        // approximation_composition.cpp, that's left to the caller via
+        // `handles`, not decided here.
+
+        Expr x0 = block(kk, blk) * id(blk);
+
+        Func codes("affine_quantize_codes");
+        if (rounding_ == RoundingMode::Nearest) {
+            // Matches Q8_0's actual (bit-exact-verified) reference: no
+            // explicit clamp, since id was derived so |x0| doesn't exceed
+            // qmax in practice.
+            codes(kk, blk) = cast<int8_t>(round(x0));
+        } else if (rounding_ == RoundingMode::TruncateHalfUpWithOffset) {
+            Expr raw = cast<int32_t>(cast<int8_t>(x0 + (float)qmax_ + 0.5f));
+            codes(kk, blk) = cast<int8_t>(min(raw, 2 * qmax_ - 1) - qmax_);
+        } else if (rounding_ == RoundingMode::SignOnly) {
+            codes(kk, blk) = cast<int8_t>(select(block(kk, blk) >= 0.0f, 1, -1));
+        } else {  // NearestEvenClampedHigh
+            Expr q_raw = nearest_int(x0);
+            codes(kk, blk) = cast<int8_t>(min(qmax_, q_raw));
+        }
+
+        return {{codes, scale}, {stat}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func codes = encoded[0], scale = encoded[1];
+        Var kk("kk"), blk("blk");
+        Func dequantized("affine_dequantized");
+        dequantized(kk, blk) = cast<float>(codes(kk, blk)) * scale(blk);
+        return {{dequantized}, {}};
+    }
+
+private:
+    int block_size_, qmax_;
+    RoundingMode rounding_;
+    ScaleAnchor anchor_;
+};
+
+// How AffineQuantize rounds+truncates code = round((x-min)*id) into its
+// final representable range -- a different formula than SymmetricAffineQuantize's
+// RoundingMode, and not a variation on it: there's no centering/offset here
+// (codes are naturally unsigned starting at 0), and GGML's two affine legacy
+// formats don't even agree on whether to clamp at all:
+//   - ClampedInt8: (int8_t)(v+0.5f), then an explicit min(.., levels) --
+//     Q4_1's exact formula.
+//   - UnclampedUint8: (uint8_t)(v+0.5f) directly, no further clamp -- Q5_1's
+//     exact formula. GGML's own reference genuinely doesn't clamp this one
+//     (verified against quantize_row_q5_1_ref); reproduced faithfully since
+//     quantize output is checked bit-exact.
+enum class AffineRounding { ClampedInt8,
+                            UnclampedUint8 };
+
+// encode(): block(kk, blk) -> {codes(kk, blk) in [0, levels], scale(blk),
+// min(blk)}. decode(): {codes, scale, min} -> cast<float>(codes)*scale + min.
+// The min-max (not max-abs) scale derivation is what makes this "affine"
+// rather than "symmetric" -- every value in a block is representable, not
+// just those centered on zero, at the cost of needing a second per-block
+// float (Q4_1/Q5_1's 'm').
+class AffineQuantize : public Halide::Approximation {
+public:
+    AffineQuantize(int block_size, int levels, AffineRounding rounding)
+        : block_size_(block_size), levels_(levels), rounding_(rounding) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func block = inputs[0];  // block(kk, blk)
+        Var kk("kk"), blk("blk");
+        RDom r(0, block_size_, "r");
+
+        // Plain min/max reduction -- unlike SymmetricAffineQuantize's
+        // ScaleAnchor::ExtremeSignedValue, min and max are independent here,
+        // forming an affine (not centered-on-zero) range.
+        Func stat("affine_quantize_minmax");
+        stat(blk) = Tuple(std::numeric_limits<float>::max(), std::numeric_limits<float>::lowest());
+        Expr v = block(r, blk);
+        stat(blk) = Tuple(min(stat(blk)[0], v), max(stat(blk)[1], v));
+
+        Func scale("affine_quantize_scale");
+        scale(blk) = (stat(blk)[1] - stat(blk)[0]) / (float)levels_;
+        Func minv("affine_quantize_min");
+        minv(blk) = stat(blk)[0];
+
+        Expr id = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        Expr x0 = (block(kk, blk) - minv(blk)) * id;
+
+        Func codes("affine_quantize_codes");
+        if (rounding_ == AffineRounding::ClampedInt8) {
+            Expr raw = cast<int32_t>(cast<int8_t>(x0 + 0.5f));
+            codes(kk, blk) = cast<int8_t>(min(raw, levels_));
+        } else {
+            codes(kk, blk) = cast<int8_t>(cast<uint8_t>(x0 + 0.5f));
+        }
+
+        return {{codes, scale, minv}, {stat}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func codes = encoded[0], scale = encoded[1], minv = encoded[2];
+        Var kk("kk"), blk("blk");
+        Func dequantized("affine_dequantized_am");
+        dequantized(kk, blk) = cast<float>(codes(kk, blk)) * scale(blk) + minv(blk);
+        return {{dequantized}, {}};
+    }
+
+private:
+    int block_size_, levels_;
+    AffineRounding rounding_;
+};
+
+// ---------------------------------------------------------------------------
+// 3a. Per-field bit packing.
+// ---------------------------------------------------------------------------
+
+// Assemble a little-endian integer word starting at bytes(base, blk). The
+// uint32 specialization below is the on-disk byte order shared by F32Pack's
+// scale and IQ3_XXS's/IQ2_XXS's aux32.
+inline Halide::Expr le_uint(Halide::Func bytes, Halide::Expr base, Halide::Var blk, int byte_count) {
+    using namespace Halide;
+    Expr result = cast<uint32_t>(0);
+    for (int i = 0; i < byte_count; i++) {
+        result = result | (cast<uint32_t>(bytes(base + i, blk)) << (8 * i));
+    }
+    return result;
+}
+
+inline Halide::Expr le_u32(Halide::Func bytes, Halide::Expr base, Halide::Var blk) {
+    using namespace Halide;
+    return le_uint(bytes, base, blk, 4);
+}
+
+// le_uint's encode-side mirror: byte `byte_idx` (little-endian) of `bits`.
+// Widening to uint32 first makes one expression work for every word size
+// used below (Fp16Pack's 16-bit word, F32Pack's 32-bit one, Int16Pack's
+// per-group 16-bit one) -- shifting a narrower type by up to 24 bits would
+// silently truncate instead.
+inline Halide::Expr word_to_le_byte(Halide::Expr bits, Halide::Expr byte_idx) {
+    using namespace Halide;
+    return cast<uint8_t>((cast<uint32_t>(bits) >> (cast<uint32_t>(byte_idx) * 8)) & 0xff);
+}
+
+// le_uint's encode-side-agnostic generalization: a little-endian 16-bit word
+// from two bytes, but via an arbitrary per-byte accessor (`byte_at(0)` = low
+// byte, `byte_at(1)` = high byte) instead of a fixed `Func bytes` indexed at
+// (base+i, blk) -- so it also covers a Halide::_-general decoder's
+// bytes(offset, blk, _) reads, or a byte pair that isn't a plain Func index
+// at all (e.g. IQ1_M's per-scale-word accumulation further below). Replaces
+// the hand-rolled "lo | (hi << 8)" pattern that used to be written out at
+// each call site.
+template<typename ByteAt>
+inline Halide::Expr le_u16(ByteAt byte_at) {
+    using namespace Halide;
+    Expr lo = cast<uint16_t>(byte_at(0));
+    Expr hi = cast<uint16_t>(byte_at(1));
+    return cast<uint16_t>(lo | (hi << 8));
+}
+
+// encode(float scale) -> 2 bytes; decode(2 bytes) -> float. Matches every
+// format's existing fp16 delta-byte code (byte0 = bits&0xff, byte1 =
+// (bits>>8)&0xff).
+class Fp16Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func scale = inputs[0];  // scale(blk)
+        Var byte("byte"), blk("blk");
+        Expr bits = reinterpret<uint16_t>(cast<float16_t>(scale(blk)));
+        Func bytes("fp16_pack_bytes");
+        bytes(byte, blk) = word_to_le_byte(bits, byte);
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk[, ...]), byte in [0, 2)
+        Var blk("blk");
+        Expr bits = le_u16([&](int i) { return bytes(i, blk, _); });
+        Func scale("fp16_pack_scale");
+        scale(blk, _) = cast<float>(reinterpret<float16_t>(bits));
+        return {{scale}, {}};
+    }
+};
+
+// encode(float scale) -> 4 bytes (plain IEEE-754 binary32, little-endian);
+// decode(4 bytes) -> float -- Q8_K's scale format, the one format here whose
+// delta is a full float, not fp16 (matching block_q8_K's `float d;`, not
+// GGML's usual `ggml_fp16_t d;`).
+class F32Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func scale = inputs[0];  // scale(blk)
+        Var byte("byte"), blk("blk");
+        Expr bits = reinterpret<uint32_t>(scale(blk));
+        Func bytes("f32_pack_bytes");
+        bytes(byte, blk) = word_to_le_byte(bits, byte);
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk[, ...]), byte in [0, 4)
+        Var blk("blk");
+        // Dimension-general via Halide::_ (matches Fp16Pack): any trailing
+        // "lane" dims -- e.g. a repack weight's columns -- ride through, so this
+        // pack can decode a repack scale field, not just a plain (byte, blk) one.
+        Expr b0 = cast<uint32_t>(bytes(0, blk, _)), b1 = cast<uint32_t>(bytes(1, blk, _));
+        Expr b2 = cast<uint32_t>(bytes(2, blk, _)), b3 = cast<uint32_t>(bytes(3, blk, _));
+        Func scale("f32_pack_scale");
+        scale(blk, _) = reinterpret<float>(b0 | (b1 << 8) | (b2 << 16) | (b3 << 24));
+        return {{scale}, {}};
+    }
+};
+
+// encode(int16 values(g, blk)) -> 2 little-endian bytes each, byte `2g`/
+// `2g+1` -- e.g. Q8_K's bsums[16] int16 array (one value per 16-element
+// group, not one per block).
+class Int16Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func values = inputs[0];  // values(g, blk)
+        Var byte_idx("byte_idx"), blk("blk");
+        Expr g = byte_idx / 2;
+        Expr bits = reinterpret<uint16_t>(values(g, blk));
+        Func bytes("int16_pack_bytes");
+        bytes(byte_idx, blk) = word_to_le_byte(bits, byte_idx % 2);
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk), byte_idx in [0, 2*num_groups)
+        Var g("g"), blk("blk");
+        Expr bits = le_u16([&](int i) { return bytes(2 * g + i, blk); });
+        Func values("int16_pack_values");
+        values(g, blk) = reinterpret<int16_t>(bits);
+        return {{values}, {}};
+    }
+};
+
+// decode(1 byte) -> float, an E8M0 power-of-two exponent (GGML's MXFP4/NVFP4
+// scale format). Reproduces ggml_e8m0_to_fp32_half's exact bit construction:
+// d = 2^(e-128) for every e in [0, 255], computed via a subnormal-exploiting
+// shift trick for e<2 instead of a normal exponent-field write (both branches
+// compute the same uniform 2^(e-128); see the comment inline). Decode-only:
+// MXFP4 quantize is extern-delegated (see ExternQuantize).
+class E8M0Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "E8M0Pack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func byte = encoded[0];  // byte(byte_idx, blk[, ...]), byte_idx in [0, 1)
+        Var blk("blk");
+        // Dimension-general via Halide::_ (matches Fp16Pack/F32Pack) so this pack
+        // can decode a repack weight's E8M0 scale header, columns riding through.
+        Expr e = cast<uint32_t>(byte(0, blk, _));
+        Expr bits = select(e < 2, cast<uint32_t>(0x00200000) << e, (e - 1) << 23);
+        Func scale("e8m0_pack_scale");
+        scale(blk, _) = reinterpret<float>(bits);
+        return {{scale}, {}};
+    }
+};
+
+// decode(1 byte per sub-block) -> float(sub, blk), a UE4M3 unsigned
+// 4-exponent/3-mantissa float (GGML's NVFP4 per-sub-block scale format).
+// Reproduces ggml_ue4m3_to_fp32_half's exact construction: subnormal
+// (exp==0) is man/512, normal is (1+man/8)*2^(exp-7), both halved; byte 0x00
+// or 0x7f (GGML's NVFP4 zero/sentinel bytes) decode to 0. Unlike
+// Fp16Pack/E8M0Pack (exactly one scale value per block), this is meant to be
+// used with LinearDequant's per-sub-block (sub_size > 0) mode: the Funcs here are indexed by
+// `sub` directly (one byte each). Decode-only: NVFP4 quantize is
+// extern-delegated (see ExternQuantize).
+class UE4M3Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "UE4M3Pack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func byte = encoded[0];  // byte(sub, blk)
+        Var sub("sub"), blk("blk");
+        Expr ue = byte(sub, blk);                                  // codespell:ignore ue
+        Expr is_zero = (ue == 0) || (ue == 0x7f);                  // codespell:ignore ue
+        Expr exp_ = cast<int32_t>(cast<uint32_t>(ue) >> 3) & 0xf;  // codespell:ignore ue
+        Expr man_ = cast<int32_t>(ue) & 0x7;                       // codespell:ignore ue
+        Expr raw = select(exp_ == 0,
+                          cast<float>(man_) / 512.0f,
+                          (1.0f + cast<float>(man_) / 8.0f) * pow(2.0f, cast<float>(exp_ - 7)));
+        Func scale("ue4m3_pack_scale");
+        scale(sub, blk) = select(is_zero, 0.0f, raw * 0.5f);
+        return {{scale}, {}};
+    }
+};
+
+// The four scalar-scale on-disk formats above (Fp16Pack/F32Pack/E8M0Pack/
+// UE4M3Pack), named so call sites that need to pick one at runtime (the
+// repack weight schemes' RepackWeightScale used to be a bespoke copy of
+// exactly this same enum; make_codebook_scheme's scale_pack/scale_bytes
+// parameter pair was the other) can hand a single value across instead of a
+// (unique_ptr<Approximation>, int width) pair kept in sync by hand.
+enum class ScaleFormat { Fp16,
+                         F32,
+                         E8M0,
+                         UE4M3 };
+
+inline std::unique_ptr<Halide::Approximation> make_scale_pack(ScaleFormat fmt) {
+    switch (fmt) {
+    case ScaleFormat::Fp16:
+        return std::make_unique<Fp16Pack>();
+    case ScaleFormat::F32:
+        return std::make_unique<F32Pack>();
+    case ScaleFormat::E8M0:
+        return std::make_unique<E8M0Pack>();
+    case ScaleFormat::UE4M3:
+        return std::make_unique<UE4M3Pack>();
+    }
+    _halide_internal_error << "unreachable ScaleFormat\n";
+}
+
+// On-disk byte width of `num_scales` consecutive values in `fmt` (e.g.
+// NVFP4's 4 per-sub-block UE4M3 bytes, or a repack weight's n_cols-wide
+// per-column scale header).
+inline int scale_width(ScaleFormat fmt, int num_scales = 1) {
+    switch (fmt) {
+    case ScaleFormat::Fp16:
+        return 2 * num_scales;
+    case ScaleFormat::F32:
+        return 4 * num_scales;
+    case ScaleFormat::E8M0:
+        return num_scales;
+    case ScaleFormat::UE4M3:
+        return num_scales;
+    }
+    _halide_internal_error << "unreachable ScaleFormat\n";
+}
+
+// The shared byte<->field decomposition behind every uniform-width,
+// non-base-3 bit-packed field in this file (nibble packs, 2-bit packs, and
+// the K-quants' rotating out-of-band high-bit arrays): a flat per-block
+// index kk factors as
+//   kk = outer*(plane_count*pos_count) + plane*pos_count + pos
+// and each byte packs `plane_count` fixed-width `field_bits`-bit fields
+// ("planes") -- byte `outer*pos_count + pos` holds element (outer, plane, pos)
+// at bit-shift `plane*field_bits`. This one decomposition is exactly (verified
+// by hand, not just formally) GGML's addressing for all of the following, each
+// just a different (field_bits, plane_count, pos_count) instantiated directly
+// at its call site below rather than through a named subclass, since the
+// class itself is the entire "component" here -- a named wrapper would only
+// rename a handful of integers, not remove any duplication:
+//   - Nibble-packed formats (Q4_0, Q4_1, Q5_0's low bits, Q4_K, NVFP4, ...):
+//     4-bit fields, 2 planes (low/high nibble), pos_count = window_size/2,
+//     outer = which independent sub-block "window" (1 for a plain whole-block
+//     low-half/high-half split, e.g. Q4_0/IQ4_NL; >1 for NVFP4/Q4_K/Q5_K's
+//     independent windows) -- GGML's actual convention for every nibble-
+//     packed format, not just the more obvious "adjacent pair per byte"
+//     layout.
+//   - TQ2_0/Q2_K/Q3_K/Q6_K's 2-bit fields: 2-bit fields, 4 planes, pos_count
+//     = block_size/8 (= window_bytes), outer = which half of the block --
+//     verified by hand against each format's original dequantize math (e.g.
+//     TQ2_0's `half = gi/half_block; l = local/window_bytes; m =
+//     local%window_bytes; byte_idx = half*window_bytes + m; shift = l*2`);
+//     NOT a plain "4 consecutive elements per byte" packing.
+//   - Q3_K's hmask / Q5_K's qh out-of-band high-bit arrays: 1-bit fields,
+//     num_windows planes, pos_count = window_size, outer always 0 (the whole
+//     array is one window_size*num_windows group) -- the "rotating bit
+//     position" scheme GGML uses for an extra high bit per element (their own
+//     source expresses this via a `half`/`iter`-based case split instead, but
+//     it's the same addressing). Always paired with a lower-bit code via
+//     CombineBits, never used standalone.
+// `qmax` recenters already-decoded codes before splitting (0 leaves the raw
+// field, e.g. a lookup-table index or a single out-of-band bit -- always 0
+// for the two cases above). encode() OR-accumulates the planes into each
+// byte via an RDom, exactly like BitPack's qh_accum-style OR-reduction.
+//
+// `plane_axis` (default false) selects an alternate decode that keeps `plane`
+// as an explicit LEADING output axis instead of folding it into a flat kk:
+//   fields(plane, pos, blk, _) = (bytes(pos, blk, _) >> plane*field_bits) & mask
+// This is the shape a *combined* (scale, min) field wants -- plane 0 = scale,
+// plane 1 = min -- so LinearDequant can read both from one func by its
+// plane index. It's dimension-general (Halide::_) and needs neither `outer` nor
+// `qmax` (raw unsigned fields), so it is exactly Q2_K's per-sub-block nibble-
+// pair (scale, min) byte array, with no bespoke leaf. Decode-only in this mode
+// (only ever an extern-delegated TrustedInverse decoder stage).
+class PlanarBitPack : public Halide::Approximation {
+public:
+    // plane_count is always 8/field_bits in every instantiation here (a byte
+    // packs exactly 8 bits' worth of same-width fields, full stop), so it's
+    // derived rather than taken as its own parameter -- see nibble_pack/
+    // crumb_pack/rotating_bit_pack/le_bit_pack below for the named-shape
+    // constructors most call sites should use instead of this directly.
+    PlanarBitPack(int field_bits, int pos_count, int qmax = 0, bool plane_axis = false)
+        : field_bits_(field_bits), plane_count_(8 / field_bits), pos_count_(pos_count), qmax_(qmax),
+          plane_axis_(plane_axis) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        if (plane_axis_) {
+            _halide_user_error << "PlanarBitPack plane-axis mode is decode-only "
+                                  "(only an extern-delegated TrustedInverse decoder stage).\n";
+            return {};
+        }
+        Func codes = inputs[0];
+        Var byte_idx("byte_idx"), blk("blk");
+        int group = plane_count_ * pos_count_;
+        Expr outer = byte_idx / pos_count_;
+        Expr pos = byte_idx % pos_count_;
+
+        RDom rp(0, plane_count_, "rp");
+        Expr kk = outer * group + rp * pos_count_ + pos;
+        Expr field = cast<uint32_t>(cast<int32_t>(codes(kk, blk)) + qmax_) & ((1u << field_bits_) - 1);
+        Func bytes("planar_bit_pack_bytes");
+        bytes(byte_idx, blk) = cast<uint8_t>(0);
+        bytes(byte_idx, blk) = bytes(byte_idx, blk) | cast<uint8_t>(field << (rp * field_bits_));
+        return {{bytes}, {bytes}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var kk("kk"), blk("blk");
+        if (plane_axis_) {
+            Var plane("plane"), pos("pos");
+            Expr field = (cast<uint32_t>(bytes(pos, blk, _)) >> (cast<uint32_t>(plane) * field_bits_)) &
+                         ((1u << field_bits_) - 1);
+            Func fields("planar_bit_pack_fields");
+            fields(plane, pos, blk, _) = cast<uint8_t>(field);
+            return {{fields}, {}};
+        }
+        // Lane-general via Halide::_ (see the DIMENSION / WILDCARD CONVENTION
+        // in the file's top comment), like the plane-axis mode above.
+        int group = plane_count_ * pos_count_;
+        Expr outer = kk / group;
+        Expr rem = kk % group;
+        Expr plane = rem / pos_count_;
+        Expr pos = rem % pos_count_;
+        Expr byte_idx = outer * pos_count_ + pos;
+        Expr field = (cast<uint32_t>(bytes(byte_idx, blk, _)) >> (plane * field_bits_)) & ((1u << field_bits_) - 1);
+        Func codes("planar_bit_pack_codes");
+        codes(kk, blk, _) = cast<int8_t>(cast<int32_t>(field) - qmax_);
+        return {{codes}, {}};
+    }
+
+private:
+    int field_bits_, plane_count_, pos_count_, qmax_;
+    bool plane_axis_;
+};
+
+// Named PlanarBitPack shapes for the four bit-widths this file actually
+// instantiates, so a call site reads as "a nibble pack over this many
+// elements" instead of a bare (field_bits, pos_count) pair the reader has to
+// re-derive the meaning of. `window` is the element span PlanarBitPack's own
+// class comment calls "window_size": for nibble_pack/crumb_pack, the size of
+// one independently low/high- (or low/high-2-bit-) split group (pos_count is
+// window/2 or window/4, one plane's share of it); for rotating_bit_pack, the
+// single rotating-bit-position span itself (pos_count = window directly,
+// since GGML's hmask/qh arrays have exactly one such group covering the
+// *whole* field, not several independent windows).
+inline std::unique_ptr<Halide::Approximation> nibble_pack(int window, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(4, window / 2, qmax);
+}
+inline std::unique_ptr<Halide::Approximation> crumb_pack(int window, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(2, window / 4, qmax);
+}
+inline std::unique_ptr<Halide::Approximation> rotating_bit_pack(int window, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(1, window, qmax);
+}
+// The Stage-2 qh addressing (make_code_pack's code_bits==5 case): one flat
+// bit per element, byte kk/8 at shift kk%8 -- PlanarBitPack{1, 1} regardless
+// of block size (pos_count is always 1; there's no "window" to parameterize).
+inline std::unique_ptr<Halide::Approximation> le_bit_pack() {
+    return std::make_unique<PlanarBitPack>(1, 1);
+}
+
+// GGML's Q5_0/Q5_1 5-bit code split (a 4-bit low nibble plus a 5th high bit,
+// OR-accumulated one bit per element into a 32-bit little-endian word) used
+// to be a bespoke FiveBitPack class here. It's deleted: verified by hand that
+// it is exactly the K-quant "combined bit code" shape (CombineBits, section
+// 5 below) already used for Q3_K/Q5_K/Q6_K's own adjacent {high-bit array;
+// low-bits array} regions -- qh's bit `kk` is element kk's own high bit,
+// i.e. le_bit_pack()'s addressing (byte kk/8, shift kk%8; GGML's own code
+// computes this via a low/high-half split for scalar-loop efficiency -- e.g.
+// `(qh >> (byte_idx+12)) & 0x10` for the high half -- but that's an
+// equivalent, more roundabout way of writing the same fact used directly
+// here), and the nibble half is exactly nibble_pack(block_size) (byte b
+// holds elements b and b+block_size/2 at shifts 0/4). See make_code_pack's
+// code_bits==5 case below, which assembles this
+// via make_combined_bit_codec exactly as make_q5_k_scheme does for its own
+// adjacent {qh; qs} region.
+
+// encode(codes(kk, blk) signed in {-1, +1}) -> block_size/8 bytes; decode
+// reverses it. Packs one sign bit per element, 8 elements per byte, bit
+// `kk % 8` of byte `kk / 8` set when code is +1 -- Q1_0's layout (paired
+// with RoundingMode::SignOnly/ScaleAnchor::MeanAbs above). Accumulates via
+// an OR-reduction exactly like PlanarBitPack::encode's per-byte accumulation,
+// just one full byte's worth of bits at a time instead of a per-plane field.
+class BitPack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0];
+        Var byte_idx("byte_idx"), blk("blk");
+
+        Func bit("bit_pack_bit");
+        Var kk("kk");
+        bit(kk, blk) = cast<uint8_t>(select(codes(kk, blk) > 0, 1, 0));
+
+        RDom rb(0, 8, "rb");
+        Func bytes("bit_pack_bytes");
+        bytes(byte_idx, blk) = cast<uint8_t>(0);
+        bytes(byte_idx, blk) = bytes(byte_idx, blk) | cast<uint8_t>(bit(byte_idx * 8 + rb, blk) << rb);
+
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk), byte_idx in [0, block_size/8)
+        Var kk("kk"), blk("blk");
+        Expr byte_idx = kk / 8;
+        Expr bit_off = kk % 8;
+        Expr bit = (cast<uint32_t>(bytes(byte_idx, blk)) >> bit_off) & 1u;
+        Func codes("bit_pack_codes");
+        codes(kk, blk) = cast<int8_t>(select(bit != 0, 1, -1));
+        return {{codes}, {}};
+    }
+};
+
+// encode(codes(kk, blk) signed int8) -> 1 code per byte, same shape (the
+// identity-shaped case PlanarBitPack's nibble/2-bit packing doesn't cover,
+// used by e.g. Q8_0). Unlike PlanarBitPack, this formula has no precondition
+// on kk's range (reinterpret is valid for any kk) -- it doesn't know or care
+// what block_size is; bounds propagate backward from whatever actually
+// consumes it.
+class BytePack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0];
+        Var kk("kk"), blk("blk");
+        Func bytes("byte_pack_bytes");
+        bytes(kk, blk) = reinterpret<uint8_t>(codes(kk, blk));
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var kk("kk"), blk("blk");
+        // Lane-general via Halide::_ (see the DIMENSION / WILDCARD CONVENTION
+        // in the file's top comment): trailing lane dims ride through.
+        Func codes("byte_pack_codes");
+        codes(kk, blk, _) = reinterpret<int8_t>(bytes(kk, blk, _));
+        return {{codes}, {}};
+    }
+};
+
+// decode(52 bytes {qs[48]; qh[4]}) -> codes(kk, blk), the raw base-3 digit in
+// [0, 3) -- GGML's TQ1_0 packing: 256 elements in 3 sections (a 160-element
+// and an 80-element run at 5 trits/byte, then a 16-element run at 4 real
+// trits/byte, its 5th digit always 0). decode() reverses the ceiling-division
+// packing (`byte = ceil(digit_number * 256 / 243)`) via the same
+// multiply-truncate-rescale trick tq1_0_generators.cpp hand-rolled: extracting
+// digit `n` needs multiplier `3^n`, `n` from the most-significant digit. Codes
+// feed a codebook index directly (TQ2_0's {-1, 0, 1, unused} convention).
+// Decode-only: TQ1_0 quantize is extern-delegated.
+class TritPack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "TritPack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk), byte_idx in [0, 52)
+        Var kk("kk"), blk("blk");
+
+        Expr n_a = kk / 32;
+        Expr byte_a = kk % 32;
+
+        Expr local_b = kk - 160;
+        Expr n_b = local_b / 16;
+        Expr byte_b = 32 + local_b % 16;
+
+        Expr local_c = kk - 240;
+        Expr n_c = local_c / 4;
+        Expr byte_c = 48 + local_c % 4;
+
+        Expr n = select(kk < 160, n_a, select(kk < 240, n_b, n_c));
+        Expr byte_abs = select(kk < 160, byte_a, select(kk < 240, byte_b, byte_c));
+
+        Expr byte_val = bytes(byte_abs, blk);
+        Expr p3 = mux(n, {1, 3, 9, 27, 81, 243});
+
+        Expr q_trunc = cast<uint8_t>(widening_mul(byte_val, p3));
+        Expr xi = cast<int32_t>((cast<uint16_t>(q_trunc) * 3) >> 8);
+
+        Func codes("trit_pack_codes");
+        codes(kk, blk) = cast<int8_t>(xi);
+        return {{codes}, {}};
+    }
+};
+
+// ---------------------------------------------------------------------------
+// 3b. Derived extra fields (computed from other already-encoded fields, not
+//     from the original values -- appended before struct-packing).
+// ---------------------------------------------------------------------------
+
+// The two ways a derived "sum of codes" extra field gets appended:
+//   - ScaledFloat: one sum for the *whole* block, already multiplied by
+//     scale into a float -- GGML's Q8_1 "s" field (group_size == block_size,
+//     a single group), letting a paired vec_dot recover sum(dequantized
+//     values) cheaply from the block's own header instead of re-reducing
+//     codes itself.
+//   - RawInt16: one sum *per group* (group_size < block_size, several
+//     groups), each a plain int32-then-int16 integer sum with no scale
+//     multiply -- GGML's Q8_K "bsums" field, letting a paired K-quant
+//     vec_dot recover each 16-element group's sum of raw int8 codes cheaply.
+enum class SumMode { ScaledFloat,
+                     RawInt16 };
+
+// encode({codes(kk, blk), scale(blk)}) -> {codes, scale, sum}: sum(blk) (no
+// group dim) for ScaledFloat, sum(g, blk) for RawInt16 -- see SumMode above.
+// decode() discards sum and passes codes/scale through unchanged in both
+// modes: it's a redundant, derivable quantity, not needed to reconstruct
+// dequantized values, so there's nothing to invert. Arity-changing like the
+// grouped FieldSpec fields make_block_layout composes (see FieldSpec/
+// FieldLayout above), but in the *encode* direction instead (2 inputs -> 3
+// outputs; decode then undoes it in the same direction rather than the
+// mirror one, since sum isn't invertible into anything -- it's simply
+// dropped).
+class AppendSums : public Halide::Approximation {
+public:
+    AppendSums(int group_size, SumMode mode)
+        : group_size_(group_size), mode_(mode) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0], scale = inputs[1];
+        Var blk("blk");
+
+        if (mode_ == SumMode::ScaledFloat) {
+            RDom r(0, group_size_, "r");
+            Func sum_i("append_sums_i");
+            sum_i(blk) = 0;
+            sum_i(blk) += cast<int32_t>(codes(r, blk));
+
+            Func sum_f("append_sums_scaled");
+            sum_f(blk) = cast<float>(sum_i(blk)) * scale(blk);
+
+            return {{codes, scale, sum_f}, {sum_i}};
+        }
+        Var g("g");
+        RDom rg(0, group_size_, "rg");
+
+        Func sum_i("append_sums_i");
+        sum_i(g, blk) = cast<int32_t>(0);
+        sum_i(g, blk) += cast<int32_t>(codes(g * group_size_ + rg, blk));
+
+        Func bsums("append_sums_raw");
+        bsums(g, blk) = cast<int16_t>(sum_i(g, blk));
+
+        return {{codes, scale, bsums}, {sum_i}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        // The sum (encoded[2]) is a redundant derived quantity -- pass
+        // codes/scale through unchanged.
+        return {{encoded[0], encoded[1]}, {}};
+    }
+
+private:
+    int group_size_;
+    SumMode mode_;
+};
+
+// ---------------------------------------------------------------------------
+// 3c. Concatenation into one byte buffer.
+// ---------------------------------------------------------------------------
+
+// Concatenates N already-packed, fixed-width byte fields into one
+// byte-addressed buffer per block, at fixed offsets -- generalizes the
+// per-format "select(byte==0, delta_byte0, byte==1, delta_byte1,
+// packed(...))" pattern duplicated in every *_generators.cpp quantize
+// function today.
+//
+// `field_widths[k]` is the k-th field's width in bytes, *in on-disk byte
+// order*: `inputs[k]`/`encoded[k]` (encode/decode respectively) must already
+// be in that same order. When the rest of a Compose/Apply chain produces
+// fields in some other order (e.g. {codes_bytes, scale_bytes} when the
+// on-disk layout is scale-then-codes, as in block_q4_0), reorder into byte
+// order with a Permute stage composed in front of/behind this one --
+// FieldLayout below is the one call site that needs this, via
+// Permute{slots_of(fields_)}.
+class StructPack : public Halide::Approximation {
+public:
+    explicit StructPack(std::vector<int> field_widths)
+        : field_widths_(std::move(field_widths)) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Var byte("byte"), blk("blk");
+        std::vector<int> offsets = offsets_in_output_order();
+
+        // select()'s branches are all evaluated unconditionally (not
+        // short-circuiting), so each field's local index is clamped to its
+        // own valid range before use -- same idiom as q4_0_generators.cpp.
+        Expr result = cast<uint8_t>(0);
+        for (int k = (int)field_widths_.size() - 1; k >= 0; k--) {
+            Expr local = clamp(byte - offsets[k], 0, field_widths_[k] - 1);
+            Expr in_range = byte >= offsets[k] && byte < offsets[k] + field_widths_[k];
+            result = select(in_range, inputs[k](local, blk), result);
+        }
+
+        Func packed("struct_pack_packed");
+        packed(byte, blk) = result;
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];
+        std::vector<int> offsets = offsets_in_output_order();
+        Var local("local"), blk("blk");
+
+        std::vector<Func> fields;
+        fields.reserve(field_widths_.size());
+        for (int k = 0; k < (int)field_widths_.size(); k++) {
+            Func field("struct_pack_field_" + std::to_string(k));
+            field(local, blk) = packed(local + offsets[k], blk);
+            fields.push_back(field);
+        }
+        return {fields, {}};
+    }
+
+private:
+    std::vector<int> offsets_in_output_order() const {
+        std::vector<int> offsets(field_widths_.size());
+        int acc = 0;
+        for (size_t k = 0; k < field_widths_.size(); k++) {
+            offsets[k] = acc;
+            acc += field_widths_[k];
+        }
+        return offsets;
+    }
+
+    std::vector<int> field_widths_;
+    std::vector<int> input_index_;
+};
+
+// One field, in ON-DISK byte order, of a struct-packed block layout: an
+// on-disk byte width plus which logical "slot" it lands in -- the index it
+// occupies in the Func vector immediately after StructPack::decode() (and,
+// symmetrically, immediately before StructPack::encode()) -- and how to
+// pack/unpack it. This is the single declaration that used to be split three
+// ways at every make_*_scheme call site: a StructPack{widths, input_index},
+// a stack of Apply{i,1,1,pack} lines whose `i` had to be kept in sync with
+// StructPack's own indices by hand, and (at the Generator call sites) a
+// hand-summed block byte count. FieldSpec/make_block_layout below fold all
+// three into one list.
+//
+// Most fields are their own arity-1 group: `pack` set, `arity` left at its
+// default of 1, one FieldSpec per on-disk field. A few fields aren't packed
+// independently, though -- make_code_pack's code_bits==5 combined codec's
+// {nibble, qh} decode into one `codes` field together (Q5_0/Q5_1's split
+// 5-bit code), and IQ4XSScalePack's {scales_h, scales_l} decode into one
+// `scale` field together. For a group like that, list every physical on-disk
+// field it spans (so StructPack still gets each one's own width/slot), but
+// only the *leader* -- conventionally, the one whose pack actually does the
+// work -- carries `pack` and `arity` (= how many consecutive slots,
+// [[slot, slot+arity), the group spans); every other member of the group
+// leaves `pack` null (a "this slot is spoken for by an earlier FieldSpec's
+// group" marker) and `arity` at its default (unused for non-leaders).
+struct FieldSpec {
+    int slot;
+    int width_bytes;
+    std::unique_ptr<Halide::Approximation> pack;
+    int arity = 1;
+};
+
+// The result of make_block_layout(): the assembled layout Approximation,
+// ready to compose in front of a scheme's lossy quantize/dequantize stage,
+// plus the on-disk block's total byte width -- summed here, once, from the
+// same field list every make_*_scheme() used to hand-sum separately at its
+// own Generator call site.
+struct BlockLayout {
+    std::unique_ptr<Halide::Approximation> layout;
+    int bytes;
+};
+
+// The plain overload, for the rare call site (make_codebook_scheme) that
+// must build its field list conditionally rather than write it out literally.
+inline BlockLayout make_block_layout(std::vector<FieldSpec> fields) {
+    using namespace Halide;
+
+    int bytes = 0;
+    std::vector<int> widths, permutation;
+    std::vector<FieldSpec *> leaders;
+    widths.reserve(fields.size());
+    permutation.reserve(fields.size());
+
+    for (FieldSpec &f : fields) {
+        permutation.push_back(f.slot);
+        widths.push_back(f.width_bytes);
+        bytes += f.width_bytes;
+        if (f.pack) {
+            leaders.push_back(&f);
+        }
+    }
+
+    std::sort(leaders.begin(), leaders.end(),
+              [](const FieldSpec *a, const FieldSpec *b) { return a->slot > b->slot; });
+
+    // StructPack first (outermost -- closest to the on-disk bytes), then
+    // Permute (byte order <-> slot order), then each group's Apply -- see
+    // StructPack's doc comment for why the Permute is needed at all.
+    ComposeBuilder packs;
+    packs.add(StructPack{widths});
+    packs.add(Permute{permutation});
+    for (FieldSpec *f : leaders) {
+        // encode_arity is always 1 (a group's pack consumes one already-
+        // combined logical Func and expands it into `arity` on-disk fields);
+        // decode_arity is `arity` (the mirror: collapse those `arity`
+        // on-disk fields back into the one logical Func).
+        packs.add(Apply{f->slot, /*encode_arity=*/1, /*decode_arity=*/f->arity, std::move(f->pack)});
+    }
+
+    return {packs.build(), bytes};
+}
+
+// Takes each field as its own argument (FieldSpec{...}, FieldSpec{...}, ...),
+// not a single braced list: FieldSpec holds a std::unique_ptr, so it isn't
+// CopyConstructible, and std::initializer_list<T> -- unlike a plain function
+// parameter pack -- requires copying its elements, even into a move-only
+// std::vector<T>. A variadic template sidesteps that entirely (each argument
+// is forwarded, never copied), at the cost of every call site needing to
+// spell out `FieldSpec{...}` instead of a bare `{...}`.
+template<typename... Fields>
+inline BlockLayout make_block_layout(Fields &&...fields) {
+    std::vector<FieldSpec> v;
+    v.reserve(sizeof...(Fields));
+    (v.push_back(std::forward<Fields>(fields)), ...);
+    return make_block_layout(std::move(v));
+}
+
+// ---------------------------------------------------------------------------
+// Shared helpers for the extern-delegated formats.
+// ---------------------------------------------------------------------------
+
+// The extern quantize body, shared by every format whose real quantizer is a
+// named GGML extern (see ggml_extern_quantize.cpp): it computes nothing
+// itself, just names the *_quantize_via_ggml symbol and returns the whole
+// packed byte buffer as a single 2-D uint8 Func. Wrapped as ExternQuantize
+// below and used as the *encoder* half of a Halide::TrustedInverse, whose
+// decoder half is the Compose that unpacks and dequantizes those bytes.
+inline Halide::EncodeResult extern_quantize_blocks(std::vector<Halide::Func> inputs,
+                                                   const std::string &extern_name) {
+    using namespace Halide;
+    Func flat = inputs[0];
+    Func blocks(extern_name + "_blocks");
+    std::vector<ExternFuncArgument> args = {flat};
+    blocks.define_extern(extern_name, args, UInt(8), 2, NameMangling::C);
+    return {{blocks}, {}};
+}
+
+// Decode the 2-byte little-endian fp16 delta stored at bytes(offset)/
+// bytes(offset+1). Returns the delta as an Expr in `blk`. Same bit twiddling
+// as Fp16Pack::decode, written inline here because the grid leaves below read
+// their delta out of a raw byte buffer at a fixed offset rather than through a
+// composed Fp16Pack stage.
+inline Halide::Expr fp16_delta(Halide::Func bytes, int offset, Halide::Var blk) {
+    using namespace Halide;
+    return cast<float>(reinterpret<float16_t>(cast<uint16_t>(le_uint(bytes, offset, blk, 2))));
+}
+
+// (grid(idx) >> (j*8)) & 0xff -- the byte-within-grid-entry extraction every
+// grid leaf below does, whether the grid buffer's entries are 64-bit
+// (iq2s_grid/iq2xs_grid/iq1s_grid) or 32-bit (iq3xxs_grid/iq3s_grid/
+// iq2xxs_grid). Templated on the grid's element type so one function covers
+// both widths; `j` indexes the byte within the (8- or 4-byte) grid entry.
+template<typename T>
+inline Halide::Expr grid_byte(Halide::Buffer<T> grid, Halide::Expr idx, Halide::Expr j) {
+    using namespace Halide;
+    Expr grid_val = grid(idx);
+    return cast<uint8_t>((grid_val >> (cast<T>(j) * 8)) & 0xff);
+}
+
+// select(bit, -1.0f, 1.0f) -- the sign-bit-to-multiplier idiom every grid
+// leaf's final dequantize multiply uses.
+inline Halide::Expr sign_select(Halide::Expr bit) {
+    using namespace Halide;
+    return select(bit, -1.0f, 1.0f);
+}
+
+// The ksigns_iq2xs indirection + bit test shared by IQ3_XXS/IQ2_XS/IQ2_XXS:
+// look `sign_idx` up in the 128-entry ksigns table, then test bit `j` of the
+// looked-up byte.
+inline Halide::Expr ksigns_sign(Halide::Buffer<uint8_t> ksigns, Halide::Expr sign_idx, Halide::Expr j) {
+    using namespace Halide;
+    Expr signs = ksigns(sign_idx);
+    return (cast<uint32_t>(signs) & (cast<uint32_t>(1) << j)) != 0;
+}
+
+// select(is_high, byte >> 4, byte & 0x0f) -- the low/high-nibble-of-a-byte
+// idiom used throughout the K-quant scale unpackers and the grid/repack
+// leaves alike.
+inline Halide::Expr nibble_of(Halide::Expr byte_expr, Halide::Expr is_high) {
+    using namespace Halide;
+    return select(is_high, byte_expr >> 4, byte_expr & 0x0f);
+}
+
+// Copy one of iq_grids_data.h's static constant codebook tables into a named
+// Halide::Buffer the grid classes below index into.
+template<typename T>
+inline Halide::Buffer<T> make_grid_buffer(const T *data, int n, const char *name) {
+    Halide::Buffer<T> buf(n, name);
+    for (int i = 0; i < n; i++) {
+        buf(i) = data[i];
+    }
+    return buf;
+}
+
+template<size_t N>
+inline Halide::Buffer<int8_t> make_static_codebook(const int8_t (&values)[N], const char *name) {
+    return Halide::Buffer<int8_t>(const_cast<int8_t *>(values), (int)N, name);
+}
+
+// ---------------------------------------------------------------------------
+// 4. Extern-delegated quantize + decode-only dequantize-math leaves.
+// ---------------------------------------------------------------------------
+//
+// The formats below (codebook, K-quant, IQ grid, IQ4_XS) all share one shape:
+// their forward map (quantize) is an opaque offline black box -- a per-block
+// nearest-codeword search, an iterative error-minimizing scale fit, a
+// transcendental scale derivation -- that no composition of Halide Funcs
+// reproduces bit-for-bit, so it is delegated to a named GGML extern (see
+// ggml_extern_quantize.cpp). Their reverse map (dequantize) IS an ordinary,
+// bit-exact composition of invertible primitives. Halide::TrustedInverse
+// pairs the two: ExternQuantize (encode()) as the encoder half, a plain
+// Compose (decode()) as the decoder half. The leaves here are the pieces of
+// that decoder that aren't already covered by the packing/reshape components
+// in sections 1-3 -- the codebook lookup and the scale-multiply math. Each is
+// decode-only: its encode() is exactly the opaque forward map deferred to the
+// extern, so it is never called (it only ever lives inside a TrustedInverse's
+// decoder) and traps if it somehow is.
+
+// The encoder half of every extern-delegated format's TrustedInverse: encode()
+// delegates to the named GGML extern (extern_quantize_blocks), producing the
+// whole packed byte buffer as one 2-D uint8 Func. decode() is never called --
+// the TrustedInverse's decoder half owns dequantize.
+class ExternQuantize : public Halide::Approximation {
+public:
+    explicit ExternQuantize(std::string extern_name)
+        : extern_name_(std::move(extern_name)) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        return extern_quantize_blocks(std::move(inputs), extern_name_);
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func>) override {
+        _halide_user_error << "ExternQuantize::decode is never valid -- it is only "
+                              "the encoder half of a TrustedInverse.\n";
+        return {};
+    }
+
+private:
+    std::string extern_name_;
+};
+
+// The encoder half for formats that have NO forward map at all (the IQ1/IQ2
+// importance-matrix-only quantizers: GGML exposes no *_quantize_via_ggml for
+// them). encode() produces a correctly-shaped `blocks(byte, blk)` uint8 Func
+// so that Func::approximate_by() -- which always builds encode() before
+// decode() -- can splice the round trip; the value is a placeholder (0),
+// because Pipeline::compute_offline() always severs this encode and binds the
+// real already-quantized Input in its place, so it is never computed. This is
+// what lets a decode-only format still go through the standard
+// approximate_by/compute_offline path (exercising the framework) instead of a
+// bespoke direct-decode generator. decode() traps -- the paired decoder half
+// of the TrustedInverse owns dequantize.
+class SeveredEncode : public Halide::Approximation {
+public:
+    // `dims` is the dimensionality of the packed buffer this stands in for: 2
+    // for a plain (byte, blk) codec, 3 for a repack weight buffer
+    // (byte, k-block, col-group).
+    explicit SeveredEncode(int block_bytes, int dims = 2)
+        : block_bytes_(block_bytes), dims_(dims) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        using namespace Halide;
+        std::vector<Var> args;
+        for (int d = 0; d < dims_; d++) {
+            args.push_back(Var("se" + std::to_string(d)));
+        }
+        Func blocks("severed_encode_blocks");
+        blocks(args) = cast<uint8_t>(0);
+        return {{blocks}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func>) override {
+        _halide_user_error << "SeveredEncode::decode is never valid -- it is only "
+                              "the (always-severed) encoder half of a TrustedInverse.\n";
+        return {};
+    }
+
+private:
+    int block_bytes_, dims_;
+};
+
+// codes(kk, blk) -> table[codes], a fixed int8 codebook lookup -- the shared
+// codes->value step of every codebook-quantized format (IQ4_NL, MXFP4, TQ1_0,
+// TQ2_0, NVFP4, IQ4_XS). Apply'd on the codes field between unpacking and the
+// scale multiply, so LinearDequant sees the looked-up value
+// in place of a raw integer code. `table` is a Buffer over `static const`
+// backing data (matching every per-format lookup_*() helper's idiom), copied
+// around as a lightweight handle. encode() is the nearest-codeword search
+// deferred to the extern, so it never runs.
+class Codebook : public Halide::Approximation {
+public:
+    explicit Codebook(Halide::Buffer<int8_t> table)
+        : table_(table) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "Codebook::encode is never valid -- the forward "
+                              "codeword search is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func codes = encoded[0];
+        Var kk("kk"), blk("blk");
+        Buffer<int8_t> table = table_;
+        // Clamp the index to the table's own extent -- a no-op on valid codes
+        // (unpackers produce in-range indices), but it gives Halide's bounds
+        // inference a provable index range across this Func boundary, instead
+        // of falling back to the full int8 range (accessing table at -128).
+        // The grid leaves clamp their grid index the same way.
+        // Dimension-general (pure) decode: the Halide::_ placeholder carries
+        // any extra trailing "lane" dims (e.g. a matmul weight's column dims)
+        // through untouched; with zero trailing dims it collapses to the
+        // familiar (kk, blk). See the DESIGN NOTE by Reblock.
+        Func values("codebook_values");
+        values(kk, blk, _) = table(clamp(cast<int32_t>(codes(kk, blk, _)), 0, table.dim(0).extent() - 1));
+        return {{values}, {}};
+    }
+
+private:
+    Halide::Buffer<int8_t> table_;
+};
+
+// The one decode-only linear dequantize behind every extern-delegated
+// format, unifying what used to be two separate leaves (ScaleDequant and
+// TwoLevelScaleDequant):
+//
+//   - One-level (has_super_d = false, always has_min = false): the flat
+//     codebook formats' cast<float>(codes) * scale. Inputs {codes, scale}.
+//     `sub_size` selects the scale's indexing: 0 means one scale for the
+//     whole block, a Func with NO sub dimension (scale(blk, _), the shape
+//     Fp16Pack/F32Pack/E8M0Pack decode to -- IQ4_NL/MXFP4/TQ*); > 0 means
+//     one scale per `sub_size`-element sub-block, indexed
+//     scale(kk / sub_size, blk, _) (NVFP4's per-sub-block UE4M3 bytes).
+//     This is SymmetricAffineQuantize::decode generalized with a sub-block
+//     scale index; the native symmetric formats keep their own invertible
+//     SymmetricAffineQuantize, so this is decode-only.
+//
+//   - Two-level (has_super_d = true): the K-quant / IQ4_XS dequantize -- a
+//     super-block-wide float `d` (and, for the affine K-quants, `dmin`)
+//     times a per-sub-block scale (and min). Inputs, in order:
+//       has_min:  {d, dmin, scale_min, codes}
+//       no min:   {d, scale, codes}
+//     When has_min, scale and min arrive *combined* in one func
+//     scale_min(plane, sub, ...) -- plane 0 = scale, plane 1 = min -- the
+//     shape PlanarBitPack's plane-axis mode and K4ScaleMinPack both produce,
+//     so a single field carries both halves of the affine per-sub-block
+//     parameters (no separate `min` slot to thread). `codes` may be raw
+//     integer codes (K-quants) or already-looked-up codebook values
+//     (IQ4_XS, via a Codebook stage).
+//
+// The two branches keep their exact original float expression shapes
+// (multiplication order matters for bit-exactness against GGML's reference
+// dequantizers) -- this class only merges the leaves, not the arithmetic.
+// Dimension-general via Halide::_: any trailing "lane" dims (a matmul
+// weight's columns) ride through untouched; zero of them collapses to the
+// familiar (kk, blk). See the DESIGN NOTE by Reblock.
+class LinearDequant : public Halide::Approximation {
+public:
+    LinearDequant(int sub_size, bool has_super_d, bool has_min)
+        : sub_size_(sub_size), has_super_d_(has_super_d), has_min_(has_min) {
+        _halide_user_assert(has_super_d || !has_min)
+            << "LinearDequant: has_min requires has_super_d (no one-level affine format exists).\n";
+        _halide_user_assert(!has_super_d || sub_size > 0)
+            << "LinearDequant: two-level mode always has a per-sub-block scale.\n";
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "LinearDequant::encode is never valid -- the forward "
+                              "quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Var kk("kk"), blk("blk");
+        Func dequantized("linear_dequantized");
+        if (!has_super_d_) {
+            Func codes = encoded[0], scale = encoded[1];
+            if (sub_size_ == 0) {
+                dequantized(kk, blk, _) = cast<float>(cast<int32_t>(codes(kk, blk, _))) * scale(blk, _);
+            } else {
+                dequantized(kk, blk, _) = cast<float>(cast<int32_t>(codes(kk, blk, _))) * scale(kk / sub_size_, blk, _);
+            }
+        } else if (has_min_) {
+            Func d = encoded[0], dmin = encoded[1], scale_min = encoded[2], codes = encoded[3];
+            Expr sub = kk / sub_size_;
+            dequantized(kk, blk, _) = d(blk, _) * cast<float>(scale_min(0, sub, blk, _)) * cast<float>(codes(kk, blk, _)) -
+                                      dmin(blk, _) * cast<float>(scale_min(1, sub, blk, _));
+        } else {
+            Func d = encoded[0], scale = encoded[1], codes = encoded[2];
+            dequantized(kk, blk, _) = d(blk, _) * cast<float>(scale(kk / sub_size_, blk, _)) * cast<float>(codes(kk, blk, _));
+        }
+        return {{dequantized}, {}};
+    }
+
+private:
+    int sub_size_;
+    bool has_super_d_, has_min_;
+};
+
+// ---------------------------------------------------------------------------
+// 5. K-quants: combined-bit codes and per-sub-block (scale, min) packing.
+// ---------------------------------------------------------------------------
+
+// The invertible arithmetic behind GGML's K-quant "combined bit" codes: a
+// wider-than-one-field code split into a low part and a high part, where
+// `code = low + high*high_weight - offset` (verified by hand to collapse
+// Q3_K's/Q5_K's/Q6_K's actual bit-OR reconstruction into one formula -- OR
+// and + agree because the low/high bit ranges never overlap: high_weight is
+// always the low part's own value range). Unlike the extern-delegated leaves
+// in section 4, this is genuinely invertible in both directions, so it
+// composes as an ordinary symmetric stage: decode() combines {low, high} ->
+// code, encode() splits code -> {low, high}. The per-field packing (each part
+// through its own PlanarBitPack/BytePack, then StructPack concatenating them
+// in the format's on-disk order) is the composition around it -- e.g. for
+// Q5_K, Compose{StructPack{{qs, qh}, order}, Apply{low_pack}, Apply{high_pack},
+// CombineBits{...}} -- not this leaf, which is only the split/combine math.
+class CombineBits : public Halide::Approximation {
+public:
+    CombineBits(int high_weight, int offset)
+        : high_weight_(high_weight), offset_(offset) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0];  // codes(kk, blk), the combined (pre-split) value
+        // `combined` is always >= 0 by construction (offset_ is exactly what
+        // decode() subtracts after reconstructing low + high*weight), so the
+        // %// below don't need to handle negative operands.
+        Expr combined = cast<int32_t>(codes(kk, blk)) + offset_;
+        Func low("combine_bits_low");
+        low(kk, blk) = cast<int8_t>(combined % high_weight_);
+        Func high("combine_bits_high");
+        high(kk, blk) = cast<int8_t>(combined / high_weight_);
+        return {{low, high}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func low = encoded[0], high = encoded[1];
+        Func code("combine_bits_code");
+        code(kk, blk) = cast<int8_t>((cast<int32_t>(low(kk, blk)) + high_weight_ * cast<int32_t>(high(kk, blk))) - offset_);
+        return {{code}, {}};
+    }
+
+private:
+    int high_weight_, offset_;
+    Halide::Var kk{"kk"}, blk{"blk"};
+};
+
+// (Q2_K's per-sub-block nibble-pair scale/min -- low nibble = scale, high
+// nibble = min -- is no longer a bespoke leaf: it is exactly PlanarBitPack's
+// plane-axis mode, PlanarBitPack{4, 16, 0, /*plane_axis=*/true}, producing
+// the same combined (plane, sub) field K4ScaleMinPack does. See make_q2_k_scheme.)
+
+// decode(bytes(byte_idx, blk), 12 bytes) -> scale_min(plane, sub, blk) for sub
+// in [0, 8), plane 0 = scale / plane 1 = min -- GGML's get_scale_min_k4 scheme,
+// shared by Q4_K and Q5_K: for sub<4, scale/min are simply the low 6 bits of
+// byte[sub]/byte[sub+4]; for sub>=4, each is a 4-bit low part from byte[sub+4]
+// combined with a 2-bit high part borrowed from the top 2 bits of an
+// earlier byte (byte[sub-4] for scale, byte[sub] for min) -- a
+// bit-interleaved packing that fits 8 six-bit values into 6 bytes' worth of
+// budget instead of 8 (see q4_k_generators.cpp's original header comment for
+// the full derivation). Emits the combined (plane, sub) field LinearDequant
+// consumes. Decode-only: Q4_K/Q5_K quantize is extern-delegated.
+class K4ScaleMinPack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "K4ScaleMinPack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk[, ...]), byte_idx in [0, 12)
+        Var plane("plane"), sub("sub"), blk("blk");
+
+        Expr jj = clamp(sub - 4, 0, 3);
+        Expr sc = select(sub < 4,
+                         bytes(sub, blk, _) & 0x3f,
+                         cast<uint8_t>((bytes(8 + jj, blk, _) & 0x0f) | ((bytes(jj, blk, _) >> 6) << 4)));
+        Expr m = select(sub < 4,
+                        bytes(sub + 4, blk, _) & 0x3f,
+                        cast<uint8_t>((bytes(8 + jj, blk, _) >> 4) | ((bytes(4 + jj, blk, _) >> 6) << 4)));
+
+        Func scale_min("k4_scale_min_pack");
+        scale_min(plane, sub, blk, _) = cast<uint8_t>(select(plane == 0, sc, m));
+        return {{scale_min}, {}};
+    }
+};
+
+// decode(bytes(byte_idx, blk), 12 bytes) -> scale(sub, blk) for sub in
+// [0, 16) -- Q3_K's 16 SIGNED 6-bit scale values (no min field), a
+// different bit-interleaving than get_scale_min_k4 above: the 2 high bits
+// always live in byte (sub%4)+8, at bit-shift 2*(sub/4); the 4 low bits
+// live in byte (sub%8), taken from the byte's low nibble if sub<8 or high
+// nibble if sub>=8. The final signed value is (low|(high<<4)) - 32 (see
+// q3_k_generators.cpp's original header comment for the full derivation).
+// Decode-only: Q3_K quantize is extern-delegated.
+class Q3KScalePack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "Q3KScalePack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk[, ...]), byte_idx in [0, 12)
+        Var sub("sub"), blk("blk");
+
+        // Dimension-general via Halide::_, matching the other scale unpackers.
+        Expr low_byte_idx = sub % 8;
+        Expr use_high_nibble = sub >= 8;
+        Expr low_byte = bytes(low_byte_idx, blk, _);
+        Expr low_val = cast<int32_t>(nibble_of(low_byte, use_high_nibble));
+        Expr high_byte_idx = (sub % 4) + 8;
+        Expr high_shift = (sub / 4) * 2;
+        Expr high = cast<int32_t>((bytes(high_byte_idx, blk, _) >> high_shift) & 0x3);
+
+        Func scale("q3k_scale_pack_scale");
+        scale(sub, blk, _) = cast<int8_t>((low_val | (high << 4)) - 32);
+        return {{scale}, {}};
+    }
+};
+
+// ---------------------------------------------------------------------------
+// 6. IQ2/IQ3 grid+sign codebook dequantize.
+// ---------------------------------------------------------------------------
+//
+// Unlike IQ4_NL/MXFP4/TQ1_0/TQ2_0/NVFP4 above, these codebooks map one index
+// to a whole *group* of 4 or 8 signed output bytes at once (GGML's published
+// iq2s_grid/iq3xxs_grid/iq3s_grid tables, embedded verbatim from
+// iq_grids_data.h), and each format combines its grid index, sign bits, and
+// per-group scale via its own distinct bit layout -- there's no shared
+// sub-formula across formats the way PlanarBitPack's instances turned out to
+// be for the K-quants. Rather than force an
+// artificial shared abstraction over 3 genuinely different bit layouts, each
+// format below is its own small, decode-only Approximation leaf, wrapped by
+// its make_*_scheme() factory in a TrustedInverse{ExternQuantize, Compose{...,
+// BlockReshape}} (GGML's own reference quantizer for these runs a per-block
+// codebook search -- see ggml_extern_quantize.cpp). decode() is a mechanical,
+// verified-unchanged transcription of iq2_s_generators.cpp's/
+// iq3_xxs_generators.cpp's/iq3_s_generators.cpp's own (already bit-exact)
+// dequantize math, just reading from a `bytes(byte, blk)` Func instead of
+// an `Input<Buffer<uint8_t, 2>>` directly, and producing block-indexed values
+// (the composed BlockReshape does the flat<->block reshape) instead of a flat
+// row itself. encode() traps: quantize is the ExternQuantize's job.
+
+// IQ2_S: 256-element superblock, 8 groups of 32 elements, grid index = an 8-
+// bit qs byte plus 2 extra high bits from a per-group qh byte (1024-entry,
+// 64-bit iq2s_grid, 8 output bytes/index); signs stored directly (no
+// ksigns_iq2xs indirection); scale a nibble byte array (2 groups/byte) via
+// `d*(0.5+nibble)*0.25` -- {fp16 d; qs[32]; signs[32]; qh[8]; scales[8];},
+// 82 bytes.
+class IQ2SGridDequantize : public Halide::Approximation {
+public:
+    IQ2SGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq2s_grid, 1024, "iq2s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ2SGridDequantize is decode-only -- quantize is "
+                              "deferred to an ExternQuantize via TrustedInverse.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk), byte in [0, 82)
+        // Superblock structure recovered by the composed BlockReshape({8, 4, 8}):
+        // j (element in [0,8)), l ([0,4)), ib32 (group in [0,8)).
+        Var j("j"), l("l"), ib32("ib32"), blk("blk");
+
+        constexpr int kQsOffset = 2;
+        constexpr int kSignsOffset = kQsOffset + 256 / 8;    // 34
+        constexpr int kQhOffset = kSignsOffset + 256 / 8;    // 66
+        constexpr int kScalesOffset = kQhOffset + 256 / 32;  // 74
+
+        Expr qs_l = bytes(kQsOffset + ib32 * 4 + l, blk);
+        Expr qh_byte = cast<uint32_t>(bytes(kQhOffset + ib32, blk));
+        Expr extra_bits = mux(l, {(qh_byte << 8) & 0x300,
+                                  (qh_byte << 6) & 0x300,
+                                  (qh_byte << 4) & 0x300,
+                                  (qh_byte << 2) & 0x300});
+        Expr grid_idx_raw = cast<uint32_t>(qs_l) + extra_bits;
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(grid_idx_raw)), 0, 1023);
+
+        Expr signs_byte = bytes(kSignsOffset + ib32 * 4 + l, blk);
+
+        Expr scales_byte = bytes(kScalesOffset + ib32, blk);
+        Expr nibble = nibble_of(scales_byte, l >= 2);
+
+        Expr d = fp16_delta(bytes, 0, blk);
+        Expr db = d * (0.5f + cast<float>(nibble)) * 0.25f;
+
+        Expr gbyte = grid_byte(grid_, grid_idx, j);
+        Expr sign_bit = (cast<uint32_t>(signs_byte) & (cast<uint32_t>(1) << j)) != 0;
+
+        Func dequantized("iq2s_grid_dequantized");
+        dequantized(j, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+};
+
+// IQ3_XXS: 256-element superblock, 8 groups of 32 elements, TWO grid indices
+// per l (plain 8-bit qs bytes, no extra bits; 256-entry, 32-bit
+// iq3xxs_grid, 4 output bytes/index); signs via the same ksigns_iq2xs
+// indirection as IQ2_XXS (a 7-bit sign_idx and a 4-bit scale exponent both
+// bit-packed into one little-endian uint32 "aux32" read from the scales-
+// and-signs field); scale via `d*(0.5+exp)*0.5` -- {fp16 d; qs[64];
+// scales_and_signs[32];}, 98 bytes.
+class IQ3XXSGridDequantize : public Halide::Approximation {
+public:
+    IQ3XXSGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq3xxs_grid, 256, "iq3xxs_grid")),
+          ksigns_(make_grid_buffer(iq_grids::ksigns_iq2xs, 128, "ksigns_iq2xs")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ3XXSGridDequantize is decode-only -- quantize is "
+                              "deferred to an ExternQuantize via TrustedInverse.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk), byte in [0, 98)
+        // Superblock structure recovered by the composed BlockReshape({8, 4, 8}):
+        // j8 (element in [0,8)), l ([0,4)), ib32 (group in [0,8)).
+        Var j8("j8"), l("l"), ib32("ib32"), blk("blk");
+        Expr j4 = j8 % 4;    // byte within the 4-byte grid entry
+        Expr half = j8 / 4;  // 0 (grid1) or 1 (grid2)
+
+        constexpr int kQsOffset = 2;
+        constexpr int kScalesSignsOffset = kQsOffset + 256 / 4;  // 66
+
+        Expr grid_qs_idx = ib32 * 8 + l * 2 + half;
+        Expr grid_idx = bytes(kQsOffset + grid_qs_idx, blk);
+
+        Expr aux32 = le_u32(bytes, kScalesSignsOffset + ib32 * 4, blk);
+
+        Expr d = fp16_delta(bytes, 0, blk);
+        Expr db = d * (0.5f + cast<float>(aux32 >> 28)) * 0.5f;
+
+        Expr sign_idx = cast<uint8_t>((aux32 >> (cast<uint32_t>(l) * 7)) & 127);
+        Expr sign_bit = ksigns_sign(ksigns_, sign_idx, j8);
+        Expr gbyte = grid_byte(grid_, grid_idx, j4);
+
+        Func dequantized("iq3xxs_grid_dequantized");
+        dequantized(j8, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint32_t> grid_;
+    Halide::Buffer<uint8_t> ksigns_;
+};
+
+// IQ3_S: 256-element superblock, 8 groups of 32 elements, TWO grid indices
+// per l (an 8-bit qs byte plus 1 extra high bit from a per-group qh byte,
+// combined into a 9-bit index; 512-entry, 32-bit iq3s_grid, 4 output
+// bytes/index); signs stored directly (no ksigns_iq2xs indirection, unlike
+// IQ2_XXS/IQ3_XXS); scale a nibble byte array (one byte per *pair* of
+// groups) via `d*(1+2*nibble)` (odd integers 1,3,...,31 -- not the
+// "0.5+x*k" formula every other type here uses) -- {fp16 d; qs[64]; qh[8];
+// signs[32]; scales[4];}, 110 bytes.
+class IQ3SGridDequantize : public Halide::Approximation {
+public:
+    IQ3SGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq3s_grid, 512, "iq3s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ3SGridDequantize is decode-only -- quantize is "
+                              "deferred to an ExternQuantize via TrustedInverse.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk), byte in [0, 110)
+        // Superblock structure recovered by the composed BlockReshape({8, 4, 8}):
+        // j8 (element in [0,8)), l ([0,4)), grp (group in [0,8)).
+        Var j8("j8"), l("l"), grp("grp"), blk("blk");
+        Expr j4 = j8 % 4;    // byte within the 4-byte grid entry
+        Expr half = j8 / 4;  // 0 (first grid index of this l) or 1 (second)
+
+        constexpr int kQsOffset = 2;
+        constexpr int kQhOffset = kQsOffset + 256 / 4;         // 66
+        constexpr int kSignsOffset = kQhOffset + 256 / 32;     // 74
+        constexpr int kScalesOffset = kSignsOffset + 256 / 8;  // 106
+
+        Expr qs_byte = bytes(kQsOffset + grp * 8 + l * 2 + half, blk);
+        Expr qh_byte = cast<uint32_t>(bytes(kQhOffset + grp, blk));
+        Expr bit_pos = l * 2 + half;
+        Expr high_bit = (qh_byte >> cast<uint32_t>(bit_pos)) & 1;
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(cast<uint32_t>(qs_byte) + (high_bit << 8))), 0, 511);
+
+        Expr signs_byte = bytes(kSignsOffset + grp * 4 + l, blk);
+        Expr sign_bit = (cast<uint32_t>(signs_byte) & (cast<uint32_t>(1) << j8)) != 0;
+
+        Expr scales_byte = bytes(kScalesOffset + grp / 2, blk);
+        Expr nibble = nibble_of(scales_byte, (grp % 2) != 0);
+
+        Expr d = fp16_delta(bytes, 0, blk);
+        Expr db = d * (1.0f + 2.0f * cast<float>(nibble));
+
+        Expr gbyte = grid_byte(grid_, grid_idx, j4);
+
+        Func dequantized("iq3s_grid_dequantized");
+        dequantized(j8, l, grp, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint32_t> grid_;
+};
+
+// The four IQ1/IQ2 importance-matrix-only formats have no from_float extern
+// (GGML exposes no *_quantize_via_ggml), so their make_*_scheme() below pairs
+// this decode leaf with a SeveredEncode via TrustedInverse (a dequantize-only /
+// vec_dot-only round trip; the placeholder encode is always severed). Each is a
+// verified-unchanged transcription of the matching *_generators.cpp dequantize,
+// reading bytes(byte, blk) and emitting the {8,4,8} superblock form
+// (j/j-elem, l, group, blk) so the composed BlockReshape does the reshape.
+
+// IQ2_XS: 74-byte block. qs[32] as 32 little-endian uint16 (4 per group): low
+// 9 bits index the 512-entry uint64 iq2xs_grid, top 7 bits index ksigns_iq2xs;
+// scale a nibble byte array (2 l's/byte) via d*(0.5+nibble)*0.25.
+class IQ2XSGridDequantize : public Halide::Approximation {
+public:
+    IQ2XSGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq2xs_grid, 512, "iq2xs_grid")),
+          ksigns_(make_grid_buffer(iq_grids::ksigns_iq2xs, 128, "ksigns_iq2xs")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ2XSGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l("l"), ib32("ib32"), blk("blk");
+        constexpr int kQsOffset = 2;
+        constexpr int kScalesOffset = 66;
+
+        Expr qs_idx = ib32 * 4 + l;
+        Expr qs_val = le_u16([&](int i) { return bytes(kQsOffset + qs_idx * 2 + i, blk); });
+        Expr grid_idx = clamp(cast<int32_t>(qs_val & 511), 0, 511);
+        Expr sign_idx = clamp(cast<int32_t>(qs_val >> 9), 0, 127);
+
+        Expr scales_byte = bytes(kScalesOffset + ib32, blk);
+        Expr nibble = nibble_of(scales_byte, l >= 2);
+        Expr db = fp16_delta(bytes, 0, blk) * (0.5f + cast<float>(nibble)) * 0.25f;
+
+        Expr gbyte = grid_byte(grid_, grid_idx, j);
+        Expr sign_bit = ksigns_sign(ksigns_, sign_idx, j);
+
+        Func dequantized("iq2xs_grid_dequantized");
+        dequantized(j, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+    Halide::Buffer<uint8_t> ksigns_;
+};
+
+// IQ2_XXS: 66-byte block. Per group, an 8-byte window: bytes 0..3 are 4 grid
+// indices (one per l) into the 256-entry uint64 iq2xxs_grid; bytes 4..7 form a
+// uint32 aux32 whose top 4 bits are a scale exponent (d*(0.5+exp)*0.25) and
+// whose low 28 bits pack four 7-bit ksigns indices.
+class IQ2XXSGridDequantize : public Halide::Approximation {
+public:
+    IQ2XXSGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq2xxs_grid, 256, "iq2xxs_grid")),
+          ksigns_(make_grid_buffer(iq_grids::ksigns_iq2xs, 128, "ksigns_iq2xs")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ2XXSGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l("l"), ib32("ib32"), blk("blk");
+        constexpr int kQsOffset = 2;
+
+        Expr grid_idx = clamp(cast<int32_t>(bytes(kQsOffset + ib32 * 8 + l, blk)), 0, 255);
+        Expr aux32 = le_u32(bytes, kQsOffset + ib32 * 8 + 4, blk);
+        Expr db = fp16_delta(bytes, 0, blk) * (0.5f + cast<float>(aux32 >> 28)) * 0.25f;
+        Expr sign_idx = clamp(cast<int32_t>((aux32 >> (cast<uint32_t>(l) * 7)) & 127), 0, 127);
+
+        Expr sign_bit = ksigns_sign(ksigns_, sign_idx, j);
+        Expr gbyte = grid_byte(grid_, grid_idx, j);
+
+        Func dequantized("iq2xxs_grid_dequantized");
+        dequantized(j, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+    Halide::Buffer<uint8_t> ksigns_;
+};
+
+// IQ1_S: 50-byte block. qs[32] low grid-index bytes (one per l); qh[8] as 8
+// uint16 (one per group): 3 bits/l give the grid index's high 3 bits, bits
+// 12..14 a per-group scale (dl = d*(2*s+1)), bit 15 selects a +/-IQ1S_DELTA
+// added to every value. iq1s_grid entries are SIGNED bytes used directly.
+class IQ1SGridDequantize : public Halide::Approximation {
+public:
+    IQ1SGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq1s_grid, 2048, "iq1s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ1SGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l("l"), ib("ib"), blk("blk");
+        constexpr float kIQ1S_DELTA = 0.125f;
+        constexpr int kQsOffset = 2;
+        constexpr int kQhOffset = 34;
+
+        Expr qh_val = le_u16([&](int i) { return bytes(kQhOffset + ib * 2 + i, blk); });
+        Expr dl_scale = cast<int32_t>((qh_val >> 12) & 7);
+        Expr dl = fp16_delta(bytes, 0, blk) * cast<float>(2 * dl_scale + 1);
+        Expr delta = select((qh_val & 0x8000) != 0, -kIQ1S_DELTA, kIQ1S_DELTA);
+
+        Expr qs_byte = bytes(kQsOffset + ib * 4 + l, blk);
+        Expr high3 = cast<uint32_t>((qh_val >> (cast<uint16_t>(l) * 3)) & 7);
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(cast<uint32_t>(qs_byte) + (high3 << 8))), 0, 2047);
+
+        Expr grid_signed = reinterpret<int8_t>(grid_byte(grid_, grid_idx, j));
+
+        Func dequantized("iq1s_grid_dequantized");
+        dequantized(j, l, ib, blk) = dl * (cast<float>(grid_signed) + delta);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+};
+
+// IQ1_M: 56-byte block, NO separate delta field. qs[32] low index bytes;
+// qh[16] (2/group) give a high-3-bit grid extension + a sign-delta bit per l2;
+// scales[8] as 4 uint16: the block's shared fp16 d is bit-gathered from the
+// top nibble of all 4 words, each word's low 12 bits holding two 3-bit
+// per-group scales. Signed codebook + /-IQ1S_DELTA, same as IQ1_S.
+class IQ1MGridDequantize : public Halide::Approximation {
+public:
+    IQ1MGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq1s_grid, 2048, "iq1s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ1MGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l2("l2"), ib("ib"), blk("blk");
+        constexpr float kIQ1S_DELTA = 0.125f;
+        constexpr int kQsOffset = 0;
+        constexpr int kQhOffset = 32;
+        constexpr int kScalesOffset = 48;
+
+        auto sc_word = [&](int k) -> Expr {
+            return le_u16([&](int i) { return bytes(kScalesOffset + k * 2 + i, blk); });
+        };
+        Expr sc0 = sc_word(0), sc1 = sc_word(1), sc2 = sc_word(2), sc3 = sc_word(3);
+        Expr d_bits = (sc0 >> 12) | ((sc1 >> 8) & 0xf0) | ((sc2 >> 4) & 0xf00) | (sc3 & 0xf000);
+        Expr d = cast<float>(reinterpret<float16_t>(cast<uint16_t>(d_bits)));
+
+        Expr qh_half = l2 / 2;
+        Expr parity = l2 % 2;
+        Expr qh_byte = cast<uint32_t>(bytes(kQhOffset + ib * 2 + qh_half, blk));
+        Expr qs_byte = bytes(kQsOffset + ib * 4 + l2, blk);
+        // Constant-amount shifts per parity arm (a variable-amount shift into a
+        // buffer index defeats Halide bounds inference even when masked).
+        Expr extra_bits = select(parity == 0, (qh_byte << 8) & 0x700, (qh_byte << 4) & 0x700);
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(cast<uint32_t>(qs_byte) + extra_bits)), 0, 2047);
+
+        Expr delta_mask = select(parity == 0, cast<uint32_t>(0x08), cast<uint32_t>(0x80));
+        Expr delta = select((qh_byte & delta_mask) != 0, -kIQ1S_DELTA, kIQ1S_DELTA);
+
+        Expr sc_idx = ib / 2;
+        Expr sc_word_val = mux(sc_idx, {sc0, sc1, sc2, sc3});
+        Expr shift = (ib % 2) * 6 + qh_half * 3;
+        Expr scale3 = (sc_word_val >> cast<uint32_t>(shift)) & 7;
+        Expr dl = d * cast<float>(2 * scale3 + 1);
+
+        Expr grid_signed = reinterpret<int8_t>(grid_byte(grid_, grid_idx, j));
+
+        Func dequantized("iq1m_grid_dequantized");
+        dequantized(j, l2, ib, blk) = dl * (cast<float>(grid_signed) + delta);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+};
+
+// decode({scales_h(2 bytes), scales_l(4 bytes)}) -> scale(sub, blk) for sub in
+// [0, 8) -- IQ4_XS's per-sub-block 6-bit scale `ls`, already minus its 32 bias
+// so it feeds LinearDequant's `d * scale(sub) * value` directly. `ls`
+// is 4 low bits from scales_l (2 sub-blocks/byte) plus 2 high bits from
+// scales_h (a little-endian uint16, 2 bits/sub-block) -- a two-field
+// bit-interleaving, the peer of Q3KScalePack/K4ScaleMinPack but reading two
+// separate byte fields. Decode-only: IQ4_XS quantize is extern-delegated.
+class IQ4XSScalePack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ4XSScalePack::encode is never valid -- IQ4_XS "
+                              "quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func scales_h = encoded[0], scales_l = encoded[1];
+        Var sub("sub"), blk("blk");
+
+        // Dimension-general via Halide::_, matching the other scale unpackers.
+        // ls = (scales_l[sub/2] >> 4*(sub%2)) & 0xf | ((scales_h >> 2*sub) & 3) << 4
+        Expr low4 = cast<int32_t>(nibble_of(scales_l(sub / 2, blk, _), (sub % 2) != 0));
+        Expr sh = le_u16([&](int i) { return scales_h(i, blk, _); });
+        Expr high2 = cast<int32_t>((sh >> cast<uint16_t>(sub * 2)) & 3);
+        Expr ls = low4 | (high2 << 4);
+
+        Func scale("iq4xs_scale");
+        scale(sub, blk, _) = cast<int8_t>(ls - 32);
+        return {{scale}, {}};
+    }
+};
+
+// ---------------------------------------------------------------------------
+// 7. Repack: interleaved multi-row activation layout (block_q8_0x4 / q8_Kx4).
+//
+// These are repack-specific instances of the deferred general block-relayout
+// (see the DESIGN NOTE by Reblock): "block-layout change prior to applying the
+// same lossy quantization Approximations". The 4 interleaved rows are folded
+// into the block index blk = ib*4 + row, so the existing (kk, blk)
+// SymmetricAffineQuantize + BytePack + Fp16Pack run unchanged; only the two
+// relayouts here (input row-blocking and output interleave/header assembly)
+// are new. n_rows is fixed at 4 (repack's "x4").
+// ---------------------------------------------------------------------------
+
+// Losslessly re-view a 2-D activation x(col, row in [0,n_rows)) as
+// block-indexed block(kk in [0, block_size), blk = ib*n_rows + row), where ib
+// is the k-block. The per-row scale then falls out of the quantizer as a
+// per-blk scale. `n_rows` is 4 for every current caller (repack's "x4"), but
+// isn't hardcoded here -- callers pass it explicitly.
+class RepackRowReshape : public Halide::Approximation {
+public:
+    RepackRowReshape(int block_size, int n_rows)
+        : block_size_(block_size), n_rows_(n_rows) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func x = inputs[0];  // x(col, row)
+        Var kk("kk"), blk("blk");
+        Func block("repack_row_block");
+        block(kk, blk) = x((blk / n_rows_) * block_size_ + kk, blk % n_rows_);
+        return {{block}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func block = encoded[0];  // block(kk, blk)
+        Var col("col"), row("row");
+        Func x("repack_row_x");
+        x(col, row) = block(col % block_size_, (col / block_size_) * n_rows_ + row);
+        return {{x}, {}};
+    }
+
+private:
+    int block_size_, n_rows_;
+};
+
+// Assemble one interleaved output block (byte, ib) from the per-(row) packed
+// code bytes and scale bytes of the 4 rows blk = ib*4 + row. Header: 4 deltas
+// (`delta_bytes` each, row order). Payload: codes interleaved in groups of
+// `blocklen` -- payload position jj -> (row = (jj % (4*bl))/bl,
+// kk = (jj/(4*bl))*bl + jj%bl), matching GGML's src_id/src_offset.
+class RepackInterleavePack : public Halide::Approximation {
+public:
+    RepackInterleavePack(int block_size, int blocklen, int delta_bytes, bool with_bsums = false)
+        : block_size_(block_size), blocklen_(blocklen), delta_bytes_(delta_bytes), with_bsums_(with_bsums) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func code = inputs[0], scale = inputs[1];
+        Var byte("byte"), ib("ib");
+        int header = 4 * delta_bytes_;
+        int payload_end = header + block_size_ * 4;
+
+        Expr row_d = clamp(byte / delta_bytes_, 0, 3);
+        Expr delta_byte = scale(clamp(byte % delta_bytes_, 0, delta_bytes_ - 1), ib * 4 + row_d);
+
+        Expr jj = clamp(byte - header, 0, block_size_ * 4 - 1);
+        Expr row_p = (jj % (4 * blocklen_)) / blocklen_;
+        Expr kk_p = (jj / (4 * blocklen_)) * blocklen_ + (jj % blocklen_);
+        Expr code_byte = code(kk_p, ib * 4 + row_p);
+
+        Func blocks("repack_interleave_blocks");
+        if (!with_bsums_) {
+            blocks(byte, ib) = select(byte < header, delta_byte, code_byte);
+            return {{blocks}, {}};
+        }
+
+        // Q8_K's block_q8_Kx4 appends `bsums`: int16 group-sums of the int8
+        // codes, scattered across rows/groups by GGML's index_q8_k mapping.
+        // Reduce over the interleaved payload order (rj), same as GGML.
+        Var g("g");
+        Func bsums("repack_bsums");
+        RDom rj(0, block_size_ * 4, "rj");
+        Expr rp = (rj % (4 * blocklen_)) / blocklen_;
+        Expr kp = (rj / (4 * blocklen_)) * blocklen_ + (rj % blocklen_);
+        Expr qval = reinterpret<int8_t>(code(kp, ib * 4 + rp));
+        int shift = blocklen_ == 8 ? 3 : 2;  // log2(blocklen)
+        Expr idx = (((rj & (4 * blocklen_ - 1)) >> shift) << 2) + ((rj >> 8) << 4) + ((rj >> 6) & 3);
+        bsums(g, ib) = cast<int32_t>(0);
+        bsums(idx, ib) += cast<int32_t>(qval);
+
+        int nbsum = (block_size_ / 16) * 4;  // 64 groups
+        Expr bsum_rel = clamp(byte - payload_end, 0, nbsum * 2 - 1);
+        Expr bsum_g = bsum_rel / 2;
+        Expr bsum_is_lo = (bsum_rel % 2) == 0;
+        Expr bsum_bits = reinterpret<uint16_t>(cast<int16_t>(bsums(bsum_g, ib)));
+        Expr bsum_byte = cast<uint8_t>(select(bsum_is_lo, bsum_bits & 0xff, (bsum_bits >> 8) & 0xff));
+
+        blocks(byte, ib) = select(byte < header, delta_byte, byte < payload_end, code_byte, bsum_byte);
+        return {{blocks}, {bsums}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func blocks = encoded[0];
+        Var kk("kk"), blk("blk"), sbyte("sbyte");
+        int header = 4 * delta_bytes_;
+
+        // blk = ib*4 + row: ib = blk/4, row = blk%4.
+        Func code("repack_interleave_code");
+        Expr jj = (kk / blocklen_) * (4 * blocklen_) + (blk % 4) * blocklen_ + (kk % blocklen_);
+        code(kk, blk) = blocks(header + jj, blk / 4);
+        Func scaleb("repack_interleave_scale");
+        scaleb(sbyte, blk) = blocks((blk % 4) * delta_bytes_ + sbyte, blk / 4);
+        return {{code, scaleb}, {}};
+    }
+
+private:
+    int block_size_, blocklen_, delta_bytes_;
+    bool with_bsums_;
+};
+
+// block_q8_0x4 codec: block-layout relayout + the same symmetric Q8_0 quantize
+// (amax/127, round) as make_symmetric_block_scheme, interleaved by `blocklen`.
+inline std::unique_ptr<Halide::Approximation> make_q8_0x4_scheme(int blocklen) {
+    using namespace Halide;
+    return std::make_unique<Compose>(
+        RepackInterleavePack{32, blocklen, /*delta_bytes=*/2},
+        Apply{0, BytePack{}},  // codes -> bytes
+        Apply{1, Fp16Pack{}},  // scale -> fp16 bytes
+        SymmetricAffineQuantize{32, 127, RoundingMode::Nearest, ScaleAnchor::AbsMax},
+        RepackRowReshape{32, /*n_rows=*/4});
+}
+
+// block_q8_Kx4 codec: same shape as make_q8_0x4_scheme but a 256-element block,
+// a float32 delta per row (F32Pack), the same round-to-even -127/max Q8_K
+// quantize as make_q8_k_scheme, and the interleaved bsums field (with_bsums).
+inline std::unique_ptr<Halide::Approximation> make_q8_kx4_scheme(int blocklen) {
+    using namespace Halide;
+    return std::make_unique<Compose>(
+        RepackInterleavePack{256, blocklen, /*delta_bytes=*/4, /*with_bsums=*/true},
+        Apply{0, BytePack{}},  // codes -> bytes
+        Apply{1, F32Pack{}},   // scale -> float32 bytes
+        SymmetricAffineQuantize{256, 127, RoundingMode::NearestEvenClampedHigh,
+                                ScaleAnchor::ExtremeSignedValueTwoStep},
+        RepackRowReshape{256, /*n_rows=*/4});
+}
+
+// ---------------------------------------------------------------------------
+// 8. Repack weight un-interleave (for gemv/gemm): decode the 3-D interleaved
+// weight buffer (byte, k-block, col-group) into per-element codes + per-column
+// scale *bytes* (the composed Fp16/F32/E8M0 pack turns those into the float
+// scale -- no scale decode duplicated in the leaf), carrying the two column
+// dims (col-in-group j, col-group x) as explicit trailing dims so the
+// dimension-general LinearDequant/Codebook/scale packs (Halide::_) run unchanged
+// and the matmul reduces over (kk, blk). Decode-only (the weight is
+// pre-quantized; SeveredEncode is the severed encoder half). This is the
+// decode twin of RepackInterleavePack, for the four "simple" weight families.
+// ---------------------------------------------------------------------------
+enum class RepackWeightCode { SignedByte,    // Q8_0: whole signed int8
+                              SignedNibble,  // Q4_0: two's-complement 4-bit (repack's XOR-0x8)
+                              RawNibble };   // IQ4_NL/MXFP4: raw 4-bit codebook index
+
+class UnInterleaveWeight : public Halide::Approximation {
+public:
+    UnInterleaveWeight(int n_cols, int blocklen, int block_size,
+                       RepackWeightCode code_kind, ScaleFormat scale_kind)
+        : n_cols_(n_cols), blocklen_(blocklen), block_size_(block_size),
+          code_kind_(code_kind), scale_kind_(scale_kind) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "UnInterleaveWeight is decode-only (SeveredEncode is the "
+                              "severed encoder half of its TrustedInverse).\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func blocks = encoded[0];  // blocks(byte, k-block, col-group)
+        Var byte("byte"), kk("kk"), blk("blk"), j("j"), x("x");
+        // Per-column scale header width: fp16 = 2, f32 = 4, e8m0 = 1 byte/column.
+        int scale_stride = scale_width(scale_kind_);
+        int header = scale_stride * n_cols_;
+
+        Func codes("uninterleave_codes");
+        if (code_kind_ == RepackWeightCode::SignedByte) {
+            Expr qs_idx = (kk / blocklen_) * n_cols_ * blocklen_ + j * blocklen_ + (kk % blocklen_);
+            codes(kk, blk, j, x) = reinterpret<int8_t>(blocks(header + qs_idx, blk, x));
+        } else {
+            Expr half = kk / (block_size_ / 2);
+            Expr el = kk % (block_size_ / 2);
+            Expr qs_idx = (el / blocklen_) * n_cols_ * blocklen_ + j * blocklen_ + (el % blocklen_);
+            Expr byte_v = blocks(header + qs_idx, blk, x);
+            Expr nib = cast<int32_t>(nibble_of(byte_v, half != 0));
+            Expr code = code_kind_ == RepackWeightCode::SignedNibble ? select(nib < 8, nib, nib - 16) : nib;
+            codes(kk, blk, j, x) = cast<int8_t>(code);
+        }
+
+        // Pure addressing: gather column j's raw scale-header bytes; the composed
+        // Fp16Pack / F32Pack / E8M0Pack (dimension-general) turns them into the
+        // float scale, exactly as KQuantDeInterleave emits d_bytes for Fp16Pack.
+        Func scale_bytes("uninterleave_scale_bytes");
+        scale_bytes(byte, blk, j, x) = blocks(scale_stride * j + byte, blk, x);
+        return {{codes, scale_bytes}, {}};
+    }
+
+private:
+    int n_cols_, blocklen_, block_size_;
+    RepackWeightCode code_kind_;
+    ScaleFormat scale_kind_;
+};
+
+// Weight-decode scheme for a "simple" repack weight family (Q4_0/Q8_0/IQ4_NL/
+// MXFP4): un-interleave -> [codebook] -> one-level scale. The col dims ride the
+// dimension-general LinearDequant/Codebook via Halide::_. SeveredEncode (3-D)
+// stands in for the pre-quantized weight buffer, severed by the gemv/gemm
+// generator's compute_offline.
+inline std::unique_ptr<Halide::Approximation> make_repack_weight_scheme(
+    int n_cols, int blocklen, int block_bytes, RepackWeightCode code_kind, ScaleFormat scale_kind,
+    Halide::Buffer<int8_t> table = {}, int block_size = 32) {
+    using namespace Halide;
+    // Interpret the scale-header bytes UnInterleaveWeight gathers with the same
+    // packs the plain codecs use -- no per-kind scale decode duplicated here.
+    return std::make_unique<TrustedInverse>(
+        SeveredEncode{block_bytes, 3},
+        Compose{UnInterleaveWeight{n_cols, blocklen, block_size, code_kind, scale_kind},
+                Choose{code_kind == RepackWeightCode::RawNibble,
+                       Apply{0, Codebook{table}}, Identity{}},
+                Apply{1, make_scale_pack(scale_kind)},
+                LinearDequant{/*sub_size=*/0, /*has_super_d=*/false, /*has_min=*/false}});
+}
+
+// K-quant repack weight decode (block_q{4,5,6,2}_Kx8, n_cols=8): a bespoke,
+// verified-unchanged transcription of the repack_gemv_generators.cpp
+// weight_value helpers -- the interleaved 256-element super-block with a
+// two-level scale (and, for Q4_K/Q5_K, get_scale_min_k4 with the column index
+// standing in for the sub-block index). Its decode reads the severed 3-D
+// weight buffer and emits Wt(kk, blk, j, x) directly (kept as one leaf rather
+// than composed from LinearDequant + the scale-min packs, because the
+// interleave is intricate enough that a direct transcription is far less
+// error-prone; the SeveredEncode encoder half is severed by the matmul
+// generator's compute_offline). blocklen is the interleave width (4 or 8).
+enum class KQuantWeightFamily { Q4_K,
+                                Q5_K,
+                                Q6_K,
+                                Q2_K };
+
+// The genuinely repack-specific half of a K-quant weight decode: the *byte
+// addressing* that maps a logical (element kk, sub-block, column j, col-group
+// x) back to the interleaved block's scattered qs/qh/ql bytes and its scale
+// region. It is the K-quant analog of UnInterleaveWeight -- pure permutation,
+// no dequant arithmetic. It emits the same logical field slots the plain
+// K-quant decode's StructPack+packs produce, so the shared downstream
+// Compose (Fp16Pack on the fp16 headers, then LinearDequant, in
+// LOGICAL element order) finishes the job identically:
+//
+//   Q4_K/Q5_K: {d_bytes, dmin_bytes, scale_min, codes}   sub_size 32
+//   Q6_K:      {d_bytes,             scale,     codes}   sub_size 16 (no min)
+//   Q2_K:      {d_bytes, dmin_bytes, scale_min, codes}   sub_size 16
+//
+// (has-min families emit scale and min combined in one scale_min(plane, sub, ...)
+// field, plane 0 = scale / 1 = min -- the same shape K4ScaleMinPack and
+// PlanarBitPack's plane-axis mode produce, so LinearDequant consumes it
+// identically.) scale/min are produced as VALUES here (not bytes) because their
+// bit layouts can't be delegated to the plain packs: get_scale_min_k4 (Q4_K/Q5_K)
+// does its bit-math on what is the *column* index j in the repack while the
+// sub-block rides a separate axis -- an axis transpose K4ScaleMinPack's
+// (sub-first) interface can't express -- and the qs/qh code stream is
+// column-interleaved, so PlanarBitPack's contiguous-window assumption doesn't
+// hold. The scale index reduces to kk/sub_size in logical order for all four families
+// (verified: Q6_K's base_l/base_h and Q2_K's sm_idx both collapse to kk/16
+// since blocklen divides 16), which is exactly what LinearDequant
+// consumes, so no re-order is needed downstream.
+class KQuantDeInterleave : public Halide::Approximation {
+public:
+    KQuantDeInterleave(KQuantWeightFamily family, int blocklen)
+        : family_(family), blocklen_(blocklen) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "KQuantDeInterleave is decode-only (SeveredEncode is the severed encoder).\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func b = encoded[0];  // blocks(byte, k-superblock, col-group)
+        Var byte("byte"), kk("kk"), blk("blk"), sub("sub"), plane("plane"), j("j"), x("x");
+        const int bl = blocklen_;
+        const int nc = 8;
+
+        // has-min families (Q4_K/Q5_K/Q2_K) emit scale and min combined in one
+        // field scale_min(plane, sub, ...) (plane 0 = scale, 1 = min), the shape
+        // LinearDequant consumes; Q6_K (no min) emits a plain scale.
+        Func codes("kq_codes"), scale("kq_scale"), scale_min("kq_scale_min"), d_bytes("kq_d"), dmin_bytes("kq_dmin");
+
+        if (family_ == KQuantWeightFamily::Q4_K || family_ == KQuantWeightFamily::Q5_K) {
+            const bool is_q5 = family_ == KQuantWeightFamily::Q5_K;
+            const int kScalesOffset = 32;
+            const int kQsOffset = is_q5 ? 384 : 128;
+            const int kQhOffset = 128;  // Q5_K only
+
+            Expr iter = kk / 64, local64 = kk % 64, half64 = local64 / 32, lpos = local64 % 32;
+            Expr k_inner = lpos / bl, ii = lpos % bl, k = iter * (32 / bl) + k_inner;
+            Expr qs_byte = b(kQsOffset + k * nc * bl + j * bl + ii, blk, x);
+            Expr nibble = cast<int32_t>(nibble_of(qs_byte, half64 != 0));
+            Expr value = nibble;
+            if (is_q5) {
+                Expr qh_byte = b(kQhOffset + k_inner * (bl * nc) + j * bl + ii, blk, x);
+                Expr h_bit = cast<int32_t>((cast<uint32_t>(qh_byte) >> (iter * 2 + half64)) & 1);
+                value = nibble | (h_bit << 4);
+            }
+            codes(kk, blk, j, x) = value;
+
+            // get_scale_min_k4, addressed by window(sub) and bit-position j,
+            // scale and min combined into one (plane, sub) field.
+            Expr window = (sub / 4) * 48 + (sub % 4) * 12;
+            Expr jj = clamp(j - 4, 0, 3);
+            Expr sc = select(j < 4, b(kScalesOffset + window + j, blk, x) & 0x3f,
+                             cast<uint8_t>((b(kScalesOffset + window + 8 + jj, blk, x) & 0x0f) |
+                                           ((b(kScalesOffset + window + jj, blk, x) >> 6) << 4)));
+            Expr mn = select(j < 4, b(kScalesOffset + window + j + 4, blk, x) & 0x3f,
+                             cast<uint8_t>((b(kScalesOffset + window + 8 + jj, blk, x) >> 4) |
+                                           ((b(kScalesOffset + window + 4 + jj, blk, x) >> 6) << 4)));
+            scale_min(plane, sub, blk, j, x) = cast<uint8_t>(select(plane == 0, sc, mn));
+
+            d_bytes(byte, blk, j, x) = b(2 * j + byte, blk, x);
+            dmin_bytes(byte, blk, j, x) = b(16 + 2 * j + byte, blk, x);
+            return {{d_bytes, dmin_bytes, scale_min, codes}, {}};
+        } else if (family_ == KQuantWeightFamily::Q6_K) {
+            const int kScalesOffset = 16, kQlOffset = 144, kQhOffset = 1168;
+            const int blocks_per_half = 64 / bl;
+            constexpr int kQlSize = (256 * 8) / 2, kQhSize = (256 * 8) / 4;
+
+            Expr local128 = kk % 128, is_high = local128 >= 64, pos64 = local128 % 64, i = pos64 % bl;
+            Expr base_l = kk - i - select(is_high, 64, 0), base_h = base_l + 64;
+            Expr k = (base_l / 128) * blocks_per_half + (base_l % 128) / bl;
+            Expr ql_byte = b(kQlOffset + clamp(k * nc * bl + j * bl + i, 0, kQlSize - 1), blk, x);
+            Expr nibble = cast<int32_t>(nibble_of(ql_byte, is_high));
+            Expr qh_shift = select(is_high, ((base_h % 128) / 32) * 2, ((base_l % 128) / 32) * 2);
+            Expr qh_idx_l = (base_l / 128) * 32 + ((base_l + i) % 32);
+            Expr qh_idx_h = (base_h / 128) * 32 + ((base_h + i) % 32);
+            Expr qh_off_l = clamp((qh_idx_l / bl) * (bl * nc) + j * bl + (qh_idx_l % bl), 0, kQhSize - 1);
+            Expr qh_off_h = clamp((qh_idx_h / bl) * (bl * nc) + j * bl + (qh_idx_h % bl), 0, kQhSize - 1);
+            Expr qh_byte = b(kQhOffset + select(is_high, qh_off_h, qh_off_l), blk, x);
+            Expr hi2 = cast<int32_t>((qh_byte >> qh_shift) & 3);
+            codes(kk, blk, j, x) = (nibble | (hi2 << 4)) - 32;
+
+            // 16 plain signed int8 scales, sub = kk/16 (see class comment).
+            scale(sub, blk, j, x) = cast<int32_t>(reinterpret<int8_t>(b(kScalesOffset + sub * nc + j, blk, x)));
+            d_bytes(byte, blk, j, x) = b(2 * j + byte, blk, x);
+            return {{d_bytes, scale, codes}, {}};
+        } else {  // Q2_K (blocklen fixed 8 upstream)
+            const int kDminOffset = 16, kScalesOffset = 32, kQsOffset = 160;
+            Expr half = kk / 128, local = kk % 128, subg = local / 32, rem32 = local % 32;
+            Expr k = half * 4 + rem32 / bl, i = rem32 % bl;
+            Expr qs_byte = b(kQsOffset + k * nc * bl + j * bl + i, blk, x);
+            codes(kk, blk, j, x) = cast<int32_t>((qs_byte >> (subg * 2)) & 3);
+
+            // scale/min nibble pair, sub = kk/16 (see class comment), combined
+            // into one (plane, sub) field (plane 0 = scale, 1 = min).
+            Expr sm_idx = (sub / 8) * 64 + ((sub % 8) / 2) * 16 + j * 2 + (sub % 2);
+            Expr sm_byte = b(kScalesOffset + sm_idx, blk, x);
+            scale_min(plane, sub, blk, j, x) = cast<int32_t>(nibble_of(sm_byte, plane != 0));
+            d_bytes(byte, blk, j, x) = b(2 * j + byte, blk, x);
+            dmin_bytes(byte, blk, j, x) = b(kDminOffset + 2 * j + byte, blk, x);
+            return {{d_bytes, dmin_bytes, scale_min, codes}, {}};
+        }
+    }
+
+private:
+    KQuantWeightFamily family_;
+    int blocklen_;
+};
+
+// K-quant repack weight scheme: the de-interleave addressing leaf composed
+// with the SAME arithmetic pieces the plain K-quant decode uses (Fp16Pack for
+// the fp16 headers, then LinearDequant in logical element order),
+// paired with a severed 3-D encoder (the weight is pre-quantized; encode is
+// severed by compute_offline). No BlockReshape: the weight stays block-indexed
+// (kk, blk) with the column dims (j, x) riding through.
+inline std::unique_ptr<Halide::Approximation> make_kquant_repack_weight_scheme(
+    KQuantWeightFamily family, int blocklen, int block_bytes) {
+    using namespace Halide;
+    const bool has_min = family != KQuantWeightFamily::Q6_K;
+    const int sub_size = family == KQuantWeightFamily::Q4_K || family == KQuantWeightFamily::Q5_K ? 32 : 16;
+    return std::make_unique<TrustedInverse>(
+        SeveredEncode{block_bytes, 3},
+        Compose{KQuantDeInterleave{family, blocklen},
+                Apply{0, Fp16Pack{}},                               // d_bytes -> d
+                Choose{has_min, Apply{1, Fp16Pack{}}, Identity{}},  // dmin_bytes -> dmin
+                LinearDequant{sub_size, /*has_super_d=*/true, has_min}});
+}
+
+// The make_*() factories below each return one owned Halide::Approximation
+// (as a std::unique_ptr<Halide::Approximation>, the framework's polymorphic
+// scheme handle) -- a single leaf, a Compose, or a TrustedInverse, whichever
+// the format actually is, never a single-element Compose wrapper. A Compose's
+// stage 0 is outermost (its encoded output is the whole thing's result) and
+// its last stage is innermost (closest to the original values).
+//
+// Native (in-Halide-quantizable) formats are a plain Compose. Extern-delegated
+// formats -- whose forward quantize is an opaque GGML extern -- are a
+// TrustedInverse pairing ExternQuantize (encode) with a Compose (decode); see
+// Halide::TrustedInverse and section 4 above.
+
+// The invertible combined-bit split/combine (see CombineBits above), with
+// its own per-part packing folded in via make_block_layout: `fields` lists
+// the {low, high} on-disk sub-fields (in on-disk order, tagged with their
+// logical slots -- 0 for low, 1 for high, so CombineBits sees {low, high}
+// regardless of on-disk order). `fields` is a trailing variadic pack (see
+// make_block_layout), so `high_weight, offset` come first.
+template<typename... Fields>
+inline std::unique_ptr<Halide::Approximation> make_combined_bit_codec(
+    int high_weight, int offset, Fields &&...fields) {
+    using namespace Halide;
+    return std::make_unique<Compose>(
+        make_block_layout(std::forward<Fields>(fields)...).layout,
+        CombineBits{high_weight, offset});
+}
+
+struct CodePackField {
+    std::unique_ptr<Halide::Approximation> pack;
+    int bytes;
+};
+
+inline CodePackField make_code_pack(int block_size, int code_bits, int qmax) {
+    using namespace Halide;
+    if (code_bits == 4) {
+        return {nibble_pack(block_size, qmax), block_size / 2};
+    }
+    if (code_bits == 5) {
+        // Q5_0/Q5_1's split 5-bit code (a 4-bit low nibble plus a 5th high
+        // bit) -- exactly the K-quant combined-bit-code shape make_q5_k_scheme
+        // uses for its own adjacent {qh; qs} region (see the deleted
+        // FiveBitPack's comment, above BitPack, for the verified-by-hand
+        // equivalence): qh's bit `kk` is element kk's own high bit, i.e.
+        // le_bit_pack()'s pos_count=1 addressing (unlike Q5_K's per-window
+        // rotating_bit_pack -- Q5_0/Q5_1's qh is one flat bit array over the
+        // *whole* block, not windowed); the nibble half is the ordinary
+        // nibble_pack. `qmax` becomes CombineBits' final recentering offset
+        // (0 for Q5_1's already-unsigned affine codes) rather than a per-part
+        // qmax, since the parts here are raw, uncentered digits.
+        return {make_combined_bit_codec(
+                    16, qmax,
+                    FieldSpec{1, 4, le_bit_pack()},                          // qh -> high bit
+                    FieldSpec{0, block_size / 2, nibble_pack(block_size)}),  // qs -> low nibble
+                block_size / 2 + 4};
+    }
+    if (code_bits == 1) {
+        return {std::make_unique<BitPack>(), block_size / 8};
+    }
+    return {std::make_unique<BytePack>(), block_size};
+}
+
+// quantize -> pack codes -> pack scale -> concatenate into one byte buffer
+// with the scale stored first (matching every GGML block_* struct's
+// `{fp16 d; ...qs;}` layout) -> reshape, plus the on-disk block byte count
+// (2 + the code field's own width) -- the source of truth every Generator
+// switch used to re-derive by hand. `layout` picks BlockReshape's flat-row
+// (quantize_row/dequantize_row) vs block-indexed (vec_dot/repack, whose
+// Inputs are already block-indexed so the reshape is a lossless passthrough)
+// shape -- see the Layout enum above; this one factory now covers what used
+// to be a separate make_symmetric_block_codec()/make_symmetric_block_scheme()
+// pair.
+inline SchemeAndBytes make_symmetric_block_scheme(
+    int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor, int code_bits,
+    Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    auto [code_pack, code_bytes] = make_code_pack(block_size, code_bits, qmax);
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},    // scale
+        FieldSpec{0, code_bytes, std::move(code_pack)});  // codes
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                SymmetricAffineQuantize{block_size, qmax, rounding, anchor},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// quantize -> pack codes -> pack scale -> pack min -> concatenate (scale, min,
+// codes; matching block_q4_1/block_q5_1's `{fp16 d; fp16 m; ...qs;}` layout)
+// -> reshape -- the affine (min+scale) counterpart to
+// make_symmetric_block_scheme(), used by Q4_1 (code_bits=4, ClampedInt8). Q5_1
+// pairs with make_affine_5bit_block_scheme() below instead, since it also
+// needs the qh high-bit field.
+inline SchemeAndBytes make_affine_block_scheme(
+    int block_size, int levels, AffineRounding rounding, int code_bits, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    CodePackField code = make_code_pack(block_size, code_bits, /*qmax=*/0);
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},    // scale
+        FieldSpec{2, 2, std::make_unique<Fp16Pack>()},    // min
+        FieldSpec{0, code.bytes, std::move(code.pack)});  // codes
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                AffineQuantize{block_size, levels, rounding},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// Symmetric quantize (like make_symmetric_block_scheme()) but 5-bit -- now
+// just make_symmetric_block_scheme with code_bits=5, since make_code_pack's
+// code_bits==5 case already assembles the {qh; qs} split (matching
+// block_q5_0's `{fp16 d; qh[4]; qs[16];}`) via the combined-bit codec.
+// `qmax` is always 16 (5-bit signed range [-16, 15]). Kept as its own named
+// entry point (rather than collapsed into SchemeKind::Symmetric in
+// symmetric_quant_generators.cpp) so Q5_0/Q5_1's CMakeLists GENERATOR_ARGS
+// (kind=symmetric_5bit/affine_5bit) don't need to change in lockstep.
+inline SchemeAndBytes make_symmetric_5bit_block_scheme(int block_size, int qmax,
+                                                       Layout layout = Layout::FlatRow) {
+    return make_symmetric_block_scheme(block_size, qmax, RoundingMode::TruncateHalfUpWithOffset,
+                                       ScaleAnchor::ExtremeSignedValue, /*code_bits=*/5, layout);
+}
+
+// Affine quantize (like make_affine_block_scheme()) but 5-bit -- likewise now
+// just make_affine_block_scheme with code_bits=5, matching block_q5_1's
+// `{fp16 d; fp16 m; qh[4]; qs[16];}`. `qmax=0` passed to make_code_pack here
+// (unlike Q5_0's 16): AffineQuantize's codes are already unsigned [0,
+// levels], not centered, so there's no offset to re-apply before splitting
+// into nibble+high-bit.
+inline SchemeAndBytes make_affine_5bit_block_scheme(int block_size, int levels,
+                                                    AffineRounding rounding,
+                                                    Layout layout = Layout::FlatRow) {
+    return make_affine_block_scheme(block_size, levels, rounding, /*code_bits=*/5, layout);
+}
+
+// Symmetric byte-packed quantize (like make_symmetric_block_scheme() with
+// code_bits=8) plus AppendSums's derived 's' field (SumMode::ScaledFloat),
+// matching block_q8_1's `{fp16 d; fp16 s; qs[32];}` -- Q8_1's scheme. Q8_1 is
+// activation-only (GGML has no public to_float for it), so there's normally
+// no dequantize_row Generator for this scheme's flat-array variant below --
+// but its decode() is still correct and used by any vec_dot pairing against
+// Q8_1 as the activation format. AppendSums needs no Apply wrapper: it
+// consumes and produces the *whole* current list (like quantize itself),
+// not just one element of it.
+inline SchemeAndBytes make_symmetric_byte_sum_block_scheme(int block_size, int qmax,
+                                                           Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},            // scale
+        FieldSpec{2, 2, std::make_unique<Fp16Pack>()},            // sum
+        FieldSpec{0, block_size, std::make_unique<BytePack>()});  // codes
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                AppendSums{block_size, SumMode::ScaledFloat},
+                SymmetricAffineQuantize{block_size, qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// Q8_K: activation-only (quantize_row only, matching Q8_1's own situation
+// above -- see q8_k_generators.cpp), one 256-element superblock, plain int8
+// codes (BytePack), one float32 (not fp16) scale (F32Pack), and 16
+// per-group int32-then-int16 sums (AppendSums, SumMode::RawInt16) -- {float d;
+// qs[256]; bsums[16];}, 292 bytes. RoundingMode::NearestEvenClampedHigh/
+// ScaleAnchor::ExtremeSignedValueTwoStep reproduce GGML's exact
+// nearest-int-then-reciprocal-pair quantizer bit-for-bit -- see their own
+// comments in SymmetricAffineQuantize above for why the usual
+// Nearest/ExtremeSignedValue formulas aren't equivalent here.
+inline SchemeAndBytes make_q8_k_scheme(int block_size, int qmax, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 4, std::make_unique<F32Pack>()},                         // scale
+        FieldSpec{0, block_size, std::make_unique<BytePack>()},               // codes
+        FieldSpec{2, (block_size / 16) * 2, std::make_unique<Int16Pack>()});  // bsums
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                AppendSums{16, SumMode::RawInt16},
+                SymmetricAffineQuantize{block_size, qmax, RoundingMode::NearestEvenClampedHigh,
+                                        ScaleAnchor::ExtremeSignedValueTwoStep},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// ---------------------------------------------------------------------------
+// Factory helpers for the shared extern-delegated shapes. Each is a plain
+// function that assembles a TrustedInverse{ExternQuantize, Compose{...}} out
+// of the section-4 leaves -- transparent (it returns exactly the Compose you'd
+// write by hand), unlike the bespoke Approximation subclasses this file used
+// to have. The canonical composition for each family lives here once; the
+// per-format make_*_scheme() below are just its parameters.
+// ---------------------------------------------------------------------------
+
+// Codebook formats (IQ4_NL/MXFP4/TQ2_0/TQ1_0/NVFP4): unpack the code field,
+// look it up in `table`, unpack the scale field (`scale_fmt`/`num_scales`
+// picking the pack and its width via make_scale_pack/scale_width, rather
+// than a (pack, width) pair kept in sync by hand), one-level scale multiply,
+// reshape. `scale_first` is the on-disk field order ({scale; codes} vs
+// {codes; scale}); StructPack normalizes both to logical {codes, scale}.
+inline SchemeAndBytes make_codebook_scheme(
+    std::string extern_name, int block_size, Halide::Buffer<int8_t> table,
+    std::unique_ptr<Halide::Approximation> code_pack, int code_bytes,
+    ScaleFormat scale_fmt, int num_scales, bool scale_first, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    int scale_bytes = scale_width(scale_fmt, num_scales);
+    std::vector<FieldSpec> fields;
+    if (scale_first) {
+        fields.push_back({1, scale_bytes, make_scale_pack(scale_fmt)});
+        fields.push_back({0, code_bytes, std::move(code_pack)});
+    } else {
+        fields.push_back({0, code_bytes, std::move(code_pack)});
+        fields.push_back({1, scale_bytes, make_scale_pack(scale_fmt)});
+    }
+    BlockLayout bl = make_block_layout(std::move(fields));
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{std::move(extern_name)},
+                Compose{
+                    std::move(bl.layout),
+                    Apply{0, Codebook{std::move(table)}},  // codes -> codebook values
+                    LinearDequant{num_scales == 1 ? 0 : block_size / num_scales,
+                                  /*has_super_d=*/false, /*has_min=*/false},
+                    BlockReshape{block_size, layout == Layout::BlockIndexed},
+                }),
+            bl.bytes};
+}
+
+// K-quant formats (Q2_K/Q3_K/Q4_K/Q5_K/Q6_K): `fields` lists every on-disk
+// field (d, [dmin,] scale_min, code -- in on-disk order, tagged with their
+// logical slots; see FieldSpec) that make_block_layout unpacks, then the
+// two-level scale multiply and reshape. This is the old KQuantDequantize
+// parameter list -- now assembling a Compose, not a class. `fields` is a
+// trailing variadic pack (see make_block_layout for why), so `layout` --
+// unlike every other make_*_scheme() here -- comes before it rather than
+// trailing with a default.
+template<typename... Fields>
+inline SchemeAndBytes make_k_quant_scheme(
+    std::string extern_name, int block_size, int sub_size, bool has_min, Layout layout,
+    Fields &&...fields) {
+    using namespace Halide;
+    BlockLayout bl = make_block_layout(std::forward<Fields>(fields)...);
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{std::move(extern_name)},
+                Compose{std::move(bl.layout),
+                        LinearDequant{sub_size, /*has_super_d=*/true, has_min},
+                        BlockReshape{block_size, layout == Layout::BlockIndexed}}),
+            bl.bytes};
+}
+
+// IQ grid formats (IQ2_S/IQ3_XXS/IQ3_S): a bespoke grid+sign+scale decode leaf
+// that emits values in the superblock's nested structure, with
+// BlockReshape(`block_extents`) doing the flat<->block reshape. `block_bytes`
+// is the leaf's own hand-verified on-disk block size -- unlike the
+// field-table schemes above it can't be derived from a FieldSpec list (the
+// grid leaves are deliberately NOT field-table-decomposed; see section 6's
+// design note), so it's declared here, once, next to the leaf that owns it,
+// and returned in SchemeAndBytes like every other make_*_scheme().
+inline SchemeAndBytes make_grid_scheme(
+    std::string extern_name, int block_bytes, std::unique_ptr<Halide::Approximation> grid_leaf,
+    std::vector<int> block_extents, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{std::move(extern_name)},
+                Compose{std::move(grid_leaf), BlockReshape{std::move(block_extents), layout == Layout::BlockIndexed}}),
+            block_bytes};
+}
+
+inline SchemeAndBytes make_severed_grid_scheme(
+    int block_bytes, std::unique_ptr<Halide::Approximation> grid_leaf,
+    std::vector<int> block_extents, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return {std::make_unique<TrustedInverse>(
+                SeveredEncode{block_bytes},
+                Compose{std::move(grid_leaf), BlockReshape{std::move(block_extents), layout == Layout::BlockIndexed}}),
+            block_bytes};
+}
+
+// IQ4_NL: 32-element blocks, 4-bit codes into a 16-value non-uniform
+// codebook, one fp16 scale per block -- {fp16 d; qs[16];}, 18 bytes. Extern
+// quantize; decode unpacks {code_bytes, scale_bytes} (ScaleFirst), looks the
+// nibbles up in the codebook, applies the one fp16 scale, and reshapes to a
+// flat row.
+inline SchemeAndBytes make_iq4_nl_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+                                       1, 13, 25, 38, 53, 69, 89, 113};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_iq4nl");
+    return make_codebook_scheme("iq4_nl_quantize_via_ggml", 32, table,
+                                nibble_pack(32), 16,
+                                ScaleFormat::Fp16, /*num_scales=*/1, /*scale_first=*/true, layout);
+}
+
+// MXFP4: 32-element blocks, 4-bit codes into the same-shaped 16-value
+// codebook as IQ4_NL (different values), one E8M0 (1-byte, power-of-two)
+// scale per block -- {e8m0 e; qs[16];}, 17 bytes.
+inline SchemeAndBytes make_mxfp4_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_mxfp4");
+    return make_codebook_scheme("mxfp4_quantize_via_ggml", 32, table,
+                                nibble_pack(32), 16,
+                                ScaleFormat::E8M0, /*num_scales=*/1, /*scale_first=*/true, layout);
+}
+
+// TQ2_0: 256-element superblock, 2-bit codes (each in {0,1,2}, meaning
+// {-1,0,1}) via crumb_pack(128)'s window-interleaved layout, one
+// fp16 scale -- {qs[64]; fp16 d;}, 66 bytes -- qs *before* d, unlike most
+// formats here (StructPack's codes-first field order below).
+inline SchemeAndBytes make_tq2_0_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[4] = {-1, 0, 1, 0};  // index 3 is never produced
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_tq2_0");
+    return make_codebook_scheme("tq2_0_quantize_via_ggml", 256, table,
+                                crumb_pack(128), 64,
+                                ScaleFormat::Fp16, /*num_scales=*/1, /*scale_first=*/false, layout);
+}
+
+// TQ1_0: 256-element superblock, base-3 codes (each in {0,1,2}, meaning
+// {-1,0,1}) via TritPack's 5-trits/byte (+4-trits/byte tail) packing, one
+// fp16 scale -- {qs[48]; qh[4]; fp16 d;}, 54 bytes -- qs+qh (combined, 52
+// bytes) *before* d, like TQ2_0. Reuses TQ2_0's exact {-1, 0, 1, unused}
+// codebook (TritPack's codes are the same raw 0/1/2 digit either way).
+inline SchemeAndBytes make_tq1_0_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[4] = {-1, 0, 1, 0};  // index 3 is never produced
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_tq1_0");
+    return make_codebook_scheme("tq1_0_quantize_via_ggml", 256, table,
+                                std::make_unique<TritPack>(), 52,
+                                ScaleFormat::Fp16, /*num_scales=*/1, /*scale_first=*/false, layout);
+}
+
+// NVFP4: 64-element block, 4 sub-blocks of 16 elements each, 4-bit codes
+// into the same 16-value codebook MXFP4 uses (NVFP4 is MXFP4 with
+// finer-grained scales), one UE4M3 scale *per sub-block* via UE4M3Pack --
+// {d[4]; qs[32];}, 36 bytes -- LinearDequant's num_scales=4 (not 1) is what
+// makes each sub-block's dequantize use its own scale byte instead of one
+// shared scale for the whole 64-element block.
+inline SchemeAndBytes make_nvfp4_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_nvfp4");
+    return make_codebook_scheme("nvfp4_quantize_via_ggml", 64, table,
+                                nibble_pack(16), 32,
+                                ScaleFormat::UE4M3, /*num_scales=*/4, /*scale_first=*/true, layout);
+}
+
+// Q4_K: 256-element superblock, 8 sub-blocks of 32 elements each, plain
+// 4-bit codes (nibble_pack(64)) and get_scale_min_k4-packed
+// (scale, min) pairs (K4ScaleMinPack) -- {fp16 d; fp16 dmin; scales[12];
+// qs[128];}, 144 bytes, fields already in {d, dmin, scale_min, code} logical
+// order. Two-level scale (d*scale(sub)*code - dmin*min(sub)).
+inline SchemeAndBytes make_q4_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q4_k_quantize_via_ggml", 256, 32, /*has_min=*/true, layout,
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},         // d
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},         // dmin
+        FieldSpec{2, 12, std::make_unique<K4ScaleMinPack>()},  // scale_min
+        FieldSpec{3, 128, nibble_pack(64)});                   // codes
+}
+
+// Q5_K: same super-block/sub-block/scale-min shape as Q4_K, but each code
+// is 5 bits: a plain 4-bit low nibble (nibble_pack(64)) plus a
+// 5th high bit from a separate 32-byte, 8-window rotating-bit array
+// (rotating_bit_pack(32)) -- {fp16 d; fp16 dmin; scales[12]; qh[32];
+// qs[128];}, 176 bytes. qh+qs are adjacent in memory, treated as one
+// 160-byte combined "code" field, split by an inner make_combined_bit_codec
+// (qh before qs on-disk), offset=0 since Q5_K's code is a plain 0..31
+// unsigned magnitude, not recentered.
+inline SchemeAndBytes make_q5_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q5_k_quantize_via_ggml", 256, 32, /*has_min=*/true, layout,
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},         // d
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},         // dmin
+        FieldSpec{2, 12, std::make_unique<K4ScaleMinPack>()},  // scale_min
+        // Combined 5-bit code: qh (high bit) + qs (low nibble), on-disk
+        // qh before qs; offset 0 (plain 0..31).
+        FieldSpec{3, 160,
+                  make_combined_bit_codec(
+                      16, 0,
+                      FieldSpec{1, 32, rotating_bit_pack(32)},  // qh -> high bit
+                      FieldSpec{0, 128, nibble_pack(64)})});    // qs -> low nibble
+}
+
+// Q2_K: 256-element superblock, 16 sub-blocks of 16 elements each, plain
+// 2-bit codes (crumb_pack(128)) and independent per-sub-block
+// nibble-pair (scale, min) via PlanarBitPack's plane-axis mode (low nibble =
+// scale = plane 0, high nibble = min = plane 1; no bit-interleaving across
+// sub-blocks) -- {scales[16]; qs[64]; fp16 d; fp16 dmin;}, 84 bytes, fields
+// on-disk in {scale_min, code, d, dmin} order (scale_min/code *before* d/dmin,
+// unlike Q4_K/Q5_K), normalized by their own slots back to {d, dmin,
+// scale_min, code}.
+inline SchemeAndBytes make_q2_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q2_k_quantize_via_ggml", 256, 16, /*has_min=*/true, layout,
+        FieldSpec{2, 16, std::make_unique<PlanarBitPack>(4, 16, 0, /*plane_axis=*/true)},  // scales
+        FieldSpec{3, 64, crumb_pack(128)},                                                 // qs
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},                                     // d
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()});                                    // dmin
+}
+
+// Q3_K: 256-element superblock, 16 sub-blocks of 16 elements each, no min
+// (symmetric, not affine) -- each code is 3 bits: 2 low bits
+// (crumb_pack(128)) plus a high bit from a 32-byte, 8-window
+// rotating-bit "hmask" array (rotating_bit_pack(32)), recentered by -4
+// (CombineBits offset=4, matching a signed [-4, 3] range); scale is 16
+// SIGNED 6-bit values, its own bit-interleaving distinct from get_scale_min_k4
+// (Q3KScalePack) -- {hmask[32]; qs[64]; scales[12]; fp16 d;}, 110 bytes.
+// hmask+qs are adjacent in memory, treated as one 96-byte combined "code"
+// field (hmask before qs on-disk); on-disk {code, scale, d} normalizes to
+// logical {d, scale, code}.
+inline SchemeAndBytes make_q3_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q3_k_quantize_via_ggml", 256, 16, /*has_min=*/false, layout,
+        // Combined 3-bit code: hmask (high bit) + qs (low 2 bits).
+        // On-disk hmask before qs; offset 4 (recenters to signed [-4, 3]).
+        FieldSpec{2, 96,
+                  make_combined_bit_codec(
+                      4, 4,
+                      FieldSpec{1, 32, rotating_bit_pack(32)},  // hmask -> high bit
+                      FieldSpec{0, 64, crumb_pack(128)})},      // qs -> low 2 bits
+        FieldSpec{1, 12, std::make_unique<Q3KScalePack>()},     // scales
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()});         // d
+}
+
+// Q6_K: 256-element superblock, 16 sub-blocks of 16 elements each, no min --
+// each code is 6 bits: a plain 4-bit low nibble over *two* 128-element
+// halves (nibble_pack(128)) plus 2 high bits
+// (crumb_pack(128)), recentered by -32; scale is 16 plain SIGNED
+// int8 values, no bit-interleaving at all (BytePack -- its plain
+// reinterpret<int8_t> is exactly what this needs) -- {ql[128]; qh[64];
+// scales[16]; fp16 d;}, 210 bytes. ql+qh are adjacent in memory, treated as
+// one 192-byte combined "code" field (ql before qh on-disk).
+inline SchemeAndBytes make_q6_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q6_k_quantize_via_ggml", 256, 16, /*has_min=*/false, layout,
+        // Combined 6-bit code: ql (low nibble) + qh (high 2 bits).
+        // On-disk ql before qh; offset 32 (recenters to signed [-32, 31]).
+        FieldSpec{2, 192,
+                  make_combined_bit_codec(
+                      16, 32,
+                      FieldSpec{0, 128, nibble_pack(128)},  // ql -> low nibble
+                      FieldSpec{1, 64, crumb_pack(128)})},  // qh -> high 2 bits
+        FieldSpec{1, 16, std::make_unique<BytePack>()},     // scales, 16 signed int8 values
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()});     // d
+}
+
+// IQ2_S/IQ3_XXS/IQ3_S: see IQ2SGridDequantize/IQ3XXSGridDequantize/
+// IQ3SGridDequantize above for the bit-layout rationale -- each is a bespoke,
+// self-contained grid+sign+scale decode leaf (its three bit layouts share no
+// sub-formula worth abstracting). Extern quantize; the grid leaf's decode
+// produces block-indexed values, and BlockReshape composes the flat<->block
+// reshape on top.
+inline SchemeAndBytes make_iq2_s_scheme(Layout layout = Layout::FlatRow) {
+    return make_grid_scheme("iq2_s_quantize_via_ggml", 82, std::make_unique<IQ2SGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq3_xxs_scheme(Layout layout = Layout::FlatRow) {
+    return make_grid_scheme("iq3_xxs_quantize_via_ggml", 98, std::make_unique<IQ3XXSGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq3_s_scheme(Layout layout = Layout::FlatRow) {
+    return make_grid_scheme("iq3_s_quantize_via_ggml", 110, std::make_unique<IQ3SGridDequantize>(), {8, 4, 8}, layout);
+}
+
+// IQ2_XS/IQ2_XXS/IQ1_S/IQ1_M: importance-matrix-only formats with no forward
+// map -- SeveredEncode stands in for the (always-severed) encode half so the
+// dequantize/vec_dot still go through approximate_by/compute_offline. Block
+// bytes: 74 / 66 / 50 / 56.
+inline SchemeAndBytes make_iq2_xs_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(74, std::make_unique<IQ2XSGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq2_xxs_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(66, std::make_unique<IQ2XXSGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq1_s_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(50, std::make_unique<IQ1SGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq1_m_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(56, std::make_unique<IQ1MGridDequantize>(), {8, 4, 8}, layout);
+}
+
+// IQ4_XS: 256-element superblock, 8 sub-blocks of 32 elements, the superblock
+// generalization of IQ4_NL's fixed 16-value codebook -- plain 4-bit codes
+// (nibble_pack(32)) into the same kvalues_iq4nl table, scaled by
+// `d * (ls - 32)`, a two-level scale (no min) whose per-sub-block `ls` is
+// bit-interleaved across two byte fields (IQ4XSScalePack) -- {fp16 d;
+// scales_h[2]; scales_l[4]; qs[128];}, 136 bytes.
+inline SchemeAndBytes make_iq4_xs_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+                                       1, 13, 25, 38, 53, 69, 89, 113};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_iq4nl_xs");
+    // scales_h leads a 2-slot group together with scales_l: IQ4XSScalePack's
+    // decode consumes both (scales_h bytes, then scales_l bytes) to recover
+    // one `scale(sub)` field, the same grouped-field shape make_code_pack's
+    // code_bits==5 combined codec uses for Q5_0/Q5_1's {nibble, qh} pair.
+    BlockLayout bl = make_block_layout(
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},                     // d
+        FieldSpec{1, 2, std::make_unique<IQ4XSScalePack>(), /*arity=*/2},  // scales_h (leads the group)
+        FieldSpec{2, 4, nullptr},                                          // scales_l (part of the group above)
+        FieldSpec{3, 128, nibble_pack(32)});                               // qs -> nibbles
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{"iq4_xs_quantize_via_ggml"},
+                Compose{
+                    std::move(bl.layout),
+                    Apply{2, Codebook{table}},  // nibbles -> codebook values (slot2: {d, scale, qs})
+                    LinearDequant{32, /*has_super_d=*/true, /*has_min=*/false},
+                    BlockReshape{256, layout == Layout::BlockIndexed},
+                }),
+            bl.bytes};
+}
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/halide/repack_matmul_generator.cpp b/apps/ggml/halide/repack_matmul_generator.cpp
new file mode 100644
index 000000000000..cf95ed192a9f
--- /dev/null
+++ b/apps/ggml/halide/repack_matmul_generator.cpp
@@ -0,0 +1,263 @@
+// Generic, family-driven repack gemv/gemm, the matmul counterpart of the
+// repack quantize_mat codecs. Like the vec_dot generators, the weight and
+// activation operands are decoded through the Approximation framework
+// (approximate_by + compute_offline), and the interleaved weight layout is a
+// lossless relayout (UnInterleaveWeight) composed in front of the same lossy
+// quant -- the col dims ride the dimension-general LinearDequant/Codebook via
+// Halide::_. One generator backs every (family, n_cols, blocklen) gemv library
+// (32 hand-rolled kernels -> 2 generic generators). This file covers the four
+// "simple" weight families (Q4_0/Q8_0/IQ4_NL/MXFP4); the interleaved K-quant
+// weights layer on top later.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class WFamily { Q4_0,
+                     Q8_0,
+                     IQ4_NL,
+                     MXFP4,
+                     Q4_K,
+                     Q5_K,
+                     Q6_K,
+                     Q2_K };
+
+inline bool is_kquant(WFamily f) {
+    return f == WFamily::Q4_K || f == WFamily::Q5_K || f == WFamily::Q6_K || f == WFamily::Q2_K;
+}
+
+struct WeightSpec {
+    std::unique_ptr<Halide::Approximation> scheme;
+    int block_bytes;
+};
+
+// Weight decode scheme + on-disk block byte width for a simple family. Byte
+// widths: fp16-delta families = 2*n_cols header + payload; mxfp4 = n_cols E8M0
+// header. Nibble payload = 16*n_cols, byte payload = 32*n_cols.
+WeightSpec weight_spec(WFamily fam, int n_cols, int blocklen) {
+    static const int8_t kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+                                      1, 13, 25, 38, 53, 69, 89, 113};
+    static const Buffer<int8_t> iq4nl_lut(const_cast<int8_t *>(kIq4nl), 16, "kvalues_iq4nl_gemv");
+    static const int8_t kMxfp4[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+    static const Buffer<int8_t> mxfp4_lut(const_cast<int8_t *>(kMxfp4), 16, "kvalues_mxfp4_gemv");
+
+    switch (fam) {
+    case WFamily::Q4_0:
+        return {make_repack_weight_scheme(n_cols, blocklen, 18 * n_cols,
+                                          RepackWeightCode::SignedNibble, ScaleFormat::Fp16),
+                18 * n_cols};
+    case WFamily::Q8_0:
+        return {make_repack_weight_scheme(n_cols, blocklen, 34 * n_cols,
+                                          RepackWeightCode::SignedByte, ScaleFormat::Fp16),
+                34 * n_cols};
+    case WFamily::IQ4_NL:
+        return {make_repack_weight_scheme(n_cols, blocklen, 18 * n_cols,
+                                          RepackWeightCode::RawNibble, ScaleFormat::Fp16, iq4nl_lut),
+                18 * n_cols};
+    case WFamily::MXFP4:
+        return {make_repack_weight_scheme(n_cols, blocklen, 17 * n_cols,
+                                          RepackWeightCode::RawNibble, ScaleFormat::E8M0, mxfp4_lut),
+                17 * n_cols};
+    // K-quant weights (n_cols=8 always): a bespoke interleaved decode leaf.
+    case WFamily::Q4_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q4_K, blocklen, 1152), 1152};
+    case WFamily::Q5_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q5_K, blocklen, 1408), 1408};
+    case WFamily::Q6_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q6_K, blocklen, 1680), 1680};
+    case WFamily::Q2_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q2_K, blocklen, 672), 672};
+    }
+    _halide_internal_error << "RepackGemvGenerator: bad family\n";
+    return {};
+}
+
+// gemv: one plain Q8_0 activation row x every column of a repack-interleaved
+// weight matrix -> s(col-in-group, col-group).
+class RepackGemvGenerator : public Generator<RepackGemvGenerator> {
+public:
+    GeneratorParam<WFamily> family{
+        "family",
+        WFamily::Q4_0,
+        {{"q4_0", WFamily::Q4_0},
+         {"q8_0", WFamily::Q8_0},
+         {"iq4_nl", WFamily::IQ4_NL},
+         {"mxfp4", WFamily::MXFP4},
+         {"q4_k", WFamily::Q4_K},
+         {"q5_k", WFamily::Q5_K},
+         {"q6_k", WFamily::Q6_K},
+         {"q2_k", WFamily::Q2_K}}};
+    GeneratorParam<int> n_cols{"n_cols", 4};
+    GeneratorParam<int> blocklen{"blocklen", 4};
+
+    void configure() {
+        bool kq = is_kquant(family);
+        int block_size = kq ? 256 : 32;
+        WeightSpec w = weight_spec(family, n_cols, blocklen);
+        // Activation: plain Q8_K for K-quant weights, plain Q8_0 otherwise.
+        auto act = kq ? make_q8_k_scheme(256, 127, Layout::BlockIndexed).scheme : make_symmetric_block_scheme(32, 127, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
+        int act_bytes = kq ? (4 + 256 + 2 * (256 / 16)) : (2 + 32);
+
+        ImageParam weight_blocks(UInt(8), 3, "weight_blocks");  // (byte, k-block, col-group)
+        ImageParam act_blocks(UInt(8), 2, "act_blocks");        // plain Q8_0/Q8_K (byte, k-block)
+
+        Var kk("kk"), blk("blk"), j("j"), x("x");
+        Func Wt("wt_naive"), Vec("act_naive");
+        Wt(kk, blk, j, x) = 0.0f;
+        Vec(kk, blk) = 0.0f;
+
+        RDom r(0, block_size, 0, weight_blocks.dim(1).extent(), "r");
+        Func s("s");
+        s(j, x) = 0.0f;
+        s(j, x) += Wt(r.x, r.y, j, x) * Vec(r.x, r.y);
+
+        ApproximationResult wr = Wt.approximate_by(*w.scheme, {s});
+        ApproximationResult ar = Vec.approximate_by(*act, {s});
+        s.update().eager_inline({wr.replacement, ar.replacement});
+
+        std::vector<Func> sever = wr.encoded;
+        sever.insert(sever.end(), ar.encoded.begin(), ar.encoded.end());
+        std::vector<ImageParam> bind = {weight_blocks, act_blocks};
+        Pipeline({s}).compute_offline(sever, bind);
+
+        for (Func h : wr.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+        for (Func h : ar.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+
+        weight_blocks.dim(0).set_bounds(0, w.block_bytes);
+        weight_blocks.dim(1).set_min(0);
+        weight_blocks.dim(2).set_min(0);
+        act_blocks.dim(0).set_bounds(0, act_bytes);
+        act_blocks.dim(1).set_min(0);
+        s.output_buffer().dim(0).set_bounds(0, n_cols);
+        s.output_buffer().dim(1).set_min(0);
+
+        add_input(weight_blocks);
+        add_input(act_blocks);
+        add_output(s);
+    }
+
+    void generate() {
+    }
+};
+
+// gemm activation: `nr` rows packed 4-at-a-time into the SAME interleaved
+// block layout the weight uses -- so it decodes through make_repack_weight_scheme
+// with n_cols=4 (row group of 4), the 4 "columns" being the 4 packed rows.
+// Q8_0x4 (fp16 scale, 32-block, 136 B) for simple weights; Q8_Kx4 (f32 scale,
+// 256-block, 1168 B incl. dropped bsums) for K-quant weights.
+struct ActSpec {
+    std::unique_ptr<Halide::Approximation> scheme;
+    int block_bytes;
+    int block_size;
+};
+ActSpec act_spec(bool kquant, int blocklen) {
+    if (kquant) {
+        // block_q8_Kx4 is 1168 B, but the gemm reads only the f32 d[4] header
+        // (16 B) + interleaved qs (1024 B) = 1040 B; the 128 B of bsums are
+        // never touched, so the input is bound to 1040, not the full 1168.
+        return {make_repack_weight_scheme(4, blocklen, 1040, RepackWeightCode::SignedByte,
+                                          ScaleFormat::F32, {}, 256),
+                1040, 256};
+    }
+    return {make_repack_weight_scheme(4, blocklen, 136, RepackWeightCode::SignedByte,
+                                      ScaleFormat::Fp16),
+            136, 32};
+}
+
+// gemm: 4 packed activation rows x every column of a repack-interleaved weight
+// matrix -> s(col-in-group j, col-group x, row-in-group m, row-group y). Both
+// operands are interleaved codecs decoded through the framework; the only
+// difference from gemv is that the activation is interleaved too (4 rows) and
+// the output gains the two activation-lane dims.
+class RepackGemmGenerator : public Generator<RepackGemmGenerator> {
+public:
+    GeneratorParam<WFamily> family{
+        "family",
+        WFamily::Q4_0,
+        {{"q4_0", WFamily::Q4_0},
+         {"q8_0", WFamily::Q8_0},
+         {"iq4_nl", WFamily::IQ4_NL},
+         {"mxfp4", WFamily::MXFP4},
+         {"q4_k", WFamily::Q4_K},
+         {"q5_k", WFamily::Q5_K},
+         {"q6_k", WFamily::Q6_K},
+         {"q2_k", WFamily::Q2_K}}};
+    GeneratorParam<int> n_cols{"n_cols", 4};
+    GeneratorParam<int> blocklen{"blocklen", 4};
+
+    void configure() {
+        bool kq = is_kquant(family);
+        int block_size = kq ? 256 : 32;
+        WeightSpec w = weight_spec(family, n_cols, blocklen);
+        ActSpec a = act_spec(kq, blocklen);
+
+        ImageParam weight_blocks(UInt(8), 3, "weight_blocks");  // (byte, k-block, col-group)
+        ImageParam act_blocks(UInt(8), 3, "act_blocks");        // (byte, k-block, row-group)
+
+        Var kk("kk"), blk("blk"), j("j"), x("x"), m("m"), y("y");
+        Func Wt("wt_naive"), Act("act_naive");
+        Wt(kk, blk, j, x) = 0.0f;
+        Act(kk, blk, m, y) = 0.0f;
+
+        RDom r(0, block_size, 0, weight_blocks.dim(1).extent(), "r");
+        Func s("s");
+        s(j, x, m, y) = 0.0f;
+        s(j, x, m, y) += Wt(r.x, r.y, j, x) * Act(r.x, r.y, m, y);
+
+        ApproximationResult wr = Wt.approximate_by(*w.scheme, {s});
+        ApproximationResult ar = Act.approximate_by(*a.scheme, {s});
+        s.update().eager_inline({wr.replacement, ar.replacement});
+
+        std::vector<Func> sever = wr.encoded;
+        sever.insert(sever.end(), ar.encoded.begin(), ar.encoded.end());
+        std::vector<ImageParam> bind = {weight_blocks, act_blocks};
+        Pipeline({s}).compute_offline(sever, bind);
+
+        for (Func h : wr.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+        for (Func h : ar.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+
+        weight_blocks.dim(0).set_bounds(0, w.block_bytes);
+        weight_blocks.dim(1).set_min(0);
+        weight_blocks.dim(2).set_min(0);
+        act_blocks.dim(0).set_bounds(0, a.block_bytes);
+        act_blocks.dim(1).set_min(0);
+        act_blocks.dim(2).set_min(0);
+        s.output_buffer().dim(0).set_bounds(0, n_cols);
+        s.output_buffer().dim(1).set_min(0);
+        s.output_buffer().dim(2).set_bounds(0, 4);
+        s.output_buffer().dim(3).set_min(0);
+
+        add_input(weight_blocks);
+        add_input(act_blocks);
+        add_output(s);
+    }
+
+    void generate() {
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(RepackGemvGenerator, repack_gemv)
+HALIDE_REGISTER_GENERATOR(RepackGemmGenerator, repack_gemm)
diff --git a/apps/ggml/halide/repack_quantize_mat_generators.cpp b/apps/ggml/halide/repack_quantize_mat_generators.cpp
new file mode 100644
index 000000000000..27425c819719
--- /dev/null
+++ b/apps/ggml/halide/repack_quantize_mat_generators.cpp
@@ -0,0 +1,121 @@
+// From-scratch Halide reimplementation of GGML's "repack" quantize_mat
+// kernels (see src/ggml-cpu/repack.cpp: ggml_quantize_mat_q8_0_4x4_generic /
+// ggml_quantize_mat_q8_0_4x8_generic / ggml_quantize_mat_q8_K_4x4_generic /
+// ggml_quantize_mat_q8_K_4x8_generic upstream, as of GGML v0.15.3). These
+// take 4 contiguous rows of `k` floats (row r at x[r*k .. r*k+k)) and
+// interleave them into ONE activation-format block per `k`-sized chunk,
+// where 4 per-row values are laid out consecutively (in groups of
+// `blck_size_interleave`) instead of one row's worth at a time -- this is
+// the packed activation format the corresponding repack_gemv/repack_gemm
+// kernels consume. There are only 4 distinct interleavings (2 activation
+// formats x 2 interleave widths), reused across every repack weight type
+// that shares that (activation, interleave) pair -- see k_repack_entries in
+// providers/ggml_provider.cpp and this file's registration in
+// halide_provider.cpp.
+//
+// block_q8_0x4 layout (136 bytes, one per 32-element chunk x 4 rows):
+//   byte 0-7:    4 fp16 deltas, one per row, in row order
+//   byte 8-135:  128 signed int8 quants, interleaved in groups of
+//                `blck_size_interleave` (4 or 8) per row
+//
+// block_q8_Kx4 layout (1168 bytes, one per 256-element chunk x 4 rows):
+//   byte 0-15:    4 float32 deltas, one per row, in row order
+//   byte 16-1039: 1024 signed int8 quants, interleaved the same way
+//   byte 1040-1167: 64 signed int16 "bsums" (sum of quants in groups of 16,
+//                  scattered across rows/groups by the same index mapping
+//                  GGML uses -- see index_q8_k below)
+//
+// Q8_0's per-row scale is the same amax/127 symmetric scale as plain Q8_0
+// (see quant_components.h's make_symmetric_block_scheme()), using round()
+// (roundf, not round-to-even). Q8_K's per-row scale is the same -127/max
+// signed scale as plain Q8_K (see quant_components.h's make_q8_k_scheme()),
+// using the same round-to-nearest-even magic-number trick -- but unlike
+// plain Q8_K's quantize_row, this repack version has no final MIN(127, ...)
+// clamp (safe here since the scale is derived from this exact block's own
+// amax, so values can never exceed +-127 already).
+//
+// This is intentionally unscheduled beyond the minimum Halide requires for
+// legality (an update-defined Func can't stay inline) -- scheduling for
+// performance is a later step.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+constexpr int kQK8_0 = 32;
+constexpr int kBlockBytesQ8_0x4 = 4 * 2 + kQK8_0 * 4;  // 136
+
+constexpr int kQK_K = 256;
+constexpr int kNumGroups = kQK_K / 16;                                     // 16
+constexpr int kBlockBytesQ8_Kx4 = 4 * 4 + kQK_K * 4 + kNumGroups * 4 * 2;  // 1168
+
+// Shared "quantize_mat" pipeline: a 2-D activation x(col, row in [0,4)) flows
+// through the codec `scheme` (block-relayout + Q8 quantize + interleave) via
+// the same approximate_by/compute_offline idiom as codec_generator_base.h's
+// Quantize direction -- the encode half is adopted as the output block buffer.
+struct QuantizeMatPipe {
+    ImageParam x;
+    Func blocks_out;
+};
+inline QuantizeMatPipe build_quantize_mat(std::unique_ptr<Approximation> scheme, int block_bytes) {
+    ImageParam x(Float(32), 2, "x");  // dim 0: col-within-row (mult. of block), dim 1: row (4)
+    Var col("col"), row("row"), byte("byte"), ib("ib");
+    Func identity("qm_identity");
+    identity(col, row) = x(col, row);
+
+    ApproximationResult r = Func(x).approximate_by(*scheme, {identity});
+    for (Func h : r.handles) {
+        if (h.has_update_definition()) {
+            h.compute_root();
+        }
+    }
+
+    ImageParam blocks_in(UInt(8), 2, "blocks_in");
+    ComputeOfflineResult q = Pipeline({identity}).compute_offline(r.encoded, {blocks_in});
+
+    Func blocks_out("blocks");
+    blocks_out(byte, ib) = q.offline.outputs()[0](byte, ib);
+    blocks_out.output_buffer().dim(0).set_bounds(0, block_bytes);
+    blocks_out.output_buffer().dim(1).set_min(0);
+    x.dim(0).set_min(0);
+    x.dim(1).set_bounds(0, 4);
+    return {x, blocks_out};
+}
+
+// q8_0_4x4 / q8_0_4x8 differ only by interleave width (blocklen).
+template<int Blocklen>
+class Q8_0QuantizeMatGenerator : public Generator<Q8_0QuantizeMatGenerator<Blocklen>> {
+public:
+    void configure() {
+        QuantizeMatPipe p = build_quantize_mat(make_q8_0x4_scheme(Blocklen), kBlockBytesQ8_0x4);
+        this->add_input(p.x);
+        this->add_output(p.blocks_out);
+    }
+    void generate() {
+    }
+};
+
+// q8_k_4x4 / q8_k_4x8: same shared pipeline, the Q8_K interleaved codec.
+template<int Blocklen>
+class Q8_KQuantizeMatGenerator : public Generator<Q8_KQuantizeMatGenerator<Blocklen>> {
+public:
+    void configure() {
+        QuantizeMatPipe p = build_quantize_mat(make_q8_kx4_scheme(Blocklen), kBlockBytesQ8_Kx4);
+        this->add_input(p.x);
+        this->add_output(p.blocks_out);
+    }
+    void generate() {
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(Q8_0QuantizeMatGenerator<4>, q8_0_4x4_quantize_mat)
+HALIDE_REGISTER_GENERATOR(Q8_0QuantizeMatGenerator<8>, q8_0_4x8_quantize_mat)
+HALIDE_REGISTER_GENERATOR(Q8_KQuantizeMatGenerator<4>, q8_k_4x4_quantize_mat)
+HALIDE_REGISTER_GENERATOR(Q8_KQuantizeMatGenerator<8>, q8_k_4x8_quantize_mat)
diff --git a/apps/ggml/halide/symmetric_quant_generators.cpp b/apps/ggml/halide/symmetric_quant_generators.cpp
new file mode 100644
index 000000000000..0f53f213a575
--- /dev/null
+++ b/apps/ggml/halide/symmetric_quant_generators.cpp
@@ -0,0 +1,120 @@
+// Generic, GeneratorParam-driven quantize/dequantize pair for GGML's legacy
+// per-block quantized formats (see quant_components.h for the reusable
+// Approximation pieces this assembles). "Q4_0"/"Q4_1"/"Q5_0"/"Q5_1"/"Q8_0"/
+// "Q8_1" are not distinct C++ classes here -- they're just different
+// GENERATOR_ARGS instantiations of the same generator template, registered
+// in CMakeLists.txt as e.g. q4_0_quantize/q4_0_dequantize.
+//
+// Quantize and dequantize share every GeneratorParam (the scheme they
+// build is identical, just run in opposite directions), so rather than two
+// classes each redeclaring the same params, this is one class template
+// parameterized on Direction, following
+// apps/linear_algebra/src/blas_l1_generators.cpp's AXPYGenerator<T>
+// precedent (one generator template, registered multiple times under
+// different names/template args).
+//
+// The whole pipeline -- for *either* direction -- is built once in
+// configure(), not generate(): a single Func::approximate_by() +
+// Pipeline::compute_offline() call on a genuinely real ImageParam (not a
+// placeholder) produces both an "offline" half (the encode/quantize side,
+// still depending on that real ImageParam) and an "online" half (the
+// decode/dequantize side, reading from whatever ImageParam
+// compute_offline() severed it to instead). Each direction just adopts
+// whichever half applies to it as its own Input/Output, via
+// GeneratorBase::add_input(const ImageParam&)/add_output(const Func&) --
+// new overloads added to Generator.h/.cpp for exactly this use (see there),
+// since the stock add_input<Buffer<>>()/add_output<Buffer<>>() only ever
+// mint fresh, undefined ports for generate() to fill in later. generate()
+// is therefore an empty stub: by the time it would run, there's nothing
+// left to do.
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by() and Pipeline::compute_offline(). This
+// configure()/generate() body is identical across every *_quant_generators.cpp
+// file in this directory, so it lives in codec_generator_base.h's
+// CodecGeneratorBase<Derived, dir> instead of being repeated here -- this
+// class only needs to supply its own GeneratorParams and a build_scheme().
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+// Which of quant_components.h's make_*_scheme() factories to use -- the one
+// axis that can't be reduced to a GeneratorParam value alone, since each
+// scheme needs a different subset/arity of the other params below.
+enum class SchemeKind { Symmetric,
+                        Affine,
+                        Symmetric5Bit,
+                        Affine5Bit,
+                        SymmetricByteSum,
+                        Q8K };
+
+template<Direction dir>
+class SymmetricCodecGenerator : public CodecGeneratorBase<SymmetricCodecGenerator<dir>, dir> {
+public:
+    GeneratorParam<int> block_size{"block_size", 32};
+    GeneratorParam<int> qmax{"qmax", 127};
+    GeneratorParam<int> code_bits{"code_bits", 8};
+    GeneratorParam<int> levels{"levels", 15};
+    GeneratorParam<RoundingMode> rounding{
+        "rounding",
+        RoundingMode::Nearest,
+        {{"nearest", RoundingMode::Nearest},
+         {"truncate_half_up_with_offset", RoundingMode::TruncateHalfUpWithOffset},
+         {"sign_only", RoundingMode::SignOnly}}};
+    GeneratorParam<ScaleAnchor> anchor{
+        "anchor",
+        ScaleAnchor::AbsMax,
+        {{"abs_max", ScaleAnchor::AbsMax},
+         {"extreme_signed", ScaleAnchor::ExtremeSignedValue},
+         {"mean_abs", ScaleAnchor::MeanAbs}}};
+    GeneratorParam<AffineRounding> affine_rounding{
+        "affine_rounding",
+        AffineRounding::ClampedInt8,
+        {{"clamped_int8", AffineRounding::ClampedInt8},
+         {"unclamped_uint8", AffineRounding::UnclampedUint8}}};
+    GeneratorParam<SchemeKind> kind{
+        "kind",
+        SchemeKind::Symmetric,
+        {{"symmetric", SchemeKind::Symmetric},
+         {"affine", SchemeKind::Affine},
+         {"symmetric_5bit", SchemeKind::Symmetric5Bit},
+         {"affine_5bit", SchemeKind::Affine5Bit},
+         {"symmetric_byte_sum", SchemeKind::SymmetricByteSum},
+         {"q8k", SchemeKind::Q8K}}};
+
+    SchemeAndBytes build_scheme() const {
+        // switch's controlling expression can't resolve GeneratorParam<T>'s
+        // implicit conversion operators unambiguously -- .value() sidesteps
+        // that by returning the plain SchemeKind directly. Each make_*_scheme()
+        // now returns its own block_bytes alongside the scheme (computed from
+        // the same field list it builds internally), so there's no byte
+        // arithmetic to duplicate here.
+        switch (kind.value()) {
+        case SchemeKind::Symmetric:
+            return make_symmetric_block_scheme(block_size, qmax, rounding, anchor, code_bits);
+        case SchemeKind::Affine:
+            return make_affine_block_scheme(block_size, levels, affine_rounding, code_bits);
+        case SchemeKind::Symmetric5Bit:
+            return make_symmetric_5bit_block_scheme(block_size, qmax);
+        case SchemeKind::Affine5Bit:
+            return make_affine_5bit_block_scheme(block_size, levels, affine_rounding);
+        case SchemeKind::SymmetricByteSum:
+            return make_symmetric_byte_sum_block_scheme(block_size, qmax);
+        case SchemeKind::Q8K:
+            return make_q8_k_scheme(block_size, qmax);
+        }
+        _halide_internal_error << "unreachable SchemeKind\n";
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(SymmetricCodecGenerator<Direction::Quantize>, symmetric_quantize)
+HALIDE_REGISTER_GENERATOR(SymmetricCodecGenerator<Direction::Dequantize>, symmetric_dequantize)
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
new file mode 100644
index 000000000000..381f50bb7b06
--- /dev/null
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -0,0 +1,148 @@
+// Generic, family-driven vec_dot for the symmetric/affine per-block formats,
+// the vec_dot counterpart of symmetric_quant_generators.cpp's
+// SymmetricCodecGenerator. "q4_0_vec_dot"/"q4_1_vec_dot"/... are PARAMS
+// instantiations of this one generator (registered in CMakeLists.txt), not
+// per-format C++ classes. Weight and activation are both block-indexed codecs
+// from quant_components.h; VecDotGeneratorBase splices them via approximate_by/
+// compute_offline (see vec_dot_generator_base.h) -- generate() never calls
+// Approximation::encode()/decode() directly.
+//
+// The weight is one of the symmetric-family kinds (symmetric / affine /
+// symmetric_5bit / affine_5bit); the activation is Q8_0 or Q8_1. Single-scale
+// symmetric weights x Q8_0 reach an SDOT Int(32) inner dot; affine (+min) and
+// mismatched-block pairings (Q1_0 block 128 x Q8_0 block 32) fall back to a
+// Float reduction.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "vec_dot_generator_base.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class WKind { Symmetric,
+                   Affine,
+                   Symmetric5Bit,
+                   Affine5Bit };
+enum class AKind { Q8_0,
+                   Q8_1 };
+
+class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGenerator> {
+public:
+    GeneratorParam<int> block_size{"block_size", 32};
+
+    GeneratorParam<WKind> w_kind{
+        "w_kind",
+        WKind::Symmetric,
+        {{"symmetric", WKind::Symmetric},
+         {"affine", WKind::Affine},
+         {"symmetric_5bit", WKind::Symmetric5Bit},
+         {"affine_5bit", WKind::Affine5Bit}}};
+    GeneratorParam<AKind> a_kind{
+        "a_kind",
+        AKind::Q8_0,
+        {{"q8_0", AKind::Q8_0},
+         {"q8_1", AKind::Q8_1}}};
+
+    // symmetric / symmetric_5bit weight params
+    GeneratorParam<int> w_qmax{"w_qmax", 8};
+    GeneratorParam<int> w_code_bits{"w_code_bits", 4};
+    GeneratorParam<RoundingMode> w_rounding{
+        "w_rounding",
+        RoundingMode::TruncateHalfUpWithOffset,
+        {{"nearest", RoundingMode::Nearest},
+         {"truncate_half_up_with_offset", RoundingMode::TruncateHalfUpWithOffset},
+         {"sign_only", RoundingMode::SignOnly}}};
+    GeneratorParam<ScaleAnchor> w_anchor{
+        "w_anchor",
+        ScaleAnchor::ExtremeSignedValue,
+        {{"abs_max", ScaleAnchor::AbsMax},
+         {"extreme_signed", ScaleAnchor::ExtremeSignedValue},
+         {"mean_abs", ScaleAnchor::MeanAbs}}};
+
+    // affine / affine_5bit weight params
+    GeneratorParam<int> w_levels{"w_levels", 15};
+    GeneratorParam<AffineRounding> w_affine_rounding{
+        "w_affine_rounding",
+        AffineRounding::ClampedInt8,
+        {{"clamped_int8", AffineRounding::ClampedInt8},
+         {"unclamped_uint8", AffineRounding::UnclampedUint8}}};
+
+    // activation param (Q8_0/Q8_1 are always 8-bit int8 codes)
+    GeneratorParam<int> a_qmax{"a_qmax", 127};
+
+    VecDotSpec build_vec_dot() const {
+        int wbs = block_size;
+
+        std::unique_ptr<Halide::Approximation> wc;
+        int wb;
+        ScheduleKind sched;
+        switch (w_kind.value()) {
+        case WKind::Symmetric:
+            wc = make_symmetric_block_scheme(wbs, w_qmax, w_rounding, w_anchor, w_code_bits, Layout::BlockIndexed).scheme;
+            wb = 2 + (w_code_bits == 4 ? wbs / 2 : (w_code_bits == 1 ? wbs / 8 : wbs));
+            // TODO(ggml-on-qk, SDOT): the mature hoist_invariants() can't lift the
+            // per-block scale out of the reduction here. Unlike
+            // test/correctness/struct_type_dot_product.cpp (which eager_inline()s
+            // the dequantizer Funcs directly, exposing the scale as a leaf), this
+            // app builds the dequant via approximate_by()'s round-trip replacement,
+            // and a single eager_inline() of that replacement leaves the scale
+            // behind decode-chain Func boundaries hoist_invariants() can't see
+            // through. Correct (non-SDOT) Float schedule for now; see the base
+            // header's SDOT branch. Restoring SDOT needs deeper inlining of the
+            // decode chain or a hoist_invariants() that sees through it.
+            sched = ScheduleKind::Float;
+            break;
+        case WKind::Affine:
+            wc = make_affine_block_scheme(wbs, w_levels, w_affine_rounding, w_code_bits, Layout::BlockIndexed).scheme;
+            wb = 2 + 2 + (w_code_bits == 4 ? wbs / 2 : wbs);
+            sched = ScheduleKind::Float;
+            break;
+        case WKind::Symmetric5Bit:
+            wc = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed).scheme;
+            wb = 2 + 4 + wbs / 2;
+            // TODO(ggml-on-qk): the alpha rfactor(HoistInvariantFactor) hoisted
+            // q5_0's per-block scale even though the 5-bit code is assembled via
+            // CombineBits (nibble | (high_bit << 4)); the mature hoist_invariants()
+            // does not recognize the scale as a distributable factor through that
+            // reconstruction and errors. Use the correct (non-SDOT) Float schedule
+            // until either hoist_invariants() is taught this shape or the 5-bit
+            // codec is restructured to expose the scale as a clean leaf.
+            sched = ScheduleKind::Float;
+            break;
+        case WKind::Affine5Bit:
+            wc = make_affine_5bit_block_scheme(wbs, w_levels, w_affine_rounding, Layout::BlockIndexed).scheme;
+            wb = 2 + 2 + 4 + wbs / 2;
+            sched = ScheduleKind::Float;
+            break;
+        }
+
+        // Q8_0/Q8_1 activations are 32-element blocks. Build the codec at that
+        // natural block size, then Reblock to the weight's block size (a no-op
+        // when they already match, e.g. Q4_0/Q8_0); the byte width stays the
+        // natural-block width since y_blocks is stored at 32-element blocks.
+        const int a_nat = 32;
+        std::unique_ptr<Halide::Approximation> ac;
+        int ab;
+        switch (a_kind.value()) {
+        case AKind::Q8_0:
+            ac = make_symmetric_block_scheme(a_nat, a_qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
+            ab = 2 + a_nat;
+            break;
+        case AKind::Q8_1:
+            ac = make_symmetric_byte_sum_block_scheme(a_nat, a_qmax, Layout::BlockIndexed).scheme;
+            ab = 2 + 2 + a_nat;
+            break;
+        }
+        ac = reblock_activation(std::move(ac), a_nat, wbs);
+
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched};
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(SymmetricVecDotGenerator, symmetric_vec_dot)
diff --git a/apps/ggml/halide/test_bf16.cpp b/apps/ggml/halide/test_bf16.cpp
new file mode 100644
index 000000000000..87aa15a2ce74
--- /dev/null
+++ b/apps/ggml/halide/test_bf16.cpp
@@ -0,0 +1,45 @@
+// Standalone round-trip check: the from-scratch Halide BF16 kernels (both
+// directions fully native) vs GGML's own reference implementation, reached
+// via the public ggml_get_type_traits() API.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_BF16);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_BF16, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_bf16(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_bf16(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_f16.cpp b/apps/ggml/halide/test_f16.cpp
new file mode 100644
index 000000000000..ec1f8f619f7b
--- /dev/null
+++ b/apps/ggml/halide/test_f16.cpp
@@ -0,0 +1,45 @@
+// Standalone round-trip check: the from-scratch Halide F16 kernels (both
+// directions fully native) vs GGML's own reference implementation, reached
+// via the public ggml_get_type_traits() API.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_F16);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_F16, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_f16(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_f16(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq1_m.cpp b/apps/ggml/halide/test_iq1_m.cpp
new file mode 100644
index 000000000000..1f9324bee8d1
--- /dev/null
+++ b/apps/ggml/halide/test_iq1_m.cpp
@@ -0,0 +1,48 @@
+// Standalone check: the from-scratch Halide IQ1_M dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq1_m doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq1_m symbol, with a uniform (all-1.0)
+// weighting for consistency with IQ2_XXS/IQ2_XS/IQ1_S (whose equivalent
+// weights are a hard requirement, not just optional here).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ1_M);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ1_M);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ1_M, k);
+
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq1_m(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq1_m(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq1_s.cpp b/apps/ggml/halide/test_iq1_s.cpp
new file mode 100644
index 000000000000..67703e23492a
--- /dev/null
+++ b/apps/ggml/halide/test_iq1_s.cpp
@@ -0,0 +1,47 @@
+// Standalone check: the from-scratch Halide IQ1_S dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq1_s doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq1_s symbol, with a uniform (all-1.0)
+// weighting since it hard-requires a non-null quant_weights.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ1_S);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ1_S);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ1_S, k);
+
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq1_s(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq1_s(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq2_s.cpp b/apps/ggml/halide/test_iq2_s.cpp
new file mode 100644
index 000000000000..ec5bce940667
--- /dev/null
+++ b/apps/ggml/halide/test_iq2_s.cpp
@@ -0,0 +1,48 @@
+// Standalone round-trip check: the from-scratch Halide IQ2_S dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ2_S);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ2_S);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ2_S, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq2_s(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq2_s(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq2_xs.cpp b/apps/ggml/halide/test_iq2_xs.cpp
new file mode 100644
index 000000000000..05d738095761
--- /dev/null
+++ b/apps/ggml/halide/test_iq2_xs.cpp
@@ -0,0 +1,47 @@
+// Standalone check: the from-scratch Halide IQ2_XS dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq2_xs doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq2_xs symbol, with a uniform (all-1.0)
+// weighting since it hard-requires a non-null quant_weights.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ2_XS);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ2_XS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ2_XS, k);
+
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq2_xs(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq2_xs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq2_xxs.cpp b/apps/ggml/halide/test_iq2_xxs.cpp
new file mode 100644
index 000000000000..abc1bfdc9aa7
--- /dev/null
+++ b/apps/ggml/halide/test_iq2_xxs.cpp
@@ -0,0 +1,52 @@
+// Standalone check: the from-scratch Halide IQ2_XXS dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq2_xxs doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq2_xxs symbol, called with nrows=1 and no
+// importance matrix to get a plain per-row reference.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    // The nearest-neighbor grid search this quantizer uses needs its lookup
+    // table built first (see ggml_provider.cpp's comment on ggml_quantize_init).
+    ggml_quantize_init(GGML_TYPE_IQ2_XXS);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ2_XXS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ2_XXS, k);
+
+    // quantize_row_iq2_xxs_impl hard-requires a non-null quant_weights
+    // (GGML_ASSERT) -- a uniform (all-1.0) weighting treats every element
+    // as equally important, the closest equivalent to "no weighting".
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq2_xxs(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq2_xxs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq3_s.cpp b/apps/ggml/halide/test_iq3_s.cpp
new file mode 100644
index 000000000000..2e5132b99486
--- /dev/null
+++ b/apps/ggml/halide/test_iq3_s.cpp
@@ -0,0 +1,48 @@
+// Standalone round-trip check: the from-scratch Halide IQ3_S dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ3_S);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ3_S);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ3_S, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq3_s(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq3_s(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq3_xxs.cpp b/apps/ggml/halide/test_iq3_xxs.cpp
new file mode 100644
index 000000000000..5dd80636469b
--- /dev/null
+++ b/apps/ggml/halide/test_iq3_xxs.cpp
@@ -0,0 +1,48 @@
+// Standalone round-trip check: the from-scratch Halide IQ3_XXS dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ3_XXS);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ3_XXS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ3_XXS, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq3_xxs(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq3_xxs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq4_nl.cpp b/apps/ggml/halide/test_iq4_nl.cpp
new file mode 100644
index 000000000000..456ae55cc725
--- /dev/null
+++ b/apps/ggml/halide/test_iq4_nl.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide IQ4_NL dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK4_NL (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ4_NL);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ4_NL, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq4_nl(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq4_nl(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq4_xs.cpp b/apps/ggml/halide/test_iq4_xs.cpp
new file mode 100644
index 000000000000..628a3a39613a
--- /dev/null
+++ b/apps/ggml/halide/test_iq4_xs.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide IQ4_XS dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ4_XS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ4_XS, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq4_xs(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq4_xs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_mxfp4.cpp b/apps/ggml/halide/test_mxfp4.cpp
new file mode 100644
index 000000000000..9d46c4bfa2df
--- /dev/null
+++ b/apps/ggml/halide/test_mxfp4.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide MXFP4 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_MXFP4 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_MXFP4);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_MXFP4, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_mxfp4(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_mxfp4(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_nvfp4.cpp b/apps/ggml/halide/test_nvfp4.cpp
new file mode 100644
index 000000000000..3e1e9208dd48
--- /dev/null
+++ b/apps/ggml/halide/test_nvfp4.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide NVFP4 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_NVFP4 (64)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_NVFP4);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_NVFP4, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_nvfp4(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_nvfp4(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q1_0.cpp b/apps/ggml/halide/test_q1_0.cpp
new file mode 100644
index 000000000000..a267bbfa5c6e
--- /dev/null
+++ b/apps/ggml/halide/test_q1_0.cpp
@@ -0,0 +1,45 @@
+// Standalone round-trip check: the from-scratch Halide Q1_0 kernels (both
+// directions fully native) vs GGML's own reference implementation, reached
+// via the public ggml_get_type_traits() API.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK1_0 (128)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q1_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q1_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q1_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q1_0(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q2_k.cpp b/apps/ggml/halide/test_q2_k.cpp
new file mode 100644
index 000000000000..0fd2c5deee59
--- /dev/null
+++ b/apps/ggml/halide/test_q2_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q2_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q2_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q2_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q2_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q2_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q3_k.cpp b/apps/ggml/halide/test_q3_k.cpp
new file mode 100644
index 000000000000..767eb193727b
--- /dev/null
+++ b/apps/ggml/halide/test_q3_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q3_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q3_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q3_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q3_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q3_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q4_0.cpp b/apps/ggml/halide/test_q4_0.cpp
new file mode 100644
index 000000000000..f568b0f17401
--- /dev/null
+++ b/apps/ggml/halide/test_q4_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q4_0 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK4_0 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q4_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q4_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q4_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q4_0(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q4_1.cpp b/apps/ggml/halide/test_q4_1.cpp
new file mode 100644
index 000000000000..bdbb4f9f0c5d
--- /dev/null
+++ b/apps/ggml/halide/test_q4_1.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q4_1 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK4_1 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q4_1);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q4_1, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q4_1(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q4_1(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q4_k.cpp b/apps/ggml/halide/test_q4_k.cpp
new file mode 100644
index 000000000000..62174d0e5e3d
--- /dev/null
+++ b/apps/ggml/halide/test_q4_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q4_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q4_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q4_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q4_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q4_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q5_0.cpp b/apps/ggml/halide/test_q5_0.cpp
new file mode 100644
index 000000000000..d64bf692ff80
--- /dev/null
+++ b/apps/ggml/halide/test_q5_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q5_0 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK5_0 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q5_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q5_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q5_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q5_0(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q5_1.cpp b/apps/ggml/halide/test_q5_1.cpp
new file mode 100644
index 000000000000..2b096dfa3ebd
--- /dev/null
+++ b/apps/ggml/halide/test_q5_1.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q5_1 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK5_1 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q5_1);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q5_1, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q5_1(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q5_1(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q5_k.cpp b/apps/ggml/halide/test_q5_k.cpp
new file mode 100644
index 000000000000..d54e62c4f9dc
--- /dev/null
+++ b/apps/ggml/halide/test_q5_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q5_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q5_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q5_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q5_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q5_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q6_k.cpp b/apps/ggml/halide/test_q6_k.cpp
new file mode 100644
index 000000000000..804fb2f35081
--- /dev/null
+++ b/apps/ggml/halide/test_q6_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q6_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q6_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q6_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q6_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q6_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q8_0.cpp b/apps/ggml/halide/test_q8_0.cpp
new file mode 100644
index 000000000000..95135acf835b
--- /dev/null
+++ b/apps/ggml/halide/test_q8_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q8_0 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK8_0 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q8_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q8_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q8_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q8_0(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q8_1.cpp b/apps/ggml/halide/test_q8_1.cpp
new file mode 100644
index 000000000000..fe570667642f
--- /dev/null
+++ b/apps/ggml/halide/test_q8_1.cpp
@@ -0,0 +1,37 @@
+// Standalone check: the from-scratch Halide Q8_1 quantize kernel vs GGML's
+// own reference implementation, reached via the public
+// ggml_get_type_traits() API. Q8_1 is an activation-only format -- GGML has
+// no public to_float for it, so there's no dequantize round-trip to check
+// here, only the quantize output.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK8_1 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q8_1);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q8_1, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q8_1(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q8_k.cpp b/apps/ggml/halide/test_q8_k.cpp
new file mode 100644
index 000000000000..5413714382c5
--- /dev/null
+++ b/apps/ggml/halide/test_q8_k.cpp
@@ -0,0 +1,40 @@
+// Standalone check: the from-scratch Halide Q8_K quantize kernel vs GGML's
+// own reference implementation. Unlike every other type here, Q8_K has no
+// public from_float_ref (see include/ggml.h's type_traits table) -- it's
+// reached the same way apps/ggml/providers/ggml_provider.cpp reaches it,
+// through the private quantize_row_q8_K_generic symbol declared in
+// ../providers/ggml_internal_abi.h (see that header's own comment for why
+// this one symbol needs the private ABI). Q8_K is activation-only, so
+// there's no dequantize round-trip to check, only the quantize output.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q8_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    quantize_row_q8_K_generic(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q8_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_tq1_0.cpp b/apps/ggml/halide/test_tq1_0.cpp
new file mode 100644
index 000000000000..3c08bb576797
--- /dev/null
+++ b/apps/ggml/halide/test_tq1_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide TQ1_0 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_TQ1_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_TQ1_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_tq1_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_tq1_0(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_tq2_0.cpp b/apps/ggml/halide/test_tq2_0.cpp
new file mode 100644
index 000000000000..19464cb6eab6
--- /dev/null
+++ b/apps/ggml/halide/test_tq2_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide TQ2_0 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_TQ2_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_TQ2_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_tq2_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_tq2_0(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
new file mode 100644
index 000000000000..a7ef4d3a9deb
--- /dev/null
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -0,0 +1,151 @@
+#pragma once
+
+// Shared configure() scaffolding for every Approximation-based vec_dot
+// Generator (the extended SymmetricVecDotGenerator, KQuantVecDotGenerator,
+// LookupTableVecDotGenerator). All three build the same "naive fp32 dot
+// product -> approximate_by both operands -> compute_offline severs the
+// (already-quantized, Input-supplied) encode halves -> schedule" pipeline;
+// they differ only in which block-indexed codecs their build_vec_dot() picks.
+// This factors that shared body out via CRTP (Derived::build_vec_dot()) -- the
+// same static-polymorphism idiom as codec_generator_base.h's
+// CodecGeneratorBase.
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by()/Pipeline::compute_offline(), exactly like the
+// codec generators. The vec_dot is the point at which the framework's splice +
+// sever path is exercised for a dot product (matching
+// test/performance/matvec_offline_split.cpp).
+//
+// Usage:
+//   class FooVecDotGenerator : public VecDotGeneratorBase<FooVecDotGenerator> {
+//   public:
+//       GeneratorParam<...> family{...};
+//       VecDotSpec build_vec_dot() const { return {weight_codec, wbytes, act_codec, abytes, block_size, sched}; }
+//   };
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"  // Direction/SchemeAndBytes live here; shared idiom
+#include "quant_components.h"
+
+namespace ggml_halide {
+
+// Whether the per-block scale factors out to a single block-invariant scalar
+// (SDOT: hoist_invariants() + rfactor() + change_type(Int(32)) -> Int(32) inner
+// dot) or not (Float: a plain vectorized float accumulation -- affine offsets,
+// two-level sub-block scales, and per-group grid scales are not
+// single-per-block-invariant).
+enum class ScheduleKind { SDOT,
+                          Float };
+
+struct VecDotSpec {
+    // Both codecs decode to a block-indexed (kk, blk) Func at the SAME
+    // block_size, so the reduction below is uniform -- Wt(r.x, r.y) * Vec(r.x,
+    // r.y). When an activation is stored in a smaller block than the weight
+    // (e.g. Q1_0/NVFP4 x Q8_0), its codec is composed with a Reblock stage
+    // (see quant_components.h) that re-views it at the weight's block_size --
+    // the block-structure reconciliation is an Approximation, not something
+    // the Generator open-codes into the reduction.
+    std::unique_ptr<Halide::Approximation> weight_codec;
+    int weight_bytes;
+    std::unique_ptr<Halide::Approximation> act_codec;
+    int act_bytes;
+    int block_size;
+    ScheduleKind sched;
+};
+
+template<typename Derived>
+class VecDotGeneratorBase : public Halide::Generator<Derived> {
+public:
+    void configure() {
+        using namespace Halide;
+        VecDotSpec spec = static_cast<Derived *>(this)->build_vec_dot();
+        int bs = spec.block_size;
+
+        // dim 0: byte-within-block, dim 1: block index.
+        ImageParam x_blocks(UInt(8), 2, "x_blocks");  // weight format
+        ImageParam y_blocks(UInt(8), 2, "y_blocks");  // activation format
+
+        // Naive fp32 placeholders -- never realized; compute_offline() severs
+        // Acc from them entirely, and the real values come from the
+        // already-quantized x_blocks/y_blocks. Block-indexed (kk, blk) to match
+        // the codecs' block-indexed decode.
+        Var kk("kk"), blk("blk"), u("u");
+        Func Wt("wt_naive"), Vec("vec_naive");
+        Wt(kk, blk) = 0.0f;
+        Vec(kk, blk) = 0.0f;
+
+        RDom r(0, bs, 0, x_blocks.dim(1).extent(), "r");
+        Func Acc("acc");
+        Acc() = 0.0f;
+        Acc() += Wt(r.x, r.y) * Vec(r.x, r.y);
+
+        ApproximationResult wt_r = Wt.approximate_by(*spec.weight_codec, {Acc});
+        ApproximationResult act_r = Vec.approximate_by(*spec.act_codec, {Acc});
+
+        // Both operands' encode halves are severed and bound to the real
+        // already-quantized Input buffers (same as symmetric_vec_dot). For the
+        // extern-delegated / SeveredEncode weight schemes the encode is likewise
+        // severed here, so its extern symbol is never computed or linked.
+        std::vector<Func> to_sever = wt_r.encoded;
+        to_sever.insert(to_sever.end(), act_r.encoded.begin(), act_r.encoded.end());
+        std::vector<ImageParam> bind_to = {x_blocks, y_blocks};
+        Pipeline({Acc}).compute_offline(to_sever, bind_to);
+
+        // Only handles with update definitions (per-block stat reductions) need
+        // explicit scheduling; pure pass-throughs stay inline (same reasoning as
+        // symmetric_vec_dot_generator.cpp).
+        for (Func h : wt_r.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+        for (Func h : act_r.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+
+        if (spec.sched == ScheduleKind::SDOT) {
+            // The per-block scale depends on the block index r.y, so it is only
+            // invariant across the *within-block* reduction r.x, not across all
+            // reduced RVars. rfactor() must therefore run first, preserving r.y
+            // as u so the partial Acc_dot reduces over r.x alone; only then can
+            // eager_inline() fold the dequantizers into that per-block partial
+            // and hoist_invariants() lift the now-invariant scale out of the r.x
+            // sum. change_type() finally retypes the scale-free inner dot to
+            // Int(32), leaving an SDOT-eligible integer dot. The alpha
+            // rfactor(HoistInvariantFactor) fused all of this; the mature API
+            // splits it (see test/correctness/struct_type_dot_product.cpp, the
+            // same q4_0 x q8_0 case).
+            Func Acc_dot = Acc.update().rfactor({{r.y, u}});
+            Func Acc_ff = Acc_dot.update().eager_inline({wt_r.replacement, act_r.replacement}).hoist_invariants();
+            Func Acc_i32 = Acc_ff.change_type(Int(32));
+            Acc_i32.compute_root()
+                .update()
+                .atomic()
+                .vectorize(r.x, bs);
+        }
+        // ScheduleKind::Float: leave the reduction at its default (legal) schedule
+        // -- correctness first; an interleave/sub-block-aware performance schedule
+        // is a separate step.
+
+        Func result("result");
+        result() = Acc();
+
+        x_blocks.dim(0).set_bounds(0, spec.weight_bytes);
+        x_blocks.dim(1).set_min(0);
+        y_blocks.dim(0).set_bounds(0, spec.act_bytes);
+        y_blocks.dim(1).set_min(0);
+
+        this->add_input(x_blocks);
+        this->add_input(y_blocks);
+        this->add_output(result);
+    }
+
+    void generate() {
+        // configure() built the whole pipeline (add_input/add_output included).
+    }
+};
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/include/kernel_registry.h b/apps/ggml/include/kernel_registry.h
new file mode 100644
index 000000000000..34ce7d3acb1b
--- /dev/null
+++ b/apps/ggml/include/kernel_registry.h
@@ -0,0 +1,156 @@
+#pragma once
+
+// Implementation-agnostic core of kernel-bench.
+//
+// A "kernel" here is identified by a ggml_type plus a category (quantize,
+// dequantize, vec_dot, repack quantize_mat/gemv/gemm). For each (category,
+// type) pair, exactly one *reference* implementation may be registered --
+// this is the correctness ground truth and baseline timing that every other
+// *candidate* implementation for that (category, type) is measured against.
+//
+// GGML's own routines are registered as the reference by providers/ggml_provider.cpp.
+// Nothing in this file knows anything about GGML internals: a future provider
+// (e.g. the user's own reference implementation) is just another call to
+// register_candidate() (or register_reference(), if it should replace GGML's
+// as the ground truth) with a function pointer matching the category's
+// signature. See providers/README.md for how to add one.
+
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <ggml.h>
+
+// Function-pointer shapes shared by every provider. These mirror the layouts
+// used throughout ggml-cpu (quantize_row_*, ggml_vec_dot_*, ggml_gemv_*/ggml_gemm_*)
+// so that both GGML's own symbols and a from-scratch implementation can be
+// registered without adapters.
+using quantize_fn_t = void (*)(const float *GGML_RESTRICT x, void *GGML_RESTRICT y, int64_t k);
+using dequantize_fn_t = void (*)(const void *GGML_RESTRICT x, float *GGML_RESTRICT y, int64_t k);
+using vec_dot_fn_t = void (*)(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx,
+                              const void *GGML_RESTRICT vy, size_t by, int nrc);
+using gemx_fn_t = void (*)(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx,
+                           const void *GGML_RESTRICT vy, int nr, int nc);
+
+template<typename Fn>
+struct Impl {
+    std::string name;
+    Fn fn;
+};
+
+template<typename Fn>
+class Registry {
+public:
+    // Exactly one reference per type. Calling this twice for the same type
+    // replaces the previous reference (last call wins) -- useful if a later
+    // provider should become the new ground truth for that type.
+    void register_reference(ggml_type type, std::string name, Fn fn) {
+        entries_[type].reference = Impl<Fn>{std::move(name), fn};
+    }
+
+    // Zero or more per type.
+    void register_candidate(ggml_type type, std::string name, Fn fn) {
+        entries_[type].candidates.push_back(Impl<Fn>{std::move(name), fn});
+    }
+
+    const Impl<Fn> *reference(ggml_type type) const {
+        auto it = entries_.find(type);
+        if (it == entries_.end() || !it->second.reference.has_value()) {
+            return nullptr;
+        }
+        return &*it->second.reference;
+    }
+
+    const std::vector<Impl<Fn>> &candidates(ggml_type type) const {
+        static const std::vector<Impl<Fn>> empty;
+        auto it = entries_.find(type);
+        return it == entries_.end() ? empty : it->second.candidates;
+    }
+
+    std::vector<ggml_type> types_with_reference() const {
+        std::vector<ggml_type> out;
+        for (const auto &[type, entry] : entries_) {
+            if (entry.reference.has_value()) {
+                out.push_back(type);
+            }
+        }
+        return out;
+    }
+
+private:
+    struct Entry {
+        std::optional<Impl<Fn>> reference;
+        std::vector<Impl<Fn>> candidates;
+    };
+    std::map<ggml_type, Entry> entries_;
+};
+
+// Repack kernels are additionally keyed by the activation (vec_dot_type)
+// they were interleaved against, and by the interleave geometry -- carried
+// alongside the ggml_type key as a small identifying suffix (e.g. "4x4",
+// "8x8") so multiple repack variants can coexist for the same base type.
+struct RepackKey {
+    ggml_type base_type;
+    ggml_type act_type;
+    int inter_size;
+    int nb_cols;
+    std::string label;  // e.g. "q4_0_4x4_q8_0", used for display and as a stable map key
+
+    bool operator<(const RepackKey &other) const {
+        return label < other.label;
+    }
+};
+
+template<typename Fn>
+class RepackRegistry {
+public:
+    void register_reference(const RepackKey &key, std::string name, Fn fn) {
+        entries_[key.label].key = key;
+        entries_[key.label].reference = Impl<Fn>{std::move(name), fn};
+    }
+    void register_candidate(const RepackKey &key, std::string name, Fn fn) {
+        entries_[key.label].key = key;
+        entries_[key.label].candidates.push_back(Impl<Fn>{std::move(name), fn});
+    }
+    const Impl<Fn> *reference(const std::string &label) const {
+        auto it = entries_.find(label);
+        if (it == entries_.end() || !it->second.reference.has_value()) {
+            return nullptr;
+        }
+        return &*it->second.reference;
+    }
+    const std::vector<Impl<Fn>> &candidates(const std::string &label) const {
+        static const std::vector<Impl<Fn>> empty;
+        auto it = entries_.find(label);
+        return it == entries_.end() ? empty : it->second.candidates;
+    }
+    std::vector<RepackKey> keys() const {
+        std::vector<RepackKey> out;
+        for (const auto &[label, entry] : entries_) {
+            if (entry.reference.has_value()) {
+                out.push_back(entry.key);
+            }
+        }
+        return out;
+    }
+
+private:
+    struct Entry {
+        RepackKey key{};
+        std::optional<Impl<Fn>> reference;
+        std::vector<Impl<Fn>> candidates;
+    };
+    std::map<std::string, Entry> entries_;
+};
+
+struct KernelRegistries {
+    Registry<quantize_fn_t> quantize;
+    Registry<dequantize_fn_t> dequantize;
+    Registry<vec_dot_fn_t> vec_dot;
+    RepackRegistry<quantize_fn_t> repack_quantize_mat;
+    RepackRegistry<gemx_fn_t> repack_gemv;
+    RepackRegistry<gemx_fn_t> repack_gemm;
+};
diff --git a/apps/ggml/providers/README.md b/apps/ggml/providers/README.md
new file mode 100644
index 000000000000..279d8bdbf775
--- /dev/null
+++ b/apps/ggml/providers/README.md
@@ -0,0 +1,60 @@
+# Adding a provider
+
+A "provider" is anything that registers one or more implementations into a
+`KernelRegistries` (see `include/kernel_registry.h`). `ggml_provider.cpp` is the
+provider shipped today; it is the *only* file that knows GGML's internal symbols
+exist. A new provider -- e.g. your own from-scratch reference implementation,
+starting with dequantize per the project's stated goal -- is just another
+translation unit that does the same thing.
+
+## Steps
+
+1. Create `providers/<name>_provider.h` declaring one function:
+
+   ```cpp
+   void register_<name>_provider(KernelRegistries & registries);
+   ```
+
+2. Create `providers/<name>_provider.cpp` implementing it. For each
+   `(ggml_type, implementation)` pair you want benchmarked, call:
+
+   ```cpp
+   registries.dequantize.register_candidate(GGML_TYPE_Q4_0, "my-dequant", my_dequantize_q4_0);
+   ```
+
+   matching the category's function-pointer typedef from `kernel_registry.h`:
+
+   | category                          | typedef           | signature                                                                                     |
+   | --------------------------------- | ----------------- | --------------------------------------------------------------------------------------------- |
+   | `quantize`, `repack_quantize_mat` | `quantize_fn_t`   | `(const float* x, void* y, int64_t k)`                                                        |
+   | `dequantize`                      | `dequantize_fn_t` | `(const void* x, float* y, int64_t k)`                                                        |
+   | `vec_dot`                         | `vec_dot_fn_t`    | `(int n, float* s, size_t bs, const void* vx, size_t bx, const void* vy, size_t by, int nrc)` |
+   | `repack_gemv`, `repack_gemm`      | `gemx_fn_t`       | `(int n, float* s, size_t bs, const void* vx, const void* vy, int nr, int nc)`                |
+
+   Use `register_candidate()` if GGML's existing reference should remain the
+   correctness ground truth for that type (the common case: you want to see
+   whether your implementation agrees with GGML and how fast it is). Use
+   `register_reference()` instead if your implementation should *become* the new
+   ground truth other candidates are compared against for that type -- the
+   harness doesn't care which provider a reference comes from.
+
+3. Add one line to `src/main.cpp`:
+
+   ```cpp
+   register_<name>_provider(registries);
+   ```
+
+   next to the existing `register_ggml_provider(registries);` call. Nothing else
+   changes -- `bench_*.cpp`, the CLI, and the reporting code iterate whatever
+   ends up in the registries and don't know or care how many providers
+   contributed to them.
+
+## Repack keys
+
+`repack_quantize_mat`/`repack_gemv`/`repack_gemm` are keyed by `RepackKey` (base
+type, activation type, interleave geometry, label string) rather than by
+`ggml_type` alone, since several interleaved weight layouts can exist for the
+same base type (e.g. `q4_0_4x4_q8_0` vs `q4_0_8x8_q8_0`). Reuse the `RepackKey`
+values already registered by `ggml_provider.cpp` (see `k_repack_entries` in
+`ggml_provider.cpp`) if you're providing an alternative gemv/gemm for an
+existing layout; define your own `RepackKey` if you're introducing a new one.
diff --git a/apps/ggml/providers/ggml_internal_abi.h b/apps/ggml/providers/ggml_internal_abi.h
new file mode 100644
index 000000000000..a53ebf1bd905
--- /dev/null
+++ b/apps/ggml/providers/ggml_internal_abi.h
@@ -0,0 +1,295 @@
+#pragma once
+
+// PRIVATE, VERSION-PINNED ABI SURFACE -- READ BEFORE TOUCHING
+//
+// The functions declared below are internal implementation details of
+// ggml-cpu (declared in the *uninstalled* headers src/ggml-cpu/quants.h and
+// src/ggml-cpu/repack.h). GGML does not install those headers, does not
+// document these symbols, does not version them, and offers no ABI
+// stability guarantee for them whatsoever.
+//
+// They are reachable from an external application ONLY because ggml-cpu is
+// built without -fvisibility=hidden: every plain, non-static C function ends
+// up with default (exported) linker visibility by accident of the build
+// configuration, not by design. This header is a hand-copied snapshot of
+// the declarations in ggml (as of the commit this file was written against;
+// see README.md) -- if a future GGML release renames, removes, or changes
+// the signature of one of these functions, this header (and only this
+// header + ggml_provider.cpp) will need updating. No other part of
+// kernel-bench depends on GGML internals.
+//
+// Why we need this at all: GGML's public API (ggml_get_type_traits /
+// ggml_get_type_traits_cpu, see include/ggml.h and include/ggml-cpu.h)
+// exposes exactly one "reference" and one "dispatched" implementation per
+// type for quantize/dequantize, which is enough for those two categories
+// without touching anything private (see ggml_provider.cpp). It does NOT
+// expose the always-available pure-C fallback for vec_dot or for the repack
+// quantize_mat/gemv/gemm kernels -- the only way to reach those, and thus
+// the only way to compare them against the (possibly arch-optimized)
+// canonical symbol, is by declaring both names ourselves and letting the
+// linker resolve them.
+//
+// IMPORTANT -- the `_generic` name does not always exist as its own link
+// symbol: src/ggml-cpu/arch-fallback.h #defines it onto the canonical name
+// -- as a textual macro substitution inside GGML's own .c/.cpp files -- for
+// whichever functions the current architecture has no distinct optimized
+// version of, and there is then only one function in the binary. (A weak
+// C++ declaration doesn't paper over this: verified empirically that
+// Darwin's ld64 hard-fails on an undefined `weak_import` symbol that has
+// zero definitions anywhere in the link, and GNU ld's "resolve undefined
+// weak to null" behavior isn't something to rely on portably either.) So
+// this header mirrors arch-fallback.h's own collapsing, using the same
+// preprocessor guards, restricted to the subset of functions declared
+// below. When a `_generic` name collapses onto its canonical counterpart
+// here exactly as it does inside GGML itself, ggml_provider.cpp's
+// pointer-equality check (`generic_fn == canonical_fn`) naturally detects
+// "single implementation, nothing to compare" for that kernel on this
+// architecture. If GGML's own arch-fallback.h changes its collapsing list,
+// this block needs updating to match; this is the one part of this file
+// most likely to need attention when moving to a newer GGML.
+#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K
+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K
+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
+#elif defined(__POWERPC__) || defined(__powerpc__) || defined(__loongarch64) || defined(__riscv) || \
+    defined(__s390x__) || defined(__wasm__)
+// PowerPC/LoongArch/RISC-V/s390x/wasm each collapse a large, differently-shaped
+// subset of quants.c/repack.cpp symbols (see arch-fallback.h) -- rather than
+// transcribe five more per-arch lists by hand, collapse everything this
+// header declares on these architectures. This is conservative in the safe
+// direction: on an arch that actually kept a real optimized/generic split
+// for some function, this makes that pairing look like "single
+// implementation" instead of reporting it, but it will never misreport two
+// genuinely different implementations as identical, and it will never
+// produce a link error.
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
+#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
+#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
+#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
+#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K
+#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K
+#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K
+#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K
+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
+#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0
+#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K
+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K
+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K
+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
+#endif
+
+#include <cstddef>
+#include <cstdint>
+
+#include <ggml-backend.h>  // ggml_backend_buffer_type_t
+#include <ggml.h>          // GGML_RESTRICT
+
+// NOTE: declared with ordinary C++ (mangled) linkage, matching the real
+// src/ggml-cpu/repack.h -- this one declaration sits *before* that header's
+// `extern "C" { ... }` block, unlike every quantize_row_*/vec_dot_*/gemv/gemm
+// declaration below.
+ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void);
+
+extern "C" {
+
+// -- src/ggml-cpu/quants.h: pure-C reference quantizer for Q8_K. Unlike
+// every other quantized type, GGML_TYPE_Q8_K has no `from_float_ref` in the
+// public ggml_get_type_traits() table (src/ggml.c) at all -- Q8_K is purely
+// an internal activation format for K-quant vec_dot/gemv/gemm, never a
+// row-conversion target -- so it needs this private symbol as its
+// reference; see the special case in ggml_provider.cpp.
+void quantize_row_q8_K_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT y, int64_t k);
+
+// -- src/ggml-quants.h: whole-matrix quantizers for the importance-matrix-
+// only codebook types (IQ2_XXS, IQ2_XS, IQ1_S, IQ1_M). Unlike every other
+// quantized type, these have no `from_float_ref` in the public
+// ggml_get_type_traits() table at all -- GGML only exposes them through
+// this differently-shaped `(src, dst, nrows, n_per_row, imatrix)` signature
+// (nrows/n_per_row instead of a flat element count k, and an optional
+// importance-matrix pointer), used only by the model-quantization tool.
+// Called here with nrows=1, n_per_row=k, imatrix=nullptr to get a plain
+// per-row reference, matching the shape every other type's from_float_ref
+// already has; see the special case in ggml_provider.cpp. Declared to
+// return size_t per GGML's real signature (the number of bytes written).
+size_t quantize_iq2_xxs(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+size_t quantize_iq2_xs(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+size_t quantize_iq1_s(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+size_t quantize_iq1_m(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+
+// -- src/ggml-cpu/quants.h: vec_dot, pure-C reference (collapsed onto the
+// canonical, arch-dispatched symbol above on architectures with no distinct
+// optimized implementation for that type) --
+void ggml_vec_dot_q1_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_1_q8_1_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_1_q8_1_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q8_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_mxfp4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_nvfp4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq1_0_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq2_0_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q2_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q3_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q6_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_s_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_s_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_s_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_m_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+
+// -- src/ggml-cpu/repack.h: activation packing (float -> interleaved q8 blocks), canonical + reference --
+void ggml_quantize_mat_q8_0_4x4(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_0_4x8(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x4(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x8(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_0_4x4_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_0_4x8_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x4_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x8_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+
+// -- src/ggml-cpu/repack.h: gemv/gemm over packed weight blocks, canonical + reference --
+void ggml_gemv_q4_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q2_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+void ggml_gemm_q4_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q2_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q2_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q2_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+}  // extern "C"
diff --git a/apps/ggml/providers/ggml_provider.cpp b/apps/ggml/providers/ggml_provider.cpp
new file mode 100644
index 000000000000..928233cdf13f
--- /dev/null
+++ b/apps/ggml/providers/ggml_provider.cpp
@@ -0,0 +1,293 @@
+#include "ggml_provider.h"
+#include "ggml_internal_abi.h"
+
+#include <ggml-cpu.h>
+#include <ggml.h>
+
+#include <vector>
+
+namespace {
+
+// type -> pure-C reference vec_dot (src/ggml-cpu/quants.h `_generic` symbols).
+// The canonical (possibly arch-optimized) candidate is obtained separately,
+// through the PUBLIC ggml_get_type_traits_cpu(type)->vec_dot.
+struct VecDotRef {
+    ggml_type type;
+    vec_dot_fn_t fn;
+};
+
+const VecDotRef k_vec_dot_refs[] = {
+    {GGML_TYPE_Q1_0, ggml_vec_dot_q1_0_q8_0_generic},
+    {GGML_TYPE_Q4_0, ggml_vec_dot_q4_0_q8_0_generic},
+    {GGML_TYPE_Q4_1, ggml_vec_dot_q4_1_q8_1_generic},
+    {GGML_TYPE_Q5_0, ggml_vec_dot_q5_0_q8_0_generic},
+    {GGML_TYPE_Q5_1, ggml_vec_dot_q5_1_q8_1_generic},
+    {GGML_TYPE_Q8_0, ggml_vec_dot_q8_0_q8_0_generic},
+    {GGML_TYPE_MXFP4, ggml_vec_dot_mxfp4_q8_0_generic},
+    {GGML_TYPE_NVFP4, ggml_vec_dot_nvfp4_q8_0_generic},
+    {GGML_TYPE_Q2_K, ggml_vec_dot_q2_K_q8_K_generic},
+    {GGML_TYPE_Q3_K, ggml_vec_dot_q3_K_q8_K_generic},
+    {GGML_TYPE_Q4_K, ggml_vec_dot_q4_K_q8_K_generic},
+    {GGML_TYPE_Q5_K, ggml_vec_dot_q5_K_q8_K_generic},
+    {GGML_TYPE_Q6_K, ggml_vec_dot_q6_K_q8_K_generic},
+    {GGML_TYPE_TQ1_0, ggml_vec_dot_tq1_0_q8_K_generic},
+    {GGML_TYPE_TQ2_0, ggml_vec_dot_tq2_0_q8_K_generic},
+    {GGML_TYPE_IQ2_XXS, ggml_vec_dot_iq2_xxs_q8_K_generic},
+    {GGML_TYPE_IQ2_XS, ggml_vec_dot_iq2_xs_q8_K_generic},
+    {GGML_TYPE_IQ2_S, ggml_vec_dot_iq2_s_q8_K_generic},
+    {GGML_TYPE_IQ3_XXS, ggml_vec_dot_iq3_xxs_q8_K_generic},
+    {GGML_TYPE_IQ3_S, ggml_vec_dot_iq3_s_q8_K_generic},
+    {GGML_TYPE_IQ1_S, ggml_vec_dot_iq1_s_q8_K_generic},
+    {GGML_TYPE_IQ1_M, ggml_vec_dot_iq1_m_q8_K_generic},
+    {GGML_TYPE_IQ4_NL, ggml_vec_dot_iq4_nl_q8_0_generic},
+    {GGML_TYPE_IQ4_XS, ggml_vec_dot_iq4_xs_q8_K_generic},
+};
+
+// The 9 repack combinations enumerated in
+// ggml_repack_get_optimal_repack_type() (src/ggml-cpu/repack.cpp:4528-4560).
+struct RepackEntry {
+    RepackKey key;
+    quantize_fn_t quantize_mat;
+    quantize_fn_t quantize_mat_generic;
+    gemx_fn_t gemv;
+    gemx_fn_t gemv_generic;
+    gemx_fn_t gemm;
+    gemx_fn_t gemm_generic;
+};
+
+const RepackEntry k_repack_entries[] = {
+    {{GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_q4_0_4x4_q8_0,
+     ggml_gemv_q4_0_4x4_q8_0_generic,
+     ggml_gemm_q4_0_4x4_q8_0,
+     ggml_gemm_q4_0_4x4_q8_0_generic},
+    {{GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_q4_0_4x8_q8_0,
+     ggml_gemv_q4_0_4x8_q8_0_generic,
+     ggml_gemm_q4_0_4x8_q8_0,
+     ggml_gemm_q4_0_4x8_q8_0_generic},
+    {{GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_q4_0_8x8_q8_0,
+     ggml_gemv_q4_0_8x8_q8_0_generic,
+     ggml_gemm_q4_0_8x8_q8_0,
+     ggml_gemm_q4_0_8x8_q8_0_generic},
+    {{GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"},
+     ggml_quantize_mat_q8_K_4x4,
+     ggml_quantize_mat_q8_K_4x4_generic,
+     ggml_gemv_q4_K_8x4_q8_K,
+     ggml_gemv_q4_K_8x4_q8_K_generic,
+     ggml_gemm_q4_K_8x4_q8_K,
+     ggml_gemm_q4_K_8x4_q8_K_generic},
+    {{GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q4_K_8x8_q8_K,
+     ggml_gemv_q4_K_8x8_q8_K_generic,
+     ggml_gemm_q4_K_8x8_q8_K,
+     ggml_gemm_q4_K_8x8_q8_K_generic},
+    {{GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"},
+     ggml_quantize_mat_q8_K_4x4,
+     ggml_quantize_mat_q8_K_4x4_generic,
+     ggml_gemv_q5_K_8x4_q8_K,
+     ggml_gemv_q5_K_8x4_q8_K_generic,
+     ggml_gemm_q5_K_8x4_q8_K,
+     ggml_gemm_q5_K_8x4_q8_K_generic},
+    {{GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q5_K_8x8_q8_K,
+     ggml_gemv_q5_K_8x8_q8_K_generic,
+     ggml_gemm_q5_K_8x8_q8_K,
+     ggml_gemm_q5_K_8x8_q8_K_generic},
+    {{GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"},
+     ggml_quantize_mat_q8_K_4x4,
+     ggml_quantize_mat_q8_K_4x4_generic,
+     ggml_gemv_q6_K_8x4_q8_K,
+     ggml_gemv_q6_K_8x4_q8_K_generic,
+     ggml_gemm_q6_K_8x4_q8_K,
+     ggml_gemm_q6_K_8x4_q8_K_generic},
+    {{GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q6_K_8x8_q8_K,
+     ggml_gemv_q6_K_8x8_q8_K_generic,
+     ggml_gemm_q6_K_8x8_q8_K,
+     ggml_gemm_q6_K_8x8_q8_K_generic},
+    {{GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q2_K_8x8_q8_K,
+     ggml_gemv_q2_K_8x8_q8_K_generic,
+     ggml_gemm_q2_K_8x8_q8_K,
+     ggml_gemm_q2_K_8x8_q8_K_generic},
+    {{GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_iq4_nl_4x4_q8_0,
+     ggml_gemv_iq4_nl_4x4_q8_0_generic,
+     ggml_gemm_iq4_nl_4x4_q8_0,
+     ggml_gemm_iq4_nl_4x4_q8_0_generic},
+    {{GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_iq4_nl_8x8_q8_0,
+     ggml_gemv_iq4_nl_8x8_q8_0_generic,
+     ggml_gemm_iq4_nl_8x8_q8_0,
+     ggml_gemm_iq4_nl_8x8_q8_0_generic},
+    {{GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_mxfp4_4x4_q8_0,
+     ggml_gemv_mxfp4_4x4_q8_0_generic,
+     ggml_gemm_mxfp4_4x4_q8_0,
+     ggml_gemm_mxfp4_4x4_q8_0_generic},
+    {{GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_mxfp4_8x8_q8_0,
+     ggml_gemv_mxfp4_8x8_q8_0_generic,
+     ggml_gemm_mxfp4_8x8_q8_0,
+     ggml_gemm_mxfp4_8x8_q8_0_generic},
+    {{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_q8_0_4x4_q8_0,
+     ggml_gemv_q8_0_4x4_q8_0_generic,
+     ggml_gemm_q8_0_4x4_q8_0,
+     ggml_gemm_q8_0_4x4_q8_0_generic},
+    {{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_q8_0_4x8_q8_0,
+     ggml_gemv_q8_0_4x8_q8_0_generic,
+     ggml_gemm_q8_0_4x8_q8_0,
+     ggml_gemm_q8_0_4x8_q8_0_generic},
+};
+
+// Thin adapters from GGML's whole-matrix `(src, dst, nrows, n_per_row,
+// imatrix)` quantizer signature (see ggml_internal_abi.h) down to
+// quantize_fn_t's flat `(x, y, k)` shape, matching what every other type's
+// from_float_ref already looks like: one row, no importance weighting. All
+// four of these implementations actually require a non-null quant_weights
+// pointer (a real GGML_ASSERT for IQ2_XXS/IQ2_XS/IQ1_S; commented out but
+// still exercised for IQ1_M) -- a uniform (all-1.0) weighting is passed so
+// every element is treated as equally important, the closest equivalent to
+// "no importance weighting" these quantizers support.
+void quantize_iq2_xxs_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq2_xxs(x, y, 1, k, w.data());
+}
+void quantize_iq2_xs_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq2_xs(x, y, 1, k, w.data());
+}
+void quantize_iq1_s_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq1_s(x, y, 1, k, w.data());
+}
+void quantize_iq1_m_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq1_m(x, y, 1, k, w.data());
+}
+
+}  // namespace
+
+void register_ggml_provider(KernelRegistries &registries) {
+    ggml_cpu_init();
+
+    // -- quantize / dequantize: fully public API --
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const ggml_type_traits *tt = ggml_get_type_traits(type);
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(type);
+        if (!tt || !tc) {
+            continue;
+        }
+        // Deliberately NOT calling ggml_quantize_init(type) here: for
+        // IQ2_XXS/IQ2_XS/IQ2_S/IQ1_S/IQ1_M/IQ3_XXS/IQ3_S it builds a nearest-
+        // neighbor lookup table via an O(43692 * grid_size) search with a
+        // qsort per row (src/ggml-quants.c, iq2xs_init_impl/iq3xs_init_impl)
+        // -- genuinely slow (hundreds of ms), and doing it here for all 42
+        // types unconditionally at registration time means paying for it
+        // before any benchmark has printed a single row, even for runs
+        // (e.g. --repack) that never touch these types at all. Each bench_*.cpp
+        // calls it lazily, once, right before it first actually invokes one
+        // of these types' quantize/dequantize/vec_dot functions instead.
+
+        if (tt->from_float_ref) {
+            registries.quantize.register_reference(type, "ggml-ref", tt->from_float_ref);
+            if (tc->from_float) {
+                registries.quantize.register_candidate(type, "ggml-cpu", tc->from_float);
+            }
+        }
+        if (tt->to_float) {
+            registries.dequantize.register_reference(type, "ggml-ref", tt->to_float);
+            // No candidate registered yet: GGML has exactly one dequantize
+            // implementation per type (src/ggml-quants.c, arch-independent).
+            // This is where a from-scratch dequantizer plugs in later.
+        }
+    }
+
+    // GGML_TYPE_Q8_K has no public from_float_ref (see quantize_row_q8_K_generic's
+    // doc comment in ggml_internal_abi.h) -- it's the activation format for every
+    // K-quant vec_dot/gemv/gemm, so without this, all of those silently have no
+    // valid input to quantize into and get skipped by the benchmarks.
+    {
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);
+        registries.quantize.register_reference(GGML_TYPE_Q8_K, "ggml-generic", quantize_row_q8_K_generic);
+        if (tc && tc->from_float) {
+            registries.quantize.register_candidate(GGML_TYPE_Q8_K, "ggml-cpu", tc->from_float);
+        }
+    }
+
+    // GGML_TYPE_IQ2_XXS/IQ2_XS/IQ1_S/IQ1_M have no public from_float_ref
+    // either (see quantize_iq2_xxs's doc comment in ggml_internal_abi.h) --
+    // they're importance-matrix-only codebook types whose only public
+    // quantizer takes a different, whole-matrix signature. Without this,
+    // these 4 types would never appear in the quantize/dequantize
+    // benchmarks at all (bench_dequantize.cpp requires both a quantize and
+    // a dequantize reference to exist before it will test a type).
+    registries.quantize.register_reference(GGML_TYPE_IQ2_XXS, "ggml-ref", quantize_iq2_xxs_row);
+    registries.quantize.register_reference(GGML_TYPE_IQ2_XS, "ggml-ref", quantize_iq2_xs_row);
+    registries.quantize.register_reference(GGML_TYPE_IQ1_S, "ggml-ref", quantize_iq1_s_row);
+    registries.quantize.register_reference(GGML_TYPE_IQ1_M, "ggml-ref", quantize_iq1_m_row);
+
+    // arch-fallback.h #defines a `_generic` symbol onto its canonical
+    // counterpart -- inside GGML's own source files -- for whichever
+    // functions the current architecture has no distinct optimized version
+    // of, and which functions that applies to varies by architecture. The
+    // `_generic` declarations in ggml_internal_abi.h are marked
+    // GGML_BENCH_WEAK precisely so that case resolves to a null function
+    // pointer here instead of a link error: when null, there is only one
+    // real implementation, so it becomes the reference with no candidate,
+    // rather than fabricating a comparison against nothing.
+    auto register_pair = [](auto &registry, const auto &key, auto generic_fn, auto canonical_fn) {
+        if (generic_fn) {
+            registry.register_reference(key, "ggml-generic", generic_fn);
+            if (canonical_fn) {
+                registry.register_candidate(key, "ggml-cpu", canonical_fn);
+            }
+        } else if (canonical_fn) {
+            registry.register_reference(key, "ggml-cpu", canonical_fn);
+        }
+    };
+
+    // -- vec_dot: public candidate, private (possibly weak-null) reference --
+    for (const auto &ref : k_vec_dot_refs) {
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(ref.type);
+        register_pair(registries.vec_dot, ref.type, ref.fn, tc ? tc->vec_dot : nullptr);
+    }
+
+    // -- repack: private reference and candidate (no public accessor exists) --
+    for (const auto &e : k_repack_entries) {
+        register_pair(registries.repack_quantize_mat, e.key, e.quantize_mat_generic, e.quantize_mat);
+        register_pair(registries.repack_gemv, e.key, e.gemv_generic, e.gemv);
+        register_pair(registries.repack_gemm, e.key, e.gemm_generic, e.gemm);
+    }
+}
diff --git a/apps/ggml/providers/ggml_provider.h b/apps/ggml/providers/ggml_provider.h
new file mode 100644
index 000000000000..bca2c7ec42fb
--- /dev/null
+++ b/apps/ggml/providers/ggml_provider.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "kernel_registry.h"
+
+// Registers GGML's own implementations into `registries`:
+//
+//   quantize / dequantize -- entirely via GGML's PUBLIC API
+//   (ggml_get_type_traits / ggml_get_type_traits_cpu, see include/ggml.h and
+//   include/ggml-cpu.h): `from_float_ref`/`to_float` (GGML's own documented
+//   "reference" routines) become the Registry reference, and the
+//   CPU-dispatched `from_float` becomes a candidate. No private header used.
+//
+//   vec_dot / repack (quantize_mat, gemv, gemm) -- these categories have no
+//   public way to reach the always-available pure-C fallback, so the
+//   `_generic`-suffixed symbol (declared in ggml_internal_abi.h) is
+//   registered as the reference and the canonical symbol (reached publicly
+//   for vec_dot via ggml_get_type_traits_cpu, and privately for repack,
+//   which has no public accessor at all) is registered as a candidate.
+//
+// This is the only file in kernel-bench that knows GGML exists.
+void register_ggml_provider(KernelRegistries &registries);
diff --git a/apps/ggml/providers/halide_provider.cpp b/apps/ggml/providers/halide_provider.cpp
new file mode 100644
index 000000000000..09df8bcc2333
--- /dev/null
+++ b/apps/ggml/providers/halide_provider.cpp
@@ -0,0 +1,236 @@
+#include "halide_provider.h"
+
+#include <ggml_quants.h>
+
+void register_halide_provider(KernelRegistries &registries) {
+    registries.quantize.register_candidate(GGML_TYPE_Q4_0, "halide", ggml_quants_halide_quantize_q4_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q4_0, "halide", ggml_quants_halide_dequantize_q4_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q4_0, "halide", ggml_quants_halide_vec_dot_q4_0_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q4_1, "halide", ggml_quants_halide_quantize_q4_1);
+    registries.dequantize.register_candidate(GGML_TYPE_Q4_1, "halide", ggml_quants_halide_dequantize_q4_1);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q4_1, "halide", ggml_quants_halide_vec_dot_q4_1_q8_1);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q5_0, "halide", ggml_quants_halide_quantize_q5_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q5_0, "halide", ggml_quants_halide_dequantize_q5_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q5_0, "halide", ggml_quants_halide_vec_dot_q5_0_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q5_1, "halide", ggml_quants_halide_quantize_q5_1);
+    registries.dequantize.register_candidate(GGML_TYPE_Q5_1, "halide", ggml_quants_halide_dequantize_q5_1);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q5_1, "halide", ggml_quants_halide_vec_dot_q5_1_q8_1);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q8_0, "halide", ggml_quants_halide_quantize_q8_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q8_0, "halide", ggml_quants_halide_dequantize_q8_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q8_0, "halide", ggml_quants_halide_vec_dot_q8_0_q8_0);
+
+    // Q8_1 is activation-only (GGML has no public to_float for it, so
+    // ggml_provider.cpp registers no dequantize reference either -- the
+    // harness's bench_dequantize.cpp already skips types with no reference).
+    registries.quantize.register_candidate(GGML_TYPE_Q8_1, "halide", ggml_quants_halide_quantize_q8_1);
+
+    // Q8_K is also activation-only, but unlike Q8_1, GGML doesn't even
+    // register a public from_float_ref for it -- ggml_provider.cpp's
+    // special case (using the private quantize_row_q8_K_generic) is what
+    // supplies the reference this candidate is compared against.
+    registries.quantize.register_candidate(GGML_TYPE_Q8_K, "halide", ggml_quants_halide_quantize_q8_k);
+
+    // Q2_K, Q6_K: dequantize is a genuine from-scratch Halide candidate.
+    // Quantize is scaffolding that itself calls out to GGML's own
+    // reference (see halide/ggml_extern_quantize.cpp) pending a
+    // from-scratch port of GGML's iterative scale search -- it's still
+    // registered as a candidate so the harness's plumbing is exercised
+    // end-to-end, but it will trivially match (same underlying code path).
+    // vec_dot is a genuine from-scratch candidate (against Q8_K).
+    registries.quantize.register_candidate(GGML_TYPE_Q2_K, "halide", ggml_quants_halide_quantize_q2_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q2_K, "halide", ggml_quants_halide_dequantize_q2_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q2_K, "halide", ggml_quants_halide_vec_dot_q2_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q6_K, "halide", ggml_quants_halide_quantize_q6_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q6_K, "halide", ggml_quants_halide_dequantize_q6_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q6_K, "halide", ggml_quants_halide_vec_dot_q6_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q4_K, "halide", ggml_quants_halide_quantize_q4_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q4_K, "halide", ggml_quants_halide_dequantize_q4_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q4_K, "halide", ggml_quants_halide_vec_dot_q4_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q5_K, "halide", ggml_quants_halide_quantize_q5_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q5_K, "halide", ggml_quants_halide_dequantize_q5_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q5_K, "halide", ggml_quants_halide_vec_dot_q5_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q3_K, "halide", ggml_quants_halide_quantize_q3_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q3_K, "halide", ggml_quants_halide_dequantize_q3_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q3_K, "halide", ggml_quants_halide_vec_dot_q3_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q1_0, "halide", ggml_quants_halide_quantize_q1_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q1_0, "halide", ggml_quants_halide_dequantize_q1_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q1_0, "halide", ggml_quants_halide_vec_dot_q1_0_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_MXFP4, "halide", ggml_quants_halide_quantize_mxfp4);
+    registries.dequantize.register_candidate(GGML_TYPE_MXFP4, "halide", ggml_quants_halide_dequantize_mxfp4);
+    registries.vec_dot.register_candidate(GGML_TYPE_MXFP4, "halide", ggml_quants_halide_vec_dot_mxfp4_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_NVFP4, "halide", ggml_quants_halide_quantize_nvfp4);
+    registries.dequantize.register_candidate(GGML_TYPE_NVFP4, "halide", ggml_quants_halide_dequantize_nvfp4);
+    registries.vec_dot.register_candidate(GGML_TYPE_NVFP4, "halide", ggml_quants_halide_vec_dot_nvfp4_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ4_NL, "halide", ggml_quants_halide_quantize_iq4_nl);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ4_NL, "halide", ggml_quants_halide_dequantize_iq4_nl);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ4_NL, "halide", ggml_quants_halide_vec_dot_iq4_nl_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ4_XS, "halide", ggml_quants_halide_quantize_iq4_xs);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ4_XS, "halide", ggml_quants_halide_dequantize_iq4_xs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ4_XS, "halide", ggml_quants_halide_vec_dot_iq4_xs_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_TQ1_0, "halide", ggml_quants_halide_quantize_tq1_0);
+    registries.dequantize.register_candidate(GGML_TYPE_TQ1_0, "halide", ggml_quants_halide_dequantize_tq1_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_TQ1_0, "halide", ggml_quants_halide_vec_dot_tq1_0_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_TQ2_0, "halide", ggml_quants_halide_quantize_tq2_0);
+    registries.dequantize.register_candidate(GGML_TYPE_TQ2_0, "halide", ggml_quants_halide_dequantize_tq2_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_TQ2_0, "halide", ggml_quants_halide_vec_dot_tq2_0_q8_k);
+
+    // IQ2_XXS: dequantize only (see ggml_quants.h for why); vec_dot is still
+    // a genuine from-scratch candidate (GGML's own reference quantizer is
+    // used to produce the test/benchmark input, via ggml_provider.cpp).
+    registries.dequantize.register_candidate(GGML_TYPE_IQ2_XXS, "halide", ggml_quants_halide_dequantize_iq2_xxs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ2_XXS, "halide", ggml_quants_halide_vec_dot_iq2_xxs_q8_k);
+
+    registries.dequantize.register_candidate(GGML_TYPE_IQ2_XS, "halide", ggml_quants_halide_dequantize_iq2_xs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ2_XS, "halide", ggml_quants_halide_vec_dot_iq2_xs_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ2_S, "halide", ggml_quants_halide_quantize_iq2_s);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ2_S, "halide", ggml_quants_halide_dequantize_iq2_s);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ2_S, "halide", ggml_quants_halide_vec_dot_iq2_s_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ3_XXS, "halide", ggml_quants_halide_quantize_iq3_xxs);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ3_XXS, "halide", ggml_quants_halide_dequantize_iq3_xxs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ3_XXS, "halide", ggml_quants_halide_vec_dot_iq3_xxs_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ3_S, "halide", ggml_quants_halide_quantize_iq3_s);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ3_S, "halide", ggml_quants_halide_dequantize_iq3_s);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ3_S, "halide", ggml_quants_halide_vec_dot_iq3_s_q8_k);
+
+    registries.dequantize.register_candidate(GGML_TYPE_IQ1_S, "halide", ggml_quants_halide_dequantize_iq1_s);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ1_S, "halide", ggml_quants_halide_vec_dot_iq1_s_q8_k);
+
+    registries.dequantize.register_candidate(GGML_TYPE_IQ1_M, "halide", ggml_quants_halide_dequantize_iq1_m);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ1_M, "halide", ggml_quants_halide_vec_dot_iq1_m_q8_k);
+
+    // F16, BF16: plain float casts, not "quantized" types, but both
+    // directions are fully native Halide. No vec_dot: not part of the
+    // quantized-format vec_dot sweep this directory otherwise covers.
+    registries.quantize.register_candidate(GGML_TYPE_F16, "halide", ggml_quants_halide_quantize_f16);
+    registries.dequantize.register_candidate(GGML_TYPE_F16, "halide", ggml_quants_halide_dequantize_f16);
+
+    registries.quantize.register_candidate(GGML_TYPE_BF16, "halide", ggml_quants_halide_quantize_bf16);
+    registries.dequantize.register_candidate(GGML_TYPE_BF16, "halide", ggml_quants_halide_dequantize_bf16);
+
+    // Repack quantize_mat: GGML itself only has 4 distinct implementations
+    // (2 activation formats x 2 interleave widths), reused across every
+    // repack weight type that shares one -- see k_repack_entries in
+    // ggml_provider.cpp, which this table mirrors label-for-label so the
+    // registered RepackKey (used by bench_repack.cpp for act_type/base_type)
+    // matches exactly. gemv/gemm repack candidates for every weight family
+    // GGML registers a repack entry for (Q4_0, Q8_0, IQ4_NL, MXFP4, Q4_K,
+    // Q5_K, Q6_K, Q2_K) are registered below.
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+
+    // gemv/gemm: same RepackKeys as the quantize_mat table above (base type,
+    // activation type, blocklen, ncols_interleaved, label must match
+    // label-for-label so bench_repack.cpp's per-key lookups line up).
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_0_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_0_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_0_4x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_0_4x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_0_8x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_0_8x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q8_0_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q8_0_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q8_0_4x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q8_0_4x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_iq4_nl_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_iq4_nl_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_iq4_nl_8x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_iq4_nl_8x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_mxfp4_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_mxfp4_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_mxfp4_8x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_mxfp4_8x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_k_8x4_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_k_8x4_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_k_8x8_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q5_k_8x4_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q5_k_8x4_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q5_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q5_k_8x8_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q6_k_8x4_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q6_k_8x4_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q6_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q6_k_8x8_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q2_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q2_k_8x8_q8_k);
+}
diff --git a/apps/ggml/providers/halide_provider.h b/apps/ggml/providers/halide_provider.h
new file mode 100644
index 000000000000..02b80d345536
--- /dev/null
+++ b/apps/ggml/providers/halide_provider.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#include "kernel_registry.h"
+
+// Registers the from-scratch Halide reimplementation of GGML's Q4_0
+// quantize/dequantize kernels (see ../halide/) as candidates against GGML's
+// own reference, which register_ggml_provider() registers first.
+void register_halide_provider(KernelRegistries &registries);
diff --git a/apps/ggml/src/bench_dequantize.cpp b/apps/ggml/src/bench_dequantize.cpp
new file mode 100644
index 000000000000..260645b80656
--- /dev/null
+++ b/apps/ggml/src/bench_dequantize.cpp
@@ -0,0 +1,67 @@
+#include "benchmarks.h"
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "timing.h"
+
+namespace {
+constexpr int64_t kTargetElements = 4096;
+}  // namespace
+
+BenchReport run_dequantize_benchmarks(const KernelRegistries &registries) {
+    BenchReport report{"dequantize_row", "GB/s", {}};
+    print_report_header(report.title, report.throughput_unit);
+
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const Impl<dequantize_fn_t> *ref = registries.dequantize.reference(type);
+        const Impl<quantize_fn_t> *qref = registries.quantize.reference(type);
+        if (!ref || !qref) {
+            continue;
+        }
+        ggml_quantize_init(type);  // one-time, cheap after the first call for this type; see ggml_provider.cpp
+
+        const int64_t blck = ggml_blck_size(type);
+        const int64_t n = ((kTargetElements + blck - 1) / blck) * blck;
+
+        AlignedBuffer src(n * sizeof(float));
+        generate_synthetic_data(src.as<float>(), n);
+
+        AlignedBuffer quantized(ggml_row_size(type, n));
+        qref->fn(src.as<float>(), quantized.data(), n);
+
+        AlignedBuffer ref_out(n * sizeof(float));
+        ref->fn(quantized.data(), ref_out.as<float>(), n);
+        const TimingResult ref_time = time_calls([&] { ref->fn(quantized.data(), ref_out.as<float>(), n); });
+
+        BenchRow row;
+        row.label = ggml_type_name(type);
+        row.ref_name = ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(n * sizeof(float), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.dequantize.candidates(type)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(n * sizeof(float));
+                cand.fn(quantized.data(), cand_out.as<float>(), n);
+                bc.correct = floats_match(ref_out.as<float>(), cand_out.as<float>(), n);
+
+                const TimingResult cand_time = time_calls([&] { cand.fn(quantized.data(), cand_out.as<float>(), n); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(n * sizeof(float), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+
+        print_row(row, report.throughput_unit);
+        report.rows.push_back(std::move(row));
+    }
+
+    return report;
+}
diff --git a/apps/ggml/src/bench_quantize.cpp b/apps/ggml/src/bench_quantize.cpp
new file mode 100644
index 000000000000..56d60dcd7f71
--- /dev/null
+++ b/apps/ggml/src/bench_quantize.cpp
@@ -0,0 +1,65 @@
+#include "benchmarks.h"
+
+#include <cstring>
+
+#include <ggml.h>
+
+#include "data_gen.h"
+#include "timing.h"
+
+namespace {
+constexpr int64_t kTargetElements = 4096;
+}
+
+BenchReport run_quantize_benchmarks(const KernelRegistries &registries) {
+    BenchReport report{"quantize_row", "GB/s", {}};
+    print_report_header(report.title, report.throughput_unit);
+
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const Impl<quantize_fn_t> *ref = registries.quantize.reference(type);
+        if (!ref) {
+            continue;
+        }
+        ggml_quantize_init(type);  // one-time, cheap after the first call for this type; see ggml_provider.cpp
+
+        const int64_t blck = ggml_blck_size(type);
+        const int64_t n = ((kTargetElements + blck - 1) / blck) * blck;
+        const size_t out_bytes = ggml_row_size(type, n);
+
+        AlignedBuffer src(n * sizeof(float));
+        generate_synthetic_data(src.as<float>(), n);
+
+        AlignedBuffer ref_out(out_bytes);
+        ref->fn(src.as<float>(), ref_out.data(), n);
+        const TimingResult ref_time = time_calls([&] { ref->fn(src.as<float>(), ref_out.data(), n); });
+
+        BenchRow row;
+        row.label = ggml_type_name(type);
+        row.ref_name = ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(n * sizeof(float), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.quantize.candidates(type)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(out_bytes);
+                cand.fn(src.as<float>(), cand_out.data(), n);
+                bc.correct = (std::memcmp(ref_out.data(), cand_out.data(), out_bytes) == 0);
+
+                const TimingResult cand_time = time_calls([&] { cand.fn(src.as<float>(), cand_out.data(), n); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(n * sizeof(float), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+
+        print_row(row, report.throughput_unit);
+        report.rows.push_back(std::move(row));
+    }
+
+    return report;
+}
diff --git a/apps/ggml/src/bench_repack.cpp b/apps/ggml/src/bench_repack.cpp
new file mode 100644
index 000000000000..18ca3b675222
--- /dev/null
+++ b/apps/ggml/src/bench_repack.cpp
@@ -0,0 +1,269 @@
+#include "benchmarks.h"
+
+#include <cstring>
+#include <vector>
+
+#include <ggml-alloc.h>
+#include <ggml-backend.h>
+#include <ggml-cpu.h>
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_internal_abi.h"  // ggml_backend_cpu_repack_buffer_type()
+#include "timing.h"
+
+namespace {
+
+// K (reduction dim): divisible by every block size in play (32 and 256).
+constexpr int64_t kK = 4096;
+// Output columns: divisible by every NB_COLS in play (4 and 8).
+constexpr int kNC = 32;
+// Activation rows for the gemm (batched) path; must be a multiple of 4 (the
+// row-group size ggml_quantize_mat_* always packs), and > 3 so production
+// code would pick gemm over gemv for this many rows (see repack.cpp's
+// forward_mul_mat_one_chunk: "if there are more than three rows in src1,
+// use gemm; otherwise, use gemv").
+constexpr int kGemmRows = 32;
+
+// Builds a packed weight buffer for `base_type`, shape [kK, kNC], by
+// quantizing a row-major staging buffer through the reference quantizer and
+// then letting the repack buffer type's set_tensor callback do the actual
+// interleaving (src/ggml-cpu/repack.cpp:4733). This avoids reimplementing
+// the private block<K,N> interleave layout by hand -- everything here is
+// public API plus the one declared-ourselves accessor for the buffer type.
+struct PackedWeight {
+    ggml_context *ctx = nullptr;
+    ggml_backend_buffer_t buffer = nullptr;
+    ggml_tensor *tensor = nullptr;
+
+    PackedWeight() = default;
+    PackedWeight(const PackedWeight &) = delete;
+    PackedWeight &operator=(const PackedWeight &) = delete;
+    PackedWeight(PackedWeight &&other) noexcept
+        : ctx(other.ctx), buffer(other.buffer), tensor(other.tensor) {
+        other.ctx = nullptr;
+        other.buffer = nullptr;
+        other.tensor = nullptr;
+    }
+    PackedWeight &operator=(PackedWeight &&other) noexcept {
+        if (this != &other) {
+            if (buffer) ggml_backend_buffer_free(buffer);
+            if (ctx) ggml_free(ctx);
+            ctx = other.ctx;
+            buffer = other.buffer;
+            tensor = other.tensor;
+            other.ctx = nullptr;
+            other.buffer = nullptr;
+            other.tensor = nullptr;
+        }
+        return *this;
+    }
+    ~PackedWeight() {
+        if (buffer) ggml_backend_buffer_free(buffer);
+        if (ctx) ggml_free(ctx);
+    }
+
+    // ggml_repack_get_optimal_repack_type() (src/ggml-cpu/repack.cpp) picks
+    // ONE interleave layout per (type, CPU features, column count) -- it is
+    // GGML's own "best kernel for this hardware" heuristic, not something
+    // this benchmark can steer towards a specific registered combo. When it
+    // finds none (e.g. Q2_K has no ARM branch at all, only AVX512/RISC-V --
+    // see that function), the buffer type's init_tensor callback leaves
+    // tensor->extra null, and calling set_tensor on it would dereference a
+    // null tensor_traits pointer. There is no supported way to force a
+    // different combo through this public mechanism, so such combos are
+    // skipped rather than benchmarked with fabricated data.
+    bool supported_on_this_cpu() const {
+        return tensor && tensor->extra != nullptr;
+    }
+};
+
+PackedWeight build_packed_weight(ggml_type base_type, const Impl<quantize_fn_t> &row_quant, int64_t k, int nc) {
+    PackedWeight pw;
+    ggml_init_params params{/*.mem_size=*/ggml_tensor_overhead() + 256, /*.mem_buffer=*/nullptr,
+                            /*.no_alloc=*/true};
+    pw.ctx = ggml_init(params);
+    pw.tensor = ggml_new_tensor_2d(pw.ctx, base_type, k, nc);
+    pw.buffer = ggml_backend_alloc_ctx_tensors_from_buft(pw.ctx, ggml_backend_cpu_repack_buffer_type());
+    if (!pw.supported_on_this_cpu()) {
+        return pw;
+    }
+
+    AlignedBuffer staging(ggml_row_size(base_type, k) * nc);
+    AlignedBuffer col(k * sizeof(float));
+    const size_t row_bytes = ggml_row_size(base_type, k);
+    for (int c = 0; c < nc; ++c) {
+        generate_synthetic_data(col.as<float>(), k, static_cast<float>(c));
+        row_quant.fn(col.as<float>(), staging.as<uint8_t>() + c * row_bytes, k);
+    }
+    ggml_backend_tensor_set(pw.tensor, staging.data(), 0, staging.size());
+    return pw;
+}
+
+}  // namespace
+
+std::vector<BenchReport> run_repack_benchmarks(const KernelRegistries &registries) {
+    BenchReport quant_mat_report{"repack_quantize_mat", "GB/s", {}};
+    BenchReport gemv_report{"repack_gemv", "GFLOP/s", {}};
+    BenchReport gemm_report{"repack_gemm", "GFLOP/s", {}};
+
+    print_report_header(quant_mat_report.title, quant_mat_report.throughput_unit);
+    for (const RepackKey &key : registries.repack_quantize_mat.keys()) {
+        const Impl<quantize_fn_t> *qm_ref = registries.repack_quantize_mat.reference(key.label);
+        if (!qm_ref) {
+            continue;
+        }
+        const int64_t blck = ggml_blck_size(key.act_type);
+        const int64_t k = ((kK + blck - 1) / blck) * blck;
+
+        // ggml_quantize_mat_* always consumes exactly 4 rows (see
+        // src/ggml-cpu/repack.cpp: ggml_quantize_mat_t<...> asserts nrow==4),
+        // regardless of the interleave geometry in the name.
+        AlignedBuffer src(4 * k * sizeof(float));
+        generate_synthetic_data(src.as<float>(), 4 * k);
+        const size_t out_bytes = 4 * ggml_row_size(key.act_type, k);
+
+        AlignedBuffer ref_out(out_bytes);
+        qm_ref->fn(src.as<float>(), ref_out.data(), k);
+        const TimingResult ref_time = time_calls([&] { qm_ref->fn(src.as<float>(), ref_out.data(), k); });
+
+        BenchRow row;
+        row.label = key.label;
+        row.ref_name = qm_ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(4 * k * sizeof(float), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.repack_quantize_mat.candidates(key.label)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == qm_ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(out_bytes);
+                cand.fn(src.as<float>(), cand_out.data(), k);
+                bc.correct = (std::memcmp(ref_out.data(), cand_out.data(), out_bytes) == 0);
+
+                const TimingResult cand_time = time_calls([&] { cand.fn(src.as<float>(), cand_out.data(), k); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(4 * k * sizeof(float), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+        print_row(row, quant_mat_report.throughput_unit);
+        quant_mat_report.rows.push_back(std::move(row));
+    }
+
+    print_report_header(gemv_report.title, gemv_report.throughput_unit);
+    for (const RepackKey &key : registries.repack_gemv.keys()) {
+        const Impl<gemx_fn_t> *gemv_ref = registries.repack_gemv.reference(key.label);
+        const Impl<quantize_fn_t> *w_quant = registries.quantize.reference(key.base_type);
+        const Impl<quantize_fn_t> *a_quant = registries.quantize.reference(key.act_type);
+        if (!gemv_ref || !w_quant || !a_quant) {
+            continue;
+        }
+        const int64_t blck = ggml_blck_size(key.base_type);
+        const int64_t k = ((kK + blck - 1) / blck) * blck;
+
+        PackedWeight weight = build_packed_weight(key.base_type, *w_quant, k, kNC);
+        if (!weight.supported_on_this_cpu()) {
+            continue;  // no repack kernel for this (type, CPU) combo -- see PackedWeight::supported_on_this_cpu
+        }
+
+        AlignedBuffer a_src(k * sizeof(float));
+        generate_synthetic_data(a_src.as<float>(), k, 3.0f);
+        AlignedBuffer vy(ggml_row_size(key.act_type, k));
+        a_quant->fn(a_src.as<float>(), vy.data(), k);
+
+        AlignedBuffer ref_out(kNC * sizeof(float));
+        gemv_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC);
+        const TimingResult ref_time =
+            time_calls([&] { gemv_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC); });
+        const double ref_flops = 2.0 * k * kNC;
+
+        BenchRow row;
+        row.label = key.label;
+        row.ref_name = gemv_ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = gflops(ref_flops, ref_time.median_ns);
+
+        for (const auto &cand : registries.repack_gemv.candidates(key.label)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == gemv_ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(kNC * sizeof(float));
+                cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC);
+                bc.correct = floats_match(ref_out.as<float>(), cand_out.as<float>(), kNC);
+
+                const TimingResult cand_time =
+                    time_calls([&] { cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = gflops(ref_flops, cand_time.median_ns);
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+        print_row(row, gemv_report.throughput_unit);
+        gemv_report.rows.push_back(std::move(row));
+    }
+
+    print_report_header(gemm_report.title, gemm_report.throughput_unit);
+    for (const RepackKey &key : registries.repack_gemm.keys()) {
+        const Impl<gemx_fn_t> *gemm_ref = registries.repack_gemm.reference(key.label);
+        const Impl<quantize_fn_t> *w_quant = registries.quantize.reference(key.base_type);
+        const Impl<quantize_fn_t> *qm_ref = registries.repack_quantize_mat.reference(key.label);
+        if (!gemm_ref || !w_quant || !qm_ref) {
+            continue;
+        }
+        const int64_t blck = ggml_blck_size(key.base_type);
+        const int64_t k = ((kK + blck - 1) / blck) * blck;
+
+        PackedWeight weight = build_packed_weight(key.base_type, *w_quant, k, kNC);
+        if (!weight.supported_on_this_cpu()) {
+            continue;  // no repack kernel for this (type, CPU) combo -- see PackedWeight::supported_on_this_cpu
+        }
+
+        AlignedBuffer a_src(kGemmRows * k * sizeof(float));
+        generate_synthetic_data(a_src.as<float>(), kGemmRows * k, 5.0f);
+        const size_t group_bytes = 4 * ggml_row_size(key.act_type, k);
+        AlignedBuffer vy(group_bytes * (kGemmRows / 4));
+        for (int g = 0; g < kGemmRows / 4; ++g) {
+            qm_ref->fn(a_src.as<float>() + g * 4 * k, vy.as<uint8_t>() + g * group_bytes, k);
+        }
+
+        AlignedBuffer ref_out(kGemmRows * kNC * sizeof(float));
+        gemm_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC);
+        const TimingResult ref_time = time_calls(
+            [&] { gemm_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC); });
+        const double ref_flops = 2.0 * k * kNC * kGemmRows;
+
+        BenchRow row;
+        row.label = key.label;
+        row.ref_name = gemm_ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = gflops(ref_flops, ref_time.median_ns);
+
+        for (const auto &cand : registries.repack_gemm.candidates(key.label)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == gemm_ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(kGemmRows * kNC * sizeof(float));
+                cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC);
+                bc.correct = floats_match(ref_out.as<float>(), cand_out.as<float>(), kGemmRows * kNC);
+
+                const TimingResult cand_time = time_calls(
+                    [&] { cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = gflops(ref_flops, cand_time.median_ns);
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+        print_row(row, gemm_report.throughput_unit);
+        gemm_report.rows.push_back(std::move(row));
+    }
+
+    return {quant_mat_report, gemv_report, gemm_report};
+}
diff --git a/apps/ggml/src/bench_vecdot.cpp b/apps/ggml/src/bench_vecdot.cpp
new file mode 100644
index 000000000000..3081e111d26a
--- /dev/null
+++ b/apps/ggml/src/bench_vecdot.cpp
@@ -0,0 +1,83 @@
+#include "benchmarks.h"
+
+#include <ggml-cpu.h>
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "timing.h"
+
+namespace {
+// Divisible by every quant block size in play (32 for the q4/q5/q8 family, 256 for the k-quants).
+constexpr int64_t kElements = 4096;
+}  // namespace
+
+BenchReport run_vecdot_benchmarks(const KernelRegistries &registries) {
+    BenchReport report{"vec_dot", "GB/s", {}};
+    print_report_header(report.title, report.throughput_unit);
+
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const Impl<vec_dot_fn_t> *ref = registries.vec_dot.reference(type);
+        if (!ref) {
+            continue;
+        }
+
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(type);
+        if (!tc) {
+            continue;
+        }
+        const ggml_type act_type = tc->vec_dot_type;
+
+        const Impl<quantize_fn_t> *x_quant = registries.quantize.reference(type);
+        const Impl<quantize_fn_t> *y_quant = registries.quantize.reference(act_type);
+        if (!x_quant || !y_quant) {
+            continue;  // shouldn't happen for any type reachable through the CPU backend
+        }
+        ggml_quantize_init(type);  // one-time, cheap after the first call for this type; see ggml_provider.cpp
+
+        AlignedBuffer x_src(kElements * sizeof(float));
+        AlignedBuffer y_src(kElements * sizeof(float));
+        generate_synthetic_data(x_src.as<float>(), kElements, 0.0f);
+        generate_synthetic_data(y_src.as<float>(), kElements, 7.0f);  // different phase so x != y
+
+        AlignedBuffer vx(ggml_row_size(type, kElements));
+        AlignedBuffer vy(ggml_row_size(act_type, kElements));
+        x_quant->fn(x_src.as<float>(), vx.data(), kElements);
+        y_quant->fn(y_src.as<float>(), vy.data(), kElements);
+
+        float ref_result = 0.0f;
+        ref->fn(kElements, &ref_result, 0, vx.data(), 0, vy.data(), 0, 1);
+        const TimingResult ref_time =
+            time_calls([&] { ref->fn(kElements, &ref_result, 0, vx.data(), 0, vy.data(), 0, 1); });
+
+        BenchRow row;
+        row.label = ggml_type_name(type);
+        row.ref_name = ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(vx.size() + vy.size(), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.vec_dot.candidates(type)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == ref->fn);
+            if (!bc.identical) {
+                float cand_result = 0.0f;
+                cand.fn(kElements, &cand_result, 0, vx.data(), 0, vy.data(), 0, 1);
+                bc.correct = floats_match(&ref_result, &cand_result, 1);
+
+                const TimingResult cand_time =
+                    time_calls([&] { cand.fn(kElements, &cand_result, 0, vx.data(), 0, vy.data(), 0, 1); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(vx.size() + vy.size(), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+
+        print_row(row, report.throughput_unit);
+        report.rows.push_back(std::move(row));
+    }
+
+    return report;
+}
diff --git a/apps/ggml/src/benchmarks.h b/apps/ggml/src/benchmarks.h
new file mode 100644
index 000000000000..75029adbc44d
--- /dev/null
+++ b/apps/ggml/src/benchmarks.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <vector>
+
+#include "kernel_registry.h"
+#include "report.h"
+
+BenchReport run_quantize_benchmarks(const KernelRegistries &registries);
+BenchReport run_dequantize_benchmarks(const KernelRegistries &registries);
+BenchReport run_vecdot_benchmarks(const KernelRegistries &registries);
+
+// One report each for quantize_mat, gemv, gemm.
+std::vector<BenchReport> run_repack_benchmarks(const KernelRegistries &registries);
diff --git a/apps/ggml/src/compare.h b/apps/ggml/src/compare.h
new file mode 100644
index 000000000000..b056c3c0d8fe
--- /dev/null
+++ b/apps/ggml/src/compare.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+
+// Relative-error comparison for floating point outputs (dequantize, vec_dot,
+// gemv/gemm results). Quantize/quantize_mat outputs are compared with an
+// exact memcmp instead (see bench_quantize.cpp) since those algorithms are
+// specified to be bit-identical across implementations.
+inline bool floats_match(const float *a, const float *b, int64_t n, float rel_tol = 1e-2f) {
+    for (int64_t i = 0; i < n; ++i) {
+        const float diff = std::fabs(a[i] - b[i]);
+        const float scale = std::max(std::fabs(a[i]), 1e-6f);
+        if (diff / scale > rel_tol) {
+            return false;
+        }
+    }
+    return true;
+}
diff --git a/apps/ggml/src/data_gen.h b/apps/ggml/src/data_gen.h
new file mode 100644
index 000000000000..22075c369abf
--- /dev/null
+++ b/apps/ggml/src/data_gen.h
@@ -0,0 +1,60 @@
+#pragma once
+
+// Deterministic synthetic data + aligned buffers, following the conventions
+// of tests/test-quantize-perf.cpp (same generator, same rationale: a fixed
+// seedless formula so every implementation under comparison sees byte-identical
+// input without carrying a PRNG dependency).
+
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+
+inline void generate_synthetic_data(float *dst, size_t n, float offset = 0.0f) {
+    for (size_t i = 0; i < n; ++i) {
+        dst[i] = 0.1f + 2.0f * cosf(static_cast<float>(i) + offset);
+    }
+}
+
+// 64-byte aligned heap buffer (covers every SIMD width in use: SSE/AVX/AVX-512/NEON/SVE).
+class AlignedBuffer {
+public:
+    explicit AlignedBuffer(size_t bytes) : size_(bytes) {
+        constexpr size_t alignment = 64;
+        size_t padded = (bytes + alignment - 1) / alignment * alignment;
+        if (padded == 0) {
+            padded = alignment;
+        }
+        ptr_ = nullptr;
+        posix_memalign(&ptr_, alignment, padded);
+    }
+    ~AlignedBuffer() {
+        std::free(ptr_);
+    }
+
+    AlignedBuffer(const AlignedBuffer &) = delete;
+    AlignedBuffer &operator=(const AlignedBuffer &) = delete;
+
+    void *data() {
+        return ptr_;
+    }
+    const void *data() const {
+        return ptr_;
+    }
+    template<typename T>
+    T *as() {
+        return static_cast<T *>(ptr_);
+    }
+    template<typename T>
+    const T *as() const {
+        return static_cast<const T *>(ptr_);
+    }
+    size_t size() const {
+        return size_;
+    }
+
+private:
+    void *ptr_;
+    size_t size_;
+};
diff --git a/apps/ggml/src/main.cpp b/apps/ggml/src/main.cpp
new file mode 100644
index 000000000000..24cb441426f8
--- /dev/null
+++ b/apps/ggml/src/main.cpp
@@ -0,0 +1,122 @@
+#include <cstdio>
+#include <fstream>
+#include <string>
+
+#include <ggml-cpu.h>
+
+#include "benchmarks.h"
+#include "ggml_provider.h"
+#include "halide_provider.h"
+#include "kernel_registry.h"
+
+namespace {
+
+// The repack buffer type logs a GGML_LOG_DEBUG line on every repack (see
+// src/ggml-cpu/repack.cpp:4733) -- benign, but this benchmark triggers many
+// of them (one per gemv/gemm sample weight built), so drop DEBUG/INFO noise
+// and keep only warnings/errors.
+void quiet_log_callback(ggml_log_level level, const char *text, void *) {
+    if (level >= GGML_LOG_LEVEL_WARN) {
+        std::fputs(text, stderr);
+    }
+}
+
+void print_ggml_version() {
+#ifdef KERNEL_BENCH_GGML_VERSION
+    std::printf("GGML version: %s\n", KERNEL_BENCH_GGML_VERSION);
+#else
+    std::printf("GGML version: unknown (GGML_VERSION not set by ggml-config.cmake)\n");
+#endif
+}
+
+void print_cpu_features() {
+    std::printf("CPU features:");
+#if defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
+    if (ggml_cpu_has_avx()) std::printf(" avx");
+    if (ggml_cpu_has_avx2()) std::printf(" avx2");
+    if (ggml_cpu_has_avx512()) std::printf(" avx512");
+    if (ggml_cpu_has_avx512_vnni()) std::printf(" avx512_vnni");
+    if (ggml_cpu_has_fma()) std::printf(" fma");
+    if (ggml_cpu_has_f16c()) std::printf(" f16c");
+    if (ggml_cpu_has_amx_int8()) std::printf(" amx_int8");
+#endif
+#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
+    if (ggml_cpu_has_neon()) std::printf(" neon");
+    if (ggml_cpu_has_dotprod()) std::printf(" dotprod");
+    if (ggml_cpu_has_matmul_int8()) std::printf(" matmul_int8");
+    if (ggml_cpu_has_fp16_va()) std::printf(" fp16_va");
+    if (ggml_cpu_has_sve()) std::printf(" sve(%d bytes)", ggml_cpu_get_sve_cnt());
+    if (ggml_cpu_has_sme()) std::printf(" sme");  // codespell:ignore sme
+#endif
+    std::printf("\n");
+}
+
+void usage(const char *argv0) {
+    std::printf("usage: %s [--quantize] [--dequantize] [--vecdot] [--repack] [--all] [--csv FILE]\n", argv0);
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    ggml_log_set(quiet_log_callback, nullptr);
+
+    bool do_quantize = false, do_dequantize = false, do_vecdot = false, do_repack = false;
+    std::string csv_path;
+
+    for (int i = 1; i < argc; ++i) {
+        const std::string arg = argv[i];
+        if (arg == "--quantize") do_quantize = true;
+        else if (arg == "--dequantize")
+            do_dequantize = true;
+        else if (arg == "--vecdot")
+            do_vecdot = true;
+        else if (arg == "--repack")
+            do_repack = true;
+        else if (arg == "--all")
+            do_quantize = do_dequantize = do_vecdot = do_repack = true;
+        else if (arg == "--csv" && i + 1 < argc)
+            csv_path = argv[++i];
+        else if (arg == "--help" || arg == "-h") {
+            usage(argv[0]);
+            return 0;
+        } else {
+            std::fprintf(stderr, "unknown argument: %s\n", arg.c_str());
+            usage(argv[0]);
+            return 1;
+        }
+    }
+    if (!do_quantize && !do_dequantize && !do_vecdot && !do_repack) {
+        do_quantize = do_dequantize = do_vecdot = do_repack = true;  // default: --all
+    }
+
+    print_ggml_version();
+    print_cpu_features();
+
+    KernelRegistries registries;
+    register_ggml_provider(registries);
+    register_halide_provider(registries);
+
+    // Each run_*_benchmarks() call prints its own header and streams a row
+    // to stdout as soon as that row is computed (see report.h/print_row) --
+    // results become visible immediately rather than only after everything
+    // finishes. The returned reports are only needed here for --csv.
+    std::vector<BenchReport> reports;
+    if (do_quantize) reports.push_back(run_quantize_benchmarks(registries));
+    if (do_dequantize) reports.push_back(run_dequantize_benchmarks(registries));
+    if (do_vecdot) reports.push_back(run_vecdot_benchmarks(registries));
+    if (do_repack) {
+        for (auto &r : run_repack_benchmarks(registries)) {
+            reports.push_back(std::move(r));
+        }
+    }
+
+    if (!csv_path.empty()) {
+        std::ofstream out(csv_path);
+        for (const auto &report : reports) {
+            write_report_csv(report, out);
+        }
+        std::printf("\nwrote %s\n", csv_path.c_str());
+    }
+
+    return 0;
+}
diff --git a/apps/ggml/src/report.cpp b/apps/ggml/src/report.cpp
new file mode 100644
index 000000000000..6527b10967ed
--- /dev/null
+++ b/apps/ggml/src/report.cpp
@@ -0,0 +1,50 @@
+#include "report.h"
+
+#include <cstdio>
+#include <ostream>
+
+void print_report_header(const std::string &title, const std::string &throughput_unit) {
+    std::printf("\n=== %s (%s) ===\n", title.c_str(), throughput_unit.c_str());
+    std::fflush(stdout);
+}
+
+void print_row(const BenchRow &row, const std::string &throughput_unit) {
+    std::printf("  %-20s  reference=%-10s %8.1f ns  %8.2f %s\n", row.label.c_str(), row.ref_name.c_str(), row.ref_ns,
+                row.ref_throughput, throughput_unit.c_str());
+    if (row.candidates.empty()) {
+        std::printf("  %-20s    (no candidates registered yet)\n", "");
+    }
+    for (const auto &c : row.candidates) {
+        if (c.identical) {
+            std::printf("  %-20s    %-12s  identical to reference\n", "", c.name.c_str());
+            continue;
+        }
+        std::printf("  %-20s    %-12s  %8.1f ns  %8.2f %s  %6.2fx%s\n", "", c.name.c_str(), c.ns, c.throughput,
+                    throughput_unit.c_str(), c.speedup, c.correct ? "" : "  [MISMATCH vs reference]");
+    }
+    std::fflush(stdout);
+}
+
+void print_report(const BenchReport &report) {
+    print_report_header(report.title, report.throughput_unit);
+    if (report.rows.empty()) {
+        std::printf("  (nothing registered)\n");
+        std::fflush(stdout);
+        return;
+    }
+    for (const auto &row : report.rows) {
+        print_row(row, report.throughput_unit);
+    }
+}
+
+void write_report_csv(const BenchReport &report, std::ostream &out) {
+    out << "table,label,role,name,ns,throughput_" << report.throughput_unit << ",speedup,identical,correct\n";
+    for (const auto &row : report.rows) {
+        out << report.title << ',' << row.label << ",reference," << row.ref_name << ',' << row.ref_ns << ','
+            << row.ref_throughput << ",1.0,0,1\n";
+        for (const auto &c : row.candidates) {
+            out << report.title << ',' << row.label << ",candidate," << c.name << ',' << c.ns << ',' << c.throughput
+                << ',' << c.speedup << ',' << (c.identical ? 1 : 0) << ',' << (c.correct ? 1 : 0) << '\n';
+        }
+    }
+}
diff --git a/apps/ggml/src/report.h b/apps/ggml/src/report.h
new file mode 100644
index 000000000000..265cab2e10eb
--- /dev/null
+++ b/apps/ggml/src/report.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+
+struct BenchCandidate {
+    std::string name;
+    double ns = 0.0;
+    double throughput = 0.0;
+    double speedup = 0.0;    // reference.ns / ns
+    bool identical = false;  // candidate fn pointer == reference fn pointer
+    bool correct = true;     // output matched the reference within tolerance
+};
+
+struct BenchRow {
+    std::string label;  // type name, or repack key label
+    double ref_ns = 0.0;
+    double ref_throughput = 0.0;
+    std::string ref_name;
+    std::vector<BenchCandidate> candidates;
+};
+
+struct BenchReport {
+    std::string title;
+    std::string throughput_unit;  // "GB/s" or "GFLOP/s"
+    std::vector<BenchRow> rows;
+};
+
+// Incremental printing: call print_report_header() once, then print_row()
+// as each row is computed (bench_*.cpp interleaves this with the actual
+// benchmarking so results stream out immediately instead of only appearing
+// after the whole category finishes). Both flush stdout so the stream is
+// visible immediately even when redirected/piped, not just on a tty.
+void print_report_header(const std::string &title, const std::string &throughput_unit);
+void print_row(const BenchRow &row, const std::string &throughput_unit);
+
+// Convenience wrapper for a fully-built report (used for the "nothing
+// registered" case, and anywhere the whole report is already in hand).
+void print_report(const BenchReport &report);
+
+void write_report_csv(const BenchReport &report, std::ostream &out);
diff --git a/apps/ggml/src/timing.h b/apps/ggml/src/timing.h
new file mode 100644
index 000000000000..ad78e859c6b8
--- /dev/null
+++ b/apps/ggml/src/timing.h
@@ -0,0 +1,73 @@
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <vector>
+
+struct TimingResult {
+    double min_ns = 0.0;
+    double median_ns = 0.0;
+};
+
+// Runs `fn` `warmup` times (discarded), then calibrates a batch size large
+// enough that timing a whole batch back-to-back amortizes clock overhead and
+// resolution (individual quantize_row/vec_dot calls on fast SIMD kernels can
+// complete in a few nanoseconds -- timing them one at a time, even with a
+// high-resolution clock, is dominated by noise), then times `iters` such
+// batches and returns the min/median per-call latency in nanoseconds.
+inline TimingResult time_calls(const std::function<void()> &fn, int warmup = 5, int iters = 20) {
+    using clock = std::chrono::steady_clock;
+
+    for (int i = 0; i < warmup; ++i) {
+        fn();
+    }
+
+    constexpr double kMinBatchNs = 200000.0;  // 0.2ms per batch
+    int batch = 1;
+    for (;;) {
+        const auto t0 = clock::now();
+        for (int i = 0; i < batch; ++i) {
+            fn();
+        }
+        const auto t1 = clock::now();
+        const double batch_ns = std::chrono::duration<double, std::nano>(t1 - t0).count();
+        if (batch_ns >= kMinBatchNs || batch >= (1 << 20)) {
+            break;
+        }
+        batch *= 4;
+    }
+
+    std::vector<double> samples_ns;
+    samples_ns.reserve(iters);
+    for (int i = 0; i < iters; ++i) {
+        const auto t0 = clock::now();
+        for (int b = 0; b < batch; ++b) {
+            fn();
+        }
+        const auto t1 = clock::now();
+        const double batch_ns = std::chrono::duration<double, std::nano>(t1 - t0).count();
+        samples_ns.push_back(batch_ns / batch);
+    }
+
+    std::sort(samples_ns.begin(), samples_ns.end());
+    TimingResult result;
+    result.min_ns = samples_ns.front();
+    result.median_ns = samples_ns[samples_ns.size() / 2];
+    return result;
+}
+
+inline double bytes_per_sec(size_t bytes, double ns) {
+    if (ns <= 0.0) {
+        return 0.0;
+    }
+    return static_cast<double>(bytes) / (ns * 1e-9);
+}
+
+inline double gflops(double flops, double ns) {
+    if (ns <= 0.0) {
+        return 0.0;
+    }
+    return flops / (ns * 1e-9) / 1e9;
+}
diff --git a/apps/ggml/vcpkg-configuration.json b/apps/ggml/vcpkg-configuration.json
new file mode 100644
index 000000000000..a0daf83101ff
--- /dev/null
+++ b/apps/ggml/vcpkg-configuration.json
@@ -0,0 +1,5 @@
+{
+  "overlay-ports": [
+    "../vcpkg/ports"
+  ]
+}
diff --git a/apps/ggml/vcpkg.json b/apps/ggml/vcpkg.json
new file mode 100644
index 000000000000..bac5a1fefe0f
--- /dev/null
+++ b/apps/ggml/vcpkg.json
@@ -0,0 +1,9 @@
+{
+  "name": "halide-ggml-app",
+  "version": "22.0.0",
+  "license": "MIT",
+  "builtin-baseline": "66c0373dc7fca549e5803087b9487edfe3aca0a1",
+  "dependencies": [
+    "ggml"
+  ]
+}
diff --git a/apps/vcpkg.json b/apps/vcpkg.json
index 1c3c5805f7ca..381ba6e211ac 100644
--- a/apps/vcpkg.json
+++ b/apps/vcpkg.json
@@ -10,6 +10,7 @@
       "platform": "(windows & x64 & !uwp & !xbox) | (linux & x64) | (linux & arm64)"
     },
     "eigen3",
+    "ggml",
     "libjpeg-turbo",
     "libpng",
     "onnx",
diff --git a/apps/vcpkg/ports/ggml/portfile.cmake b/apps/vcpkg/ports/ggml/portfile.cmake
new file mode 100644
index 000000000000..942b99f5f5b4
--- /dev/null
+++ b/apps/vcpkg/ports/ggml/portfile.cmake
@@ -0,0 +1,26 @@
+vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
+
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO ggml-org/ggml
+    REF eced84c86f8b012c752c016f7fe789adea168e1e  # v0.15.3
+    SHA512 3295c064aff295b0387249d5dec7860b620de82c8361197888df186be18270ede253ab7bce3358b1fb3020f11d01f0f8a29f7d268bff44666f8a8f3ea832781e
+)
+
+# We set GGML_BACKEND_DL=OFF to keep the CPU backend linked, not dlopen'ed,
+# because apps/ggml needs ggml-cpu's internal symbols at link time.
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+    -DBUILD_SHARED_LIBS=OFF
+    -DGGML_BACKEND_DL=OFF
+    -DGGML_BUILD_TESTS=OFF
+    -DGGML_BUILD_EXAMPLES=OFF
+)
+
+vcpkg_cmake_install()
+vcpkg_cmake_config_fixup(PACKAGE_NAME ggml CONFIG_PATH lib/cmake/ggml)
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE" "${SOURCE_PATH}/AUTHORS")
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share")
diff --git a/apps/vcpkg/ports/ggml/vcpkg.json b/apps/vcpkg/ports/ggml/vcpkg.json
new file mode 100644
index 000000000000..043c2b93892d
--- /dev/null
+++ b/apps/vcpkg/ports/ggml/vcpkg.json
@@ -0,0 +1,17 @@
+{
+  "name": "ggml",
+  "version": "0.15.3",
+  "description": "Tensor library for machine learning",
+  "homepage": "https://github.com/ggml-org/ggml",
+  "license": "MIT",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}

From bc36efa7e78bc166590bafaf872e6614369155c1 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sat, 1 Aug 2026 20:53:45 -0400
Subject: [PATCH 06/22] apps/ggml: struct-type the symmetric codec block layout
 (Phase 3 pilot)

Adopts mature Type::Struct/field()/pack_struct() for the symmetric
quantize/dequantize codecs (Q4_0/Q8_0/Q1_0), replacing the hand-rolled
byte-layout engine for those formats:

  - StructBlockLayout (quant_components.h): a struct-typed leaf that reads
    the block's scale straight out of a typed `d` field (subsuming Fp16Pack's
    manual lo|(hi<<8) reassembly + reinterpret) and hands the `qs` bytes to
    the same code_pack the byte path uses. It replaces StructPack + Permute +
    the scale's Fp16Pack; the compiler owns the field offsets and the total
    byte size (block_type.bytes()), so no hand-summed block_bytes.
  - make_symmetric_block_scheme() gains an opt-in struct_layout flag; the
    symmetric *codec* generator sets it (vec_dot/repack stay on the byte path
    for now). SchemeAndBytes carries the block's Type::Struct.
  - codec_generator_base.h declares a 1-D Type::Struct ImageParam/Output when
    the scheme is structured (vs the 2-D (byte, blk) UInt(8) buffer), and
    compute_offline() binds/severs it as an ordinary struct-typed Func.
  - ggml_quants.cpp: the Q4_0/Q8_0/Q1_0 wrappers hand GGML's raw bytes to the
    struct kernel via a 1-D halide_buffer_t whose type is
    {halide_type_struct, 8, reserved=block_bytes} -- Type::to_abi()'s exact
    ABI form, constructed with Halide::Runtime::Buffer<void,1>.

Validated: q4_0/q8_0/q1_0 roundtrip tests pass and kernel-bench --all reports
zero mismatches vs GGML -- the struct-typed buffer flows through
approximate_by/compute_offline and the GGML byte-buffer ABI boundary unchanged.

Deferred: converting the vec_dot/repack operands (which would also eliminate
the re-derived block widths in symmetric_vec_dot_generator.cpp), and rolling
out to the affine (min), split-code (Q5_0), K-quant, and IQ layouts. The old
StructPack/FieldSpec/make_block_layout engine stays for those.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/halide/codec_generator_base.h       |  53 ++++-----
 apps/ggml/halide/ggml_quants.cpp              |  30 +++---
 apps/ggml/halide/quant_components.h           | 102 +++++++++++++++++-
 .../halide/symmetric_quant_generators.cpp     |   5 +-
 4 files changed, 152 insertions(+), 38 deletions(-)

diff --git a/apps/ggml/halide/codec_generator_base.h b/apps/ggml/halide/codec_generator_base.h
index 0145349690be..1974b3430505 100644
--- a/apps/ggml/halide/codec_generator_base.h
+++ b/apps/ggml/halide/codec_generator_base.h
@@ -46,6 +46,12 @@ class CodecGeneratorBase : public Halide::Generator<Derived> {
         using namespace Halide;
         SchemeAndBytes sb = static_cast<Derived *>(this)->build_scheme();
 
+        // A structured scheme's encoded form is a first-class 1-D Type::Struct
+        // block (one struct per block index); an unported one is the flat 2-D
+        // (byte, blk) UInt(8) buffer. block_type.bytes() is the on-disk width in
+        // the struct case -- no separately-threaded block_bytes needed.
+        const bool structured = sb.block_type.is_struct();
+
         // The "obvious" identity: a real ImageParam (never a placeholder --
         // that's what lets *both* directions share this one call below)
         // flowing through unchanged.
@@ -59,45 +65,44 @@ class CodecGeneratorBase : public Halide::Generator<Derived> {
             h.compute_root();
         }
 
-        // Every scheme here produces a single 2-D uint8 packed byte buffer
-        // as its encoded form -- bind compute_offline() to a properly-named
-        // ImageParam of that shape up front, instead of letting it mint one
-        // named after whatever internal Func happened to produce
-        // r.encoded[0] (e.g. "struct_pack_packed"). Only Dequantize below
-        // adopts it as a port (named "blocks_in" rather than reusing
-        // Quantize's output name "blocks_out" below, so the two don't
-        // collide and get uniquified within this same configure() call --
-        // they're never both real ports at once, but both objects always
-        // exist).
-        ImageParam blocks_in(UInt(8), 2, "blocks_in");
+        // Bind compute_offline() to a properly-named ImageParam of the packed
+        // block's shape up front, instead of letting it mint one named after
+        // whatever internal Func produced r.encoded[0]. Only Dequantize below
+        // adopts it as a port (named "blocks_in" rather than reusing Quantize's
+        // output name "blocks_out"); the two are never both real ports at once,
+        // but both objects always exist.
+        ImageParam blocks_in = structured ? ImageParam(sb.block_type, 1, "blocks_in") : ImageParam(UInt(8), 2, "blocks_in");
 
         // Severs `identity` from `input`/encode() entirely: `q.offline`
         // recomputes r.encoded (quantize) from `input`, while `identity`
         // (post-severance) instead reads from `blocks_in` (dequantize).
-        // Each direction below adopts exactly one of these two independent
-        // halves; the other is simply never registered as a port and so
-        // never gets compiled in.
         ComputeOfflineResult q = Pipeline({identity}).compute_offline(r.encoded, {blocks_in});
 
         if constexpr (dir == Direction::Quantize) {
             input.dim(0).set_min(0);
 
-            // q.offline.outputs()[0] is r.encoded[0] itself (an internally-
-            // named Func) -- a thin renamed passthrough is the only way to
-            // give the compiled Output a clean name, the same way
-            // `blocks_in` above did for the Input side; Halide inlines it
-            // away, so this costs nothing.
+            // A thin renamed passthrough gives the compiled Output a clean name
+            // (the way `blocks_in` did for the Input side); Halide inlines it.
             Func blocks_out("blocks_out");
             Var byte("byte"), blk("blk");
-            blocks_out(byte, blk) = q.offline.outputs()[0](byte, blk);
-            blocks_out.output_buffer().dim(0).set_bounds(0, sb.block_bytes);
-            blocks_out.output_buffer().dim(1).set_min(0);
+            if (structured) {
+                blocks_out(blk) = q.offline.outputs()[0](blk);
+                blocks_out.output_buffer().dim(0).set_min(0);
+            } else {
+                blocks_out(byte, blk) = q.offline.outputs()[0](byte, blk);
+                blocks_out.output_buffer().dim(0).set_bounds(0, sb.block_bytes);
+                blocks_out.output_buffer().dim(1).set_min(0);
+            }
 
             this->add_input(input);
             this->add_output(blocks_out);
         } else {
-            blocks_in.dim(0).set_bounds(0, sb.block_bytes);
-            blocks_in.dim(1).set_min(0);
+            if (structured) {
+                blocks_in.dim(0).set_min(0);
+            } else {
+                blocks_in.dim(0).set_bounds(0, sb.block_bytes);
+                blocks_in.dim(1).set_min(0);
+            }
             identity.output_buffer().dim(0).set_min(0);
 
             this->add_input(blocks_in);
diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
index 2fe2ec3a1bcb..6d095070f27d 100644
--- a/apps/ggml/halide/ggml_quants.cpp
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -124,6 +124,18 @@ void check(int result, const char *what) {
     }
 }
 
+// A 1-D Type::Struct block buffer over `nb` blocks of `block_bytes` each,
+// wrapping GGML's raw packed bytes at `data`. The struct's ABI tag is
+// {halide_type_struct, bits=8, reserved=block_bytes}; each element is one whole
+// block, dim-0 stride 1 in struct units (see Halide::Type::to_abi()). This is
+// how a Phase-3 struct-typed codec kernel receives GGML's byte layout unchanged.
+Halide::Runtime::Buffer<void, 1> struct_block_buffer(const void *data, int nb, int block_bytes) {
+    halide_type_t ty(halide_type_struct, 8);
+    ty.reserved = static_cast<uint16_t>(block_bytes);
+    halide_dimension_t shape[1] = {{0, nb, 1}};
+    return Halide::Runtime::Buffer<void, 1>(ty, const_cast<void *>(data), 1, shape);
+}
+
 }  // namespace
 
 extern "C" {
@@ -136,16 +148,14 @@ void ggml_quants_halide_quantize_q4_0(const float *x, void *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
     Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
     check(q4_0_quantize(xb, blocks), "q4_0_quantize");
 }
 
 void ggml_quants_halide_dequantize_q4_0(const void *x, float *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
     Buffer<float, 1> yb(y, static_cast<int>(k));
     check(q4_0_dequantize(blocks, yb), "q4_0_dequantize");
 }
@@ -272,16 +282,14 @@ void ggml_quants_halide_quantize_q8_0(const float *x, void *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 2 + kQK;
     Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
     check(q8_0_quantize(xb, blocks), "q8_0_quantize");
 }
 
 void ggml_quants_halide_dequantize_q8_0(const void *x, float *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
     Buffer<float, 1> yb(y, static_cast<int>(k));
     check(q8_0_dequantize(blocks, yb), "q8_0_dequantize");
 }
@@ -518,16 +526,14 @@ void ggml_quants_halide_quantize_q1_0(const float *x, void *y, int64_t k) {
     constexpr int kQK = 128, kBlockBytes = 2 + kQK / 8;
     Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
     check(q1_0_quantize(xb, blocks), "q1_0_quantize");
 }
 
 void ggml_quants_halide_dequantize_q1_0(const void *x, float *y, int64_t k) {
     constexpr int kQK = 128, kBlockBytes = 2 + kQK / 8;
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
     Buffer<float, 1> yb(y, static_cast<int>(k));
     check(q1_0_dequantize(blocks, yb), "q1_0_dequantize");
 }
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index 11517cfef869..a94ffe822f79 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -83,6 +83,13 @@ namespace ggml_halide {
 struct SchemeAndBytes {
     std::unique_ptr<Halide::Approximation> scheme;
     int block_bytes;
+    // When set (is_struct()), the scheme's encoded form is a first-class
+    // Type::Struct block (one struct per block index) rather than a 2-D
+    // (byte, blk) UInt(8) buffer. The Generator uses this to declare a
+    // struct-typed, 1-D packed ImageParam/Output, and block_bytes is then just
+    // block_type.bytes() -- the single source of truth for the on-disk width.
+    // Default-constructed (invalid) for the byte-buffer schemes not yet ported.
+    Halide::Type block_type;
 };
 
 // Every make_*_scheme() factory below takes a Layout, selecting what its
@@ -1161,6 +1168,81 @@ class StructPack : public Halide::Approximation {
     std::vector<int> input_index_;
 };
 
+// A struct-typed replacement for StructPack + Permute + the scale field's
+// Fp16Pack, backed by a first-class Halide::Type::Struct instead of hand-summed
+// byte offsets. `block_type` is the on-disk block's struct type (e.g.
+// block_q4_0's `{fp16 d; uint8 qs[16]}`); the compiler owns the field offsets
+// and the total byte size (`block_type.bytes()`), so nothing here computes them.
+//
+// This leaf sits at the OUTERMOST (on-disk-byte) end of a scheme's Compose,
+// exactly where the old make_block_layout() stack did. It produces the same two
+// logical Funcs the symmetric/affine quantize stage consumes, in slot order:
+//   slot 0: `codes_bytes(local, blk)` -- the raw UInt(8) bytes of the codes
+//           field, still to be interpreted by the code_pack (nibble/byte/bit
+//           extraction) composed just inside this leaf. Struct types subsume the
+//           *layout* of these bytes, not the packing trick that reads sub-byte
+//           codes out of them.
+//   slot 1: `scale(blk)` -- the block's scale, read straight out of the typed
+//           `d` field (Float(16) -> Float(32)); this is what subsumes Fp16Pack's
+//           manual `lo | (hi<<8)` reassembly + reinterpret.
+// v1 pilot: exactly one scalar scale field + one UInt(8) array codes field (the
+// block_q4_0/block_q8_0 shape). Affine (min) and split-code (q5_0) layouts stay
+// on make_block_layout for now.
+class StructBlockLayout : public Halide::Approximation {
+public:
+    StructBlockLayout(Halide::Type block_type, std::string scale_field, std::string codes_field)
+        : block_type_(block_type), scale_field_(std::move(scale_field)), codes_field_(std::move(codes_field)) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes_bytes = inputs[0];  // codes_bytes(local, blk), UInt(8)
+        Func scale = inputs[1];        // scale(blk), Float(32)
+        Var blk("blk");
+
+        // Assemble one value per field element, in field declaration order, for
+        // the flattened pack_struct() form. The compiler places each at its own
+        // offset -- no hand-rolled shifts/masks/reinterpret in the reverse
+        // direction the way the old encode path needed.
+        const StructTypeInfo *info = block_type_.struct_type();
+        std::vector<Expr> vals;
+        for (const StructField &f : info->fields) {
+            int extent = f.array_extent.value_or(1);
+            for (int i = 0; i < extent; i++) {
+                if (f.name == scale_field_) {
+                    vals.push_back(cast(f.type, scale(blk)));
+                } else if (f.name == codes_field_) {
+                    vals.push_back(cast(f.type, codes_bytes(i, blk)));
+                } else {
+                    _halide_internal_error << "StructBlockLayout: unexpected field \"" << f.name << "\"\n";
+                }
+            }
+        }
+
+        Func packed("struct_block_packed");
+        packed(blk) = pack_struct(block_type_, vals);
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];  // packed(blk), struct-typed
+        Var local("local"), blk("blk");
+
+        Func scale("struct_block_scale");
+        scale(blk) = cast<float>(field(packed(blk), scale_field_));
+
+        Func codes_bytes("struct_block_codes_bytes");
+        codes_bytes(local, blk) = cast<uint8_t>(field(packed(blk), codes_field_)[local]);
+
+        return {{codes_bytes, scale}, {}};
+    }
+
+private:
+    Halide::Type block_type_;
+    std::string scale_field_, codes_field_;
+};
+
 // One field, in ON-DISK byte order, of a struct-packed block layout: an
 // on-disk byte width plus which logical "slot" it lands in -- the index it
 // occupies in the Func vector immediately after StructPack::decode() (and,
@@ -2565,9 +2647,27 @@ inline CodePackField make_code_pack(int block_size, int code_bits, int qmax) {
 // pair.
 inline SchemeAndBytes make_symmetric_block_scheme(
     int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor, int code_bits,
-    Layout layout = Layout::FlatRow) {
+    Layout layout = Layout::FlatRow, bool struct_layout = false) {
     using namespace Halide;
     auto [code_pack, code_bytes] = make_code_pack(block_size, code_bits, qmax);
+
+    if (struct_layout) {
+        // The on-disk block as a first-class struct: `{fp16 d; uint8 qs[...]}`,
+        // matching every symmetric GGML block_* layout. The compiler owns the
+        // offsets and the total byte size; StructBlockLayout reads/writes `d` as
+        // a typed field (subsuming Fp16Pack) and hands the `qs` bytes to the same
+        // code_pack the byte-buffer path uses. Apply{0, code_pack} interprets
+        // those bytes (nibble/byte/bit extraction) exactly as before.
+        Type block_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), code_bytes}});
+        return {std::make_unique<Compose>(
+                    StructBlockLayout{block_type, "d", "qs"},
+                    Apply{0, std::move(code_pack)},
+                    SymmetricAffineQuantize{block_size, qmax, rounding, anchor},
+                    BlockReshape{block_size, layout == Layout::BlockIndexed}),
+                block_type.bytes(),
+                block_type};
+    }
+
     BlockLayout bl = make_block_layout(
         FieldSpec{1, 2, std::make_unique<Fp16Pack>()},    // scale
         FieldSpec{0, code_bytes, std::move(code_pack)});  // codes
diff --git a/apps/ggml/halide/symmetric_quant_generators.cpp b/apps/ggml/halide/symmetric_quant_generators.cpp
index 0f53f213a575..5b2d6863d06a 100644
--- a/apps/ggml/halide/symmetric_quant_generators.cpp
+++ b/apps/ggml/halide/symmetric_quant_generators.cpp
@@ -98,7 +98,10 @@ class SymmetricCodecGenerator : public CodecGeneratorBase<SymmetricCodecGenerato
         // arithmetic to duplicate here.
         switch (kind.value()) {
         case SchemeKind::Symmetric:
-            return make_symmetric_block_scheme(block_size, qmax, rounding, anchor, code_bits);
+            // Phase 3 pilot: the symmetric codecs (Q4_0/Q8_0/Q1_0) build their
+            // on-disk block as a first-class Type::Struct.
+            return make_symmetric_block_scheme(block_size, qmax, rounding, anchor, code_bits,
+                                               Layout::FlatRow, /*struct_layout=*/true);
         case SchemeKind::Affine:
             return make_affine_block_scheme(block_size, levels, affine_rounding, code_bits);
         case SchemeKind::Symmetric5Bit:

From 7cad24305f390dcc2c2a7d54b41d4677c0e2c799 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 03:34:23 -0400
Subject: [PATCH 07/22] apps/ggml: restore SDOT vec-dot via deep decode-chain
 inlining

The SDOT integer-dot schedule was downgraded to a plain Float reduction in
Phase 2 because hoist_invariants() could not find the per-block scale as a
distributable factor: a single eager_inline() of approximate_by()'s
.replacement only peels the outermost relayout wrapper, leaving the scale*codes
product buried inside the dequantizer Func the Approximation combinators build.

Fix: in the SDOT branch, after rfactor() preserves the block index, fold the
*entire* decode chain of both operands into the per-block partial's update.
eager_inline() no-ops on any Func not currently directly called and flattens
exposed calls left to right, so inlining the whole set of inlinable decode
handles -- one pass per possible chain level -- flattens it regardless of build
order, leaving (codes*scale)*(codes*scale) with the scales as loop-invariant
leaves. hoist_invariants() then lifts them and change_type(Int(32)) retypes the
scale-free inner dot, which CodeGen_ARM matches to SDOT.

Re-enables SDOT for Symmetric (Q4_0/Q8_0), Symmetric5Bit (Q5_0 -- its
CombineBits code reconstruction is all inside the r.x-dependent codes leaf, so
the scale stays a top-level factor), and the single-scale codebook families
(IQ4_NL/MXFP4). Q1_0 stays Float: its 128-wide 1-bit block trips
change_type(Int(32))'s overflow proof.

Verified: kernel-bench --all reports zero mismatches vs GGML and all 28
roundtrip tests pass. q4_0 vec_dot now runs at 1.48x ggml-cpu (was 0.07x on the
Float schedule) and q5_0 at 2.24x -- both beating GGML's hand-written kernels.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../halide/lookup_table_vec_dot_generator.cpp | 13 ++---
 .../halide/symmetric_vec_dot_generator.cpp    | 29 ++++-------
 apps/ggml/halide/vec_dot_generator_base.h     | 48 ++++++++++++++-----
 3 files changed, 52 insertions(+), 38 deletions(-)

diff --git a/apps/ggml/halide/lookup_table_vec_dot_generator.cpp b/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
index 609b250e0ea4..b6b86e8bca03 100644
--- a/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
+++ b/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
@@ -62,15 +62,12 @@ class LookupTableVecDotGenerator : public VecDotGeneratorBase<LookupTableVecDotG
         switch (family.value()) {
         case Family::IQ4_NL:
             // 4-bit codebook, single fp16 scale x Q8_0: single per-block scale
-            // and int8 codebook values -> SDOT-eligible in principle. See the
-            // TODO in symmetric_vec_dot_generator.cpp: mature hoist_invariants()
-            // can't lift the scale through approximate_by()'s round-trip
-            // replacement, so use the correct Float schedule for now.
-            return {make_iq4_nl_scheme(Layout::BlockIndexed).scheme, 18, q8_0_codec(), 34, 32, ScheduleKind::Float};
+            // and int8 codebook values -> SDOT-eligible (the base header's
+            // deep-inline SDOT exposes the scale even through the codebook LUT).
+            return {make_iq4_nl_scheme(Layout::BlockIndexed).scheme, 18, q8_0_codec(), 34, 32, ScheduleKind::SDOT};
         case Family::MXFP4:
-            // Same single-scale codebook shape as IQ4_NL (E8M0 scale) x Q8_0;
-            // same SDOT/hoist_invariants limitation -> Float for now.
-            return {make_mxfp4_scheme(Layout::BlockIndexed).scheme, 17, q8_0_codec(), 34, 32, ScheduleKind::Float};
+            // Same single-scale codebook shape as IQ4_NL (E8M0 scale) x Q8_0.
+            return {make_mxfp4_scheme(Layout::BlockIndexed).scheme, 17, q8_0_codec(), 34, 32, ScheduleKind::SDOT};
         case Family::NVFP4:
             // 64-element block (4 sub-scales) x Q8_0 (32-block): the activation
             // is Reblocked 32 -> 64 so both share the weight's block. Sub-block
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index 381f50bb7b06..9b79ade24a5c 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -84,17 +84,10 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         case WKind::Symmetric:
             wc = make_symmetric_block_scheme(wbs, w_qmax, w_rounding, w_anchor, w_code_bits, Layout::BlockIndexed).scheme;
             wb = 2 + (w_code_bits == 4 ? wbs / 2 : (w_code_bits == 1 ? wbs / 8 : wbs));
-            // TODO(ggml-on-qk, SDOT): the mature hoist_invariants() can't lift the
-            // per-block scale out of the reduction here. Unlike
-            // test/correctness/struct_type_dot_product.cpp (which eager_inline()s
-            // the dequantizer Funcs directly, exposing the scale as a leaf), this
-            // app builds the dequant via approximate_by()'s round-trip replacement,
-            // and a single eager_inline() of that replacement leaves the scale
-            // behind decode-chain Func boundaries hoist_invariants() can't see
-            // through. Correct (non-SDOT) Float schedule for now; see the base
-            // header's SDOT branch. Restoring SDOT needs deeper inlining of the
-            // decode chain or a hoist_invariants() that sees through it.
-            sched = ScheduleKind::Float;
+            // 1-bit (Q1_0) stays Float: change_type(Int(32)) can't prove its
+            // deep-inlined per-term range fits Int(32) (its 128-wide block trips
+            // the overflow check), and it's a niche format. Q4_0/Q8_0 -> SDOT.
+            sched = w_code_bits == 1 ? ScheduleKind::Float : ScheduleKind::SDOT;
             break;
         case WKind::Affine:
             wc = make_affine_block_scheme(wbs, w_levels, w_affine_rounding, w_code_bits, Layout::BlockIndexed).scheme;
@@ -104,14 +97,12 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         case WKind::Symmetric5Bit:
             wc = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed).scheme;
             wb = 2 + 4 + wbs / 2;
-            // TODO(ggml-on-qk): the alpha rfactor(HoistInvariantFactor) hoisted
-            // q5_0's per-block scale even though the 5-bit code is assembled via
-            // CombineBits (nibble | (high_bit << 4)); the mature hoist_invariants()
-            // does not recognize the scale as a distributable factor through that
-            // reconstruction and errors. Use the correct (non-SDOT) Float schedule
-            // until either hoist_invariants() is taught this shape or the 5-bit
-            // codec is restructured to expose the scale as a clean leaf.
-            sched = ScheduleKind::Float;
+            // Q5_0's 5-bit code is assembled via CombineBits (nibble |
+            // (high_bit << 4)); that reconstruction is all inside the (r.x-
+            // dependent) codes leaf, so the per-block scale is still a top-level
+            // invariant factor once the decode chain is fully inlined -- SDOT
+            // works the same as Q4_0 (see the base header's deep-inline SDOT).
+            sched = ScheduleKind::SDOT;
             break;
         case WKind::Affine5Bit:
             wc = make_affine_5bit_block_scheme(wbs, w_levels, w_affine_rounding, Layout::BlockIndexed).scheme;
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index a7ef4d3a9deb..7ea04f1187d6 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -108,18 +108,44 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
 
         if (spec.sched == ScheduleKind::SDOT) {
             // The per-block scale depends on the block index r.y, so it is only
-            // invariant across the *within-block* reduction r.x, not across all
-            // reduced RVars. rfactor() must therefore run first, preserving r.y
-            // as u so the partial Acc_dot reduces over r.x alone; only then can
-            // eager_inline() fold the dequantizers into that per-block partial
-            // and hoist_invariants() lift the now-invariant scale out of the r.x
-            // sum. change_type() finally retypes the scale-free inner dot to
-            // Int(32), leaving an SDOT-eligible integer dot. The alpha
-            // rfactor(HoistInvariantFactor) fused all of this; the mature API
-            // splits it (see test/correctness/struct_type_dot_product.cpp, the
-            // same q4_0 x q8_0 case).
+            // invariant across the *within-block* reduction r.x. rfactor() runs
+            // first, preserving r.y as u so the partial Acc_dot reduces over r.x
+            // alone.
             Func Acc_dot = Acc.update().rfactor({{r.y, u}});
-            Func Acc_ff = Acc_dot.update().eager_inline({wt_r.replacement, act_r.replacement}).hoist_invariants();
+
+            // hoist_invariants() needs each operand's per-block scale to appear
+            // as a top-level factor of Acc_dot's update product. A single
+            // eager_inline() of the two .replacements only peels the outermost
+            // relayout wrapper; the scale*codes product lives deeper, inside the
+            // dequantizer Func the Approximation combinators build (see the SDOT
+            // investigation). So fold in the *entire* decode chain of both
+            // operands: eager_inline() no-ops on any Func not (currently)
+            // directly called and flattens exposed calls left to right, so
+            // inlining the whole set of inlinable decode handles, one pass per
+            // possible chain level, flattens it regardless of build order --
+            // leaving Acc_dot's update as (codes*scale)*(codes*scale) with the
+            // scales as loop-invariant leaves.
+            std::vector<Func> decode_funcs = {wt_r.replacement, act_r.replacement};
+            for (const Func &h : wt_r.handles) {
+                if (h.function().can_be_inlined()) {
+                    decode_funcs.push_back(h);
+                }
+            }
+            for (const Func &h : act_r.handles) {
+                if (h.function().can_be_inlined()) {
+                    decode_funcs.push_back(h);
+                }
+            }
+            for (size_t pass = 0; pass < decode_funcs.size(); pass++) {
+                Acc_dot.update().eager_inline(decode_funcs);
+            }
+
+            // hoist_invariants() lifts the now-invariant scales out of the r.x
+            // sum; change_type() retypes the scale-free inner dot to Int(32),
+            // leaving an SDOT-eligible integer dot (CodeGen_ARM matches it to
+            // SDOT). See test/correctness/struct_type_dot_product.cpp for the
+            // same q4_0 x q8_0 case written directly (single dequant Func).
+            Func Acc_ff = Acc_dot.update().hoist_invariants();
             Func Acc_i32 = Acc_ff.change_type(Int(32));
             Acc_i32.compute_root()
                 .update()

From 9a5fa5e0a55b34bc4e1e6b0f48a7cc6c707a546f Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 03:42:14 -0400
Subject: [PATCH 08/22] apps/ggml: efficient SDOT schedule for repack gemv/gemm

Factors the vec_dot SDOT schedule into a shared sdot_partial() helper
(sdot_schedule.h): rfactor the block-index reduction, flatten every operand's
decode chain (deep eager_inline so hoist_invariants() sees each per-block scale
as a top-level factor), hoist the scales, and change_type(Int(32)) the
scale-free inner dot. vec_dot now calls it instead of open-coding the sequence.

Applies the same schedule to the repack matmul generators for the simple
single-scale weight families (Q4_0/Q8_0/IQ4_NL/MXFP4): in gemv the weight scale
depends only on (block, column) and the activation scale only on the block, so
both hoist out of the within-block reduction, same as gemm. K-quant weights
carry two-level (super/sub-block) scales that aren't a single per-block-
invariant factor, so they keep the default schedule.

Verified: kernel-bench --repack reports zero mismatches vs GGML. The simple
families go from the default schedule (~0.06-0.20x) to ~0.6x of GGML's
hand-written interleaved kernels for both gemv and gemm -- an efficient,
SDOT-based schedule is now writable for repack, matching the vec_dot path.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/halide/repack_matmul_generator.cpp | 22 +++++++
 apps/ggml/halide/sdot_schedule.h             | 60 ++++++++++++++++++++
 apps/ggml/halide/vec_dot_generator_base.h    | 46 ++-------------
 3 files changed, 88 insertions(+), 40 deletions(-)
 create mode 100644 apps/ggml/halide/sdot_schedule.h

diff --git a/apps/ggml/halide/repack_matmul_generator.cpp b/apps/ggml/halide/repack_matmul_generator.cpp
index cf95ed192a9f..2059a0ee07e7 100644
--- a/apps/ggml/halide/repack_matmul_generator.cpp
+++ b/apps/ggml/halide/repack_matmul_generator.cpp
@@ -12,6 +12,7 @@
 #include "Halide.h"
 
 #include "quant_components.h"
+#include "sdot_schedule.h"
 
 using namespace Halide;
 using namespace ggml_halide;
@@ -136,6 +137,18 @@ class RepackGemvGenerator : public Generator<RepackGemvGenerator> {
             }
         }
 
+        // Simple single-scale weight families (Q4_0/Q8_0/IQ4_NL/MXFP4) reduce
+        // as a scale-free Int(32) dot (SDOT), same as the vec_dot path: the
+        // per-block weight scale depends only on (block, column) and the
+        // activation scale only on the block, so both hoist out of the r.x sum.
+        // K-quant weights carry two-level (super/sub-block) scales that aren't a
+        // single per-block-invariant factor, so they keep the default schedule.
+        if (!kq) {
+            Var u("u");
+            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar});
+            s_i32.compute_root().update().atomic().vectorize(r.x, block_size);
+        }
+
         weight_blocks.dim(0).set_bounds(0, w.block_bytes);
         weight_blocks.dim(1).set_min(0);
         weight_blocks.dim(2).set_min(0);
@@ -237,6 +250,15 @@ class RepackGemmGenerator : public Generator<RepackGemmGenerator> {
             }
         }
 
+        // Simple single-scale weight families reduce as a scale-free Int(32) dot
+        // (SDOT), same as gemv/vec_dot; K-quant's two-level scales keep the
+        // default schedule. See sdot_schedule.h.
+        if (!is_kquant(family)) {
+            Var u("u");
+            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar});
+            s_i32.compute_root().update().atomic().vectorize(r.x, block_size);
+        }
+
         weight_blocks.dim(0).set_bounds(0, w.block_bytes);
         weight_blocks.dim(1).set_min(0);
         weight_blocks.dim(2).set_min(0);
diff --git a/apps/ggml/halide/sdot_schedule.h b/apps/ggml/halide/sdot_schedule.h
new file mode 100644
index 000000000000..62eb1b54367d
--- /dev/null
+++ b/apps/ggml/halide/sdot_schedule.h
@@ -0,0 +1,60 @@
+#pragma once
+
+// Shared "make an Approximation-decoded block reduction accumulate as an
+// integer dot product" schedule, used by both the vec_dot and repack matmul
+// Generators. Given a reduction `acc` of the shape
+//   acc(...) += decode(Wt)(r.x, r.y, ...) * decode(Vec)(r.x, r.y)
+// whose operands' per-block scales are invariant across the within-block
+// reduction r.x (but vary with the block index r.y and any output dims), this
+// derives the scale-free Int(32) inner dot the caller then schedules.
+//
+// Why the deep inline: hoist_invariants() needs each operand's per-block scale
+// to appear as a top-level factor of the (rfactored) update product. A single
+// eager_inline() of an ApproximationResult's .replacement only peels the
+// outermost relayout wrapper; the scale*codes product lives deeper, inside the
+// dequantizer Func the Approximation combinators build. eager_inline() no-ops
+// on any Func not currently directly called and flattens exposed calls left to
+// right, so inlining the whole set of inlinable decode handles -- one pass per
+// possible chain level -- flattens the decode chains of every operand
+// regardless of build order, leaving (codes*scale)*... with the scales as
+// loop-invariant leaves. See doc: the SDOT investigation on ggml-on-qk.
+
+#include "Halide.h"
+
+#include <vector>
+
+namespace ggml_halide {
+
+// rfactor `acc` preserving `preserved` (typically {{r.y, u}} -- the block
+// index), flatten every operand's decode chain into the resulting per-block
+// partial, hoist the now-invariant scales out of the remaining reduction, and
+// retype the scale-free inner dot to Int(32). Returns that Int(32) Func -- it
+// holds the real reduction, so the caller schedules *it* (compute_root,
+// vectorize the within-block RVar, etc.).
+inline Halide::Func sdot_partial(Halide::Func &acc,
+                                 const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
+                                 const std::vector<Halide::ApproximationResult> &operands) {
+    using namespace Halide;
+
+    Func acc_dot = acc.update().rfactor(preserved);
+
+    std::vector<Func> decode_funcs;
+    for (const ApproximationResult &op : operands) {
+        decode_funcs.push_back(op.replacement);
+    }
+    for (const ApproximationResult &op : operands) {
+        for (const Func &h : op.handles) {
+            if (h.function().can_be_inlined()) {
+                decode_funcs.push_back(h);
+            }
+        }
+    }
+    for (size_t pass = 0; pass < decode_funcs.size(); pass++) {
+        acc_dot.update().eager_inline(decode_funcs);
+    }
+
+    Func acc_ff = acc_dot.update().hoist_invariants();
+    return acc_ff.change_type(Int(32));
+}
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 7ea04f1187d6..fb3934db9173 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -27,6 +27,7 @@
 
 #include "codec_generator_base.h"  // Direction/SchemeAndBytes live here; shared idiom
 #include "quant_components.h"
+#include "sdot_schedule.h"
 
 namespace ggml_halide {
 
@@ -107,46 +108,11 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         }
 
         if (spec.sched == ScheduleKind::SDOT) {
-            // The per-block scale depends on the block index r.y, so it is only
-            // invariant across the *within-block* reduction r.x. rfactor() runs
-            // first, preserving r.y as u so the partial Acc_dot reduces over r.x
-            // alone.
-            Func Acc_dot = Acc.update().rfactor({{r.y, u}});
-
-            // hoist_invariants() needs each operand's per-block scale to appear
-            // as a top-level factor of Acc_dot's update product. A single
-            // eager_inline() of the two .replacements only peels the outermost
-            // relayout wrapper; the scale*codes product lives deeper, inside the
-            // dequantizer Func the Approximation combinators build (see the SDOT
-            // investigation). So fold in the *entire* decode chain of both
-            // operands: eager_inline() no-ops on any Func not (currently)
-            // directly called and flattens exposed calls left to right, so
-            // inlining the whole set of inlinable decode handles, one pass per
-            // possible chain level, flattens it regardless of build order --
-            // leaving Acc_dot's update as (codes*scale)*(codes*scale) with the
-            // scales as loop-invariant leaves.
-            std::vector<Func> decode_funcs = {wt_r.replacement, act_r.replacement};
-            for (const Func &h : wt_r.handles) {
-                if (h.function().can_be_inlined()) {
-                    decode_funcs.push_back(h);
-                }
-            }
-            for (const Func &h : act_r.handles) {
-                if (h.function().can_be_inlined()) {
-                    decode_funcs.push_back(h);
-                }
-            }
-            for (size_t pass = 0; pass < decode_funcs.size(); pass++) {
-                Acc_dot.update().eager_inline(decode_funcs);
-            }
-
-            // hoist_invariants() lifts the now-invariant scales out of the r.x
-            // sum; change_type() retypes the scale-free inner dot to Int(32),
-            // leaving an SDOT-eligible integer dot (CodeGen_ARM matches it to
-            // SDOT). See test/correctness/struct_type_dot_product.cpp for the
-            // same q4_0 x q8_0 case written directly (single dequant Func).
-            Func Acc_ff = Acc_dot.update().hoist_invariants();
-            Func Acc_i32 = Acc_ff.change_type(Int(32));
+            // Accumulate the per-block dot as a scale-free Int(32) reduction
+            // (preserving the block index r.y) so CodeGen_ARM matches it to
+            // SDOT; sdot_partial() flattens both operands' decode chains and
+            // hoists their per-block scales out. See sdot_schedule.h.
+            Func Acc_i32 = sdot_partial(Acc, {{r.y, u}}, {wt_r, act_r});
             Acc_i32.compute_root()
                 .update()
                 .atomic()

From 72ac30605ea6127ba558501fa4fcca1cbd9ed59a Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 03:49:11 -0400
Subject: [PATCH 09/22] apps/ggml: struct-type the symmetric vec_dot weight
 operand

Rolls the Type::Struct weight blocks from the codec path into the vec_dot
generator for the symmetric weight families (Q4_0/Q8_0/Q1_0):

  - VecDotSpec carries an optional Type per operand; the base generator declares
    a 1-D Type::Struct ImageParam (block index only) when set, vs the 2-D
    (byte, blk) UInt(8) buffer otherwise, and indexes the reduction's block RDom
    off the right dimension. compute_offline() binds/severs the struct-typed
    operand as an ordinary Func.
  - The symmetric weight uses make_symmetric_block_scheme(struct_layout=true) and
    reports its block Type. The activation stays on the byte path for now (Q8_0/
    Q8_1 are shared across many weight formats and the Reblock relayout is
    byte-based), so operands are struct-typed independently.
  - The Q4_0/Q8_0/Q1_0 vec_dot wrappers wrap GGML's raw weight bytes in the same
    1-D struct halide_buffer_t helper the codecs use.

SDOT survives the struct decode unchanged: the base header's deep inline
flattens StructBlockLayout's field()-based dequantizer the same way it flattens
the byte path's, so hoist_invariants() still lifts the per-block scale.

Verified: kernel-bench --vecdot reports zero mismatches; q4_0 vec_dot runs at
1.55x ggml-cpu (struct-typed weight, unchanged from the byte-path SDOT result).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/halide/ggml_quants.cpp              |  9 ++---
 .../halide/symmetric_vec_dot_generator.cpp    | 17 ++++++++--
 apps/ggml/halide/vec_dot_generator_base.h     | 34 ++++++++++++++-----
 3 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
index 6d095070f27d..abfb03f8a3b3 100644
--- a/apps/ggml/halide/ggml_quants.cpp
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -164,9 +164,8 @@ void ggml_quants_halide_vec_dot_q4_0_q8_0(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytesX = 2 + kQK / 2, kBlockBytesY = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
     halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
-    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    auto xb = struct_block_buffer(vx, nb, kBlockBytesX);  // weight: struct-typed
     Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
     Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
     check(q4_0_vec_dot(xb, yb, result), "q4_0_vec_dot");
@@ -298,9 +297,8 @@ void ggml_quants_halide_vec_dot_q8_0_q8_0(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytes = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t xshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
     halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    auto xb = struct_block_buffer(vx, nb, kBlockBytes);  // weight: struct-typed
     Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
     Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
     check(q8_0_vec_dot(xb, yb, result), "q8_0_vec_dot");
@@ -546,9 +544,8 @@ void ggml_quants_halide_vec_dot_q1_0_q8_0(int n, float *s, size_t bs, const void
     constexpr int kQKY = 32, kBlockBytesY = 2 + kQKY;
     const int32_t nbx = static_cast<int32_t>(n / kQKX);
     const int32_t nby = static_cast<int32_t>(n / kQKY);
-    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nbx, kBlockBytesX}};
     halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nby, kBlockBytesY}};
-    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    auto xb = struct_block_buffer(vx, nbx, kBlockBytesX);  // weight: struct-typed
     Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
     Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
     check(q1_0_vec_dot(xb, yb, result), "q1_0_vec_dot");
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index 9b79ade24a5c..c2e3d17c8ea2 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -80,15 +80,23 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         std::unique_ptr<Halide::Approximation> wc;
         int wb;
         ScheduleKind sched;
+        Halide::Type weight_type;  // set -> weight blocks are a 1-D Type::Struct buffer
         switch (w_kind.value()) {
-        case WKind::Symmetric:
-            wc = make_symmetric_block_scheme(wbs, w_qmax, w_rounding, w_anchor, w_code_bits, Layout::BlockIndexed).scheme;
+        case WKind::Symmetric: {
+            // Struct-typed weight blocks (`{fp16 d; uint8 qs[...]}`); SDOT still
+            // works because the base header's deep inline flattens the struct
+            // decode's dequantizer just like the byte path's.
+            SchemeAndBytes sb = make_symmetric_block_scheme(wbs, w_qmax, w_rounding, w_anchor, w_code_bits,
+                                                            Layout::BlockIndexed, /*struct_layout=*/true);
+            wc = std::move(sb.scheme);
+            weight_type = sb.block_type;
             wb = 2 + (w_code_bits == 4 ? wbs / 2 : (w_code_bits == 1 ? wbs / 8 : wbs));
             // 1-bit (Q1_0) stays Float: change_type(Int(32)) can't prove its
             // deep-inlined per-term range fits Int(32) (its 128-wide block trips
             // the overflow check), and it's a niche format. Q4_0/Q8_0 -> SDOT.
             sched = w_code_bits == 1 ? ScheduleKind::Float : ScheduleKind::SDOT;
             break;
+        }
         case WKind::Affine:
             wc = make_affine_block_scheme(wbs, w_levels, w_affine_rounding, w_code_bits, Layout::BlockIndexed).scheme;
             wb = 2 + 2 + (w_code_bits == 4 ? wbs / 2 : wbs);
@@ -130,7 +138,10 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         }
         ac = reblock_activation(std::move(ac), a_nat, wbs);
 
-        return {std::move(wc), wb, std::move(ac), ab, wbs, sched};
+        // Activation stays on the byte path for now (Q8_0/Q8_1 are shared across
+        // many weight formats, and the Reblock relayout is byte-based); only the
+        // weight operand is struct-typed here.
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, weight_type, Halide::Type{}};
     }
 };
 
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index fb3934db9173..85fa9d9c205b 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -53,6 +53,13 @@ struct VecDotSpec {
     int act_bytes;
     int block_size;
     ScheduleKind sched;
+    // When set (is_struct()), the corresponding operand's packed blocks are a
+    // first-class 1-D Type::Struct buffer (one struct per block) rather than a
+    // 2-D (byte, blk) UInt(8) buffer. Default-invalid keeps a codec on the byte
+    // path -- so an operand whose codec isn't struct-typed (e.g. a Reblock'd
+    // activation) just leaves its type unset.
+    Halide::Type weight_type;
+    Halide::Type act_type;
 };
 
 template<typename Derived>
@@ -63,9 +70,12 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         VecDotSpec spec = static_cast<Derived *>(this)->build_vec_dot();
         int bs = spec.block_size;
 
-        // dim 0: byte-within-block, dim 1: block index.
-        ImageParam x_blocks(UInt(8), 2, "x_blocks");  // weight format
-        ImageParam y_blocks(UInt(8), 2, "y_blocks");  // activation format
+        // A struct-typed operand's packed blocks are a 1-D Type::Struct buffer
+        // (block index only); a byte-path operand is 2-D (byte, blk).
+        const bool wt_struct = spec.weight_type.is_struct();
+        const bool act_struct = spec.act_type.is_struct();
+        ImageParam x_blocks = wt_struct ? ImageParam(spec.weight_type, 1, "x_blocks") : ImageParam(UInt(8), 2, "x_blocks");
+        ImageParam y_blocks = act_struct ? ImageParam(spec.act_type, 1, "y_blocks") : ImageParam(UInt(8), 2, "y_blocks");
 
         // Naive fp32 placeholders -- never realized; compute_offline() severs
         // Acc from them entirely, and the real values come from the
@@ -76,7 +86,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         Wt(kk, blk) = 0.0f;
         Vec(kk, blk) = 0.0f;
 
-        RDom r(0, bs, 0, x_blocks.dim(1).extent(), "r");
+        RDom r(0, bs, 0, x_blocks.dim(wt_struct ? 0 : 1).extent(), "r");
         Func Acc("acc");
         Acc() = 0.0f;
         Acc() += Wt(r.x, r.y) * Vec(r.x, r.y);
@@ -125,10 +135,18 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         Func result("result");
         result() = Acc();
 
-        x_blocks.dim(0).set_bounds(0, spec.weight_bytes);
-        x_blocks.dim(1).set_min(0);
-        y_blocks.dim(0).set_bounds(0, spec.act_bytes);
-        y_blocks.dim(1).set_min(0);
+        if (wt_struct) {
+            x_blocks.dim(0).set_min(0);
+        } else {
+            x_blocks.dim(0).set_bounds(0, spec.weight_bytes);
+            x_blocks.dim(1).set_min(0);
+        }
+        if (act_struct) {
+            y_blocks.dim(0).set_min(0);
+        } else {
+            y_blocks.dim(0).set_bounds(0, spec.act_bytes);
+            y_blocks.dim(1).set_min(0);
+        }
 
         this->add_input(x_blocks);
         this->add_input(y_blocks);

From 00d21ff9531644c0e2165232bbf6f2ece1e54902 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 09:16:28 -0400
Subject: [PATCH 10/22] apps/ggml: cast int8 codes straight to float in
 LinearDequant (enables SDOT)

LinearDequant dequantized as cast<float>(cast<int32_t>(codes)) * scale. The
redundant int32 detour broke change_type(Int(32))'s SDOT rewrite pattern
(cast<f32>(int8) * cast<f32>(int8)), so the repack gemv/gemm and codebook
(IQ4_NL/MXFP4) inner products fell back to a float16 SIMD multiply instead of an
int8 dot -- confirmed by `-e stmt`: widening_mul(float16x32, ...) vs vec_dot's
widening_mul(int8x32, int8x32).

Casting the int8 codes straight to float restores the SDOT match. Repack q4_0
gemv ~doubles (8.5 -> 14 GFLOP/s), IQ4_NL vec_dot 1188 -> 837 ns; 0 mismatches,
28/28 roundtrip tests still pass.

Note on the remaining repack gap: this is the best Halide can do here. ggml-cpu
matches the machine's `matmul_int8` (i8mm/SMMLA) instruction for its repack
kernels (~166 GFLOP/s on q4_0 4x4), but Halide's CodeGen_ARM has no SMMLA path
-- only SDOT (CodeGen_ARM.cpp:793-801). So a Halide repack tops out at SDOT
throughput (~12x below ggml-cpu's SMMLA matmul), regardless of schedule; closing
it would require i8mm support in the Halide backend. (vec_dot, a pure dot
product, has no such gap -- ggml-cpu uses SDOT there too, and Halide matches/beats
it.)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/halide/quant_components.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index a94ffe822f79..ee153a934d2a 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -1603,10 +1603,15 @@ class LinearDequant : public Halide::Approximation {
         Func dequantized("linear_dequantized");
         if (!has_super_d_) {
             Func codes = encoded[0], scale = encoded[1];
+            // cast<float> straight off the int8 codes (no int32 detour): the
+            // extra cast<int32_t> would break change_type()'s
+            // cast<f32>(int8)*cast<f32>(int8) SDOT pattern, forcing the vec_dot/
+            // repack inner product into a float16 multiply instead of an int8
+            // dot. See sdot_schedule.h.
             if (sub_size_ == 0) {
-                dequantized(kk, blk, _) = cast<float>(cast<int32_t>(codes(kk, blk, _))) * scale(blk, _);
+                dequantized(kk, blk, _) = cast<float>(codes(kk, blk, _)) * scale(blk, _);
             } else {
-                dequantized(kk, blk, _) = cast<float>(cast<int32_t>(codes(kk, blk, _))) * scale(kk / sub_size_, blk, _);
+                dequantized(kk, blk, _) = cast<float>(codes(kk, blk, _)) * scale(kk / sub_size_, blk, _);
             }
         } else if (has_min_) {
             Func d = encoded[0], dmin = encoded[1], scale_min = encoded[2], codes = encoded[3];

From 98555636e1c422c93c1df27c77f318ebcdc864a8 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 10:59:43 -0400
Subject: [PATCH 11/22] apps/ggml: bring q4_0/q8_0 vec_dot up to ggml-cpu speed

The SDOT vec_dot schedule collapsed each block's integer dot to a scalar
and accumulated it into a single float, which cost a horizontal reduce per
block and left the whole kernel bound by one nb-deep multiply-add chain.
Restructure it into ggml's shape:

  - Take the accumulator's vector lanes from the within-block reduction, so
    the sdot's four Int(32) lanes survive into the float accumulator and no
    block pays for a horizontal reduce. Lanes have to come from r.x, not
    r.y: blocks are interleaved {scale, codes} records, so a lane per block
    would gather both the codes and the scales.
  - Cut r.x so one sdot covers 16 int8s, with the chunks run serially into
    the same register. Reducing straight to 4 lanes instead makes Halide
    lower the wide reduce as two independent sdots plus an addp.
  - Interleave four blocks into independent accumulators. Widening the
    vector does not help here -- every lane of one accumulator advances on
    every block, so only interleaving blocks shortens the chain.
  - Give the main reduction a block count divisible by the interleave and
    sweep the remainder in a second update. A predicated tail is not a
    local cost: it makes the per-block sdot a dynamic-extent allocation
    that has to be zeroed and accumulated through memory, roughly doubling
    the cost of every block.
  - Reduce the lanes-by-blocks accumulators vectorially rather than as a
    serial chain of scalar adds.

vec_dot is called once per output element of a matvec, so the argument
marshalling is not amortized over anything; at these row lengths building
three Halide::Runtime::Buffers cost about as much as the dot product.
Fill a halide_buffer_t in place in the vec_dot wrappers instead, and build
these kernels with no_asserts/no_bounds_query.

Also pin the byte-path block stride, which was costing a serial pointer-add
chain per block, and add KERNEL_BENCH_N to the harness so per-call overhead
can be separated from per-block cost.

vec_dot vs ggml-cpu on M3 Max, n=4096: q4_0 0.52x -> 0.97x,
q8_0 0.61x -> 0.96x, q5_0 0.38x -> 0.51x. All 28 roundtrip tests pass and
kernel-bench --all reports no mismatches.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 apps/ggml/halide/CMakeLists.txt           |   3 +
 apps/ggml/halide/ggml_quants.cpp          |  74 +++++++++++----
 apps/ggml/halide/vec_dot_generator_base.h | 109 ++++++++++++++++++++--
 apps/ggml/src/bench_vecdot.cpp            |  45 +++++++--
 4 files changed, 199 insertions(+), 32 deletions(-)

diff --git a/apps/ggml/halide/CMakeLists.txt b/apps/ggml/halide/CMakeLists.txt
index e73cc8cdf175..78f8d58572b2 100644
--- a/apps/ggml/halide/CMakeLists.txt
+++ b/apps/ggml/halide/CMakeLists.txt
@@ -37,6 +37,7 @@ add_halide_library(
     q4_0_vec_dot
     FROM quants.generator
     GENERATOR symmetric_vec_dot
+    FEATURES no_asserts no_bounds_query
     PARAMS
     w_kind=symmetric block_size=32 w_qmax=8 w_code_bits=4 w_rounding=truncate_half_up_with_offset
     w_anchor=extreme_signed a_kind=q8_0 a_qmax=127
@@ -59,6 +60,7 @@ add_halide_library(
     q4_1_vec_dot
     FROM quants.generator
     GENERATOR symmetric_vec_dot
+    FEATURES no_asserts no_bounds_query
     PARAMS
     w_kind=affine block_size=32 w_levels=15 w_code_bits=4 w_affine_rounding=clamped_int8 a_kind=q8_1
     a_qmax=127
@@ -122,6 +124,7 @@ add_halide_library(
     q8_0_vec_dot
     FROM quants.generator
     GENERATOR symmetric_vec_dot
+    FEATURES no_asserts no_bounds_query
     PARAMS
     w_kind=symmetric block_size=32 w_qmax=127 w_code_bits=8 w_rounding=nearest w_anchor=abs_max
     a_kind=q8_0 a_qmax=127
diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
index abfb03f8a3b3..c2adb7e0c040 100644
--- a/apps/ggml/halide/ggml_quants.cpp
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -136,6 +136,49 @@ Halide::Runtime::Buffer<void, 1> struct_block_buffer(const void *data, int nb, i
     return Halide::Runtime::Buffer<void, 1>(ty, const_cast<void *>(data), 1, shape);
 }
 
+// vec_dot is called once per output element of a matvec, so the argument
+// marshalling is not amortized over anything: at the row lengths GGML actually
+// uses, building three Halide::Runtime::Buffers costs about as much as the dot
+// product itself. These kernels know their shapes exactly, so the vec_dot
+// wrappers below fill a halide_buffer_t in place instead. (The quantize /
+// dequantize / gemv wrappers stay on Buffer -- they run over whole rows or
+// tiles, where the difference is noise.)
+struct StackBuffer {
+    halide_buffer_t buf{};
+    halide_dimension_t dims[2]{};
+
+    // Packed blocks as a 1-D Type::Struct buffer: one struct per block, with the
+    // block width in the type's `reserved` field (see Halide::Type::to_abi()).
+    halide_buffer_t *blocks_struct(const void *data, int nb, int block_bytes) {
+        buf.type = halide_type_t(halide_type_struct, 8);
+        buf.type.reserved = static_cast<uint16_t>(block_bytes);
+        dims[0] = {0, nb, 1, 0};
+        return init(data, 1);
+    }
+
+    // Packed blocks as a 2-D (byte, block) UInt(8) buffer.
+    halide_buffer_t *blocks_bytes(const void *data, int nb, int block_bytes) {
+        buf.type = halide_type_t(halide_type_uint, 8);
+        dims[0] = {0, block_bytes, 1, 0};
+        dims[1] = {0, nb, block_bytes, 0};
+        return init(data, 2);
+    }
+
+    // The 0-D float32 result.
+    halide_buffer_t *scalar_f32(float *data) {
+        buf.type = halide_type_t(halide_type_float, 32);
+        return init(data, 0);
+    }
+
+private:
+    halide_buffer_t *init(const void *data, int dimensions) {
+        buf.host = const_cast<uint8_t *>(static_cast<const uint8_t *>(data));
+        buf.dimensions = dimensions;
+        buf.dim = dimensions ? dims : nullptr;
+        return &buf;
+    }
+};
+
 }  // namespace
 
 extern "C" {
@@ -164,11 +207,11 @@ void ggml_quants_halide_vec_dot_q4_0_q8_0(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytesX = 2 + kQK / 2, kBlockBytesY = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
-    auto xb = struct_block_buffer(vx, nb, kBlockBytesX);  // weight: struct-typed
-    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
-    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
-    check(q4_0_vec_dot(xb, yb, result), "q4_0_vec_dot");
+    StackBuffer xb, yb, result;
+    check(q4_0_vec_dot(xb.blocks_struct(vx, nb, kBlockBytesX),  // weight: struct-typed
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       result.scalar_f32(s)),
+          "q4_0_vec_dot");
 }
 
 //
@@ -197,12 +240,11 @@ void ggml_quants_halide_vec_dot_q4_1_q8_1(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytesX = 4 + kQK / 2, kBlockBytesY = 4 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
-    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
-    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
-    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
-    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
-    check(q4_1_vec_dot(xb, yb, result), "q4_1_vec_dot");
+    StackBuffer xb, yb, result;
+    check(q4_1_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       result.scalar_f32(s)),
+          "q4_1_vec_dot");
 }
 
 //
@@ -297,11 +339,11 @@ void ggml_quants_halide_vec_dot_q8_0_q8_0(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytes = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    auto xb = struct_block_buffer(vx, nb, kBlockBytes);  // weight: struct-typed
-    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
-    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
-    check(q8_0_vec_dot(xb, yb, result), "q8_0_vec_dot");
+    StackBuffer xb, yb, result;
+    check(q8_0_vec_dot(xb.blocks_struct(vx, nb, kBlockBytes),  // weight: struct-typed
+                       yb.blocks_bytes(vy, nb, kBlockBytes),
+                       result.scalar_f32(s)),
+          "q8_0_vec_dot");
 }
 
 //
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 85fa9d9c205b..47ac0b8abe52 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -65,6 +65,11 @@ struct VecDotSpec {
 template<typename Derived>
 class VecDotGeneratorBase : public Halide::Generator<Derived> {
 public:
+    // How many blocks the SDOT schedule keeps in flight as independent float
+    // accumulators. Four is enough to hide the accumulate latency without
+    // running the block loop out of vector registers.
+    static constexpr int kUnrollBlocks = 4;
+
     void configure() {
         using namespace Halide;
         VecDotSpec spec = static_cast<Derived *>(this)->build_vec_dot();
@@ -77,6 +82,14 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         ImageParam x_blocks = wt_struct ? ImageParam(spec.weight_type, 1, "x_blocks") : ImageParam(UInt(8), 2, "x_blocks");
         ImageParam y_blocks = act_struct ? ImageParam(spec.act_type, 1, "y_blocks") : ImageParam(UInt(8), 2, "y_blocks");
 
+        // The packed-block buffers are quantized GGML rows: their base pointers
+        // are cache-line aligned. Without this Halide assumes 1-byte alignment
+        // and lowers every strided / reinterpreted read (the interleaved fp16
+        // scales, the int8 codes) to byte-wise ld1.b + orr reassembly instead
+        // of wide vector loads -- which dominates these tiny vec_dots.
+        x_blocks.set_host_alignment(64);
+        y_blocks.set_host_alignment(64);
+
         // Naive fp32 placeholders -- never realized; compute_offline() severs
         // Acc from them entirely, and the real values come from the
         // already-quantized x_blocks/y_blocks. Block-indexed (kk, blk) to match
@@ -86,10 +99,26 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         Wt(kk, blk) = 0.0f;
         Vec(kk, blk) = 0.0f;
 
-        RDom r(0, bs, 0, x_blocks.dim(wt_struct ? 0 : 1).extent(), "r");
+        // The SDOT schedule interleaves kUnrollBlocks blocks into independent
+        // accumulators, so it wants a block count divisible by that. Letting
+        // Halide's split produce the odd tail instead is not a local cost: the
+        // predicate it inserts makes the per-block sdot a dynamic-extent
+        // allocation that has to be zeroed and accumulated through memory,
+        // roughly doubling the cost of *every* block. So the main reduction gets
+        // an exactly divisible extent and a second update sweeps the remainder
+        // at the default schedule (at most kUnrollBlocks - 1 blocks).
+        const bool sdot = spec.sched == ScheduleKind::SDOT;
+        Expr nblocks = x_blocks.dim(wt_struct ? 0 : 1).extent();
+        Expr main_blocks = sdot ? (nblocks / kUnrollBlocks) * kUnrollBlocks : nblocks;
+
+        RDom r(0, bs, 0, main_blocks, "r");
         Func Acc("acc");
         Acc() = 0.0f;
         Acc() += Wt(r.x, r.y) * Vec(r.x, r.y);
+        RDom r_tail(0, bs, main_blocks, nblocks - main_blocks, "r_tail");
+        if (sdot) {
+            Acc() += Wt(r_tail.x, r_tail.y) * Vec(r_tail.x, r_tail.y);
+        }
 
         ApproximationResult wt_r = Wt.approximate_by(*spec.weight_codec, {Acc});
         ApproximationResult act_r = Vec.approximate_by(*spec.act_codec, {Acc});
@@ -118,15 +147,72 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         }
 
         if (spec.sched == ScheduleKind::SDOT) {
-            // Accumulate the per-block dot as a scale-free Int(32) reduction
-            // (preserving the block index r.y) so CodeGen_ARM matches it to
-            // SDOT; sdot_partial() flattens both operands' decode chains and
-            // hoists their per-block scales out. See sdot_schedule.h.
-            Func Acc_i32 = sdot_partial(Acc, {{r.y, u}}, {wt_r, act_r});
-            Acc_i32.compute_root()
+            // The reduction is over (within-block r.x) x (block r.y). The lanes
+            // of the accumulator come from r.x, so the sdot's four Int(32) lanes
+            // survive all the way into the float accumulator and no block pays
+            // for a horizontal reduce. They must come from r.x rather than r.y:
+            // blocks are interleaved {scale, codes} records, so a lane per block
+            // would gather both the codes and the scales, while a lane per
+            // within-block group keeps every code load contiguous.
+            //
+            // One sdot consumes 16 int8s and lands in a 4-lane Int(32) register,
+            // so r.x is cut three ways: chunks of 16 (one sdot each, run
+            // serially so they accumulate into the *same* register), then within
+            // a chunk a 4-wide lane index and the 4 elements the lane sums.
+            // Reducing straight to 4 lanes instead would make Halide lower the
+            // wide reduce as two independent sdots plus an addp to merge them --
+            // an extra zeroing and an extra reduction per block.
+            const int lanes = 4;
+            const int chunk = 4 * lanes;
+            RVar rxc("rxc"), rxr("rxr"), rxo("rxo"), rxi("rxi");
+            Acc.update(0).split(r.x, rxc, rxr, chunk);
+            Acc.update(0).split(rxr, rxo, rxi, 4);
+
+            // sdot_partial() flattens both operands' decode chains and hoists
+            // their per-block scales out of the surviving rxi reduction, leaving
+            // the scale-free Int(32) dot. See sdot_schedule.h.
+            Var lane("lane");
+            Func Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r});
+
+            // Acc's update now reduces over (rxo, r.y). Peel rxo back off as the
+            // vector lanes, and peel kUnrollBlocks consecutive blocks off
+            // alongside it into separate accumulators. The accumulators have to
+            // be split over *blocks*: every lane of one accumulator advances on
+            // every block, so widening the vector does not shorten the
+            // multiply-add chain, only interleaving blocks does. At ~3-4 cycles
+            // of accumulate latency, an un-interleaved chain is what bounds the
+            // whole kernel.
+            RVar ryo("ryo"), ryi("ryi");
+            Var lv("lv"), bacc("bacc");
+            Acc.update(0).split(r.y, ryo, ryi, kUnrollBlocks);
+            Func acc_vec = Acc.update(0).rfactor({{rxo, lv}, {ryi, bacc}});
+            acc_vec.compute_root().vectorize(lv, lanes).unroll(bacc);
+            acc_vec.update().vectorize(lv, lanes).unroll(bacc);
+
+            // Inside the unrolled body, not at the block-group loop: at `bacc` the
+            // sdot is one block's worth of registers, whereas at `ryo` it is a
+            // kUnrollBlocks-long buffer that Halide has to allocate, zero, and
+            // accumulate through memory.
+            Acc_i32.compute_at(acc_vec, bacc)
                 .update()
                 .atomic()
-                .vectorize(r.x, bs);
+                .vectorize(rxi, 4)
+                .vectorize(lane, lanes)
+                .unroll(rxc);
+
+            // Collapsing the lanes x unrolled-blocks accumulators is a fixed
+            // cost, but at the row lengths GGML uses it is not a negligible one:
+            // left alone it is a serial chain of lanes*unroll_blocks scalar
+            // adds. Sum the blocks vectorially first, then reduce the lanes
+            // horizontally, so it costs a handful of vector ops instead.
+            Var lv2("lv2");
+            Func acc_lanes = Acc.update(0).rfactor(rxo, lv2);
+            acc_lanes.compute_root().vectorize(lv2, lanes);
+            acc_lanes.update().vectorize(lv2, lanes);
+            Acc.update(0).atomic().vectorize(rxo, lanes);
+
+            // The odd-block tail deliberately keeps the default schedule.
+            Acc.update(1).unscheduled();
         }
         // ScheduleKind::Float: leave the reduction at its default (legal) schedule
         // -- correctness first; an interleave/sub-block-aware performance schedule
@@ -135,17 +221,20 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         Func result("result");
         result() = Acc();
 
+        // A byte-path operand's block stride is pinned to its block width: these
+        // are densely packed GGML rows, and leaving the stride dynamic costs a
+        // serial pointer-add chain per block instead of an immediate offset.
         if (wt_struct) {
             x_blocks.dim(0).set_min(0);
         } else {
             x_blocks.dim(0).set_bounds(0, spec.weight_bytes);
-            x_blocks.dim(1).set_min(0);
+            x_blocks.dim(1).set_min(0).set_stride(spec.weight_bytes);
         }
         if (act_struct) {
             y_blocks.dim(0).set_min(0);
         } else {
             y_blocks.dim(0).set_bounds(0, spec.act_bytes);
-            y_blocks.dim(1).set_min(0);
+            y_blocks.dim(1).set_min(0).set_stride(spec.act_bytes);
         }
 
         this->add_input(x_blocks);
diff --git a/apps/ggml/src/bench_vecdot.cpp b/apps/ggml/src/bench_vecdot.cpp
index 3081e111d26a..c2e522794d28 100644
--- a/apps/ggml/src/bench_vecdot.cpp
+++ b/apps/ggml/src/bench_vecdot.cpp
@@ -7,9 +7,39 @@
 #include "data_gen.h"
 #include "timing.h"
 
+#include <cstdlib>
+#include <cstring>
+
 namespace {
 // Divisible by every quant block size in play (32 for the q4/q5/q8 family, 256 for the k-quants).
-constexpr int64_t kElements = 4096;
+// Dev iteration aid: KERNEL_BENCH_N overrides it (must stay a multiple of 256),
+// which is how per-call overhead is separated from per-block cost.
+int64_t elements() {
+    const char *n = std::getenv("KERNEL_BENCH_N");
+    return (n && *n) ? std::atoll(n) : 4096;
+}
+const int64_t kElements = elements();
+
+// Dev iteration aid: KERNEL_BENCH_FILTER=q4_0,q8_0 restricts the sweep to
+// matching type names (substring match, comma-separated). Empty => all.
+bool passes_filter(const char *name) {
+    const char *f = std::getenv("KERNEL_BENCH_FILTER");
+    if (!f || !*f) {
+        return true;
+    }
+    std::string filt(f), item;
+    size_t pos = 0;
+    while (pos <= filt.size()) {
+        size_t comma = filt.find(',', pos);
+        if (comma == std::string::npos) comma = filt.size();
+        item = filt.substr(pos, comma - pos);
+        if (!item.empty() && std::strstr(name, item.c_str())) {
+            return true;
+        }
+        pos = comma + 1;
+    }
+    return false;
+}
 }  // namespace
 
 BenchReport run_vecdot_benchmarks(const KernelRegistries &registries) {
@@ -22,6 +52,9 @@ BenchReport run_vecdot_benchmarks(const KernelRegistries &registries) {
         if (!ref) {
             continue;
         }
+        if (!passes_filter(ggml_type_name(type))) {
+            continue;
+        }
 
         const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(type);
         if (!tc) {
@@ -54,8 +87,8 @@ BenchReport run_vecdot_benchmarks(const KernelRegistries &registries) {
         BenchRow row;
         row.label = ggml_type_name(type);
         row.ref_name = ref->name;
-        row.ref_ns = ref_time.median_ns;
-        row.ref_throughput = bytes_per_sec(vx.size() + vy.size(), ref_time.median_ns) / 1e9;
+        row.ref_ns = ref_time.min_ns;
+        row.ref_throughput = bytes_per_sec(vx.size() + vy.size(), ref_time.min_ns) / 1e9;
 
         for (const auto &cand : registries.vec_dot.candidates(type)) {
             BenchCandidate bc;
@@ -68,9 +101,9 @@ BenchReport run_vecdot_benchmarks(const KernelRegistries &registries) {
 
                 const TimingResult cand_time =
                     time_calls([&] { cand.fn(kElements, &cand_result, 0, vx.data(), 0, vy.data(), 0, 1); });
-                bc.ns = cand_time.median_ns;
-                bc.throughput = bytes_per_sec(vx.size() + vy.size(), cand_time.median_ns) / 1e9;
-                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+                bc.ns = cand_time.min_ns;
+                bc.throughput = bytes_per_sec(vx.size() + vy.size(), cand_time.min_ns) / 1e9;
+                bc.speedup = ref_time.min_ns / cand_time.min_ns;
             }
             row.candidates.push_back(bc);
         }

From a0a449abd19806a8612e9123f101dcb01cd545bb Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 11:40:55 -0400
Subject: [PATCH 12/22] apps/ggml: take the affine vec_dots (q4_1, q5_1) to
 SDOT

The affine formats decode to d*code + m, so the per-block product
(d*code + m) * (d_act*act) has no single scale to hoist and the generator
left them on the plain float reduction -- an unscheduled, fully scalar
loop over every element, redoing the nibble extraction and both fp16 scale
loads per element.

Multiplying the product out with the new Stage::distribute() gives
  d*d_act * sum(code*act)  +  m*d_act * sum(act)
and hoist_invariants() puts each term in its own accumulator. Both bodies
are integer, so both reach SDOT: the first as the ordinary code-by-
activation dot, the second as a dot with a vector of ones. That is ggml's
own decomposition of these formats, except that ggml reads sum(act) from
the s field block_q8_1 stores at quantize time rather than recomputing it,
which is worth about four SIMD ops per block.

vec_dot vs ggml-cpu on M3 Max, n=4096: q4_1 0.03x -> 0.80x (3843 ns ->
134 ns), q5_1 0.04x -> 0.50x (4059 ns -> 289 ns). All 28 roundtrip tests
pass, kernel-bench --all reports no mismatches, and the odd-block tail is
correct for block counts that are not a multiple of the interleave.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 apps/ggml/halide/sdot_schedule.h                | 12 +++++++++++-
 .../ggml/halide/symmetric_vec_dot_generator.cpp | 17 ++++++++++++++---
 apps/ggml/halide/vec_dot_generator_base.h       |  7 ++++++-
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/apps/ggml/halide/sdot_schedule.h b/apps/ggml/halide/sdot_schedule.h
index 62eb1b54367d..1d64e581733d 100644
--- a/apps/ggml/halide/sdot_schedule.h
+++ b/apps/ggml/halide/sdot_schedule.h
@@ -31,9 +31,16 @@ namespace ggml_halide {
 // retype the scale-free inner dot to Int(32). Returns that Int(32) Func -- it
 // holds the real reduction, so the caller schedules *it* (compute_root,
 // vectorize the within-block RVar, etc.).
+// `distribute` multiplies the per-block product out before hoisting, for
+// formats whose decode carries an offset: an affine weight makes the product
+// (d*code + m) * (d_act*act), which has no single scale to hoist. Multiplied out
+// it is d*d_act * sum(code*act) + m*d_act * sum(act), and hoist_invariants()
+// gives each term its own accumulator -- both with integer bodies, so both reach
+// SDOT. That is ggml's own decomposition of the affine formats.
 inline Halide::Func sdot_partial(Halide::Func &acc,
                                  const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
-                                 const std::vector<Halide::ApproximationResult> &operands) {
+                                 const std::vector<Halide::ApproximationResult> &operands,
+                                 bool distribute = false) {
     using namespace Halide;
 
     Func acc_dot = acc.update().rfactor(preserved);
@@ -53,6 +60,9 @@ inline Halide::Func sdot_partial(Halide::Func &acc,
         acc_dot.update().eager_inline(decode_funcs);
     }
 
+    if (distribute) {
+        acc_dot.update().distribute();
+    }
     Func acc_ff = acc_dot.update().hoist_invariants();
     return acc_ff.change_type(Int(32));
 }
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index c2e3d17c8ea2..29090385762b 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -80,6 +80,7 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         std::unique_ptr<Halide::Approximation> wc;
         int wb;
         ScheduleKind sched;
+        bool distribute = false;   // set for the affine (offset-carrying) weights
         Halide::Type weight_type;  // set -> weight blocks are a 1-D Type::Struct buffer
         switch (w_kind.value()) {
         case WKind::Symmetric: {
@@ -100,7 +101,13 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         case WKind::Affine:
             wc = make_affine_block_scheme(wbs, w_levels, w_affine_rounding, w_code_bits, Layout::BlockIndexed).scheme;
             wb = 2 + 2 + (w_code_bits == 4 ? wbs / 2 : wbs);
-            sched = ScheduleKind::Float;
+            // The affine decode is (d*code + m), so the per-block product is not
+            // a single scaled term. hoist_invariants() multiplies it out into
+            //   d*d_act * sum(code*act)  +  m*d_act * sum(act)
+            // and gives each term its own accumulator, which is ggml's own
+            // decomposition -- both inner sums are integer, so both reach SDOT.
+            sched = ScheduleKind::SDOT;
+            distribute = true;
             break;
         case WKind::Symmetric5Bit:
             wc = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed).scheme;
@@ -115,7 +122,11 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         case WKind::Affine5Bit:
             wc = make_affine_5bit_block_scheme(wbs, w_levels, w_affine_rounding, Layout::BlockIndexed).scheme;
             wb = 2 + 2 + 4 + wbs / 2;
-            sched = ScheduleKind::Float;
+            // Same offset-carrying decode as the 4-bit affine case above; the
+            // 5-bit code reconstruction stays inside the codes leaf, so the
+            // multiplied-out terms are integer just the same.
+            sched = ScheduleKind::SDOT;
+            distribute = true;
             break;
         }
 
@@ -141,7 +152,7 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         // Activation stays on the byte path for now (Q8_0/Q8_1 are shared across
         // many weight formats, and the Reblock relayout is byte-based); only the
         // weight operand is struct-typed here.
-        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, weight_type, Halide::Type{}};
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}};
     }
 };
 
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 47ac0b8abe52..6dea4d4625ae 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -53,6 +53,11 @@ struct VecDotSpec {
     int act_bytes;
     int block_size;
     ScheduleKind sched;
+    // Whether the SDOT schedule must multiply the per-block product out before
+    // hoisting. Set for the formats whose weight decode carries an offset (the
+    // affine family): (d*code + m) * (d_act*act) has no single scale to hoist
+    // until it is expanded. See sdot_schedule.h.
+    bool distribute_terms;
     // When set (is_struct()), the corresponding operand's packed blocks are a
     // first-class 1-D Type::Struct buffer (one struct per block) rather than a
     // 2-D (byte, blk) UInt(8) buffer. Default-invalid keeps a codec on the byte
@@ -172,7 +177,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // their per-block scales out of the surviving rxi reduction, leaving
             // the scale-free Int(32) dot. See sdot_schedule.h.
             Var lane("lane");
-            Func Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r});
+            Func Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms);
 
             // Acc's update now reduces over (rxo, r.y). Peel rxo back off as the
             // vector lanes, and peel kUnrollBlocks consecutive blocks off

From 72330ec73a391de0288ed0b0ae745d0993a6c88b Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 12:08:21 -0400
Subject: [PATCH 13/22] apps/ggml: propagate hoist_invariants()'s vector<Func>
 return through sdot_partial()

sdot_partial() was written against hoist_invariants()'s old single-Func
return, so it retyped the intermediate and handed back one Func. Now that
hoist_invariants() gives each term of a split reduction its own
single-valued accumulator, thread that through: sdot_partial() returns a
std::vector<Func> -- one Int(32) part per term -- and
vec_dot_generator_base.h carries the vector through instead of narrowing it
back down to one Func.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 apps/ggml/halide/repack_matmul_generator.cpp |  4 ++--
 apps/ggml/halide/sdot_schedule.h             | 19 +++++++++++++------
 apps/ggml/halide/vec_dot_generator_base.h    | 16 +++++++++-------
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/apps/ggml/halide/repack_matmul_generator.cpp b/apps/ggml/halide/repack_matmul_generator.cpp
index 2059a0ee07e7..118eea1aeb24 100644
--- a/apps/ggml/halide/repack_matmul_generator.cpp
+++ b/apps/ggml/halide/repack_matmul_generator.cpp
@@ -145,7 +145,7 @@ class RepackGemvGenerator : public Generator<RepackGemvGenerator> {
         // single per-block-invariant factor, so they keep the default schedule.
         if (!kq) {
             Var u("u");
-            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar});
+            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar})[0];
             s_i32.compute_root().update().atomic().vectorize(r.x, block_size);
         }
 
@@ -255,7 +255,7 @@ class RepackGemmGenerator : public Generator<RepackGemmGenerator> {
         // default schedule. See sdot_schedule.h.
         if (!is_kquant(family)) {
             Var u("u");
-            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar});
+            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar})[0];
             s_i32.compute_root().update().atomic().vectorize(r.x, block_size);
         }
 
diff --git a/apps/ggml/halide/sdot_schedule.h b/apps/ggml/halide/sdot_schedule.h
index 1d64e581733d..6612bf49289f 100644
--- a/apps/ggml/halide/sdot_schedule.h
+++ b/apps/ggml/halide/sdot_schedule.h
@@ -37,10 +37,10 @@ namespace ggml_halide {
 // it is d*d_act * sum(code*act) + m*d_act * sum(act), and hoist_invariants()
 // gives each term its own accumulator -- both with integer bodies, so both reach
 // SDOT. That is ggml's own decomposition of the affine formats.
-inline Halide::Func sdot_partial(Halide::Func &acc,
-                                 const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
-                                 const std::vector<Halide::ApproximationResult> &operands,
-                                 bool distribute = false) {
+inline std::vector<Halide::Func> sdot_partial(Halide::Func &acc,
+                                              const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
+                                              const std::vector<Halide::ApproximationResult> &operands,
+                                              bool distribute = false) {
     using namespace Halide;
 
     Func acc_dot = acc.update().rfactor(preserved);
@@ -63,8 +63,15 @@ inline Halide::Func sdot_partial(Halide::Func &acc,
     if (distribute) {
         acc_dot.update().distribute();
     }
-    Func acc_ff = acc_dot.update().hoist_invariants();
-    return acc_ff.change_type(Int(32));
+
+    // One accumulator per term -- one for a symmetric weight, two once an affine
+    // weight's product has been multiplied out. Each is its own Func, so each
+    // retypes on its own.
+    std::vector<Func> parts;
+    for (Func &part : acc_dot.update().hoist_invariants()) {
+        parts.push_back(part.change_type(Int(32)));
+    }
+    return parts;
 }
 
 }  // namespace ggml_halide
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 6dea4d4625ae..cebde7cd8c30 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -177,7 +177,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // their per-block scales out of the surviving rxi reduction, leaving
             // the scale-free Int(32) dot. See sdot_schedule.h.
             Var lane("lane");
-            Func Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms);
+            std::vector<Func> Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms);
 
             // Acc's update now reduces over (rxo, r.y). Peel rxo back off as the
             // vector lanes, and peel kUnrollBlocks consecutive blocks off
@@ -198,12 +198,14 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // sdot is one block's worth of registers, whereas at `ryo` it is a
             // kUnrollBlocks-long buffer that Halide has to allocate, zero, and
             // accumulate through memory.
-            Acc_i32.compute_at(acc_vec, bacc)
-                .update()
-                .atomic()
-                .vectorize(rxi, 4)
-                .vectorize(lane, lanes)
-                .unroll(rxc);
+            for (Func &part : Acc_i32) {
+                part.compute_at(acc_vec, bacc)
+                    .update()
+                    .atomic()
+                    .vectorize(rxi, 4)
+                    .vectorize(lane, lanes)
+                    .unroll(rxc);
+            }
 
             // Collapsing the lanes x unrolled-blocks accumulators is a fixed
             // cost, but at the row lengths GGML uses it is not a negligible one:

From fdb5376380acb5645b86b87bd739e51db3ed29f9 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 12:52:11 -0400
Subject: [PATCH 14/22] apps/ggml: notes for resuming the vec_dot performance
 work

Where q4_0/q8_0/q4_1 landed and why, the build/measure workflow (including
the traps: core-type noise, the generator's env var not invalidating ninja,
WITH_TESTS=OFF), the design decisions behind distribute() and per-term
accumulator Funcs, and the measured numbers for the stored-block-sum route
that is still outstanding.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 apps/ggml/PERF_NOTES.md | 277 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 277 insertions(+)
 create mode 100644 apps/ggml/PERF_NOTES.md

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
new file mode 100644
index 000000000000..ecefef2fccec
--- /dev/null
+++ b/apps/ggml/PERF_NOTES.md
@@ -0,0 +1,277 @@
+# vec_dot performance notes
+
+Working notes for bringing `apps/ggml`'s vec_dot kernels up to `ggml-cpu` speed
+on ARM (measured on an M3 Max). Branch `alexreinking/ggml-on-qk`, worktree
+`~/dev/Halide/ggml-on-qk`.
+
+## Where things stand
+
+Measured at n=4096, best of many runs (see "Measuring" below):
+
+| type | ggml-cpu | halide   | ratio | at session start |
+| ---- | -------- | -------- | ----- | ---------------- |
+| q4_0 | 94.2 ns  | 97.3 ns  | 0.97x | 0.52x            |
+| q4_1 | 106.7 ns | 133.8 ns | 0.80x | 0.03x            |
+| q5_0 | 122.2 ns | 241.7 ns | 0.51x | 0.38x            |
+| q5_1 | 143.4 ns | 288.1 ns | 0.50x | 0.04x            |
+| q8_0 | 71.9 ns  | 75.6 ns  | 0.95x | 0.61x            |
+
+Everything else in the table is still on the unscheduled float path
+(0.01x-0.12x) and untouched. All 28 roundtrip tests pass; `kernel-bench --all`
+reports no mismatches; odd block-count tails are correct (verified at n = 32,
+96, 160, 224, 1056).
+
+Commits, oldest first:
+
+- `c66ab336d` apps/ggml: bring q4_0/q8_0 vec_dot up to ggml-cpu speed
+- `c8ec1934d` Add `Stage::distribute()`, and one accumulator per term in
+  `hoist_invariants()`
+- `6c72fb4c1` apps/ggml: take the affine vec_dots (q4_1, q5_1) to SDOT
+- `a6e9b0962` `hoist_invariants()`: return one Func per accumulator, not a Tuple
+
+## Uncommitted
+
+`apps/ggml/halide/vec_dot_generator_base.h` has a probe branch guarded by
+`getenv("GGML_PER_BLOCK_PROBE")` -- "variant A", the per-block (single rfactor)
+schedule. It exists only to measure; delete it or keep it as the base for the
+next step (see below). The default path is unchanged and measures as above.
+
+Note the generator reads the env var at *generator run time*, so changing it
+does not invalidate ninja's outputs. Force regeneration:
+
+```sh
+rm -f build/apps/ggml/halide/q4_1_vec_dot.o build/apps/ggml/halide/libq4_1_vec_dot.a
+GGML_PER_BLOCK_PROBE=1 cmake --build build/apps/ggml -j --target q4_1_vec_dot
+cmake --build build/apps/ggml -j
+```
+
+## Building
+
+libHalide is consumed from `install/macOS`, so an app change needs only the app
+build, but a Halide change needs build + install first:
+
+```sh
+cmake --build build/macOS -j --target Halide
+cmake --install build/macOS --prefix install/macOS
+cmake --build build/apps/ggml -j
+```
+
+`build/macOS` is configured with `WITH_TESTS=OFF`, so `correctness_*` targets do
+not exist. Compile a test directly instead:
+
+```sh
+c++ -O1 -std=c++17 -DHALIDE_KEEP_MACROS -DHALIDE_WITH_EXCEPTIONS \
+	-I install/macOS/include -I test/common -I tools \
+	test/correctness/rfactor.cpp \
+	-L install/macOS/lib -lHalide -Wl,-rpath,$PWD/install/macOS/lib -o /tmp/rfactor && /tmp/rfactor
+```
+
+`HALIDE_KEEP_MACROS` is required (`internal_assert` is `#undef`'d at the end of
+the installed `Halide.h`); `HALIDE_WITH_EXCEPTIONS` is required or the
+exception-guarded tests silently do not compile. To find which sub-test fails,
+shard it: `TEST_TOTAL_SHARDS=40 TEST_SHARD_INDEX=N ./rfactor`.
+
+## Measuring
+
+macOS moves the process between P- and E-cores between runs, so a single run's
+absolute numbers are not comparable -- swings of 30%+ are normal. Take the best
+of several runs and always read the halide/ggml-cpu *ratio* within a run.
+
+`src/bench_vecdot.cpp` has two dev aids added this session:
+`KERNEL_BENCH_FILTER=q4_0,q8_0` (substring match on type name) and
+`KERNEL_BENCH_N=<n>` (vector length; default 4096). The n sweep is what
+separates per-call overhead from per-block cost -- fit the slope.
+
+Repeat-and-take-best wrapper:
+
+```sh
+#!/bin/zsh
+B=~/dev/Halide/ggml-on-qk/build/apps/ggml
+N=${N:-5}
+FILTER=${FILTER:-q4_0,q4_1,q8_0}
+TMP=$(mktemp -d)
+for i in $(seq $N); do KERNEL_BENCH_FILTER=$FILTER $B/kernel-bench --vecdot --csv $TMP/r$i.csv >/dev/null; done
+cat $TMP/*.csv | awk -F, '
+  $3!="role" && $1=="vec_dot" { k=$2 SUBSEP $4; if (!(k in best) || $5+0 < best[k]) best[k]=$5+0; ok[k]=$9; types[$2]=1 }
+  END { n=0; for (t in types) st[++n]=t
+        for(a=1;a<n;a++)for(b=a+1;b<=n;b++)if(st[a]>st[b]){tmp=st[a];st[a]=st[b];st[b]=tmp}
+        for (i=1;i<=n;i++) { t=st[i]; c=best[t,"ggml-cpu"]; h=best[t,"halide"]
+          printf "%-8s %9.1f ns %10.1f ns %7.2fx  %s\n", t, c, h, (h>0?c/h:0), (ok[t,"halide"]==1?"yes":"NO") } }'
+rm -rf $TMP
+```
+
+To read generated code, re-run the generator by hand with extra outputs. Grab
+the exact command from `build.ninja` (`grep 'COMMAND = .*-n q4_1_vec_dot '`) and
+swap `-e c_header,object` for `-e stmt,assembly`. Add
+`-no_asserts-no_bounds_query` to the target to see what actually ships.
+
+Diagnose SDOT vs fallback by grepping the `.s` for `sdot.4s`; grep the `.stmt`
+for `vector_reduce_add(int32x..(widening_mul(int8x.., int8x..)))`.
+
+## What the q4_0/q8_0 speedup actually was
+
+Four independent things, roughly equal in size:
+
+1. **Lanes from `r.x`, not `r.y`.** `rfactor({{rxo, lane}, {r.y, u}})` keeps the
+   sdot's four Int(32) lanes alive into the float accumulator, so no block pays
+   a horizontal reduce. Lanes must come from `r.x`: blocks are interleaved
+   `{scale, codes}` records, so a lane per block gathers both the codes and the
+   scales.
+2. **Chained sdot.** Cut `r.x` into chunks of 16 run serially, so both sdots
+   accumulate into the *same* register. Reducing straight to 4 lanes makes
+   `CodeGen_ARM` lower the wide reduce as two independent sdots plus an `addp`
+   (`codegen_dot_product_vector_reduce` only matches factor 4 and recurses).
+3. **Interleave 4 blocks into independent accumulators.** Widening the vector
+   does not help -- every lane of one accumulator advances on every block, so
+   only interleaving blocks shortens the multiply-add chain. Un-interleaved, the
+   kernel is latency-bound at ~4 cycles/block.
+4. **Per-call overhead.** vec_dot is called once per output element of a matvec,
+   so nothing amortizes. Three `Halide::Runtime::Buffer` constructions cost ~13
+   ns flat; the assert/bounds-query prologue another ~10 ns. Fixed by filling a
+   `halide_buffer_t` in place (`StackBuffer` in `ggml_quants.cpp`) and building
+   these libraries with `FEATURES no_asserts no_bounds_query`.
+
+Two traps found along the way:
+
+- **A predicated tail is not a local cost.** Splitting the block RVar with
+  `GuardWithIf` makes the per-block sdot a dynamic-extent allocation that Halide
+  has to `bzero` and accumulate *through memory*, roughly doubling the cost of
+  every block. Fixed by giving the main reduction an exactly divisible extent
+  (`(nb / kUnrollBlocks) * kUnrollBlocks`) and sweeping the remainder in a
+  second update at the default schedule. `kUnrollBlocks` must be a power of two
+  -- 3 and 6 measured 40% worse because the simplifier cannot discharge the
+  tail.
+- **`specialize()` inherits the schedule as of the call**, so scheduling
+  directives applied *after* `specialize()` do not reach the specialized branch.
+  It silently dropped the vectorize/unroll and made things 4x slower.
+
+## The q4_1 story so far
+
+q4_1 is affine: the weight decodes to `d*code + m`, so the per-block product
+`(d*code + m) * (d_act*act)` has no single scale to hoist and it was left on the
+default (fully scalar, unscheduled) float reduction at 3843 ns.
+
+`Stage::distribute()` multiplies the product out to
+`d*d_act * sum(code*act) + m*d_act * sum(act)`, and `hoist_invariants()` gives
+each term its own accumulator. Both bodies are integer, so both reach SDOT --
+the first as the ordinary dot, the second as a dot with a vector of ones (ARM
+already matches `i32(int8x)`). That is ggml's own decomposition, and it got q4_1
+to 133.8 ns / 0.80x.
+
+**Design decisions worth not re-litigating:**
+
+- Multiplying out is `distribute()`, a separate schedule directive, *not*
+  something `hoist_invariants()` decides. A first attempt did it unconditionally
+  and broke `hoist_invariants test (predicated RDom)`: `require(...) * (r + 1)`
+  distributes into two accumulators when one was optimal. The predicate that
+  would have rescued it ("ignore constants, don't traverse call arguments...")
+  is exactly the kind of heuristic that needs tuning forever. Whether to
+  multiply out depends on what the terms turn out to contain, which is a cost
+  question, so it belongs in the schedule.
+- `hoist_invariants()` returns **one single-valued Func per accumulator**, not a
+  Tuple. The Tuple version worked and measured identically, but it blocked
+  `compute_offline` (which cannot sever one value of a Tuple) and forced
+  `change_type` to grow multi-output support. Fusing the terms' loop nests is
+  `compute_with`'s job.
+
+## Next step: the stored block sum
+
+ggml does *not* recompute `sum(act)` at vec_dot time -- it reads the `s` field
+that `block_q8_1` stores at quantize time
+(`{ggml_half d; ggml_half s; int8_t qs[32]}`, 36 bytes). Our Q8_1 codec already
+computes that field (`AppendSums{block_size, SumMode::ScaledFloat}` in
+`make_symmetric_byte_sum_block_scheme`).
+
+**This needs no new Halide directive.** `compute_offline`'s contract already
+*is* this claim: it severs a Func's computation, replaces calls with an
+ImageParam read, and hands back an `offline` Pipeline that computes exactly
+those values -- which for Q8_1 is the quantizer that writes `s`. Same claim
+already made for the packed codes. Proven end to end in a standalone probe:
+
+```
+accumulators: 2
+inner accumulator type: int32
+reference = -1535.121338
+severed   = -1535.112305  (rel err 5.88e-06)
+```
+
+The recipe that works:
+
+1. Leave the **activation decode un-inlined** through `distribute()` and the
+   first hoist, so the offset term's accumulator body is exactly `Act(r.x, u)`
+   and the accumulator *is* `sum_k decode_act(k, blk)`. (`sdot_partial()`
+   currently eager-inlines both operands' whole decode chains; for this path it
+   must inline the weight's only.)
+2. `parts[1].change_type(Float(16))` -- the stored field is fp16, and this is
+   what makes the severed Func's type match the data. Faithful: the encoder
+   rounds to fp16 too. The 5.9e-06 error above is that rounding, same as ggml's.
+3. `Pipeline({Acc}).compute_offline({sum16}, {stored_param})`.
+4. `parts[0].update().eager_inline({Act})` then hoist again to pull the
+   activation scale out, then `change_type(Int(32))` -- the survivor still
+   reaches SDOT.
+
+**The structural conflict.** The two terms need different rfactors:
+
+| term            | preserved dims        | why                                                                                 |
+| --------------- | --------------------- | ----------------------------------------------------------------------------------- |
+| `sum(code*act)` | `{rxo->lane, r.y->u}` | the lane split is the q4_0/q8_0 win                                                 |
+| `sum(act)`      | `{r.y->u}` only       | must be the *whole-block* sum to equal `s`; lane-split gives four per-lane partials |
+
+One update definition can only be rfactored one way, and `distribute()` +
+`hoist_invariants()` produce two accumulator Funcs from a *single* update, so
+both inherit its rfactor.
+
+**Measured cost of each way out** (this is the important part -- an earlier
+estimate that this was net-zero was wrong):
+
+| structure             | q4_0 (1 accumulator) | q4_1 (2 accumulators) | cost of the 2nd |
+| --------------------- | -------------------- | --------------------- | --------------- |
+| lane-split (current)  | 97.3 ns              | 133.8 ns              | 36.5 ns         |
+| per-block (variant A) | 105.7 ns             | 169.9 ns              | 64.2 ns         |
+
+The per-block structure costs only **+8.4 ns** on the base (97.3 -> 105.7). Most
+of the apparent 30.8 ns q4_1 penalty is the second accumulator getting more
+expensive -- exactly what severing removes. ggml's own q4_1 - q4_0 delta is
+**12.6 ns**, which is what a well-scheduled stored-`s` offset term costs.
+
+So:
+
+| route              | projection                 | ratio  | machinery                            |
+| ------------------ | -------------------------- | ------ | ------------------------------------ |
+| today              | 133.8 ns                   | 0.80x  | --                                   |
+| variant A + sever  | ~105.7 + 12.6 = **118 ns** | ~0.90x | severing plumbing only               |
+| lane-split + sever | ~97.3 + 12.6 = **110 ns**  | ~0.97x | + distribute-into-update-definitions |
+
+`compute_with` **is** verified to fuse a lane-split reduction with a block-only
+one over the same block loop (probe: one block-group loop, correct result),
+using `LoopAlignStrategy::AlignStart` and *matching split-variable names* -- the
+loop variables must be named identically in both stages or it errors with
+"cannot find <var> in <stage>".
+
+**Recommended order:** build the severing plumbing on variant A first (it is
+shared by both routes), measure the real number, then decide whether the last ~8
+ns justifies teaching `distribute()` to split into separate update definitions
+so each term can be rfactored on its own.
+
+Plumbing still to write:
+
+- A third generator input: 1-D `Float(16)` ImageParam for the block sums.
+- The ABI wrapper passes a zero-copy view of the `s` field: `host = base + 2`,
+  1-D, extent `nb`, **stride 18 in fp16 units** (= 36 bytes). `StackBuffer` in
+  `ggml_quants.cpp` already builds `halide_buffer_t`s in place; add a
+  `blocks_field_f16` helper.
+- Where the "this activation stores its block sums" knowledge lives. It is
+  format knowledge, so it belongs with the codec --
+  `make_symmetric_byte_sum_block_scheme` is the Approximation construction site,
+  and `SchemeAndBytes`/`VecDotSpec` is the natural carrier. Do not put it in the
+  generator's schedule block.
+
+## Not started
+
+q5_0 (0.51x) and q5_1 (0.50x) are on the SDOT path but well behind; the 5-bit
+high bit comes from a separate 4-byte field and the reconstruction may not be
+folding into the codes leaf as cleanly as q4_0's nibble unpack. Everything else
+(k-quants, IQ family, tq\*) is still on the default unscheduled float reduction
+and would benefit from the same treatment as q4_1 -- the k-quants' two-level
+scales are also sums of scaled sub-reductions, which is what `distribute()` was
+built for.

From bdb0229bad6d6dd05f9f26926ab0bdb1cb2aca46 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 13:20:31 -0400
Subject: [PATCH 15/22] apps/ggml: sever q4_1/q5_1 offset term to Q8_1's stored
 block sum

The affine vec_dots decompose (d*code + m)*(d_act*act) into
d*d_act*sum(code*act) + m*d_act*sum(act). ggml does not recompute the
second sum -- it reads the `s` field block_q8_1 stores at quantize time.
Do the same: leave the activation decode un-inlined through distribute()
and the first hoist so the offset term's accumulator is exactly the
whole-block activation sum, change_type it to Float(16), and
compute_offline it to a new third Input bound to a zero-copy fp16 view of
the stored `s`. The product term re-inlines the activation's full decode
chain and re-hoists to recover the scale-free Int(32) SDOT.

q4_1 0.80x -> 0.97x (110.2 ns), q5_1 0.50x -> 0.64x. This matched the
lane-split+sever projection, so the harder route (per-term rfactors via
distribute-into-update-definitions) is unnecessary. All 28 roundtrips
pass, kernel-bench --all clean, odd tails correct.

Format knowledge lives with the codec: SchemeAndBytes::has_block_sums,
set by make_symmetric_byte_sum_block_scheme, carried to
VecDotSpec::act_has_block_sums (guarded a_nat == wbs). The
GGML_PER_BLOCK_PROBE variant-A branch is kept for the symmetric formats.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/PERF_NOTES.md                       | 201 +++++++++---------
 apps/ggml/halide/ggml_quants.cpp              |  25 ++-
 apps/ggml/halide/quant_components.h           |  11 +-
 .../halide/symmetric_vec_dot_generator.cpp    |  14 +-
 apps/ggml/halide/vec_dot_generator_base.h     | 106 ++++++++-
 5 files changed, 239 insertions(+), 118 deletions(-)

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
index ecefef2fccec..4b5f73f8d2e9 100644
--- a/apps/ggml/PERF_NOTES.md
+++ b/apps/ggml/PERF_NOTES.md
@@ -8,13 +8,13 @@ on ARM (measured on an M3 Max). Branch `alexreinking/ggml-on-qk`, worktree
 
 Measured at n=4096, best of many runs (see "Measuring" below):
 
-| type | ggml-cpu | halide   | ratio | at session start |
-| ---- | -------- | -------- | ----- | ---------------- |
-| q4_0 | 94.2 ns  | 97.3 ns  | 0.97x | 0.52x            |
-| q4_1 | 106.7 ns | 133.8 ns | 0.80x | 0.03x            |
-| q5_0 | 122.2 ns | 241.7 ns | 0.51x | 0.38x            |
-| q5_1 | 143.4 ns | 288.1 ns | 0.50x | 0.04x            |
-| q8_0 | 71.9 ns  | 75.6 ns  | 0.95x | 0.61x            |
+| type | ggml-cpu | halide   | ratio | before stored-s |
+| ---- | -------- | -------- | ----- | --------------- |
+| q4_0 | 87.8 ns  | 90.7 ns  | 0.97x | 0.97x           |
+| q4_1 | 106.7 ns | 110.2 ns | 0.97x | 0.80x           |
+| q5_0 | 115.6 ns | 225.3 ns | 0.51x | 0.51x           |
+| q5_1 | 133.5 ns | 207.8 ns | 0.64x | 0.50x           |
+| q8_0 | 69.1 ns  | 70.4 ns  | 0.98x | 0.95x           |
 
 Everything else in the table is still on the unscheduled float path
 (0.01x-0.12x) and untouched. All 28 roundtrip tests pass; `kernel-bench --all`
@@ -31,17 +31,30 @@ Commits, oldest first:
 
 ## Uncommitted
 
-`apps/ggml/halide/vec_dot_generator_base.h` has a probe branch guarded by
-`getenv("GGML_PER_BLOCK_PROBE")` -- "variant A", the per-block (single rfactor)
-schedule. It exists only to measure; delete it or keep it as the base for the
-next step (see below). The default path is unchanged and measures as above.
+The stored-block-sum work (see "The stored block sum" below) is implemented and
+uncommitted. Touched files:
+
+- `halide/vec_dot_generator_base.h` -- the `sever_sum` branch (default for
+  affine x Q8_1), the third `s_blocks` Input, its pinned fp16 stride.
+- `halide/quant_components.h` -- `SchemeAndBytes::has_block_sums`, set by
+  `make_symmetric_byte_sum_block_scheme`.
+- `halide/symmetric_vec_dot_generator.cpp` -- captures the flag into
+  `VecDotSpec::act_has_block_sums`.
+- `halide/ggml_quants.cpp` -- `StackBuffer::blocks_field_f16` + the q4_1/q5_1
+  wrappers passing the `s` view (q5_1 also moved off `Buffer` onto
+  `StackBuffer`).
+
+The `getenv("GGML_PER_BLOCK_PROBE")` branch (original "variant A" via
+`sdot_partial`) is kept, now reachable only for the *symmetric* SDOT formats
+(q4_0/q8_0/q5_0) -- a dev aid to measure per-block vs the lane-split default.
+The affine formats reach `sever_sum` first and never see it.
 
 Note the generator reads the env var at *generator run time*, so changing it
 does not invalidate ninja's outputs. Force regeneration:
 
 ```sh
-rm -f build/apps/ggml/halide/q4_1_vec_dot.o build/apps/ggml/halide/libq4_1_vec_dot.a
-GGML_PER_BLOCK_PROBE=1 cmake --build build/apps/ggml -j --target q4_1_vec_dot
+rm -f build/apps/ggml/halide/q4_0_vec_dot.o build/apps/ggml/halide/libq4_0_vec_dot.a
+GGML_PER_BLOCK_PROBE=1 cmake --build build/apps/ggml -j --target q4_0_vec_dot
 cmake --build build/apps/ggml -j
 ```
 
@@ -174,104 +187,80 @@ to 133.8 ns / 0.80x.
   `change_type` to grow multi-output support. Fusing the terms' loop nests is
   `compute_with`'s job.
 
-## Next step: the stored block sum
+## The stored block sum -- DONE (q4_1 0.97x)
 
 ggml does *not* recompute `sum(act)` at vec_dot time -- it reads the `s` field
 that `block_q8_1` stores at quantize time
 (`{ggml_half d; ggml_half s; int8_t qs[32]}`, 36 bytes). Our Q8_1 codec already
 computes that field (`AppendSums{block_size, SumMode::ScaledFloat}` in
-`make_symmetric_byte_sum_block_scheme`).
-
-**This needs no new Halide directive.** `compute_offline`'s contract already
-*is* this claim: it severs a Func's computation, replaces calls with an
-ImageParam read, and hands back an `offline` Pipeline that computes exactly
-those values -- which for Q8_1 is the quantizer that writes `s`. Same claim
-already made for the packed codes. Proven end to end in a standalone probe:
-
-```
-accumulators: 2
-inner accumulator type: int32
-reference = -1535.121338
-severed   = -1535.112305  (rel err 5.88e-06)
-```
-
-The recipe that works:
-
-1. Leave the **activation decode un-inlined** through `distribute()` and the
-   first hoist, so the offset term's accumulator body is exactly `Act(r.x, u)`
-   and the accumulator *is* `sum_k decode_act(k, blk)`. (`sdot_partial()`
-   currently eager-inlines both operands' whole decode chains; for this path it
-   must inline the weight's only.)
-2. `parts[1].change_type(Float(16))` -- the stored field is fp16, and this is
-   what makes the severed Func's type match the data. Faithful: the encoder
-   rounds to fp16 too. The 5.9e-06 error above is that rounding, same as ggml's.
-3. `Pipeline({Acc}).compute_offline({sum16}, {stored_param})`.
-4. `parts[0].update().eager_inline({Act})` then hoist again to pull the
-   activation scale out, then `change_type(Int(32))` -- the survivor still
-   reaches SDOT.
-
-**The structural conflict.** The two terms need different rfactors:
-
-| term            | preserved dims        | why                                                                                 |
-| --------------- | --------------------- | ----------------------------------------------------------------------------------- |
-| `sum(code*act)` | `{rxo->lane, r.y->u}` | the lane split is the q4_0/q8_0 win                                                 |
-| `sum(act)`      | `{r.y->u}` only       | must be the *whole-block* sum to equal `s`; lane-split gives four per-lane partials |
-
-One update definition can only be rfactored one way, and `distribute()` +
-`hoist_invariants()` produce two accumulator Funcs from a *single* update, so
-both inherit its rfactor.
-
-**Measured cost of each way out** (this is the important part -- an earlier
-estimate that this was net-zero was wrong):
-
-| structure             | q4_0 (1 accumulator) | q4_1 (2 accumulators) | cost of the 2nd |
-| --------------------- | -------------------- | --------------------- | --------------- |
-| lane-split (current)  | 97.3 ns              | 133.8 ns              | 36.5 ns         |
-| per-block (variant A) | 105.7 ns             | 169.9 ns              | 64.2 ns         |
-
-The per-block structure costs only **+8.4 ns** on the base (97.3 -> 105.7). Most
-of the apparent 30.8 ns q4_1 penalty is the second accumulator getting more
-expensive -- exactly what severing removes. ggml's own q4_1 - q4_0 delta is
-**12.6 ns**, which is what a well-scheduled stored-`s` offset term costs.
-
-So:
-
-| route              | projection                 | ratio  | machinery                            |
-| ------------------ | -------------------------- | ------ | ------------------------------------ |
-| today              | 133.8 ns                   | 0.80x  | --                                   |
-| variant A + sever  | ~105.7 + 12.6 = **118 ns** | ~0.90x | severing plumbing only               |
-| lane-split + sever | ~97.3 + 12.6 = **110 ns**  | ~0.97x | + distribute-into-update-definitions |
-
-`compute_with` **is** verified to fuse a lane-split reduction with a block-only
-one over the same block loop (probe: one block-group loop, correct result),
-using `LoopAlignStrategy::AlignStart` and *matching split-variable names* -- the
-loop variables must be named identically in both stages or it errors with
-"cannot find <var> in <stage>".
-
-**Recommended order:** build the severing plumbing on variant A first (it is
-shared by both routes), measure the real number, then decide whether the last ~8
-ns justifies teaching `distribute()` to split into separate update definitions
-so each term can be rfactored on its own.
-
-Plumbing still to write:
-
-- A third generator input: 1-D `Float(16)` ImageParam for the block sums.
-- The ABI wrapper passes a zero-copy view of the `s` field: `host = base + 2`,
-  1-D, extent `nb`, **stride 18 in fp16 units** (= 36 bytes). `StackBuffer` in
-  `ggml_quants.cpp` already builds `halide_buffer_t`s in place; add a
-  `blocks_field_f16` helper.
-- Where the "this activation stores its block sums" knowledge lives. It is
-  format knowledge, so it belongs with the codec --
-  `make_symmetric_byte_sum_block_scheme` is the Approximation construction site,
-  and `SchemeAndBytes`/`VecDotSpec` is the natural carrier. Do not put it in the
-  generator's schedule block.
+`make_symmetric_byte_sum_block_scheme`). We now sever the offset term's
+accumulator straight to it via `compute_offline`, which needs no new Halide
+directive: its contract already *is* this claim -- sever a Func's computation,
+replace calls with an ImageParam read, discard the recomputing reduction.
+
+**Result: variant A + sever measured 110.2 ns / 0.97x** (q5_1: 208 ns / 0.64x).
+This *matched* the lane-split+sever projection, so the second route below
+(teaching `distribute()` to split into separate update definitions for per-term
+rfactors) is **not needed** -- skip it. All correct: 28 roundtrips,
+`kernel-bench --all` clean, odd tails at n = 32/96/160/224/1056.
+
+The recipe, as implemented in `vec_dot_generator_base.h`'s `sever_sum` branch
+(reached when `distribute_terms && act_has_block_sums`, i.e. affine x Q8_1):
+
+1. `acc_dot = Acc.update().rfactor({{r.y, u}})` -- whole-block partials (variant
+   A). Inline **only the weight's** decode chain (replacement + inlinable
+   handles, multi-pass), leaving the activation decode `Act`
+   (`act_r.replacement`) whole. `distribute()`, then `hoist_invariants()`. The
+   offset term's accumulator body is then `Act(r.x, u)`, so the accumulator *is*
+   `sum_k decode_act(k, blk)` = the stored `s`.
+2. `parts[1].change_type(Float(16))` -- makes the severed Func's type match the
+   data. Faithful: the encoder rounds `s` to fp16 too (the ~6e-06 rel err is
+   exactly that rounding, same as ggml's).
+3. `Pipeline({Acc}).compute_offline({s16}, {s_blocks})` -- second
+   `compute_offline` on the pipeline (the first, at configure top, severs the
+   encode halves to x_blocks/y_blocks). `s_blocks` is the third Input.
+4. Product term: inline `Act`'s **full** chain into `parts[0]` (one Act
+   eager_inline is not enough -- the chain has intermediate levels, and a single
+   inline leaves the second hoist with no visible d_act factor and it errors),
+   re-`hoist_invariants()` to pull d_act out, `change_type(Int(32))` -- the
+   survivor reaches SDOT. Verified: 8 `sdot` in the `.s`; writeback is
+   `d_w*(d_act*int32_dot) + m_w*s_blocks[blk]`, ggml's exact decomposition, with
+   no `sum(act)` reduction left anywhere in the stmt.
+
+Plumbing (all landed, see Uncommitted):
+
+- `s_blocks`: 1-D `Float(16)` Input, `dim(0).set_stride(act_bytes/2)` (= 18 for
+  Q8_1) -- pinning it makes the read an immediate offset, *and* is required:
+  left dynamic, Halide's default constrains the innermost stride to 1 and the
+  bound-check fails against the strided view.
+- ABI: `StackBuffer::blocks_field_f16(base, nb, byte_offset=2, block_bytes=36)`
+  -- a zero-copy fp16 view of the `s` slot, stride `block_bytes/2`.
+- Format knowledge lives with the codec: `SchemeAndBytes::has_block_sums` set by
+  `make_symmetric_byte_sum_block_scheme`, carried to
+  `VecDotSpec::act_has_block_sums` (guarded `&& a_nat == wbs`, so a Reblock'd
+  activation stays off it).
+
+**The structural conflict (why variant A, not lane-split).** The two terms want
+different rfactors: `sum(code*act)` wants `{rxo->lane, r.y->u}` (the lane split
+is the q4_0/q8_0 win); `sum(act)` must be the *whole-block* sum to equal `s`, so
+`{r.y->u}` only. One update rfactors one way. Variant A gives both `{r.y->u}`;
+severing then deletes the offset accumulator entirely, so its per-lane-partial
+problem never arises -- and the surviving product dot, alone in its block loop,
+schedules close enough to the lane-split base that the ~8 ns gap projected
+between the routes did not materialize. `compute_with` (verified to fuse a
+lane-split with a block-only reduction using `AlignStart` + matching split-var
+names) is therefore unnecessary here; keep it in mind for formats that keep two
+live accumulators.
 
 ## Not started
 
-q5_0 (0.51x) and q5_1 (0.50x) are on the SDOT path but well behind; the 5-bit
-high bit comes from a separate 4-byte field and the reconstruction may not be
-folding into the codes leaf as cleanly as q4_0's nibble unpack. Everything else
-(k-quants, IQ family, tq\*) is still on the default unscheduled float reduction
-and would benefit from the same treatment as q4_1 -- the k-quants' two-level
-scales are also sums of scaled sub-reductions, which is what `distribute()` was
-built for.
+q5_0 (0.51x) and q5_1 (0.64x) are on the SDOT path but well behind. q5_1 got the
+stored-`s` sever above (0.50x -> 0.64x), so its *offset* term is now free --
+what remains is the **5-bit product term**, which it shares with q5_0 (unmoved
+at 0.51x). The 5-bit high bit comes from a separate 4-byte field and the
+reconstruction may not be folding into the codes leaf as cleanly as q4_0's
+nibble unpack; that leaf, not the offset, is the q5_x bottleneck now. Everything
+else (k-quants, IQ family, tq\*) is still on the default unscheduled float
+reduction and would benefit from the same treatment as q4_1 -- the k-quants'
+two-level scales are also sums of scaled sub-reductions, which is what
+`distribute()` was built for.
diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
index c2adb7e0c040..0f3d1249b308 100644
--- a/apps/ggml/halide/ggml_quants.cpp
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -164,6 +164,16 @@ struct StackBuffer {
         return init(data, 2);
     }
 
+    // A gathered 1-D fp16 view of one field within each packed block -- e.g.
+    // Q8_1's stored `s` (scaled code sum) at byte_offset 2 of its 36-byte block.
+    // Zero-copy: the field repeats every block_bytes, so the fp16 stride is
+    // block_bytes/2. Used to sever a stored per-block quantity into a vec_dot.
+    halide_buffer_t *blocks_field_f16(const void *base, int nb, int byte_offset, int block_bytes) {
+        buf.type = halide_type_t(halide_type_float, 16);
+        dims[0] = {0, nb, block_bytes / 2, 0};  // stride in fp16 units
+        return init(static_cast<const uint8_t *>(base) + byte_offset, 1);
+    }
+
     // The 0-D float32 result.
     halide_buffer_t *scalar_f32(float *data) {
         buf.type = halide_type_t(halide_type_float, 32);
@@ -240,9 +250,10 @@ void ggml_quants_halide_vec_dot_q4_1_q8_1(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytesX = 4 + kQK / 2, kBlockBytesY = 4 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    StackBuffer xb, yb, result;
+    StackBuffer xb, yb, sb, result;
     check(q4_1_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
                        yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       sb.blocks_field_f16(vy, nb, 2, kBlockBytesY),  // Q8_1 stored `s`
                        result.scalar_f32(s)),
           "q4_1_vec_dot");
 }
@@ -307,12 +318,12 @@ void ggml_quants_halide_vec_dot_q5_1_q8_1(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytesX = 4 + 4 + kQK / 2, kBlockBytesY = 4 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
-    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
-    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
-    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
-    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
-    check(q5_1_vec_dot(xb, yb, result), "q5_1_vec_dot");
+    StackBuffer xb, yb, sb, result;
+    check(q5_1_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       sb.blocks_field_f16(vy, nb, 2, kBlockBytesY),  // Q8_1 stored `s`
+                       result.scalar_f32(s)),
+          "q5_1_vec_dot");
 }
 
 //
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index ee153a934d2a..18a664f1dd5f 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -90,6 +90,13 @@ struct SchemeAndBytes {
     // block_type.bytes() -- the single source of truth for the on-disk width.
     // Default-constructed (invalid) for the byte-buffer schemes not yet ported.
     Halide::Type block_type;
+    // Set when the scheme appends a per-block scaled sum of its codes (Q8_1's
+    // `s` field -- AppendSums{SumMode::ScaledFloat}). A vec_dot pairing an
+    // affine weight against such an activation can sever the offset term's
+    // sum(act) accumulator straight to this stored field instead of recomputing
+    // it -- see VecDotGeneratorBase::configure(). Purely a byte-path,
+    // 32-element-block property today (Q8_1); wider layouts leave it false.
+    bool has_block_sums = false;
 };
 
 // Every make_*_scheme() factory below takes a Layout, selecting what its
@@ -2751,7 +2758,9 @@ inline SchemeAndBytes make_symmetric_byte_sum_block_scheme(int block_size, int q
                 AppendSums{block_size, SumMode::ScaledFloat},
                 SymmetricAffineQuantize{block_size, qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax},
                 BlockReshape{block_size, layout == Layout::BlockIndexed}),
-            bl.bytes};
+            bl.bytes,
+            /*block_type=*/Halide::Type{},
+            /*has_block_sums=*/true};
 }
 
 // Q8_K: activation-only (quantize_row only, matching Q8_1's own situation
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index 29090385762b..a6d3a43faf4e 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -137,22 +137,30 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         const int a_nat = 32;
         std::unique_ptr<Halide::Approximation> ac;
         int ab;
+        bool act_has_block_sums = false;
         switch (a_kind.value()) {
         case AKind::Q8_0:
             ac = make_symmetric_block_scheme(a_nat, a_qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
             ab = 2 + a_nat;
             break;
-        case AKind::Q8_1:
-            ac = make_symmetric_byte_sum_block_scheme(a_nat, a_qmax, Layout::BlockIndexed).scheme;
+        case AKind::Q8_1: {
+            SchemeAndBytes sb = make_symmetric_byte_sum_block_scheme(a_nat, a_qmax, Layout::BlockIndexed);
+            ac = std::move(sb.scheme);
             ab = 2 + 2 + a_nat;
+            // Q8_1 stores its per-block scaled code sum (`s`); the affine vec_dot
+            // severs the offset term straight to it. Only meaningful when the
+            // activation block matches the weight block (a_nat == wbs, i.e. no
+            // Reblock) -- true for the q4_1/q5_1 pairings.
+            act_has_block_sums = sb.has_block_sums && a_nat == wbs;
             break;
         }
+        }
         ac = reblock_activation(std::move(ac), a_nat, wbs);
 
         // Activation stays on the byte path for now (Q8_0/Q8_1 are shared across
         // many weight formats, and the Reblock relayout is byte-based); only the
         // weight operand is struct-typed here.
-        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}};
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}, act_has_block_sums};
     }
 };
 
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index cebde7cd8c30..e775ad72d60f 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -65,6 +65,12 @@ struct VecDotSpec {
     // activation) just leaves its type unset.
     Halide::Type weight_type;
     Halide::Type act_type;
+    // Set when the activation stores a per-block scaled sum of its codes (Q8_1's
+    // `s` field). Together with distribute_terms (an affine weight), this lets
+    // configure() sever the offset term's sum(act) accumulator to a stored fp16
+    // field supplied as a third Input, instead of recomputing it -- ggml's own
+    // q4_1/q5_1 optimization. See SchemeAndBytes::has_block_sums.
+    bool act_has_block_sums = false;
 };
 
 template<typename Derived>
@@ -87,6 +93,15 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         ImageParam x_blocks = wt_struct ? ImageParam(spec.weight_type, 1, "x_blocks") : ImageParam(UInt(8), 2, "x_blocks");
         ImageParam y_blocks = act_struct ? ImageParam(spec.act_type, 1, "y_blocks") : ImageParam(UInt(8), 2, "y_blocks");
 
+        // Third Input, present only for the affine-x-Q8_1 pairings: the
+        // activation's stored per-block scaled code sum (`s`). configure() severs
+        // the offset term's sum(act) accumulator to this field rather than
+        // recomputing it -- a zero-copy 1-D fp16 view of the `s` slot the ABI
+        // wrapper passes (stride = block width). See the SDOT sever branch below.
+        const bool sever_sum = spec.sched == ScheduleKind::SDOT &&
+                               spec.distribute_terms && spec.act_has_block_sums;
+        ImageParam s_blocks = sever_sum ? ImageParam(Float(16), 1, "s_blocks") : ImageParam();
+
         // The packed-block buffers are quantized GGML rows: their base pointers
         // are cache-line aligned. Without this Halide assumes 1-byte alignment
         // and lowers every strided / reinterpreted read (the interleaved fp16
@@ -151,7 +166,86 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             }
         }
 
-        if (spec.sched == ScheduleKind::SDOT) {
+        if (sever_sum) {
+            // Affine weight x Q8_1: the per-block product (d*code + m)*(d_act*act)
+            // distributes into d*d_act*sum(code*act) + m*d_act*sum(act). ggml does
+            // not recompute the second sum -- it reads the `s` field Q8_1 stores.
+            // We do the same by severing that term's accumulator to the stored
+            // field (the third Input, s_blocks), leaving only the Int(32) dot.
+            //
+            // rfactor to whole-block partials (variant A), inlining only the
+            // WEIGHT's decode chain so the activation decode (Act) stays whole:
+            // the offset term's accumulator is then sum_k Act(k, blk), which *is*
+            // the stored `s`. The product term re-inlines Act's full chain and
+            // re-hoists to recover the scale-free Int(32) dot.
+            Func acc_dot = Acc.update().rfactor({{r.y, u}});
+
+            std::vector<Func> winl = {wt_r.replacement};
+            for (const Func &h : wt_r.handles) {
+                if (h.function().can_be_inlined()) {
+                    winl.push_back(h);
+                }
+            }
+            for (size_t pass = 0; pass < winl.size(); pass++) {
+                acc_dot.update().eager_inline(winl);
+            }
+
+            acc_dot.update().distribute();
+            std::vector<Func> parts = acc_dot.update().hoist_invariants();
+            // parts[0] = product term (scale * sum code_w*Act); parts[1] = offset
+            // term (min * sum Act == stored s).
+
+            // Sever the offset term to the stored fp16 field. change_type(Float16)
+            // makes the severed accumulator's type match the data (the encoder
+            // rounds `s` to fp16 too, so this reproduces ggml's own rounding);
+            // compute_offline then replaces every call to it with a read of
+            // s_blocks and discards the recomputing reduction.
+            Func s16 = parts[1].change_type(Float(16));
+            Pipeline({Acc}).compute_offline({s16}, {s_blocks});
+
+            // Product term: flatten the activation's full decode chain and
+            // re-hoist to pull d_act out, leaving the scale-free Int(32) dot.
+            std::vector<Func> ainl = {act_r.replacement};
+            for (const Func &h : act_r.handles) {
+                if (h.function().can_be_inlined()) {
+                    ainl.push_back(h);
+                }
+            }
+            for (size_t pass = 0; pass < ainl.size(); pass++) {
+                parts[0].update().eager_inline(ainl);
+            }
+            Func prod_i32 = parts[0].update().hoist_invariants()[0].change_type(Int(32));
+
+            RVar ryo("ryo"), ryi("ryi");
+            Var bacc("bacc");
+            Acc.update(0).split(r.y, ryo, ryi, kUnrollBlocks);
+            Func acc_vec = Acc.update(0).rfactor(ryi, bacc);
+            acc_vec.compute_root().unroll(bacc);
+            acc_vec.update().unroll(bacc);
+
+            // Only the product dot is computed per block now; the offset term is
+            // a severed read of s_blocks (nothing to schedule).
+            prod_i32.compute_at(acc_vec, bacc).update().atomic().vectorize(r.x, bs);
+            Acc.update(1).unscheduled();
+        } else if (spec.sched == ScheduleKind::SDOT && getenv("GGML_PER_BLOCK_PROBE")) {
+            // PROBE (variant A): rfactor only the block index, so both terms are
+            // whole-block reductions. Costs a horizontal reduce per block and a
+            // scalar cross-block accumulator; kept for measuring the non-sum
+            // formats against the lane-split default below.
+            std::vector<Func> parts = sdot_partial(Acc, {{r.y, u}}, {wt_r, act_r}, spec.distribute_terms);
+
+            RVar ryo("ryo"), ryi("ryi");
+            Var bacc("bacc");
+            Acc.update(0).split(r.y, ryo, ryi, kUnrollBlocks);
+            Func acc_vec = Acc.update(0).rfactor(ryi, bacc);
+            acc_vec.compute_root().unroll(bacc);
+            acc_vec.update().unroll(bacc);
+
+            for (Func &part : parts) {
+                part.compute_at(acc_vec, bacc).update().atomic().vectorize(r.x, bs);
+            }
+            Acc.update(1).unscheduled();
+        } else if (spec.sched == ScheduleKind::SDOT) {
             // The reduction is over (within-block r.x) x (block r.y). The lanes
             // of the accumulator come from r.x, so the sdot's four Int(32) lanes
             // survive all the way into the float accumulator and no block pays
@@ -243,9 +337,19 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             y_blocks.dim(0).set_bounds(0, spec.act_bytes);
             y_blocks.dim(1).set_min(0).set_stride(spec.act_bytes);
         }
+        if (sever_sum) {
+            // A gathered view of the `s` slot within each packed block: the field
+            // repeats every act_bytes, so the fp16 stride is act_bytes/2 (18 for
+            // Q8_1's 36-byte block). Pinning it makes the per-block read a
+            // compile-time immediate offset rather than a dynamic pointer chain.
+            s_blocks.dim(0).set_min(0).set_stride(spec.act_bytes / 2);
+        }
 
         this->add_input(x_blocks);
         this->add_input(y_blocks);
+        if (sever_sum) {
+            this->add_input(s_blocks);
+        }
         this->add_output(result);
     }
 

From 4b01ba2ab63001d477a25d2ea173a63c1115babe Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 16:40:20 -0400
Subject: [PATCH 16/22] apps/ggml: expand q5_x qh high bit via a compile-time
 LUT (0.51->0.63x)

The q5_0/q5_1 gap over q4_x is the per-element high bit unpacked from the
qh field. PlanarBitPack's 1-bit decode emitted (qh[kk/8] >> (kk%8)) & 1,
which lowers to a per-lane variable shift plus a transpose_vector to
broadcast the qh bytes across the sdot lanes (~24 NEON ops/block). ggml
avoids this with a byte->8-byte memory LUT (table_b2b).

Mirror it with a compile-time Buffer<uint8_t> b2b(bit, byte) embedded in
the binary. The table read is a contiguous 8-byte load only when the qh
byte is a scalar and the 8 bit positions are the vector lanes, so
materialize the reconstructed codes per block (combine_bits_code
compute_at the block loop, kk split (byte, pos): pos vectorizes the load,
byte unrolls to a scalar index). Inlined into the sdot the byte varies
per lane -> a 16-wide gather (0.12x), so materialization is required.

Two enablers: sdot_partial() gained a keep_out list to hold the codes
leaf out of its deep inline (can_be_inlined() ignores compute level, so a
schedule alone doesn't stop eager_inline); and the odd-block tail gets its
own decode chain (a second Wt/Vec + approximate_by bound to the same
inputs) so the main reduction's compute_at can fuse without the tail --
which also stops Halide hoisting the codes buffer to whole-row.

q5_0 0.51->0.63x, q5_1 0.64->0.68x; q5_K on the float path 8615->6730 ns
from the shared decode. Transpose is gone (no uzp2/dup.8b). All 28
roundtrips pass, kernel-bench --all clean, odd tails correct. The residual
gap to 0.95x is the per-block codes store/reload, which needs a
CodeGen_ARM change or ggml's hand-scheduled load balance -- see PERF_NOTES.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/PERF_NOTES.md                   | 91 +++++++++++++++--------
 apps/ggml/halide/quant_components.h       | 25 ++++++-
 apps/ggml/halide/sdot_schedule.h          | 18 ++++-
 apps/ggml/halide/vec_dot_generator_base.h | 76 ++++++++++++++++++-
 4 files changed, 174 insertions(+), 36 deletions(-)

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
index 4b5f73f8d2e9..f05b76e5d2aa 100644
--- a/apps/ggml/PERF_NOTES.md
+++ b/apps/ggml/PERF_NOTES.md
@@ -8,13 +8,16 @@ on ARM (measured on an M3 Max). Branch `alexreinking/ggml-on-qk`, worktree
 
 Measured at n=4096, best of many runs (see "Measuring" below):
 
-| type | ggml-cpu | halide   | ratio | before stored-s |
-| ---- | -------- | -------- | ----- | --------------- |
-| q4_0 | 87.8 ns  | 90.7 ns  | 0.97x | 0.97x           |
-| q4_1 | 106.7 ns | 110.2 ns | 0.97x | 0.80x           |
-| q5_0 | 115.6 ns | 225.3 ns | 0.51x | 0.51x           |
-| q5_1 | 133.5 ns | 207.8 ns | 0.64x | 0.50x           |
-| q8_0 | 69.1 ns  | 70.4 ns  | 0.98x | 0.95x           |
+| type | ggml-cpu | halide   | ratio | before this work |
+| ---- | -------- | -------- | ----- | ---------------- |
+| q4_0 | 87.8 ns  | 90.7 ns  | 0.97x | 0.97x            |
+| q4_1 | 106.7 ns | 110.2 ns | 0.97x | 0.80x            |
+| q5_0 | 115.6 ns | 183.0 ns | 0.63x | 0.51x            |
+| q5_1 | 133.5 ns | 195.8 ns | 0.68x | 0.50x            |
+| q8_0 | 69.1 ns  | 70.4 ns  | 0.98x | 0.95x            |
+
+q5_K (still on the float path) also improved 8615 -> 6730 ns from the same
+qh-expansion table (it shares the 1-bit `PlanarBitPack` decode).
 
 Everything else in the table is still on the unscheduled float path
 (0.01x-0.12x) and untouched. All 28 roundtrip tests pass; `kernel-bench --all`
@@ -29,20 +32,7 @@ Commits, oldest first:
 - `6c72fb4c1` apps/ggml: take the affine vec_dots (q4_1, q5_1) to SDOT
 - `a6e9b0962` `hoist_invariants()`: return one Func per accumulator, not a Tuple
 
-## Uncommitted
-
-The stored-block-sum work (see "The stored block sum" below) is implemented and
-uncommitted. Touched files:
-
-- `halide/vec_dot_generator_base.h` -- the `sever_sum` branch (default for
-  affine x Q8_1), the third `s_blocks` Input, its pinned fp16 stride.
-- `halide/quant_components.h` -- `SchemeAndBytes::has_block_sums`, set by
-  `make_symmetric_byte_sum_block_scheme`.
-- `halide/symmetric_vec_dot_generator.cpp` -- captures the flag into
-  `VecDotSpec::act_has_block_sums`.
-- `halide/ggml_quants.cpp` -- `StackBuffer::blocks_field_f16` + the q4_1/q5_1
-  wrappers passing the `s` view (q5_1 also moved off `Buffer` onto
-  `StackBuffer`).
+## Dev aids
 
 The `getenv("GGML_PER_BLOCK_PROBE")` branch (original "variant A" via
 `sdot_partial`) is kept, now reachable only for the *symmetric* SDOT formats
@@ -252,15 +242,56 @@ lane-split with a block-only reduction using `AlignStart` + matching split-var
 names) is therefore unnecessary here; keep it in mind for formats that keep two
 live accumulators.
 
+## The q5_x 5-bit high bit -- partway (0.51 -> 0.63x, 0.50 -> 0.68x)
+
+q5_0/q5_1 reach SDOT, but every code carries a per-element high bit unpacked
+from the `qh` field, and that reconstruction -- not the dot, not (for q5_1) the
+offset -- is the whole gap vs q4_x.
+
+**What the reconstruction was.** `PlanarBitPack::decode`'s 1-bit case emitted
+`(qh[kk/8] >> (kk%8)) & 1`: a per-lane variable shift plus a `transpose_vector`
+to broadcast the two `qh` bytes across the 16 sdot lanes
+(`dup.8b + dup.4h + uzp2` per sdot, ~24 NEON ops/block). ggml avoids this with a
+1 KB `table_b2b` memory LUT (byte -> 8 expanded bytes, one contiguous load per
+`qh` byte).
+
+**What was done.** Mirrored the LUT: a compile-time `Buffer<uint8_t>` b2b(bit,
+byte) embedded in the binary. Two things make it pay:
+
+1. The table read is only a *contiguous* 8-byte load
+   (`b0[ramp(qh_byte*8, 1, 8)]`, matching ggml) when the `qh` byte is a
+   **scalar** and the 8 bit positions are the vector lanes. Inlined into the
+   sdot it is the opposite (the byte varies per lane -> a 16-wide per-lane
+   gather, `ld1` per lane, measured 0.12x). So the reconstructed codes are
+   **materialized** per block (`combine_bits_code` compute_at the block loop,
+   `kk` split `(byte, pos)`: pos vectorizes the load, byte unrolls to a scalar
+   index).
+2. Materializing needs the codes leaf kept out of `sdot_partial()`'s deep inline
+   (`can_be_inlined()` ignores compute level, so a `compute_root`/`compute_at`
+   schedule alone does not stop `eager_inline` -- a `keep_out` name list does).
+3. The odd-block **tail** reads the same codes but is a separate update
+   `compute_at` cannot reach. Fixed by giving the tail its **own** decode chain:
+   a second `Wt/Vec` placeholder pair, its own `approximate_by`, both bound to
+   the same `x_blocks/y_blocks`. Main materializes; the tail reconstructs inline
+   (< kUnrollBlocks blocks, negligible). This is also what unlocks the per-block
+   fusion -- with a shared chain Halide hoists the codes buffer to whole-row.
+
+The transpose is gone (verified: no `uzp2`/`dup.8b`); reconstruction is a
+handful of ops + 4 contiguous LUT loads/block. Bonus: the same decode change
+sped up q5_K on the float path (8615 -> 6730 ns).
+
+**Ceiling.** This lands q5_0 at 0.63x, q5_1 at 0.68x -- real, but short of the
+0.95x the symmetric formats hit. The residual is the materialization
+**store/reload**: the codes must round-trip a per-block stack buffer (the sdot's
+lane layout can't consume them in registers without bringing the per-lane gather
+back). Group-level compute (all 4 blocks first) measured *worse* (0.50x), not
+better. Closing the last ~0.3x looks like it needs either a Halide CodeGen_ARM
+improvement (feed materialized codes to the sdot without the L1 round-trip) or
+ggml's exact hand-scheduled load/compute balance; the LUT approach caps here.
+
 ## Not started
 
-q5_0 (0.51x) and q5_1 (0.64x) are on the SDOT path but well behind. q5_1 got the
-stored-`s` sever above (0.50x -> 0.64x), so its *offset* term is now free --
-what remains is the **5-bit product term**, which it shares with q5_0 (unmoved
-at 0.51x). The 5-bit high bit comes from a separate 4-byte field and the
-reconstruction may not be folding into the codes leaf as cleanly as q4_0's
-nibble unpack; that leaf, not the offset, is the q5_x bottleneck now. Everything
-else (k-quants, IQ family, tq\*) is still on the default unscheduled float
-reduction and would benefit from the same treatment as q4_1 -- the k-quants'
-two-level scales are also sums of scaled sub-reductions, which is what
+Everything else (k-quants, IQ family, tq\*) is still on the default unscheduled
+float reduction and would benefit from the same treatment as q4_1 -- the
+k-quants' two-level scales are also sums of scaled sub-reductions, which is what
 `distribute()` was built for.
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index 18a664f1dd5f..be09af4bc3e6 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -859,7 +859,30 @@ class PlanarBitPack : public Halide::Approximation {
         Expr plane = rem / pos_count_;
         Expr pos = rem % pos_count_;
         Expr byte_idx = outer * pos_count_ + pos;
-        Expr field = (cast<uint32_t>(bytes(byte_idx, blk, _)) >> (plane * field_bits_)) & ((1u << field_bits_) - 1);
+        Expr byte = cast<uint32_t>(bytes(byte_idx, blk, _));
+        Expr field;
+        if (field_bits_ == 1) {
+            // A single-bit field (Q5_0/Q5_1's per-element high bit). Read it from
+            // a compile-time byte->bits expansion table (ggml's table_b2b idea):
+            // b2b(bit, byte) = (byte >> bit) & 1. The alternative arithmetic form
+            // `(byte >> plane) & 1` lowers to a per-lane byte-broadcast transpose
+            // plus mask (~24 NEON ops per 16 lanes), which is the whole
+            // q5_x-vs-q4_x gap. A Buffer<> is embedded in the binary as constant
+            // data, so this is a pure lookup, no runtime input.
+            static const Halide::Buffer<uint8_t> b2b = [] {
+                Halide::Buffer<uint8_t> t(8, 256);
+                t.set_min(0, 0);
+                for (int by = 0; by < 256; by++) {
+                    for (int bit = 0; bit < 8; bit++) {
+                        t(bit, by) = (by >> bit) & 1;
+                    }
+                }
+                return t;
+            }();
+            field = cast<uint32_t>(b2b(cast<int32_t>(plane * field_bits_), cast<int32_t>(byte)));
+        } else {
+            field = (byte >> (plane * field_bits_)) & ((1u << field_bits_) - 1);
+        }
         Func codes("planar_bit_pack_codes");
         codes(kk, blk, _) = cast<int8_t>(cast<int32_t>(field) - qmax_);
         return {{codes}, {}};
diff --git a/apps/ggml/halide/sdot_schedule.h b/apps/ggml/halide/sdot_schedule.h
index 6612bf49289f..3d279de9a71b 100644
--- a/apps/ggml/halide/sdot_schedule.h
+++ b/apps/ggml/halide/sdot_schedule.h
@@ -21,6 +21,8 @@
 
 #include "Halide.h"
 
+#include <algorithm>
+#include <string>
 #include <vector>
 
 namespace ggml_halide {
@@ -37,21 +39,33 @@ namespace ggml_halide {
 // it is d*d_act * sum(code*act) + m*d_act * sum(act), and hoist_invariants()
 // gives each term its own accumulator -- both with integer bodies, so both reach
 // SDOT. That is ggml's own decomposition of the affine formats.
+// `keep_out` names decode-chain Funcs that must NOT be flattened -- a caller
+// that has scheduled one as a materialization boundary (e.g. Q5_x's
+// reconstructed `combine_bits_code`, computed once per block so its qh
+// byte->bits table read is a contiguous load instead of a per-lane gather).
+// can_be_inlined() only checks purity, so a compute_root schedule alone does not
+// stop eager_inline from flattening it; excluding it here does. The scale still
+// hoists past it, since it stays an opaque r.x-dependent factor of the product.
 inline std::vector<Halide::Func> sdot_partial(Halide::Func &acc,
                                               const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
                                               const std::vector<Halide::ApproximationResult> &operands,
-                                              bool distribute = false) {
+                                              bool distribute = false,
+                                              const std::vector<std::string> &keep_out = {}) {
     using namespace Halide;
 
     Func acc_dot = acc.update().rfactor(preserved);
 
+    auto excluded = [&](const Func &f) {
+        return std::find(keep_out.begin(), keep_out.end(), f.name()) != keep_out.end();
+    };
+
     std::vector<Func> decode_funcs;
     for (const ApproximationResult &op : operands) {
         decode_funcs.push_back(op.replacement);
     }
     for (const ApproximationResult &op : operands) {
         for (const Func &h : op.handles) {
-            if (h.function().can_be_inlined()) {
+            if (h.function().can_be_inlined() && !excluded(h)) {
                 decode_funcs.push_back(h);
             }
         }
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index e775ad72d60f..3f438378a4b9 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -135,9 +135,19 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         Func Acc("acc");
         Acc() = 0.0f;
         Acc() += Wt(r.x, r.y) * Vec(r.x, r.y);
+
+        // The odd-block tail decodes through its OWN placeholder Funcs, given a
+        // separate decode chain by a second approximate_by below. This lets the
+        // main reduction materialize a reconstructed-codes leaf (Q5_x) via
+        // compute_at while the tail -- a different loop nest that could not see
+        // that per-block buffer -- reconstructs inline, at negligible cost (fewer
+        // than kUnrollBlocks blocks). Both chains read the same x_blocks/y_blocks.
+        Func WtT("wt_naive_tail"), VecT("vec_naive_tail");
+        WtT(kk, blk) = 0.0f;
+        VecT(kk, blk) = 0.0f;
         RDom r_tail(0, bs, main_blocks, nblocks - main_blocks, "r_tail");
         if (sdot) {
-            Acc() += Wt(r_tail.x, r_tail.y) * Vec(r_tail.x, r_tail.y);
+            Acc() += WtT(r_tail.x, r_tail.y) * VecT(r_tail.x, r_tail.y);
         }
 
         ApproximationResult wt_r = Wt.approximate_by(*spec.weight_codec, {Acc});
@@ -150,8 +160,55 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         std::vector<Func> to_sever = wt_r.encoded;
         to_sever.insert(to_sever.end(), act_r.encoded.begin(), act_r.encoded.end());
         std::vector<ImageParam> bind_to = {x_blocks, y_blocks};
+        if (sdot) {
+            ApproximationResult wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
+            ApproximationResult actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
+            to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
+            to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
+            bind_to.push_back(x_blocks);
+            bind_to.push_back(y_blocks);
+            for (Func h : wtT_r.handles) {
+                if (h.has_update_definition()) {
+                    h.compute_root();
+                }
+            }
+            for (Func h : actT_r.handles) {
+                if (h.has_update_definition()) {
+                    h.compute_root();
+                }
+            }
+        }
         Pipeline({Acc}).compute_offline(to_sever, bind_to);
 
+        // Q5_0/Q5_1 reconstruct each code from a nibble plus a per-element high
+        // bit read from the qh field's byte->bits expansion table (see
+        // PlanarBitPack::decode). That table read is only a *contiguous* 8-byte
+        // load -- matching ggml's table_b2b -- when the qh byte is a scalar and
+        // the 8 bit positions are the vector lanes. Inlined into the sdot it is
+        // the opposite (qh byte per lane -> a per-lane gather), so the SDOT
+        // branches materialize the reconstructed int8 codes per block (compute_at
+        // the block loop, kk split as (byte, pos): pos vectorizes the table load,
+        // byte unrolls to a scalar index). The odd-block tail decodes through its
+        // own inline chain (see above), so it does not need this buffer.
+        Func codes_leaf;
+        for (const Func &h : wt_r.handles) {
+            if (h.name() == "combine_bits_code") {
+                codes_leaf = h;
+                break;
+            }
+        }
+        auto schedule_codes = [&](LoopLevel level) {
+            // Split kk into (byte, pos): pos (8) vectorizes the contiguous table
+            // load, byte unrolls to a scalar index. Computed at the block-group
+            // level so all kUnrollBlocks blocks' codes are reconstructed before
+            // the sdots, hiding the table-load latency behind them.
+            Var kc = codes_leaf.args()[0], ko("ko"), ki("ki");
+            codes_leaf.compute_at(level)
+                .split(kc, ko, ki, 8)
+                .vectorize(ki, 8)
+                .unroll(ko);
+        };
+
         // Only handles with update definitions (per-block stat reductions) need
         // explicit scheduling; pure pass-throughs stay inline (same reasoning as
         // symmetric_vec_dot_generator.cpp).
@@ -182,7 +239,10 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
 
             std::vector<Func> winl = {wt_r.replacement};
             for (const Func &h : wt_r.handles) {
-                if (h.function().can_be_inlined()) {
+                // Keep the materialized codes leaf (Q5_1) out of the flatten, so
+                // its qh table read stays a per-block contiguous load.
+                if (h.function().can_be_inlined() &&
+                    !(codes_leaf.defined() && h.name() == codes_leaf.name())) {
                     winl.push_back(h);
                 }
             }
@@ -226,6 +286,9 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // Only the product dot is computed per block now; the offset term is
             // a severed read of s_blocks (nothing to schedule).
             prod_i32.compute_at(acc_vec, bacc).update().atomic().vectorize(r.x, bs);
+            if (codes_leaf.defined()) {
+                schedule_codes(LoopLevel(acc_vec, bacc));
+            }
             Acc.update(1).unscheduled();
         } else if (spec.sched == ScheduleKind::SDOT && getenv("GGML_PER_BLOCK_PROBE")) {
             // PROBE (variant A): rfactor only the block index, so both terms are
@@ -271,7 +334,11 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // their per-block scales out of the surviving rxi reduction, leaving
             // the scale-free Int(32) dot. See sdot_schedule.h.
             Var lane("lane");
-            std::vector<Func> Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms);
+            std::vector<std::string> keep_out;
+            if (codes_leaf.defined()) {
+                keep_out.push_back(codes_leaf.name());
+            }
+            std::vector<Func> Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms, keep_out);
 
             // Acc's update now reduces over (rxo, r.y). Peel rxo back off as the
             // vector lanes, and peel kUnrollBlocks consecutive blocks off
@@ -300,6 +367,9 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
                     .vectorize(lane, lanes)
                     .unroll(rxc);
             }
+            if (codes_leaf.defined()) {
+                schedule_codes(LoopLevel(acc_vec, bacc));
+            }
 
             // Collapsing the lanes x unrolled-blocks accumulators is a fixed
             // cost, but at the row lengths GGML uses it is not a negligible one:

From 2672e7a9879f47603620570adfb724c433e7659e Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 17:01:58 -0400
Subject: [PATCH 17/22] apps/ggml: q5_0/q5_1 no_asserts + register-resident
 codes (0.63->0.69x, 0.68->0.78x)

Two fixes on top of the qh LUT:

- q5_0_vec_dot / q5_1_vec_dot were missing FEATURES no_asserts
  no_bounds_query (every other tuned vec_dot has it), so they paid the
  assert/bounds-query prologue -- ~11 ns of startup on a ~170 ns call.
  Worth ~0.04x on its own.

- The materialized reconstructed codes were round-tripping a per-block
  stack buffer. store_in(MemoryType::Register), with kk split into 16-code
  units (one sdot chunk = two qh bytes x 8 positions) so the table-load
  store width matches the sdot read width, keeps them in the vector
  register file (stmt: "in Register", str drops to 2).

q5_0 0.63->0.69x (166.6 ns), q5_1 0.68->0.78x (170.3 ns). All 28
roundtrips pass, kernel-bench --all clean (24/24), odd tails correct. The
residual gap to 0.95x is the reconstruction op count (ggml's table folds
the +16/-16 into one sub) and qh load traffic -- see PERF_NOTES.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/PERF_NOTES.md                   | 35 +++++++++++++++--------
 apps/ggml/halide/CMakeLists.txt           |  6 ++--
 apps/ggml/halide/vec_dot_generator_base.h | 18 +++++++-----
 3 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
index f05b76e5d2aa..a54568c59738 100644
--- a/apps/ggml/PERF_NOTES.md
+++ b/apps/ggml/PERF_NOTES.md
@@ -12,8 +12,8 @@ Measured at n=4096, best of many runs (see "Measuring" below):
 | ---- | -------- | -------- | ----- | ---------------- |
 | q4_0 | 87.8 ns  | 90.7 ns  | 0.97x | 0.97x            |
 | q4_1 | 106.7 ns | 110.2 ns | 0.97x | 0.80x            |
-| q5_0 | 115.6 ns | 183.0 ns | 0.63x | 0.51x            |
-| q5_1 | 133.5 ns | 195.8 ns | 0.68x | 0.50x            |
+| q5_0 | 115.6 ns | 166.6 ns | 0.69x | 0.51x            |
+| q5_1 | 133.7 ns | 170.3 ns | 0.78x | 0.50x            |
 | q8_0 | 69.1 ns  | 70.4 ns  | 0.98x | 0.95x            |
 
 q5_K (still on the float path) also improved 8615 -> 6730 ns from the same
@@ -242,7 +242,7 @@ lane-split with a block-only reduction using `AlignStart` + matching split-var
 names) is therefore unnecessary here; keep it in mind for formats that keep two
 live accumulators.
 
-## The q5_x 5-bit high bit -- partway (0.51 -> 0.63x, 0.50 -> 0.68x)
+## The q5_x 5-bit high bit -- partway (0.51 -> 0.69x, 0.50 -> 0.78x)
 
 q5_0/q5_1 reach SDOT, but every code carries a per-element high bit unpacked
 from the `qh` field, and that reconstruction -- not the dot, not (for q5_1) the
@@ -265,7 +265,11 @@ byte) embedded in the binary. Two things make it pay:
    gather, `ld1` per lane, measured 0.12x). So the reconstructed codes are
    **materialized** per block (`combine_bits_code` compute_at the block loop,
    `kk` split `(byte, pos)`: pos vectorizes the load, byte unrolls to a scalar
-   index).
+   index). To keep the codes in the vector register file rather than round-trip
+   a stack buffer, the materialization is `store_in(MemoryType::Register)` with
+   `kk` split further into 16-code units (one sdot chunk = two `qh` bytes x 8
+   positions) so the store width matches the sdot's read width -- otherwise the
+   8-wide table-load store vs 16-wide sdot load mismatch keeps it in memory.
 2. Materializing needs the codes leaf kept out of `sdot_partial()`'s deep inline
    (`can_be_inlined()` ignores compute level, so a `compute_root`/`compute_at`
    schedule alone does not stop `eager_inline` -- a `keep_out` name list does).
@@ -280,14 +284,21 @@ The transpose is gone (verified: no `uzp2`/`dup.8b`); reconstruction is a
 handful of ops + 4 contiguous LUT loads/block. Bonus: the same decode change
 sped up q5_K on the float path (8615 -> 6730 ns).
 
-**Ceiling.** This lands q5_0 at 0.63x, q5_1 at 0.68x -- real, but short of the
-0.95x the symmetric formats hit. The residual is the materialization
-**store/reload**: the codes must round-trip a per-block stack buffer (the sdot's
-lane layout can't consume them in registers without bringing the per-lane gather
-back). Group-level compute (all 4 blocks first) measured *worse* (0.50x), not
-better. Closing the last ~0.3x looks like it needs either a Halide CodeGen_ARM
-improvement (feed materialized codes to the sdot without the L1 round-trip) or
-ggml's exact hand-scheduled load/compute balance; the LUT approach caps here.
+**Build-flag bug.** q5_0/q5_1's `add_halide_library` were missing
+`FEATURES no_asserts no_bounds_query` (every other tuned vec_dot has it), so
+they paid the assert/bounds-query prologue -- ~11 ns of pure startup on a ~170
+ns call. Adding it was worth 0.63 -> 0.67x on its own; the register store
+another 0.67 -> 0.69x (q5_1 to 0.78x). Check this first on any new kernel.
+
+**Ceiling.** This lands q5_0 at 0.69x, q5_1 at 0.78x -- real, still short of the
+0.95x the symmetric formats hit. The residual is the reconstruction op count and
+load traffic vs ggml's hand-tuned kernel: ggml's `table_b2b_1[byte]` stores
+`(!bit)<<4`, so one `vsubq_s8` does both the high-bit add *and* the -16 offset;
+ours stores the raw bit, so `CombineBits` does `shl #4` + `add` (nibble) +
+`sub #16` (three ops). Folding the table like ggml (and loading `qh` as one word
+instead of four `ldrb`) is the next lever. Group-level codes compute (all 4
+blocks first) measured *worse* (0.50x) than per-block. See the "what differs
+from ggml" analysis in the session log.
 
 ## Not started
 
diff --git a/apps/ggml/halide/CMakeLists.txt b/apps/ggml/halide/CMakeLists.txt
index 78f8d58572b2..8adce0e4b9ea 100644
--- a/apps/ggml/halide/CMakeLists.txt
+++ b/apps/ggml/halide/CMakeLists.txt
@@ -85,6 +85,7 @@ add_halide_library(
     FROM quants.generator
     GENERATOR symmetric_vec_dot
     PARAMS w_kind=symmetric_5bit block_size=32 w_qmax=16 a_kind=q8_0 a_qmax=127
+    FEATURES no_asserts no_bounds_query
 )
 # Q5_1 is affine like Q4_1, but 5-bit: quant_components.h's AffineQuantize +
 # FiveBitPack, matching block_q5_1's {fp16 d; fp16 m; qh[4]; qs[16];}.
@@ -105,8 +106,9 @@ add_halide_library(
     FROM quants.generator
     GENERATOR symmetric_vec_dot
     PARAMS
-    w_kind=affine_5bit block_size=32 w_levels=31 w_affine_rounding=unclamped_uint8 a_kind=q8_1
-    a_qmax=127
+        w_kind=affine_5bit block_size=32 w_levels=31 w_affine_rounding=unclamped_uint8 a_kind=q8_1
+        a_qmax=127
+    FEATURES no_asserts no_bounds_query
 )
 add_halide_library(
     q8_0_quantize
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 3f438378a4b9..d972720908fc 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -199,14 +199,18 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         }
         auto schedule_codes = [&](LoopLevel level) {
             // Split kk into (byte, pos): pos (8) vectorizes the contiguous table
-            // load, byte unrolls to a scalar index. Computed at the block-group
-            // level so all kUnrollBlocks blocks' codes are reconstructed before
-            // the sdots, hiding the table-load latency behind them.
-            Var kc = codes_leaf.args()[0], ko("ko"), ki("ki");
+            // load, byte unrolls to a scalar index. store_in(Register) keeps the
+            // reconstructed codes in the vector register file straight into the
+            // sdot instead of round-tripping a stack buffer (all stores are at
+            // constant coordinates once ki is vectorized and ko unrolled).
+            Var kc = codes_leaf.args()[0], co("co"), ci("ci"), byte("byte"), pos("pos");
             codes_leaf.compute_at(level)
-                .split(kc, ko, ki, 8)
-                .vectorize(ki, 8)
-                .unroll(ko);
+                .store_in(MemoryType::Register)
+                .split(kc, co, ci, 16)    // 16-code register unit = one sdot chunk
+                .split(ci, byte, pos, 8)  // within it, two qh bytes x 8 positions
+                .vectorize(pos, 8)
+                .unroll(byte)
+                .unroll(co);
         };
 
         // Only handles with update definitions (per-block stat reductions) need

From e2dc0aad2490debdc5b416cda0989cc15a1f91ef Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 17:07:25 -0400
Subject: [PATCH 18/22] apps/ggml: q5_0 vec_dot ABI wrapper on StackBuffer
 (166->155 ns)

q5_0_vec_dot still built three Halide::Runtime::Buffer objects per call
(~13 ns of marshalling on a ~160 ns kernel) while q4_0/q8_0/q5_1 fill a
halide_buffer_t in place via StackBuffer. Migrate it. q5_0 0.69->0.73x.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ggml/halide/ggml_quants.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
index 0f3d1249b308..7583c35aedb8 100644
--- a/apps/ggml/halide/ggml_quants.cpp
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -284,12 +284,11 @@ void ggml_quants_halide_vec_dot_q5_0_q8_0(int n, float *s, size_t bs, const void
                                           size_t by, int nrc) {
     constexpr int kQK = 32, kBlockBytesX = 2 + 4 + kQK / 2, kBlockBytesY = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
-    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
-    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
-    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
-    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
-    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
-    check(q5_0_vec_dot(xb, yb, result), "q5_0_vec_dot");
+    StackBuffer xb, yb, result;
+    check(q5_0_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       result.scalar_f32(s)),
+          "q5_0_vec_dot");
 }
 
 //

From 9a2f4fa5e23831b857873f06ac2422faa6040349 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 21:58:15 -0400
Subject: [PATCH 19/22] Improve q5_0/q5_1 performance (note: breaks contracts)

---
 apps/ggml/PERF_NOTES.md                       |  72 ++++++--
 apps/ggml/halide/ggml_quants.cpp              |  16 +-
 apps/ggml/halide/quant_components.h           | 168 +++++++++++++++---
 .../halide/symmetric_vec_dot_generator.cpp    |  18 +-
 apps/ggml/halide/vec_dot_generator_base.h     | 137 +++++++++++---
 5 files changed, 335 insertions(+), 76 deletions(-)

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
index a54568c59738..859a58a0f7f6 100644
--- a/apps/ggml/PERF_NOTES.md
+++ b/apps/ggml/PERF_NOTES.md
@@ -12,8 +12,8 @@ Measured at n=4096, best of many runs (see "Measuring" below):
 | ---- | -------- | -------- | ----- | ---------------- |
 | q4_0 | 87.8 ns  | 90.7 ns  | 0.97x | 0.97x            |
 | q4_1 | 106.7 ns | 110.2 ns | 0.97x | 0.80x            |
-| q5_0 | 115.6 ns | 166.6 ns | 0.69x | 0.51x            |
-| q5_1 | 133.7 ns | 170.3 ns | 0.78x | 0.50x            |
+| q5_0 | 122.2 ns | 130.2 ns | 0.94x | 0.51x            |
+| q5_1 | 143.5 ns | 153.7 ns | 0.93x | 0.50x            |
 | q8_0 | 69.1 ns  | 70.4 ns  | 0.98x | 0.95x            |
 
 q5_K (still on the float path) also improved 8615 -> 6730 ns from the same
@@ -140,8 +140,8 @@ Two traps found along the way:
   `GuardWithIf` makes the per-block sdot a dynamic-extent allocation that Halide
   has to `bzero` and accumulate *through memory*, roughly doubling the cost of
   every block. Fixed by giving the main reduction an exactly divisible extent
-  (`(nb / kUnrollBlocks) * kUnrollBlocks`) and sweeping the remainder in a
-  second update at the default schedule. `kUnrollBlocks` must be a power of two
+  (`(nb / unroll_blocks) * unroll_blocks`) and sweeping the remainder in a
+  second update at the default schedule. `unroll_blocks` must be a power of two
   -- 3 and 6 measured 40% worse because the simplifier cannot discharge the
   tail.
 - **`specialize()` inherits the schedule as of the call**, so scheduling
@@ -242,7 +242,7 @@ lane-split with a block-only reduction using `AlignStart` + matching split-var
 names) is therefore unnecessary here; keep it in mind for formats that keep two
 live accumulators.
 
-## The q5_x 5-bit high bit -- partway (0.51 -> 0.69x, 0.50 -> 0.78x)
+## The q5_x 5-bit high bit -- DONE (q5_0 0.94x, q5_1 0.93x)
 
 q5_0/q5_1 reach SDOT, but every code carries a per-element high bit unpacked
 from the `qh` field, and that reconstruction -- not the dot, not (for q5_1) the
@@ -255,8 +255,11 @@ to broadcast the two `qh` bytes across the 16 sdot lanes
 1 KB `table_b2b` memory LUT (byte -> 8 expanded bytes, one contiguous load per
 `qh` byte).
 
-**What was done.** Mirrored the LUT: a compile-time `Buffer<uint8_t>` b2b(bit,
-byte) embedded in the binary. Two things make it pay:
+**What was done.** Mirrored the LUT: compile-time b2b tables embedded in the
+binary. The q5 tables contain the final high-bit contribution (`-16/0` for q5_0,
+`0/16` for q5_1), so the lookup folds the shift and q5_0 zero-point into the
+load rather than reconstructing a raw bit and applying them afterward. Several
+other details are necessary:
 
 1. The table read is only a *contiguous* 8-byte load
    (`b0[ramp(qh_byte*8, 1, 8)]`, matching ggml) when the `qh` byte is a
@@ -273,32 +276,63 @@ byte) embedded in the binary. Two things make it pay:
 2. Materializing needs the codes leaf kept out of `sdot_partial()`'s deep inline
    (`can_be_inlined()` ignores compute level, so a `compute_root`/`compute_at`
    schedule alone does not stop `eager_inline` -- a `keep_out` name list does).
-3. The odd-block **tail** reads the same codes but is a separate update
+3. `block_q5_0` and `block_q5_1` are represented as packed Halide struct types,
+   including `qh` as a scalar `UInt(32)`. `LowerStructTypes` now preserves a
+   multi-byte packed field read as `concat_bits()` of adjacent bytes instead of
+   immediately expanding it into a shift/or tree. LLVM can consequently issue
+   one unaligned `ldr/ldur w`, matching ggml, rather than four `ldrb`s. A
+   correctness/codegen regression test covers the deliberately unaligned qh
+   field at byte offset 2.
+4. The odd-block **tail** reads the same codes but is a separate update
    `compute_at` cannot reach. Fixed by giving the tail its **own** decode chain:
    a second `Wt/Vec` placeholder pair, its own `approximate_by`, both bound to
    the same `x_blocks/y_blocks`. Main materializes; the tail reconstructs inline
-   (< kUnrollBlocks blocks, negligible). This is also what unlocks the per-block
-   fusion -- with a shared chain Halide hoists the codes buffer to whole-row.
+   (< `unroll_blocks` blocks, negligible). This is also what unlocks the
+   per-block fusion -- with a shared chain Halide hoists the codes buffer to
+   whole-row.
 
 The transpose is gone (verified: no `uzp2`/`dup.8b`); reconstruction is a
 handful of ops + 4 contiguous LUT loads/block. Bonus: the same decode change
 sped up q5_K on the float path (8615 -> 6730 ns).
 
+**The schedule changes still matter after the compiler fix.** The compiler
+change is the dominant improvement, but it does not make the schedule work
+redundant:
+
+- Removing the explicit qh `compute_at(...).store_in(Register)` schedule made
+  q5_0 regress from about 122 to 129 ns and q5_1 from about 152 to 162 ns, even
+  though both generated versions contained a word load. Keep it: it changes
+  placement/reuse around the four bit extracts, not merely the load width.
+- Interleaving two q5 blocks gives better latency hiding/register pressure than
+  the four-block setting used by q4/q8. Applying two globally regressed q4_0 and
+  q4_1, so this is a per-format `VecDotSpec` choice.
+- Explicitly unrolling the fixed-size final rfactor reductions removes a stack
+  spill and one-iteration epilogue loops (roughly another 1 ns here).
+
+**q5_1 needs a different reduction shape.** The generic affine path horizontally
+reduced each block's Int32 dot before accumulating floats. The q5_1-specific
+schedule keeps four float dot lanes live across the entire block loop, maintains
+two scalar `m*s` accumulators alongside them, and fuses their two-block loops.
+The generated steady state now matches the important shape of ggml's kernel: two
+SDOTs per block, persistent vector FMAs, and scalar offset FMAs, with one
+horizontal reduction at the end.
+
+One representative paired n=4096 run after the final epilogue change was q5_0
+122.2 ns ggml / 130.2 ns Halide (0.94x), and q5_1 143.5 / 153.7 ns (0.93x). Core
+migration moves both the absolute numbers and ratios; another run measured 121.8
+/ 121.8 and 143.2 / 144.3 before the final ~1 ns epilogue improvement. Use
+repeated paired runs rather than treating either sample as a fixed score.
+
 **Build-flag bug.** q5_0/q5_1's `add_halide_library` were missing
 `FEATURES no_asserts no_bounds_query` (every other tuned vec_dot has it), so
 they paid the assert/bounds-query prologue -- ~11 ns of pure startup on a ~170
 ns call. Adding it was worth 0.63 -> 0.67x on its own; the register store
 another 0.67 -> 0.69x (q5_1 to 0.78x). Check this first on any new kernel.
 
-**Ceiling.** This lands q5_0 at 0.69x, q5_1 at 0.78x -- real, still short of the
-0.95x the symmetric formats hit. The residual is the reconstruction op count and
-load traffic vs ggml's hand-tuned kernel: ggml's `table_b2b_1[byte]` stores
-`(!bit)<<4`, so one `vsubq_s8` does both the high-bit add *and* the -16 offset;
-ours stores the raw bit, so `CombineBits` does `shl #4` + `add` (nibble) +
-`sub #16` (three ops). Folding the table like ggml (and loading `qh` as one word
-instead of four `ldrb`) is the next lever. Group-level codes compute (all 4
-blocks first) measured *worse* (0.50x) than per-block. See the "what differs
-from ggml" analysis in the session log.
+Group-level codes compute (all blocks first) measured worse than the final
+per-block materialization. Likewise, removing qh materialization because the new
+compiler lowering already produced `ldr w` was a measured regression; the two
+optimizations address different parts of the generated loop.
 
 ## Not started
 
diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
index 7583c35aedb8..18acc52c35aa 100644
--- a/apps/ggml/halide/ggml_quants.cpp
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -266,16 +266,14 @@ void ggml_quants_halide_quantize_q5_0(const float *x, void *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 2 + 4 + kQK / 2;
     Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
     check(q5_0_quantize(xb, blocks), "q5_0_quantize");
 }
 
 void ggml_quants_halide_dequantize_q5_0(const void *x, float *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 2 + 4 + kQK / 2;
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
     Buffer<float, 1> yb(y, static_cast<int>(k));
     check(q5_0_dequantize(blocks, yb), "q5_0_dequantize");
 }
@@ -285,7 +283,7 @@ void ggml_quants_halide_vec_dot_q5_0_q8_0(int n, float *s, size_t bs, const void
     constexpr int kQK = 32, kBlockBytesX = 2 + 4 + kQK / 2, kBlockBytesY = 2 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
     StackBuffer xb, yb, result;
-    check(q5_0_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
+    check(q5_0_vec_dot(xb.blocks_struct(vx, nb, kBlockBytesX),
                        yb.blocks_bytes(vy, nb, kBlockBytesY),
                        result.scalar_f32(s)),
           "q5_0_vec_dot");
@@ -299,16 +297,14 @@ void ggml_quants_halide_quantize_q5_1(const float *x, void *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 4 + 4 + kQK / 2;
     Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
     check(q5_1_quantize(xb, blocks), "q5_1_quantize");
 }
 
 void ggml_quants_halide_dequantize_q5_1(const void *x, float *y, int64_t k) {
     constexpr int kQK = 32, kBlockBytes = 4 + 4 + kQK / 2;
     const int32_t nb = static_cast<int32_t>(k / kQK);
-    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
-    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
     Buffer<float, 1> yb(y, static_cast<int>(k));
     check(q5_1_dequantize(blocks, yb), "q5_1_dequantize");
 }
@@ -318,7 +314,7 @@ void ggml_quants_halide_vec_dot_q5_1_q8_1(int n, float *s, size_t bs, const void
     constexpr int kQK = 32, kBlockBytesX = 4 + 4 + kQK / 2, kBlockBytesY = 4 + kQK;
     const int32_t nb = static_cast<int32_t>(n / kQK);
     StackBuffer xb, yb, sb, result;
-    check(q5_1_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
+    check(q5_1_vec_dot(xb.blocks_struct(vx, nb, kBlockBytesX),
                        yb.blocks_bytes(vy, nb, kBlockBytesY),
                        sb.blocks_field_f16(vy, nb, 2, kBlockBytesY),  // Q8_1 stored `s`
                        result.scalar_f32(s)),
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index be09af4bc3e6..5756e8e5ae9d 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -812,9 +812,10 @@ class PlanarBitPack : public Halide::Approximation {
     // derived rather than taken as its own parameter -- see nibble_pack/
     // crumb_pack/rotating_bit_pack/le_bit_pack below for the named-shape
     // constructors most call sites should use instead of this directly.
-    PlanarBitPack(int field_bits, int pos_count, int qmax = 0, bool plane_axis = false)
+    PlanarBitPack(int field_bits, int pos_count, int qmax = 0, bool plane_axis = false,
+                  int value_scale = 1)
         : field_bits_(field_bits), plane_count_(8 / field_bits), pos_count_(pos_count), qmax_(qmax),
-          plane_axis_(plane_axis) {
+          plane_axis_(plane_axis), value_scale_(value_scale) {
     }
 
     Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
@@ -832,7 +833,8 @@ class PlanarBitPack : public Halide::Approximation {
 
         RDom rp(0, plane_count_, "rp");
         Expr kk = outer * group + rp * pos_count_ + pos;
-        Expr field = cast<uint32_t>(cast<int32_t>(codes(kk, blk)) + qmax_) & ((1u << field_bits_) - 1);
+        Expr field = cast<uint32_t>((cast<int32_t>(codes(kk, blk)) + qmax_) / value_scale_) &
+                     ((1u << field_bits_) - 1);
         Func bytes("planar_bit_pack_bytes");
         bytes(byte_idx, blk) = cast<uint8_t>(0);
         bytes(byte_idx, blk) = bytes(byte_idx, blk) | cast<uint8_t>(field << (rp * field_bits_));
@@ -869,8 +871,8 @@ class PlanarBitPack : public Halide::Approximation {
             // plus mask (~24 NEON ops per 16 lanes), which is the whole
             // q5_x-vs-q4_x gap. A Buffer<> is embedded in the binary as constant
             // data, so this is a pure lookup, no runtime input.
-            static const Halide::Buffer<uint8_t> b2b = [] {
-                Halide::Buffer<uint8_t> t(8, 256);
+            static const Halide::Buffer<int8_t> b2b = [] {
+                Halide::Buffer<int8_t> t(8, 256);
                 t.set_min(0, 0);
                 for (int by = 0; by < 256; by++) {
                     for (int bit = 0; bit < 8; bit++) {
@@ -879,18 +881,47 @@ class PlanarBitPack : public Halide::Approximation {
                 }
                 return t;
             }();
-            field = cast<uint32_t>(b2b(cast<int32_t>(plane * field_bits_), cast<int32_t>(byte)));
+            static const Halide::Buffer<int8_t> b2b_shifted = [] {
+                Halide::Buffer<int8_t> t(8, 256);
+                t.set_min(0, 0);
+                for (int by = 0; by < 256; by++) {
+                    for (int bit = 0; bit < 8; bit++) {
+                        t(bit, by) = ((by >> bit) & 1) << 4;
+                    }
+                }
+                return t;
+            }();
+            static const Halide::Buffer<int8_t> b2b_shifted_signed = [] {
+                Halide::Buffer<int8_t> t(8, 256);
+                t.set_min(0, 0);
+                for (int by = 0; by < 256; by++) {
+                    for (int bit = 0; bit < 8; bit++) {
+                        t(bit, by) = (((by >> bit) & 1) << 4) - 16;
+                    }
+                }
+                return t;
+            }();
+            const Expr bit_index = cast<int32_t>(plane * field_bits_);
+            const Expr byte_value = cast<int32_t>(byte);
+            if (value_scale_ == 16 && qmax_ == 16) {
+                field = b2b_shifted_signed(bit_index, byte_value);
+            } else if (value_scale_ == 16 && qmax_ == 0) {
+                field = b2b_shifted(bit_index, byte_value);
+            } else {
+                field = cast<int32_t>(b2b(bit_index, byte_value)) * value_scale_ - qmax_;
+            }
         } else {
-            field = (byte >> (plane * field_bits_)) & ((1u << field_bits_) - 1);
+            field = cast<int32_t>((byte >> (plane * field_bits_)) & ((1u << field_bits_) - 1)) * value_scale_ - qmax_;
         }
         Func codes("planar_bit_pack_codes");
-        codes(kk, blk, _) = cast<int8_t>(cast<int32_t>(field) - qmax_);
+        codes(kk, blk, _) = cast<int8_t>(field);
         return {{codes}, {}};
     }
 
 private:
     int field_bits_, plane_count_, pos_count_, qmax_;
     bool plane_axis_;
+    int value_scale_;
 };
 
 // Named PlanarBitPack shapes for the four bit-widths this file actually
@@ -915,8 +946,8 @@ inline std::unique_ptr<Halide::Approximation> rotating_bit_pack(int window, int
 // The Stage-2 qh addressing (make_code_pack's code_bits==5 case): one flat
 // bit per element, byte kk/8 at shift kk%8 -- PlanarBitPack{1, 1} regardless
 // of block size (pos_count is always 1; there's no "window" to parameterize).
-inline std::unique_ptr<Halide::Approximation> le_bit_pack() {
-    return std::make_unique<PlanarBitPack>(1, 1);
+inline std::unique_ptr<Halide::Approximation> le_bit_pack(int value_scale = 1, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(1, 1, qmax, false, value_scale);
 }
 
 // GGML's Q5_0/Q5_1 5-bit code split (a 4-bit low nibble plus a 5th high bit,
@@ -1273,6 +1304,74 @@ class StructBlockLayout : public Halide::Approximation {
     std::string scale_field_, codes_field_;
 };
 
+// Struct-typed layout for the split-code Q5_0/Q5_1 blocks. Keeping qh as a
+// typed UInt(32) field lets LowerStructTypes expose one four-byte load to LLVM;
+// the byte-buffer form presents four unrelated UInt(8) loads, which cannot be
+// coalesced before the byte-indexed expansion-table lookups. `codes_bytes`
+// retains the existing logical {qh[4], qs[16]} layout, so the reusable
+// combined-bit codec inside this leaf is unchanged.
+class Q5StructBlockLayout : public Halide::Approximation {
+public:
+    Q5StructBlockLayout(Halide::Type block_type, bool affine)
+        : block_type_(block_type), affine_(affine) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes_bytes = inputs[0];
+        Func scale = inputs[1];
+        Func min = affine_ ? inputs[2] : Func();
+        Var blk("blk");
+
+        Expr qh = cast<uint32_t>(codes_bytes(0, blk)) |
+                  (cast<uint32_t>(codes_bytes(1, blk)) << 8) |
+                  (cast<uint32_t>(codes_bytes(2, blk)) << 16) |
+                  (cast<uint32_t>(codes_bytes(3, blk)) << 24);
+        std::vector<Expr> vals = {cast<float16_t>(scale(blk))};
+        if (affine_) {
+            vals.push_back(cast<float16_t>(min(blk)));
+        }
+        vals.push_back(qh);
+        for (int i = 0; i < 16; i++) {
+            vals.push_back(codes_bytes(i + 4, blk));
+        }
+
+        Func packed("q5_struct_block_packed");
+        packed(blk) = pack_struct(block_type_, vals);
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];
+        Var local("local"), blk("blk");
+
+        Func scale("q5_struct_block_scale");
+        scale(blk) = cast<float>(field(packed(blk), "d"));
+        Func qh("q5_struct_block_qh");
+        qh(blk) = field(packed(blk), "qh");
+        Func qh_bytes("q5_struct_block_qh_bytes");
+        qh_bytes(local, blk) = cast<uint8_t>(qh(blk) >> (local * 8));
+        Func qs_bytes("q5_struct_block_qs_bytes");
+        qs_bytes(local, blk) = cast<uint8_t>(field(packed(blk), "qs")[local]);
+        Func codes_bytes("q5_struct_block_codes_bytes");
+        codes_bytes(local, blk) = select(local < 4,
+                                         qh_bytes(local, blk),
+                                         qs_bytes(local - 4, blk));
+
+        if (affine_) {
+            Func min("q5_struct_block_min");
+            min(blk) = cast<float>(field(packed(blk), "m"));
+            return {{codes_bytes, scale, min}, {qh}};
+        }
+        return {{codes_bytes, scale}, {qh}};
+    }
+
+private:
+    Halide::Type block_type_;
+    bool affine_;
+};
+
 // One field, in ON-DISK byte order, of a struct-packed block layout: an
 // on-disk byte width plus which logical "slot" it lands in -- the index it
 // occupies in the Func vector immediately after StructPack::decode() (and,
@@ -1679,8 +1778,8 @@ class LinearDequant : public Halide::Approximation {
 // CombineBits{...}} -- not this leaf, which is only the split/combine math.
 class CombineBits : public Halide::Approximation {
 public:
-    CombineBits(int high_weight, int offset)
-        : high_weight_(high_weight), offset_(offset) {
+    CombineBits(int high_weight, int offset, bool expanded_high = false)
+        : high_weight_(high_weight), offset_(offset), expanded_high_(expanded_high) {
     }
 
     Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
@@ -1693,7 +1792,9 @@ class CombineBits : public Halide::Approximation {
         Func low("combine_bits_low");
         low(kk, blk) = cast<int8_t>(combined % high_weight_);
         Func high("combine_bits_high");
-        high(kk, blk) = cast<int8_t>(combined / high_weight_);
+        high(kk, blk) = cast<int8_t>(expanded_high_ ?
+                                         (combined / high_weight_) * high_weight_ - offset_ :
+                                         combined / high_weight_);
         return {{low, high}, {}};
     }
 
@@ -1701,12 +1802,15 @@ class CombineBits : public Halide::Approximation {
         using namespace Halide;
         Func low = encoded[0], high = encoded[1];
         Func code("combine_bits_code");
-        code(kk, blk) = cast<int8_t>((cast<int32_t>(low(kk, blk)) + high_weight_ * cast<int32_t>(high(kk, blk))) - offset_);
+        code(kk, blk) = cast<int8_t>(expanded_high_ ?
+                                         cast<int32_t>(low(kk, blk)) + cast<int32_t>(high(kk, blk)) :
+                                         (cast<int32_t>(low(kk, blk)) + high_weight_ * cast<int32_t>(high(kk, blk))) - offset_);
         return {{code}, {}};
     }
 
 private:
     int high_weight_, offset_;
+    bool expanded_high_;
     Halide::Var kk{"kk"}, blk{"blk"};
 };
 
@@ -2658,10 +2762,12 @@ inline CodePackField make_code_pack(int block_size, int code_bits, int qmax) {
         // nibble_pack. `qmax` becomes CombineBits' final recentering offset
         // (0 for Q5_1's already-unsigned affine codes) rather than a per-part
         // qmax, since the parts here are raw, uncentered digits.
-        return {make_combined_bit_codec(
-                    16, qmax,
-                    FieldSpec{1, 4, le_bit_pack()},                          // qh -> high bit
-                    FieldSpec{0, block_size / 2, nibble_pack(block_size)}),  // qs -> low nibble
+        return {std::make_unique<Compose>(
+                    make_block_layout(
+                        FieldSpec{1, 4, le_bit_pack(16, qmax)},                 // qh -> folded high bit and offset
+                        FieldSpec{0, block_size / 2, nibble_pack(block_size)})  // qs -> low nibble
+                        .layout,
+                    CombineBits{16, qmax, /*expanded_high=*/true}),
                 block_size / 2 + 4};
     }
     if (code_bits == 1) {
@@ -2744,8 +2850,18 @@ inline SchemeAndBytes make_affine_block_scheme(
 // (kind=symmetric_5bit/affine_5bit) don't need to change in lockstep.
 inline SchemeAndBytes make_symmetric_5bit_block_scheme(int block_size, int qmax,
                                                        Layout layout = Layout::FlatRow) {
-    return make_symmetric_block_scheme(block_size, qmax, RoundingMode::TruncateHalfUpWithOffset,
-                                       ScaleAnchor::ExtremeSignedValue, /*code_bits=*/5, layout);
+    using namespace Halide;
+    _halide_user_assert(block_size == 32) << "The Q5 struct layout requires a 32-element block.\n";
+    Type block_type = Type::Struct({{"d", Float(16)}, {"qh", UInt(32)}, {"qs", UInt(8), 16}});
+    CodePackField code = make_code_pack(block_size, /*code_bits=*/5, qmax);
+    return {std::make_unique<Compose>(
+                Q5StructBlockLayout{block_type, /*affine=*/false},
+                Apply{0, std::move(code.pack)},
+                SymmetricAffineQuantize{block_size, qmax, RoundingMode::TruncateHalfUpWithOffset,
+                                        ScaleAnchor::ExtremeSignedValue},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            block_type.bytes(),
+            block_type};
 }
 
 // Affine quantize (like make_affine_block_scheme()) but 5-bit -- likewise now
@@ -2757,7 +2873,17 @@ inline SchemeAndBytes make_symmetric_5bit_block_scheme(int block_size, int qmax,
 inline SchemeAndBytes make_affine_5bit_block_scheme(int block_size, int levels,
                                                     AffineRounding rounding,
                                                     Layout layout = Layout::FlatRow) {
-    return make_affine_block_scheme(block_size, levels, rounding, /*code_bits=*/5, layout);
+    using namespace Halide;
+    _halide_user_assert(block_size == 32) << "The Q5 struct layout requires a 32-element block.\n";
+    Type block_type = Type::Struct({{"d", Float(16)}, {"m", Float(16)}, {"qh", UInt(32)}, {"qs", UInt(8), 16}});
+    CodePackField code = make_code_pack(block_size, /*code_bits=*/5, /*qmax=*/0);
+    return {std::make_unique<Compose>(
+                Q5StructBlockLayout{block_type, /*affine=*/true},
+                Apply{0, std::move(code.pack)},
+                AffineQuantize{block_size, levels, rounding},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            block_type.bytes(),
+            block_type};
 }
 
 // Symmetric byte-packed quantize (like make_symmetric_block_scheme() with
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index a6d3a43faf4e..97517ab28358 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -109,8 +109,10 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
             sched = ScheduleKind::SDOT;
             distribute = true;
             break;
-        case WKind::Symmetric5Bit:
-            wc = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed).scheme;
+        case WKind::Symmetric5Bit: {
+            SchemeAndBytes sb = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed);
+            wc = std::move(sb.scheme);
+            weight_type = sb.block_type;
             wb = 2 + 4 + wbs / 2;
             // Q5_0's 5-bit code is assembled via CombineBits (nibble |
             // (high_bit << 4)); that reconstruction is all inside the (r.x-
@@ -119,8 +121,11 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
             // works the same as Q4_0 (see the base header's deep-inline SDOT).
             sched = ScheduleKind::SDOT;
             break;
-        case WKind::Affine5Bit:
-            wc = make_affine_5bit_block_scheme(wbs, w_levels, w_affine_rounding, Layout::BlockIndexed).scheme;
+        }
+        case WKind::Affine5Bit: {
+            SchemeAndBytes sb = make_affine_5bit_block_scheme(wbs, w_levels, w_affine_rounding, Layout::BlockIndexed);
+            wc = std::move(sb.scheme);
+            weight_type = sb.block_type;
             wb = 2 + 2 + 4 + wbs / 2;
             // Same offset-carrying decode as the 4-bit affine case above; the
             // 5-bit code reconstruction stays inside the codes leaf, so the
@@ -129,6 +134,7 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
             distribute = true;
             break;
         }
+        }
 
         // Q8_0/Q8_1 activations are 32-element blocks. Build the codec at that
         // natural block size, then Reblock to the weight's block size (a no-op
@@ -160,7 +166,9 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         // Activation stays on the byte path for now (Q8_0/Q8_1 are shared across
         // many weight formats, and the Reblock relayout is byte-based); only the
         // weight operand is struct-typed here.
-        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}, act_has_block_sums};
+        const bool five_bit = w_kind.value() == WKind::Symmetric5Bit || w_kind.value() == WKind::Affine5Bit;
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}, act_has_block_sums,
+                five_bit ? 2 : 4};
     }
 };
 
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index d972720908fc..569d265202a0 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -71,20 +71,20 @@ struct VecDotSpec {
     // field supplied as a third Input, instead of recomputing it -- ggml's own
     // q4_1/q5_1 optimization. See SchemeAndBytes::has_block_sums.
     bool act_has_block_sums = false;
+    // Number of adjacent blocks kept in flight by the SDOT schedule. The
+    // ordinary q4/q8 paths prefer four; q5's table-expansion live ranges make
+    // two faster and match GGML's paired-block loop.
+    int unroll_blocks = 4;
 };
 
 template<typename Derived>
 class VecDotGeneratorBase : public Halide::Generator<Derived> {
 public:
-    // How many blocks the SDOT schedule keeps in flight as independent float
-    // accumulators. Four is enough to hide the accumulate latency without
-    // running the block loop out of vector registers.
-    static constexpr int kUnrollBlocks = 4;
-
     void configure() {
         using namespace Halide;
         VecDotSpec spec = static_cast<Derived *>(this)->build_vec_dot();
         int bs = spec.block_size;
+        const int unroll_blocks = spec.unroll_blocks;
 
         // A struct-typed operand's packed blocks are a 1-D Type::Struct buffer
         // (block index only); a byte-path operand is 2-D (byte, blk).
@@ -119,17 +119,17 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         Wt(kk, blk) = 0.0f;
         Vec(kk, blk) = 0.0f;
 
-        // The SDOT schedule interleaves kUnrollBlocks blocks into independent
+        // The SDOT schedule interleaves unroll_blocks blocks into independent
         // accumulators, so it wants a block count divisible by that. Letting
         // Halide's split produce the odd tail instead is not a local cost: the
         // predicate it inserts makes the per-block sdot a dynamic-extent
         // allocation that has to be zeroed and accumulated through memory,
         // roughly doubling the cost of *every* block. So the main reduction gets
         // an exactly divisible extent and a second update sweeps the remainder
-        // at the default schedule (at most kUnrollBlocks - 1 blocks).
+        // at the default schedule (at most unroll_blocks - 1 blocks).
         const bool sdot = spec.sched == ScheduleKind::SDOT;
         Expr nblocks = x_blocks.dim(wt_struct ? 0 : 1).extent();
-        Expr main_blocks = sdot ? (nblocks / kUnrollBlocks) * kUnrollBlocks : nblocks;
+        Expr main_blocks = sdot ? (nblocks / unroll_blocks) * unroll_blocks : nblocks;
 
         RDom r(0, bs, 0, main_blocks, "r");
         Func Acc("acc");
@@ -141,7 +141,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         // main reduction materialize a reconstructed-codes leaf (Q5_x) via
         // compute_at while the tail -- a different loop nest that could not see
         // that per-block buffer -- reconstructs inline, at negligible cost (fewer
-        // than kUnrollBlocks blocks). Both chains read the same x_blocks/y_blocks.
+        // than unroll_blocks blocks). Both chains read the same x_blocks/y_blocks.
         Func WtT("wt_naive_tail"), VecT("vec_naive_tail");
         WtT(kk, blk) = 0.0f;
         VecT(kk, blk) = 0.0f;
@@ -160,9 +160,10 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         std::vector<Func> to_sever = wt_r.encoded;
         to_sever.insert(to_sever.end(), act_r.encoded.begin(), act_r.encoded.end());
         std::vector<ImageParam> bind_to = {x_blocks, y_blocks};
+        ApproximationResult wtT_r, actT_r;
         if (sdot) {
-            ApproximationResult wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
-            ApproximationResult actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
+            wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
+            actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
             to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
             to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
             bind_to.push_back(x_blocks);
@@ -190,11 +191,12 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         // the block loop, kk split as (byte, pos): pos vectorizes the table load,
         // byte unrolls to a scalar index). The odd-block tail decodes through its
         // own inline chain (see above), so it does not need this buffer.
-        Func codes_leaf;
+        Func codes_leaf, qh_leaf;
         for (const Func &h : wt_r.handles) {
             if (h.name() == "combine_bits_code") {
                 codes_leaf = h;
-                break;
+            } else if (h.name() == "q5_struct_block_qh") {
+                qh_leaf = h;
             }
         }
         auto schedule_codes = [&](LoopLevel level) {
@@ -211,6 +213,9 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
                 .vectorize(pos, 8)
                 .unroll(byte)
                 .unroll(co);
+            if (qh_leaf.defined()) {
+                qh_leaf.compute_at(level).store_in(MemoryType::Register);
+            }
         };
 
         // Only handles with update definitions (per-block stat reductions) need
@@ -227,7 +232,97 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             }
         }
 
-        if (sever_sum) {
+        Func final_value = Acc;
+        if (sever_sum && codes_leaf.defined()) {
+            // Q5_1 needs two differently shaped reductions: a four-lane dot
+            // accumulator that survives across blocks, and the scalar m*s
+            // term supplied by Q8_1's stored sum. Express them independently,
+            // then fuse their paired-block loops. This mirrors GGML and avoids
+            // the generic sever path's horizontal Int(32) reduction per block.
+            Func min_leaf, scale_leaf;
+            for (const Func &h : wt_r.handles) {
+                if (h.name() == "q5_struct_block_min") {
+                    min_leaf = h;
+                } else if (h.name() == "q5_struct_block_scale") {
+                    scale_leaf = h;
+                }
+            }
+            Func codes_tail, scale_tail;
+            for (const Func &h : wtT_r.handles) {
+                if (h.name().find("combine_bits_code") == 0) {
+                    codes_tail = h;
+                } else if (h.name().find("q5_struct_block_scale") == 0) {
+                    scale_tail = h;
+                }
+            }
+            _halide_internal_assert(min_leaf.defined() && scale_leaf.defined() &&
+                                    codes_tail.defined() && scale_tail.defined());
+
+            Func wt_product("q5_1_product_weight");
+            wt_product(kk, blk) = cast<float>(codes_leaf(kk, blk)) * scale_leaf(blk);
+            Func wt_product_tail("q5_1_product_weight_tail");
+            wt_product_tail(kk, blk) = cast<float>(codes_tail(kk, blk)) * scale_tail(blk);
+
+            RDom rd(0, bs, 0, main_blocks, "rd");
+            RDom rd_tail(0, bs, main_blocks, nblocks - main_blocks, "rd_tail");
+            Func dot_acc("q5_1_dot_acc");
+            dot_acc() = 0.0f;
+            dot_acc() += wt_product(rd.x, rd.y) * act_r.replacement(rd.x, rd.y);
+            dot_acc() += wt_product_tail(rd_tail.x, rd_tail.y) * actT_r.replacement(rd_tail.x, rd_tail.y);
+
+            const int lanes = 4;
+            RVar rxc("rxc"), rxr("rxr"), rxo("rxo"), rxi("rxi");
+            dot_acc.update(0).split(rd.x, rxc, rxr, 4 * lanes);
+            dot_acc.update(0).split(rxr, rxo, rxi, 4);
+            dot_acc.update(0).eager_inline(wt_product);
+            Var lane("lane");
+            std::vector<Func> dot_i32 = sdot_partial(dot_acc, {{rxo, lane}, {rd.y, u}},
+                                                     {act_r}, false, {codes_leaf.name()});
+
+            RVar ryo("ryo"), ryi("ryi");
+            Var lv("lv"), bacc("bacc");
+            dot_acc.update(0).split(rd.y, ryo, ryi, unroll_blocks);
+            Func acc_vec = dot_acc.update(0).rfactor({{rxo, lv}, {ryi, bacc}});
+            acc_vec.compute_root().vectorize(lv, lanes).unroll(bacc);
+            acc_vec.update().vectorize(lv, lanes).unroll(bacc);
+            for (Func &part : dot_i32) {
+                part.compute_at(acc_vec, bacc)
+                    .update()
+                    .atomic()
+                    .vectorize(rxi, 4)
+                    .vectorize(lane, lanes)
+                    .unroll(rxc);
+            }
+            schedule_codes(LoopLevel(acc_vec, bacc));
+
+            Var lv2("lv2");
+            Func dot_lanes = dot_acc.update(0).rfactor(rxo, lv2);
+            dot_lanes.compute_root().vectorize(lv2, lanes);
+            dot_lanes.update().vectorize(lv2, lanes).unroll(ryi);
+            dot_acc.update(0).atomic().vectorize(rxo, lanes);
+            dot_acc.update(1).unscheduled();
+
+            RDom rb(0, main_blocks, "rb");
+            RDom rb_tail(main_blocks, nblocks - main_blocks, "rb_tail");
+            Func offset_acc("q5_1_offset_acc");
+            offset_acc() = 0.0f;
+            offset_acc() += min_leaf(rb) * cast<float>(s_blocks(rb));
+            offset_acc() += min_leaf(rb_tail) * cast<float>(s_blocks(rb_tail));
+
+            RVar rbi("rbi");
+            Var offset_bacc("offset_bacc");
+            offset_acc.update(0).split(rb.x, ryo, rbi, unroll_blocks);
+            Func offset_vec = offset_acc.update(0).rfactor(rbi, offset_bacc);
+            offset_vec.compute_root().unroll(offset_bacc);
+            offset_vec.update().unroll(offset_bacc);
+            offset_vec.update().compute_with(acc_vec.update(), ryo, LoopAlignStrategy::AlignStart);
+            offset_acc.update(0).unroll(rbi);
+            offset_acc.update(1).unscheduled();
+
+            Func combined("q5_1_combined_acc");
+            combined() = dot_acc() + offset_acc();
+            final_value = combined;
+        } else if (sever_sum) {
             // Affine weight x Q8_1: the per-block product (d*code + m)*(d_act*act)
             // distributes into d*d_act*sum(code*act) + m*d_act*sum(act). ggml does
             // not recompute the second sum -- it reads the `s` field Q8_1 stores.
@@ -282,7 +377,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
 
             RVar ryo("ryo"), ryi("ryi");
             Var bacc("bacc");
-            Acc.update(0).split(r.y, ryo, ryi, kUnrollBlocks);
+            Acc.update(0).split(r.y, ryo, ryi, unroll_blocks);
             Func acc_vec = Acc.update(0).rfactor(ryi, bacc);
             acc_vec.compute_root().unroll(bacc);
             acc_vec.update().unroll(bacc);
@@ -303,7 +398,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
 
             RVar ryo("ryo"), ryi("ryi");
             Var bacc("bacc");
-            Acc.update(0).split(r.y, ryo, ryi, kUnrollBlocks);
+            Acc.update(0).split(r.y, ryo, ryi, unroll_blocks);
             Func acc_vec = Acc.update(0).rfactor(ryi, bacc);
             acc_vec.compute_root().unroll(bacc);
             acc_vec.update().unroll(bacc);
@@ -345,7 +440,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             std::vector<Func> Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms, keep_out);
 
             // Acc's update now reduces over (rxo, r.y). Peel rxo back off as the
-            // vector lanes, and peel kUnrollBlocks consecutive blocks off
+            // vector lanes, and peel unroll_blocks consecutive blocks off
             // alongside it into separate accumulators. The accumulators have to
             // be split over *blocks*: every lane of one accumulator advances on
             // every block, so widening the vector does not shorten the
@@ -354,14 +449,14 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // whole kernel.
             RVar ryo("ryo"), ryi("ryi");
             Var lv("lv"), bacc("bacc");
-            Acc.update(0).split(r.y, ryo, ryi, kUnrollBlocks);
+            Acc.update(0).split(r.y, ryo, ryi, unroll_blocks);
             Func acc_vec = Acc.update(0).rfactor({{rxo, lv}, {ryi, bacc}});
             acc_vec.compute_root().vectorize(lv, lanes).unroll(bacc);
             acc_vec.update().vectorize(lv, lanes).unroll(bacc);
 
             // Inside the unrolled body, not at the block-group loop: at `bacc` the
             // sdot is one block's worth of registers, whereas at `ryo` it is a
-            // kUnrollBlocks-long buffer that Halide has to allocate, zero, and
+            // unroll_blocks-long buffer that Halide has to allocate, zero, and
             // accumulate through memory.
             for (Func &part : Acc_i32) {
                 part.compute_at(acc_vec, bacc)
@@ -383,7 +478,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             Var lv2("lv2");
             Func acc_lanes = Acc.update(0).rfactor(rxo, lv2);
             acc_lanes.compute_root().vectorize(lv2, lanes);
-            acc_lanes.update().vectorize(lv2, lanes);
+            acc_lanes.update().vectorize(lv2, lanes).unroll(ryi);
             Acc.update(0).atomic().vectorize(rxo, lanes);
 
             // The odd-block tail deliberately keeps the default schedule.
@@ -394,7 +489,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         // is a separate step.
 
         Func result("result");
-        result() = Acc();
+        result() = final_value();
 
         // A byte-path operand's block stride is pinned to its block width: these
         // are densely packed GGML rows, and leaving the stride dynamic costs a

From 3e57421841367f6662ae4a452863cbca30bc6fc8 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 23:37:27 -0400
Subject: [PATCH 20/22] Checkpoint

---
 apps/ggml/PERF_NOTES.md                       |  50 +-
 apps/ggml/Q5_0_APPROXIMATION_PLAN.md          | 150 +++++
 apps/ggml/halide/quant_components.h           | 227 ++------
 apps/ggml/halide/sdot_schedule.h              |   8 +-
 .../halide/symmetric_vec_dot_generator.cpp    |   5 +-
 apps/ggml/halide/vec_dot_generator_base.h     |  71 ++-
 src/Approximation.cpp                         |  49 +-
 src/Approximation.h                           |  64 ++-
 src/ApproximationComponents.h                 | 521 ++++++++++++++++++
 src/CMakeLists.txt                            |   1 +
 src/Func.cpp                                  |   4 +-
 test/correctness/CMakeLists.txt               |   1 +
 test/correctness/approximate_by.cpp           |  51 ++
 test/correctness/approximation_components.cpp | 229 ++++++++
 14 files changed, 1215 insertions(+), 216 deletions(-)
 create mode 100644 apps/ggml/Q5_0_APPROXIMATION_PLAN.md
 create mode 100644 src/ApproximationComponents.h
 create mode 100644 test/correctness/approximation_components.cpp

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
index 859a58a0f7f6..1528e2af2ffc 100644
--- a/apps/ggml/PERF_NOTES.md
+++ b/apps/ggml/PERF_NOTES.md
@@ -275,21 +275,19 @@ other details are necessary:
    8-wide table-load store vs 16-wide sdot load mismatch keeps it in memory.
 2. Materializing needs the codes leaf kept out of `sdot_partial()`'s deep inline
    (`can_be_inlined()` ignores compute level, so a `compute_root`/`compute_at`
-   schedule alone does not stop `eager_inline` -- a `keep_out` name list does).
-3. `block_q5_0` and `block_q5_1` are represented as packed Halide struct types,
-   including `qh` as a scalar `UInt(32)`. `LowerStructTypes` now preserves a
-   multi-byte packed field read as `concat_bits()` of adjacent bytes instead of
-   immediately expanding it into a shift/or tree. LLVM can consequently issue
-   one unaligned `ldr/ldur w`, matching ggml, rather than four `ldrb`s. A
-   correctness/codegen regression test covers the deliberately unaligned qh
-   field at byte offset 2.
-4. The odd-block **tail** reads the same codes but is a separate update
-   `compute_at` cannot reach. Fixed by giving the tail its **own** decode chain:
-   a second `Wt/Vec` placeholder pair, its own `approximate_by`, both bound to
-   the same `x_blocks/y_blocks`. Main materializes; the tail reconstructs inline
-   (< `unroll_blocks` blocks, negligible). This is also what unlocks the
-   per-block fusion -- with a shared chain Halide hoists the codes buffer to
-   whole-row.
+   schedule alone does not stop `eager_inline`). `sdot_partial()` now takes
+   resolved `Func` identities, not generated-name strings; q5_0 obtains the
+   identity from its `AdditiveRadixSplit` stage key.
+3. q5_0 uses its faithful packed type `{Float16 d; UInt8 qh[4]; UInt8 qs[16]}`.
+   `LittleEndianScalarPack<uint32_t>` decodes qh with `concat_bits()`;
+   `LowerStructTypes` and LLVM consequently issue one unaligned `ldr/ldur w`,
+   matching ggml, rather than four `ldrb`s. q5_1 retains the older scalar
+   `UInt(32)` struct field as explicit transitional debt.
+4. q5_0's odd-block **tail** now shares the main Approximation graph. Before the
+   main update materializes the keyed reconstructed-code and qh-word Funcs, the
+   tail update alone eagerly inlines that decode chain. This removes the second
+   q5_0 weight `approximate_by` graph while keeping the tail inline and the main
+   loop register-resident. q5_1 retains its separate legacy tail chain.
 
 The transpose is gone (verified: no `uzp2`/`dup.8b`); reconstruction is a
 handful of ops + 4 contiguous LUT loads/block. Bonus: the same decode change
@@ -317,11 +315,23 @@ The generated steady state now matches the important shape of ggml's kernel: two
 SDOTs per block, persistent vector FMAs, and scalar offset FMAs, with one
 horizontal reduction at the end.
 
-One representative paired n=4096 run after the final epilogue change was q5_0
-122.2 ns ggml / 130.2 ns Halide (0.94x), and q5_1 143.5 / 153.7 ns (0.93x). Core
-migration moves both the absolute numbers and ratios; another run measured 121.8
-/ 121.8 and 143.2 / 144.3 before the final ~1 ns epilogue improvement. Use
-repeated paired runs rather than treating either sample as a fixed score.
+After the core-composition migration, ten paired n=4096 runs had q5_0 medians of
+122.147 ns GGML / 130.645 ns Halide and a median paired ratio of 0.9349x. The
+committed baseline was 122.172 / 130.259 ns and 0.9379x, so the refactor is
+performance-neutral (+0.30% Halide time). q5_1 remained within noise at 143.728
+/ 153.956 ns and 0.9336x. Use repeated paired runs rather than treating either
+sample as a fixed score.
+
+**Core composition now mirrors the representation.** q5_0 is
+`StructLayout -> StorageCast -> LittleEndianScalarPack -> BinaryAlphabetPack -> PlanarFieldPack -> AdditiveRadixSplit -> SymmetricBlockQuantize -> BlockReshape`.
+Opaque Approximation stage keys carry the reconstructed-code and qh-word
+identities into `VecDotSpec`; generated Func names no longer control q5_0's
+schedule. `BlockReshape` and symmetric block quantization now live in the public
+Approximation component library, with ggml compatibility aliases/wrappers.
+
+Reusable performance experiments should become discoverable `kernel-bench`
+modes. In particular, promote the odd-tail n sweep (32, 96, 160, 224, 1056)
+instead of preserving it only as a shell loop.
 
 **Build-flag bug.** q5_0/q5_1's `add_halide_library` were missing
 `FEATURES no_asserts no_bounds_query` (every other tuned vec_dot has it), so
diff --git a/apps/ggml/Q5_0_APPROXIMATION_PLAN.md b/apps/ggml/Q5_0_APPROXIMATION_PLAN.md
new file mode 100644
index 000000000000..0b3d3c7eb710
--- /dev/null
+++ b/apps/ggml/Q5_0_APPROXIMATION_PLAN.md
@@ -0,0 +1,150 @@
+# Clean q5_0 Approximation Implementation
+
+## Current status
+
+- Baseline commit: `ca685e94e91c33d924f10c238ffa4ae6dab183a4`
+- Status: complete; all acceptance gates passed
+- [x] Phase 0: confirm clean baseline and create this progress document
+- [x] Phase 1: collect ten paired baseline runs at n=4096
+- [x] Phase 2: add stage-key tracing and reusable core Approximation components
+- [x] Phase 3: add focused core correctness coverage
+- [x] Phase 4: refactor q5_0 composition and schedule lookup
+- [x] Phase 5: correctness, odd-tail, generated-code, and full-suite validation
+- [x] Phase 6: paired performance validation and durable documentation
+
+## Goal and constraints
+
+Refactor q5_0 so its generator contains only the base reduction,
+`approximate_by`, `compute_offline`, and scheduling. Compose all q5_0
+representation logic from reusable core Halide Approximations. Preserve
+correctness, keep median paired performance within 5% of the committed baseline,
+and remain at least 0.90x GGML. q5_1 is explicitly unchanged transitional debt.
+
+## Core Approximation APIs
+
+- Add an opaque, copyable `ApproximationStageKey` to every Approximation
+  instance.
+- Extend traced invocation through `Compose`, `Apply`, `TrustedInverse`, and
+  `Func::approximate_by` so `ApproximationResult` resolves encoded and decoded
+  outputs by `(StageKey, port)` while preserving flat `handles` and `encoded`.
+- Add public standard components in a dedicated header included by `Halide.h`:
+  `StructLayout`, `StorageCast`, `LittleEndianScalarPack`, `BinaryAlphabetPack`,
+  `AdditiveRadixSplit`, `PlanarFieldPack`, `BlockReshape`, and the symmetric
+  block quantizer/policies.
+- Leave ggml compatibility aliases/wrappers so unrelated formats do not migrate.
+- Test nested combinators, repeated types with distinct keys, invalid lookups,
+  encode/decode lookup, and component correctness, including one- and
+  two-dimensional `StructLayout` records.
+
+## q5_0 target composition and schedule
+
+Faithful packed type: `{d: Float16, qh: UInt8[4], qs: UInt8[16]}`.
+
+Outer to inner:
+
+1. `StructLayout`, logical `{qs, qh, d}` to physical fields.
+2. `Apply` `StorageCast<Float32, Float16>` to `d`.
+3. `Apply` `LittleEndianScalarPack<UInt32>` to `qh`.
+4. `Apply` `BinaryAlphabetPack<int8_t>{32, UInt32, -16, 0}` to high
+   contributions.
+5. `Apply` `PlanarFieldPack{4, 16}` to low nibbles.
+6. `Apply` `AdditiveRadixSplit{16, 16}` to signed codes.
+7. Symmetric block quantization, qmax 16, extreme-signed scale selection,
+   truncate-half-up rounding.
+8. `BlockReshape{32, block_indexed}`.
+
+Capture stage keys for reconstructed signed codes (`AdditiveRadixSplit`) and the
+qh word (`LittleEndianScalarPack`), carry them through scheme metadata into
+`VecDotSpec`, and resolve scheduling Funcs from `ApproximationResult`. Replace
+the duplicate q5_0 tail Approximation with stage-scoped eager inlining into the
+tail update. Preserve the measured two-block SDOT schedule and use Func
+identity, not names, for `sdot_partial` exclusions.
+
+## Validation gates
+
+- q5_0 quantize is bit-exact with GGML; dequantize and vec_dot pass tolerances.
+- `kernel-bench --all` has no failures.
+- Odd block counts pass at n=32, 96, 160, 224, and 1056.
+- ARM assembly contains SDOT, one qh word load per block, contiguous LUT loads,
+  no qh byte-load sequence in the main loop, and no accumulator stack spill or
+  one-iteration epilogue loop.
+- Median paired q5_0 is no more than 5% slower than baseline and at least 0.90x
+  GGML; shared formats show no accidental regression.
+- Any compiler optimization is general, separately tested, and
+  target-independent where possible. No new scheduling directive is planned.
+
+## Benchmark experiment policy
+
+Any experimental setup that proves useful or repeatable should be promoted into
+`kernel-bench` as a named mode rather than left as an ad hoc shell recipe. This
+includes scaling/size sweeps such as the q5_0 odd-block checks at n=32, 96, 160,
+224, and 1056. One-off scripts are acceptable for initial exploration, but the
+durable form should make the experiment discoverable, reproducible, and usable
+for future regressions from the benchmark utility itself.
+
+## Baseline results
+
+Ten paired filtered runs, `KERNEL_BENCH_N=4096`, filter
+`q4_0,q4_1,q5_0,q5_1,q8_0`:
+
+Median of ten per-run timings and median paired ratio:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide |
+| ------ | ---------- | ---------- | ------------------ |
+| q4_0   | 94.437 ns  | 96.893 ns  | 0.9726x            |
+| q4_1   | 107.819 ns | 118.424 ns | 0.9020x            |
+| q5_0   | 122.172 ns | 130.259 ns | 0.9379x            |
+| q5_1   | 143.575 ns | 153.717 ns | 0.9339x            |
+| q8_0   | 73.202 ns  | 74.829 ns  | 0.9764x            |
+
+All candidate correctness flags were true. Raw CSV files are in
+`/tmp/q50-baseline.tX9dNF` for this work session.
+
+## Experiment log
+
+| #   | Change                                                                                                                | Correctness                                                                                        | GGML / Halide timings                                             | Generated-code observations                                                                                          | Decision  |
+| --- | --------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | --------- |
+| 0   | Clean committed baseline                                                                                              | All filtered vec_dot checks passed                                                                 | q5_0: 122.172 / 130.259 ns, 0.9379x paired median                 | Existing q5_0 path is the generated-code reference                                                                   | Reference |
+| 1   | Add opaque stage keys and traces through Compose, Apply, TrustedInverse, and approximate_by                           | Nested/repeated/directional/invalid lookup tests pass                                              | Not performance-sensitive                                         | Flat encoded/handle compatibility retained                                                                           | Keep      |
+| 2   | Add public core components; move BlockReshape and symmetric quantization behind ggml compatibility alias/wrapper      | Focused component suite passes, including 1-D and 2-D StructLayout and inline StorageCast rounding | Not measured independently                                        | `strict_float` makes fp16 storage rounding survive eager inlining                                                    | Keep      |
+| 3   | Replace q5_0 legacy split-code composition with the eight reusable stages                                             | q5_0 quantize/dequantize/vec_dot pass                                                              | Focused q5_0 run: 117.7 / 126.2 ns in one sample                  | Faithful UInt8[4] qh lowers through concat_bits to a word load                                                       | Keep      |
+| 4   | Resolve codes/qh by stage key, stop sdot inlining by Func identity, and share/eager-inline the q5_0 tail decode graph | Odd n=32/96/160/224/1056 all pass                                                                  | Included in final medians                                         | Main loop has two blocks in flight, four SDOTs/pair, persistent vector accumulators; tail is scalar and fully inline | Keep      |
+| 5   | Full validation and ten final paired runs after the final StorageCast change                                          | `kernel-bench --all` clean; focused tests and odd sizes pass; all paired flags true                | q5_0: 122.147 / 130.645 ns, 0.9349x; baseline delta +0.30% Halide | One unaligned qh word load/block, contiguous LUT loads, no accumulator spill or one-iteration epilogue               | Final     |
+
+## Final paired results
+
+Median of ten n=4096 paired runs:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide | Halide vs baseline |
+| ------ | ---------- | ---------- | ------------------ | ------------------ |
+| q4_0   | 94.187 ns  | 96.832 ns  | 0.9727x            | -0.06%             |
+| q4_1   | 106.760 ns | 118.423 ns | 0.9015x            | 0.00%              |
+| q5_0   | 122.147 ns | 130.645 ns | 0.9349x            | +0.30%             |
+| q5_1   | 143.728 ns | 153.956 ns | 0.9336x            | +0.16%             |
+| q8_0   | 72.774 ns  | 74.687 ns  | 0.9744x            | -0.19%             |
+
+Negative deltas are improvements. Raw final CSV files are in
+`/tmp/q50-final-strict.vULMku` for this work session.
+
+## Framework/compiler issues
+
+- No compiler change was needed. Existing `LowerStructTypes` handling of
+  `concat_bits()` recovered one unaligned qh word load from the faithful
+  `UInt8[4]` field.
+- The existing Halide build had `WITH_TESTS=OFF`; it was reconfigured with tests
+  enabled to build the two focused correctness targets.
+- Installing only the development component updates headers and GenGen but not
+  the changed shared library; a full `cmake --install` was required before the
+  standalone ggml generator could link the new stage lookup methods.
+- A plain fp32-to-fp16-to-fp32 cast chain can fuse away when fully inlined.
+  `StorageCast` uses `strict_float` on both conversions so storage rounding is
+  schedule-independent; its correctness test intentionally leaves the stage
+  inline. q5_0 also materializes the fp16 value in its packed struct field.
+
+## Final follow-up items
+
+- Add a `kernel-bench` scaling/odd-tail mode that captures the useful n-sweep
+  performed during this work; follow the benchmark experiment policy above for
+  future reusable setups.
+- Remove the legacy q5-specific components and specialized q5_1 reduction after
+  q5_1 is migrated to reusable components; this work intentionally leaves it.
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index 5756e8e5ae9d..410bbc4659d8 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -97,6 +97,12 @@ struct SchemeAndBytes {
     // it -- see VecDotGeneratorBase::configure(). Purely a byte-path,
     // 32-element-block property today (Q8_1); wider layouts leave it false.
     bool has_block_sums = false;
+    // Optional scheduling identities exported by composed schemes. q5_0 uses
+    // these to materialize reconstructed codes and its packed qh word without
+    // relying on generated Func names. Legacy q5_1 intentionally leaves them
+    // undefined until its separate migration.
+    Halide::ApproximationStageKey reconstructed_codes_stage;
+    Halide::ApproximationStageKey packed_high_word_stage;
 };
 
 // Every make_*_scheme() factory below takes a Layout, selecting what its
@@ -140,81 +146,8 @@ enum class Layout { FlatRow,
 // In block-indexed mode a single-extent reshape is a (kk,blk) passthrough,
 // and a multi-extent one collapses the nested dims (elem,l,group,blk) into
 // (kk,blk) -- the only difference from flat mode is folding blk into k or not.
-class BlockReshape : public Halide::Approximation {
-public:
-    explicit BlockReshape(int block_size, bool block_indexed = false)
-        : extents_{block_size}, block_indexed_(block_indexed) {
-    }
-    explicit BlockReshape(std::vector<int> extents, bool block_indexed = false)
-        : extents_(std::move(extents)), block_indexed_(block_indexed) {
-    }
-
-    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
-        using namespace Halide;
-        Func flat = inputs[0];  // f(k), or f(kk, blk) when block_indexed_
-        std::vector<Var> dims = block_vars();
-        Var blk("blk");
-
-        // packed(d0, d1, ..., blk) = flat(<within>), within = d0 + e0*d1 + ...
-        Expr within = cast<int>(0);
-        int stride = 1;
-        for (size_t i = 0; i < dims.size(); i++) {
-            within += dims[i] * stride;
-            stride *= extents_[i];
-        }
-        std::vector<Var> args = dims;
-        args.push_back(blk);
-        Func packed("block_reshape_packed");
-        packed(args) = block_indexed_ ? flat(within, blk) : flat(blk * block_size() + within);
-        return {{packed}, {}};
-    }
-
-    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
-        using namespace Halide;
-        Func packed = encoded[0];
-        Var k("k"), kk("kk"), blk("blk");
-
-        // Read packed(within%e0, (within/e0)%e1, ..., block); the within-block
-        // index and block index come from either a flat k or a (kk, blk) pair.
-        Expr within = block_indexed_ ? (Expr)kk : k % block_size();
-        Expr block = block_indexed_ ? (Expr)blk : k / block_size();
-        std::vector<Expr> args;
-        Expr rem = within;
-        for (int e : extents_) {
-            args.push_back(rem % e);
-            rem = rem / e;
-        }
-        args.push_back(block);
-        Func out("block_reshape_unpacked");
-        if (block_indexed_) {
-            out(kk, blk) = packed(args);
-        } else {
-            out(k) = packed(args);
-        }
-        return {{out}, {}};
-    }
-
-private:
-    std::vector<int> extents_;
-    bool block_indexed_;
-
-    int block_size() const {
-        int p = 1;
-        for (int e : extents_) {
-            p *= e;
-        }
-        return p;
-    }
-    // One Var per within-block dimension; the familiar "kk" in the common
-    // one-dimensional case, "d0"/"d1"/... otherwise.
-    std::vector<Halide::Var> block_vars() const {
-        std::vector<Halide::Var> vs;
-        for (size_t i = 0; i < extents_.size(); i++) {
-            vs.push_back(extents_.size() == 1 ? Halide::Var("kk") : Halide::Var("d" + std::to_string(i)));
-        }
-        return vs;
-    }
-};
+// Compatibility alias: the implementation is now a public core component.
+using BlockReshape = Halide::BlockReshape;
 
 // ---------------------------------------------------------------------------
 // Lossless block-layout relayouts (Reblock, and the repack Interleave below).
@@ -361,95 +294,40 @@ enum class ScaleAnchor { AbsMax,
 // decode(): {codes, scale} -> cast<float>(codes) * scale -- this half is
 // exactly the same regardless of rounding/anchor (both Q4_0's and Q8_0's
 // existing hand-written dequantize math already reduce to this one formula).
-class SymmetricAffineQuantize : public Halide::Approximation {
+class SymmetricAffineQuantize : public Halide::SymmetricBlockQuantize {
 public:
     SymmetricAffineQuantize(int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor)
-        : block_size_(block_size), qmax_(qmax), rounding_(rounding), anchor_(anchor) {
+        : Halide::SymmetricBlockQuantize(block_size, qmax, core_rounding(rounding), core_anchor(anchor)) {
     }
 
-    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
-        using namespace Halide;
-        Func block = inputs[0];  // block(kk, blk)
-        Var kk("kk"), blk("blk");
-        RDom r(0, block_size_, "r");
-
-        Func stat("affine_quantize_stat");
-        Func scale("affine_quantize_scale");
-        Func id("affine_quantize_id");
-        auto define_extreme_signed_stat = [&]() {
-            stat(blk) = Tuple(0.0f, 0.0f);  // {amax, extreme_signed}
-            Expr v = block(r, blk);
-            Expr take = abs(v) > stat(blk)[0];
-            stat(blk) = Tuple(select(take, abs(v), stat(blk)[0]),
-                              select(take, v, stat(blk)[1]));
-        };
-        if (anchor_ == ScaleAnchor::AbsMax) {
-            stat(blk) = 0.0f;
-            stat(blk) = max(stat(blk), abs(block(r, blk)));
-            scale(blk) = stat(blk) / (float)qmax_;
-            id(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
-        } else if (anchor_ == ScaleAnchor::ExtremeSignedValue) {
-            define_extreme_signed_stat();
-            scale(blk) = stat(blk)[1] * (-1.0f / (float)qmax_);
-            id(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
-        } else if (anchor_ == ScaleAnchor::MeanAbs) {
-            stat(blk) = 0.0f;
-            stat(blk) += abs(block(r, blk));
-            scale(blk) = stat(blk) / (float)block_size_;
-            id(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
-        } else {  // ExtremeSignedValueTwoStep
-            define_extreme_signed_stat();
-            // `id` (== GGML's `iscale`) is computed FIRST here, and `scale`
-            // is derived from it -- the reverse order of every other
-            // anchor above -- because GGML's own reference computes
-            // `iscale = -qmax/extreme` then `d = 1/iscale` as two
-            // *separate* divisions, and quantizes using `iscale` directly.
-            // Re-deriving `id` as `1/scale` afterward (like every other
-            // anchor does) would round through an extra reciprocal
-            // (`1/(1/iscale)`) that isn't guaranteed to reproduce `iscale`
-            // bit-for-bit.
-            id(blk) = select(stat(blk)[0] == 0.0f, 0.0f, (-1.0f * (float)qmax_) / stat(blk)[1]);
-            scale(blk) = select(id(blk) != 0.0f, 1.0f / id(blk), 0.0f);
+private:
+    static Halide::BlockRoundingMode core_rounding(RoundingMode mode) {
+        switch (mode) {
+        case RoundingMode::Nearest:
+            return Halide::BlockRoundingMode::Nearest;
+        case RoundingMode::TruncateHalfUpWithOffset:
+            return Halide::BlockRoundingMode::TruncateHalfUpWithOffset;
+        case RoundingMode::SignOnly:
+            return Halide::BlockRoundingMode::SignOnly;
+        case RoundingMode::NearestEvenClampedHigh:
+            return Halide::BlockRoundingMode::NearestEvenClampedHigh;
         }
-        // stat has an update definition, so it must be scheduled somewhere
-        // (Halide can't inline it) -- like SymmetricRowQuantize's `amax` in
-        // approximation_composition.cpp, that's left to the caller via
-        // `handles`, not decided here.
-
-        Expr x0 = block(kk, blk) * id(blk);
-
-        Func codes("affine_quantize_codes");
-        if (rounding_ == RoundingMode::Nearest) {
-            // Matches Q8_0's actual (bit-exact-verified) reference: no
-            // explicit clamp, since id was derived so |x0| doesn't exceed
-            // qmax in practice.
-            codes(kk, blk) = cast<int8_t>(round(x0));
-        } else if (rounding_ == RoundingMode::TruncateHalfUpWithOffset) {
-            Expr raw = cast<int32_t>(cast<int8_t>(x0 + (float)qmax_ + 0.5f));
-            codes(kk, blk) = cast<int8_t>(min(raw, 2 * qmax_ - 1) - qmax_);
-        } else if (rounding_ == RoundingMode::SignOnly) {
-            codes(kk, blk) = cast<int8_t>(select(block(kk, blk) >= 0.0f, 1, -1));
-        } else {  // NearestEvenClampedHigh
-            Expr q_raw = nearest_int(x0);
-            codes(kk, blk) = cast<int8_t>(min(qmax_, q_raw));
+        _halide_internal_error << "Unknown symmetric rounding mode\n";
+    }
+
+    static Halide::BlockScaleAnchor core_anchor(ScaleAnchor anchor) {
+        switch (anchor) {
+        case ScaleAnchor::AbsMax:
+            return Halide::BlockScaleAnchor::AbsMax;
+        case ScaleAnchor::ExtremeSignedValue:
+            return Halide::BlockScaleAnchor::ExtremeSignedValue;
+        case ScaleAnchor::MeanAbs:
+            return Halide::BlockScaleAnchor::MeanAbs;
+        case ScaleAnchor::ExtremeSignedValueTwoStep:
+            return Halide::BlockScaleAnchor::ExtremeSignedValueTwoStep;
         }
-
-        return {{codes, scale}, {stat}};
+        _halide_internal_error << "Unknown symmetric scale anchor\n";
     }
-
-    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
-        using namespace Halide;
-        Func codes = encoded[0], scale = encoded[1];
-        Var kk("kk"), blk("blk");
-        Func dequantized("affine_dequantized");
-        dequantized(kk, blk) = cast<float>(codes(kk, blk)) * scale(blk);
-        return {{dequantized}, {}};
-    }
-
-private:
-    int block_size_, qmax_;
-    RoundingMode rounding_;
-    ScaleAnchor anchor_;
 };
 
 // How AffineQuantize rounds+truncates code = round((x-min)*id) into its
@@ -2852,16 +2730,31 @@ inline SchemeAndBytes make_symmetric_5bit_block_scheme(int block_size, int qmax,
                                                        Layout layout = Layout::FlatRow) {
     using namespace Halide;
     _halide_user_assert(block_size == 32) << "The Q5 struct layout requires a 32-element block.\n";
-    Type block_type = Type::Struct({{"d", Float(16)}, {"qh", UInt(32)}, {"qs", UInt(8), 16}});
-    CodePackField code = make_code_pack(block_size, /*code_bits=*/5, qmax);
-    return {std::make_unique<Compose>(
-                Q5StructBlockLayout{block_type, /*affine=*/false},
-                Apply{0, std::move(code.pack)},
-                SymmetricAffineQuantize{block_size, qmax, RoundingMode::TruncateHalfUpWithOffset,
-                                        ScaleAnchor::ExtremeSignedValue},
-                BlockReshape{block_size, layout == Layout::BlockIndexed}),
-            block_type.bytes(),
-            block_type};
+    _halide_user_assert(qmax == 16) << "The q5_0 additive representation requires qmax 16.\n";
+
+    // Faithful physical declaration: qh remains an array of four bytes in the
+    // public type. LittleEndianScalarPack's concat_bits decode lets
+    // LowerStructTypes recover a single unaligned word load from those bytes.
+    Type block_type = Type::Struct({{"d", Float(16)}, {"qh", UInt(8), 4}, {"qs", UInt(8), 16}});
+
+    Halide::LittleEndianScalarPack<uint32_t> qh_word;
+    Halide::AdditiveRadixSplit radix_split(16, 16);
+    ApproximationStageKey qh_word_key = qh_word.stage_key();
+    ApproximationStageKey reconstructed_codes_key = radix_split.stage_key();
+
+    auto scheme = std::make_unique<Compose>(
+        Halide::StructLayout{block_type, {"qs", "qh", "d"}},
+        Apply{2, Halide::StorageCast<float, float16_t>{}},
+        Apply{1, std::move(qh_word)},
+        Apply{1, Halide::BinaryAlphabetPack<int8_t>{32, UInt(32), -16, 0}},
+        Apply{0, Halide::PlanarFieldPack{4, 16}},
+        Apply{0, /*encode_arity=*/1, /*decode_arity=*/2, std::move(radix_split)},
+        Halide::SymmetricBlockQuantize{block_size, qmax,
+                                       Halide::BlockRoundingMode::TruncateHalfUpWithOffset,
+                                       Halide::BlockScaleAnchor::ExtremeSignedValue},
+        Halide::BlockReshape{block_size, layout == Layout::BlockIndexed});
+    return {std::move(scheme), block_type.bytes(), block_type, false,
+            reconstructed_codes_key, qh_word_key};
 }
 
 // Affine quantize (like make_affine_block_scheme()) but 5-bit -- likewise now
diff --git a/apps/ggml/halide/sdot_schedule.h b/apps/ggml/halide/sdot_schedule.h
index 3d279de9a71b..e2ae4c2ed8ce 100644
--- a/apps/ggml/halide/sdot_schedule.h
+++ b/apps/ggml/halide/sdot_schedule.h
@@ -39,7 +39,7 @@ namespace ggml_halide {
 // it is d*d_act * sum(code*act) + m*d_act * sum(act), and hoist_invariants()
 // gives each term its own accumulator -- both with integer bodies, so both reach
 // SDOT. That is ggml's own decomposition of the affine formats.
-// `keep_out` names decode-chain Funcs that must NOT be flattened -- a caller
+// `keep_out` identifies decode-chain Funcs that must NOT be flattened -- a caller
 // that has scheduled one as a materialization boundary (e.g. Q5_x's
 // reconstructed `combine_bits_code`, computed once per block so its qh
 // byte->bits table read is a contiguous load instead of a per-lane gather).
@@ -50,13 +50,15 @@ inline std::vector<Halide::Func> sdot_partial(Halide::Func &acc,
                                               const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
                                               const std::vector<Halide::ApproximationResult> &operands,
                                               bool distribute = false,
-                                              const std::vector<std::string> &keep_out = {}) {
+                                              const std::vector<Halide::Func> &keep_out = {}) {
     using namespace Halide;
 
     Func acc_dot = acc.update().rfactor(preserved);
 
     auto excluded = [&](const Func &f) {
-        return std::find(keep_out.begin(), keep_out.end(), f.name()) != keep_out.end();
+        return std::any_of(keep_out.begin(), keep_out.end(), [&](const Func &kept) {
+            return kept.defined() && kept.function().same_as(f.function());
+        });
     };
 
     std::vector<Func> decode_funcs;
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index 97517ab28358..c2da055da7e4 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -82,6 +82,7 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         ScheduleKind sched;
         bool distribute = false;   // set for the affine (offset-carrying) weights
         Halide::Type weight_type;  // set -> weight blocks are a 1-D Type::Struct buffer
+        Halide::ApproximationStageKey reconstructed_codes_stage, packed_high_word_stage;
         switch (w_kind.value()) {
         case WKind::Symmetric: {
             // Struct-typed weight blocks (`{fp16 d; uint8 qs[...]}`); SDOT still
@@ -113,6 +114,8 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
             SchemeAndBytes sb = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed);
             wc = std::move(sb.scheme);
             weight_type = sb.block_type;
+            reconstructed_codes_stage = sb.reconstructed_codes_stage;
+            packed_high_word_stage = sb.packed_high_word_stage;
             wb = 2 + 4 + wbs / 2;
             // Q5_0's 5-bit code is assembled via CombineBits (nibble |
             // (high_bit << 4)); that reconstruction is all inside the (r.x-
@@ -168,7 +171,7 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         // weight operand is struct-typed here.
         const bool five_bit = w_kind.value() == WKind::Symmetric5Bit || w_kind.value() == WKind::Affine5Bit;
         return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}, act_has_block_sums,
-                five_bit ? 2 : 4};
+                five_bit ? 2 : 4, reconstructed_codes_stage, packed_high_word_stage};
     }
 };
 
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 569d265202a0..0fbed23a4a18 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -75,6 +75,8 @@ struct VecDotSpec {
     // ordinary q4/q8 paths prefer four; q5's table-expansion live ranges make
     // two faster and match GGML's paired-block loop.
     int unroll_blocks = 4;
+    Halide::ApproximationStageKey reconstructed_codes_stage;
+    Halide::ApproximationStageKey packed_high_word_stage;
 };
 
 template<typename Derived>
@@ -128,6 +130,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         // an exactly divisible extent and a second update sweeps the remainder
         // at the default schedule (at most unroll_blocks - 1 blocks).
         const bool sdot = spec.sched == ScheduleKind::SDOT;
+        const bool keyed_q5 = spec.reconstructed_codes_stage.defined();
         Expr nblocks = x_blocks.dim(wt_struct ? 0 : 1).extent();
         Expr main_blocks = sdot ? (nblocks / unroll_blocks) * unroll_blocks : nblocks;
 
@@ -147,7 +150,11 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         VecT(kk, blk) = 0.0f;
         RDom r_tail(0, bs, main_blocks, nblocks - main_blocks, "r_tail");
         if (sdot) {
-            Acc() += WtT(r_tail.x, r_tail.y) * VecT(r_tail.x, r_tail.y);
+            // q5_0 shares the main decode graph with its tiny odd-block tail.
+            // The tail update is eagerly inlined below before the reconstructed
+            // codes leaf is materialized for the main paired-block update.
+            Func tail_weight = keyed_q5 ? Wt : WtT;
+            Acc() += tail_weight(r_tail.x, r_tail.y) * VecT(r_tail.x, r_tail.y);
         }
 
         ApproximationResult wt_r = Wt.approximate_by(*spec.weight_codec, {Acc});
@@ -162,15 +169,23 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         std::vector<ImageParam> bind_to = {x_blocks, y_blocks};
         ApproximationResult wtT_r, actT_r;
         if (sdot) {
-            wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
+            if (!keyed_q5) {
+                wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
+            }
             actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
-            to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
+            if (!keyed_q5) {
+                to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
+            }
             to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
-            bind_to.push_back(x_blocks);
+            if (!keyed_q5) {
+                bind_to.push_back(x_blocks);
+            }
             bind_to.push_back(y_blocks);
-            for (Func h : wtT_r.handles) {
-                if (h.has_update_definition()) {
-                    h.compute_root();
+            if (!keyed_q5) {
+                for (Func h : wtT_r.handles) {
+                    if (h.has_update_definition()) {
+                        h.compute_root();
+                    }
                 }
             }
             for (Func h : actT_r.handles) {
@@ -191,12 +206,36 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         // the block loop, kk split as (byte, pos): pos vectorizes the table load,
         // byte unrolls to a scalar index). The odd-block tail decodes through its
         // own inline chain (see above), so it does not need this buffer.
-        Func codes_leaf, qh_leaf;
-        for (const Func &h : wt_r.handles) {
-            if (h.name() == "combine_bits_code") {
-                codes_leaf = h;
-            } else if (h.name() == "q5_struct_block_qh") {
-                qh_leaf = h;
+        Func codes_leaf = wt_r.decoded_by(spec.reconstructed_codes_stage);
+        Func qh_leaf = wt_r.decoded_by(spec.packed_high_word_stage);
+        if (!keyed_q5) {
+            // Transitional q5_1 debt: its legacy composition has not yet
+            // exported stage keys. Keep its existing generated-name discovery
+            // local to that path; all scheduling boundaries below use Func
+            // identity once resolved.
+            for (const Func &h : wt_r.handles) {
+                if (h.name() == "combine_bits_code") {
+                    codes_leaf = h;
+                } else if (h.name() == "q5_struct_block_qh") {
+                    qh_leaf = h;
+                }
+            }
+        }
+        _halide_internal_assert(!keyed_q5 || (codes_leaf.defined() && qh_leaf.defined()));
+
+        if (keyed_q5) {
+            // This update has at most one block (the paired q5 schedule's odd
+            // remainder). Flatten only its weight decode chain so it reconstructs
+            // eagerly, while the same stage Funcs remain materialization
+            // boundaries in the main update.
+            std::vector<Func> tail_inline = {wt_r.replacement};
+            for (const Func &h : wt_r.handles) {
+                if (h.function().can_be_inlined()) {
+                    tail_inline.push_back(h);
+                }
+            }
+            for (size_t pass = 0; pass < tail_inline.size(); ++pass) {
+                Acc.update(1).eager_inline(tail_inline);
             }
         }
         auto schedule_codes = [&](LoopLevel level) {
@@ -277,7 +316,7 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             dot_acc.update(0).eager_inline(wt_product);
             Var lane("lane");
             std::vector<Func> dot_i32 = sdot_partial(dot_acc, {{rxo, lane}, {rd.y, u}},
-                                                     {act_r}, false, {codes_leaf.name()});
+                                                     {act_r}, false, {codes_leaf});
 
             RVar ryo("ryo"), ryi("ryi");
             Var lv("lv"), bacc("bacc");
@@ -433,9 +472,9 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // their per-block scales out of the surviving rxi reduction, leaving
             // the scale-free Int(32) dot. See sdot_schedule.h.
             Var lane("lane");
-            std::vector<std::string> keep_out;
+            std::vector<Func> keep_out;
             if (codes_leaf.defined()) {
-                keep_out.push_back(codes_leaf.name());
+                keep_out.push_back(codes_leaf);
             }
             std::vector<Func> Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms, keep_out);
 
diff --git a/src/Approximation.cpp b/src/Approximation.cpp
index dc7777b236fe..ba02ed6f1644 100644
--- a/src/Approximation.cpp
+++ b/src/Approximation.cpp
@@ -4,13 +4,41 @@
 
 namespace Halide {
 
+namespace {
+
+Func find_stage_output(const std::vector<ApproximationStageOutputs> &outputs,
+                       const ApproximationStageKey &stage, size_t port) {
+    if (!stage.defined()) {
+        return Func();
+    }
+    for (auto it = outputs.rbegin(); it != outputs.rend(); ++it) {
+        if (it->stage == stage) {
+            return port < it->ports.size() ? it->ports[port] : Func();
+        }
+    }
+    return Func();
+}
+
+}  // namespace
+
+Func ApproximationResult::encoded_by(const ApproximationStageKey &stage, size_t port) const {
+    return find_stage_output(encoded_stage_outputs, stage, port);
+}
+
+Func ApproximationResult::decoded_by(const ApproximationStageKey &stage, size_t port) const {
+    return find_stage_output(decoded_stage_outputs, stage, port);
+}
+
 EncodeResult Compose::encode(std::vector<Func> inputs) {
     user_assert(!stages_.empty()) << "Compose::encode: no stages\n";
 
     std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
     std::vector<Func> current = std::move(inputs);
     for (int i = (int)stages_.size() - 1; i >= 0; i--) {
         EncodeResult r = stages_[i]->encode(std::move(current));
+        stage_outputs.insert(stage_outputs.end(), r.stage_outputs.begin(), r.stage_outputs.end());
+        stage_outputs.push_back({stages_[i]->stage_key(), r.encoded});
         if (i > 0) {
             // Not the final (outermost) stage -- its encoded output is an
             // intermediate between stages, so it needs scheduling like any
@@ -21,23 +49,26 @@ EncodeResult Compose::encode(std::vector<Func> inputs) {
         handles.insert(handles.end(), r.handles.begin(), r.handles.end());
         current = std::move(r.encoded);
     }
-    return {current, handles};
+    return {current, handles, stage_outputs};
 }
 
 DecodeResult Compose::decode(std::vector<Func> encoded) {
     user_assert(!stages_.empty()) << "Compose::decode: no stages\n";
 
     std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
     std::vector<Func> current = std::move(encoded);
     for (int i = 0; i < (int)stages_.size(); i++) {
         DecodeResult r = stages_[i]->decode(std::move(current));
+        stage_outputs.insert(stage_outputs.end(), r.stage_outputs.begin(), r.stage_outputs.end());
+        stage_outputs.push_back({stages_[i]->stage_key(), r.decoded});
         if (i + 1 < (int)stages_.size()) {
             handles.insert(handles.end(), r.decoded.begin(), r.decoded.end());
         }
         handles.insert(handles.end(), r.handles.begin(), r.handles.end());
         current = std::move(r.decoded);
     }
-    return {current, handles};
+    return {current, handles, stage_outputs};
 }
 
 EncodeResult Apply::encode(std::vector<Func> inputs) {
@@ -50,7 +81,8 @@ EncodeResult Apply::encode(std::vector<Func> inputs) {
     std::vector<Func> encoded(inputs.begin(), inputs.begin() + idx_);
     encoded.insert(encoded.end(), inner_result.encoded.begin(), inner_result.encoded.end());
     encoded.insert(encoded.end(), inputs.begin() + idx_ + encode_arity_, inputs.end());
-    return {encoded, inner_result.handles};
+    inner_result.stage_outputs.push_back({inner_->stage_key(), inner_result.encoded});
+    return {encoded, inner_result.handles, inner_result.stage_outputs};
 }
 
 DecodeResult Apply::decode(std::vector<Func> encoded) {
@@ -63,15 +95,20 @@ DecodeResult Apply::decode(std::vector<Func> encoded) {
     std::vector<Func> decoded(encoded.begin(), encoded.begin() + idx_);
     decoded.insert(decoded.end(), inner_result.decoded.begin(), inner_result.decoded.end());
     decoded.insert(decoded.end(), encoded.begin() + idx_ + decode_arity_, encoded.end());
-    return {decoded, inner_result.handles};
+    inner_result.stage_outputs.push_back({inner_->stage_key(), inner_result.decoded});
+    return {decoded, inner_result.handles, inner_result.stage_outputs};
 }
 
 EncodeResult TrustedInverse::encode(std::vector<Func> inputs) {
-    return encoder_->encode(std::move(inputs));
+    EncodeResult r = encoder_->encode(std::move(inputs));
+    r.stage_outputs.push_back({encoder_->stage_key(), r.encoded});
+    return r;
 }
 
 DecodeResult TrustedInverse::decode(std::vector<Func> encoded) {
-    return decoder_->decode(std::move(encoded));
+    DecodeResult r = decoder_->decode(std::move(encoded));
+    r.stage_outputs.push_back({decoder_->stage_key(), r.decoded});
+    return r;
 }
 
 }  // namespace Halide
diff --git a/src/Approximation.h b/src/Approximation.h
index d11ba0b65b47..fc4d5e7dbb5c 100644
--- a/src/Approximation.h
+++ b/src/Approximation.h
@@ -10,6 +10,7 @@
  */
 
 #include <memory>
+#include <string>
 #include <type_traits>
 #include <utility>
 #include <vector>
@@ -18,6 +19,42 @@
 
 namespace Halide {
 
+/** An opaque identity for one Approximation stage. Keys are cheap to copy and
+ * remain stable when an Approximation is moved into Compose/Apply. A copied
+ * Approximation deliberately retains its key: the key identifies the logical
+ * stage selected by the caller, not a particular C++ address. */
+class ApproximationStageKey {
+public:
+    ApproximationStageKey() = default;
+
+    bool defined() const {
+        return token_ != nullptr;
+    }
+
+    friend bool operator==(const ApproximationStageKey &a, const ApproximationStageKey &b) {
+        return a.token_ == b.token_;
+    }
+    friend bool operator!=(const ApproximationStageKey &a, const ApproximationStageKey &b) {
+        return !(a == b);
+    }
+
+private:
+    explicit ApproximationStageKey(std::shared_ptr<const char> token)
+        : token_(std::move(token)) {
+    }
+
+    std::shared_ptr<const char> token_;
+    friend class Approximation;
+};
+
+/** The ports produced by one stage during encode or decode. This trace is
+ * supplemental scheduling metadata; it does not alter the signature contract
+ * or the legacy flat handle lists. */
+struct ApproximationStageOutputs {
+    ApproximationStageKey stage;
+    std::vector<Func> ports;
+};
+
 /** The result of Approximation::encode(): the Func(s) that make up the
  * signature contract other code is expected to consume, plus any extra
  * intermediate Funcs ("handles") that have no meaning outside scheduling
@@ -26,6 +63,7 @@ namespace Halide {
 struct EncodeResult {
     std::vector<Func> encoded;
     std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
 };
 
 /** The result of Approximation::decode(): decoded is the round-trip
@@ -37,6 +75,7 @@ struct EncodeResult {
 struct DecodeResult {
     std::vector<Func> decoded;
     std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
 };
 
 /** Approximation is the base class for a lossy, quantified transformation
@@ -63,8 +102,15 @@ struct DecodeResult {
  * for splicing an Approximation into an existing call graph. */
 class Approximation {
 public:
+    Approximation()
+        : stage_key_(std::make_shared<const char>(0)) {
+    }
     virtual ~Approximation() = default;
 
+    ApproximationStageKey stage_key() const {
+        return stage_key_;
+    }
+
     /** Produce the encoded form of `inputs`. EncodeResult::encoded's
      * elements are not required to have the same type, dimensionality, or
      * count as `inputs` -- an Approximation is free to choose a packed
@@ -78,6 +124,9 @@ class Approximation {
      * encoded form. See DecodeResult for the constraint on `decoded`'s
      * size, which depends on how this Approximation is used. */
     virtual DecodeResult decode(std::vector<Func> encoded) = 0;
+
+private:
+    ApproximationStageKey stage_key_;
 };
 
 namespace Internal {
@@ -121,6 +170,13 @@ struct ApproximationResult {
      * -- without calling Approximation::encode() themselves. */
     std::vector<Func> encoded;
     std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> encoded_stage_outputs;
+    std::vector<ApproximationStageOutputs> decoded_stage_outputs;
+
+    /** Return a stage output port, or an undefined Func if the key was not
+     * invoked in this direction or the port is out of range. */
+    Func encoded_by(const ApproximationStageKey &stage, size_t port = 0) const;
+    Func decoded_by(const ApproximationStageKey &stage, size_t port = 0) const;
 };
 
 /** Sequentially composes any number of Approximations into a pipeline:
@@ -273,11 +329,15 @@ class Choose : public Approximation {
     }
 
     EncodeResult encode(std::vector<Func> inputs) {
-        return chosen_->encode(std::move(inputs));
+        EncodeResult r = chosen_->encode(std::move(inputs));
+        r.stage_outputs.push_back({chosen_->stage_key(), r.encoded});
+        return r;
     }
 
     DecodeResult decode(std::vector<Func> encoded) {
-        return chosen_->decode(std::move(encoded));
+        DecodeResult r = chosen_->decode(std::move(encoded));
+        r.stage_outputs.push_back({chosen_->stage_key(), r.decoded});
+        return r;
     }
 
 private:
diff --git a/src/ApproximationComponents.h b/src/ApproximationComponents.h
new file mode 100644
index 000000000000..1a71ca061e1d
--- /dev/null
+++ b/src/ApproximationComponents.h
@@ -0,0 +1,521 @@
+#ifndef HALIDE_APPROXIMATION_COMPONENTS_H
+#define HALIDE_APPROXIMATION_COMPONENTS_H
+
+/** \file
+ * Reusable Approximation building blocks for typed records, scalar storage,
+ * fixed-width bit fields, and block quantization.
+ */
+
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "Approximation.h"
+#include "Buffer.h"
+#include "IROperator.h"
+#include "RDom.h"
+
+namespace Halide {
+
+namespace Internal {
+
+inline std::vector<Var> approximation_component_vars(int dimensions, const std::string &prefix) {
+    std::vector<Var> vars;
+    vars.reserve(dimensions);
+    for (int i = 0; i < dimensions; ++i) {
+        vars.emplace_back(prefix + std::to_string(i));
+    }
+    return vars;
+}
+
+inline std::vector<Expr> approximation_component_exprs(const std::vector<Var> &vars) {
+    return std::vector<Expr>(vars.begin(), vars.end());
+}
+
+}  // namespace Internal
+
+/** Losslessly reshape a flat row into fixed-size records. In block-indexed
+ * mode the flat side is `(within, record)` rather than a single flat index. */
+class BlockReshape : public Approximation {
+public:
+    explicit BlockReshape(int block_size, bool block_indexed = false)
+        : extents_{block_size}, block_indexed_(block_indexed) {
+    }
+    explicit BlockReshape(std::vector<int> extents, bool block_indexed = false)
+        : extents_(std::move(extents)), block_indexed_(block_indexed) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1) << "BlockReshape::encode expects one input\n";
+        Func flat = inputs[0];
+        std::vector<Var> dims = block_vars();
+        Var blk("blk");
+        Expr within = cast<int>(0);
+        int stride = 1;
+        for (size_t i = 0; i < dims.size(); ++i) {
+            within += dims[i] * stride;
+            stride *= extents_[i];
+        }
+        std::vector<Var> args = dims;
+        args.push_back(blk);
+        Func packed("block_reshape_packed");
+        packed(args) = block_indexed_ ? flat(within, blk) : flat(blk * block_size() + within);
+        return {{packed}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1) << "BlockReshape::decode expects one input\n";
+        Func packed = encoded[0];
+        Var k("k"), kk("kk"), blk("blk");
+        Expr within = block_indexed_ ? Expr(kk) : k % block_size();
+        Expr block = block_indexed_ ? Expr(blk) : k / block_size();
+        std::vector<Expr> args;
+        Expr rem = within;
+        for (int extent : extents_) {
+            args.push_back(rem % extent);
+            rem /= extent;
+        }
+        args.push_back(block);
+        Func out("block_reshape_unpacked");
+        if (block_indexed_) {
+            out(kk, blk) = packed(args);
+        } else {
+            out(k) = packed(args);
+        }
+        return {{out}, {}};
+    }
+
+private:
+    std::vector<int> extents_;
+    bool block_indexed_;
+
+    int block_size() const {
+        int size = 1;
+        for (int extent : extents_) {
+            size *= extent;
+        }
+        return size;
+    }
+
+    std::vector<Var> block_vars() const {
+        std::vector<Var> vars;
+        for (size_t i = 0; i < extents_.size(); ++i) {
+            vars.emplace_back(extents_.size() == 1 ? "kk" : "d" + std::to_string(i));
+        }
+        return vars;
+    }
+};
+
+/** Map consecutive logical Func slots to named fields of an exact struct
+ * type. Scalar fields have `record_dimensions` dimensions; array fields have
+ * an additional leading element dimension. */
+class StructLayout : public Approximation {
+public:
+    StructLayout(Type record_type, std::vector<std::string> logical_fields,
+                 int record_dimensions = 1)
+        : record_type_(record_type), logical_fields_(std::move(logical_fields)),
+          record_dimensions_(record_dimensions) {
+        user_assert(record_type_.is_struct()) << "StructLayout requires a struct Type\n";
+        user_assert(record_dimensions_ > 0) << "StructLayout record dimensionality must be positive\n";
+        const StructTypeInfo *info = record_type_.struct_type();
+        user_assert(logical_fields_.size() == info->fields.size())
+            << "StructLayout requires exactly one logical slot per physical field\n";
+        for (const std::string &name : logical_fields_) {
+            int matches = 0;
+            for (const StructField &field : info->fields) {
+                matches += field.name == name;
+            }
+            user_assert(matches == 1) << "StructLayout: no unique field named '" << name << "'\n";
+            int logical_matches = 0;
+            for (const std::string &logical_name : logical_fields_) {
+                logical_matches += logical_name == name;
+            }
+            user_assert(logical_matches == 1) << "StructLayout: duplicate logical field '" << name << "'\n";
+        }
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == logical_fields_.size())
+            << "StructLayout::encode input count does not match logical field count\n";
+        const StructTypeInfo *info = record_type_.struct_type();
+        std::vector<Var> records = Internal::approximation_component_vars(record_dimensions_, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        std::vector<Expr> values;
+        for (const StructField &field : info->fields) {
+            size_t slot = logical_slot(field.name);
+            Func input = inputs[slot];
+            user_assert(input.outputs() == 1 && input.types()[0] == field.type)
+                << "StructLayout field '" << field.name << "' requires exact type " << field.type
+                << " but slot " << slot << " has " << input.types()[0] << "\n";
+            int extent = field.array_extent.value_or(1);
+            user_assert(input.dimensions() == record_dimensions_ + (field.array_extent ? 1 : 0))
+                << "StructLayout field '" << field.name << "' has the wrong dimensionality\n";
+            for (int element = 0; element < extent; ++element) {
+                std::vector<Expr> args = record_args;
+                if (field.array_extent) {
+                    args.insert(args.begin(), element);
+                }
+                values.push_back(input(args));
+            }
+        }
+        Func packed("struct_layout_packed");
+        packed(records) = pack_struct(record_type_, values);
+        return {{packed}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].outputs() == 1 &&
+                    encoded[0].types()[0] == record_type_)
+            << "StructLayout::decode requires one Func of the exact record type\n";
+        Func packed = encoded[0];
+        std::vector<Var> records = Internal::approximation_component_vars(record_dimensions_, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        Expr record = packed(record_args);
+        std::vector<Func> outputs;
+        outputs.reserve(logical_fields_.size());
+        for (const std::string &name : logical_fields_) {
+            const StructField &physical = physical_field(name);
+            Func output("struct_layout_" + name);
+            if (physical.array_extent) {
+                Var element("element");
+                std::vector<Var> args = records;
+                args.insert(args.begin(), element);
+                output(args) = field(record, name)[element];
+            } else {
+                output(records) = field(record, name);
+            }
+            outputs.push_back(output);
+        }
+        return {outputs, {}};
+    }
+
+private:
+    Type record_type_;
+    std::vector<std::string> logical_fields_;
+    int record_dimensions_;
+
+    size_t logical_slot(const std::string &name) const {
+        for (size_t i = 0; i < logical_fields_.size(); ++i) {
+            if (logical_fields_[i] == name) {
+                return i;
+            }
+        }
+        user_error << "StructLayout internal field mapping failure\n";
+        return 0;
+    }
+
+    const StructField &physical_field(const std::string &name) const {
+        for (const StructField &field : record_type_.struct_type()->fields) {
+            if (field.name == name) {
+                return field;
+            }
+        }
+        user_error << "StructLayout internal physical field failure\n";
+        return record_type_.struct_type()->fields[0];
+    }
+};
+
+/** Explicit numeric conversion between a decoded computation type and the
+ * exact type stored in a representation. */
+template<typename Decoded, typename Storage>
+class StorageCast : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].types() == std::vector<Type>{type_of<Decoded>()})
+            << "StorageCast::encode input type mismatch\n";
+        Func input = inputs[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "cast");
+        Func stored("storage_cast_stored");
+        stored(args) = strict_float(cast<Storage>(input(Internal::approximation_component_exprs(args))));
+        return {{stored}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{type_of<Storage>()})
+            << "StorageCast::decode storage type mismatch\n";
+        Func input = encoded[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "cast");
+        Func decoded("storage_cast_decoded");
+        decoded(args) = strict_float(cast<Decoded>(input(Internal::approximation_component_exprs(args))));
+        return {{decoded}, {}};
+    }
+};
+
+/** Convert a scalar integral word per record to/from a leading little-endian
+ * byte dimension. Decode deliberately uses concat_bits so struct lowering and
+ * ordinary byte buffers share the same wide-load optimization path. */
+template<typename Word>
+class LittleEndianScalarPack : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].types() == std::vector<Type>{type_of<Word>()})
+            << "LittleEndianScalarPack::encode word type mismatch\n";
+        Func word = inputs[0];
+        std::vector<Var> records = Internal::approximation_component_vars(word.dimensions(), "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        Var byte("byte");
+        std::vector<Var> args = records;
+        args.insert(args.begin(), byte);
+        Func bytes("little_endian_scalar_bytes");
+        Expr bits = cast(type_of<Word>(), word(record_args));
+        bytes(args) = cast<uint8_t>(bits >> (byte * 8));
+        return {{bytes}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{UInt(8)} &&
+                    encoded[0].dimensions() >= 2)
+            << "LittleEndianScalarPack::decode requires byte arrays per record\n";
+        Func bytes = encoded[0];
+        std::vector<Var> records = Internal::approximation_component_vars(bytes.dimensions() - 1, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        std::vector<Expr> pieces;
+        for (size_t i = 0; i < sizeof(Word); ++i) {
+            std::vector<Expr> args = record_args;
+            args.insert(args.begin(), (int)i);
+            pieces.push_back(bytes(args));
+        }
+        Func word("little_endian_scalar_word");
+        word(records) = cast<Word>(concat_bits(pieces));
+        return {{word}, {}};
+    }
+};
+
+/** Pack a fixed vector containing exactly two values into an integer word.
+ * Decode uses an embedded 8x256 byte-expansion LUT, allowing each source byte
+ * to expand through one contiguous eight-byte load. */
+template<typename Value>
+class BinaryAlphabetPack : public Approximation {
+public:
+    BinaryAlphabetPack(int vector_size, Type word_type, Value zero_value, Value one_value)
+        : vector_size_(vector_size), word_type_(word_type), zero_value_(zero_value), one_value_(one_value),
+          expansion_(8, 256) {
+        user_assert(word_type_.is_uint() && word_type_.bits() >= vector_size_)
+            << "BinaryAlphabetPack word is too small for its vector\n";
+        user_assert(vector_size_ > 0 && vector_size_ % 8 == 0)
+            << "BinaryAlphabetPack vector size must be a positive multiple of eight\n";
+        for (int byte = 0; byte < 256; ++byte) {
+            for (int bit = 0; bit < 8; ++bit) {
+                expansion_(bit, byte) = (byte & (1 << bit)) ? one_value_ : zero_value_;
+            }
+        }
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].dimensions() >= 2)
+            << "BinaryAlphabetPack::encode requires (element, record...)\n";
+        Func values = inputs[0];
+        int records_n = values.dimensions() - 1;
+        std::vector<Var> records = Internal::approximation_component_vars(records_n, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        RDom bit(0, vector_size_, "binary_bit");
+        std::vector<Expr> value_args = record_args;
+        value_args.insert(value_args.begin(), bit);
+        Expr value = values(value_args);
+        Func word("binary_alphabet_word");
+        word(records) = cast(word_type_, 0);
+        word(records) = word(record_args) |
+                        select(value == cast<Value>(one_value_),
+                               cast(word_type_, 1) << bit, cast(word_type_, 0));
+        return {{word}, {word}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{word_type_})
+            << "BinaryAlphabetPack::decode word type mismatch\n";
+        Func word = encoded[0];
+        std::vector<Var> records = Internal::approximation_component_vars(word.dimensions(), "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        Var element("element");
+        Expr bits = word(record_args);
+        Expr byte = cast<int32_t>((bits >> ((element / 8) * 8)) & 0xff);
+        std::vector<Var> args = records;
+        args.insert(args.begin(), element);
+        Func values("binary_alphabet_values");
+        values(args) = expansion_(element % 8, byte);
+        return {{values}, {}};
+    }
+
+private:
+    int vector_size_;
+    Type word_type_;
+    Value zero_value_, one_value_;
+    Buffer<Value> expansion_;
+};
+
+/** Split a signed code into an unsigned low digit and an additive weighted
+ * high contribution. The second parameter recenters the signed code before
+ * taking the low digit; decode is simply `code = low + high`. */
+class AdditiveRadixSplit : public Approximation {
+public:
+    AdditiveRadixSplit(int radix, int offset)
+        : radix_(radix), offset_(offset) {
+        user_assert(radix_ > 1 && offset_ >= 0) << "Invalid AdditiveRadixSplit parameters\n";
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1) << "AdditiveRadixSplit::encode expects one code Func\n";
+        Func code = inputs[0];
+        std::vector<Var> args = Internal::approximation_component_vars(code.dimensions(), "code");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Expr value = cast<int32_t>(code(call_args));
+        Expr low_value = (value + offset_) % radix_;
+        Func low("additive_radix_low"), high("additive_radix_high");
+        low(args) = cast<uint8_t>(low_value);
+        high(args) = cast<int8_t>(value - low_value);
+        return {{low, high}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 2 && encoded[0].dimensions() == encoded[1].dimensions())
+            << "AdditiveRadixSplit::decode expects low and high contributions\n";
+        std::vector<Var> args = Internal::approximation_component_vars(encoded[0].dimensions(), "code");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Func code("additive_radix_code");
+        code(args) = cast<int8_t>(cast<int32_t>(encoded[0](call_args)) +
+                                  cast<int32_t>(encoded[1](call_args)));
+        return {{code}, {}};
+    }
+
+private:
+    int radix_, offset_;
+};
+
+/** Exact fixed-width planar packing. For `(field_bits, positions)`, one byte
+ * contains `8/field_bits` planes, each plane spanning `positions` consecutive
+ * elements. This component applies no recentering and no lookup policy. */
+class PlanarFieldPack : public Approximation {
+public:
+    PlanarFieldPack(int field_bits, int positions)
+        : field_bits_(field_bits), positions_(positions), planes_(8 / field_bits) {
+        user_assert(field_bits_ > 0 && 8 % field_bits_ == 0 && positions_ > 0)
+            << "Invalid PlanarFieldPack shape\n";
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].dimensions() == 2)
+            << "PlanarFieldPack::encode currently requires (element, record)\n";
+        Func fields = inputs[0];
+        Var position("position"), record("record");
+        RDom plane(0, planes_, "plane");
+        Expr element = plane * positions_ + position;
+        Expr value = cast<uint8_t>(fields(element, record)) & ((1 << field_bits_) - 1);
+        Func bytes("planar_field_bytes");
+        bytes(position, record) = cast<uint8_t>(0);
+        bytes(position, record) = bytes(position, record) |
+                                  cast<uint8_t>(value << (plane * field_bits_));
+        return {{bytes}, {bytes}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{UInt(8)} &&
+                    encoded[0].dimensions() == 2)
+            << "PlanarFieldPack::decode currently requires (position, record) bytes\n";
+        Func bytes = encoded[0];
+        Var element("element"), record("record");
+        Expr plane = element / positions_;
+        Expr position = element % positions_;
+        Func fields("planar_field_values");
+        fields(element, record) = cast<uint8_t>((bytes(position, record) >> (plane * field_bits_)) &
+                                                ((1 << field_bits_) - 1));
+        return {{fields}, {}};
+    }
+
+private:
+    int field_bits_, positions_, planes_;
+};
+
+enum class BlockRoundingMode {
+    Nearest,
+    TruncateHalfUpWithOffset,
+    SignOnly,
+    NearestEvenClampedHigh
+};
+
+enum class BlockScaleAnchor {
+    AbsMax,
+    ExtremeSignedValue,
+    MeanAbs,
+    ExtremeSignedValueTwoStep
+};
+
+inline Expr approximation_nearest_int(Expr value) {
+    Expr rounded = value + 12582912.0f;
+    Expr bits = reinterpret<int32_t>(rounded);
+    return (bits & 0x007fffff) - 0x00400000;
+}
+
+/** Symmetric per-block int8 quantization with explicit rounding and scale
+ * selection policies. */
+class SymmetricBlockQuantize : public Approximation {
+public:
+    SymmetricBlockQuantize(int block_size, int qmax, BlockRoundingMode rounding, BlockScaleAnchor anchor)
+        : block_size_(block_size), qmax_(qmax), rounding_(rounding), anchor_(anchor) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1) << "SymmetricBlockQuantize::encode expects one block Func\n";
+        Func block = inputs[0];
+        Var kk("kk"), blk("blk");
+        RDom r(0, block_size_, "r");
+        Func stat("symmetric_quantize_stat"), scale("symmetric_quantize_scale"), reciprocal("symmetric_quantize_reciprocal");
+        auto define_extreme = [&]() {
+            stat(blk) = Tuple(0.0f, 0.0f);
+            Expr value = block(r, blk);
+            Expr take = abs(value) > stat(blk)[0];
+            stat(blk) = Tuple(select(take, abs(value), stat(blk)[0]),
+                              select(take, value, stat(blk)[1]));
+        };
+        if (anchor_ == BlockScaleAnchor::AbsMax) {
+            stat(blk) = 0.0f;
+            stat(blk) = max(stat(blk), abs(block(r, blk)));
+            scale(blk) = stat(blk) / (float)qmax_;
+            reciprocal(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else if (anchor_ == BlockScaleAnchor::ExtremeSignedValue) {
+            define_extreme();
+            scale(blk) = stat(blk)[1] * (-1.0f / (float)qmax_);
+            reciprocal(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else if (anchor_ == BlockScaleAnchor::MeanAbs) {
+            stat(blk) = 0.0f;
+            stat(blk) += abs(block(r, blk));
+            scale(blk) = stat(blk) / (float)block_size_;
+            reciprocal(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else {
+            define_extreme();
+            reciprocal(blk) = select(stat(blk)[0] == 0.0f, 0.0f,
+                                     (-1.0f * (float)qmax_) / stat(blk)[1]);
+            scale(blk) = select(reciprocal(blk) != 0.0f, 1.0f / reciprocal(blk), 0.0f);
+        }
+        Expr scaled = block(kk, blk) * reciprocal(blk);
+        Func codes("symmetric_quantize_codes");
+        if (rounding_ == BlockRoundingMode::Nearest) {
+            codes(kk, blk) = cast<int8_t>(round(scaled));
+        } else if (rounding_ == BlockRoundingMode::TruncateHalfUpWithOffset) {
+            Expr raw = cast<int32_t>(cast<int8_t>(scaled + (float)qmax_ + 0.5f));
+            codes(kk, blk) = cast<int8_t>(min(raw, 2 * qmax_ - 1) - qmax_);
+        } else if (rounding_ == BlockRoundingMode::SignOnly) {
+            codes(kk, blk) = cast<int8_t>(select(block(kk, blk) >= 0.0f, 1, -1));
+        } else {
+            codes(kk, blk) = cast<int8_t>(min(qmax_, approximation_nearest_int(scaled)));
+        }
+        return {{codes, scale}, {stat}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 2) << "SymmetricBlockQuantize::decode expects codes and scale\n";
+        Var kk("kk"), blk("blk");
+        Func dequantized("symmetric_dequantized");
+        dequantized(kk, blk) = cast<float>(encoded[0](kk, blk)) * encoded[1](blk);
+        return {{dequantized}, {}};
+    }
+
+private:
+    int block_size_, qmax_;
+    BlockRoundingMode rounding_;
+    BlockScaleAnchor anchor_;
+};
+
+}  // namespace Halide
+
+#endif
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 7ea25905b3a9..f139c32eba10 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -62,6 +62,7 @@ target_sources(
     AllocationBoundsInference.h
     ApplySplit.h
     Approximation.h
+    ApproximationComponents.h
     Argument.h
     AssociativeOpsTable.h
     Associativity.h
diff --git a/src/Func.cpp b/src/Func.cpp
index c895ea799bc8..c403cb8488fc 100644
--- a/src/Func.cpp
+++ b/src/Func.cpp
@@ -2885,7 +2885,9 @@ ApproximationResult Func::approximate_by(Approximation &p, const vector<Func> &c
     vector<Func> handles = enc.encoded;
     handles.insert(handles.end(), enc.handles.begin(), enc.handles.end());
     handles.insert(handles.end(), dec.handles.begin(), dec.handles.end());
-    return {round_trip, enc.encoded, handles};
+    enc.stage_outputs.push_back({p.stage_key(), enc.encoded});
+    dec.stage_outputs.push_back({p.stage_key(), dec.decoded});
+    return {round_trip, enc.encoded, handles, enc.stage_outputs, dec.stage_outputs};
 }
 
 Func Func::copy_to_device(DeviceAPI d) {
diff --git a/test/correctness/CMakeLists.txt b/test/correctness/CMakeLists.txt
index b72b4f159e0c..0a92b81c8739 100644
--- a/test/correctness/CMakeLists.txt
+++ b/test/correctness/CMakeLists.txt
@@ -6,6 +6,7 @@ tests(
     # keep-sorted start case=no
     align_bounds.cpp
     approximate_by.cpp
+    approximation_components.cpp
     argmax.cpp
     arm_cpu_detect.cpp
     associativity.cpp
diff --git a/test/correctness/approximate_by.cpp b/test/correctness/approximate_by.cpp
index 00a79833d07b..e36192b08c84 100644
--- a/test/correctness/approximate_by.cpp
+++ b/test/correctness/approximate_by.cpp
@@ -63,6 +63,57 @@ int main(int argc, char **argv) {
     SymmetricQuantizer quant;
     ApproximationResult result = f.approximate_by(quant, {g});
 
+    // Stage tracing preserves opaque identities through nested combinators,
+    // including repeated component types and Apply/TrustedInverse ownership.
+    Identity first_identity, second_identity, trusted_encoder, trusted_decoder;
+    const ApproximationStageKey first_key = first_identity.stage_key();
+    const ApproximationStageKey second_key = second_identity.stage_key();
+    const ApproximationStageKey encoder_key = trusted_encoder.stage_key();
+    const ApproximationStageKey decoder_key = trusted_decoder.stage_key();
+    Apply applied(0, second_identity);
+    const ApproximationStageKey apply_key = applied.stage_key();
+    TrustedInverse trusted(trusted_encoder, trusted_decoder);
+    const ApproximationStageKey trusted_key = trusted.stage_key();
+    Compose nested(first_identity, std::move(applied), std::move(trusted));
+    const ApproximationStageKey nested_key = nested.stage_key();
+
+    Func traced_source("traced_source"), traced_consumer("traced_consumer");
+    traced_source(x) = cast<float>(x);
+    traced_consumer(x) = traced_source(x);
+    ApproximationResult traced = traced_source.approximate_by(nested, {traced_consumer});
+
+    auto require_port = [&](const ApproximationStageKey &key, const char *label) {
+        if (!traced.encoded_by(key).defined() || !traced.decoded_by(key).defined()) {
+            printf("Missing encode/decode stage trace for %s\n", label);
+            return false;
+        }
+        return true;
+    };
+    if (!require_port(first_key, "first repeated Identity") ||
+        !require_port(second_key, "second repeated Identity") ||
+        !require_port(apply_key, "Apply") ||
+        !require_port(trusted_key, "TrustedInverse") ||
+        !require_port(nested_key, "outer Compose")) {
+        return 1;
+    }
+    if (!traced.encoded_by(encoder_key).defined() || traced.decoded_by(encoder_key).defined() ||
+        traced.encoded_by(decoder_key).defined() || !traced.decoded_by(decoder_key).defined()) {
+        printf("TrustedInverse did not preserve direction-specific child traces\n");
+        return 1;
+    }
+    if (first_key == second_key) {
+        printf("Distinct Approximation instances unexpectedly share a stage key\n");
+        return 1;
+    }
+    Identity unused;
+    if (traced.encoded_by(unused.stage_key()).defined() ||
+        traced.decoded_by(unused.stage_key()).defined() ||
+        traced.encoded_by(first_key, 1).defined() ||
+        traced.decoded_by(first_key, 1).defined()) {
+        printf("Invalid stage key or port unexpectedly resolved\n");
+        return 1;
+    }
+
     if (result.handles.empty()) {
         printf("Expected approximate_by() to return scheduling handles\n");
         return 1;
diff --git a/test/correctness/approximation_components.cpp b/test/correctness/approximation_components.cpp
new file mode 100644
index 000000000000..c273f1ed2088
--- /dev/null
+++ b/test/correctness/approximation_components.cpp
@@ -0,0 +1,229 @@
+#include "Halide.h"
+
+#include <cmath>
+#include <cstdio>
+
+using namespace Halide;
+
+namespace {
+
+int test_struct_layout_1d() {
+    Type record_type = Type::Struct({{"d", Float(16)}, {"qh", UInt(8), 4}, {"qs", UInt(8), 16}});
+    Var element("element"), record("record");
+    Func qs("qs"), qh("qh"), d("d");
+    qs(element, record) = cast<uint8_t>(element + 3 * record);
+    qh(element, record) = cast<uint8_t>(0x80 + element + record);
+    d(record) = cast<float16_t>(cast<float>(record) + 0.5f);
+
+    StructLayout layout(record_type, {"qs", "qh", "d"});
+    DecodeResult decoded = layout.decode(layout.encode({qs, qh, d}).encoded);
+    Buffer<uint8_t> out_qs = decoded.decoded[0].realize({16, 3});
+    Buffer<uint8_t> out_qh = decoded.decoded[1].realize({4, 3});
+    Buffer<float16_t> out_d = decoded.decoded[2].realize({3});
+    for (int r = 0; r < 3; ++r) {
+        if ((float)out_d(r) != r + 0.5f) {
+            return 1;
+        }
+        for (int i = 0; i < 16; ++i) {
+            if (out_qs(i, r) != (uint8_t)(i + 3 * r)) {
+                return 1;
+            }
+        }
+        for (int i = 0; i < 4; ++i) {
+            if (out_qh(i, r) != (uint8_t)(0x80 + i + r)) {
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+int test_struct_layout_2d() {
+    Type record_type = Type::Struct({{"tag", UInt(16)}, {"pixels", UInt(8), 3}});
+    Var element("element"), x("x"), y("y");
+    Func pixels("pixels"), tag("tag");
+    pixels(element, x, y) = cast<uint8_t>(element + 10 * x + 30 * y);
+    tag(x, y) = cast<uint16_t>(100 + x + 4 * y);
+    StructLayout layout(record_type, {"pixels", "tag"}, 2);
+    DecodeResult decoded = layout.decode(layout.encode({pixels, tag}).encoded);
+    Buffer<uint8_t> out_pixels = decoded.decoded[0].realize({3, 4, 2});
+    Buffer<uint16_t> out_tag = decoded.decoded[1].realize({4, 2});
+    for (int yy = 0; yy < 2; ++yy) {
+        for (int xx = 0; xx < 4; ++xx) {
+            if (out_tag(xx, yy) != 100 + xx + 4 * yy) {
+                return 1;
+            }
+            for (int i = 0; i < 3; ++i) {
+                if (out_pixels(i, xx, yy) != i + 10 * xx + 30 * yy) {
+                    return 1;
+                }
+            }
+        }
+    }
+    return 0;
+}
+
+int test_struct_layout_contract_errors() {
+    Type record_type = Type::Struct({{"tag", UInt(16)}, {"pixels", UInt(8), 3}});
+    Var element("element"), record("record");
+    Func pixels("pixels"), wrong_tag("wrong_tag");
+    pixels(element, record) = cast<uint8_t>(element);
+    wrong_tag(record) = cast<int16_t>(record);
+    try {
+        StructLayout layout(record_type, {"pixels", "tag"});
+        (void)layout.encode({pixels, wrong_tag});
+        return 1;
+    } catch (const CompileError &) {
+    }
+    try {
+        StructLayout duplicate(record_type, {"pixels", "pixels"});
+        return 1;
+    } catch (const CompileError &) {
+    }
+    return 0;
+}
+
+int test_scalar_components() {
+    Var record("record");
+    Func values("values");
+    values(record) = cast<float>(record) / 3.0f;
+    StorageCast<float, float16_t> storage;
+    EncodeResult stored = storage.encode({values});
+    DecodeResult cast_roundtrip = storage.decode(stored.encoded);
+    Buffer<float> out = cast_roundtrip.decoded[0].realize({8});
+    for (int i = 0; i < 8; ++i) {
+        float expected = (float)(float16_t)(i / 3.0f);
+        if (out(i) != expected) {
+            printf("StorageCast mismatch at %d: %g vs %g\n", i, out(i), expected);
+            return 1;
+        }
+    }
+
+    Func words("words");
+    words(record) = cast<uint32_t>((int32_t)0x10203040) + cast<uint32_t>(record);
+    LittleEndianScalarPack<uint32_t> little_endian;
+    EncodeResult bytes = little_endian.encode({words});
+    DecodeResult word_roundtrip = little_endian.decode(bytes.encoded);
+    Buffer<uint8_t> packed = bytes.encoded[0].realize({4, 5});
+    Buffer<uint32_t> unpacked = word_roundtrip.decoded[0].realize({5});
+    for (int r = 0; r < 5; ++r) {
+        if (unpacked(r) != 0x10203040u + r || packed(0, r) != (uint8_t)(0x40 + r) ||
+            packed(1, r) != 0x30 || packed(2, r) != 0x20 || packed(3, r) != 0x10) {
+            printf("LittleEndian mismatch at %d: %08x [%02x %02x %02x %02x]\n", r,
+                   unpacked(r), packed(0, r), packed(1, r), packed(2, r), packed(3, r));
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int test_code_components() {
+    Var element("element"), record("record");
+    Func high("high");
+    high(element, record) = cast<int8_t>(select(((element + record) & 1) != 0, 0, -16));
+    BinaryAlphabetPack<int8_t> binary(32, UInt(32), -16, 0);
+    EncodeResult word = binary.encode({high});
+    word.encoded[0].compute_root();
+    DecodeResult expanded = binary.decode(word.encoded);
+    Buffer<uint32_t> packed_word = word.encoded[0].realize({2});
+    Buffer<int8_t> out_high = expanded.decoded[0].realize({32, 2});
+    if (packed_word(0) != 0xaaaaaaaau || packed_word(1) != 0x55555555u) {
+        return 1;
+    }
+    for (int r = 0; r < 2; ++r) {
+        for (int i = 0; i < 32; ++i) {
+            int8_t expected = ((i + r) & 1) ? 0 : -16;
+            if (out_high(i, r) != expected) {
+                return 1;
+            }
+        }
+    }
+
+    Func codes("codes");
+    codes(element, record) = cast<int8_t>((element % 32) - 16);
+    AdditiveRadixSplit split(16, 16);
+    EncodeResult parts = split.encode({codes});
+    DecodeResult combined = split.decode(parts.encoded);
+    Buffer<uint8_t> low = parts.encoded[0].realize({32, 1});
+    Buffer<int8_t> high_part = parts.encoded[1].realize({32, 1});
+    Buffer<int8_t> combined_codes = combined.decoded[0].realize({32, 1});
+    for (int i = 0; i < 32; ++i) {
+        if (low(i, 0) != (i & 15) || high_part(i, 0) != (i < 16 ? -16 : 0) ||
+            combined_codes(i, 0) != i - 16) {
+            return 1;
+        }
+    }
+
+    PlanarFieldPack planar(4, 16);
+    EncodeResult planar_bytes = planar.encode({parts.encoded[0]});
+    planar_bytes.encoded[0].compute_root();
+    DecodeResult planar_fields = planar.decode(planar_bytes.encoded);
+    Buffer<uint8_t> bytes_out = planar_bytes.encoded[0].realize({16, 1});
+    Buffer<uint8_t> fields_out = planar_fields.decoded[0].realize({32, 1});
+    for (int i = 0; i < 16; ++i) {
+        if (bytes_out(i, 0) != (uint8_t)(i | (i << 4)) ||
+            fields_out(i, 0) != i || fields_out(i + 16, 0) != i) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int test_block_components() {
+    Var k("k");
+    Func flat("flat");
+    flat(k) = cast<float>(k);
+    BlockReshape reshape(32);
+    DecodeResult reshaped = reshape.decode(reshape.encode({flat}).encoded);
+    Buffer<float> roundtrip = reshaped.decoded[0].realize({96});
+    for (int i = 0; i < 96; ++i) {
+        if (roundtrip(i) != i) {
+            return 1;
+        }
+    }
+
+    Var element("element"), record("record");
+    Func blocks("blocks");
+    blocks(element, record) = cast<float>(element - 16);
+    SymmetricBlockQuantize quantize(32, 16, BlockRoundingMode::TruncateHalfUpWithOffset,
+                                    BlockScaleAnchor::ExtremeSignedValue);
+    EncodeResult quantized = quantize.encode({blocks});
+    for (Func handle : quantized.handles) {
+        handle.compute_root();
+    }
+    DecodeResult dequantized = quantize.decode(quantized.encoded);
+    Buffer<int8_t> codes = quantized.encoded[0].realize({32, 1});
+    Buffer<float> scale = quantized.encoded[1].realize({1});
+    Buffer<float> values = dequantized.decoded[0].realize({32, 1});
+    if (scale(0) != 1.0f) {
+        return 1;
+    }
+    for (int i = 0; i < 32; ++i) {
+        if (codes(i, 0) != i - 16 || values(i, 0) != i - 16) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    struct Test {
+        const char *name;
+        int (*run)();
+    } tests[] = {{"StructLayout 1-D", test_struct_layout_1d},
+                 {"StructLayout 2-D", test_struct_layout_2d},
+                 {"StructLayout contract errors", test_struct_layout_contract_errors},
+                 {"scalar packs", test_scalar_components},
+                 {"code packs", test_code_components},
+                 {"block components", test_block_components}};
+    for (const Test &test : tests) {
+        if (test.run()) {
+            printf("Approximation component test failed: %s\n", test.name);
+            return 1;
+        }
+    }
+    printf("Success!\n");
+    return 0;
+}

From c4de4f8a76c288b0d74ecfe901d0c356cc4ebc8b Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 23:49:28 -0400
Subject: [PATCH 21/22] Checkpoint

---
 apps/ggml/PERF_NOTES.md                       |  41 ++++++
 apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md     | 138 ++++++++++++++++++
 apps/ggml/halide/quant_components.h           | 100 ++++++++-----
 .../halide/symmetric_vec_dot_generator.cpp    |  25 +++-
 apps/ggml/halide/vec_dot_generator_base.h     |  68 ++++++---
 src/ApproximationComponents.h                 |  43 ++++++
 test/correctness/approximation_components.cpp |  70 ++++++++-
 7 files changed, 420 insertions(+), 65 deletions(-)
 create mode 100644 apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md

diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
index 1528e2af2ffc..da18185c6be0 100644
--- a/apps/ggml/PERF_NOTES.md
+++ b/apps/ggml/PERF_NOTES.md
@@ -148,6 +148,47 @@ Two traps found along the way:
   directives applied *after* `specialize()` do not reach the specialized branch.
   It silently dropped the vectorize/unroll and made things 4x slower.
 
+### q4_0/q8_0 core-composition cleanup
+
+The tuned schedule above is now fed by faithful, public core Approximation
+compositions rather than ggml's legacy `StructBlockLayout` and code-pack
+wrappers:
+
+- q4_0 `{Float16 d; UInt8 qs[16]}` is
+  `StructLayout -> StorageCast -> PlanarFieldPack -> AdditiveOffset -> SymmetricBlockQuantize -> BlockReshape`.
+  `AdditiveOffset<int8_t, uint8_t>{8}` is the representation policy that maps
+  signed codes `[-8, 7]` to stored nibbles `[0, 15]`; planar packing remains an
+  exact, policy-free bit layout.
+- q8_0 `{Float16 d; Int8 qs[32]}` is
+  `StructLayout -> StorageCast -> SymmetricBlockQuantize -> BlockReshape`. Its
+  faithful signed array means the old UInt8 `BytePack` reinterpretation is
+  unnecessary.
+
+Both formats share one traced weight/activation decode graph between the main
+and remainder updates. The shared stages are eagerly inlined only into the tail
+before the four-block main update is transformed. The main assembly remains
+eight SDOTs per iteration with paired 128-bit code loads, four persistent vector
+accumulators, an explicitly unrolled epilogue, and no accumulator spill. The
+scalar tail is still intentionally proportional to its one-to-three-block
+remainder; include those cases in the planned named scaling/odd-tail
+`kernel-bench` mode.
+
+A struct-typed q8_0 activation input was also tested. It was correct, but it
+broadened load-shape changes across q4_0 and q5_0 without removing any target
+weight/codec representation debt, so the shared activation ABI remains on its
+stable byte path. The standalone q8_0 codecs and q8_0 weight operand use the
+faithful struct composition. Q1_0 and struct-aware reblocking are the remaining
+symmetric compatibility work.
+
+Ten final paired n=4096 runs measured q4_0 at 95.088 ns GGML / 97.000 ns Halide
+(0.9803x), versus a worktree baseline of 92.585 / 95.784 ns (0.9666x). Q8_0
+measured 72.973 / 74.951 ns (0.9736x), versus 70.618 / 74.870 ns (0.9432x).
+Absolute times moved with core placement; the paired ratios improved, and Halide
+time changed by +1.27% and +0.11%, respectively. No compiler change was
+required: the simplifier folded q4's offset/planar stages into the expected
+mask, shifts, and vector add, while q8's signed field lowered to direct vector
+loads.
+
 ## The q4_1 story so far
 
 q4_1 is affine: the weight decodes to `d*code + m`, so the per-block product
diff --git a/apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md b/apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md
new file mode 100644
index 000000000000..079fe18e1921
--- /dev/null
+++ b/apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md
@@ -0,0 +1,138 @@
+# Clean q4_0 and q8_0 Approximation Implementation
+
+## Current status
+
+- Source baseline commit: `ca685e94e91c33d924f10c238ffa4ae6dab183a4`
+- Working baseline: completed q5_0 core-composition refactor in this worktree
+- Status: complete; all acceptance gates passed
+- [x] Phase 0: inspect the existing schemes and create this progress document
+- [x] Phase 1: collect ten paired baseline runs at n=4096
+- [x] Phase 2: compose q4_0 and q8_0 from reusable core components
+- [x] Phase 3: add focused correctness coverage
+- [x] Phase 4: preserve identity-based scheduling and tuned tails
+- [x] Phase 5: correctness, odd-tail, generated-code, and full-suite validation
+- [x] Phase 6: paired performance validation and durable documentation
+
+## Goal and constraints
+
+Apply the q5_0 cleanup methodology to q4_0 and q8_0. Their schemes should use
+faithful packed struct types and reusable public Halide Approximations; the
+generic vec-dot generator should retain only the base reduction,
+`approximate_by`, `compute_offline`, and scheduling. Preserve bit-exact
+quantization, dequantization/vec-dot correctness, and the tuned four-block SDOT
+shape. Keep median paired performance within 5% of this worktree baseline and at
+least 0.90x GGML.
+
+Unrelated legacy formats remain compatibility debt. In particular, q1_0 keeps
+the legacy symmetric layout until it is migrated deliberately, and q4_1/q5_1
+remain on their affine/legacy paths.
+
+## Target compositions
+
+q4_0 faithful type: `{d: Float16, qs: UInt8[16]}`.
+
+1. `StructLayout`, logical `{qs, d}` to physical fields.
+2. `Apply` `StorageCast<Float32, Float16>` to `d`.
+3. `Apply` `PlanarFieldPack{4, 16}` to stored nibbles.
+4. `Apply` a reusable additive-offset component mapping signed codes to on-disk
+   `[0, 15]` values.
+5. `SymmetricBlockQuantize`, qmax 8, extreme-signed scale selection, and
+   truncate-half-up-with-offset rounding.
+6. `BlockReshape{32}` with the requested row/block-indexed layout.
+
+q8_0 faithful type: `{d: Float16, qs: Int8[32]}`.
+
+1. `StructLayout`, logical `{qs, d}` to physical fields.
+2. `Apply` `StorageCast<Float32, Float16>` to `d`.
+3. `SymmetricBlockQuantize`, qmax 127, absolute-max scale selection, and nearest
+   rounding.
+4. `BlockReshape{32}` with the requested row/block-indexed layout.
+
+The standalone q8_0 codecs and q8_0 weight path use the faithful core scheme.
+The shared activation ABI remains byte-addressed: a struct-typed experiment
+broadened generated-code changes across q4_0/q5_0 without removing reusable
+representation logic from either target's weight/codec pipeline. Mismatched
+consumers also require that byte path for the existing `Reblock` component.
+
+## Validation gates
+
+- q4_0 and q8_0 quantize outputs are bit-exact with GGML; dequantize and vec-dot
+  pass existing tolerances.
+- Focused component tests cover the additive offset and both compositions.
+- `kernel-bench --all` has no failures.
+- Odd block counts pass at n=32, 96, 160, 224, and 1056.
+- ARM main loops retain SDOT, four blocks in flight, wide contiguous code loads,
+  persistent accumulators, and fully unrolled fixed-size epilogues without
+  accumulator stack spills or one-iteration epilogue loops.
+- Median paired performance is no more than 5% slower than baseline and remains
+  at least 0.90x GGML; q5_0 and affine shared-format checks show no accidental
+  regression.
+
+## Benchmark experiment policy
+
+Any useful or repeatable experimental setup must be promoted into `kernel-bench`
+as a named mode rather than left as an ad hoc shell recipe. Size and scaling
+sweeps used here should feed the same scaling/odd-tail mode already identified
+by the q5_0 work.
+
+## Baseline results
+
+Ten paired filtered runs at `KERNEL_BENCH_N=4096`:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide |
+| ------ | ---------- | ---------- | ------------------ |
+| q4_0   | 92.585 ns  | 95.784 ns  | 0.9666x            |
+| q4_1   | 104.726 ns | 118.332 ns | 0.8850x            |
+| q5_0   | 119.370 ns | 128.931 ns | 0.9258x            |
+| q5_1   | 135.748 ns | 153.946 ns | 0.8818x            |
+| q8_0   | 70.618 ns  | 74.870 ns  | 0.9432x            |
+
+All candidate correctness flags were true. Raw CSV files are in
+`/tmp/q48-baseline.4xCRUB` for this work session.
+
+## Experiment log
+
+| #   | Change                                                                                                                              | Correctness                                               | GGML / Halide timings                                                                                 | Generated-code observations                                                                              | Decision                                         |
+| --- | ----------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
+| 0   | Completed q5_0 worktree baseline                                                                                                    | All filtered vec-dot checks passed                        | q4_0: 92.585 / 95.784 ns, 0.9666x; q8_0: 70.618 / 74.870 ns, 0.9432x                                  | Existing four-block SDOT paths are the generated-code reference                                          | Reference                                        |
+| 1   | Add core `AdditiveOffset` and compose faithful q4_0/q8_0 structs from public components                                             | Component composition and standalone q4_0/q8_0 tests pass | Ten-run probe: q4_0 94.177 / 96.975 ns, 0.9712x; q8_0 72.942 / 74.824 ns, 0.9748x                     | Core stages simplify to the existing signed-code SDOT inputs                                             | Keep                                             |
+| 2   | Use faithful struct q8_0 for the shared activation operand                                                                          | Correct, including q5_0                                   | Single sample moved absolute timings with core placement; paired ratios did not indicate a regression | Broadened input/load-shape changes across q4_0 and q5_0                                                  | Revert; keep the established byte activation ABI |
+| 3   | Share the traced weight and activation decode graphs between q4_0/q8_0 main and tail updates, eagerly inlining only the tail update | Standalone and odd-size tests pass                        | Ten-run probe: q4_0 96.039 / 99.147 ns, 0.9687x; q8_0 75.801 / 77.443 ns, 0.9788x                     | Removes duplicate tail Approximation graphs; main remains four-block SDOT and the remainder stays scalar | Keep                                             |
+| 4   | Full validation and ten final paired runs                                                                                           | `kernel-bench --all` clean; all paired flags true         | q4_0: 95.088 / 97.000 ns, 0.9803x; q8_0: 72.973 / 74.951 ns, 0.9736x                                  | Eight SDOTs/four blocks, paired 128-bit code loads, persistent accumulators, no accumulator spill        | Final                                            |
+
+## Final paired results
+
+Median of ten n=4096 paired runs:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide | Halide vs baseline |
+| ------ | ---------- | ---------- | ------------------ | ------------------ |
+| q4_0   | 95.088 ns  | 97.000 ns  | 0.9803x            | +1.27%             |
+| q4_1   | 106.761 ns | 117.951 ns | 0.9051x            | -0.32%             |
+| q5_0   | 122.167 ns | 130.589 ns | 0.9355x            | +1.29%             |
+| q5_1   | 143.560 ns | 153.727 ns | 0.9339x            | -0.14%             |
+| q8_0   | 72.973 ns  | 74.951 ns  | 0.9736x            | +0.11%             |
+
+Negative deltas are improvements. Raw final CSV files are in
+`/tmp/q48-final.5IktYc` for this work session.
+
+## Framework/compiler issues
+
+- No compiler change was needed. The simplifier folds q4_0's core
+  `AdditiveOffset` and `PlanarFieldPack` into the same mask/shift/vector-add
+  operations consumed by SDOT, and q8_0's signed struct array lowers to direct
+  128-bit loads.
+- A struct-typed q8_0 activation was correct but unnecessarily broadened load
+  shape changes across q4_0/q5_0. The stable shared activation ABI remains the
+  compatibility byte path; q8_0's codecs and weight path are fully
+  core-composed.
+- Shared q4_0/q8_0 tails must be eagerly inlined into the tail update before the
+  main SDOT schedule is applied. The remainder is deliberately scalar and its
+  cost scales with one to three blocks; this reinforces the need for a named
+  size-sweep benchmark mode.
+
+## Final follow-up items
+
+- Add the reusable size/scaling experiments from this work as named
+  `kernel-bench` modes.
+- Migrate q1_0 from `StructBlockLayout` and make `Reblock` struct-aware before
+  removing the symmetric compatibility layout and byte activation path.
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
index 410bbc4659d8..71c27cbca5cd 100644
--- a/apps/ggml/halide/quant_components.h
+++ b/apps/ggml/halide/quant_components.h
@@ -294,39 +294,39 @@ enum class ScaleAnchor { AbsMax,
 // decode(): {codes, scale} -> cast<float>(codes) * scale -- this half is
 // exactly the same regardless of rounding/anchor (both Q4_0's and Q8_0's
 // existing hand-written dequantize math already reduce to this one formula).
+inline Halide::BlockRoundingMode core_rounding_mode(RoundingMode mode) {
+    switch (mode) {
+    case RoundingMode::Nearest:
+        return Halide::BlockRoundingMode::Nearest;
+    case RoundingMode::TruncateHalfUpWithOffset:
+        return Halide::BlockRoundingMode::TruncateHalfUpWithOffset;
+    case RoundingMode::SignOnly:
+        return Halide::BlockRoundingMode::SignOnly;
+    case RoundingMode::NearestEvenClampedHigh:
+        return Halide::BlockRoundingMode::NearestEvenClampedHigh;
+    }
+    _halide_internal_error << "Unknown symmetric rounding mode\n";
+}
+
+inline Halide::BlockScaleAnchor core_scale_anchor(ScaleAnchor anchor) {
+    switch (anchor) {
+    case ScaleAnchor::AbsMax:
+        return Halide::BlockScaleAnchor::AbsMax;
+    case ScaleAnchor::ExtremeSignedValue:
+        return Halide::BlockScaleAnchor::ExtremeSignedValue;
+    case ScaleAnchor::MeanAbs:
+        return Halide::BlockScaleAnchor::MeanAbs;
+    case ScaleAnchor::ExtremeSignedValueTwoStep:
+        return Halide::BlockScaleAnchor::ExtremeSignedValueTwoStep;
+    }
+    _halide_internal_error << "Unknown symmetric scale anchor\n";
+}
+
 class SymmetricAffineQuantize : public Halide::SymmetricBlockQuantize {
 public:
     SymmetricAffineQuantize(int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor)
-        : Halide::SymmetricBlockQuantize(block_size, qmax, core_rounding(rounding), core_anchor(anchor)) {
-    }
-
-private:
-    static Halide::BlockRoundingMode core_rounding(RoundingMode mode) {
-        switch (mode) {
-        case RoundingMode::Nearest:
-            return Halide::BlockRoundingMode::Nearest;
-        case RoundingMode::TruncateHalfUpWithOffset:
-            return Halide::BlockRoundingMode::TruncateHalfUpWithOffset;
-        case RoundingMode::SignOnly:
-            return Halide::BlockRoundingMode::SignOnly;
-        case RoundingMode::NearestEvenClampedHigh:
-            return Halide::BlockRoundingMode::NearestEvenClampedHigh;
-        }
-        _halide_internal_error << "Unknown symmetric rounding mode\n";
-    }
-
-    static Halide::BlockScaleAnchor core_anchor(ScaleAnchor anchor) {
-        switch (anchor) {
-        case ScaleAnchor::AbsMax:
-            return Halide::BlockScaleAnchor::AbsMax;
-        case ScaleAnchor::ExtremeSignedValue:
-            return Halide::BlockScaleAnchor::ExtremeSignedValue;
-        case ScaleAnchor::MeanAbs:
-            return Halide::BlockScaleAnchor::MeanAbs;
-        case ScaleAnchor::ExtremeSignedValueTwoStep:
-            return Halide::BlockScaleAnchor::ExtremeSignedValueTwoStep;
-        }
-        _halide_internal_error << "Unknown symmetric scale anchor\n";
+        : Halide::SymmetricBlockQuantize(block_size, qmax,
+                                         core_rounding_mode(rounding), core_scale_anchor(anchor)) {
     }
 };
 
@@ -2668,15 +2668,43 @@ inline SchemeAndBytes make_symmetric_block_scheme(
     int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor, int code_bits,
     Layout layout = Layout::FlatRow, bool struct_layout = false) {
     using namespace Halide;
+
+    if (struct_layout && code_bits == 4) {
+        _halide_user_assert(block_size == 32 && qmax == 8)
+            << "The core q4_0 layout requires a 32-element, qmax=8 block\n";
+        Type block_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), 16}});
+        return {std::make_unique<Compose>(
+                    Halide::StructLayout{block_type, {"qs", "d"}},
+                    Apply{1, Halide::StorageCast<float, float16_t>{}},
+                    Apply{0, Halide::PlanarFieldPack{4, 16}},
+                    Apply{0, Halide::AdditiveOffset<int8_t, uint8_t>{8}},
+                    Halide::SymmetricBlockQuantize{block_size, qmax,
+                                                   core_rounding_mode(rounding), core_scale_anchor(anchor)},
+                    Halide::BlockReshape{block_size, layout == Layout::BlockIndexed}),
+                block_type.bytes(),
+                block_type};
+    }
+
+    if (struct_layout && code_bits == 8) {
+        _halide_user_assert(block_size == 32 && qmax == 127)
+            << "The core q8_0 layout requires a 32-element, qmax=127 block\n";
+        Type block_type = Type::Struct({{"d", Float(16)}, {"qs", Int(8), 32}});
+        return {std::make_unique<Compose>(
+                    Halide::StructLayout{block_type, {"qs", "d"}},
+                    Apply{1, Halide::StorageCast<float, float16_t>{}},
+                    Halide::SymmetricBlockQuantize{block_size, qmax,
+                                                   core_rounding_mode(rounding), core_scale_anchor(anchor)},
+                    Halide::BlockReshape{block_size, layout == Layout::BlockIndexed}),
+                block_type.bytes(),
+                block_type};
+    }
+
     auto [code_pack, code_bytes] = make_code_pack(block_size, code_bits, qmax);
 
     if (struct_layout) {
-        // The on-disk block as a first-class struct: `{fp16 d; uint8 qs[...]}`,
-        // matching every symmetric GGML block_* layout. The compiler owns the
-        // offsets and the total byte size; StructBlockLayout reads/writes `d` as
-        // a typed field (subsuming Fp16Pack) and hands the `qs` bytes to the same
-        // code_pack the byte-buffer path uses. Apply{0, code_pack} interprets
-        // those bytes (nibble/byte/bit extraction) exactly as before.
+        // Compatibility path for Q1_0. Q4_0/Q8_0 use their faithful, fully
+        // core-composed layouts above; other symmetric formats migrate
+        // deliberately rather than changing behavior through this fallback.
         Type block_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), code_bytes}});
         return {std::make_unique<Compose>(
                     StructBlockLayout{block_type, "d", "qs"},
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
index c2da055da7e4..e5ce474d7203 100644
--- a/apps/ggml/halide/symmetric_vec_dot_generator.cpp
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -147,11 +147,22 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         std::unique_ptr<Halide::Approximation> ac;
         int ab;
         bool act_has_block_sums = false;
+        Halide::Type act_type;
         switch (a_kind.value()) {
-        case AKind::Q8_0:
-            ac = make_symmetric_block_scheme(a_nat, a_qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
+        case AKind::Q8_0: {
+            // Keep the shared activation ABI byte-addressed. A struct-typed
+            // activation was tested for same-size blocks, but disturbed the
+            // tuned q4_0/q5_0 load shape; the faithful struct scheme remains in
+            // use for q8_0's codecs and weight operand.
+            const bool structured_q8 = false;
+            SchemeAndBytes sb = make_symmetric_block_scheme(
+                a_nat, a_qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8,
+                Layout::BlockIndexed, structured_q8);
+            ac = std::move(sb.scheme);
+            act_type = sb.block_type;
             ab = 2 + a_nat;
             break;
+        }
         case AKind::Q8_1: {
             SchemeAndBytes sb = make_symmetric_byte_sum_block_scheme(a_nat, a_qmax, Layout::BlockIndexed);
             ac = std::move(sb.scheme);
@@ -166,12 +177,12 @@ class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGener
         }
         ac = reblock_activation(std::move(ac), a_nat, wbs);
 
-        // Activation stays on the byte path for now (Q8_0/Q8_1 are shared across
-        // many weight formats, and the Reblock relayout is byte-based); only the
-        // weight operand is struct-typed here.
         const bool five_bit = w_kind.value() == WKind::Symmetric5Bit || w_kind.value() == WKind::Affine5Bit;
-        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, Halide::Type{}, act_has_block_sums,
-                five_bit ? 2 : 4, reconstructed_codes_stage, packed_high_word_stage};
+        const bool clean_symmetric = w_kind.value() == WKind::Symmetric &&
+                                     (w_code_bits == 4 || w_code_bits == 8);
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, act_type, act_has_block_sums,
+                five_bit ? 2 : 4, reconstructed_codes_stage, packed_high_word_stage,
+                clean_symmetric, clean_symmetric};
     }
 };
 
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index 0fbed23a4a18..aeacecbc2430 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -77,6 +77,11 @@ struct VecDotSpec {
     int unroll_blocks = 4;
     Halide::ApproximationStageKey reconstructed_codes_stage;
     Halide::ApproximationStageKey packed_high_word_stage;
+    // Clean core-composed schemes can share one traced decode graph between
+    // the four-block main update and its tiny remainder update. Legacy schemes
+    // retain separate traces until their representation stages are migrated.
+    bool share_weight_tail = false;
+    bool share_act_tail = false;
 };
 
 template<typename Derived>
@@ -131,6 +136,8 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         // at the default schedule (at most unroll_blocks - 1 blocks).
         const bool sdot = spec.sched == ScheduleKind::SDOT;
         const bool keyed_q5 = spec.reconstructed_codes_stage.defined();
+        const bool share_weight_tail = keyed_q5 || spec.share_weight_tail;
+        const bool share_act_tail = spec.share_act_tail;
         Expr nblocks = x_blocks.dim(wt_struct ? 0 : 1).extent();
         Expr main_blocks = sdot ? (nblocks / unroll_blocks) * unroll_blocks : nblocks;
 
@@ -153,8 +160,9 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
             // q5_0 shares the main decode graph with its tiny odd-block tail.
             // The tail update is eagerly inlined below before the reconstructed
             // codes leaf is materialized for the main paired-block update.
-            Func tail_weight = keyed_q5 ? Wt : WtT;
-            Acc() += tail_weight(r_tail.x, r_tail.y) * VecT(r_tail.x, r_tail.y);
+            Func tail_weight = share_weight_tail ? Wt : WtT;
+            Func tail_act = share_act_tail ? Vec : VecT;
+            Acc() += tail_weight(r_tail.x, r_tail.y) * tail_act(r_tail.x, r_tail.y);
         }
 
         ApproximationResult wt_r = Wt.approximate_by(*spec.weight_codec, {Acc});
@@ -169,28 +177,36 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         std::vector<ImageParam> bind_to = {x_blocks, y_blocks};
         ApproximationResult wtT_r, actT_r;
         if (sdot) {
-            if (!keyed_q5) {
+            if (!share_weight_tail) {
                 wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
             }
-            actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
-            if (!keyed_q5) {
+            if (!share_act_tail) {
+                actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
+            }
+            if (!share_weight_tail) {
                 to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
             }
-            to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
-            if (!keyed_q5) {
+            if (!share_act_tail) {
+                to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
+            }
+            if (!share_weight_tail) {
                 bind_to.push_back(x_blocks);
             }
-            bind_to.push_back(y_blocks);
-            if (!keyed_q5) {
+            if (!share_act_tail) {
+                bind_to.push_back(y_blocks);
+            }
+            if (!share_weight_tail) {
                 for (Func h : wtT_r.handles) {
                     if (h.has_update_definition()) {
                         h.compute_root();
                     }
                 }
             }
-            for (Func h : actT_r.handles) {
-                if (h.has_update_definition()) {
-                    h.compute_root();
+            if (!share_act_tail) {
+                for (Func h : actT_r.handles) {
+                    if (h.has_update_definition()) {
+                        h.compute_root();
+                    }
                 }
             }
         }
@@ -223,15 +239,25 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         }
         _halide_internal_assert(!keyed_q5 || (codes_leaf.defined() && qh_leaf.defined()));
 
-        if (keyed_q5) {
-            // This update has at most one block (the paired q5 schedule's odd
-            // remainder). Flatten only its weight decode chain so it reconstructs
-            // eagerly, while the same stage Funcs remain materialization
-            // boundaries in the main update.
-            std::vector<Func> tail_inline = {wt_r.replacement};
-            for (const Func &h : wt_r.handles) {
-                if (h.function().can_be_inlined()) {
-                    tail_inline.push_back(h);
+        if (share_weight_tail || share_act_tail) {
+            // Flatten shared decode stages into this update only. The main
+            // update is scheduled independently below, so its SDOT boundaries
+            // and any keyed materializations remain intact.
+            std::vector<Func> tail_inline;
+            if (share_weight_tail) {
+                tail_inline.push_back(wt_r.replacement);
+                for (const Func &h : wt_r.handles) {
+                    if (h.function().can_be_inlined()) {
+                        tail_inline.push_back(h);
+                    }
+                }
+            }
+            if (share_act_tail) {
+                tail_inline.push_back(act_r.replacement);
+                for (const Func &h : act_r.handles) {
+                    if (h.function().can_be_inlined()) {
+                        tail_inline.push_back(h);
+                    }
                 }
             }
             for (size_t pass = 0; pass < tail_inline.size(); ++pass) {
diff --git a/src/ApproximationComponents.h b/src/ApproximationComponents.h
index 1a71ca061e1d..12d617ed302c 100644
--- a/src/ApproximationComponents.h
+++ b/src/ApproximationComponents.h
@@ -8,6 +8,7 @@
 
 #include <cstdint>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
@@ -242,6 +243,48 @@ class StorageCast : public Approximation {
     }
 };
 
+/** Losslessly translate an integral decoded alphabet by a fixed offset into
+ * its stored integral alphabet. This is representation policy, not bit
+ * packing: e.g. signed q4 codes [-8, 7] become stored nibbles [0, 15] before
+ * PlanarFieldPack handles their physical layout. */
+template<typename Decoded, typename Storage>
+class AdditiveOffset : public Approximation {
+    static_assert(std::is_integral_v<Decoded> && std::is_integral_v<Storage>,
+                  "AdditiveOffset requires integral types");
+
+public:
+    explicit AdditiveOffset(int64_t offset)
+        : offset_(offset) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].types() == std::vector<Type>{type_of<Decoded>()})
+            << "AdditiveOffset::encode input type mismatch\n";
+        Func input = inputs[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "offset");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Func stored("additive_offset_stored");
+        Expr offset = Internal::make_const(Int(64), offset_);
+        stored(args) = cast<Storage>(cast<int64_t>(input(call_args)) + offset);
+        return {{stored}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{type_of<Storage>()})
+            << "AdditiveOffset::decode storage type mismatch\n";
+        Func input = encoded[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "offset");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Func decoded("additive_offset_decoded");
+        Expr offset = Internal::make_const(Int(64), offset_);
+        decoded(args) = cast<Decoded>(cast<int64_t>(input(call_args)) - offset);
+        return {{decoded}, {}};
+    }
+
+private:
+    int64_t offset_;
+};
+
 /** Convert a scalar integral word per record to/from a leading little-endian
  * byte dimension. Decode deliberately uses concat_bits so struct lowering and
  * ordinary byte buffers share the same wide-load optimization path. */
diff --git a/test/correctness/approximation_components.cpp b/test/correctness/approximation_components.cpp
index c273f1ed2088..9e4d1e3627c9 100644
--- a/test/correctness/approximation_components.cpp
+++ b/test/correctness/approximation_components.cpp
@@ -119,6 +119,21 @@ int test_scalar_components() {
 
 int test_code_components() {
     Var element("element"), record("record");
+    Func signed_nibbles("signed_nibbles");
+    signed_nibbles(element, record) = cast<int8_t>((element % 16) - 8);
+    AdditiveOffset<int8_t, uint8_t> offset(8);
+    EncodeResult offset_codes = offset.encode({signed_nibbles});
+    DecodeResult signed_roundtrip = offset.decode(offset_codes.encoded);
+    Buffer<uint8_t> stored_nibbles = offset_codes.encoded[0].realize({16, 2});
+    Buffer<int8_t> restored_nibbles = signed_roundtrip.decoded[0].realize({16, 2});
+    for (int r = 0; r < 2; ++r) {
+        for (int i = 0; i < 16; ++i) {
+            if (stored_nibbles(i, r) != i || restored_nibbles(i, r) != i - 8) {
+                return 1;
+            }
+        }
+    }
+
     Func high("high");
     high(element, record) = cast<int8_t>(select(((element + record) & 1) != 0, 0, -16));
     BinaryAlphabetPack<int8_t> binary(32, UInt(32), -16, 0);
@@ -206,6 +221,58 @@ int test_block_components() {
     return 0;
 }
 
+int test_standard_quant_compositions() {
+    Var k("k");
+
+    Type q4_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), 16}});
+    Func q4_values("q4_values");
+    q4_values(k) = cast<float>((k % 16) - 8);
+    Compose q4(
+        StructLayout{q4_type, {"qs", "d"}},
+        Apply{1, StorageCast<float, float16_t>{}},
+        Apply{0, PlanarFieldPack{4, 16}},
+        Apply{0, AdditiveOffset<int8_t, uint8_t>{8}},
+        SymmetricBlockQuantize{32, 8, BlockRoundingMode::TruncateHalfUpWithOffset,
+                               BlockScaleAnchor::ExtremeSignedValue},
+        BlockReshape{32});
+    EncodeResult q4_encoded = q4.encode({q4_values});
+    for (Func handle : q4_encoded.handles) {
+        handle.compute_root();
+    }
+    DecodeResult q4_decoded = q4.decode(q4_encoded.encoded);
+    Buffer<float> q4_roundtrip = q4_decoded.decoded[0].realize({64});
+    for (int i = 0; i < 64; ++i) {
+        if (q4_roundtrip(i) != (i % 16) - 8) {
+            return 1;
+        }
+    }
+
+    Type q8_type = Type::Struct({{"d", Float(16)}, {"qs", Int(8), 32}});
+    Func q8_values("q8_values");
+    Expr local = k % 32;
+    q8_values(k) = cast<float>(select(local == 0, -127, local - 16));
+    Compose q8(
+        StructLayout{q8_type, {"qs", "d"}},
+        Apply{1, StorageCast<float, float16_t>{}},
+        SymmetricBlockQuantize{32, 127, BlockRoundingMode::Nearest,
+                               BlockScaleAnchor::AbsMax},
+        BlockReshape{32});
+    EncodeResult q8_encoded = q8.encode({q8_values});
+    for (Func handle : q8_encoded.handles) {
+        handle.compute_root();
+    }
+    DecodeResult q8_decoded = q8.decode(q8_encoded.encoded);
+    Buffer<float> q8_roundtrip = q8_decoded.decoded[0].realize({64});
+    for (int i = 0; i < 64; ++i) {
+        int local_i = i % 32;
+        float expected = local_i == 0 ? -127.0f : local_i - 16.0f;
+        if (q8_roundtrip(i) != expected) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
 }  // namespace
 
 int main(int argc, char **argv) {
@@ -217,7 +284,8 @@ int main(int argc, char **argv) {
                  {"StructLayout contract errors", test_struct_layout_contract_errors},
                  {"scalar packs", test_scalar_components},
                  {"code packs", test_code_components},
-                 {"block components", test_block_components}};
+                 {"block components", test_block_components},
+                 {"standard quant compositions", test_standard_quant_compositions}};
     for (const Test &test : tests) {
         if (test.run()) {
             printf("Approximation component test failed: %s\n", test.name);

From 9bd41cc7af4eb80245fd5ca7ab9896009e04dce8 Mon Sep 17 00:00:00 2001
From: Alex Reinking <areinking@adobe.com>
Date: Sun, 2 Aug 2026 23:57:11 -0400
Subject: [PATCH 22/22] Manual cleanup

---
 apps/ggml/halide/vec_dot_generator_base.h | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
index aeacecbc2430..f246e2db434a 100644
--- a/apps/ggml/halide/vec_dot_generator_base.h
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -179,23 +179,8 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
         if (sdot) {
             if (!share_weight_tail) {
                 wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
-            }
-            if (!share_act_tail) {
-                actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
-            }
-            if (!share_weight_tail) {
                 to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
-            }
-            if (!share_act_tail) {
-                to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
-            }
-            if (!share_weight_tail) {
                 bind_to.push_back(x_blocks);
-            }
-            if (!share_act_tail) {
-                bind_to.push_back(y_blocks);
-            }
-            if (!share_weight_tail) {
                 for (Func h : wtT_r.handles) {
                     if (h.has_update_definition()) {
                         h.compute_root();
@@ -203,6 +188,9 @@ class VecDotGeneratorBase : public Halide::Generator<Derived> {
                 }
             }
             if (!share_act_tail) {
+                actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
+                to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
+                bind_to.push_back(y_blocks);
                 for (Func h : actT_r.handles) {
                     if (h.has_update_definition()) {
                         h.compute_root();