feat(protobuf): expose protoc as a host tool

.github/workflows/validate.yml

@@ -14,6 +14,25 @@ on:
   workflow_dispatch:
 
 env:
+  # 2026.8.5.3 carries two things this workflow depends on:
+  #   .5.1  `tools = [...]` — how a consumer asks for a dependency's
+  #         `kind = "bin"` target, which tests/examples/protobuf-protoc is built
+  #         on. Before it: "tools must be a string, inline dep table, or nested
+  #         table".
+  #   .5.3  link response files separate objects by NEWLINES. link.exe caps a
+  #         response-file LINE at 128 KiB, and opencv-module / opencv-module-dnn
+  #         went past it on windows — after 795s / 1166s of compiling:
+  #             fatal error LNK1170: line in command file contains 135135 or
+  #             more characters
+  #         Together with re-enabling the global package cache below, this is
+  #         what makes a green FULL run possible again: .5.3 removes the
+  #         windows link failure, the cache removes the 150-minute timeout.
+  #
+  # Neither of them moves index.toml's min_mcpp: exposing compat.protobuf's `protoc`
+  # target is additive, and 2026.8.3.3 still parses that descriptor with an
+  # empty unknown_keys. The floor an index publishes decides whether older
+  # clients keep working at all (mcpp#349), so it moves only when a descriptor
+  # genuinely stops being readable — which is not the case here.
   # 2026.8.3.1: on macOS, a global object that touches std::cout during static
   # init crashes on sight (mcpp#336). Mach-O has no priority-ordered init
   # section and libc++'s <iostream> carries no ios_base::Init guard of its own,
@@ -88,7 +107,7 @@ env:
   # 0.0.94 fixed feature-gated `sources` under `mcpp test` (mcpp#218); 0.0.91
   # added standard = "c++fly" to the resolver grammar, so c++fly descriptors
   # get the lint WARN below, not a hard grammar-parse rejection.
-  MCPP_VERSION: "2026.8.3.3"
+  MCPP_VERSION: "2026.8.5.3"
 
 jobs:
   lint:
@@ -253,21 +272,21 @@ jobs:
             ext: tar.gz
             mcpp: bin/mcpp
             xlings: registry/bin/xlings
-            mcpp_version: "2026.8.3.3"   # keep in sync with env.MCPP_VERSION
+            mcpp_version: "2026.8.5.3"   # keep in sync with env.MCPP_VERSION
           - platform: macos
             os: macos-15
             suffix: macosx-arm64
             ext: tar.gz
             mcpp: bin/mcpp
             xlings: registry/bin/xlings
-            mcpp_version: "2026.8.3.3"   # keep in sync with env.MCPP_VERSION
+            mcpp_version: "2026.8.5.3"   # keep in sync with env.MCPP_VERSION
           - platform: windows
             os: windows-latest
             suffix: windows-x86_64
             ext: zip
             mcpp: bin/mcpp.exe
             xlings: registry/bin/xlings.exe
-            mcpp_version: "2026.8.3.3"   # keep in sync with env.MCPP_VERSION
+            mcpp_version: "2026.8.5.3"   # keep in sync with env.MCPP_VERSION
     env:
       MCPP_EFFECTIVE: ${{ matrix.mcpp_version }}
     steps:
@@ -437,21 +456,30 @@ jobs:
         shell: bash
         env:
           MCPP_INDEX_MIRROR: GLOBAL
-          # Dependencies build inside each member's own target/ instead of
-          # through the global package build cache (mcpp >= 2026.7.30.2).
-          # That cache is unusable here: mcpp#233's object-path disambiguation
-          # fires on basename collisions across the WHOLE build dir — i.e. on
-          # which packages the CONSUMER pulls in — while the cache key covers
-          # only the dependency itself, so one entry can hold two different
-          # layouts. `tests/examples/archive` pulls zlib AND bzip2 (both ship
-          # compress.c) and stores obj/compat_zlib/zlib-1.3.2/compress.o;
-          # every zlib consumer without bzip2 then asks the same key for a
-          # flat obj/compress.o and ninja dies at graph time with
-          # "missing and no known rule to make it". Reproduced both ways round
-          # on 2026.8.3.3 and filed as mcpp-community/mcpp#344; drop this once
-          # it lands. `local` still caches the std BMI, which is the expensive
-          # one — only package entries are bypassed.
-          MCPP_BUILD_CACHE: local
+          # The GLOBAL package build cache is on (mcpp >= 2026.7.30.2), which
+          # is the default — this step used to set `MCPP_BUILD_CACHE: local`
+          # and no longer does.
+          #
+          # That bypass existed for mcpp#344: object-path disambiguation fires
+          # on basename collisions across the WHOLE build dir — i.e. on what
+          # the CONSUMER pulls in — while the cache key covered only the
+          # dependency, so one entry could hold two layouts and ninja died at
+          # graph time with "missing and no known rule to make it". #344
+          # landed in 2026.8.3.4 with per-package Merkle keys that cover the
+          # consumer-dependent layout, so the reason is gone.
+          #
+          # Keeping it cost real time, and the full run is where it showed:
+          # with `local`, EVERY member recompiles EVERY dependency from
+          # scratch. 59 members that mostly share abseil / protobuf / opencv
+          # meant the same sources were built over and over —
+          #
+          #   linux    2h30m -> cancelled at the 150-minute timeout
+          #   windows  2h20m
+          #   macos    1h26m
+          #
+          # — and a workspace cannot be validated by a job that cannot finish.
+          # With the cache on, a given (package, version, features, toolchain)
+          # is built once per run and every later member hits it.
         run: |
           "$MCPP" --version
           # No `timeout` wrapper: absent on macOS runners; job-level timeout-minutes bounds it.

README.md

@@ -39,7 +39,7 @@ Two kinds of packages live here:
 |------|------|
 | Native module library (Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](pkgs/o/opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) |
 | C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) |
-| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) |
+| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) |
 | header-only (with `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) |
 | Runtime loader compat (pure sources, sidestepping upstream codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua) (the Khronos loader: `loader/generated/` is checked in, and the assembly path degrades to plain C through `UNKNOWN_FUNCTIONS_SUPPORTED`, so no CMake/Python/assembler is needed; windows deferred) · [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) |
 | Whole-source direct build + generated config (only where a platform lacks one) | [`compat.curl`](pkgs/c/compat.curl.lua) (win32 uses upstream's checked-in config, unix generates one) · [`compat.sdl2`](pkgs/c/compat.sdl2.lua) (win/mac use upstream's checked-in config; linux generates one and enables X11 by hand) · [`compat.c-ares`](pkgs/c/compat.c-ares.lua) (91 TUs; the release tarball already ships `ares_build.h` and a Windows config, so only `ares_config.h` is snapshotted per OS) |

README.zh-CN.md

@@ -36,7 +36,7 @@ mcpp self config --mirror CN   # 切换至国内镜像,默认使用 GLOBAL 上
 |------|------|
 | 原生模块库(Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](pkgs/o/opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)|
 | C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) |
-| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) |
+| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) |
 | header-only(含 `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) |
 | 运行时 loader compat(纯源码,绕开上游 codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua)(Khronos loader:`loader/generated/` 已签入,汇编路径经 `UNKNOWN_FUNCTIONS_SUPPORTED` 降级为纯 C,故无需 CMake/Python/汇编器;windows 延后)· [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) |
 | 全源码直编 + 生成 config(仅缺口平台) | [`compat.curl`](pkgs/c/compat.curl.lua)(win32 用上游签入 config,unix 生成) · [`compat.sdl2`](pkgs/c/compat.sdl2.lua)(win/mac 用上游签入 config,linux 生成 + 手工开 X11) · [`compat.c-ares`](pkgs/c/compat.c-ares.lua)(91 TU;release tarball 已自带 `ares_build.h` 与 Windows 配置,故只需按 OS 冻结 `ares_config.h`) |

docs/package-types.md

@@ -18,6 +18,7 @@ combined as needed.
 | **E. Whole-source direct build with a generated config** | upstream generates its config header through configure/CMake; here a snapshot of it lands in `generated_files` | `pkgs/c/compat.libpng.lua`, `compat.curl.lua`, `compat.sdl2.lua`, `compat.ffmpeg.lua` | `generated_files` + `include_dirs` |
 | **F. Shared-library compat** | has to be the **only** copy of that `.so` in the process (third parties `dlopen` it) | the X11 family such as `pkgs/c/compat.x11.lua`, and `compat.vulkan.lua` (linux) | `targets = { kind = "shared", soname = … }` |
 | **G. Host runtime adaptation** | things that cannot be vendored, such as drivers — only a symlink farm plus metadata | `pkgs/c/compat.glx-runtime.lua`, `compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` |
+| **H. Host tool provider** | the upstream tarball also holds a **code generator** consumers run at build time | `pkgs/c/compat.protobuf.lua` (`protoc`) | a `targets` entry with `kind = "bin"` + `main`, plus `required_features` |
 
 For the complete sample index, see the
 [Reference examples table in the root README](../README.md#reference-examples-lua-descriptors).
@@ -200,6 +201,50 @@ Two details that keep biting:
 - **The closure has to be complete.** A farm holding `libxcb.so.1` but not the `libXau.so.6` it depends on shadows the
   host copy that would otherwise have resolved, and the executable simply fails to start.
 
+## H. Host tool provider (`compat.protobuf`'s `protoc`)
+
+Some tarballs hold both a library and the code generator that emits code against it. Declare the generator as a second
+target, and consumers ask for it with `tools = [...]` (mcpp 2026.8.5.1+):
+
+```lua
+targets = {
+    ["protobuf"] = { kind = "lib" },
+    ["protoc"]   = { kind = "bin",
+                     main = "*/src/google/protobuf/compiler/main.cc",
+                     required_features = { "protoc", "upb" } },
+},
+features = {
+    ["protoc"] = { sources = { … the compiler's own sources … } },
+},
+```
+
+```toml
+# consumer side — one dependency, two roles
+compat.protobuf = { version = "35.1", tools = ["protoc"] }
+```
+
+mcpp then builds that target **for the build machine** in a nested sub-build and hands the path to the consumer's
+`build.mcpp` through `mcpp::dep_bin("protobuf", "protoc")`. This is the whole reason the shape is worth naming: the
+tool's version **is** the dependency's version, so a generator/runtime mismatch — a *runtime* failure everywhere else,
+and the classic protobuf footgun — is not expressible. Under `mcpp build --target <triple>` the tool is still built for
+the host, because a code generator has to run here.
+
+Four things to get right:
+
+- **Gate the compiler's sources behind a feature**, and name it in the target's `required_features`. Consumers who only
+  link the library must not pay for the generator's TUs; consumers who ask for the tool must not have to know which
+  features it needs. `compat.protobuf`'s `protoc` also requires `upb`, because libprotoc's upb generator links the upb
+  runtime — get that wrong and it fails at **link** time with missing `upb_*` symbols.
+- **Transcribe the source list from upstream**, exactly as for a library — protobuf's 138 entries come from
+  `libprotoc_srcs` in its own `src/file_lists.cmake`.
+- **`main` needs the same `*/` wrap glob as `sources`**; it is expanded the same way.
+- **A generator that reads data files at runtime still needs a path to them.** protoc does not embed the well-known
+  types: `import "google/protobuf/timestamp.proto"` is read from disk. Consumers derive that directory from
+  `mcpp::dep_dir("protobuf")` — see `tests/examples/protobuf-protoc/build.mcpp`.
+
+The matching member is `tests/examples/protobuf-protoc`, and it is the complement of `tests/examples/protobuf`: that
+one deliberately uses no generated code, this one is generated code end to end.
+
 ---
 
 ## The minimal project (`tests/examples/<short>/`)

docs/zh/package-types.md

@@ -16,6 +16,7 @@ A–D 是四种**基础**形态,先按它们判定;E–G 是在基础形态之
 | **E. 生成 config 的全源码直编** | 上游用 configure/CMake 生成配置头,此处以 `generated_files` 落一份快照 | `pkgs/c/compat.libpng.lua`、`compat.curl.lua`、`compat.sdl2.lua`、`compat.ffmpeg.lua` | `generated_files` + `include_dirs` |
 | **F. 共享库 compat** | 必须是**唯一**的那个 `.so`(会被第三方 `dlopen`) | `pkgs/c/compat.x11.lua` 等 X11 家族、`compat.vulkan.lua`(linux) | `targets = { kind = "shared", soname = … }` |
 | **G. 宿主运行时适配** | 驱动之类无法 vendor 的东西,只做符号链接农场 + 元数据 | `pkgs/c/compat.glx-runtime.lua`、`compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` |
+| **H. 宿主工具提供方** | 上游 tarball 里除了库,还带着消费者在构建期要跑的**代码生成器** | `pkgs/c/compat.protobuf.lua`(`protoc`) | `targets` 里一条 `kind = "bin"` + `main`,配 `required_features` |
 
 完整的样例索引见[根 README 的「参考示例」表](../../README.zh-CN.md#参考示例lua-描述符)。
 
@@ -184,6 +185,48 @@ runtime = {
 - **闭包必须完整**。农场里有 `libxcb.so.1` 却没有它依赖的 `libXau.so.6`,会遮蔽掉本来能解析的宿主副本,可执行
   文件直接起不来。
 
+## H. 宿主工具提供方(`compat.protobuf` 的 `protoc`)
+
+有些 tarball 里同时装着一个库,和「针对这个库生成代码」的那个生成器。把生成器声明成第二个 target,
+消费者用 `tools = [...]` 索取(mcpp 2026.8.5.1 起):
+
+```lua
+targets = {
+    ["protobuf"] = { kind = "lib" },
+    ["protoc"]   = { kind = "bin",
+                     main = "*/src/google/protobuf/compiler/main.cc",
+                     required_features = { "protoc", "upb" } },
+},
+features = {
+    ["protoc"] = { sources = { … 编译器自身的源码 … } },
+},
+```
+
+```toml
+# 消费者侧 —— 一条依赖,两种角色
+compat.protobuf = { version = "35.1", tools = ["protoc"] }
+```
+
+mcpp 会在一次嵌套子构建里把这个 target 编成**构建机**的二进制,并把路径经
+`mcpp::dep_bin("protobuf", "protoc")` 交给消费者的 `build.mcpp`。这个形态值得单列的全部理由在于:
+工具的版本**就是**那条依赖的版本,于是「生成器与运行时版本错配」——在别处是**运行期**才炸、也正是
+protobuf 最经典的坑——在这里**语法上无法表达**。`mcpp build --target <triple>` 下工具仍为宿主构建,
+因为代码生成器必须在本机跑。
+
+四个要点:
+
+- **把编译器的源码关进一个 feature**,并在 target 的 `required_features` 里写明。只链库的消费者不该为
+  生成器的 TU 买单;索取工具的消费者也不该需要知道它要哪些 feature。`compat.protobuf` 的 `protoc` 还
+  必须要 `upb`,因为 libprotoc 的 upb 生成器要链 upb 运行时——搞错了会在**链接期**缺一批 `upb_*` 符号。
+- **源码列表照旧逐条转录自上游**:protobuf 这 138 项来自它自己的 `src/file_lists.cmake` 的 `libprotoc_srcs`。
+- **`main` 和 `sources` 一样需要 `*/` 那层 wrap glob**,展开方式相同。
+- **运行期还要读数据文件的生成器,仍然需要一个路径**。protoc 并不内嵌 well-known types:
+  `import "google/protobuf/timestamp.proto"` 是从磁盘读的。消费者用 `mcpp::dep_dir("protobuf")` 推出那个
+  目录——见 `tests/examples/protobuf-protoc/build.mcpp`。
+
+对应的成员是 `tests/examples/protobuf-protoc`,它与 `tests/examples/protobuf` 互为补集:那个刻意**不用**
+任何生成代码,这个从头到尾都是生成代码。
+
 ---
 
 ## 最小工程(`tests/examples/<short>/`)