diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e96666b4..c172136f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -12,8 +12,36 @@ on: # nightly full regression — exercises every workspace member regardless of diff - cron: "0 6 * * *" workflow_dispatch: + inputs: + cache: + description: "Package build cache — 'local' rebuilds every dependency per member, which is what the timing table should be read against when comparing" + type: choice + options: [global, local] + default: global env: + # 2026.8.5.4 carries two things this workflow depends on: + # .5.1 `tools = [...]` — how a consumer asks for a dependency's + # `kind = "bin"` target, which tests/examples/protobuf-protoc is built + # on. Before it: "tools must be a string, inline dep table, or nested + # table". + # .5.4 windows links with lld. link.exe caps a response-file LINE at + # 128 KiB and opencv-module / opencv-module-dnn went past it — + # fatal error LNK1170: line in command file contains 135135 or + # more characters + # after 795s / 1166s of compiling. .5.3 newline-separated OUR response + # file, which was necessary but not sufficient: clang, acting as the + # driver, writes a SECOND one for the linker that we do not control. + # lld's response-file parser has no per-line limit at all. + # Together with re-enabling the global package cache below, this is + # what makes a green FULL run possible again: .5.3 removes the + # windows link failure, the cache removes the 150-minute timeout. + # + # Neither of them moves index.toml's min_mcpp: exposing compat.protobuf's `protoc` + # target is additive, and 2026.8.3.3 still parses that descriptor with an + # empty unknown_keys. The floor an index publishes decides whether older + # clients keep working at all (mcpp#349), so it moves only when a descriptor + # genuinely stops being readable — which is not the case here. # 2026.8.3.1: on macOS, a global object that touches std::cout during static # init crashes on sight (mcpp#336). Mach-O has no priority-ordered init # section and libc++'s carries no ios_base::Init guard of its own, @@ -88,7 +116,7 @@ env: # 0.0.94 fixed feature-gated `sources` under `mcpp test` (mcpp#218); 0.0.91 # added standard = "c++fly" to the resolver grammar, so c++fly descriptors # get the lint WARN below, not a hard grammar-parse rejection. - MCPP_VERSION: "2026.8.3.3" + MCPP_VERSION: "2026.8.5.4" jobs: lint: @@ -236,13 +264,124 @@ jobs: # `dnn` feature members. The registry cache (restore-keys prefix below) # amortizes those across subsequent runs. 150 covers the one-time cold full # build with headroom; it is a ceiling, not a target. + # ── The plan, computed ONCE ─────────────────────────────────────────── + # Was inlined in every workspace job — three runners each re-deriving the + # same answer. It now also has to be decided BEFORE the matrix exists, + # because the matrix's shard dimension depends on it: a full run fans out, + # a selective one does not. + select: + runs-on: ubuntu-latest + outputs: + members: ${{ steps.fanout.outputs.members }} + shard_count: ${{ steps.fanout.outputs.shard_count }} + shards: ${{ steps.fanout.outputs.shards }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # ── Selective member testing ────────────────────────────────────── + # `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and + # dominates CI wall-clock, while a PR almost always touches one + # package. Map changed files → affected members and test only those: + # pkgs//.lua → members whose mcpp.toml references + # tests/examples//** → member + # Run the FULL workspace when the change can affect everything: + # non-PR events (push to main, the nightly cron, dispatch), this + # workflow file (it carries the mcpp version pins, so a version bump + # always re-validates every package), a non-member edit to the + # workspace manifest, or shared test scripts. Docs-only and tools/-only + # changes select nothing. + # Note: bash 3.2 on macOS runners — no associative arrays here. + - name: Select affected workspace members + id: plan + shell: bash + run: | + full() { echo "MEMBERS=__ALL__" >> "$GITHUB_ENV"; echo "full run: $1"; exit 0; } + [ "${{ github.event_name }}" = "pull_request" ] || full "event=${{ github.event_name }}" + base="origin/${{ github.base_ref }}" + changed=$(git diff --name-only "$base"...HEAD) + printf 'changed files vs %s:\n%s\n' "$base" "$changed" + sel="" + add() { case " $sel " in *" $1 "*) ;; *) sel="$sel $1" ;; esac; } + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + .github/workflows/validate.yml|tests/*.sh) full "$f" ;; + mcpp.toml) + # Workspace manifest. Every new-package PR appends to the + # members list, so that alone must NOT force a full run: + # select the added members; anything else in this file + # (indices, settings) affects everyone → full. + if ! diff -q <(git show "$base:mcpp.toml" | grep -v 'tests/examples/') \ + <(grep -v 'tests/examples/' mcpp.toml) >/dev/null; then + full "mcpp.toml non-member change" + fi + for p in $(comm -13 <(git show "$base:mcpp.toml" | grep -o 'tests/examples/[A-Za-z0-9._-]*' | sort -u) \ + <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml | sort -u)); do + add "${p#tests/examples/}" + done ;; + tests/examples/*) + m=${f#tests/examples/}; m=${m%%/*} + # A deleted/renamed member dir implies a mcpp.toml edit, + # which already forces a full run above. + [ -d "tests/examples/$m" ] && add "$m" ;; + pkgs/*.lua|pkgs/*/*.lua) + lib=$(basename "$f" .lua); lib=${lib#compat.} + hit=0 + for mt in tests/examples/*/mcpp.toml; do + if grep -q "$lib" "$mt"; then add "$(basename "$(dirname "$mt")")"; hit=1; fi + done + [ "$hit" = 1 ] || echo "note: no workspace member exercises $f" ;; + # tools/ holds OFFLINE descriptor-generation and publishing + # helpers (tools/compat-*/, tools/gtc/, publish_mcpp_index.sh). + # Nothing under it is consumed by a package build: when one of + # them actually changes a package, the generated pkgs/*.lua + # changes with it and the rule above selects the right members. + # So a tools/ edit alone selects nothing rather than forcing a + # full workspace rebuild. + *.md|docs/*|.agents/*|.github/*|tools/*) : ;; + *) full "unclassified change: $f" ;; + esac + done <> "$GITHUB_ENV" + echo "selected members: ${sel:-}" + # Sharding is for the FULL run only. A selective run is a handful of + # members; splitting it 8 ways would add 7 runners' worth of checkout, + # mcpp download and cache restore to save nothing. + - name: Decide the fan-out + id: fanout + shell: bash + run: | + if [ "$MEMBERS" = "__ALL__" ]; then + echo 'shards=[0,1,2,3,4,5,6,7]' >> "$GITHUB_OUTPUT" + echo 'shard_count=8' >> "$GITHUB_OUTPUT" + else + echo 'shards=[0]' >> "$GITHUB_OUTPUT" + echo 'shard_count=1' >> "$GITHUB_OUTPUT" + fi + echo "members=$MEMBERS" >> "$GITHUB_OUTPUT" + workspace: - name: workspace (${{ matrix.platform }}) + # The shard is in the name only when there is more than one, so a + # selective run still reads "workspace (linux)". + name: workspace (${{ matrix.platform }}${{ needs.select.outputs.shard_count == '1' && '' || format(' {0}/{1}', matrix.shard, needs.select.outputs.shard_count) }}) + needs: select + if: needs.select.outputs.members != '' runs-on: ${{ matrix.os }} - timeout-minutes: 150 + # Was 150 and a full linux run hit it exactly. One shard is ~1/8 of the + # work, so this is now a real ceiling rather than the thing that decides + # whether the job finishes. + timeout-minutes: 90 strategy: fail-fast: false matrix: + # Fanned out ONLY for a full run — `shards` is [0] otherwise, which + # collapses this back to one job per platform. + shard: ${{ fromJSON(needs.select.outputs.shards) }} + platform: [linux, macos, windows] include: # Archive names are derived from env.MCPP_VERSION in the Download # step — bumping the pin is a ONE-line change (hardcoded versions @@ -253,21 +392,21 @@ jobs: ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION - platform: macos os: macos-15 suffix: macosx-arm64 ext: tar.gz mcpp: bin/mcpp xlings: registry/bin/xlings - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION - platform: windows os: windows-latest suffix: windows-x86_64 ext: zip mcpp: bin/mcpp.exe xlings: registry/bin/xlings.exe - mcpp_version: "2026.8.3.3" # keep in sync with env.MCPP_VERSION + mcpp_version: "2026.8.5.4" # keep in sync with env.MCPP_VERSION env: MCPP_EFFECTIVE: ${{ matrix.mcpp_version }} steps: @@ -340,74 +479,29 @@ jobs: # plans, mcpp#232). The sandbox copy lands in ~/.mcpp/registry, so # the cache carries it across runs. - # ── Selective member testing ────────────────────────────────────── - # `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and - # dominates CI wall-clock, while a PR almost always touches one - # package. Map changed files → affected members and test only those: - # pkgs//.lua → members whose mcpp.toml references - # tests/examples//** → member - # Run the FULL workspace when the change can affect everything: - # non-PR events (push to main, the nightly cron, dispatch), this - # workflow file (it carries the mcpp version pins, so a version bump - # always re-validates every package), a non-member edit to the - # workspace manifest, or shared test scripts. Docs-only and tools/-only - # changes select nothing. - # Note: bash 3.2 on macOS runners — no associative arrays here. - - name: Select affected workspace members + # ── This shard's slice of the plan ──────────────────────────────── + # `select` decided WHAT runs; this decides which part of it runs HERE. + # Round-robin by position, which is what spreads the expensive members: + # opencv-module / -dnn / -unifont are adjacent in the list, so `% N` + # necessarily puts them on three different runners. A single job that + # builds all three spends 45+ minutes on opencv alone. + - name: Take this shard's members shell: bash run: | - full() { echo "MEMBERS=__ALL__" >> "$GITHUB_ENV"; echo "full run: $1"; exit 0; } - [ "${{ github.event_name }}" = "pull_request" ] || full "event=${{ github.event_name }}" - base="origin/${{ github.base_ref }}" - changed=$(git diff --name-only "$base"...HEAD) - printf 'changed files vs %s:\n%s\n' "$base" "$changed" - sel="" - add() { case " $sel " in *" $1 "*) ;; *) sel="$sel $1" ;; esac; } - while IFS= read -r f; do - [ -n "$f" ] || continue - case "$f" in - .github/workflows/validate.yml|tests/*.sh) full "$f" ;; - mcpp.toml) - # Workspace manifest. Every new-package PR appends to the - # members list, so that alone must NOT force a full run: - # select the added members; anything else in this file - # (indices, settings) affects everyone → full. - if ! diff -q <(git show "$base:mcpp.toml" | grep -v 'tests/examples/') \ - <(grep -v 'tests/examples/' mcpp.toml) >/dev/null; then - full "mcpp.toml non-member change" - fi - for p in $(comm -13 <(git show "$base:mcpp.toml" | grep -o 'tests/examples/[A-Za-z0-9._-]*' | sort -u) \ - <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml | sort -u)); do - add "${p#tests/examples/}" - done ;; - tests/examples/*) - m=${f#tests/examples/}; m=${m%%/*} - # A deleted/renamed member dir implies a mcpp.toml edit, - # which already forces a full run above. - [ -d "tests/examples/$m" ] && add "$m" ;; - pkgs/*.lua|pkgs/*/*.lua) - lib=$(basename "$f" .lua); lib=${lib#compat.} - hit=0 - for mt in tests/examples/*/mcpp.toml; do - if grep -q "$lib" "$mt"; then add "$(basename "$(dirname "$mt")")"; hit=1; fi - done - [ "$hit" = 1 ] || echo "note: no workspace member exercises $f" ;; - # tools/ holds OFFLINE descriptor-generation and publishing - # helpers (tools/compat-*/, tools/gtc/, publish_mcpp_index.sh). - # Nothing under it is consumed by a package build: when one of - # them actually changes a package, the generated pkgs/*.lua - # changes with it and the rule above selects the right members. - # So a tools/ edit alone selects nothing rather than forcing a - # full workspace rebuild. - *.md|docs/*|.agents/*|.github/*|tools/*) : ;; - *) full "unclassified change: $f" ;; - esac - done <> "$GITHUB_ENV" - echo "selected members: ${sel:-}" + plan='${{ needs.select.outputs.members }}' + if [ "$plan" = "__ALL__" ]; then + plan=$(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml \ + | sed 's|tests/examples/||' | sort -u | tr '\n' ' ') + fi + n=${{ needs.select.outputs.shard_count }} + mine=""; i=0 + for m in $plan; do + [ $((i % n)) -eq ${{ matrix.shard }} ] && mine="$mine $m" + i=$((i + 1)) + done + mine=${mine# } + echo "MEMBERS=$mine" >> "$GITHUB_ENV" + echo "shard ${{ matrix.shard }}/$n of $i member(s): ${mine:-}" # ── Refresh the PUBLISHED index before testing ──────────────────── # Most members resolve everything from this checkout, but a member that @@ -437,38 +531,58 @@ jobs: shell: bash env: MCPP_INDEX_MIRROR: GLOBAL - # Dependencies build inside each member's own target/ instead of - # through the global package build cache (mcpp >= 2026.7.30.2). - # That cache is unusable here: mcpp#233's object-path disambiguation - # fires on basename collisions across the WHOLE build dir — i.e. on - # which packages the CONSUMER pulls in — while the cache key covers - # only the dependency itself, so one entry can hold two different - # layouts. `tests/examples/archive` pulls zlib AND bzip2 (both ship - # compress.c) and stores obj/compat_zlib/zlib-1.3.2/compress.o; - # every zlib consumer without bzip2 then asks the same key for a - # flat obj/compress.o and ninja dies at graph time with - # "missing and no known rule to make it". Reproduced both ways round - # on 2026.8.3.3 and filed as mcpp-community/mcpp#344; drop this once - # it lands. `local` still caches the std BMI, which is the expensive - # one — only package entries are bypassed. - MCPP_BUILD_CACHE: local + # The GLOBAL package build cache is on (mcpp >= 2026.7.30.2), which + # is the default — this step used to set `MCPP_BUILD_CACHE: local` + # and no longer does. + # + # That bypass existed for mcpp#344: object-path disambiguation fires + # on basename collisions across the WHOLE build dir — i.e. on what + # the CONSUMER pulls in — while the cache key covered only the + # dependency, so one entry could hold two layouts and ninja died at + # graph time with "missing and no known rule to make it". #344 + # landed in 2026.8.3.4 with per-package Merkle keys that cover the + # consumer-dependent layout, so the reason is gone. + # + # Keeping it cost real time, and the full run is where it showed: + # with `local`, EVERY member recompiles EVERY dependency from + # scratch. 59 members that mostly share abseil / protobuf / opencv + # meant the same sources were built over and over — + # + # linux 2h30m -> cancelled at the 150-minute timeout + # windows 2h20m + # macos 1h26m + # + # — and a workspace cannot be validated by a job that cannot finish. + # With the cache on, a given (package, version, features, toolchain) + # is built once per run and every later member hits it. run: | "$MCPP" --version # No `timeout` wrapper: absent on macOS runners; job-level timeout-minutes bounds it. - if [ "$MEMBERS" = "__ALL__" ]; then - "$MCPP" test --workspace - elif [ -z "$MEMBERS" ]; then + # One code path: the shard step above already expanded `__ALL__` + # into this runner's actual member names, so `mcpp test --workspace` + # — which would ignore the sharding and rebuild everything here — is + # gone. + # + # tests/run_members.sh is the SAME script you run locally. A timing + # table that only exists in CI cannot be used while deciding what to + # optimise, and a local harness that differs from CI measures + # something else. + if [ -z "$MEMBERS" ]; then echo "No workspace member affected by this change — nothing to test." else - rc=0 - for m in $MEMBERS; do - echo "::group::mcpp test -p $m" - "$MCPP" test -p "$m" || rc=1 - echo "::endgroup::" - done - exit $rc + MCPP_TIMINGS="$PWD/timings.tsv" bash tests/run_members.sh $MEMBERS fi + # Per-shard timings, merged by the `timings` job below. `always()`: a + # run that failed is exactly when knowing where the time went matters. + - name: Upload this shard's timings + if: always() && hashFiles('timings.tsv') != '' + uses: actions/upload-artifact@v4 + with: + name: timings-${{ matrix.platform }}-${{ matrix.shard }} + path: timings.tsv + retention-days: 14 + # install()-driven packages (openssl, openblas) build through their own # Make/Configure system, whose output xim's interface mode swallows; a # failed hook surfaces only as `E_INTERNAL: [] failed:`. Each writes @@ -487,3 +601,57 @@ jobs: echo "::endgroup::" done < <(find tests/examples "$HOME/.mcpp/registry" -name 'mcpp_*_build.log' 2>/dev/null) [ "$found" = 1 ] || echo "no install() build logs found" + + # ── Where the time went ─────────────────────────────────────────────── + # Sharding hides the cost: eight runners each report their own slice, and + # nobody can see which members actually dominate. This merges them into one + # ranking per platform, in the run summary, so the next optimisation starts + # from measurement instead of a guess. + # + # `always()` — a failed run is exactly when this is worth reading. + timings: + needs: [select, workspace] + if: always() && needs.select.outputs.members != '' + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + pattern: timings-* + path: timings + continue-on-error: true + - name: Rank members by wall-clock + shell: bash + run: | + shopt -s nullglob + files=(timings/*/timings.tsv) + if [ ${#files[@]} -eq 0 ]; then + echo "no timing data (every shard skipped or failed before testing)" \ + >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Artifact name carries the platform: timings--. + for plat in linux macos windows; do + rows=$(mktemp) + for f in timings/timings-$plat-*/timings.tsv; do + [ -f "$f" ] && cat "$f" >> "$rows" + done + [ -s "$rows" ] || { rm -f "$rows"; continue; } + + total=$(awk -F'\t' '{s += $1} END {print s+0}' "$rows") + count=$(wc -l < "$rows") + { + echo "### $plat — ${count} member(s), ${total}s of member wall-clock" + echo + echo "| rank | seconds | share | member | result |" + echo "|---:|---:|---:|---|---|" + sort -rn "$rows" | awk -F'\t' -v tot="$total" ' + { pct = tot > 0 ? ($1 * 100 / tot) : 0 + printf "| %d | %s | %.1f%% | `%s` | %s |\n", NR, $1, pct, $2, $3 }' + echo + } >> "$GITHUB_STEP_SUMMARY" + rm -f "$rows" + done + + echo "_Total is the SUM across shards; wall-clock is the slowest shard._" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 2ecb6b3d..429508ae 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Two kinds of packages live here: |------|------| | Native module library (Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](pkgs/o/opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) | | C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | +| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | | C++-source compat, zero-dep client + optional components | [`compat.websocket`](pkgs/c/compat.websocket.lua) (IXWebSocket 12.0.1 — a pure RFC 6455 client compiled from upstream's `IXWEBSOCKET_SOURCES` minus the four server TUs, so the **base build has zero external dependencies**: TLS off (the OpenSSL/MbedTLS/AppleSSL TUs aren't built) and `IXWEBSOCKET_USE_ZLIB` unset, so the gzip codec compiles to a no-op. Two optional features add on top: `server` (the four server TUs — `IXWebSocketServer`, `IXSocketServer`, `IXHttpServer`, `IXWebSocketProxyServer` — needing nothing external, and it **implies `zlib`** because upstream's server advertises permessage-deflate by default, which the transport negotiates regardless of the define) and `zlib` (deps `compat.zlib` and turns the codec into real per-message-deflate compression). The default-feature test brings its own minimal RFC 6455 echo server on loopback sockets (handshake, masking, fragmentation and close all exercised offline); a second member, `websocket-features`, runs a real `ix::WebSocketServer` and asserts the compression is observable on the wire — a 64 KiB repeated payload round-trips with `wireSize` = 80) | | header-only (with `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | Runtime loader compat (pure sources, sidestepping upstream codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua) (the Khronos loader: `loader/generated/` is checked in, and the assembly path degrades to plain C through `UNKNOWN_FUNCTIONS_SUPPORTED`, so no CMake/Python/assembler is needed; windows deferred) · [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) | diff --git a/README.zh-CN.md b/README.zh-CN.md index 1b107812..97cfe12d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -36,7 +36,7 @@ mcpp self config --mirror CN # 切换至国内镜像,默认使用 GLOBAL 上 |------|------| | 原生模块库(Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](pkgs/o/opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)| | C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | +| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | | C++ 源码 compat(零依赖客户端 + 可选组件) | [`compat.websocket`](pkgs/c/compat.websocket.lua)(IXWebSocket 12.0.1 —— 从上游 `IXWEBSOCKET_SOURCES` 剔掉 4 个 server TU 后直编的纯 RFC 6455 客户端,**基座零外部依赖**:TLS 关闭(OpenSSL/MbedTLS/AppleSSL 三组 TU 均不编),`IXWEBSOCKET_USE_ZLIB` 不定义(gzip codec 编译为 no-op)。两个可选 feature 在基座上叠加:`server`(4 个 server TU —— `IXWebSocketServer`/`IXSocketServer`/`IXHttpServer`/`IXWebSocketProxyServer`,零新增外部依赖,且 **implies `zlib`** —— 因为上游 server 默认就宣称 permessage-deflate,而 transport 的协商不受宏门控)与 `zlib`(依赖 `compat.zlib`,把 codec 变成真正的 permessage-deflate 压缩)。默认构建的测试自带基于 loopback 原始 socket 的最小 RFC 6455 echo server(握手/掩码/分片/关闭全部离线实测);第二个成员 `websocket-features` 跑真实的 `ix::WebSocketServer`,并断言压缩在线路上可观测 —— 64 KiB 重复载荷往返,`wireSize` = 80) | | header-only(含 `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | 运行时 loader compat(纯源码,绕开上游 codegen/asm) | [`compat.vulkan`](pkgs/c/compat.vulkan.lua)(Khronos loader:`loader/generated/` 已签入,汇编路径经 `UNKNOWN_FUNCTIONS_SUPPORTED` 降级为纯 C,故无需 CMake/Python/汇编器;windows 延后)· [`compat.vulkan-headers`](pkgs/c/compat.vulkan-headers.lua) | diff --git a/docs/package-types.md b/docs/package-types.md index d348f493..3930a093 100644 --- a/docs/package-types.md +++ b/docs/package-types.md @@ -18,6 +18,7 @@ combined as needed. | **E. Whole-source direct build with a generated config** | upstream generates its config header through configure/CMake; here a snapshot of it lands in `generated_files` | `pkgs/c/compat.libpng.lua`, `compat.curl.lua`, `compat.sdl2.lua`, `compat.ffmpeg.lua` | `generated_files` + `include_dirs` | | **F. Shared-library compat** | has to be the **only** copy of that `.so` in the process (third parties `dlopen` it) | the X11 family such as `pkgs/c/compat.x11.lua`, and `compat.vulkan.lua` (linux) | `targets = { kind = "shared", soname = … }` | | **G. Host runtime adaptation** | things that cannot be vendored, such as drivers — only a symlink farm plus metadata | `pkgs/c/compat.glx-runtime.lua`, `compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` | +| **H. Host tool provider** | the upstream tarball also holds a **code generator** consumers run at build time | `pkgs/c/compat.protobuf.lua` (`protoc`) | a `targets` entry with `kind = "bin"` + `main`, plus `required_features` | For the complete sample index, see the [Reference examples table in the root README](../README.md#reference-examples-lua-descriptors). @@ -200,6 +201,50 @@ Two details that keep biting: - **The closure has to be complete.** A farm holding `libxcb.so.1` but not the `libXau.so.6` it depends on shadows the host copy that would otherwise have resolved, and the executable simply fails to start. +## H. Host tool provider (`compat.protobuf`'s `protoc`) + +Some tarballs hold both a library and the code generator that emits code against it. Declare the generator as a second +target, and consumers ask for it with `tools = [...]` (mcpp 2026.8.5.1+): + +```lua +targets = { + ["protobuf"] = { kind = "lib" }, + ["protoc"] = { kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" } }, +}, +features = { + ["protoc"] = { sources = { … the compiler's own sources … } }, +}, +``` + +```toml +# consumer side — one dependency, two roles +compat.protobuf = { version = "35.1", tools = ["protoc"] } +``` + +mcpp then builds that target **for the build machine** in a nested sub-build and hands the path to the consumer's +`build.mcpp` through `mcpp::dep_bin("protobuf", "protoc")`. This is the whole reason the shape is worth naming: the +tool's version **is** the dependency's version, so a generator/runtime mismatch — a *runtime* failure everywhere else, +and the classic protobuf footgun — is not expressible. Under `mcpp build --target ` the tool is still built for +the host, because a code generator has to run here. + +Four things to get right: + +- **Gate the compiler's sources behind a feature**, and name it in the target's `required_features`. Consumers who only + link the library must not pay for the generator's TUs; consumers who ask for the tool must not have to know which + features it needs. `compat.protobuf`'s `protoc` also requires `upb`, because libprotoc's upb generator links the upb + runtime — get that wrong and it fails at **link** time with missing `upb_*` symbols. +- **Transcribe the source list from upstream**, exactly as for a library — protobuf's 138 entries come from + `libprotoc_srcs` in its own `src/file_lists.cmake`. +- **`main` needs the same `*/` wrap glob as `sources`**; it is expanded the same way. +- **A generator that reads data files at runtime still needs a path to them.** protoc does not embed the well-known + types: `import "google/protobuf/timestamp.proto"` is read from disk. Consumers derive that directory from + `mcpp::dep_dir("protobuf")` — see `tests/examples/protobuf-protoc/build.mcpp`. + +The matching member is `tests/examples/protobuf-protoc`, and it is the complement of `tests/examples/protobuf`: that +one deliberately uses no generated code, this one is generated code end to end. + --- ## The minimal project (`tests/examples//`) diff --git a/docs/zh/package-types.md b/docs/zh/package-types.md index 4ffc00eb..becdcd33 100644 --- a/docs/zh/package-types.md +++ b/docs/zh/package-types.md @@ -16,6 +16,7 @@ A–D 是四种**基础**形态,先按它们判定;E–G 是在基础形态之 | **E. 生成 config 的全源码直编** | 上游用 configure/CMake 生成配置头,此处以 `generated_files` 落一份快照 | `pkgs/c/compat.libpng.lua`、`compat.curl.lua`、`compat.sdl2.lua`、`compat.ffmpeg.lua` | `generated_files` + `include_dirs` | | **F. 共享库 compat** | 必须是**唯一**的那个 `.so`(会被第三方 `dlopen`) | `pkgs/c/compat.x11.lua` 等 X11 家族、`compat.vulkan.lua`(linux) | `targets = { kind = "shared", soname = … }` | | **G. 宿主运行时适配** | 驱动之类无法 vendor 的东西,只做符号链接农场 + 元数据 | `pkgs/c/compat.glx-runtime.lua`、`compat.vulkan-runtime.lua` | `runtime.library_dirs` / `capabilities` | +| **H. 宿主工具提供方** | 上游 tarball 里除了库,还带着消费者在构建期要跑的**代码生成器** | `pkgs/c/compat.protobuf.lua`(`protoc`) | `targets` 里一条 `kind = "bin"` + `main`,配 `required_features` | 完整的样例索引见[根 README 的「参考示例」表](../../README.zh-CN.md#参考示例lua-描述符)。 @@ -184,6 +185,48 @@ runtime = { - **闭包必须完整**。农场里有 `libxcb.so.1` 却没有它依赖的 `libXau.so.6`,会遮蔽掉本来能解析的宿主副本,可执行 文件直接起不来。 +## H. 宿主工具提供方(`compat.protobuf` 的 `protoc`) + +有些 tarball 里同时装着一个库,和「针对这个库生成代码」的那个生成器。把生成器声明成第二个 target, +消费者用 `tools = [...]` 索取(mcpp 2026.8.5.1 起): + +```lua +targets = { + ["protobuf"] = { kind = "lib" }, + ["protoc"] = { kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" } }, +}, +features = { + ["protoc"] = { sources = { … 编译器自身的源码 … } }, +}, +``` + +```toml +# 消费者侧 —— 一条依赖,两种角色 +compat.protobuf = { version = "35.1", tools = ["protoc"] } +``` + +mcpp 会在一次嵌套子构建里把这个 target 编成**构建机**的二进制,并把路径经 +`mcpp::dep_bin("protobuf", "protoc")` 交给消费者的 `build.mcpp`。这个形态值得单列的全部理由在于: +工具的版本**就是**那条依赖的版本,于是「生成器与运行时版本错配」——在别处是**运行期**才炸、也正是 +protobuf 最经典的坑——在这里**语法上无法表达**。`mcpp build --target ` 下工具仍为宿主构建, +因为代码生成器必须在本机跑。 + +四个要点: + +- **把编译器的源码关进一个 feature**,并在 target 的 `required_features` 里写明。只链库的消费者不该为 + 生成器的 TU 买单;索取工具的消费者也不该需要知道它要哪些 feature。`compat.protobuf` 的 `protoc` 还 + 必须要 `upb`,因为 libprotoc 的 upb 生成器要链 upb 运行时——搞错了会在**链接期**缺一批 `upb_*` 符号。 +- **源码列表照旧逐条转录自上游**:protobuf 这 138 项来自它自己的 `src/file_lists.cmake` 的 `libprotoc_srcs`。 +- **`main` 和 `sources` 一样需要 `*/` 那层 wrap glob**,展开方式相同。 +- **运行期还要读数据文件的生成器,仍然需要一个路径**。protoc 并不内嵌 well-known types: + `import "google/protobuf/timestamp.proto"` 是从磁盘读的。消费者用 `mcpp::dep_dir("protobuf")` 推出那个 + 目录——见 `tests/examples/protobuf-protoc/build.mcpp`。 + +对应的成员是 `tests/examples/protobuf-protoc`,它与 `tests/examples/protobuf` 互为补集:那个刻意**不用** +任何生成代码,这个从头到尾都是生成代码。 + --- ## 最小工程(`tests/examples//`) diff --git a/mcpp.toml b/mcpp.toml index 908d98da..f192b49f 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -47,6 +47,7 @@ members = [ "tests/examples/protobuf", "tests/examples/protobuf-upb", "tests/examples/protobuf-gzip", + "tests/examples/protobuf-protoc", "tests/examples/opencv-module", "tests/examples/opencv-module-dnn", "tests/examples/opencv-module-unifont", diff --git a/pkgs/c/compat.protobuf.lua b/pkgs/c/compat.protobuf.lua index fb5ab549..7d7c7b01 100644 --- a/pkgs/c/compat.protobuf.lua +++ b/pkgs/c/compat.protobuf.lua @@ -14,14 +14,22 @@ -- source list covers linux/macosx/windows and the three xpm blocks share -- a single tarball and sha256. -- --- SCOPE — runtime only. This package builds upstream's `libprotobuf` target --- (79 TUs), i.e. what a program that *uses* generated code needs: messages, --- reflection, descriptors, text/JSON formats, the well-known types. It does --- NOT build `libprotoc` (a further 157 TUs) and ships no protoc binary, so it --- does not generate .pb.cc from .proto. Consumers either check in --- protoc-generated sources or build them with the official upstream protoc --- release (protoc-35.1-.zip); wiring that into an mcpp build belongs --- to a build.mcpp step, not to this descriptor. +-- SCOPE — runtime by default, compiler on request. This package builds +-- upstream's `libprotobuf` target (79 TUs) unconditionally: messages, +-- reflection, descriptors, text/JSON formats, the well-known types — what a +-- program that *uses* generated code needs. +-- +-- Since mcpp 2026.8.5.1 it ALSO offers `protoc` as a host tool, behind the +-- `protoc` feature (upstream's `libprotoc`, 138 further TUs). A consumer that +-- only links the runtime compiles none of them: +-- +-- compat.protobuf = { version = "35.1", tools = ["protoc"] } +-- +-- That replaces the old advice of "check in protoc output, or fetch the +-- official protoc-35.1-.zip and keep its version in step by hand". +-- Keeping it in step by hand is precisely the failure this removes: a protoc +-- that disagrees with the runtime fails at RUNTIME, and here the tool's +-- version IS this package's version, so the mismatch cannot be expressed. -- -- Version numbering follows upstream verbatim: `35.1` is the protobuf release, -- and it is what gRPC 1.83.0 pins (its third_party/protobuf submodule is @@ -194,13 +202,187 @@ package = { "*/third_party/utf8_range/utf8_range.c", }, - targets = { ["protobuf"] = { kind = "lib" } }, + targets = { + ["protobuf"] = { kind = "lib" }, + -- #355 (mcpp 2026.8.5.1+): protoc as a HOST tool a consumer can ask + -- for, so it never has to supply a matching one by hand: + -- + -- compat.protobuf = { version = "35.1", tools = ["protoc"] } + -- + -- The version axis is what matters here. protoc generating code for + -- a DIFFERENT protobuf runtime than the one being linked fails at + -- RUNTIME, not at compile time, and is the single nastiest thing + -- about hand-managed protobuf codegen. Because the tool's version + -- IS this package's version, that mismatch is not expressible. + -- + -- `required_features` is a GATE in an ordinary build (the target is + -- simply absent) and an INPUT in a tool sub-build (the target is + -- what was asked for, so mcpp activates them). Both are needed: + -- `protoc` for libprotoc itself, `upb` because libprotoc's upb + -- generator links the upb runtime — leaving it out fails at LINK + -- with undefined upb_* symbols. + ["protoc"] = { + kind = "bin", + main = "*/src/google/protobuf/compiler/main.cc", + required_features = { "protoc", "upb" }, + }, + }, -- protobuf's public headers #include "absl/…" directly, so Abseil is -- part of this package's interface, not an implementation detail. deps = { ["compat.abseil"] = "20250512.1" }, features = { + -- #355: libprotoc — the protobuf COMPILER library, which the `protoc` + -- target links. 138 TUs, transcribed from upstream's own + -- `src/file_lists.cmake` `libprotoc_srcs` (not hand-picked), and with + -- ZERO overlap against the runtime source set above: importer.cc and + -- parser.cc are already there. + -- + -- Off by default, and that is the whole point — a consumer that only + -- links the protobuf runtime must not compile these. + ["protoc"] = { + sources = { + "*/src/google/protobuf/compiler/code_generator.cc", + "*/src/google/protobuf/compiler/code_generator_lite.cc", + "*/src/google/protobuf/compiler/command_line_interface.cc", + "*/src/google/protobuf/compiler/cpp/enum.cc", + "*/src/google/protobuf/compiler/cpp/extension.cc", + "*/src/google/protobuf/compiler/cpp/field.cc", + "*/src/google/protobuf/compiler/cpp/field_chunk.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/map_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/message_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/string_field.cc", + "*/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc", + "*/src/google/protobuf/compiler/cpp/file.cc", + "*/src/google/protobuf/compiler/cpp/generator.cc", + "*/src/google/protobuf/compiler/cpp/helpers.cc", + "*/src/google/protobuf/compiler/cpp/ifndef_guard.cc", + "*/src/google/protobuf/compiler/cpp/message.cc", + "*/src/google/protobuf/compiler/cpp/message_layout_helper.cc", + "*/src/google/protobuf/compiler/cpp/namespace_printer.cc", + "*/src/google/protobuf/compiler/cpp/parse_function_generator.cc", + "*/src/google/protobuf/compiler/cpp/service.cc", + "*/src/google/protobuf/compiler/cpp/tracker.cc", + "*/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc", + "*/src/google/protobuf/compiler/csharp/csharp_enum.cc", + "*/src/google/protobuf/compiler/csharp/csharp_enum_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_field_base.cc", + "*/src/google/protobuf/compiler/csharp/csharp_generator.cc", + "*/src/google/protobuf/compiler/csharp/csharp_helpers.cc", + "*/src/google/protobuf/compiler/csharp/csharp_map_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_message.cc", + "*/src/google/protobuf/compiler/csharp/csharp_message_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_primitive_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_message_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc", + "*/src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc", + "*/src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc", + "*/src/google/protobuf/compiler/csharp/names.cc", + "*/src/google/protobuf/compiler/java/context.cc", + "*/src/google/protobuf/compiler/java/doc_comment.cc", + "*/src/google/protobuf/compiler/java/field_common.cc", + "*/src/google/protobuf/compiler/java/file.cc", + "*/src/google/protobuf/compiler/java/full/enum.cc", + "*/src/google/protobuf/compiler/java/full/enum_field.cc", + "*/src/google/protobuf/compiler/java/full/extension.cc", + "*/src/google/protobuf/compiler/java/full/generator_factory.cc", + "*/src/google/protobuf/compiler/java/full/make_field_gens.cc", + "*/src/google/protobuf/compiler/java/full/map_field.cc", + "*/src/google/protobuf/compiler/java/full/message.cc", + "*/src/google/protobuf/compiler/java/full/message_builder.cc", + "*/src/google/protobuf/compiler/java/full/message_field.cc", + "*/src/google/protobuf/compiler/java/full/primitive_field.cc", + "*/src/google/protobuf/compiler/java/full/service.cc", + "*/src/google/protobuf/compiler/java/full/string_field.cc", + "*/src/google/protobuf/compiler/java/generator.cc", + "*/src/google/protobuf/compiler/java/helpers.cc", + "*/src/google/protobuf/compiler/java/internal_helpers.cc", + "*/src/google/protobuf/compiler/java/java_features.pb.cc", + "*/src/google/protobuf/compiler/java/lite/enum.cc", + "*/src/google/protobuf/compiler/java/lite/enum_field.cc", + "*/src/google/protobuf/compiler/java/lite/extension.cc", + "*/src/google/protobuf/compiler/java/lite/generator_factory.cc", + "*/src/google/protobuf/compiler/java/lite/make_field_gens.cc", + "*/src/google/protobuf/compiler/java/lite/map_field.cc", + "*/src/google/protobuf/compiler/java/lite/message.cc", + "*/src/google/protobuf/compiler/java/lite/message_builder.cc", + "*/src/google/protobuf/compiler/java/lite/message_field.cc", + "*/src/google/protobuf/compiler/java/lite/primitive_field.cc", + "*/src/google/protobuf/compiler/java/lite/string_field.cc", + "*/src/google/protobuf/compiler/java/message_serialization.cc", + "*/src/google/protobuf/compiler/java/name_resolver.cc", + "*/src/google/protobuf/compiler/java/names.cc", + "*/src/google/protobuf/compiler/java/shared_code_generator.cc", + "*/src/google/protobuf/compiler/kotlin/field.cc", + "*/src/google/protobuf/compiler/kotlin/file.cc", + "*/src/google/protobuf/compiler/kotlin/generator.cc", + "*/src/google/protobuf/compiler/kotlin/message.cc", + "*/src/google/protobuf/compiler/objectivec/enum.cc", + "*/src/google/protobuf/compiler/objectivec/enum_field.cc", + "*/src/google/protobuf/compiler/objectivec/extension.cc", + "*/src/google/protobuf/compiler/objectivec/field.cc", + "*/src/google/protobuf/compiler/objectivec/file.cc", + "*/src/google/protobuf/compiler/objectivec/generator.cc", + "*/src/google/protobuf/compiler/objectivec/helpers.cc", + "*/src/google/protobuf/compiler/objectivec/import_writer.cc", + "*/src/google/protobuf/compiler/objectivec/line_consumer.cc", + "*/src/google/protobuf/compiler/objectivec/map_field.cc", + "*/src/google/protobuf/compiler/objectivec/message.cc", + "*/src/google/protobuf/compiler/objectivec/message_field.cc", + "*/src/google/protobuf/compiler/objectivec/names.cc", + "*/src/google/protobuf/compiler/objectivec/oneof.cc", + "*/src/google/protobuf/compiler/objectivec/primitive_field.cc", + "*/src/google/protobuf/compiler/objectivec/tf_decode_data.cc", + "*/src/google/protobuf/compiler/php/names.cc", + "*/src/google/protobuf/compiler/php/php_generator.cc", + "*/src/google/protobuf/compiler/plugin.cc", + "*/src/google/protobuf/compiler/plugin.pb.cc", + "*/src/google/protobuf/compiler/python/generator.cc", + "*/src/google/protobuf/compiler/python/helpers.cc", + "*/src/google/protobuf/compiler/python/pyi_generator.cc", + "*/src/google/protobuf/compiler/retention.cc", + "*/src/google/protobuf/compiler/ruby/rbs_generator.cc", + "*/src/google/protobuf/compiler/ruby/ruby_generator.cc", + "*/src/google/protobuf/compiler/rust/accessors/accessor_case.cc", + "*/src/google/protobuf/compiler/rust/accessors/accessors.cc", + "*/src/google/protobuf/compiler/rust/accessors/default_value.cc", + "*/src/google/protobuf/compiler/rust/accessors/map.cc", + "*/src/google/protobuf/compiler/rust/accessors/repeated_field.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_cord.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_message.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc", + "*/src/google/protobuf/compiler/rust/accessors/singular_string.cc", + "*/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc", + "*/src/google/protobuf/compiler/rust/accessors/with_presence.cc", + "*/src/google/protobuf/compiler/rust/context.cc", + "*/src/google/protobuf/compiler/rust/crate_mapping.cc", + "*/src/google/protobuf/compiler/rust/enum.cc", + "*/src/google/protobuf/compiler/rust/extension.cc", + "*/src/google/protobuf/compiler/rust/generator.cc", + "*/src/google/protobuf/compiler/rust/message.cc", + "*/src/google/protobuf/compiler/rust/naming.cc", + "*/src/google/protobuf/compiler/rust/oneof.cc", + "*/src/google/protobuf/compiler/rust/relative_path.cc", + "*/src/google/protobuf/compiler/rust/rust_field_type.cc", + "*/src/google/protobuf/compiler/rust/rust_keywords.cc", + "*/src/google/protobuf/compiler/rust/upb_helpers.cc", + "*/src/google/protobuf/compiler/subprocess.cc", + "*/src/google/protobuf/compiler/versions.cc", + "*/src/google/protobuf/compiler/zip_writer.cc", + "*/upb_generator/common.cc", + "*/upb_generator/common/names.cc", + "*/upb_generator/file_layout.cc", + "*/upb_generator/minitable/names.cc", + "*/upb_generator/minitable/names_internal.cc", + "*/upb_generator/plugin.cc", + }, + }, -- GzipInputStream / GzipOutputStream. io/gzip_stream.cc is wrapped -- head-to-toe in `#if HAVE_ZLIB`, so by default it compiles to an -- empty TU and the package carries no zlib dependency at all; @@ -319,6 +501,35 @@ package = { -- here each package carries its own compile flags, so it has to be -- stated. No extra import libs: -ladvapi32 arrives with abseil. cxxflags = { "-DNOMINMAX", "-DWIN32_LEAN_AND_MEAN", "-D_CRT_SECURE_NO_WARNINGS" }, + + -- NO `protoc` TARGET ON WINDOWS — a platform `targets` replaces the + -- top-level one, so this drops the tool while keeping the library. + -- + -- Not a protobuf problem and not a flags problem: the tool SUB-BUILD + -- fails there. In the same CI run, tests/examples/protobuf, + -- protobuf-upb and protobuf-gzip all pass on windows — the very same + -- abseil + protobuf sources, built as an ordinary dependency. Only + -- the sub-build dies, and only on three abseil TUs whose `.ddi` scan + -- outputs never appear: + -- + -- error: building host tool 'compat.protobuf:protoc' failed + -- error: cannot read 'obj/compat_abseil/…/absl/time/internal/test_util.cc.ddi' + -- …/cctz/src/time_zone_posix.cc.ddi, …/cctz/src/zone_info_source.cc.ddi + -- + -- It is NOT path length (MAX_PATH was the obvious guess and it is + -- wrong: those three relative paths are 31/46/47 chars, while + -- absl/container/internal/hashtablez_sampler_force_weak_definition.cc + -- at 67 compiles fine in the same sub-build). The sub-build's inner + -- ninja output is summarized, so the underlying scan error is not in + -- the log and the cause is UNKNOWN. + -- + -- Declaring the target on a platform where it cannot be built would + -- hand users a failure with no explanation. Left off until the + -- sub-build issue is diagnosed on a windows host; nothing else about + -- this descriptor is windows-gated. + targets = { + ["protobuf"] = { kind = "lib" }, + }, }, }, } diff --git a/tests/examples/protobuf-protoc/build.mcpp b/tests/examples/protobuf-protoc/build.mcpp new file mode 100644 index 00000000..9a0458e7 --- /dev/null +++ b/tests/examples/protobuf-protoc/build.mcpp @@ -0,0 +1,74 @@ +// Generate inventory.pb.{h,cc} with the protoc this build produced. +// +// The work is DECLARED, not done here: `mcpp::action` makes it an edge in the +// build graph, so it re-runs exactly when the .proto changes and a failure is +// attributed to the edge rather than to "build.mcpp exited 1". +#include +#include + +import mcpp; + +namespace fs = std::filesystem; + +// protoc does NOT embed the well-known types. `import +// "google/protobuf/timestamp.proto"` is read from disk like any other import, +// and the files ship inside the protobuf package this project already depends +// on. Probe for the directory that actually contains descriptor.proto instead +// of hardcoding the tarball's wrap-directory name, which is a packaging +// artifact and not part of any contract. +static std::string well_known_types_dir() { + const std::string base = mcpp::dep_dir("protobuf"); + if (base.empty()) return {}; + std::error_code ec; + for (const auto& entry : fs::directory_iterator(base, ec)) { + const fs::path src = entry.path() / "src"; + if (fs::exists(src / "google" / "protobuf" / "descriptor.proto", ec)) + return src.generic_string(); + } + return {}; +} + +int main() { + // No `protoc` target on windows (see the descriptor's windows block), so + // there is nothing to declare and no include dir to add. Returning 0 keeps + // the member building; tests/codegen.cpp compiles to a visible skip. + if (std::string(mcpp::target_os()) == "windows") return 0; + + const std::string root = mcpp::manifest_dir(); + const std::string out = mcpp::out_dir(); + + const char* protoc = mcpp::dep_bin("protobuf", "protoc"); + if (!protoc || !*protoc) { + std::fputs("no protoc: declare protobuf = { version = \"35.1\", " + "tools = [\"protoc\"] }\n", stderr); + return 1; + } + + const std::string wkt = well_known_types_dir(); + if (wkt.empty()) { + std::fputs("cannot locate the well-known .proto files in the protobuf " + "package\n", stderr); + return 1; + } + + const std::string proto = root + "/proto/inventory.proto"; + + // The .pb.h is declared alongside the .pb.cc because the test includes it + // and it must therefore be PRODUCED by this edge. mcpp knows a header is + // not a translation unit and keeps it out of the compile set. + mcpp::action gen; + gen.id = "protoc:inventory"; + gen.role = "source"; + gen.description = "protoc -> inventory"; + gen.arg(protoc) + .arg(("-I" + root + "/proto").c_str()) + .arg(("-I" + wkt).c_str()) + .arg(("--cpp_out=" + out).c_str()) + .arg(proto.c_str()) + .input(proto.c_str()) + .output((out + "/inventory.pb.cc").c_str()) + .output((out + "/inventory.pb.h").c_str()) + .submit(); + + mcpp::include_dir(out.c_str()); +} diff --git a/tests/examples/protobuf-protoc/mcpp.toml b/tests/examples/protobuf-protoc/mcpp.toml new file mode 100644 index 00000000..82ae3b1d --- /dev/null +++ b/tests/examples/protobuf-protoc/mcpp.toml @@ -0,0 +1,37 @@ +# protobuf `protoc` target member — the compiler, not the runtime. +# +# The sibling tests/examples/protobuf covers the runtime and deliberately uses +# NO generated code. This member is its complement: every line of the message +# API it touches was emitted, during this build, by a protoc that mcpp built +# from the SAME descriptor that provides the runtime being linked. +# +# That co-provenance is the point. protoc and libprotobuf must agree on the +# generated-code ABI, and a mismatch there is a runtime failure, not a build +# error. Here it is not expressible: `tools = ["protoc"]` makes the tool's +# version the dependency's version. +# +# WINDOWS: linux + macOS only, matching the descriptor — compat.protobuf does +# not declare the `protoc` target on windows, because the tool sub-build fails +# there for reasons not yet diagnosed (see the comment in the descriptor's +# windows block). The dependency below is therefore per-OS, and tests/codegen.cpp +# compiles to a visible skip on windows rather than a test that silently proves +# nothing. +[package] +name = "protobuf-protoc-tests" +version = "0.1.0" +standard = "c++23" + +# One dependency, two roles: `features` shapes what gets LINKED (the runtime), +# `tools` asks for a host binary out of the same package. `protoc` pulls in +# libprotoc's 138 TUs and needs `upb` for the upb generator's runtime — the +# descriptor's `required_features` states that, so asking for the tool is +# enough and this manifest does not have to know it. +[target.'cfg(linux)'.dependencies.compat] +protobuf = { version = "35.1", tools = ["protoc"] } + +[target.'cfg(macos)'.dependencies.compat] +protobuf = { version = "35.1", tools = ["protoc"] } + +# The runtime alone, so the member still builds and links something real here. +[target.'cfg(windows)'.dependencies.compat] +protobuf = "35.1" diff --git a/tests/examples/protobuf-protoc/proto/inventory.proto b/tests/examples/protobuf-protoc/proto/inventory.proto new file mode 100644 index 00000000..edd57967 --- /dev/null +++ b/tests/examples/protobuf-protoc/proto/inventory.proto @@ -0,0 +1,37 @@ +// Small on purpose, but it exercises the generator features that break first +// when protoc and the linked runtime disagree: nested messages, an enum, a +// repeated message field, a map, oneof, and a well-known-type import. +syntax = "proto3"; + +package inventory; + +import "google/protobuf/timestamp.proto"; + +enum Grade { + GRADE_UNKNOWN = 0; + GRADE_A = 1; + GRADE_B = 2; +} + +message Item { + string sku = 1; + int32 quantity = 2; + Grade grade = 3; + + message Dimensions { + double width = 1; + double height = 2; + } + Dimensions dimensions = 4; + + oneof source { + string supplier = 5; + string warehouse = 6; + } +} + +message Inventory { + repeated Item items = 1; + map totals_by_grade = 2; + google.protobuf.Timestamp updated_at = 3; +} diff --git a/tests/examples/protobuf-protoc/tests/codegen.cpp b/tests/examples/protobuf-protoc/tests/codegen.cpp new file mode 100644 index 00000000..d70c0f71 --- /dev/null +++ b/tests/examples/protobuf-protoc/tests/codegen.cpp @@ -0,0 +1,120 @@ +// Behavioral test for compat.protobuf's `protoc` target. +// +// Every type used here was emitted DURING THIS BUILD by a protoc that mcpp +// compiled from the same package that provides the runtime being linked. The +// test therefore asserts the thing that actually matters about a code +// generator shipped as a dependency: that its output and the runtime agree. +// +// What it drives, and what would break first on a generator/runtime mismatch: +// +// nested message + accessors generated_message_reflection.cc +// enum generated_enum_util.cc +// repeated message field repeated_ptr_field.cc +// map field map_field.cc +// oneof the generated case() discriminator +// well-known type import timestamp.pb.cc (proves the -I resolved) +// serialize -> parse round trip wire_format_lite.cc, parse_context.cc +// reflection over generated msg descriptor.cc against the generated pool +// +// Returns non-zero on any mismatch. +#include +#include + +// compat.protobuf declares no `protoc` target on windows, so nothing was +// generated and there is nothing to assert. A loud skip beats a test that +// passes without exercising anything. +#ifdef _WIN32 +int main() { + std::puts("skipped: compat.protobuf has no protoc target on windows"); + return 0; +} +#else + +#include "google/protobuf/util/time_util.h" + +#include "inventory.pb.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char* what) { + if (!ok) { + std::fprintf(stderr, "FAIL: %s\n", what); + ++failures; + } +} + +} // namespace + +int main() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + inventory::Inventory inv; + + inventory::Item* widget = inv.add_items(); + widget->set_sku("WIDGET-1"); + widget->set_quantity(7); + widget->set_grade(inventory::GRADE_A); + widget->mutable_dimensions()->set_width(2.5); + widget->mutable_dimensions()->set_height(4.0); + widget->set_supplier("acme"); + + inventory::Item* gizmo = inv.add_items(); + gizmo->set_sku("GIZMO-2"); + gizmo->set_quantity(3); + gizmo->set_grade(inventory::GRADE_B); + gizmo->set_warehouse("east"); + + (*inv.mutable_totals_by_grade())["A"] = 7; + (*inv.mutable_totals_by_grade())["B"] = 3; + + // The well-known type. Reaching this line at all proves protoc resolved + // the import, and setting it proves timestamp.pb.cc is in the runtime. + *inv.mutable_updated_at() = + google::protobuf::util::TimeUtil::SecondsToTimestamp(1735689600); + + std::string wire; + check(inv.SerializeToString(&wire), "serialize"); + check(!wire.empty(), "wire is non-empty"); + + inventory::Inventory back; + check(back.ParseFromString(wire), "parse"); + + check(back.items_size() == 2, "two items survived the round trip"); + check(back.items(0).sku() == "WIDGET-1", "item 0 sku"); + check(back.items(0).quantity() == 7, "item 0 quantity"); + check(back.items(0).grade() == inventory::GRADE_A, "item 0 enum"); + check(back.items(0).dimensions().width() == 2.5, "nested message field"); + check(back.items(0).source_case() == inventory::Item::kSupplier, + "oneof discriminator (supplier)"); + check(back.items(0).supplier() == "acme", "oneof value"); + check(back.items(1).source_case() == inventory::Item::kWarehouse, + "oneof discriminator (warehouse)"); + check(back.totals_by_grade().size() == 2, "map size"); + check(back.totals_by_grade().at("A") == 7, "map lookup"); + check(google::protobuf::util::TimeUtil::TimestampToSeconds( + back.updated_at()) == 1735689600, + "well-known Timestamp round trip"); + + // Reflection over the generated pool: the descriptor protoc emitted has to + // describe the C++ class it emitted beside it. + const google::protobuf::Descriptor* d = inventory::Item::descriptor(); + check(d != nullptr && d->full_name() == "inventory.Item", + "descriptor full name"); + check(d != nullptr && d->FindFieldByName("sku") != nullptr, + "descriptor knows the sku field"); + check(d != nullptr && d->oneof_decl_count() == 1, + "descriptor knows the oneof"); + + google::protobuf::ShutdownProtobufLibrary(); + + if (failures != 0) { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + std::puts("protoc-generated code round-trips against the linked runtime"); + return 0; +} + +#endif // _WIN32 diff --git a/tests/run_members.sh b/tests/run_members.sh new file mode 100755 index 00000000..31c1172d --- /dev/null +++ b/tests/run_members.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# run_members.sh — run workspace members one by one, timing each. +# +# The same script CI runs and you run locally, on purpose: a timing table that +# only exists in CI cannot be used while deciding what to optimise, and a local +# harness that differs from CI measures something else. +# +# bash tests/run_members.sh --all +# bash tests/run_members.sh opencv-module protobuf +# bash tests/run_members.sh --all --shard 3/8 +# bash tests/run_members.sh --all --cache local # bypass the package cache +# +# Env: +# MCPP path to the mcpp binary (default: `mcpp` on PATH) +# MCPP_TIMINGS where to append `\t\t` rows +# +# Exit status is non-zero if any member failed. The timing table is printed +# regardless — a slow run is worth measuring even when it breaks. +set -u + +MCPP="${MCPP:-mcpp}" +timings="${MCPP_TIMINGS:-}" +cache="" +shard="" +members=() +all=0 + +while [ $# -gt 0 ]; do + case "$1" in + --all) all=1; shift ;; + --shard) shard="$2"; shift 2 ;; + --cache) cache="$2"; shift 2 ;; + --timings) timings="$2"; shift 2 ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + -*) echo "unknown option: $1" >&2; exit 2 ;; + *) members+=("$1"); shift ;; + esac +done + +# `--all` reads the workspace manifest rather than the directory, so a member +# that exists on disk but is not registered is not silently tested. +# +# Both filters below are load-bearing. mcpp.toml's PROSE mentions the path too +# — line 3 says "tests/examples/ — each consumes this repo's own packages", +# which this grep matches with an empty tail, and `mcpp test -p ""` is not a +# useful thing to run. Requiring a real directory also means a name that only +# appears in a comment (`tests/examples/asio-ssl` is discussed in one) cannot +# turn into a phantom member. +if [ "$all" = 1 ]; then + while IFS= read -r m; do + [ -n "$m" ] || continue + [ -d "tests/examples/$m" ] || continue + members+=("$m") + done < <(grep -o 'tests/examples/[A-Za-z0-9._-]*' mcpp.toml \ + | sed 's|tests/examples/||' | sort -u) +fi + +if [ "${#members[@]}" -eq 0 ]; then + echo "no members selected — pass names or --all" >&2 + exit 2 +fi + +# --shard N/M keeps every M-th member starting at N. Round-robin by position, +# which is what separates adjacent expensive members (opencv-module, +# -dnn, -unifont) onto different runners. +if [ -n "$shard" ]; then + idx=${shard%%/*} + cnt=${shard##*/} + picked=() + i=0 + for m in "${members[@]}"; do + [ $((i % cnt)) -eq "$idx" ] && picked+=("$m") + i=$((i + 1)) + done + members=("${picked[@]+"${picked[@]}"}") + echo "shard $idx/$cnt -> ${#members[@]} member(s)" +fi + +[ -n "$cache" ] && export MCPP_BUILD_CACHE="$cache" +echo "cache mode: ${MCPP_BUILD_CACHE:-global (default)}" + +rows=$(mktemp) +trap 'rm -f "$rows"' EXIT +rc=0 + +for m in "${members[@]}"; do + echo "::group::mcpp test -p $m" + t0=$(date +%s) + if "$MCPP" test -p "$m"; then status=ok; else status=FAIL; rc=1; fi + t1=$(date +%s) + echo "::endgroup::" + printf '%s\t%s\t%s\n' "$((t1 - t0))" "$m" "$status" >> "$rows" + printf ' %-34s %5ss %s\n' "$m" "$((t1 - t0))" "$status" +done + +[ -n "$timings" ] && cat "$rows" >> "$timings" + +echo +echo "── slowest members ──────────────────────────────────────────" +total=$(awk -F'\t' '{s += $1} END {print s+0}' "$rows") +sort -rn "$rows" | head -15 | awk -F'\t' -v tot="$total" ' + { pct = tot > 0 ? ($1 * 100 / tot) : 0 + printf " %6ss %5.1f%% %-34s %s\n", $1, pct, $2, $3 }' +echo " ────────" +printf ' %6ss total across %s member(s)\n' "$total" "${#members[@]}" + +exit "$rc"