From 6b96c9362b0c3661cd705b0df1264df188265d41 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Fri, 9 Oct 2026 18:17:17 +1100 Subject: [PATCH] Add published Action and npm smoke workflow Adds .github/workflows/published-action-smoke.yml, run on dispatch and weekly. It checks out the fixture at the v0 tag and runs the Action from stackql/mcp-wringer@v0 and the CLI from the npm registry on Linux x64 and arm64, macOS and Windows: a clean run must report zero findings and a defective run must fail at the high threshold. It also checks that npm latest matches the tag's package version, since npm and the Action must be released from the same commit. The README's npx verification step now says to run it from outside the checkout. Inside the checkout, npx matches the repository's own package name and looks for a bin link that does not exist, which is what produced "'mcp-wringer' is not recognized" during the 0.1.0 release verification; the published package itself was fine. Also records the first consumer, the StackQL repository's pull request gate, and logs three ideas from that integration: a diagnostic when a run stops at durationMs before reaching cases, accepting a UTF-8 BOM in config files, and a spec input on the Action. Verified: both npm CLI commands behave as the workflow expects against the local fixture (exit 0 clean, exit 1 with crash.process-exit defective) from outside the checkout on Windows. actionlint 1.6.23 reports only its outdated runner label list for ubuntu-24.04-arm. Not verified: the workflow itself has not run yet; it needs the v0 tag and is dispatched after merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/published-action-smoke.yml | 157 +++++++++++++++++++ README.md | 7 +- docs/dev/ideas.md | 3 + docs/dev/release-checklist.md | 3 +- 4 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/published-action-smoke.yml diff --git a/.github/workflows/published-action-smoke.yml b/.github/workflows/published-action-smoke.yml new file mode 100644 index 0000000..f6d843e --- /dev/null +++ b/.github/workflows/published-action-smoke.yml @@ -0,0 +1,157 @@ +# Exercises the published artifacts the way a consumer does: the Action +# pulled from the moving v0 tag, and the npm package pulled from the +# registry. Both run against the fixture server at the same tag, so a +# failure here means the published tag or package is broken, not that main +# has drifted. Run it after moving the v0 tag, and on a weekly schedule as a +# canary. +# +# The self-test workflow (action-self-test.yml) covers the checked-out +# source; this one covers what users actually install. + +name: Published Action smoke + +on: + workflow_dispatch: + schedule: + # Mondays 06:00 UTC + - cron: '0 6 * * 1' + +permissions: {} + +env: + # The tag the Action reference below points at. Keep both in step. + ACTION_REF: v0 + +jobs: + action: + name: published Action (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + steps: + - name: Checkout fixture at the Action tag + uses: actions/checkout@v7 + with: + ref: ${{ env.ACTION_REF }} + + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: 24.x + + - name: Run clean fixture through published Action + id: clean + uses: stackql/mcp-wringer@v0 + with: + command: node + args: '["fixtures/stdio-server.mjs","--revision","2025-11-25"]' + profile: quick + seed: "20261118" + report_directory: ${{ runner.temp }}/mcp-wringer-clean + + - name: Verify clean run and report outputs + env: + FINDING_COUNT: ${{ steps.clean.outputs.finding_count }} + SARIF_PATH: ${{ steps.clean.outputs.sarif_path }} + REPORT_DIRECTORY: ${{ steps.clean.outputs.report_directory }} + EXPECTED_SEED: ${{ steps.clean.outputs.seed }} + HIGH_COUNT: ${{ steps.clean.outputs.high_count }} + MEDIUM_COUNT: ${{ steps.clean.outputs.medium_count }} + LOW_COUNT: ${{ steps.clean.outputs.low_count }} + INFO_COUNT: ${{ steps.clean.outputs.info_count }} + run: node scripts/assert-action-output.mjs clean + + - name: Run defective fixture through published Action + id: defective + continue-on-error: true + uses: stackql/mcp-wringer@v0 + with: + command: node + args: '["fixtures/stdio-server.mjs","--revision","2025-11-25"]' + env: '{"MCP_WRINGER_DEFECTS":"exit-on-decode-error"}' + profile: quick + seed: "20261118" + fail_on: high + report_directory: ${{ runner.temp }}/mcp-wringer-defective + + - name: Verify defective run fails at the configured threshold + if: always() + env: + OUTCOME: ${{ steps.defective.outcome }} + FINDING_COUNT: ${{ steps.defective.outputs.finding_count }} + HIGH_COUNT: ${{ steps.defective.outputs.high_count }} + SARIF_PATH: ${{ steps.defective.outputs.sarif_path }} + REPORT_DIRECTORY: ${{ steps.defective.outputs.report_directory }} + run: node scripts/assert-action-output.mjs defective + + npm: + name: published npm package (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + steps: + - name: Checkout fixture at the Action tag + uses: actions/checkout@v7 + with: + ref: ${{ env.ACTION_REF }} + + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: 24.x + + - name: Check the npm latest version matches the tag + shell: bash + run: | + set -euo pipefail + tagged="$(node -p "require('./package.json').version")" + published="$(npm view @stackql/mcp-wringer version --registry=https://registry.npmjs.org/)" + echo "tag ${ACTION_REF} has ${tagged}; npm latest is ${published}" + if [ "$tagged" != "$published" ]; then + echo "npm latest and the ${ACTION_REF} tag are out of step" >&2 + exit 1 + fi + + # npx is run outside the checkout on purpose: inside it, npx matches + # the repository's own package name and looks for a local bin link + # that does not exist. + - name: Run clean fixture through the npm CLI + shell: bash + working-directory: ${{ runner.temp }} + env: + FIXTURE: ${{ github.workspace }}/fixtures/stdio-server.mjs + run: | + set -euo pipefail + npx --yes --registry=https://registry.npmjs.org/ @stackql/mcp-wringer@latest run \ + --spec 2025-11-25 --profile quick --seed 20261118 \ + --report-dir "$RUNNER_TEMP/npm-clean" \ + -- node "$FIXTURE" --revision 2025-11-25 + + - name: Run defective fixture through the npm CLI and expect exit 1 + shell: bash + working-directory: ${{ runner.temp }} + env: + FIXTURE: ${{ github.workspace }}/fixtures/stdio-server.mjs + run: | + set -uo pipefail + npx --yes --registry=https://registry.npmjs.org/ @stackql/mcp-wringer@latest run \ + --spec 2025-11-25 --profile quick --seed 20261118 --fail-on high \ + --env MCP_WRINGER_DEFECTS=exit-on-decode-error \ + --report-dir "$RUNNER_TEMP/npm-defective" \ + -- node "$FIXTURE" --revision 2025-11-25 + status=$? + if [ "$status" -ne 1 ]; then + echo "expected exit code 1 for a high-severity finding, got ${status}" >&2 + exit 1 + fi + echo "defective fixture failed the run with exit code 1 as expected" diff --git a/README.md b/README.md index 2207bba..e878e3e 100644 --- a/README.md +++ b/README.md @@ -128,11 +128,11 @@ npm publish --access public --registry=https://registry.npmjs.org/ Complete the browser login and any 2FA prompts. `--access public` is required for this scoped public package; the package also sets public access in `publishConfig`. Do not put an OTP or npm token into source, scripts, or GitHub secrets. Local publication does not provide GitHub Actions OIDC provenance; do not add `--provenance` to this manual procedure. -Verify the published version (substitute the release version for later releases): +Verify the published version (substitute the release version for later releases). Run the `npx` command from a directory outside the repository checkout: inside it, `npx` matches the checkout's own package name and fails with `'mcp-wringer' is not recognized` or `command not found`, because the package's bin is not linked into its own `node_modules/.bin`. ```sh npm view @stackql/mcp-wringer@0.1.0 version dist.integrity --registry=https://registry.npmjs.org/ -npx --yes --registry=https://registry.npmjs.org/ @stackql/mcp-wringer@0.1.0 --help +cd "$(mktemp -d)" && npx --yes --registry=https://registry.npmjs.org/ @stackql/mcp-wringer@0.1.0 --help ``` Confirm the package is public on [npmjs](https://www.npmjs.com/package/@stackql/mcp-wringer). The [npm scoped-package guide](https://docs.npmjs.com/creating-and-publishing-scoped-public-packages) describes account and publication requirements. @@ -149,5 +149,6 @@ The repository is public and has one root [action.yml](./action.yml), with displ 6. Create tag `v0.1.0` at the exact validated release commit (use the matching `vX.Y.Z` for later releases), title the release `v0.1.0`, and include release notes. Do not select a prerelease for the stable release. 7. Click **Publish release** and complete GitHub's authentication prompts. Confirm the Marketplace listing is visible and the **Validate release** workflow succeeds. This workflow does not publish to npm. 8. After validation, create or update the moving `v0` major tag to the same commit. Keep `v0.1.0` immutable. The workflow examples use `@v0` to receive updates within that major version. +9. Run the [Published Action smoke](./.github/workflows/published-action-smoke.yml) workflow from the Actions tab. It pulls the Action from `@v0` and the package from the npm registry, runs both against the fixture server at that tag on Linux (x64 and arm64), macOS and Windows, and checks that the npm `latest` version matches the tag. It also runs weekly as a canary. -Test the published Action in a small workflow against a local fixture or an isolated authorized server before integrating it into StackQL CI. Record the Marketplace URL and release commit SHA. See [GitHub's Marketplace publication guide](https://docs.github.com/en/actions/how-tos/create-and-publish-actions/publish-in-github-marketplace) and the [maintainer release checklist](./docs/dev/release-checklist.md). +Test the published Action against an isolated authorized server before integrating it into another project's CI. The StackQL repository runs it as a pull request gate in its `mcp-wringer` workflow, which builds the server from the pull request and fuzzes stdio and Streamable HTTP in both protocol revisions on each platform it ships. Record the Marketplace URL and release commit SHA. See [GitHub's Marketplace publication guide](https://docs.github.com/en/actions/how-tos/create-and-publish-actions/publish-in-github-marketplace) and the [maintainer release checklist](./docs/dev/release-checklist.md). diff --git a/docs/dev/ideas.md b/docs/dev/ideas.md index b8c954f..b9dc1c8 100644 --- a/docs/dev/ideas.md +++ b/docs/dev/ideas.md @@ -9,3 +9,6 @@ This file collects ideas that are out of scope for the current work order. An en | Optional bearer token or custom request headers for attach mode, with values redacted from traces and reports | Some servers, for example StackQL over HTTP, require a token unless they are started with unauthenticated access. Needs a maintainer decision because it brings credentials into the tool | | Per-target concurrency limit for attach mode | Parallel workers against one attached server can push a request past its 1000 ms step timeout. Confirmation correctly records these as flaky, but a limit would reduce the noise | | Report when minimisation stops at its time budget | `minimizeScenario` returns a partly reduced scenario with no indication that the budget ran out. A `truncated` flag would make this visible, but it changes the result shape and needs a maintainer decision | +| Diagnostic when a run stops at `durationMs` before reaching `cases` | Against StackQL over stdio, the `standard` profile ran 197 of 256 cases before its 120 s cap and exited 0 with only the case count as a hint. A diagnostic line, and perhaps a non-zero exit option, would stop a CI gate from silently covering fewer cases on a slow runner | +| Accept a UTF-8 byte order mark in config files | Windows PowerShell 5 `Set-Content -Encoding UTF8` and some editors write a BOM, which the loader currently rejects as invalid JSON. Stripping a leading BOM before parsing is low risk | +| `spec` input on the Action | The Action has no input for the protocol revision. Consumers set `MCP_WRINGER_SPEC_REVISION` on the step instead, which works but is less discoverable than the other inputs | diff --git a/docs/dev/release-checklist.md b/docs/dev/release-checklist.md index 9db5d1a..43b616b 100644 --- a/docs/dev/release-checklist.md +++ b/docs/dev/release-checklist.md @@ -17,7 +17,8 @@ The release job has only `contents: read`. It verifies the tag against the packa 5. Verify the exact published npm version and CLI before proceeding to Marketplace publication. 6. The maintainer creates the immutable `vX.Y.Z` tag and GitHub release through the Marketplace release form. The tag must match `package.json` exactly; publishing the release triggers the validation-only `.github/workflows/release.yml`. 7. Confirm the validation workflow succeeds, verify the Marketplace listing, and then create or update the moving `vX` tag. Do not overwrite or move a published npm version or immutable tag. +8. Dispatch the `Published Action smoke` workflow and confirm it passes. It consumes the Action from the moving tag and the package from the npm registry, so it is the first check that runs what users install. ## GitHub Marketplace -Follow the README's Marketplace steps, including the Developer Agreement, metadata/name validation, categories, and exact release commit selection. Marketplace publication must be explicitly approved and performed by the maintainer; the release validation workflow does not list the Action. Validate the published Action against a local target before the separate StackQL CI integration. +Follow the README's Marketplace steps, including the Developer Agreement, metadata/name validation, categories, and exact release commit selection. Marketplace publication must be explicitly approved and performed by the maintainer; the release validation workflow does not list the Action. The published Action is consumed as a pull request gate by the StackQL repository's `mcp-wringer` workflow.