diff --git a/.github/workflows/external-plugin-pr-quality-gates.yml b/.github/workflows/external-plugin-pr-quality-gates.yml new file mode 100644 index 000000000..8a7d3ac4d --- /dev/null +++ b/.github/workflows/external-plugin-pr-quality-gates.yml @@ -0,0 +1,235 @@ +name: External Plugin PR Quality Gates + +on: + pull_request: + branches: [staged] + paths: + - "plugins/external.json" + types: [opened, synchronize, reopened, edited, ready_for_review] + +concurrency: + group: external-plugin-pr-quality-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + detect-changed-plugins: + runs-on: ubuntu-latest + outputs: + changed-plugins: ${{ steps.detect.outputs.changed-plugins }} + changed-count: ${{ steps.detect.outputs.changed-count }} + should-run: ${{ steps.detect.outputs.should-run }} + steps: + - name: Detect changed external plugins + id: detect + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const filePath = 'plugins/external.json'; + const baseRef = context.payload.pull_request.base.sha; + const headRef = context.payload.pull_request.head.sha; + + function normalizePath(value) { + if (!value || value === '/') { + return ''; + } + return String(value).trim().replace(/^\/+|\/+$/g, '').toLowerCase(); + } + + function toIdentity(plugin) { + return [ + String(plugin?.name ?? '').trim().toLowerCase(), + String(plugin?.source?.repo ?? '').trim().toLowerCase(), + normalizePath(plugin?.source?.path), + ].join('|'); + } + + async function readExternalJson(ref) { + const response = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: filePath, + ref, + }); + + const encoded = response.data?.content ?? ''; + const decoded = Buffer.from(encoded, 'base64').toString('utf8'); + return JSON.parse(decoded); + } + + const basePlugins = await readExternalJson(baseRef); + const headPlugins = await readExternalJson(headRef); + const baseByIdentity = new Map(basePlugins.map((plugin) => [toIdentity(plugin), plugin])); + + const changedPlugins = headPlugins.filter((plugin) => { + const identity = toIdentity(plugin); + const basePlugin = baseByIdentity.get(identity); + return !basePlugin || JSON.stringify(basePlugin) !== JSON.stringify(plugin); + }); + + core.setOutput('changed-plugins', JSON.stringify(changedPlugins)); + core.setOutput('changed-count', String(changedPlugins.length)); + core.setOutput('should-run', changedPlugins.length > 0 ? 'true' : 'false'); + + run-quality-gates: + runs-on: ubuntu-latest + needs: detect-changed-plugins + if: needs.detect-changed-plugins.outputs.should-run == 'true' + outputs: + quality-result: ${{ steps.quality.outputs.quality-result }} + steps: + - name: Checkout staged branch + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: staged + persist-credentials: false + submodules: false + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + + - name: Install GitHub Copilot CLI + run: npm install -g @github/copilot + + - name: Run external plugin PR quality gates + id: quality + env: + CHANGED_PLUGINS_JSON: ${{ needs.detect-changed-plugins.outputs.changed-plugins }} + run: | + result=$(node ./eng/external-plugin-pr-quality-gates.mjs --plugins-json "$CHANGED_PLUGINS_JSON") + { + echo 'quality-result<> "$GITHUB_OUTPUT" + + sync-pr-state: + runs-on: ubuntu-latest + needs: [detect-changed-plugins, run-quality-gates] + if: always() + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Checkout staged branch + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: staged + + - name: Sync labels and PR status comment + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + DETECT_JOB_RESULT: ${{ needs.detect-changed-plugins.result }} + SHOULD_RUN: ${{ needs.detect-changed-plugins.outputs.should-run }} + CHANGED_COUNT: ${{ needs.detect-changed-plugins.outputs.changed-count }} + QUALITY_RESULT_JSON: ${{ needs.run-quality-gates.outputs.quality-result }} + QUALITY_JOB_RESULT: ${{ needs.run-quality-gates.result }} + with: + script: | + const path = require('path'); + const { pathToFileURL } = require('url'); + + const intakeState = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'eng', 'external-plugin-intake-state.mjs')).href); + const marker = ''; + + const detectJobResult = process.env.DETECT_JOB_RESULT; + const shouldRun = process.env.SHOULD_RUN === 'true'; + const changedCount = Number.parseInt(process.env.CHANGED_COUNT || '0', 10); + const qualityJobResult = process.env.QUALITY_JOB_RESULT; + + let qualityResult = { + overall_status: 'not_run', + failure_class: 'none', + checked_plugins: [], + summary: 'No changed external plugin entries were detected in this PR.', + }; + + if (detectJobResult === 'failure' || detectJobResult === 'cancelled') { + qualityResult = { + overall_status: 'infra_error', + failure_class: 'infra', + checked_plugins: [], + summary: 'External plugin PR change detection failed unexpectedly. Re-run this workflow.', + }; + } else if (shouldRun) { + if (qualityJobResult === 'failure' || qualityJobResult === 'cancelled') { + qualityResult = { + overall_status: 'infra_error', + failure_class: 'infra', + checked_plugins: [], + summary: 'External plugin PR quality checks failed unexpectedly. Re-run this workflow.', + }; + } else if (process.env.QUALITY_RESULT_JSON) { + qualityResult = JSON.parse(process.env.QUALITY_RESULT_JSON); + } else { + qualityResult = { + overall_status: 'infra_error', + failure_class: 'infra', + checked_plugins: [], + summary: 'External plugin PR quality checks did not return a result payload.', + }; + } + } + + const stateLabel = qualityResult.failure_class === 'submitter_fixes' + ? 'requires-submitter-fixes' + : qualityResult.overall_status === 'pass' || !shouldRun + ? 'ready-for-review' + : 'awaiting-review'; + + const desiredLabels = new Set(['external-plugin', stateLabel]); + await intakeState.syncExternalPluginIntakeLabels({ + github, + owner: context.repo.owner, + repo: context.repo.repo, + issueNumber: context.issue.number, + desiredLabels, + }); + + const checkedPlugins = Array.isArray(qualityResult.checked_plugins) ? qualityResult.checked_plugins : []; + const header = qualityResult.failure_class === 'submitter_fixes' + ? '## ⚠️ External plugin PR checks require submitter fixes' + : qualityResult.overall_status === 'pass' || !shouldRun + ? '## ✅ External plugin PR checks passed' + : '## ⚠️ External plugin PR checks need maintainer follow-up'; + + const rows = checkedPlugins.length > 0 + ? checkedPlugins.map((entry) => { + const name = String(entry?.name || 'unknown'); + const quality = entry?.quality || {}; + const sourceUrl = String(entry?.source_tree_url || ''); + const locator = String(entry?.source?.sha || entry?.source?.ref || 'repository'); + const sourceCell = sourceUrl ? `[${locator}](${sourceUrl})` : locator; + return `| ${name} | ${quality.skill_validator_status || 'not_run'} | ${quality.smoke_status || 'not_run'} | ${quality.overall_status || 'not_run'} | ${sourceCell} |`; + }) + : ['| _none_ | not_run | not_run | not_run | _n/a_ |']; + + const body = [ + marker, + header, + '', + `- **Changed entries detected:** ${changedCount}`, + `- **Workflow state label:** \`${stateLabel}\``, + '', + '### Per-plugin quality summary', + '', + '| Plugin | skill-validator | install smoke test | overall | source tree |', + '|---|---|---|---|---|', + ...rows, + '', + String(qualityResult.summary || '').trim() || '_No summary provided._', + ].join('\n'); + + await intakeState.upsertExternalPluginIntakeComment({ + github, + owner: context.repo.owner, + repo: context.repo.repo, + issueNumber: context.issue.number, + marker, + body, + }); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 23233906e..d76726fec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -241,6 +241,18 @@ The public-submission policy builds on those rules and also requires `license` p 9. **Approval path**: on `/approve`, automation removes `ready-for-review`, adds `approved`, closes the issue, and opens or updates a PR against `staged` that updates `plugins/external.json` and generated marketplace outputs. 10. **Rejection path**: on `/reject `, automation removes `ready-for-review`, adds `rejected`, closes the issue, and records the reason in an issue comment. After addressing the feedback, update the same issue and use `/rerun-intake` to re-queue intake. +##### Updating listed external plugins via PR + +When a pull request updates `plugins/external.json` (for example, version updates for a previously approved listing), automation runs PR quality checks and posts the result directly on the PR: + +1. **Detect changed entries**: automation identifies added/updated external plugin entries in the PR. +2. **Run quality gates**: automation runs install smoke tests and `skill-validator` checks against each changed plugin source ref/SHA/path. +3. **Post source links**: automation updates a bot comment with per-plugin results and direct GitHub tree links to each plugin source location. +4. **Sync workflow-state labels on the PR**: + - `ready-for-review` when all checks pass + - `requires-submitter-fixes` when quality checks fail due to plugin issues + - `awaiting-review` when checks cannot complete because of infrastructure/transient errors + ##### Maintainer review responsibilities Maintainers are responsible for confirming that the submission: diff --git a/eng/external-plugin-pr-quality-gates.mjs b/eng/external-plugin-pr-quality-gates.mjs new file mode 100644 index 000000000..44158322f --- /dev/null +++ b/eng/external-plugin-pr-quality-gates.mjs @@ -0,0 +1,125 @@ +#!/usr/bin/env node + +import { runExternalPluginQualityGates } from "./external-plugin-quality-gates.mjs"; + +function normalizePluginPath(pluginPath) { + if (!pluginPath || pluginPath === "/") { + return ""; + } + + return String(pluginPath).trim().replace(/^\/+|\/+$/g, ""); +} + +function encodePathLikeValue(value) { + return String(value) + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); +} + +export function buildSourceTreeUrl(plugin) { + const sourceRepo = plugin?.source?.repo; + if (!sourceRepo) { + return ""; + } + + const sourceLocator = plugin?.source?.sha || plugin?.source?.ref; + if (!sourceLocator) { + return `https://github.com/${sourceRepo}`; + } + + const encodedLocator = encodeURIComponent(sourceLocator); + const normalizedPath = normalizePluginPath(plugin?.source?.path); + if (!normalizedPath) { + return `https://github.com/${sourceRepo}/tree/${encodedLocator}`; + } + + const encodedPath = encodePathLikeValue(normalizedPath); + return `https://github.com/${sourceRepo}/tree/${encodedLocator}/${encodedPath}`; +} + +function aggregateResultStatus(pluginResults) { + if (pluginResults.some((entry) => entry.quality?.overall_status === "fail")) { + return { + overallStatus: "fail", + failureClass: "submitter_fixes", + }; + } + + if (pluginResults.some((entry) => entry.quality?.overall_status === "infra_error")) { + return { + overallStatus: "infra_error", + failureClass: "infra", + }; + } + + if (pluginResults.length === 0) { + return { + overallStatus: "not_run", + failureClass: "none", + }; + } + + return { + overallStatus: "pass", + failureClass: "none", + }; +} + +export function runExternalPluginPrQualityGates(plugins) { + if (!Array.isArray(plugins)) { + throw new Error("plugins must be an array"); + } + + const checkedPlugins = plugins.map((plugin) => { + const quality = runExternalPluginQualityGates(plugin); + return { + name: plugin?.name ?? "unknown", + source: plugin?.source ?? {}, + source_tree_url: buildSourceTreeUrl(plugin), + quality, + }; + }); + + const aggregate = aggregateResultStatus(checkedPlugins); + const summary = checkedPlugins.length === 0 + ? "No changed external plugin entries were detected in plugins/external.json." + : checkedPlugins + .map((entry) => + `- ${entry.name}: skill-validator=${entry.quality.skill_validator_status}, install-smoke=${entry.quality.smoke_status}, overall=${entry.quality.overall_status}` + ) + .join("\n"); + + return { + overall_status: aggregate.overallStatus, + failure_class: aggregate.failureClass, + summary, + checked_plugins: checkedPlugins, + }; +} + +function parseCliArgs(argv) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith("--")) { + continue; + } + + args[key.slice(2)] = argv[index + 1]; + index += 1; + } + return args; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = parseCliArgs(process.argv.slice(2)); + if (!args["plugins-json"]) { + console.error("Usage: node ./eng/external-plugin-pr-quality-gates.mjs --plugins-json ''"); + process.exit(1); + } + + const plugins = JSON.parse(args["plugins-json"]); + const result = runExternalPluginPrQualityGates(plugins); + process.stdout.write(`${JSON.stringify(result)}\n`); +}