From 431ed8be7e0b89e7cf95af358543c11f077e9368 Mon Sep 17 00:00:00 2001 From: Jon Martin Date: Tue, 19 May 2026 16:26:05 -0400 Subject: [PATCH 1/3] Adding GitHub actions for generated PR summaries --- .github/PULL_REQUEST_TEMPLATE.md | 14 ++ .github/copilot-instructions.md | 53 ++++++ .github/scripts/eval-pr-summary.js | 127 +++++++++++++++ .../workflows/pr-summary-eval-reusable.yml | 101 ++++++++++++ .github/workflows/pr-summary-eval.yml | 15 ++ .github/workflows/pr-summary-generate.yml | 151 ++++++++++++++++++ 6 files changed, 461 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/copilot-instructions.md create mode 100644 .github/scripts/eval-pr-summary.js create mode 100644 .github/workflows/pr-summary-eval-reusable.yml create mode 100644 .github/workflows/pr-summary-eval.yml create mode 100644 .github/workflows/pr-summary-generate.yml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..1bfbcaf9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,14 @@ +## + +**** + +### Changes + +- **New samples:** +- **Updated samples:** +- **Updated kits:** +- **General:** + +### Additional Notes + + diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..97db0e5d --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,53 @@ +# Copilot Instructions for PR Summaries + +When generating a pull request summary for this repository, follow the format used +in our release notes. The summary should help reviewers quickly understand what +changed, which samples were affected, and any infrastructure/kit updates. + +## Required Format + +Use this structure for every PR summary: + +``` +## + +**** explaining the purpose and scope of the PR. + +### Changes +- **New samples:** +- **Updated samples:** +- **Updated kits:** +- **General:** +``` + +## Formatting Rules + +- Use an H2 (`##`) heading for the PR title/theme +- The first line after the heading must be a **bold summary sentence** +- Use a bulleted list under a `### Changes` subheading +- Each bullet should have a **bold category prefix** followed by a colon +- Sample and kit names should be wrapped in **bold** +- Omit category lines that have no items (do not include empty categories) +- Keep the summary concise — aim for clarity over length + +## Category Detection + +Determine categories by inspecting the diff: +- Files under `Samples/` that are newly added → **New samples** +- Files under `Samples/` that are modified → **Updated samples** +- Files under `Kits/` that are modified → **Updated kits** (ATGTK or OpenSource/imgui) +- Changes to build files (`.vcxproj`, `.sln`), configs, docs, `Media/`, or vcpkg manifests → **General** + +## Examples + +Good summary: +``` +## ARM64 Build Target Support + +**Added ARM64 (Desktop) build configurations across all samples** to enable building and running on ARM64-based Windows PCs. + +### Changes +- **Updated samples:** All samples (ARM64 platform targets added to project files) +- **Updated kits:** ATGTK +- **General:** Updated `.sln` solution files with ARM64 platform entries, updated vcpkg configuration +``` diff --git a/.github/scripts/eval-pr-summary.js b/.github/scripts/eval-pr-summary.js new file mode 100644 index 00000000..c190d2be --- /dev/null +++ b/.github/scripts/eval-pr-summary.js @@ -0,0 +1,127 @@ +// PR Summary Format Evaluator +// Scores a PR body against the expected release-note format criteria. + +const CRITERIA = [ + { + id: 'h2_heading', + name: 'H2 Heading', + description: 'PR body starts with an H2 (##) heading', + weight: 20, + test: (body) => /^##\s+\S/m.test(body), + }, + { + id: 'bold_summary', + name: 'Bold Summary', + description: 'Contains a bold (**...**) summary sentence after the heading', + weight: 25, + test: (body) => /^##\s+.+\n+\*\*.+\*\*/m.test(body), + }, + { + id: 'changes_section', + name: 'Changes Section', + description: 'Contains a "### Changes" subheading', + weight: 15, + test: (body) => /^###\s+Changes/mi.test(body), + }, + { + id: 'bullet_list', + name: 'Bullet List', + description: 'Contains at least one bullet point (- or *)', + weight: 15, + test: (body) => /^[\-\*]\s+/m.test(body), + }, + { + id: 'bold_category', + name: 'Bold Category Prefix', + description: 'At least one bullet has a bold category prefix (e.g., **Updated samples:**)', + weight: 15, + test: (body) => /^[\-\*]\s+\*\*.+?\*\*:?/m.test(body), + }, + { + id: 'recognized_category', + name: 'Recognized Category', + description: 'Uses at least one recognized category (New samples, Updated samples, Updated kits, General)', + weight: 10, + test: (body) => { + const categories = ['new samples', 'updated samples', 'updated kits', 'general']; + const lower = body.toLowerCase(); + return categories.some((cat) => lower.includes(cat)); + }, + }, +]; + +function evaluate(prBody) { + if (!prBody || prBody.trim().length === 0) { + return { + score: 0, + maxScore: 100, + pass: false, + results: CRITERIA.map((c) => ({ + ...c, + passed: false, + earned: 0, + })), + feedback: 'PR body is empty. Please add a summary following the release-note format.', + }; + } + + const results = CRITERIA.map((criterion) => { + const passed = criterion.test(prBody); + return { + id: criterion.id, + name: criterion.name, + description: criterion.description, + weight: criterion.weight, + passed, + earned: passed ? criterion.weight : 0, + }; + }); + + const score = results.reduce((sum, r) => sum + r.earned, 0); + const maxScore = results.reduce((sum, r) => sum + r.weight, 0); + const pass = score >= 60; + + const missing = results.filter((r) => !r.passed); + let feedback = ''; + if (pass && missing.length === 0) { + feedback = 'PR summary follows the expected format.'; + } else if (pass) { + feedback = + 'PR summary mostly follows the format. Consider adding: ' + + missing.map((r) => r.name).join(', ') + + '.'; + } else { + feedback = + 'PR summary does not meet the format requirements. Missing: ' + + missing.map((r) => r.name).join(', ') + + '. See .github/copilot-instructions.md for the expected format.'; + } + + return { score, maxScore, pass, results, feedback }; +} + +// Main: read PR body from environment or stdin +async function main() { + const prBody = process.env.PR_BODY || ''; + const result = evaluate(prBody); + + console.log(JSON.stringify(result, null, 2)); + + // Output for GitHub Actions + const githubOutput = process.env.GITHUB_OUTPUT; + if (githubOutput) { + const fs = require('fs'); + fs.appendFileSync(githubOutput, `score=${result.score}\n`); + fs.appendFileSync(githubOutput, `max_score=${result.maxScore}\n`); + fs.appendFileSync(githubOutput, `pass=${result.pass}\n`); + fs.appendFileSync(githubOutput, `feedback=${result.feedback}\n`); + } + + process.exit(result.pass ? 0 : 1); +} + +module.exports = { evaluate, CRITERIA }; + +if (require.main === module) { + main(); +} diff --git a/.github/workflows/pr-summary-eval-reusable.yml b/.github/workflows/pr-summary-eval-reusable.yml new file mode 100644 index 00000000..21b4285d --- /dev/null +++ b/.github/workflows/pr-summary-eval-reusable.yml @@ -0,0 +1,101 @@ +name: Evaluate PR Summary + +on: + workflow_call: + inputs: + pr-number: + required: true + type: number + +permissions: + pull-requests: write + +jobs: + eval: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Fetch current PR body + uses: actions/github-script@v7 + with: + script: | + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: ${{ inputs.pr-number }}, + }); + const fs = require('fs'); + fs.writeFileSync('/tmp/pr-body.txt', pr.body || ''); + + - name: Evaluate PR summary format + run: | + set +e + export PR_BODY="$(cat /tmp/pr-body.txt)" + node .github/scripts/eval-pr-summary.js > eval-result.json 2>&1 + echo "Eval exit code: $?" + + - name: Post evaluation comment + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + let result; + try { + result = JSON.parse(fs.readFileSync('eval-result.json', 'utf8')); + } catch (e) { + result = { score: 0, maxScore: 100, pass: false, feedback: 'Could not parse eval results.', results: [] }; + } + + const icon = result.pass ? '✅' : '⚠️'; + const status = result.pass ? 'PASS' : 'NEEDS IMPROVEMENT'; + + let body = `## ${icon} PR Summary Format Eval: ${status}\n\n`; + body += `**Score:** ${result.score}/${result.maxScore}\n\n`; + body += `${result.feedback}\n\n`; + + if (result.results && result.results.length > 0) { + body += `### Criteria Breakdown\n\n`; + body += `| Criterion | Status | Points |\n`; + body += `|-----------|--------|--------|\n`; + for (const r of result.results) { + const checkmark = r.passed ? '✅' : '❌'; + body += `| ${r.name} | ${checkmark} | ${r.earned}/${r.weight} |\n`; + } + } + + body += `\n---\n*Format reference: [.github/copilot-instructions.md](.github/copilot-instructions.md)*`; + + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ inputs.pr-number }}, + }); + + const marker = '## ✅ PR Summary Format Eval'; + const markerAlt = '## ⚠️ PR Summary Format Eval'; + const existing = comments.data.find(c => + c.body.startsWith(marker) || c.body.startsWith(markerAlt) + ); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ inputs.pr-number }}, + body, + }); + } diff --git a/.github/workflows/pr-summary-eval.yml b/.github/workflows/pr-summary-eval.yml new file mode 100644 index 00000000..f5fa0f5e --- /dev/null +++ b/.github/workflows/pr-summary-eval.yml @@ -0,0 +1,15 @@ +name: PR Summary Eval + +on: + pull_request: + types: [opened, edited, synchronize] + +jobs: + evaluate-summary: + # Skip the initial 'opened' event for bot PRs — the generate workflow will + # handle eval after updating the body. + if: >- + !(github.event.action == 'opened' && endsWith(github.event.pull_request.user.login, '[bot]')) + uses: ./.github/workflows/pr-summary-eval-reusable.yml + with: + pr-number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/pr-summary-generate.yml b/.github/workflows/pr-summary-generate.yml new file mode 100644 index 00000000..9291a243 --- /dev/null +++ b/.github/workflows/pr-summary-generate.yml @@ -0,0 +1,151 @@ +name: Generate PR Summary + +on: + pull_request: + types: [opened] + +permissions: + contents: read + pull-requests: write + models: read + +jobs: + generate-summary: + # Only run for bot-created PRs + if: endsWith(github.event.pull_request.user.login, '[bot]') + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get PR diff summary + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr diff ${{ github.event.pull_request.number }} --stat > /tmp/diff-stat.txt 2>/dev/null || echo "Unable to retrieve diff" > /tmp/diff-stat.txt + gh pr diff ${{ github.event.pull_request.number }} --name-only > /tmp/changed-files.txt 2>/dev/null || echo "" > /tmp/changed-files.txt + + - name: Read format instructions + run: | + if [ -f .github/copilot-instructions.md ]; then + cat .github/copilot-instructions.md > /tmp/format-instructions.txt + else + echo "No format instructions found" > /tmp/format-instructions.txt + fi + + - name: Generate summary with GitHub Models + id: generate + uses: actions/github-script@v7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + const fs = require('fs'); + + const diffStat = fs.readFileSync('/tmp/diff-stat.txt', 'utf8'); + const changedFiles = fs.readFileSync('/tmp/changed-files.txt', 'utf8'); + const formatInstructions = fs.readFileSync('/tmp/format-instructions.txt', 'utf8'); + + const prompt = `You are generating a pull request description for an automated sync from an internal development repository to the public GitHub repository. + + Here are the format instructions to follow: + --- + ${formatInstructions} + --- + + Here are the changed files in this PR: + --- + ${changedFiles} + --- + + Here is the diff stat: + --- + ${diffStat} + --- + + Generate a PR description that follows the format instructions exactly. Categorize files correctly: + - Files under Samples/ that appear new → New samples + - Files under Samples/ that are modified → Updated samples (use the sample directory name) + - Files under Kits/ → Updated kits + - Everything else (Media/, configs, docs) → General + + Also generate a concise PR title (not the H2 heading) that summarizes the changes in + a few words (e.g., "Add TriangleWave sample, update ATGTK helpers"). + + Output a JSON object with two fields: + - "title": the PR title string + - "body": the formatted PR description following the instructions + + Output ONLY valid JSON, nothing else.`; + + const response = await fetch('https://models.github.ai/inference/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${process.env.GITHUB_TOKEN}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'openai/gpt-4o-mini', + messages: [ + { role: 'user', content: prompt } + ], + temperature: 0.3, + response_format: { type: 'json_object' }, + }), + }); + + if (!response.ok) { + const error = await response.text(); + core.setFailed(`GitHub Models API error: ${response.status} - ${error}`); + return; + } + + const result = await response.json(); + let content = result.choices[0].message.content.trim(); + + // Strip markdown code fences if present + content = content.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?```\s*$/i, ''); + + let parsed; + try { + parsed = JSON.parse(content); + } catch (e) { + // Fallback: treat entire response as body if JSON parsing fails + parsed = { title: '', body: content }; + } + + fs.writeFileSync('/tmp/pr-summary.md', parsed.body); + core.setOutput('summary', parsed.body); + core.setOutput('title', parsed.title); + + - name: Update PR title and description + if: steps.generate.outputs.summary != '' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const summary = fs.readFileSync('/tmp/pr-summary.md', 'utf8'); + const title = '${{ steps.generate.outputs.title }}'; + + const update = { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + body: summary, + }; + + if (title) { + update.title = title; + } + + await github.rest.pulls.update(update); + + console.log('PR title and description updated successfully'); + + evaluate-summary: + needs: generate-summary + uses: ./.github/workflows/pr-summary-eval-reusable.yml + with: + pr-number: ${{ github.event.pull_request.number }} From e81af7e28f2e6a732418f444a573e9e560ece5e4 Mon Sep 17 00:00:00 2001 From: Jon Martin Date: Tue, 19 May 2026 16:59:15 -0400 Subject: [PATCH 2/3] Generate PR summary for any PR opened without a description Removes the bot-only filter so the workflow triggers for all PRs that are opened with an empty body, ensuring the formatted summary is auto-generated regardless of who creates the PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-summary-generate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-summary-generate.yml b/.github/workflows/pr-summary-generate.yml index 9291a243..70d98c17 100644 --- a/.github/workflows/pr-summary-generate.yml +++ b/.github/workflows/pr-summary-generate.yml @@ -11,8 +11,8 @@ permissions: jobs: generate-summary: - # Only run for bot-created PRs - if: endsWith(github.event.pull_request.user.login, '[bot]') + # Run when a PR is opened without a description + if: github.event.pull_request.body == '' || github.event.pull_request.body == null runs-on: ubuntu-latest steps: - name: Checkout From 100e7b5cb728fc3133b8463b49445c2d371f2e24 Mon Sep 17 00:00:00 2001 From: Jon Martin Date: Tue, 19 May 2026 17:33:20 -0400 Subject: [PATCH 3/3] Add /generate-summary comment command to trigger PR summary Users can now comment '/generate-summary' on any PR to regenerate the formatted description on demand. The workflow still auto-triggers for PRs opened without a body. A rocket emoji reaction confirms the command was received. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-summary-generate.yml | 41 +++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-summary-generate.yml b/.github/workflows/pr-summary-generate.yml index 70d98c17..93e7edb5 100644 --- a/.github/workflows/pr-summary-generate.yml +++ b/.github/workflows/pr-summary-generate.yml @@ -3,6 +3,8 @@ name: Generate PR Summary on: pull_request: types: [opened] + issue_comment: + types: [created] permissions: contents: read @@ -11,21 +13,48 @@ permissions: jobs: generate-summary: - # Run when a PR is opened without a description - if: github.event.pull_request.body == '' || github.event.pull_request.body == null + # Run when: + # 1. A PR is opened without a description, OR + # 2. Someone comments "/generate-summary" on a PR + if: | + (github.event_name == 'pull_request' && (github.event.pull_request.body == '' || github.event.pull_request.body == null)) || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/generate-summary')) runs-on: ubuntu-latest + outputs: + pr-number: ${{ steps.pr-number.outputs.number }} steps: + - name: Determine PR number + id: pr-number + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT" + else + echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT" + fi + - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: React to comment + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket', + }); + - name: Get PR diff summary env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh pr diff ${{ github.event.pull_request.number }} --stat > /tmp/diff-stat.txt 2>/dev/null || echo "Unable to retrieve diff" > /tmp/diff-stat.txt - gh pr diff ${{ github.event.pull_request.number }} --name-only > /tmp/changed-files.txt 2>/dev/null || echo "" > /tmp/changed-files.txt + gh pr diff ${{ steps.pr-number.outputs.number }} --stat > /tmp/diff-stat.txt 2>/dev/null || echo "Unable to retrieve diff" > /tmp/diff-stat.txt + gh pr diff ${{ steps.pr-number.outputs.number }} --name-only > /tmp/changed-files.txt 2>/dev/null || echo "" > /tmp/changed-files.txt - name: Read format instructions run: | @@ -132,7 +161,7 @@ jobs: const update = { owner: context.repo.owner, repo: context.repo.repo, - pull_number: context.payload.pull_request.number, + pull_number: ${{ steps.pr-number.outputs.number }}, body: summary, }; @@ -148,4 +177,4 @@ jobs: needs: generate-summary uses: ./.github/workflows/pr-summary-eval-reusable.yml with: - pr-number: ${{ github.event.pull_request.number }} + pr-number: ${{ needs.generate-summary.outputs.pr-number }}