diff --git a/.github/workflows/daily-grader-audit.lock.yml b/.github/workflows/daily-grader-audit.lock.yml new file mode 100644 index 00000000000..d72987c658a --- /dev/null +++ b/.github/workflows/daily-grader-audit.lock.yml @@ -0,0 +1,2184 @@ +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"5e6396dbc7d894cc1d5c93823b0c3e0ffc629c5beeb46fcdefc12cef54692eaa","body_hash":"af214016bcec2674f9b7e150e0f747ba37087eb4c6dcbec7e96f52b3bd1431cc","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.245"}} +# gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"37fe631027851001ddb9b187196cc803df7f5f0e","version":"v4.3.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.7","digest":"sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.7@sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.7","digest":"sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.7@sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.7","digest":"sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.7@sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.10","digest":"sha256:08bb5fa417aed94b40a14e2b7b3ae457531a5f22b143a32fe58317139d9b8f42","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.10@sha256:08bb5fa417aed94b40a14e2b7b3ae457531a5f22b143a32fe58317139d9b8f42"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.10.1","digest":"sha256:1817b57d43916532dc002bdc5f344d639bd9fb54a9148d42168458f7c3280567","pinned_image":"ghcr.io/github/github-mcp-server:v1.10.1@sha256:1817b57d43916532dc002bdc5f344d639bd9fb54a9148d42168458f7c3280567"}],"mcp_servers":[{"name":"agenticworkflows","tools":["*"]},{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","upload_artifact"]}]} +# This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Daily audit of workflow runs with deterministic grader results, producing a downloadable 24-hour report artifact +# +# Resolved workflow manifest: +# Imports: +# - shared/reporting.md +# +# Secrets used: +# - ANTHROPIC_API_KEY +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 +# - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 +# - docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 +# +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.28.7@sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.28.7@sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1 +# - ghcr.io/github/gh-aw-firewall/squid:0.28.7@sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9 +# - ghcr.io/github/gh-aw-mcpg:v0.4.10@sha256:08bb5fa417aed94b40a14e2b7b3ae457531a5f22b143a32fe58317139d9b8f42 +# - ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e +# - ghcr.io/github/github-mcp-server:v1.10.1@sha256:1817b57d43916532dc002bdc5f344d639bd9fb54a9148d42168458f7c3280567 + +name: "Daily Grader Audit" +on: + schedule: + - cron: "40 7 * * *" # Friendly format: daily around 8:00 (scattered) + workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + queue: max + +run-name: "Daily Grader Audit" + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + actions: read + contents: read + env: + GH_AW_MAX_DAILY_AI_CREDITS: "5000" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + comment_id: "" + comment_repo: "" + daily_ai_credits_exceeded: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }} + daily_ai_credits_guardrail_status: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_guardrail_status || '' }} + daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }} + daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }} + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + oauth_token_check_failed: ${{ steps.check-oauth-tokens.outputs.oauth_token_check_failed == 'true' }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + safe-output-artifact-client: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "claude" + GH_AW_INFO_ENGINE_NAME: "Claude Code" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || 'agent' }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AGENT_VERSION: "2.1.245" + GH_AW_INFO_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_AGENT_RUNTIME: "" + GH_AW_INFO_FRONTMATTER_EMOJI: "๐Ÿ“Š" + GH_AW_COMPILED_STRICT: "true" + GH_AW_INFO_FEATURES: '{"gh-aw-detection":true}' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_aw_info.cjs')); + await main(core, context); + - name: Restore daily AIC usage cache + id: restore-daily-aic-cache + if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-workflow-usage-dailygraderaudit-${{ github.run_id }} + restore-keys: agentic-workflow-usage-dailygraderaudit- + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + - name: Restore daily AIC usage cache (artifact fallback) + id: restore-daily-aic-cache-fallback + if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_RESTORE_DAILY_AIC_CACHE_HIT: ${{ steps.restore-daily-aic-cache.outputs.cache-hit }} + GH_AW_RESTORE_DAILY_AIC_CACHE_MATCHED_KEY: ${{ steps.restore-daily-aic-cache.outputs.cache-matched-key }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'restore_aic_usage_cache_fallback.cjs')); + await main(); + - name: Check daily workflow token guardrail + id: daily-effective-workflow-guardrail + if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_ID: "daily-grader-audit" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_WORKFLOW_DISPATCH_AW_CONTEXT: ${{ github.event.inputs.aw_context || '' }} + GH_AW_HAS_SLASH_COMMAND: "false" + GH_AW_HAS_LABEL_COMMAND: "false" + GH_AW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_AW_MAX_DAILY_AI_CREDITS: "5000" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'check_daily_aic_workflow_guardrail.cjs')); + await main(); + - name: Validate ANTHROPIC_API_KEY secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" ANTHROPIC_API_KEY 'Claude Code' https://github.github.com/gh-aw/reference/engines/#anthropic-claude-code + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + - name: Check for OAuth tokens + id: check-oauth-tokens + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_oauth_tokens.sh" + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + actions/setup + .claude + .codex + .gemini + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .claude .github" + GH_AW_AGENT_FILES: "AGENTS.md CLAUDE.md" + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_WORKFLOW_FILE: "daily-grader-audit.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'check_workflow_timestamp_api.cjs')); + await main(); + - name: Log runtime features + if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" + - name: Create prompt with built-in context + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_ACTIONS_DIR: ${{ runner.temp }}/gh-aw/actions + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0006\"}]}" + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_PROMPT_CONTENT_0000: "\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0002: "\n" + GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" + GH_AW_PROMPT_CONTENT_0004: "\n" + GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/shared/reporting.md}}\n" + GH_AW_PROMPT_CONTENT_0006: "{{#runtime-import .github/workflows/daily-grader-audit.md}}\n" + with: + script: | + const { setupGlobals } = require(process.env.GH_AW_ACTIONS_DIR + '/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(process.env.GH_AW_ACTIONS_DIR + '/create_prompt.cjs'); + await main(core); + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "claude" + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'interpolate_prompt.cjs')); + await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: "- `agenticworkflows` โ€” run `agenticworkflows --help` to see available tools\n- `safeoutputs` โ€” run `safeoutputs --help` to see available tools" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require(path.join(actionsDir, 'substitute_placeholders.cjs')); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Stage prompt files for artifact upload + run: | + mkdir -p /tmp/gh-aw/aw-prompts + cp -a "${RUNNER_TEMP}/gh-aw/aw-prompts/." /tmp/gh-aw/aw-prompts/ + - name: Upload activation artifact + if: success() || failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/models.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.claude/agents + /tmp/gh-aw/.claude/skills + if-no-files-found: ignore + retention-days: 1 + + agent: + needs: activation + if: needs.activation.outputs.daily_ai_credits_exceeded != 'true' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + concurrency: + group: "gh-aw-claude-${{ github.workflow }}" + queue: max + timeout-minutes: 60 + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_PROJECT_UTC: "-08:00" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + GH_AW_WORKFLOW_ID_SANITIZED: dailygraderaudit + outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} + ai_credits_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.ai_credits_rate_limit_error || 'false' }} + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + ambient_context: ${{ steps.parse-mcp-gateway.outputs.ambient_context }} + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + http_400_response_error: ${{ steps.detect-agent-errors.outputs.http_400_response_error || 'false' }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + invocation_cap_exceeded: ${{ steps.detect-agent-errors.outputs.invocation_cap_exceeded || 'false' }} + max_cache_misses_exceeded: ${{ steps.detect-agent-errors.outputs.max_cache_misses_exceeded || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + missing_model_pricing_error: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_error || 'false' }} + missing_model_pricing_model_name: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_model_name || '' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + shell_expansion_guard_rejected: ${{ steps.detect-agent-errors.outputs.shell_expansion_guard_rejected || 'false' }} + unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Set runtime paths + id: set-runtime-paths + run: | + if [ -z "${RUNNER_TOOL_CACHE:-}" ]; then + echo "RUNNER_TOOL_CACHE=${{ runner.tool_cache }}" >> "$GITHUB_ENV" + fi + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Go for CLI build + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Build gh-aw CLI + run: | + echo "Building gh-aw CLI for linux/amd64..." + mkdir -p dist + VERSION=$(git describe --tags --always --dirty) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -ldflags "-s -w -X main.version=${VERSION}" \ + -o dist/gh-aw-linux-amd64 \ + ./cmd/gh-aw + # Copy binary to root for direct execution in user-defined steps + cp dist/gh-aw-linux-amd64 ./gh-aw + chmod +x ./gh-aw + echo "โœ“ Built gh-aw CLI successfully" + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - name: Build gh-aw Docker image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + platforms: linux/amd64 + push: false + load: true + tags: localhost/gh-aw:dev + build-args: | + BINARY=dist/gh-aw-linux-amd64 + - name: Create gh-aw temp directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs + path: .github/aw/logs + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request || github.event.issue.pull_request || github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.aw_context || '{}').item_type == 'pull_request' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'checkout_pr_branch.cjs')); + await main(); + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.7 --rootless + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.245 + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); + await determineAutomaticLockdown(github, context, core); + - name: Restore agent config folders from base branch + if: steps.checkout-pr.outcome == 'success' + env: + GH_AW_AGENT_FOLDERS: ".agents .claude .github" + GH_AW_AGENT_FILES: "AGENTS.md CLAUDE.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".claude/agents" + GH_AW_SUB_AGENT_EXT: ".md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".claude/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.7@sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.7@sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1 ghcr.io/github/gh-aw-firewall/squid:0.28.7@sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9 ghcr.io/github/gh-aw-mcpg:v0.4.10@sha256:08bb5fa417aed94b40a14e2b7b3ae457531a5f22b143a32fe58317139d9b8f42 ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e ghcr.io/github/github-mcp-server:v1.10.1@sha256:1817b57d43916532dc002bdc5f344d639bd9fb54a9148d42168458f7c3280567 + - name: Build and install gh-aw CLI from source + run: | + gh extension remove aw || true + make build + gh extension install . + gh aw version + env: + GH_TOKEN: ${{ github.token }} + - name: Copy gh-aw binary for MCP Server + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/copy_gh_aw_binary_for_mcp.sh" + - name: Prepare Safe Outputs Directories + run: | + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts" + - name: Generate Safe Outputs Config + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" + GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{},\"upload_artifact\":{\"max-size-bytes\":104857600,\"max-uploads\":1,\"retention-days\":1,\"skip-archive\":true}}" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'create_files.cjs')); + await main(); + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | + { + "description_suffixes": {}, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + }, + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 + } + } + }, + "upload_artifact": { + "defaultMax": 10, + "fields": { + "filters": { + "type": "object" + }, + "path": { + "type": "string" + }, + "temporary_id": { + "type": "string", + "pattern": "^#?aw_[A-Za-z0-9_]{3,12}$" + } + } + } + } + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_safe_outputs_tools.cjs')); + await main(); + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} + GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} + GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} + GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" + if [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -r "${GITHUB_EVENT_PATH}" ]; then + GH_AW_SAFEOUTPUTS_EVENT_PATH="${RUNNER_TEMP}/gh-aw/safeoutputs/github_event.json" + cp "${GITHUB_EVENT_PATH}" "${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + export GITHUB_EVENT_PATH="${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + fi + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="8080" + export MCP_GATEWAY_DOMAIN="awmg-mcpg" + export MCP_GATEWAY_HOST_DOMAIN="localhost" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export MCP_GATEWAY_ALLOWED_MOUNT_ROOTS="${GITHUB_WORKSPACE}:rw,${RUNNER_TEMP}/gh-aw:ro,${RUNNER_TEMP}/gh-aw/safeoutputs:rw,/opt:ro,/tmp:rw,/usr/bin/gh:ro" + export DEBUG="*" + + export GH_AW_ENGINE="claude" + export GH_AW_MCP_CLI_SERVERS='["agenticworkflows","safeoutputs"]' + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e RUNNER_TOOL_CACHE -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.10' + + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_53d65ee105018bcf_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + { + "mcpServers": { + "agenticworkflows": { + "container": "localhost/gh-aw:dev", + "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"], + "args": ["--network", "host", "-w", "\${GITHUB_WORKSPACE}"], + "env": { + "DEBUG": "*", + "GITHUB_TOKEN": "\${GITHUB_TOKEN}", + "GITHUB_ACTOR": "\${GITHUB_ACTOR}", + "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ], + "sink-visibility": "${GH_AW_SINK_VISIBILITY}" + } + } + }, + "github": { + "container": "ghcr.io/github/github-mcp-server:v1.10.1", + "env": { + "GITHUB_FEATURES": "fields_param", + "GITHUB_HOST": "$GITHUB_SERVER_URL", + "GITHUB_PERSONAL_ACCESS_TOKEN": "$GITHUB_MCP_SERVER_TOKEN", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", + "repos": "$GITHUB_MCP_GUARD_REPOS" + } + } + }, + "safeoutputs": { + "container": "ghcr.io/github/gh-aw-node", + "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "${RUNNER_TEMP}/gh-aw/safeoutputs:${RUNNER_TEMP}/gh-aw/safeoutputs:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"], + "args": ["-w", "\${GITHUB_WORKSPACE}"], + "entrypoint": "sh", + "entrypointArgs": ["-c", "sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh"], + "env": { + "DEBUG": "*", + "DEFAULT_BRANCH": "\${DEFAULT_BRANCH}", + "GH_AW_ASSETS_ALLOWED_EXTS": "\${GH_AW_ASSETS_ALLOWED_EXTS}", + "GH_AW_ASSETS_BRANCH": "\${GH_AW_ASSETS_BRANCH}", + "GH_AW_ASSETS_MAX_SIZE_KB": "\${GH_AW_ASSETS_MAX_SIZE_KB}", + "GH_AW_MCP_LOG_DIR": "\${GH_AW_MCP_LOG_DIR}", + "GH_AW_SAFE_OUTPUTS": "\${GH_AW_SAFE_OUTPUTS}", + "GH_AW_SAFE_OUTPUTS_CONFIG_PATH": "\${GH_AW_SAFE_OUTPUTS_CONFIG_PATH}", + "GH_AW_SAFE_OUTPUTS_TOOLS_PATH": "\${GH_AW_SAFE_OUTPUTS_TOOLS_PATH}", + "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST": "\${GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST}", + "GITHUB_EVENT_NAME": "\${GITHUB_EVENT_NAME}", + "GITHUB_EVENT_PATH": "\${GITHUB_EVENT_PATH}", + "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}", + "GITHUB_SHA": "\${GITHUB_SHA}", + "GITHUB_TOKEN": "\${GITHUB_TOKEN}", + "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}", + "RUNNER_TEMP": "\${RUNNER_TEMP}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ], + "sink-visibility": "${GH_AW_SINK_VISIBILITY}" + } + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}", + "toolTimeout": 600, + "startupTimeout": 120 + } + } + GH_AW_MCP_CONFIG_53d65ee105018bcf_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true + env: + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io); + const { main } = require(path.join(actionsDir, 'mount_mcp_as_cli.cjs')); + await main(); + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" + - name: Execute Claude Code CLI + id: agentic_execution + # Allowed tools (sorted): + # - Bash + # - BashOutput + # - Edit + # - Edit(/tmp/*) + # - ExitPlanMode + # - Glob + # - Grep + # - KillBash + # - LS + # - MultiEdit + # - MultiEdit(/tmp/*) + # - NotebookEdit + # - NotebookRead + # - Read + # - Read(/tmp/*) + # - Task + # - TodoWrite + # - Write + # - Write(/tmp/*) + # - mcp__agenticworkflows + # - mcp__github__actions_get + # - mcp__github__actions_list + # - mcp__github__get_code_scanning_alert + # - mcp__github__get_commit + # - mcp__github__get_dependabot_alert + # - mcp__github__get_discussion + # - mcp__github__get_discussion_comments + # - mcp__github__get_file_contents + # - mcp__github__get_job_logs + # - mcp__github__get_label + # - mcp__github__get_latest_release + # - mcp__github__get_me + # - mcp__github__get_notification_details + # - mcp__github__get_pull_request + # - mcp__github__get_pull_request_comments + # - mcp__github__get_pull_request_diff + # - mcp__github__get_pull_request_files + # - mcp__github__get_pull_request_review_comments + # - mcp__github__get_pull_request_reviews + # - mcp__github__get_pull_request_status + # - mcp__github__get_release_by_tag + # - mcp__github__get_secret_scanning_alert + # - mcp__github__get_tag + # - mcp__github__issue_read + # - mcp__github__list_branches + # - mcp__github__list_code_scanning_alerts + # - mcp__github__list_commits + # - mcp__github__list_dependabot_alerts + # - mcp__github__list_discussion_categories + # - mcp__github__list_discussions + # - mcp__github__list_issue_types + # - mcp__github__list_issues + # - mcp__github__list_label + # - mcp__github__list_notifications + # - mcp__github__list_pull_requests + # - mcp__github__list_releases + # - mcp__github__list_secret_scanning_alerts + # - mcp__github__list_starred_repositories + # - mcp__github__list_tags + # - mcp__github__pull_request_read + # - mcp__github__search_code + # - mcp__github__search_issues + # - mcp__github__search_orgs + # - mcp__github__search_pull_requests + # - mcp__github__search_repositories + # - mcp__github__search_users + # - mcp__safeoutputs + timeout-minutes: 30 + run: | + set -o pipefail + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; if [ "$gh_aw_exit_code" -ne 0 ]; then echo "::error::Agent execution exited with code $gh_aw_exit_code"; fi' EXIT + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + mkdir -p /tmp/gh-aw/agent/ && (umask 177 && touch /tmp/gh-aw/agent/claude-debug.log) && touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + # shellcheck disable=SC2016 + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.28.7/awf-config.schema.json","network":{"allowDomains":["*.githubusercontent.com","anthropic.com","api.anthropic.com","api.github.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","cdn.playwright.dev","codeload.github.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","ghcr.io","github-cloud.githubusercontent.com","github-cloud.s3.amazonaws.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","lfs.github.com","objects.githubusercontent.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","playwright.download.prss.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","s.symcb.com","s.symcd.com","security.ubuntu.com","sentry.io","statsig.anthropic.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"],"isolation":true,"topologyAttach":["awmg-mcpg"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxCacheMisses":5,"maxAiCredits":1500,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.5","gpt-5.6","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"auto":["copilot/auto","large"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex","kimi"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"detection":["small"],"evals":["small"],"fable":["copilot/*fable*","anthropic/*fable*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","google/nano-banana*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-3.6-flash":["copilot/gemini-3.6*flash*","google/gemini-3.6*flash*","gemini/gemini-3.6*flash*"],"gemini-3.7-flash":["copilot/gemini-3.7*flash*","google/gemini-3.7*flash*","gemini/gemini-3.7*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-omni":["copilot/gemini-omni*","google/gemini-omni*","gemini/gemini-omni*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.1":["copilot/gpt-5.1*","openai/gpt-5.1*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"gpt-5.6":["copilot/gpt-5.6*","openai/gpt-5.6*"],"grok":["copilot/*grok*","openai/*grok*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"image-generation":["copilot/gpt-image*","openai/gpt-image*","openai/chatgpt-image*","copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","google/imagen*"],"kimi":["copilot/kimi*","openai/kimi*"],"kiwi":["copilot/kiwi*","openai/kiwi*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"lyria":["google/lyria*","gemini/lyria*","copilot/lyria*"],"mai-code":["copilot/MAI-Code*","copilot/mai-code*","openai/MAI-Code*"],"mai-code-1-flash-picker":["copilot/MAI-Code-1-Flash-picker*","copilot/mai-code-1-flash-picker*","openai/MAI-Code-1-Flash-picker*"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"nano-banana":["copilot/nano-banana*","google/nano-banana*","gemini/nano-banana*"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"raptor-mini":["copilot/raptor*","openai/raptor*"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"small-agent":["haiku","gpt-5-mini","gemini-flash"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4.5*","copilot/*sonnet-4.6*","copilot/*sonnet-5*","copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*","anthropic/*sonnet-5*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"veo":["google/veo*","gemini/veo*"],"vision":["copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.28.7,squid=sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9,agent=sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059,api-proxy=sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1,cli-proxy=sha256:ebc8758c9b085ca244234e3e3ee22300d150095f9bfe7312ca8a61c5acb34a78"},"logging":{"proxyLogsDir":"/tmp/gh-aw/sandbox/firewall/logs","auditDir":"/tmp/gh-aw/sandbox/firewall/audit"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + GH_AW_AWF_ENGINE_NAME=claude \ + GH_AW_AWF_HARNESS_MARKER='[claude-harness]' \ + GH_AW_AWF_LOG_FILE=/tmp/gh-aw/agent-stdio.log \ + GH_AW_AWF_ATTEMPT_LOG_NAME=claude \ + bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts:${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts:rw" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --tty --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env ANTHROPIC_API_KEY --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner โ€” check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit,MultiEdit(/tmp/*),NotebookEdit,NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write,Write(/tmp/*),mcp__agenticworkflows,mcp__github__actions_get,mcp__github__actions_list,mcp__github__get_code_scanning_alert,mcp__github__get_commit,mcp__github__get_dependabot_alert,mcp__github__get_discussion,mcp__github__get_discussion_comments,mcp__github__get_file_contents,mcp__github__get_job_logs,mcp__github__get_label,mcp__github__get_latest_release,mcp__github__get_me,mcp__github__get_notification_details,mcp__github__get_pull_request,mcp__github__get_pull_request_comments,mcp__github__get_pull_request_diff,mcp__github__get_pull_request_files,mcp__github__get_pull_request_review_comments,mcp__github__get_pull_request_reviews,mcp__github__get_pull_request_status,mcp__github__get_release_by_tag,mcp__github__get_secret_scanning_alert,mcp__github__get_tag,mcp__github__issue_read,mcp__github__list_branches,mcp__github__list_code_scanning_alerts,mcp__github__list_commits,mcp__github__list_dependabot_alerts,mcp__github__list_discussion_categories,mcp__github__list_discussions,mcp__github__list_issue_types,mcp__github__list_issues,mcp__github__list_label,mcp__github__list_notifications,mcp__github__list_pull_requests,mcp__github__list_releases,mcp__github__list_secret_scanning_alerts,mcp__github__list_starred_repositories,mcp__github__list_tags,mcp__github__pull_request_read,mcp__github__search_code,mcp__github__search_issues,mcp__github__search_orgs,mcp__github__search_pull_requests,mcp__github__search_repositories,mcp__github__search_users,mcp__safeoutputs'\'' --debug-file /tmp/gh-aw/agent/claude-debug.log --verbose --permission-mode acceptEdits --output-format stream-json --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt${GH_AW_MODEL_AGENT_CLAUDE:+ --model "$GH_AW_MODEL_AGENT_CLAUDE"}' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_MAX_RETRIES: 0 + BASH_DEFAULT_TIMEOUT_MS: 60000 + BASH_MAX_TIMEOUT_MS: 60000 + CLAUDE_CODE_DISABLE_FAST_MODE: 1 + DISABLE_BUG_COMMAND: 1 + DISABLE_ERROR_REPORTING: 1 + DISABLE_TELEMETRY: 1 + GH_AW_LLM_PROVIDER: anthropic + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/mcp-servers.json + GH_AW_MODEL_AGENT_CLAUDE: ${{ vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || 'claude-sonnet-5' }} + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + MCP_TIMEOUT: 120000 + MCP_TOOL_TIMEOUT: 60000 + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + env: + GH_AW_AGENTIC_EXECUTION_OUTCOME: ${{ steps.agentic_execution.outcome }} + GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: 30 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'detect_agent_errors.cjs')); + await main(); + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'redact_secrets.cjs')); + await main(); + env: + GH_AW_SECRET_NAMES: 'ANTHROPIC_API_KEY,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Append agent step summary + if: always() + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs + if: always() + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,anthropic.com,api.anthropic.com,api.github.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,cdn.playwright.dev,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,ghcr.io,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,playwright.download.prss.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,s.symcb.com,s.symcd.com,security.ubuntu.com,sentry.io,statsig.anthropic.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'collect_ndjson_output.cjs')); + await main(); + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/agent-stdio.log + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_claude_log.cjs')); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_mcp_gateway_log.cjs')); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_firewall_logs.sh" --rootless + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'awf_reflect_summary.cjs')); + await main(); + - name: Run graders + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'trace_graders.cjs')); + await main('eyJ2ZXJzaW9uIjoxLCJncmFkZXJzIjpbeyJpZCI6InRvb2wtc3VjY2Vzcy1yYXRlIiwibmFtZSI6IlRvb2wgU3VjY2VzcyBSYXRlIiwiZGVzY3JpcHRpb24iOiJGcmFjdGlvbiBvZiB0b29sIGNhbGxzIHRoYXQgc3VjY2VlZGVkIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJyYXRpbyIsImRpcmVjdGlvbiI6ImhpZ2hlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjAuOCwibWF4IjoxLCJtaW4iOjB9LHsiaWQiOiJ0b29sLWZhaWx1cmUtY291bnQiLCJuYW1lIjoiVG9vbCBGYWlsdXJlIENvdW50IiwiZGVzY3JpcHRpb24iOiJOdW1iZXIgb2YgdG9vbCBjYWxscyB0aGF0IGZhaWxlZCIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjV9LHsiaWQiOiJyZXRyaWVzIiwibmFtZSI6IlJldHJpZXMiLCJkZXNjcmlwdGlvbiI6Ik51bWJlciBvZiByZXRyeSBldmVudHMgZGV0ZWN0ZWQgaW4gZ2F0ZXdheSBsb2dzIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJjb3VudCIsImRpcmVjdGlvbiI6Imxvd2VyX2lzX2JldHRlciIsInRocmVzaG9sZCI6MTB9LHsiaWQiOiJsb29wcyIsIm5hbWUiOiJMb29wcyIsImRlc2NyaXB0aW9uIjoiQ29uc2VjdXRpdmUgaWRlbnRpY2FsIHRvb2wgY2FsbHMgKHNhbWUgbmFtZSBhbmQgYXJndW1lbnRzKSIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJ0aHJlc2hvbGQiOjN9LHsiaWQiOiJ0cmFqZWN0b3J5LWVmZmljaWVuY3kiLCJuYW1lIjoiVHJhamVjdG9yeSBFZmZpY2llbmN5IiwiZGVzY3JpcHRpb24iOiJSYXRpbyBvZiB1bmlxdWUgdG9vbCBuYW1lcyB0byB0b3RhbCB0b29sIGNhbGxzIChoaWdoZXIgPSBtb3JlIGRpdmVyc2UgdXNhZ2UpIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJyYXRpbyIsImRpcmVjdGlvbiI6ImhpZ2hlcl9pc19iZXR0ZXIiLCJtYXgiOjEsIm1pbiI6MH0seyJpZCI6ImV4ZWN1dGlvbi1zdGVwLWNvdW50IiwibmFtZSI6IkV4ZWN1dGlvbiBTdGVwIENvdW50IiwiZGVzY3JpcHRpb24iOiJUb3RhbCBMTE0gcmVxdWVzdCBjb3VudCIsInNvdXJjZSI6ImJ1aWx0aW4iLCJlbmFibGVkIjp0cnVlLCJ1bml0IjoiY291bnQiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIifSx7ImlkIjoiZXhlY3V0aW9uLWR1cmF0aW9uIiwibmFtZSI6IkV4ZWN1dGlvbiBEdXJhdGlvbiIsImRlc2NyaXB0aW9uIjoiVG90YWwgZXhlY3V0aW9uIGR1cmF0aW9uIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJtcyIsImRpcmVjdGlvbiI6Imxvd2VyX2lzX2JldHRlciJ9LHsiaWQiOiJ3b3JraW5nLXNldC1yZWJ1aWxkLWZhY3RvciIsIm5hbWUiOiJXb3JraW5nLVNldCBSZWJ1aWxkIEZhY3RvciIsImRlc2NyaXB0aW9uIjoiQ3VtdWxhdGl2ZSBpbnB1dCB0b2tlbnMgZGl2aWRlZCBieSBwZWFrIGludm9jYXRpb24gaW5wdXQgdG9rZW5zIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJmYWN0b3IiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIiLCJtaW4iOjF9LHsiaWQiOiJjb250ZXh0LWdyb3d0aCIsIm5hbWUiOiJDb250ZXh0IEdyb3d0aCIsImRlc2NyaXB0aW9uIjoiUmF0aW8gb2YgdG90YWwgdG9rZW5zIHRvIGZpcnN0LXJlcXVlc3QgdG9rZW5zIiwic291cmNlIjoiYnVpbHRpbiIsImVuYWJsZWQiOnRydWUsInVuaXQiOiJmYWN0b3IiLCJkaXJlY3Rpb24iOiJsb3dlcl9pc19iZXR0ZXIifSx7ImlkIjoiYXJ0aWZhY3QtcHJvZHVjdGlvbiIsIm5hbWUiOiJBcnRpZmFjdCBQcm9kdWN0aW9uIiwiZGVzY3JpcHRpb24iOiJDb3VudCBvZiBvdXRwdXRzL2FydGlmYWN0cyBwcm9kdWNlZCBieSB0aGUgYWdlbnQiLCJzb3VyY2UiOiJidWlsdGluIiwiZW5hYmxlZCI6dHJ1ZSwidW5pdCI6ImNvdW50IiwiZGlyZWN0aW9uIjoiaGlnaGVyX2lzX2JldHRlciJ9XX0=', 'bnVsbA=='); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi + # Upload safe-outputs upload-artifact staging for the upload_artifact job + - name: Upload upload-artifact staging + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-upload-artifacts + path: ${{ runner.temp }}/gh-aw/safeoutputs/upload-artifacts/ + retention-days: 1 + if-no-files-found: ignore + # Small dedicated copy of the agent output so safe-output processing + # survives a failed or timed-out upload of the larger agent artifact + - name: Upload agent output fallback artifact + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-output-fallback + path: | + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_results.json + if-no-files-found: ignore + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/agent_usage.json + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt + /tmp/gh-aw/agent/ + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/agent/graders/grader_manifest.json + /tmp/gh-aw/agent/graders/grader_results.json + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - evals + - push_evals_state + - safe_outputs + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.oauth_token_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true' || + needs.activation.outputs.secret_verification_result == 'failed' || needs.activation.outputs.daily_ai_credits_exceeded == 'true') + runs-on: ubuntu-slim + permissions: + actions: read + issues: write + concurrency: + group: "gh-aw-conclusion-daily-grader-audit" + cancel-in-progress: false + queue: max + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Download detection artifact + id: download-detection-artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: detection + path: /tmp/gh-aw/threat-detection/ + - name: Download Safe Outputs Items Manifest + id: download-safe-outputs-manifest + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: safe-outputs-items + merge-multiple: true + path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: evals + merge-multiple: true + path: /tmp/gh-aw/evals/ + - name: Collect usage artifact files + if: always() + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/collect_usage_artifact_files.sh" + - name: Upload usage artifact + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: usage + path: | + /tmp/gh-aw/usage/aw_info.json + /tmp/gh-aw/usage/aw-info.jsonl + /tmp/gh-aw/usage/agent_usage.json + /tmp/gh-aw/usage/agent_usage.jsonl + /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/evals.jsonl + /tmp/gh-aw/usage/graders/grader_manifest.json + /tmp/gh-aw/usage/graders/grader_results.json + /tmp/gh-aw/usage/github_rate_limits.jsonl + /tmp/gh-aw/usage/agent/token_usage.jsonl + /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json + if-no-files-found: ignore + - name: Restore daily AIC usage cache + id: restore-daily-aic-cache-conclusion + if: always() + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-workflow-usage-dailygraderaudit-${{ github.run_id }} + restore-keys: agentic-workflow-usage-dailygraderaudit- + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + - name: Write daily AIC usage cache entry + id: write-daily-aic-cache + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context); + const { main } = require(path.join(actionsDir, 'write_daily_aic_usage_cache.cjs')); + await main(); + - name: Save daily AIC usage cache + id: save-daily-aic-cache + if: always() + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-workflow-usage-dailygraderaudit-${{ github.run_id }} + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + - name: Upload daily AIC usage cache artifact + id: upload-daily-aic-cache + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: aic-usage-cache + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + if-no-files-found: ignore + retention-days: 7 + - name: Process no-op messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "false" + GH_AW_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} + GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} + GH_AW_WORKFLOW_ID: "daily-grader-audit" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_noop_message.cjs')); + await main(); + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_detection_runs.cjs')); + await main(); + - name: Record missing tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'missing_tool.cjs')); + await main(); + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'report_incomplete_handler.cjs')); + await main(); + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "daily-grader-audit" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "12" + GH_AW_ENGINE_ID: "claude" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_AI_CREDITS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.ai_credits_rate_limit_error || 'false' }} + GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} + GH_AW_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} + GH_AW_MAX_AI_CREDITS: "1500" + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_HTTP_400_RESPONSE_ERROR: ${{ needs.agent.outputs.http_400_response_error }} + GH_AW_MAX_CACHE_MISSES_EXCEEDED: ${{ needs.agent.outputs.max_cache_misses_exceeded }} + GH_AW_MISSING_MODEL_PRICING_ERROR: ${{ needs.agent.outputs.missing_model_pricing_error }} + GH_AW_MISSING_MODEL_PRICING_MODEL_NAME: ${{ needs.agent.outputs.missing_model_pricing_model_name }} + GH_AW_SHELL_EXPANSION_GUARD_REJECTED: ${{ needs.agent.outputs.shell_expansion_guard_rejected }} + GH_AW_ENGINE_API_HOSTS: "api.anthropic.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} + GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} + GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "30" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_agent_failure.cjs')); + await main(); + - name: Report failed jobs + id: report_failed_jobs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_REPORT_FAILED_JOBS: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'report_failed_jobs.cjs')); + await main(); + + detection: + needs: + - activation + - agent + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 10 + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse_detection_token_usage.outputs.aic }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download activation artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.7@sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.7@sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1 ghcr.io/github/gh-aw-firewall/squid:0.28.7@sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9 + - name: Check if detection needed + id: detection_guard + if: always() + env: + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f "$HOME/.copilot/mcp-config.json" + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/prepare_threat_detection_files.sh" + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + WORKFLOW_NAME: "Daily Grader Audit" + WORKFLOW_DESCRIPTION: "Daily audit of workflow runs with deterministic grader results, producing a downloadable 24-hour report artifact" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'setup_threat_detection.cjs')); + await main(); + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.7 --rootless + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.245 + - name: Install threat-detect binary + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/install_threat_detect_binary.sh" v0.4.12 + - name: Execute threat detection with AWF + id: detection_agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + timeout-minutes: 10 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_MODEL: detection + BASH_DEFAULT_TIMEOUT_MS: 60000 + BASH_MAX_TIMEOUT_MS: 60000 + CLAUDE_CODE_DISABLE_FAST_MODE: 1 + DISABLE_BUG_COMMAND: 1 + DISABLE_ERROR_REPORTING: 1 + DISABLE_TELEMETRY: 1 + GH_AW_HARNESS_MAX_RETRIES: 0 + GH_AW_LLM_PROVIDER: anthropic + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: detection + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + MCP_TIMEOUT: 120000 + MCP_TOOL_TIMEOUT: 60000 + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + WORKFLOW_NAME: "Daily Grader Audit" + WORKFLOW_DESCRIPTION: "Daily audit of workflow runs with deterministic grader results, producing a downloadable 24-hour report artifact" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.7/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.githubusercontent.com\",\"anthropic.com\",\"api.anthropic.com\",\"api.github.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"cdn.playwright.dev\",\"codeload.github.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"ghcr.io\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"playwright.download.prss.microsoft.com\",\"ppa.launchpad.net\",\"raw.githubusercontent.com\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"sentry.io\",\"statsig.anthropic.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.7,squid=sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9,agent=sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059,api-proxy=sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1,cli-proxy=sha256:ebc8758c9b085ca244234e3e3ee22300d150095f9bfe7312ca8a61c5acb34a78\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + _GH_AW_CHROOT_JSON=$(jq -c --arg src "${RUNNER_TEMP}/gh-aw" --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home "${RUNNER_TEMP}/gh-aw/home" '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; } + printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env ANTHROPIC_API_KEY --mount /tmp/gh-aw:/tmp/gh-aw:rw --mount /tmp/gh-aw/threat-detection:/tmp/gh-aw/threat-detection:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && threat-detect --engine claude --output /tmp/gh-aw/threat-detection/detection_result.json /tmp/gh-aw/threat-detection' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + - name: Render detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'render_detection_log.cjs')); + await main(); + - name: Copy detection firewall logs + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + run: | + mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall + if [ -d /tmp/gh-aw/sandbox/firewall/logs ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/logs && cp -r /tmp/gh-aw/sandbox/firewall/logs/. /tmp/gh-aw/threat-detection/sandbox/firewall/logs/; fi + if [ -d /tmp/gh-aw/sandbox/firewall/audit ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/audit && cp -r /tmp/gh-aw/sandbox/firewall/audit/. /tmp/gh-aw/threat-detection/sandbox/firewall/audit/; fi + - name: Upload threat detection artifact + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: detection + path: | + /tmp/gh-aw/threat-detection/detection_result.json + /tmp/gh-aw/threat-detection/sandbox/firewall/logs/ + /tmp/gh-aw/threat-detection/sandbox/firewall/audit/ + if-no-files-found: ignore + - name: Parse threat detection token usage for step summary + id: parse_detection_token_usage + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); + await main(); + - name: Conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json + + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.7@sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.7@sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1 ghcr.io/github/gh-aw-firewall/squid:0.28.7@sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"grader_runs_audited","question":"Did the agent inspect workflow runs with deterministic grader results from the last 24 hours?"},{"id":"downloadable_report_uploaded","question":"Did the agent generate and upload a grader audit report artifact retained for 24 hours?"}]' + GH_AW_EVALS_MODEL: "evals" + GH_AW_EVALS_PHASE: setup + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'run_evals.cjs')); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.7 --rootless + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.245 + - name: Execute Claude Code CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Allowed tools (sorted): + # - Bash + # - BashOutput + # - Edit(/tmp/*) + # - ExitPlanMode + # - Glob + # - Grep + # - KillBash + # - LS + # - MultiEdit(/tmp/*) + # - NotebookRead + # - Read + # - Read(/tmp/*) + # - Task + # - TodoWrite + # - Write(/tmp/*) + timeout-minutes: 30 + run: | + set -o pipefail + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; if [ "$gh_aw_exit_code" -ne 0 ]; then echo "::error::Agent execution exited with code $gh_aw_exit_code"; fi' EXIT + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + mkdir -p /tmp/gh-aw/agent/ && (umask 177 && touch /tmp/gh-aw/agent/claude-debug.log) && touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + # shellcheck disable=SC2016 + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.28.7/awf-config.schema.json","network":{"allowDomains":["*.githubusercontent.com","anthropic.com","api.anthropic.com","api.github.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","cdn.playwright.dev","codeload.github.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","ghcr.io","github-cloud.githubusercontent.com","github-cloud.s3.amazonaws.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","lfs.github.com","objects.githubusercontent.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","playwright.download.prss.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","s.symcb.com","s.symcd.com","security.ubuntu.com","sentry.io","statsig.anthropic.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxCacheMisses":5,"maxAiCredits":1500,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.5","gpt-5.6","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"auto":["copilot/auto","large"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex","kimi"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"detection":["small"],"evals":["small"],"fable":["copilot/*fable*","anthropic/*fable*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","google/nano-banana*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-3.6-flash":["copilot/gemini-3.6*flash*","google/gemini-3.6*flash*","gemini/gemini-3.6*flash*"],"gemini-3.7-flash":["copilot/gemini-3.7*flash*","google/gemini-3.7*flash*","gemini/gemini-3.7*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-omni":["copilot/gemini-omni*","google/gemini-omni*","gemini/gemini-omni*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.1":["copilot/gpt-5.1*","openai/gpt-5.1*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"gpt-5.6":["copilot/gpt-5.6*","openai/gpt-5.6*"],"grok":["copilot/*grok*","openai/*grok*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"image-generation":["copilot/gpt-image*","openai/gpt-image*","openai/chatgpt-image*","copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","google/imagen*"],"kimi":["copilot/kimi*","openai/kimi*"],"kiwi":["copilot/kiwi*","openai/kiwi*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"lyria":["google/lyria*","gemini/lyria*","copilot/lyria*"],"mai-code":["copilot/MAI-Code*","copilot/mai-code*","openai/MAI-Code*"],"mai-code-1-flash-picker":["copilot/MAI-Code-1-Flash-picker*","copilot/mai-code-1-flash-picker*","openai/MAI-Code-1-Flash-picker*"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"nano-banana":["copilot/nano-banana*","google/nano-banana*","gemini/nano-banana*"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"raptor-mini":["copilot/raptor*","openai/raptor*"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"small-agent":["haiku","gpt-5-mini","gemini-flash"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4.5*","copilot/*sonnet-4.6*","copilot/*sonnet-5*","copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*","anthropic/*sonnet-5*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"veo":["google/veo*","gemini/veo*"],"vision":["copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.28.7,squid=sha256:fb362a08d4d2f0da6c036e3f5d3b2fd87931e857fec3ca4a241cd2f2b61131f9,agent=sha256:40a1e30b1b8d70642d4292485146cd5af612730d7a6a2e12706ddd13df375059,api-proxy=sha256:4f209dd4cbc74d47a6c7379956143de293429d1b1b2fb2647776cdcbf65836a1,cli-proxy=sha256:ebc8758c9b085ca244234e3e3ee22300d150095f9bfe7312ca8a61c5acb34a78"},"logging":{"proxyLogsDir":"/tmp/gh-aw/sandbox/firewall/logs","auditDir":"/tmp/gh-aw/sandbox/firewall/audit"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + GH_AW_AWF_ENGINE_NAME=claude \ + GH_AW_AWF_HARNESS_MARKER='[claude-harness]' \ + GH_AW_AWF_LOG_FILE=/tmp/gh-aw/evals/evals.log \ + GH_AW_AWF_ATTEMPT_LOG_NAME=claude \ + bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --tty --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env ANTHROPIC_API_KEY --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner โ€” check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/agent/claude-debug.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_MAX_RETRIES: 0 + ANTHROPIC_MODEL: evals + BASH_DEFAULT_TIMEOUT_MS: 60000 + BASH_MAX_TIMEOUT_MS: 60000 + CLAUDE_CODE_DISABLE_FAST_MODE: 1 + DISABLE_BUG_COMMAND: 1 + DISABLE_ERROR_REPORTING: 1 + DISABLE_TELEMETRY: 1 + GH_AW_LLM_PROVIDER: anthropic + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + MCP_TIMEOUT: 120000 + MCP_TOOL_TIMEOUT: 60000 + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_mcp_gateway_log.cjs')); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"grader_runs_audited","question":"Did the agent inspect workflow runs with deterministic grader results from the last 24 hours?"},{"id":"downloadable_report_uploaded","question":"Did the agent generate and upload a grader audit report artifact retained for 24 hours?"}]' + GH_AW_EVALS_MODEL: "evals" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'run_evals.cjs')); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'redact_evals_results.cjs')); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'render_evals_summary.cjs')); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + pattern: evals + merge-multiple: true + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/dailygraderaudit + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'push_experiment_state.cjs')); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + + safe_outputs: + needs: + - activation + - agent + - detection + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' + runs-on: ubuntu-slim + permissions: + issues: write + timeout-minutes: 45 + env: + GH_AW_AGENT_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/daily-grader-audit" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} + GH_AW_ENGINE_ID: "claude" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_PROJECT_UTC: "-08:00" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_WORKFLOW_EMOJI: "๐Ÿ“Š" + GH_AW_WORKFLOW_ID: "daily-grader-audit" + GH_AW_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-grader-audit.md" + outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_items_applied: ${{ steps.process_safe_outputs.outputs.items_applied }} + process_safe_outputs_items_cancelled: ${{ steps.process_safe_outputs.outputs.items_cancelled }} + process_safe_outputs_items_deferred: ${{ steps.process_safe_outputs.outputs.items_deferred }} + process_safe_outputs_items_failed: ${{ steps.process_safe_outputs.outputs.items_failed }} + process_safe_outputs_items_skipped: ${{ steps.process_safe_outputs.outputs.items_skipped }} + process_safe_outputs_items_succeeded: ${{ steps.process_safe_outputs.outputs.items_succeeded }} + process_safe_outputs_items_warnings: ${{ steps.process_safe_outputs.outputs.items_warnings }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_status: ${{ steps.process_safe_outputs.outputs.status }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + upload_artifact_count: ${{ steps.process_safe_outputs.outputs.upload_artifact_count }} + upload_artifact_slot_0_tmp_id: ${{ steps.process_safe_outputs.outputs.slot_0_tmp_id }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + safe-output-artifact-client: 'true' + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Grader Audit" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-grader-audit.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.245" + GH_AW_INFO_AWF_VERSION: "v0.28.7" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash + run: | # zizmor: ignore[github-env] - GITHUB_SERVER_URL is set by GitHub Actions, not user input. + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Download upload-artifact staging + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: safe-outputs-upload-artifacts + path: ${{ runner.temp }}/gh-aw/safeoutputs/upload-artifacts/ + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,anthropic.com,api.anthropic.com,api.github.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,cdn.playwright.dev,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,ghcr.io,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,playwright.download.prss.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,s.symcb.com,s.symcd.com,security.ubuntu.com,sentry.io,statsig.anthropic.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{},\"upload_artifact\":{\"max-size-bytes\":104857600,\"max-uploads\":1,\"retention-days\":1,\"skip-archive\":true}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'process_safe_outputs.cjs')); + await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + /tmp/gh-aw/safe-output-errors.json + if-no-files-found: ignore diff --git a/.github/workflows/daily-grader-audit.md b/.github/workflows/daily-grader-audit.md new file mode 100644 index 00000000000..a7c38ee4fb3 --- /dev/null +++ b/.github/workflows/daily-grader-audit.md @@ -0,0 +1,156 @@ +--- +private: true +emoji: "๐Ÿ“Š" +description: Daily audit of workflow runs with deterministic grader results, producing a downloadable 24-hour report artifact +on: + schedule: daily around 8:00 + workflow_dispatch: +permissions: + contents: read + actions: read +max-ai-credits: 1500 +max-daily-ai-credits: 5000 +engine: + id: claude + mcp: + tool-timeout: 10m +tools: + cli-proxy: true + agentic-workflows: +safe-outputs: + upload-artifact: + max-uploads: 1 + retention-days: 1 + skip-archive: true +timeout-minutes: 30 +features: + gh-aw-detection: true +imports: + - shared/reporting.md +graders: {} +evals: + - id: grader_runs_audited + question: Did the agent inspect workflow runs with deterministic grader results from the last 24 hours? + - id: downloadable_report_uploaded + question: Did the agent generate and upload a grader audit report artifact retained for 24 hours? +--- + +# Daily Grader Audit + +You are the grader audit reporter for GitHub Agentic Workflows. + +## Mission + +Inspect workflow runs from the last 24 full hours that produced deterministic grader results, summarize grader health, and upload a Markdown report artifact retained for 24 hours. + +## Context + +- Repository: `${{ github.repository }}` +- Run ID: `${{ github.run_id }}` +- Window: last 24 full hours ending at workflow start (UTC) +- Report path: `/tmp/gh-aw/agent/grader-audit-report.md` + +## Phase 1: Fetch Grader Runs + +Use the agentic-workflows MCP `logs` tool once: + +```json +{ + "workflow_name": "", + "count": 100, + "start_date": "-1d", + "graders": true, + "artifacts": ["graders", "usage"] +} +``` + +Read the returned JSON file path. The logs tool filters to runs with grader results and each `runs[]` entry may include: + +- `run_id` +- `workflow_name` +- `conclusion` +- `created_at` +- `url` +- `graders.total` +- `graders.passed` +- `graders.failed` +- `graders.error_count` +- `graders.unavailable_count` +- `graders.results[]` + +If no matching runs are found, still write a report explaining that no grader results were available for the window. + +## Phase 2: Analyze Results + +For each run with `graders.results[]`, compute: + +- Per-run totals: total, passed, failed, errors, unavailable. +- Per-grader totals grouped by `id`: runs observed, pass count, fail count, error count, unavailable count. +- Pass rate per grader: `pass count / runs observed * 100`. +- Notable failures: any grader result where `status` is `fail`, `error`, or `unavailable`. + +Do not infer missing graders from workflow files. Only report grader data present in the logs tool output. + +## Phase 3: Generate Report + +Create `/tmp/gh-aw/agent/grader-audit-report.md` with: + +### Summary + +One short paragraph covering the window, number of runs, number of workflows, and overall grader health. + +### Key Metrics + +| Metric | Value | +|---|---| +| Runs with grader results | N | +| Workflows with grader results | N | +| Grader result rows | N | +| Passing results | N | +| Failing results | N | +| Error results | N | +| Unavailable results | N | + +### Per-Grader Results + +| Grader | Runs | Pass | Fail | Error | Unavailable | Pass Rate | +|---|---:|---:|---:|---:|---:|---:| +| grader-id | N | N | N | N | N | X% | + +### Per-Run Results + +| Run | Workflow | Conclusion | Total | Pass | Fail | Error | Unavailable | +|---|---|---|---:|---:|---:|---:|---:| +| [ยง123](https://github.com/owner/repo/actions/runs/123) | workflow-name | success | N | N | N | N | N | + +
+Notable grader failures + +| Run | Workflow | Grader | Status | Value | Threshold | Message | +|---|---|---|---|---:|---|---| +| [ยง123](https://github.com/owner/repo/actions/runs/123) | workflow-name | grader-id | fail | 0.2 | >= 0.8 | explanation | + +
+ +### Recommendations + +List concrete follow-up actions only when failed, errored, or unavailable grader results exist. Otherwise state that no immediate action is needed. + +### Context + +- Window: last 24 full hours ending at workflow start (UTC) +- Source: `gh aw logs --graders --start-date -1d --artifacts graders,usage` +- Generated by run `${{ github.run_id }}` + +## Phase 4: Upload Report + +Stage and upload the report: + +1. Copy `/tmp/gh-aw/agent/grader-audit-report.md` to `${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts/grader-audit-report.md`. +2. Call the `upload_artifact` safe-output tool with: + +```json +{ "path": "grader-audit-report.md" } +``` + +The workflow safe-output configuration retains the uploaded report for 1 day. diff --git a/docs/adr/56359-add-graders-run-filter-and-daily-audit-workflow.md b/docs/adr/56359-add-graders-run-filter-and-daily-audit-workflow.md new file mode 100644 index 00000000000..9d76143afae --- /dev/null +++ b/docs/adr/56359-add-graders-run-filter-and-daily-audit-workflow.md @@ -0,0 +1,49 @@ +# ADR-56359: Add Grader Run Filter and Daily Grader Audit Workflow + +**Date**: 2026-08-27 +**Status**: Draft +**Deciders**: pelikhan, copilot-swe-agent + +--- + +### Context + +After grader results were surfaced in the `usage` artifact (see ADR-56066), operators and automation needed two new capabilities: (1) a way to narrow `gh aw logs` downloads to only workflow runs that produced grader results, analogous to the existing `--evals` filter, and (2) a scheduled audit workflow that aggregates 24-hour grader health across all agentic workflows and uploads a downloadable Markdown report. Without a run-level grader filter, every `gh aw logs` call in a grader-focused investigation would download all runs regardless of whether they contained grader output, wasting bandwidth and time. Without a dedicated audit workflow, grader health could only be inspected by manually running ad-hoc CLI queries. + +### Decision + +We will add `--graders` as a boolean run-level filter to `gh aw logs` (forwarded through the MCP `logs` tool as the `graders` parameter) and introduce a new daily scheduled workflow (`daily-grader-audit.md`) that fetches the last 24 hours of runs with grader results, computes per-grader pass rates, and uploads a Markdown report artifact retained for 24 hours. The filter follows the same pattern as `--evals`: when active it auto-expands the artifact set to include `graders`, then applies `skipByGradersFilter` to exclude runs that lack grader result files. The core `collectProcessedWorkflowRuns` loop was refactored into `fetchAndProcessLogsBatch` and `handleLogsBatchError` helpers as part of this work. + +### Alternatives Considered + +#### Alternative 1: Post-Download Filtering Inside the Audit Workflow Prompt + +Instead of adding a `--graders` CLI flag, the daily audit workflow could download all runs and have the agent discard entries without grader data after downloading. + +This would avoid any CLI or MCP changes. It was rejected because it wastes bandwidth and runner time proportional to total run volume, which scales poorly as the number of agentic workflow runs grows. The `--evals` precedent shows that run-level filtering at download time is the correct pattern for optional, artifact-backed features; diverging from that convention for graders would create an inconsistent developer experience. + +#### Alternative 2: Extend `gh aw audit` with a Cross-Run Grader Health Section + +Retrofit `gh aw audit` to aggregate grader outcomes across multiple runs rather than building a separate scheduled workflow. + +This would reuse existing audit infrastructure. It was rejected because `gh aw audit` operates on a single run's downloaded data and its API is not designed for cross-run aggregation. Restructuring it to support a rolling 24-hour window would require significant scope expansion. A standalone scheduled workflow with its own agent prompt is a cleaner fit and keeps `gh aw audit` single-run-scoped. + +### Consequences + +#### Positive +- Operators and automated agents can scope `gh aw logs` downloads exclusively to grader-bearing runs, reducing bandwidth and latency for grader-focused investigations. +- A daily Markdown artifact gives teams a lightweight, downloadable grader health report with per-grader pass rates and notable failures without any custom queries. +- The implementation follows established conventions (`--evals` pattern, `skipByXxxFilter` chain, artifact-set auto-expansion), keeping the filter layer consistent and predictable. + +#### Negative +- The `collectProcessedWorkflowRuns` function required structural refactoring (extraction of `fetchAndProcessLogsBatch` and `handleLogsBatchError`) to accommodate the new code paths, introducing churn to a core pagination loop. +- The new scheduled workflow consumes AI credits daily (up to 1,500 per run, capped at 5,000 per day), adding recurring cost even when grader health is stable. + +#### Neutral +- The MCP `logs` tool schema gains a `graders` boolean parameter; this is a backwards-compatible addition that does not break existing callers. +- The `graders` artifact set is automatically included when `--graders` is active, consistent with how `--evals` auto-includes the evals artifact set. +- The compiled lock file (`.github/workflows/daily-grader-audit.lock.yml`) is generated by `gh aw compile` from the `.md` source; the lock file itself is not hand-maintained. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* diff --git a/pkg/cli/logs_artifact_set.go b/pkg/cli/logs_artifact_set.go index b8134036b0c..68b8a760cdc 100644 --- a/pkg/cli/logs_artifact_set.go +++ b/pkg/cli/logs_artifact_set.go @@ -362,6 +362,20 @@ func applyEvalsArtifact(artifacts []string, evalsOnly bool) []string { return artifacts } +// applyGradersArtifact appends the graders artifact set to artifacts when gradersOnly is true +// and neither ArtifactSetGraders nor ArtifactSetAll is already present. +func applyGradersArtifact(artifacts []string, gradersOnly bool) []string { + if len(artifacts) == 0 { + return artifacts + } + if gradersOnly && + !slices.Contains(artifacts, string(ArtifactSetGraders)) && + !slices.Contains(artifacts, string(ArtifactSetAll)) { + return append(artifacts, string(ArtifactSetGraders)) + } + return artifacts +} + // isEvalsArtifactRequested reports whether evals were explicitly requested, // either via --evals or by including --artifacts evals. Callers use this to // decide whether to bypass stale cache entries and trigger legacy dedicated-evals @@ -369,3 +383,9 @@ func applyEvalsArtifact(artifacts []string, evalsOnly bool) []string { func isEvalsArtifactRequested(evalsOnly bool, artifactSets []string) bool { return evalsOnly || slices.Contains(artifactSets, string(ArtifactSetEvals)) } + +// isGradersArtifactRequested reports whether grader artifacts were explicitly requested, +// either via --graders or by including --artifacts graders. +func isGradersArtifactRequested(gradersOnly bool, artifactSets []string) bool { + return gradersOnly || slices.Contains(artifactSets, string(ArtifactSetGraders)) +} diff --git a/pkg/cli/logs_artifact_set_test.go b/pkg/cli/logs_artifact_set_test.go index 19edd64ed93..4f09bd9d495 100644 --- a/pkg/cli/logs_artifact_set_test.go +++ b/pkg/cli/logs_artifact_set_test.go @@ -293,6 +293,32 @@ func TestApplyEvalsArtifact(t *testing.T) { }) } +func TestApplyGradersArtifact(t *testing.T) { + t.Parallel() + t.Run("returns empty slice unchanged when artifact list is empty", func(t *testing.T) { + t.Parallel() + assert.Empty(t, applyGradersArtifact(nil, true)) + assert.Empty(t, applyGradersArtifact([]string{}, true)) + }) + + t.Run("appends graders when graders requested and artifact list narrowed", func(t *testing.T) { + t.Parallel() + assert.Equal(t, []string{"usage", "graders"}, applyGradersArtifact([]string{"usage"}, true)) + }) + + t.Run("does not append graders when already present", func(t *testing.T) { + t.Parallel() + assert.Equal(t, []string{"graders"}, applyGradersArtifact([]string{"graders"}, true)) + }) +} + +func TestIsGradersArtifactRequested(t *testing.T) { + t.Parallel() + assert.True(t, isGradersArtifactRequested(true, nil)) + assert.True(t, isGradersArtifactRequested(false, []string{"graders"})) + assert.False(t, isGradersArtifactRequested(false, []string{"usage"})) +} + func TestIsEvalsArtifactRequested(t *testing.T) { t.Parallel() tests := []struct { diff --git a/pkg/cli/logs_command.go b/pkg/cli/logs_command.go index 0f3fef2264d..88780ae4404 100644 --- a/pkg/cli/logs_command.go +++ b/pkg/cli/logs_command.go @@ -63,6 +63,7 @@ const logsCommandExampleTemplate = ` # Basic usage %[1]s logs --ref feature-xyz # Filter logs by feature branch %[1]s logs --filtered-integrity # Filter logs containing items that were filtered by gateway integrity checks %[1]s logs --evals # Filter logs from workflows with evals results + %[1]s logs --graders # Filter logs from workflows with grader results %[1]s logs --exclude-staged # Exclude staged workflow runs from results # Run ID range filtering @@ -203,6 +204,7 @@ func loadStdinLogsOptions(cmd *cobra.Command) (StdinLogsOptions, error) { SafeOutputType: values.SafeOutputType, FilteredIntegrity: values.FilteredIntegrity, EvalsOnly: values.EvalsOnly, + GradersOnly: values.GradersOnly, Train: values.Train, Format: values.Format, ReportFile: values.ReportFile, @@ -267,6 +269,7 @@ func loadCommonLogsOptions(cmd *cobra.Command) (LogsDownloadOptions, error) { SafeOutputType: getStringFlag(cmd, "safe-output"), FilteredIntegrity: getBoolFlag(cmd, "filtered-integrity"), EvalsOnly: getBoolFlag(cmd, "evals"), + GradersOnly: getBoolFlag(cmd, "graders"), Train: getBoolFlag(cmd, "train"), Format: getStringFlag(cmd, "format"), ReportFile: getStringFlag(cmd, "report-file"), @@ -277,6 +280,7 @@ func loadCommonLogsOptions(cmd *cobra.Command) (LogsDownloadOptions, error) { } if len(options.ArtifactSets) > 0 { options.ArtifactSets = applyEvalsArtifact(options.ArtifactSets, options.EvalsOnly) + options.ArtifactSets = applyGradersArtifact(options.ArtifactSets, options.GradersOnly) } return options, nil } @@ -404,6 +408,7 @@ func addLogsCommandFlags(logsCmd *cobra.Command, validArtifactSets string) { logsCmd.Flags().String("safe-output", "", "Filter to runs containing a specific safe output type (e.g., create-issue, missing-tool, missing-data, noop, report-incomplete)") logsCmd.Flags().Bool("filtered-integrity", false, "Filter to runs containing items that were filtered by gateway integrity checks") logsCmd.Flags().Bool("evals", false, "Filter to runs containing evals results (evals.jsonl); automatically includes the usage artifact (which contains evals)") + logsCmd.Flags().Bool("graders", false, "Filter to runs containing deterministic grader results; automatically includes grader artifacts") logsCmd.Flags().Bool("parse", false, "Run JavaScript parsers on agent logs and firewall logs, writing Markdown to log.md and firewall.md") addJSONFlag(logsCmd) logsCmd.Flags().Int("timeout", 0, "Download timeout in minutes (0 = no timeout)") diff --git a/pkg/cli/logs_filtering_test.go b/pkg/cli/logs_filtering_test.go index 0588f1b6611..64599f33ed5 100644 --- a/pkg/cli/logs_filtering_test.go +++ b/pkg/cli/logs_filtering_test.go @@ -17,7 +17,7 @@ func TestLogsCommandFlags(t *testing.T) { cmd := NewLogsCommand() // Check that all expected flags are present - expectedFlags := []string{"count", "start-date", "end-date", "output", "engine", "ref", "before-run-id", "after-run-id", "filtered-integrity"} + expectedFlags := []string{"count", "start-date", "end-date", "output", "engine", "ref", "before-run-id", "after-run-id", "filtered-integrity", "graders"} for _, flagName := range expectedFlags { flag := cmd.Flags().Lookup(flagName) @@ -602,3 +602,23 @@ func TestFilteredIntegrityFlag(t *testing.T) { t.Error("Expected 'filtered-integrity' flag to have usage text") } } + +// TestGradersFlag verifies the --graders flag is registered correctly. +func TestGradersFlag(t *testing.T) { + t.Parallel() + + cmd := NewLogsCommand() + + flag := cmd.Flags().Lookup("graders") + if flag == nil { + t.Fatal("Expected flag 'graders' not found in logs command") + } + + if flag.DefValue != "false" { + t.Errorf("Expected 'graders' default to be 'false', got: %s", flag.DefValue) + } + + if !strings.Contains(flag.Usage, "grader results") { + t.Errorf("Expected 'graders' usage to mention grader results, got: %s", flag.Usage) + } +} diff --git a/pkg/cli/logs_orchestrator_download.go b/pkg/cli/logs_orchestrator_download.go index 8a5b38b2797..ca6f33dd225 100644 --- a/pkg/cli/logs_orchestrator_download.go +++ b/pkg/cli/logs_orchestrator_download.go @@ -68,12 +68,13 @@ func prepareLogsDownload(ctx context.Context, opts LogsDownloadOptions) (logsDow safeOutputType: opts.SafeOutputType, filteredIntegrity: opts.FilteredIntegrity, evalsOnly: opts.EvalsOnly, + gradersOnly: opts.GradersOnly, }, }, nil } func logLogsDownloadStart(opts LogsDownloadOptions) { - logsOrchestratorLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s, summaryFile=%s, safeOutputType=%s, filteredIntegrity=%v, evalsOnly=%v, train=%v, format=%s, artifactSets=%v, after=%s", opts.WorkflowName, opts.Count, opts.StartDate, opts.EndDate, opts.OutputDir, opts.SummaryFile, opts.SafeOutputType, opts.FilteredIntegrity, opts.EvalsOnly, opts.Train, opts.Format, opts.ArtifactSets, opts.After) + logsOrchestratorLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s, summaryFile=%s, safeOutputType=%s, filteredIntegrity=%v, evalsOnly=%v, gradersOnly=%v, train=%v, format=%s, artifactSets=%v, after=%s", opts.WorkflowName, opts.Count, opts.StartDate, opts.EndDate, opts.OutputDir, opts.SummaryFile, opts.SafeOutputType, opts.FilteredIntegrity, opts.EvalsOnly, opts.GradersOnly, opts.Train, opts.Format, opts.ArtifactSets, opts.After) } func resolveLogsArtifactFilter(artifactSets []string, verbose bool) ([]string, error) { @@ -182,102 +183,107 @@ var ( logsProcessWorkflowRunBatch = processWorkflowRunBatch ) +type logsCollectionState struct { + processedRuns []ProcessedRun + beforeDate string + iteration int + timeoutReached bool + countLimitReached bool +} + func collectProcessedWorkflowRuns(runtime logsDownloadRuntime, opts LogsDownloadOptions) ([]ProcessedRun, bool, bool, string, error) { - var processedRuns []ProcessedRun - var beforeDate string - var iteration int - var timeoutReached, countLimitReached bool - for iteration < MaxIterations { + state := logsCollectionState{} + for state.iteration < MaxIterations { stop, timedOut, err := shouldStopLogsIteration(runtime, opts) if err != nil { - return processedRuns, timeoutReached || timedOut, countLimitReached, beforeDate, err + return state.processedRuns, state.timeoutReached || timedOut, state.countLimitReached, state.beforeDate, err } if stop { - timeoutReached = timeoutReached || timedOut + state.timeoutReached = state.timeoutReached || timedOut break } - if len(processedRuns) >= opts.Count { - countLimitReached = runtime.fetchAllInRange + if len(state.processedRuns) >= opts.Count { + state.countLimitReached = runtime.fetchAllInRange break } - if err := waitForLogsRateLimit(runtime.activeCtx, opts.Verbose, iteration); err != nil { - continue - } - iteration++ - logLogsIterationFetch(opts, runtime.fetchAllInRange, iteration, len(processedRuns)) - batch, err := logsFetchWorkflowRunBatch(runtime.activeCtx, opts, beforeDate, len(processedRuns), runtime.fetchAllInRange) - if err != nil { - // Context deadline exceeded means our own timeout fired mid-call. - // Treat it like a graceful timeout: return whatever was collected so far. + if err := waitForLogsRateLimit(runtime.activeCtx, opts.Verbose, state.iteration); err != nil { if errors.Is(err, context.DeadlineExceeded) { - timeoutReached = true + state.timeoutReached = true break } - // Context cancelled (e.g. user signal or outer gateway timeout): propagate - // the error without partial results. The caller discards partial runs on - // error, so returning them here would be misleading. We leave - // timeoutReached=false because this is an external cancellation, not the - // internal --timeout deadline firing. if errors.Is(err, context.Canceled) { return nil, false, false, "", err } - return nil, false, false, "", err - } - if len(batch.runs) == 0 { - cursor, shouldContinue, shouldStop := handleEmptyWorkflowRunBatch(batch, opts.Verbose) - if shouldStop { - break - } - if shouldContinue { - beforeDate = cursor - continue - } + logsOrchestratorLog.Printf("Rate limit wait failed, retrying iteration: %v", err) + state.iteration++ + continue } - logWorkflowRunBatchFound(batch, iteration, opts.Verbose) - var batchProcessed int - var allRunsConsumed, batchTimedOut bool - // Assign (not :=) so the accumulated runs are not shadowed by a loop-scoped variable. - processedRuns, batchProcessed, allRunsConsumed, batchTimedOut = logsProcessWorkflowRunBatch(runtime.activeCtx, batch, processedRuns, processWorkflowRunBatchOptions{ - count: opts.Count, - outputDir: opts.OutputDir, - verbose: opts.Verbose, - repoOverride: opts.RepoOverride, - artifactFilter: runtime.artifactFilter, - evalsOnly: opts.EvalsOnly, - artifactSets: opts.ArtifactSets, - parse: opts.Parse, - filters: runtime.filters, - }) - timeoutReached = timeoutReached || batchTimedOut - logProcessedWorkflowRunBatch(opts, runtime.fetchAllInRange, iteration, batchProcessed, len(processedRuns), opts.Verbose) - if allRunsConsumed { - if cursor, ok := selectPaginationCursorDate(batch.runs, batch.oldestFetchedCreatedAt); ok { - beforeDate = cursor - } + state.iteration++ + stop, err = fetchAndProcessLogsBatch(&state, runtime, opts) + if err != nil { + return state.processedRuns, state.timeoutReached, state.countLimitReached, state.beforeDate, err } - if shouldStopAfterWorkflowRunBatch(batch, opts.Verbose) { + if stop { break } } - logLogsIterationLimit(runtime.fetchAllInRange, iteration, len(processedRuns), opts.Count) - logLogsTimeoutResult(timeoutReached, len(processedRuns)) - // Hitting MaxIterations without reaching the requested count or the timeout - // means pagination stopped before the explicit date range was fully scanned - // (e.g. a burst of non-matching runs consumed every iteration's batch before - // the cursor could walk across the whole window). Previously this silently - // returned whatever partial data had accumulated with no continuation cursor, - // so callers requesting a wide start_date/end_date window could mistake a - // narrow, stale slice of results for a complete scan of the range (see - // github/gh-aw#53995). Treat it the same as a count-limit break so a - // continuation is always surfaced for explicit date-range queries. - if runtime.fetchAllInRange && !timeoutReached && iteration >= MaxIterations { - countLimitReached = true - } - // beforeDate reflects the pagination cursor collectProcessedWorkflowRuns actually - // advanced to (including iterations that fetched runs but matched none of them), - // so callers building a continuation can resume the scan from here instead of - // re-fetching pages already visited (see github/gh-aw#54110). - return processedRuns, timeoutReached, countLimitReached, beforeDate, nil + logLogsIterationLimit(runtime.fetchAllInRange, state.iteration, len(state.processedRuns), opts.Count) + logLogsTimeoutResult(state.timeoutReached, len(state.processedRuns)) + if runtime.fetchAllInRange && !state.timeoutReached && state.iteration >= MaxIterations { + state.countLimitReached = true + } + return state.processedRuns, state.timeoutReached, state.countLimitReached, state.beforeDate, nil +} + +func fetchAndProcessLogsBatch(state *logsCollectionState, runtime logsDownloadRuntime, opts LogsDownloadOptions) (bool, error) { + logLogsIterationFetch(opts, runtime.fetchAllInRange, state.iteration, len(state.processedRuns)) + batch, err := logsFetchWorkflowRunBatch(runtime.activeCtx, opts, state.beforeDate, len(state.processedRuns), runtime.fetchAllInRange) + if err != nil { + return handleLogsBatchError(state, runtime.activeCtx, err) + } + if len(batch.runs) == 0 { + cursor, shouldContinue, shouldStop := handleEmptyWorkflowRunBatch(batch, opts.Verbose) + if shouldStop { + return true, nil + } + if shouldContinue { + state.beforeDate = cursor + return false, nil + } + } + logWorkflowRunBatchFound(batch, state.iteration, opts.Verbose) + var batchProcessed int + var allRunsConsumed, batchTimedOut bool + state.processedRuns, batchProcessed, allRunsConsumed, batchTimedOut = logsProcessWorkflowRunBatch(runtime.activeCtx, batch, state.processedRuns, processWorkflowRunBatchOptions{ + count: opts.Count, + outputDir: opts.OutputDir, + verbose: opts.Verbose, + repoOverride: opts.RepoOverride, + artifactFilter: runtime.artifactFilter, + evalsOnly: opts.EvalsOnly, + artifactSets: opts.ArtifactSets, + parse: opts.Parse, + filters: runtime.filters, + }) + state.timeoutReached = state.timeoutReached || batchTimedOut + logProcessedWorkflowRunBatch(opts, runtime.fetchAllInRange, state.iteration, batchProcessed, len(state.processedRuns), opts.Verbose) + if allRunsConsumed { + if cursor, ok := selectPaginationCursorDate(batch.runs, batch.oldestFetchedCreatedAt); ok { + state.beforeDate = cursor + } + } + return shouldStopAfterWorkflowRunBatch(batch, opts.Verbose), nil +} + +func handleLogsBatchError(state *logsCollectionState, ctx context.Context, err error) (bool, error) { + if errors.Is(err, context.DeadlineExceeded) { + state.timeoutReached = true + return true, nil + } + if errors.Is(err, context.Canceled) { + return true, err + } + return false, err } func shouldStopLogsIteration(runtime logsDownloadRuntime, opts LogsDownloadOptions) (bool, bool, error) { diff --git a/pkg/cli/logs_orchestrator_filters.go b/pkg/cli/logs_orchestrator_filters.go index 5747d03ac47..dca1cd3d175 100644 --- a/pkg/cli/logs_orchestrator_filters.go +++ b/pkg/cli/logs_orchestrator_filters.go @@ -25,6 +25,7 @@ type runFilterOpts struct { safeOutputType string filteredIntegrity bool evalsOnly bool + gradersOnly bool } var fetchJobStatusesForProcessedRun = fetchJobStatuses @@ -54,7 +55,6 @@ func matchRuntimeFilter(awInfo *AwInfo, awInfoErr error, filterRuntime string) ( // It parses aw_info.json once (lazily) when any filter that needs it is active. // Returns true when the run should be skipped / excluded from results. func applyRunFilters(ctx context.Context, result DownloadResult, opts runFilterOpts, verbose bool) bool { - // Parse aw_info.json once for all filters that need it (optimization). var awInfo *AwInfo var awInfoErr error if opts.engine != "" || opts.runtime != "" || opts.noStaged || opts.firewallOnly || opts.noFirewall { @@ -62,116 +62,140 @@ func applyRunFilters(ctx context.Context, result DownloadResult, opts runFilterO awInfo, awInfoErr = parseAwInfo(awInfoPath, verbose) } - // Apply engine filtering if specified. - if opts.engine != "" { - engineMatches, detectedEngineID := matchEngineFilter(awInfo, awInfoErr, opts.engine) - if !engineMatches { - if detectedEngineID == "" { - detectedEngineID = "unknown" - } - logsOrchestratorLog.Printf("Skipping run %d: engine filter=%s, detected=%s", result.Run.DatabaseID, opts.engine, detectedEngineID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: engine '%s' does not match filter '%s'", result.Run.DatabaseID, detectedEngineID, opts.engine))) - } - return true - } + return skipByEngineFilter(result, opts, awInfo, awInfoErr, verbose) || + skipByRuntimeFilter(result, opts, awInfo, awInfoErr, verbose) || + skipByStagedFilter(result, opts, awInfo, awInfoErr, verbose) || + skipByFirewallFilter(result, opts, awInfo, awInfoErr, verbose) || + skipBySafeOutputFilter(result, opts, verbose) || + skipByFilteredIntegrityFilter(result, opts, verbose) || + skipByEvalsFilter(ctx, result, opts, verbose) || + skipByGradersFilter(result, opts, verbose) +} + +func skipByEngineFilter(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + if opts.engine == "" { + return false + } + engineMatches, detectedEngineID := matchEngineFilter(awInfo, awInfoErr, opts.engine) + if engineMatches { + return false + } + if detectedEngineID == "" { + detectedEngineID = "unknown" + } + logsOrchestratorLog.Printf("Skipping run %d: engine filter=%s, detected=%s", result.Run.DatabaseID, opts.engine, detectedEngineID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: engine '%s' does not match filter '%s'", result.Run.DatabaseID, detectedEngineID, opts.engine))) } + return true +} - // Apply runtime filtering if specified. - if opts.runtime != "" { - runtimeMatches, detectedRuntime := matchRuntimeFilter(awInfo, awInfoErr, opts.runtime) - if !runtimeMatches { - if detectedRuntime == "" { - detectedRuntime = "unknown" - } - logsOrchestratorLog.Printf("Skipping run %d: runtime filter=%s, detected=%s", result.Run.DatabaseID, opts.runtime, detectedRuntime) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: runtime '%s' does not match filter '%s'", result.Run.DatabaseID, detectedRuntime, opts.runtime))) - } - return true - } +func skipByRuntimeFilter(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + if opts.runtime == "" { + return false + } + runtimeMatches, detectedRuntime := matchRuntimeFilter(awInfo, awInfoErr, opts.runtime) + if runtimeMatches { + return false + } + if detectedRuntime == "" { + detectedRuntime = "unknown" + } + logsOrchestratorLog.Printf("Skipping run %d: runtime filter=%s, detected=%s", result.Run.DatabaseID, opts.runtime, detectedRuntime) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: runtime '%s' does not match filter '%s'", result.Run.DatabaseID, detectedRuntime, opts.runtime))) } + return true +} - // Apply staged filtering if --exclude-staged flag is specified. - if opts.noStaged { - var isStaged bool - if awInfoErr == nil && awInfo != nil { - isStaged = awInfo.Staged - } - if isStaged { - logsOrchestratorLog.Printf("Skipping run %d: staged workflow filtered by --exclude-staged", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow is staged (filtered out by --exclude-staged)", result.Run.DatabaseID))) - } - return true - } +func skipByStagedFilter(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + if !opts.noStaged { + return false } + isStaged := awInfoErr == nil && awInfo != nil && awInfo.Staged + if !isStaged { + return false + } + logsOrchestratorLog.Printf("Skipping run %d: staged workflow filtered by --exclude-staged", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow is staged (filtered out by --exclude-staged)", result.Run.DatabaseID))) + } + return true +} - // Apply firewall filtering if --firewall or --no-firewall flag is specified. - if opts.firewallOnly || opts.noFirewall { - var hasFirewall bool - if awInfoErr == nil && awInfo != nil { - // Firewall is enabled if steps.firewall is non-empty (e.g. "squid"). - hasFirewall = awInfo.Steps.Firewall != "" - } - if opts.firewallOnly && !hasFirewall { - logsOrchestratorLog.Printf("Skipping run %d: no firewall detected, filtered by --firewall", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not use firewall (filtered by --firewall)", result.Run.DatabaseID))) - } - return true - } - if opts.noFirewall && hasFirewall { - logsOrchestratorLog.Printf("Skipping run %d: firewall detected, filtered by --no-firewall", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow uses firewall (filtered by --no-firewall)", result.Run.DatabaseID))) - } - return true - } +func skipByFirewallFilter(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + if !opts.firewallOnly && !opts.noFirewall { + return false + } + hasFirewall := awInfoErr == nil && awInfo != nil && awInfo.Steps.Firewall != "" + if opts.firewallOnly && !hasFirewall { + logAndMaybeExplainSkip(result.Run.DatabaseID, "no firewall detected, filtered by --firewall", "workflow does not use firewall (filtered by --firewall)", verbose) + return true + } + if opts.noFirewall && hasFirewall { + logAndMaybeExplainSkip(result.Run.DatabaseID, "firewall detected, filtered by --no-firewall", "workflow uses firewall (filtered by --no-firewall)", verbose) + return true } + return false +} - // Apply safe output type filtering if --safe-output flag is specified. - if opts.safeOutputType != "" { - hasSafeOutputType, checkErr := runContainsSafeOutputType(result.LogsPath, opts.safeOutputType, verbose) - if checkErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check safe output type for run %d: %v", result.Run.DatabaseID, checkErr))) - } - if !hasSafeOutputType { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no '%s' safe output messages found", result.Run.DatabaseID, opts.safeOutputType))) - } - return true - } +func skipBySafeOutputFilter(result DownloadResult, opts runFilterOpts, verbose bool) bool { + if opts.safeOutputType == "" { + return false + } + hasSafeOutputType, checkErr := runContainsSafeOutputType(result.LogsPath, opts.safeOutputType, verbose) + if checkErr != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check safe output type for run %d: %v", result.Run.DatabaseID, checkErr))) + } + if hasSafeOutputType { + return false } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no '%s' safe output messages found", result.Run.DatabaseID, opts.safeOutputType))) + } + return true +} - // Apply filtered-integrity filtering if --filtered-integrity flag is specified. - if opts.filteredIntegrity { - hasFiltered, checkErr := runHasDifcFilteredItems(result.LogsPath, verbose) - if checkErr != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check DIFC filtered items for run %d: %v", result.Run.DatabaseID, checkErr))) - return true - } - if !hasFiltered { - logsOrchestratorLog.Printf("Skipping run %d: no DIFC filtered items found", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no DIFC integrity-filtered items found in gateway logs", result.Run.DatabaseID))) - } - return true - } +func skipByFilteredIntegrityFilter(result DownloadResult, opts runFilterOpts, verbose bool) bool { + if !opts.filteredIntegrity { + return false + } + hasFiltered, checkErr := runHasDifcFilteredItems(result.LogsPath, verbose) + if checkErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check DIFC filtered items for run %d: %v", result.Run.DatabaseID, checkErr))) + return true } + if hasFiltered { + return false + } + logAndMaybeExplainSkip(result.Run.DatabaseID, "no DIFC filtered items found", "no DIFC integrity-filtered items found in gateway logs", verbose) + return true +} - // Apply evals filtering if --evals flag is specified. - if opts.evalsOnly { - if !runHasEvals(result.LogsPath, verbose) && !ensureEvalsResultsFromBranch(ctx, result.Run, result.LogsPath, "", "", "", verbose) { - logsOrchestratorLog.Printf("Skipping run %d: no evals results found, filtered by --evals", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", result.Run.DatabaseID))) - } - return true - } +func skipByEvalsFilter(ctx context.Context, result DownloadResult, opts runFilterOpts, verbose bool) bool { + if !opts.evalsOnly { + return false + } + if runHasEvals(result.LogsPath, verbose) || ensureEvalsResultsFromBranch(ctx, result.Run, result.LogsPath, "", "", "", verbose) { + return false + } + logAndMaybeExplainSkip(result.Run.DatabaseID, "no evals results found, filtered by --evals", "workflow does not have evals results (filtered by --evals)", verbose) + return true +} + +func skipByGradersFilter(result DownloadResult, opts runFilterOpts, verbose bool) bool { + if !opts.gradersOnly || runHasGraders(result.LogsPath) { + return false } + logAndMaybeExplainSkip(result.Run.DatabaseID, "no grader results found, filtered by --graders", "workflow does not have grader results (filtered by --graders)", verbose) + return true +} - return false +func logAndMaybeExplainSkip(runID int64, logReason, message string, verbose bool) { + logsOrchestratorLog.Printf("Skipping run %d: %s", runID, logReason) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: %s", runID, message))) + } } // buildProcessedRun constructs a ProcessedRun from a DownloadResult, computing diff --git a/pkg/cli/logs_orchestrator_filters_test.go b/pkg/cli/logs_orchestrator_filters_test.go index 2cfdc5f88f0..71aa8f7d802 100644 --- a/pkg/cli/logs_orchestrator_filters_test.go +++ b/pkg/cli/logs_orchestrator_filters_test.go @@ -250,6 +250,28 @@ func TestApplyRunFilters_FilteredIntegrity(t *testing.T) { }) } +func TestApplyRunFilters_Graders(t *testing.T) { + t.Run("missing grader results are skipped", func(t *testing.T) { + result := makeDownloadResult(t, "") + + skip := applyRunFilters(context.Background(), result, runFilterOpts{gradersOnly: true}, false) + + assert.True(t, skip) + }) + + t.Run("grader results pass", func(t *testing.T) { + tmpDir := t.TempDir() + gradersDir := filepath.Join(tmpDir, "usage", "graders") + require.NoError(t, os.MkdirAll(gradersDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(gradersDir, "grader_results.json"), []byte(`{"version":1,"results":[{"id":"quality","status":"pass","value":1,"passed":true}]}`), 0o600)) + result := DownloadResult{RunAnalysis: RunAnalysis{Run: WorkflowRun{DatabaseID: 9}}, LogsPath: tmpDir} + + skip := applyRunFilters(context.Background(), result, runFilterOpts{gradersOnly: true}, false) + + assert.False(t, skip) + }) +} + // TestBuildProcessedRun verifies that buildProcessedRun correctly populates // the ProcessedRun fields from a DownloadResult. func TestBuildProcessedRun(t *testing.T) { diff --git a/pkg/cli/logs_orchestrator_types.go b/pkg/cli/logs_orchestrator_types.go index 3fbdf2bd3c9..c3f20e485e4 100644 --- a/pkg/cli/logs_orchestrator_types.go +++ b/pkg/cli/logs_orchestrator_types.go @@ -30,6 +30,7 @@ type LogsDownloadOptions struct { SafeOutputType string FilteredIntegrity bool EvalsOnly bool + GradersOnly bool Train bool Format string ArtifactSets []string @@ -61,6 +62,7 @@ type StdinLogsOptions struct { SafeOutputType string FilteredIntegrity bool EvalsOnly bool + GradersOnly bool Train bool Format string ReportFile string diff --git a/pkg/cli/mcp_tools_privileged.go b/pkg/cli/mcp_tools_privileged.go index 33cbf87c835..1262a1d6dca 100644 --- a/pkg/cli/mcp_tools_privileged.go +++ b/pkg/cli/mcp_tools_privileged.go @@ -106,12 +106,13 @@ type logsArgs struct { Firewall bool `json:"firewall,omitempty" jsonschema:"Filter to only runs with firewall enabled"` NoFirewall bool `json:"no_firewall,omitempty" jsonschema:"Filter to only runs without firewall enabled"` FilteredIntegrity bool `json:"filtered_integrity,omitempty" jsonschema:"Filter to only runs that contain DIFC integrity-filtered events in gateway logs"` + Graders bool `json:"graders,omitempty" jsonschema:"Filter to only runs with deterministic grader results"` Branch string `json:"branch,omitempty" jsonschema:"Filter runs by branch name"` AfterRunID int64 `json:"after_run_id,omitempty" jsonschema:"Filter runs with database ID after this value (exclusive)"` BeforeRunID int64 `json:"before_run_id,omitempty" jsonschema:"Filter runs with database ID before this value (exclusive)"` Timeout int `json:"timeout,omitempty" jsonschema:"Maximum time in minutes to spend downloading logs (default: auto-scales with count in the MCP server, rounded up in 40-run increments; e.g. 1 minute up to 40, 2 minutes for 41-80, 3 minutes for 81-120, and so on)"` MaxTokens int `json:"max_tokens,omitempty" jsonschema:"Deprecated: accepted for backward compatibility but ignored. Output is always written to a file."` - Artifacts []string `json:"artifacts,omitempty" jsonschema:"Artifact sets to download (default: usage). Valid sets: all, activation, agent, detection, experiment, firewall, github-api, mcp, usage"` + Artifacts []string `json:"artifacts,omitempty" jsonschema:"Artifact sets to download (default: usage). Valid sets: all, activation, agent, detection, evals, experiment, firewall, github-api, graders, mcp, usage"` } func defaultMCPLogsToolTimeoutMinutesForCount(count int) int { @@ -239,8 +240,8 @@ func newLogsToolHandler(execCmd execCmdFunc, actor string, validateActor bool) f cmdArgs, effectiveCount, timeoutValue := buildLogsCommandArgs(ctx, args) // Log the command being executed for debugging - mcpLog.Printf("Executing logs tool: workflow=%s, requested_count=%d, effective_count=%d, firewall=%v, no_firewall=%v, filtered_integrity=%v, timeout=%d, command_args=%v", - args.WorkflowName, args.Count, effectiveCount, args.Firewall, args.NoFirewall, args.FilteredIntegrity, timeoutValue, cmdArgs) + mcpLog.Printf("Executing logs tool: workflow=%s, requested_count=%d, effective_count=%d, firewall=%v, no_firewall=%v, filtered_integrity=%v, graders=%v, timeout=%d, command_args=%v", + args.WorkflowName, args.Count, effectiveCount, args.Firewall, args.NoFirewall, args.FilteredIntegrity, args.Graders, timeoutValue, cmdArgs) notifyProgress(ctx, req, 0, 100, "Downloading workflow logs...") @@ -347,6 +348,9 @@ func appendLogsFilterArgs(cmdArgs []string, args logsArgs) []string { if args.FilteredIntegrity { cmdArgs = append(cmdArgs, "--filtered-integrity") } + if args.Graders { + cmdArgs = append(cmdArgs, "--graders") + } if args.Branch != "" { // The MCP parameter is named "branch" for backwards compatibility, // but the logs CLI flag is --ref (which accepts branches and tags). diff --git a/pkg/cli/mcp_tools_privileged_test.go b/pkg/cli/mcp_tools_privileged_test.go index 5cf16f00c77..823ef9badd6 100644 --- a/pkg/cli/mcp_tools_privileged_test.go +++ b/pkg/cli/mcp_tools_privileged_test.go @@ -262,6 +262,29 @@ func TestLogsToolPassesArtifactsArgument(t *testing.T) { t.Fatal("expected --artifacts flag in command args") } +func TestLogsToolPassesGradersArgument(t *testing.T) { + var capturedArgs []string + mockExecCmd := func(ctx context.Context, args ...string) *exec.Cmd { + capturedArgs = append([]string(nil), args...) + return exec.CommandContext(ctx, "sh", "-c", `printf '%s' "$1"`, "sh", `{"file_path":"/tmp/gh-aw/aw-mcp/logs/runs.json"}`) + } + + server := mcp.NewServer(&mcp.Implementation{Name: "test", Version: "1.0"}, nil) + err := registerLogsTool(server, mockExecCmd, "", false) + require.NoError(t, err, "registerLogsTool should succeed") + + session := connectInMemory(t, server) + _, err = session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: "logs", + Arguments: map[string]any{ + "graders": true, + }, + }) + require.NoError(t, err, "logs tool should succeed") + + require.Contains(t, capturedArgs, "--graders", "logs tool should pass --graders through to the CLI") +} + func TestLogsToolUsesEffectiveCountForTimeoutScaling(t *testing.T) { t.Run("omitted count and no workflow name use all-workflow minimum timeout", func(t *testing.T) { var capturedArgs []string