|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Open the PR for one collect.yml run from your own account: |
| 3 | +# |
| 4 | +# bash .github/scripts/open-collect-pr.sh <run-id> |
| 5 | +# |
| 6 | +# collect.yml can't push or open PRs: its token is read-only. A run that |
| 7 | +# changes data uploads its commit as collect.patch, with pr-body.md and |
| 8 | +# pr.json, in a collect-* artifact. Run this by hand from a TechAPI clone to |
| 9 | +# push the run's changes to the task's collect/* branch and open the PR, or |
| 10 | +# update the task's open one. It never touches your checkout or your branches. |
| 11 | +# The PR comes from your account, so pr-checks runs on it like on any PR. |
| 12 | +# |
| 13 | +# The artifact is the run's output, so nothing in it is trusted. GitHub's own |
| 14 | +# record of the run, its title, says which task ran, and so which branch and PR |
| 15 | +# it may update. Only the patch's changes are used: they are applied to the |
| 16 | +# run's develop base in a scratch index, and may only add or change plain files |
| 17 | +# under data/. The commit is made here, with the task's fixed title as its |
| 18 | +# whole message, and the body loses every @ and issue reference before your |
| 19 | +# account posts it. |
| 20 | +set -euo pipefail |
| 21 | + |
| 22 | +run=${1:?usage: bash .github/scripts/open-collect-pr.sh <run-id>} |
| 23 | +repo=GetTechAPI/TechAPI |
| 24 | + |
| 25 | +cd "$(git rev-parse --show-toplevel)" |
| 26 | +# Both where origin fetches from and where it pushes to. |
| 27 | +urls=$(git remote get-url --all origin; git remote get-url --push --all origin) |
| 28 | +while IFS= read -r url; do |
| 29 | + case "$url" in |
| 30 | + *[/:]GetTechAPI/TechAPI | *[/:]GetTechAPI/TechAPI.git) ;; |
| 31 | + *) echo "Run this from a clone of $repo (as origin, to fetch and push)." >&2; exit 1 ;; |
| 32 | + esac |
| 33 | +done <<<"$urls" |
| 34 | +# Run each name: on Windows, python3 is often a Store stub that only says so. |
| 35 | +py= |
| 36 | +for c in python3 python; do |
| 37 | + if "$c" -I -c 'import sys; sys.exit(sys.version_info < (3,))' >/dev/null 2>&1; then py=$c; break; fi |
| 38 | +done |
| 39 | +[ -n "$py" ] || { echo "Python 3 is needed on PATH, as python3 or python." >&2; exit 1; } |
| 40 | + |
| 41 | +# collect.yml titles each run "collect ingest <category>", "collect refresh" or |
| 42 | +# "collect promote" from its inputs, and the run can't change that. |
| 43 | +info=$(gh run view "$run" --repo "$repo" --json workflowName,event,headBranch,conclusion,displayTitle \ |
| 44 | + --jq '"\(.workflowName)|\(.event)|\(.headBranch)|\(.conclusion)|\(.displayTitle)"') |
| 45 | +case "$info" in |
| 46 | + "collect|workflow_dispatch|develop|success|collect "*) task=${info##*|collect } ;; |
| 47 | + *) echo "Run $run is \"$info\", not a successful collect run on develop." >&2; exit 1 ;; |
| 48 | +esac |
| 49 | +want=collect/${task// /-} |
| 50 | + |
| 51 | +dir=$(mktemp -d "${TMPDIR:-/tmp}/open-collect-pr.XXXXXX") |
| 52 | +trap 'rm -rf "$dir"' EXIT |
| 53 | +gh run download "$run" --repo "$repo" --pattern 'collect-*' --dir "$dir/artifact" || { |
| 54 | + echo "Run $run has no collect-* artifact: it changed no data, or the artifact expired." >&2 |
| 55 | + exit 1 |
| 56 | +} |
| 57 | +patch=$(find "$dir/artifact" -type f -name collect.patch) |
| 58 | +if [ -z "$patch" ] || [ "$(printf '%s\n' "$patch" | wc -l)" -ne 1 ]; then |
| 59 | + echo "Run $run should have exactly one collect.patch." >&2 |
| 60 | + exit 1 |
| 61 | +fi |
| 62 | +art=$(dirname "$patch") |
| 63 | + |
| 64 | +# Check pr.json against the run's task, and write the body to post: the task's |
| 65 | +# tracker link, then the run's report cleaned like final-check.yml's comment, |
| 66 | +# so it can't mention anyone or link an issue. |
| 67 | +meta=$("$py" -I - "$art/pr.json" "$art/pr-body.md" "$dir/body.md" "$want" <<'PY' |
| 68 | +import json, re, sys |
| 69 | +
|
| 70 | +meta_path, body_path, out_path, want = sys.argv[1:5] |
| 71 | +m = json.load(open(meta_path, encoding="utf-8")) |
| 72 | +if not isinstance(m, dict): |
| 73 | + sys.exit("pr.json is not what collect.yml writes.") |
| 74 | +branch, title, draft, base = (m.get(k) for k in ("branch", "title", "draft", "base")) |
| 75 | +# The branch and title each TechEngine collector sets. The title is also the |
| 76 | +# commit message and the PR title, so it must be one of these exactly. |
| 77 | +titles = {"collect/refresh": "chore(data): refresh benchmark data", |
| 78 | + "collect/promote": "data(verify): promote records to verified"} |
| 79 | +for category in ("cpu", "gpu", "smartphone"): |
| 80 | + titles["collect/ingest-" + category] = "feat(data/%s): ingest new SKUs" % category |
| 81 | +if want not in titles: |
| 82 | + sys.exit("%s is not a task this script knows." % want) |
| 83 | +if not (branch == want and title == titles[want] |
| 84 | + and isinstance(draft, bool) |
| 85 | + and isinstance(base, str) and re.fullmatch(r"[0-9a-f]{40}", base)): |
| 86 | + sys.exit("pr.json is not what %s's collector writes." % want) |
| 87 | +
|
| 88 | +link = "Closes #295" if want.startswith("collect/ingest-") else "Closes #296" |
| 89 | +zw = chr(0x200B) # zero-width space |
| 90 | +lines = open(body_path, encoding="utf-8", errors="replace").read().split("\n") |
| 91 | +if lines and lines[0].strip() == link: |
| 92 | + lines = lines[1:] |
| 93 | +rest = "\n".join(lines).replace("@", "") |
| 94 | +rest = re.sub(r"(?<!&)#(?=\d+\b)", "#" + zw, rest) |
| 95 | +rest = re.sub(r"\bgh-(?=\d+\b)", lambda g: g.group(0) + zw, rest, flags=re.IGNORECASE) |
| 96 | +rest = re.sub(r"/(issues|pull|discussions)/(?=\d)", lambda g: g.group(0) + zw, rest) |
| 97 | +open(out_path, "w", encoding="utf-8").write((link + "\n" + rest)[:65000]) |
| 98 | +
|
| 99 | +print(branch, title, "true" if draft else "false", base, sep="\t") |
| 100 | +PY |
| 101 | +) |
| 102 | +IFS=$'\t' read -r branch title draft base <<<"$meta" |
| 103 | + |
| 104 | +# The full ref name: a local branch or tag called origin/develop would win over |
| 105 | +# the short one, and a single-branch clone's refspec may not cover develop. |
| 106 | +git fetch --quiet origin +refs/heads/develop:refs/remotes/origin/develop |
| 107 | +if ! git merge-base --is-ancestor "$base" refs/remotes/origin/develop 2>/dev/null; then |
| 108 | + if [ "$(git rev-parse --is-shallow-repository)" = true ]; then |
| 109 | + echo "This clone is shallow, so the run's base $base can't be checked: run git fetch --unshallow origin first." >&2 |
| 110 | + else |
| 111 | + echo "The run's base $base is not on develop." >&2 |
| 112 | + fi |
| 113 | + exit 1 |
| 114 | +fi |
| 115 | + |
| 116 | +# Apply the patch's changes to the base in a scratch index: your checkout and |
| 117 | +# branches stay as they are, and nothing of the patch but its changes is kept. |
| 118 | +index=$dir/index |
| 119 | +GIT_INDEX_FILE=$index git read-tree "$base" |
| 120 | +if ! GIT_INDEX_FILE=$index git apply --cached "$patch"; then |
| 121 | + echo "collect.patch does not apply on $base." >&2 |
| 122 | + exit 1 |
| 123 | +fi |
| 124 | +tree=$(GIT_INDEX_FILE=$index git write-tree) |
| 125 | + |
| 126 | +# The collectors only add and change records. Check the tree that would be |
| 127 | +# pushed, not the patch text: --no-renames so that a rename into data/ can't |
| 128 | +# hide the file it removes outside it, and --ignore-submodules=none so that no |
| 129 | +# submodule setting can hide a change to the TechEngine pin. |
| 130 | +changes=$(git diff-tree -r --no-renames --ignore-submodules=none "$base" "$tree") |
| 131 | +bad=$(printf '%s\n' "$changes" | |
| 132 | + grep -vE '^:(000000|100644) 100644 [0-9a-f]+ [0-9a-f]+ [AM]'$'\t''data/' || true) |
| 133 | +if [ -z "$changes" ] || [ -n "$bad" ]; then |
| 134 | + echo "collect.patch may only add or change plain files under data/, so it is not pushed:" >&2 |
| 135 | + printf '%s\n' "${bad:-(it changes nothing)}" >&2 |
| 136 | + exit 1 |
| 137 | +fi |
| 138 | + |
| 139 | +# The collector is the author and you are the committer, as with any run. |
| 140 | +commit=$(GIT_AUTHOR_NAME='github-actions[bot]' \ |
| 141 | + GIT_AUTHOR_EMAIL='41898282+github-actions[bot]@users.noreply.github.com' \ |
| 142 | + git commit-tree "$tree" -p "$base" -m "$title") |
| 143 | +# The task's branch belongs to the collector, so each run replaces it. |
| 144 | +git push --quiet --force origin "$commit:refs/heads/$branch" |
| 145 | + |
| 146 | +# Not gh pr list --head: that also matches a fork's branch of the same name. |
| 147 | +number=$(gh api "repos/$repo/pulls?state=open&base=develop&head=${repo%/*}:$branch" \ |
| 148 | + --jq '.[0].number // empty') |
| 149 | +if [ -n "$number" ]; then |
| 150 | + # gh pr edit can't set draft. A drafts run writes incomplete records, so its |
| 151 | + # PR goes back to draft first, even if it was marked ready. |
| 152 | + if [ "$draft" = true ]; then gh pr ready "$number" --repo "$repo" --undo; fi |
| 153 | + gh pr edit "$number" --repo "$repo" --title "$title" --body-file "$dir/body.md" \ |
| 154 | + --add-assignee Seungpyo1007 --add-assignee TechEngineBot \ |
| 155 | + --add-label data --add-label enhancement \ |
| 156 | + --milestone "Massive dataset rebuild (1989-2026)" |
| 157 | +else |
| 158 | + set -- --repo "$repo" --base develop --head "$branch" --title "$title" \ |
| 159 | + --body-file "$dir/body.md" \ |
| 160 | + --assignee Seungpyo1007 --assignee TechEngineBot \ |
| 161 | + --label data --label enhancement \ |
| 162 | + --milestone "Massive dataset rebuild (1989-2026)" |
| 163 | + if [ "$draft" = true ]; then set -- "$@" --draft; fi |
| 164 | + gh pr create "$@" |
| 165 | +fi |
0 commit comments