From 15ed754b098817375f3d772bbf32dd9d1b3262bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C4=81ris=20Pop=C4=93ns?= Date: Mon, 5 Oct 2026 21:36:26 +0300 Subject: [PATCH] fix: label job histograms with job_name and the workflow's own name Prometheus owns the job label (the scrape job), so the histograms' job label arrived as exported_job and every dashboard row read github-actions-exporter. Rename it to job_name. Dependabot titles each run ("npm_and_yarn in / - Update #123") and the run's name field carries that title, so every Dependabot run made new series. Take the workflow label from the workflow list by ID instead, falling back to the run's name only for a workflow no longer active. The dashboard groups by job_name and drops NaN rows and empty legend entries for jobs with no runs in the selected window. --- README.md | 12 ++--- .../github-actions-runner-exporter.json | 35 ++++++++------- internal/orgstats/feed.go | 27 ++++++++--- internal/orgstats/feed_test.go | 45 ++++++++++++------- 4 files changed, 75 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index d25e2a4..cc35197 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ scrape. | `github_repo_ci_last_run_conclusion` | `repo`, `workflow`, `url`, `conclusion` | Always 1 - an "info" metric. `conclusion` is GitHub's own string verbatim (`success`, `failure`, `cancelled`, `skipped`, `neutral`, `timed_out`, `action_required`, `stale`), not collapsed to pass/fail here - what counts as "actually broken" is a dashboard-level call. `url` links to the run on github.com. Absent if the workflow has never run | | `github_repo_ci_last_run_timestamp_seconds` | `repo`, `workflow` | Unix timestamp of that workflow's latest completed run | | `github_repo_ci_last_run_duration_seconds` | `repo`, `workflow` | Duration of that workflow's latest completed run, created → last update, so it includes queue time (the `github_job_*` histograms split the two) | -| `github_job_queue_seconds` | `repo`, `workflow`, `job`, `runner`, `runner_label`, `conclusion` | Histogram: time each finished job waited for a runner (`created_at` → `started_at`) | +| `github_job_queue_seconds` | `repo`, `workflow`, `job_name`, `runner`, `runner_label`, `conclusion` | Histogram: time each finished job waited for a runner (`created_at` → `started_at`) | | `github_job_run_seconds` | same as above | Histogram: time each finished job ran on its runner (`started_at` → `completed_at`) | | `github_workflow_runs_total` | `repo`, `workflow`, `conclusion` | Counter: completed workflow runs, once per run attempt | | `github_repo_dependabot_alerts_open` | `repo`, `severity` | Open Dependabot alerts by severity. Absent for a severity with zero open alerts | @@ -74,7 +74,9 @@ Prometheus does. replayed. On startup one `LatestRunForWorkflow` call per active workflow fills the `github_repo_ci_last_run_*` metrics. - Skipped jobs and jobs cancelled before a runner picked them up aren't - recorded. GitHub-hosted runners show as `runner="github-hosted"` (their + recorded. The job's label is `job_name`, since Prometheus reserves `job` + for the scrape job. `workflow` is the workflow's own name, never a run's + title (Dependabot titles every run differently). GitHub-hosted runners show as `runner="github-hosted"` (their names are unique per job). `runner_label` is the job's `runs-on` labels minus `self-hosted`, e.g. `oracle-x64` / `oracle-arm64`: one per pool. - Buckets: 5s, 10s, 30s, 1m, 2m, 3m, 5m, 10m, 15m, 20m, 30m, 60m. @@ -85,11 +87,11 @@ Example PromQL: ```promql # average job run time per job over the last day -sum by (repo, workflow, job) (rate(github_job_run_seconds_sum[1d])) - / sum by (repo, workflow, job) (rate(github_job_run_seconds_count[1d])) +sum by (repo, workflow, job_name) (rate(github_job_run_seconds_sum[1d])) + / sum by (repo, workflow, job_name) (rate(github_job_run_seconds_count[1d])) # p95 job run time per job -histogram_quantile(0.95, sum by (le, repo, workflow, job) (rate(github_job_run_seconds_bucket[1d]))) +histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (rate(github_job_run_seconds_bucket[1d]))) # p95 queue time per runner pool histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket[1d]))) diff --git a/dashboards/github-actions-runner-exporter.json b/dashboards/github-actions-runner-exporter.json index 14a8a5a..7afe471 100644 --- a/dashboards/github-actions-runner-exporter.json +++ b/dashboards/github-actions-runner-exporter.json @@ -8,7 +8,7 @@ "prometheus" ], "schemaVersion": 39, - "version": 5, + "version": 6, "editable": true, "time": { "from": "now-24h", @@ -997,35 +997,35 @@ }, "targets": [ { - "expr": "sum by (repo, workflow, job) (increase(github_job_run_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))", + "expr": "(sum by (repo, workflow, job_name) (increase(github_job_run_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job_name) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))) >= 0", "format": "table", "instant": true, "range": false, "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (increase(github_job_run_seconds_bucket{repo=~\"$repo\"}[$__range])))", + "expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (increase(github_job_run_seconds_bucket{repo=~\"$repo\"}[$__range])))) >= 0", "format": "table", "instant": true, "range": false, "refId": "B" }, { - "expr": "sum by (repo, workflow, job) (increase(github_job_queue_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job) (increase(github_job_queue_seconds_count{repo=~\"$repo\"}[$__range]))", + "expr": "(sum by (repo, workflow, job_name) (increase(github_job_queue_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job_name) (increase(github_job_queue_seconds_count{repo=~\"$repo\"}[$__range]))) >= 0", "format": "table", "instant": true, "range": false, "refId": "C" }, { - "expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (increase(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$__range])))", + "expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (increase(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$__range])))) >= 0", "format": "table", "instant": true, "range": false, "refId": "D" }, { - "expr": "sum by (repo, workflow, job) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))", + "expr": "(sum by (repo, workflow, job_name) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))) > 0", "format": "table", "instant": true, "range": false, @@ -1048,17 +1048,18 @@ "Value #B": "p95 run", "Value #C": "Avg queue", "Value #D": "p95 queue", - "Value #E": "Jobs" + "Value #E": "Jobs", + "job_name": "job" }, "indexByName": { "repo": 0, "workflow": 1, - "job": 2, "Avg run": 3, "p95 run": 4, "Avg queue": 5, "p95 queue": 6, - "Jobs": 7 + "Jobs": 7, + "job_name": 2 } } }, @@ -1140,8 +1141,8 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (rate(github_job_run_seconds_bucket{repo=~\"$repo\"}[$window])))", - "legendFormat": "{{repo}} / {{workflow}} / {{job}}", + "expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (rate(github_job_run_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0", + "legendFormat": "{{repo}} / {{workflow}} / {{job_name}}", "refId": "A", "range": true, "instant": false @@ -1189,8 +1190,8 @@ }, "targets": [ { - "expr": "sum by (repo, workflow, job) (rate(github_job_run_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (repo, workflow, job) (rate(github_job_run_seconds_count{repo=~\"$repo\"}[$window]))", - "legendFormat": "{{repo}} / {{workflow}} / {{job}}", + "expr": "(sum by (repo, workflow, job_name) (rate(github_job_run_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (repo, workflow, job_name) (rate(github_job_run_seconds_count{repo=~\"$repo\"}[$window]))) >= 0", + "legendFormat": "{{repo}} / {{workflow}} / {{job_name}}", "refId": "A", "range": true, "instant": false @@ -1238,14 +1239,14 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))", + "expr": "(histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0", "legendFormat": "p95 {{runner_label}}", "refId": "A", "range": true, "instant": false }, { - "expr": "sum by (runner_label) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner_label) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))", + "expr": "(sum by (runner_label) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner_label) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))) >= 0", "legendFormat": "avg {{runner_label}}", "refId": "B", "range": true, @@ -1294,14 +1295,14 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, runner) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))", + "expr": "(histogram_quantile(0.95, sum by (le, runner) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0", "legendFormat": "p95 {{runner}}", "refId": "A", "range": true, "instant": false }, { - "expr": "sum by (runner) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))", + "expr": "(sum by (runner) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))) >= 0", "legendFormat": "avg {{runner}}", "refId": "B", "range": true, diff --git a/internal/orgstats/feed.go b/internal/orgstats/feed.go index 1592dc6..7856070 100644 --- a/internal/orgstats/feed.go +++ b/internal/orgstats/feed.go @@ -56,7 +56,9 @@ type Feed struct { func NewFeed() *Feed { return newFeed(time.Now) } func newFeed(now func() time.Time) *Feed { - jobLabels := []string{"repo", "workflow", "job", "runner", "runner_label", "conclusion"} + // job_name, not job: Prometheus owns `job` (the scrape job) and would + // rename ours to exported_job + jobLabels := []string{"repo", "workflow", "job_name", "runner", "runner_label", "conclusion"} return &Feed{ now: now, start: now(), @@ -107,7 +109,11 @@ func (f *Feed) Poll(ctx context.Context, client Client, repo string, active []gi st = f.bootstrap(ctx, client, repo, active) f.repos[repo] = st } - f.advance(ctx, client, repo, st, now) + names := make(map[int64]string, len(active)) + for _, wf := range active { + names[wf.ID] = wf.Name + } + f.advance(ctx, client, repo, st, names, now) out := make([]WorkflowCI, 0, len(active)) for _, wf := range active { @@ -139,7 +145,10 @@ func (f *Feed) bootstrap(ctx context.Context, client Client, repo string, active return st } -func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repoState, now time.Time) { +// names maps workflow ID to the workflow's own name. A run's name can't be +// used for the label: Dependabot's runs carry a per-run title there +// ("npm_and_yarn in / - Update #123"), a new series each run. +func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repoState, names map[int64]string, now time.Time) { runs, err := client.RunsCreatedSince(ctx, repo, st.watermark) if err != nil { return @@ -171,8 +180,12 @@ func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repo pin(r.CreatedAt) continue } - f.observe(repo, r, jobs, now) - f.runs.WithLabelValues(repo, r.Name, r.Conclusion).Inc() + workflow := names[r.WorkflowID] + if workflow == "" { + workflow = r.Name // workflow disabled or deleted since + } + f.observe(repo, workflow, jobs, now) + f.runs.WithLabelValues(repo, workflow, r.Conclusion).Inc() } f.seenRuns[key] = now if prev, ok := st.last[r.WorkflowID]; !ok || r.UpdatedAt.After(prev.LastRunAt) { @@ -189,7 +202,7 @@ func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repo } } -func (f *Feed) observe(repo string, r github.WorkflowRun, jobs []github.Job, now time.Time) { +func (f *Feed) observe(repo, workflow string, jobs []github.Job, now time.Time) { for _, j := range jobs { // skipped never queued or ran; no runner means cancelled before pickup if j.Conclusion == "skipped" || j.RunnerName == "" { @@ -201,7 +214,7 @@ func (f *Feed) observe(repo string, r github.WorkflowRun, jobs []github.Job, now } f.seenJobs[j.ID] = now - labels := []string{repo, r.Name, j.Name, runnerName(j), runnerLabel(j.Labels), j.Conclusion} + labels := []string{repo, workflow, j.Name, runnerName(j), runnerLabel(j.Labels), j.Conclusion} if !j.CreatedAt.IsZero() && !j.StartedAt.Before(j.CreatedAt) { f.queue.WithLabelValues(labels...).Observe(j.StartedAt.Sub(j.CreatedAt).Seconds()) } diff --git a/internal/orgstats/feed_test.go b/internal/orgstats/feed_test.go index 0037e6d..918c8a6 100644 --- a/internal/orgstats/feed_test.go +++ b/internal/orgstats/feed_test.go @@ -191,21 +191,21 @@ func TestDedupe(t *testing.T) { want := ` # HELP github_job_run_seconds Time a job ran on its runner (started to completed), recorded once per finished job. # TYPE github_job_run_seconds histogram -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="5"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="10"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="30"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="60"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="120"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="180"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="300"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="600"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="900"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1200"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1800"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="3600"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="+Inf"} 3 -github_job_run_seconds_sum{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 540 -github_job_run_seconds_count{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="5"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="10"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="30"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="60"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="120"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="180"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="300"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="600"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="900"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1200"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1800"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="3600"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="+Inf"} 3 +github_job_run_seconds_sum{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 540 +github_job_run_seconds_count{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 3 ` if err := testutil.CollectAndCompare(f, strings.NewReader(want), "github_job_run_seconds"); err != nil { t.Fatal(err) @@ -251,3 +251,18 @@ func TestBuild(t *testing.T) { t.Fatal("Repos error not propagated") } } + +func TestWorkflowLabelFromWorkflowList(t *testing.T) { + r := run(1, 1, "completed", t0.Add(time.Minute)) + r.Name = "npm_and_yarn in / - Update #123" // Dependabot's per-run title + c := &fakeClient{runs: []github.WorkflowRun{r}, jobs: map[int64][]github.Job{1: {job(100, r.CreatedAt)}}} + f := newFeed((&clock{t0}).now) + + f.Poll(context.Background(), c, "repo", []github.Workflow{{ID: 10, Name: "Dependabot Updates", State: "active"}}) + if n := testutil.ToFloat64(f.runs.WithLabelValues("repo", "Dependabot Updates", "success")); n != 1 { + t.Fatalf("runs_total{workflow=\"Dependabot Updates\"} = %v, want 1", n) + } + if n := testutil.CollectAndCount(f, "github_workflow_runs_total"); n != 1 { + t.Fatalf("%d runs_total series, want only the workflow-named one", n) + } +}