Skip to content

fix: label job histograms with job_name and the workflow's own name #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JustMaris merged 1 commit into from
Oct 5, 2026
Merged

fix: label job histograms with job_name and the workflow's own name #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ scrape.
| `github_repo_ci_last_run_conclusion` | `repo`, `workflow`, `url`, `conclusion` | Always 1 - an "info" metric. `conclusion` is GitHub's own string verbatim (`success`, `failure`, `cancelled`, `skipped`, `neutral`, `timed_out`, `action_required`, `stale`), not collapsed to pass/fail here - what counts as "actually broken" is a dashboard-level call. `url` links to the run on github.com. Absent if the workflow has never run |
| `github_repo_ci_last_run_timestamp_seconds` | `repo`, `workflow` | Unix timestamp of that workflow's latest completed run |
| `github_repo_ci_last_run_duration_seconds` | `repo`, `workflow` | Duration of that workflow's latest completed run, created → last update, so it includes queue time (the `github_job_*` histograms split the two) |
| `github_job_queue_seconds` | `repo`, `workflow`, `job`, `runner`, `runner_label`, `conclusion` | Histogram: time each finished job waited for a runner (`created_at` → `started_at`) |
| `github_job_queue_seconds` | `repo`, `workflow`, `job_name`, `runner`, `runner_label`, `conclusion` | Histogram: time each finished job waited for a runner (`created_at` → `started_at`) |
| `github_job_run_seconds` | same as above | Histogram: time each finished job ran on its runner (`started_at` → `completed_at`) |
| `github_workflow_runs_total` | `repo`, `workflow`, `conclusion` | Counter: completed workflow runs, once per run attempt |
| `github_repo_dependabot_alerts_open` | `repo`, `severity` | Open Dependabot alerts by severity. Absent for a severity with zero open alerts |
Expand All @@ -74,7 +74,9 @@ Prometheus does.
replayed. On startup one `LatestRunForWorkflow` call per active workflow
fills the `github_repo_ci_last_run_*` metrics.
- Skipped jobs and jobs cancelled before a runner picked them up aren't
recorded. GitHub-hosted runners show as `runner="github-hosted"` (their
recorded. The job's label is `job_name`, since Prometheus reserves `job`
for the scrape job. `workflow` is the workflow's own name, never a run's
title (Dependabot titles every run differently). GitHub-hosted runners show as `runner="github-hosted"` (their
names are unique per job). `runner_label` is the job's `runs-on` labels
minus `self-hosted`, e.g. `oracle-x64` / `oracle-arm64`: one per pool.
- Buckets: 5s, 10s, 30s, 1m, 2m, 3m, 5m, 10m, 15m, 20m, 30m, 60m.
Expand All @@ -85,11 +87,11 @@ Example PromQL:

```promql
# average job run time per job over the last day
sum by (repo, workflow, job) (rate(github_job_run_seconds_sum[1d]))
/ sum by (repo, workflow, job) (rate(github_job_run_seconds_count[1d]))
sum by (repo, workflow, job_name) (rate(github_job_run_seconds_sum[1d]))
/ sum by (repo, workflow, job_name) (rate(github_job_run_seconds_count[1d]))

# p95 job run time per job
histogram_quantile(0.95, sum by (le, repo, workflow, job) (rate(github_job_run_seconds_bucket[1d])))
histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (rate(github_job_run_seconds_bucket[1d])))

# p95 queue time per runner pool
histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket[1d])))
Expand Down
35 changes: 18 additions & 17 deletions dashboards/github-actions-runner-exporter.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"prometheus"
],
"schemaVersion": 39,
"version": 5,
"version": 6,
"editable": true,
"time": {
"from": "now-24h",
Expand Down Expand Up @@ -997,35 +997,35 @@
},
"targets": [
{
"expr": "sum by (repo, workflow, job) (increase(github_job_run_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))",
"expr": "(sum by (repo, workflow, job_name) (increase(github_job_run_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job_name) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))) >= 0",
"format": "table",
"instant": true,
"range": false,
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (increase(github_job_run_seconds_bucket{repo=~\"$repo\"}[$__range])))",
"expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (increase(github_job_run_seconds_bucket{repo=~\"$repo\"}[$__range])))) >= 0",
"format": "table",
"instant": true,
"range": false,
"refId": "B"
},
{
"expr": "sum by (repo, workflow, job) (increase(github_job_queue_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job) (increase(github_job_queue_seconds_count{repo=~\"$repo\"}[$__range]))",
"expr": "(sum by (repo, workflow, job_name) (increase(github_job_queue_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job_name) (increase(github_job_queue_seconds_count{repo=~\"$repo\"}[$__range]))) >= 0",
"format": "table",
"instant": true,
"range": false,
"refId": "C"
},
{
"expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (increase(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$__range])))",
"expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (increase(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$__range])))) >= 0",
"format": "table",
"instant": true,
"range": false,
"refId": "D"
},
{
"expr": "sum by (repo, workflow, job) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))",
"expr": "(sum by (repo, workflow, job_name) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))) > 0",
"format": "table",
"instant": true,
"range": false,
Expand All @@ -1048,17 +1048,18 @@
"Value #B": "p95 run",
"Value #C": "Avg queue",
"Value #D": "p95 queue",
"Value #E": "Jobs"
"Value #E": "Jobs",
"job_name": "job"
},
"indexByName": {
"repo": 0,
"workflow": 1,
"job": 2,
"Avg run": 3,
"p95 run": 4,
"Avg queue": 5,
"p95 queue": 6,
"Jobs": 7
"Jobs": 7,
"job_name": 2
}
}
},
Expand Down Expand Up @@ -1140,8 +1141,8 @@
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (rate(github_job_run_seconds_bucket{repo=~\"$repo\"}[$window])))",
"legendFormat": "{{repo}} / {{workflow}} / {{job}}",
"expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (rate(github_job_run_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0",
"legendFormat": "{{repo}} / {{workflow}} / {{job_name}}",
"refId": "A",
"range": true,
"instant": false
Expand Down Expand Up @@ -1189,8 +1190,8 @@
},
"targets": [
{
"expr": "sum by (repo, workflow, job) (rate(github_job_run_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (repo, workflow, job) (rate(github_job_run_seconds_count{repo=~\"$repo\"}[$window]))",
"legendFormat": "{{repo}} / {{workflow}} / {{job}}",
"expr": "(sum by (repo, workflow, job_name) (rate(github_job_run_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (repo, workflow, job_name) (rate(github_job_run_seconds_count{repo=~\"$repo\"}[$window]))) >= 0",
"legendFormat": "{{repo}} / {{workflow}} / {{job_name}}",
"refId": "A",
"range": true,
"instant": false
Expand Down Expand Up @@ -1238,14 +1239,14 @@
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))",
"expr": "(histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0",
"legendFormat": "p95 {{runner_label}}",
"refId": "A",
"range": true,
"instant": false
},
{
"expr": "sum by (runner_label) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner_label) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))",
"expr": "(sum by (runner_label) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner_label) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))) >= 0",
"legendFormat": "avg {{runner_label}}",
"refId": "B",
"range": true,
Expand Down Expand Up @@ -1294,14 +1295,14 @@
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le, runner) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))",
"expr": "(histogram_quantile(0.95, sum by (le, runner) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0",
"legendFormat": "p95 {{runner}}",
"refId": "A",
"range": true,
"instant": false
},
{
"expr": "sum by (runner) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))",
"expr": "(sum by (runner) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))) >= 0",
"legendFormat": "avg {{runner}}",
"refId": "B",
"range": true,
Expand Down
27 changes: 20 additions & 7 deletions internal/orgstats/feed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ type Feed struct {
func NewFeed() *Feed { return newFeed(time.Now) }

func newFeed(now func() time.Time) *Feed {
jobLabels := []string{"repo", "workflow", "job", "runner", "runner_label", "conclusion"}
// job_name, not job: Prometheus owns `job` (the scrape job) and would
// rename ours to exported_job
jobLabels := []string{"repo", "workflow", "job_name", "runner", "runner_label", "conclusion"}
return &Feed{
now: now,
start: now(),
Expand Down Expand Up @@ -107,7 +109,11 @@ func (f *Feed) Poll(ctx context.Context, client Client, repo string, active []gi
st = f.bootstrap(ctx, client, repo, active)
f.repos[repo] = st
}
f.advance(ctx, client, repo, st, now)
names := make(map[int64]string, len(active))
for _, wf := range active {
names[wf.ID] = wf.Name
}
f.advance(ctx, client, repo, st, names, now)

out := make([]WorkflowCI, 0, len(active))
for _, wf := range active {
Expand Down Expand Up @@ -139,7 +145,10 @@ func (f *Feed) bootstrap(ctx context.Context, client Client, repo string, active
return st
}

func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repoState, now time.Time) {
// names maps workflow ID to the workflow's own name. A run's name can't be
// used for the label: Dependabot's runs carry a per-run title there
// ("npm_and_yarn in / - Update #123"), a new series each run.
func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repoState, names map[int64]string, now time.Time) {
runs, err := client.RunsCreatedSince(ctx, repo, st.watermark)
if err != nil {
return
Expand Down Expand Up @@ -171,8 +180,12 @@ func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repo
pin(r.CreatedAt)
continue
}
f.observe(repo, r, jobs, now)
f.runs.WithLabelValues(repo, r.Name, r.Conclusion).Inc()
workflow := names[r.WorkflowID]
if workflow == "" {
workflow = r.Name // workflow disabled or deleted since
}
f.observe(repo, workflow, jobs, now)
f.runs.WithLabelValues(repo, workflow, r.Conclusion).Inc()
}
f.seenRuns[key] = now
if prev, ok := st.last[r.WorkflowID]; !ok || r.UpdatedAt.After(prev.LastRunAt) {
Expand All @@ -189,7 +202,7 @@ func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repo
}
}

func (f *Feed) observe(repo string, r github.WorkflowRun, jobs []github.Job, now time.Time) {
func (f *Feed) observe(repo, workflow string, jobs []github.Job, now time.Time) {
for _, j := range jobs {
// skipped never queued or ran; no runner means cancelled before pickup
if j.Conclusion == "skipped" || j.RunnerName == "" {
Expand All @@ -201,7 +214,7 @@ func (f *Feed) observe(repo string, r github.WorkflowRun, jobs []github.Job, now
}
f.seenJobs[j.ID] = now

labels := []string{repo, r.Name, j.Name, runnerName(j), runnerLabel(j.Labels), j.Conclusion}
labels := []string{repo, workflow, j.Name, runnerName(j), runnerLabel(j.Labels), j.Conclusion}
if !j.CreatedAt.IsZero() && !j.StartedAt.Before(j.CreatedAt) {
f.queue.WithLabelValues(labels...).Observe(j.StartedAt.Sub(j.CreatedAt).Seconds())
}
Expand Down
45 changes: 30 additions & 15 deletions internal/orgstats/feed_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,21 +191,21 @@ func TestDedupe(t *testing.T) {
want := `
# HELP github_job_run_seconds Time a job ran on its runner (started to completed), recorded once per finished job.
# TYPE github_job_run_seconds histogram
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="5"} 0
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="10"} 0
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="30"} 0
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="60"} 0
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="120"} 0
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="180"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="300"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="600"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="900"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1200"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1800"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="3600"} 3
github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="+Inf"} 3
github_job_run_seconds_sum{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 540
github_job_run_seconds_count{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="5"} 0
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="10"} 0
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="30"} 0
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="60"} 0
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="120"} 0
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="180"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="300"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="600"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="900"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1200"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1800"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="3600"} 3
github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="+Inf"} 3
github_job_run_seconds_sum{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 540
github_job_run_seconds_count{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 3
`
if err := testutil.CollectAndCompare(f, strings.NewReader(want), "github_job_run_seconds"); err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -251,3 +251,18 @@ func TestBuild(t *testing.T) {
t.Fatal("Repos error not propagated")
}
}

func TestWorkflowLabelFromWorkflowList(t *testing.T) {
r := run(1, 1, "completed", t0.Add(time.Minute))
r.Name = "npm_and_yarn in / - Update #123" // Dependabot's per-run title
c := &fakeClient{runs: []github.WorkflowRun{r}, jobs: map[int64][]github.Job{1: {job(100, r.CreatedAt)}}}
f := newFeed((&clock{t0}).now)

f.Poll(context.Background(), c, "repo", []github.Workflow{{ID: 10, Name: "Dependabot Updates", State: "active"}})
if n := testutil.ToFloat64(f.runs.WithLabelValues("repo", "Dependabot Updates", "success")); n != 1 {
t.Fatalf("runs_total{workflow=\"Dependabot Updates\"} = %v, want 1", n)
}
if n := testutil.CollectAndCount(f, "github_workflow_runs_total"); n != 1 {
t.Fatalf("%d runs_total series, want only the workflow-named one", n)
}
}
Loading