diff --git a/README.md b/README.md index d25e2a4..cc35197 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ scrape. | `github_repo_ci_last_run_conclusion` | `repo`, `workflow`, `url`, `conclusion` | Always 1 - an "info" metric. `conclusion` is GitHub's own string verbatim (`success`, `failure`, `cancelled`, `skipped`, `neutral`, `timed_out`, `action_required`, `stale`), not collapsed to pass/fail here - what counts as "actually broken" is a dashboard-level call. `url` links to the run on github.com. Absent if the workflow has never run | | `github_repo_ci_last_run_timestamp_seconds` | `repo`, `workflow` | Unix timestamp of that workflow's latest completed run | | `github_repo_ci_last_run_duration_seconds` | `repo`, `workflow` | Duration of that workflow's latest completed run, created → last update, so it includes queue time (the `github_job_*` histograms split the two) | -| `github_job_queue_seconds` | `repo`, `workflow`, `job`, `runner`, `runner_label`, `conclusion` | Histogram: time each finished job waited for a runner (`created_at` → `started_at`) | +| `github_job_queue_seconds` | `repo`, `workflow`, `job_name`, `runner`, `runner_label`, `conclusion` | Histogram: time each finished job waited for a runner (`created_at` → `started_at`) | | `github_job_run_seconds` | same as above | Histogram: time each finished job ran on its runner (`started_at` → `completed_at`) | | `github_workflow_runs_total` | `repo`, `workflow`, `conclusion` | Counter: completed workflow runs, once per run attempt | | `github_repo_dependabot_alerts_open` | `repo`, `severity` | Open Dependabot alerts by severity. Absent for a severity with zero open alerts | @@ -74,7 +74,9 @@ Prometheus does. replayed. On startup one `LatestRunForWorkflow` call per active workflow fills the `github_repo_ci_last_run_*` metrics. - Skipped jobs and jobs cancelled before a runner picked them up aren't - recorded. GitHub-hosted runners show as `runner="github-hosted"` (their + recorded. The job's label is `job_name`, since Prometheus reserves `job` + for the scrape job. `workflow` is the workflow's own name, never a run's + title (Dependabot titles every run differently). GitHub-hosted runners show as `runner="github-hosted"` (their names are unique per job). `runner_label` is the job's `runs-on` labels minus `self-hosted`, e.g. `oracle-x64` / `oracle-arm64`: one per pool. - Buckets: 5s, 10s, 30s, 1m, 2m, 3m, 5m, 10m, 15m, 20m, 30m, 60m. @@ -85,11 +87,11 @@ Example PromQL: ```promql # average job run time per job over the last day -sum by (repo, workflow, job) (rate(github_job_run_seconds_sum[1d])) - / sum by (repo, workflow, job) (rate(github_job_run_seconds_count[1d])) +sum by (repo, workflow, job_name) (rate(github_job_run_seconds_sum[1d])) + / sum by (repo, workflow, job_name) (rate(github_job_run_seconds_count[1d])) # p95 job run time per job -histogram_quantile(0.95, sum by (le, repo, workflow, job) (rate(github_job_run_seconds_bucket[1d]))) +histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (rate(github_job_run_seconds_bucket[1d]))) # p95 queue time per runner pool histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket[1d]))) diff --git a/dashboards/github-actions-runner-exporter.json b/dashboards/github-actions-runner-exporter.json index 14a8a5a..7afe471 100644 --- a/dashboards/github-actions-runner-exporter.json +++ b/dashboards/github-actions-runner-exporter.json @@ -8,7 +8,7 @@ "prometheus" ], "schemaVersion": 39, - "version": 5, + "version": 6, "editable": true, "time": { "from": "now-24h", @@ -997,35 +997,35 @@ }, "targets": [ { - "expr": "sum by (repo, workflow, job) (increase(github_job_run_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))", + "expr": "(sum by (repo, workflow, job_name) (increase(github_job_run_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job_name) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))) >= 0", "format": "table", "instant": true, "range": false, "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (increase(github_job_run_seconds_bucket{repo=~\"$repo\"}[$__range])))", + "expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (increase(github_job_run_seconds_bucket{repo=~\"$repo\"}[$__range])))) >= 0", "format": "table", "instant": true, "range": false, "refId": "B" }, { - "expr": "sum by (repo, workflow, job) (increase(github_job_queue_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job) (increase(github_job_queue_seconds_count{repo=~\"$repo\"}[$__range]))", + "expr": "(sum by (repo, workflow, job_name) (increase(github_job_queue_seconds_sum{repo=~\"$repo\"}[$__range])) / sum by (repo, workflow, job_name) (increase(github_job_queue_seconds_count{repo=~\"$repo\"}[$__range]))) >= 0", "format": "table", "instant": true, "range": false, "refId": "C" }, { - "expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (increase(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$__range])))", + "expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (increase(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$__range])))) >= 0", "format": "table", "instant": true, "range": false, "refId": "D" }, { - "expr": "sum by (repo, workflow, job) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))", + "expr": "(sum by (repo, workflow, job_name) (increase(github_job_run_seconds_count{repo=~\"$repo\"}[$__range]))) > 0", "format": "table", "instant": true, "range": false, @@ -1048,17 +1048,18 @@ "Value #B": "p95 run", "Value #C": "Avg queue", "Value #D": "p95 queue", - "Value #E": "Jobs" + "Value #E": "Jobs", + "job_name": "job" }, "indexByName": { "repo": 0, "workflow": 1, - "job": 2, "Avg run": 3, "p95 run": 4, "Avg queue": 5, "p95 queue": 6, - "Jobs": 7 + "Jobs": 7, + "job_name": 2 } } }, @@ -1140,8 +1141,8 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, repo, workflow, job) (rate(github_job_run_seconds_bucket{repo=~\"$repo\"}[$window])))", - "legendFormat": "{{repo}} / {{workflow}} / {{job}}", + "expr": "(histogram_quantile(0.95, sum by (le, repo, workflow, job_name) (rate(github_job_run_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0", + "legendFormat": "{{repo}} / {{workflow}} / {{job_name}}", "refId": "A", "range": true, "instant": false @@ -1189,8 +1190,8 @@ }, "targets": [ { - "expr": "sum by (repo, workflow, job) (rate(github_job_run_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (repo, workflow, job) (rate(github_job_run_seconds_count{repo=~\"$repo\"}[$window]))", - "legendFormat": "{{repo}} / {{workflow}} / {{job}}", + "expr": "(sum by (repo, workflow, job_name) (rate(github_job_run_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (repo, workflow, job_name) (rate(github_job_run_seconds_count{repo=~\"$repo\"}[$window]))) >= 0", + "legendFormat": "{{repo}} / {{workflow}} / {{job_name}}", "refId": "A", "range": true, "instant": false @@ -1238,14 +1239,14 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))", + "expr": "(histogram_quantile(0.95, sum by (le, runner_label) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0", "legendFormat": "p95 {{runner_label}}", "refId": "A", "range": true, "instant": false }, { - "expr": "sum by (runner_label) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner_label) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))", + "expr": "(sum by (runner_label) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner_label) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))) >= 0", "legendFormat": "avg {{runner_label}}", "refId": "B", "range": true, @@ -1294,14 +1295,14 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, runner) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))", + "expr": "(histogram_quantile(0.95, sum by (le, runner) (rate(github_job_queue_seconds_bucket{repo=~\"$repo\"}[$window])))) >= 0", "legendFormat": "p95 {{runner}}", "refId": "A", "range": true, "instant": false }, { - "expr": "sum by (runner) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))", + "expr": "(sum by (runner) (rate(github_job_queue_seconds_sum{repo=~\"$repo\"}[$window])) / sum by (runner) (rate(github_job_queue_seconds_count{repo=~\"$repo\"}[$window]))) >= 0", "legendFormat": "avg {{runner}}", "refId": "B", "range": true, diff --git a/internal/orgstats/feed.go b/internal/orgstats/feed.go index 1592dc6..7856070 100644 --- a/internal/orgstats/feed.go +++ b/internal/orgstats/feed.go @@ -56,7 +56,9 @@ type Feed struct { func NewFeed() *Feed { return newFeed(time.Now) } func newFeed(now func() time.Time) *Feed { - jobLabels := []string{"repo", "workflow", "job", "runner", "runner_label", "conclusion"} + // job_name, not job: Prometheus owns `job` (the scrape job) and would + // rename ours to exported_job + jobLabels := []string{"repo", "workflow", "job_name", "runner", "runner_label", "conclusion"} return &Feed{ now: now, start: now(), @@ -107,7 +109,11 @@ func (f *Feed) Poll(ctx context.Context, client Client, repo string, active []gi st = f.bootstrap(ctx, client, repo, active) f.repos[repo] = st } - f.advance(ctx, client, repo, st, now) + names := make(map[int64]string, len(active)) + for _, wf := range active { + names[wf.ID] = wf.Name + } + f.advance(ctx, client, repo, st, names, now) out := make([]WorkflowCI, 0, len(active)) for _, wf := range active { @@ -139,7 +145,10 @@ func (f *Feed) bootstrap(ctx context.Context, client Client, repo string, active return st } -func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repoState, now time.Time) { +// names maps workflow ID to the workflow's own name. A run's name can't be +// used for the label: Dependabot's runs carry a per-run title there +// ("npm_and_yarn in / - Update #123"), a new series each run. +func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repoState, names map[int64]string, now time.Time) { runs, err := client.RunsCreatedSince(ctx, repo, st.watermark) if err != nil { return @@ -171,8 +180,12 @@ func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repo pin(r.CreatedAt) continue } - f.observe(repo, r, jobs, now) - f.runs.WithLabelValues(repo, r.Name, r.Conclusion).Inc() + workflow := names[r.WorkflowID] + if workflow == "" { + workflow = r.Name // workflow disabled or deleted since + } + f.observe(repo, workflow, jobs, now) + f.runs.WithLabelValues(repo, workflow, r.Conclusion).Inc() } f.seenRuns[key] = now if prev, ok := st.last[r.WorkflowID]; !ok || r.UpdatedAt.After(prev.LastRunAt) { @@ -189,7 +202,7 @@ func (f *Feed) advance(ctx context.Context, client Client, repo string, st *repo } } -func (f *Feed) observe(repo string, r github.WorkflowRun, jobs []github.Job, now time.Time) { +func (f *Feed) observe(repo, workflow string, jobs []github.Job, now time.Time) { for _, j := range jobs { // skipped never queued or ran; no runner means cancelled before pickup if j.Conclusion == "skipped" || j.RunnerName == "" { @@ -201,7 +214,7 @@ func (f *Feed) observe(repo string, r github.WorkflowRun, jobs []github.Job, now } f.seenJobs[j.ID] = now - labels := []string{repo, r.Name, j.Name, runnerName(j), runnerLabel(j.Labels), j.Conclusion} + labels := []string{repo, workflow, j.Name, runnerName(j), runnerLabel(j.Labels), j.Conclusion} if !j.CreatedAt.IsZero() && !j.StartedAt.Before(j.CreatedAt) { f.queue.WithLabelValues(labels...).Observe(j.StartedAt.Sub(j.CreatedAt).Seconds()) } diff --git a/internal/orgstats/feed_test.go b/internal/orgstats/feed_test.go index 0037e6d..918c8a6 100644 --- a/internal/orgstats/feed_test.go +++ b/internal/orgstats/feed_test.go @@ -191,21 +191,21 @@ func TestDedupe(t *testing.T) { want := ` # HELP github_job_run_seconds Time a job ran on its runner (started to completed), recorded once per finished job. # TYPE github_job_run_seconds histogram -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="5"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="10"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="30"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="60"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="120"} 0 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="180"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="300"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="600"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="900"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1200"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1800"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="3600"} 3 -github_job_run_seconds_bucket{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="+Inf"} 3 -github_job_run_seconds_sum{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 540 -github_job_run_seconds_count{conclusion="success",job="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="5"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="10"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="30"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="60"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="120"} 0 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="180"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="300"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="600"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="900"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1200"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="1800"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="3600"} 3 +github_job_run_seconds_bucket{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI",le="+Inf"} 3 +github_job_run_seconds_sum{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 540 +github_job_run_seconds_count{conclusion="success",job_name="build",repo="repo",runner="gha-arm-1",runner_label="oracle-arm64",workflow="CI"} 3 ` if err := testutil.CollectAndCompare(f, strings.NewReader(want), "github_job_run_seconds"); err != nil { t.Fatal(err) @@ -251,3 +251,18 @@ func TestBuild(t *testing.T) { t.Fatal("Repos error not propagated") } } + +func TestWorkflowLabelFromWorkflowList(t *testing.T) { + r := run(1, 1, "completed", t0.Add(time.Minute)) + r.Name = "npm_and_yarn in / - Update #123" // Dependabot's per-run title + c := &fakeClient{runs: []github.WorkflowRun{r}, jobs: map[int64][]github.Job{1: {job(100, r.CreatedAt)}}} + f := newFeed((&clock{t0}).now) + + f.Poll(context.Background(), c, "repo", []github.Workflow{{ID: 10, Name: "Dependabot Updates", State: "active"}}) + if n := testutil.ToFloat64(f.runs.WithLabelValues("repo", "Dependabot Updates", "success")); n != 1 { + t.Fatalf("runs_total{workflow=\"Dependabot Updates\"} = %v, want 1", n) + } + if n := testutil.CollectAndCount(f, "github_workflow_runs_total"); n != 1 { + t.Fatalf("%d runs_total series, want only the workflow-named one", n) + } +}