Skip to content

Commit fdb14e2

Browse files
committed
Staleness report and digest for repositories and branches nobody touches
A repository is stale when nothing landed in it for 180 days by default: no push, no commit on its default branch and none on any branch. A branch is stale when its newest commit is that old. Branches are listed only in repositories that are still active, and a stale repository says how many it holds instead. Archived repositories are left out unless asked for, and one with no dates at all is not called stale. The scanner now keeps each branch's newest commit time, which it already read for the quiet branch filter and threw away. Branch staleness fills in as each repository is scanned again, and the report says how many are still waiting. Report files gets staleness.csv and staleness.json, rewritten with the others after every scan. The Notifications page has two tabs: Criticality Notifications, which is what was there, and Staleness, which sets the number of days, whether archived repositories count, and a weekly or monthly email digest with its own addresses, in the time zone set in Settings. The digest counts stale repositories by organization, names the ones idle longest with their owner, and attaches every row. Send it now sends one straight away.
1 parent 26ff75e commit fdb14e2

14 files changed

Lines changed: 595 additions & 2 deletions

‎app/core/ingest.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -938,6 +938,12 @@ def ingest_report(session, report_path: Path, scanned_repos: set = None,
938938
heads = entry.get("branch_heads")
939939
if isinstance(heads, dict) and heads:
940940
repo.branch_heads = json.dumps(heads)
941+
# Each branch's newest commit time. A report without them, from an
942+
# older scanner or a run that never cloned, leaves the last ones.
943+
dates = entry.get("branch_dates")
944+
if isinstance(dates, dict) and dates:
945+
repo.branch_dates = json.dumps({str(k)[:255]: int(v) for k, v in dates.items()
946+
if isinstance(v, (int, float))})
941947
# When the default branch last got a commit. Only from a run that
942948
# read it, so a report without it leaves the last answer standing.
943949
head = entry.get("default_head")

‎app/core/models.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,10 @@ class Repository(Base):
424424
# and a commit that introduced no finding is still a commit nobody
425425
# reviewed.
426426
branch_heads = Column(LONG_JSON, default="{}")
427+
# Branch name to the unix time of its newest commit, for every branch that
428+
# is there, as of the last scan. What the staleness report reads. Empty
429+
# until a scan from a release that records it.
430+
branch_dates = Column(LONG_JSON, default="{}")
427431
# Whether a dynamic pass could run against this repository, and why not
428432
# when it could not. Read only: nothing is built and nothing is started.
429433
dast_verdict = Column(String(16), default="")

‎app/core/reports.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,13 @@ def write_reports(session, reports_dir: Path = None) -> dict:
720720
except Exception: # noqa: BLE001 - a report must still be written
721721
log.warning("could not write the VEX documents", exc_info=True)
722722

723+
# Repositories and branches nobody has touched, from the same moment.
724+
try:
725+
from app.core import staleness
726+
staleness.write(session, reports_dir)
727+
except Exception: # noqa: BLE001 - a report must still be written
728+
log.warning("could not write the staleness report", exc_info=True)
729+
723730
return {"findings": len(findings), "generated_at": generated,
724731
"by_severity": counts}
725732

‎app/core/scheduler.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,13 @@ def start() -> None:
300300
CronTrigger(hour=3, minute=30, timezone=_zone()),
301301
id="external-refresh", replace_existing=True,
302302
misfire_grace_time=3600, coalesce=True, max_instances=1)
303+
# The staleness digest, on the schedule set on the Notifications page.
304+
# Checked every five minutes; it does nothing until the digest is due.
305+
from app.core import staleness
306+
307+
_scheduler.add_job(staleness.tick, IntervalTrigger(minutes=5),
308+
id="staleness-digest", replace_existing=True,
309+
misfire_grace_time=240, coalesce=True, max_instances=1)
303310
# Certificates about to expire, after the night's telemetry is in.
304311
from app.core import external_certs
305312

‎app/core/settings_store.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,23 @@
465465
# Which applications the last message already named, so the next one only
466466
# says what is new.
467467
"notify.dast_due_last": ([], False, "Applications already reported as due."),
468+
# ---- staleness: repositories and branches nobody has touched ----
469+
"notify.stale_days": (180, False,
470+
"A repository or branch with no commit in this many "
471+
"days is stale."),
472+
"notify.stale_include_archived": (False, False,
473+
"Count archived repositories too. Off, since an "
474+
"archived repository is already dealt with."),
475+
"notify.stale_enabled": (False, False, "Mail the staleness digest on a schedule."),
476+
"notify.stale_to": ("", False,
477+
"Where the staleness digest goes. Several, separated "
478+
"by commas. Empty uses the Criticality To list."),
479+
"notify.stale_frequency": ("weekly", False, "weekly or monthly."),
480+
"notify.stale_weekday": (0, False, "For weekly: 0 is Monday."),
481+
"notify.stale_day": (1, False, "For monthly: the day of the month, 1 to 28."),
482+
"notify.stale_time": ("08:00", False, "HH:MM, in the time zone set in Settings."),
483+
"notify.stale_next": ("", False, "When the next digest is due, in UTC."),
484+
"notify.stale_last": ({}, False, "What the last digest said and whether it went."),
468485

469486
# ---- how times are shown and when schedules fire ----
470487
"ui.timezone": ("GMT", False,

‎app/core/staleness.py‎

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
"""Repositories and branches nobody has touched lately.
2+
3+
A repository is stale when nothing landed in it for the number of days set on
4+
the Notifications page, 180 by default: no push the host knows of, no commit
5+
on its default branch, and no commit on any branch. A branch is stale when
6+
its newest commit is that old.
7+
8+
Branches are listed only in repositories that are not stale themselves. A
9+
stale repository's branches are stale by definition, and naming each one
10+
would bury the few branches worth a look under thousands of rows. The
11+
repository row says how many branches it holds instead.
12+
13+
Branch dates come from the scanner, which reads them off the clone. A
14+
repository not scanned since this release has none yet, and the report says
15+
how many those are rather than calling their branches fresh.
16+
17+
Written to the report folder with the other files, and mailed on the schedule
18+
set on the Staleness tab of the Notifications page.
19+
"""
20+
21+
from __future__ import annotations
22+
23+
import csv
24+
import io
25+
import json
26+
import logging
27+
from datetime import datetime, timedelta, timezone
28+
from urllib.parse import quote
29+
30+
from app.core import settings_store as store
31+
32+
log = logging.getLogger("gcr.staleness")
33+
34+
DEFAULT_DAYS = 180
35+
CSV_NAME = "staleness.csv"
36+
JSON_NAME = "staleness.json"
37+
COLUMNS = ["kind", "organization", "repository", "branch", "default_branch",
38+
"last_activity", "days_idle", "last_push", "last_default_commit",
39+
"branches", "archived", "application", "application_owner",
40+
"technical_owner", "url"]
41+
# How many repositories the digest names before pointing at the attachment.
42+
DIGEST_LINES = 40
43+
44+
45+
def days(session) -> int:
46+
try:
47+
value = int(store.get(session, "notify.stale_days") or DEFAULT_DAYS)
48+
except (TypeError, ValueError):
49+
value = DEFAULT_DAYS
50+
return min(max(value, 1), 3650)
51+
52+
53+
def _aware(value):
54+
if value is None:
55+
return None
56+
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
57+
58+
59+
def _date(value) -> str:
60+
return value.strftime("%Y-%m-%d") if value else ""
61+
62+
63+
def _branch_url(base: str, provider: str, branch: str) -> str:
64+
if not base:
65+
return ""
66+
middle = "/-/tree/" if provider == "gitlab" else "/tree/"
67+
return base.rstrip("/") + middle + quote(branch, safe="")
68+
69+
70+
def find(session, limit_days: int = None, now: datetime = None,
71+
include_archived: bool = None) -> dict:
72+
"""Everything stale as of now. -> {days, cutoff, repositories, branches, ...}"""
73+
from app.core import tagging
74+
from app.core.models import Connection, Repository
75+
76+
limit_days = limit_days or days(session)
77+
if include_archived is None:
78+
include_archived = bool(store.get(session, "notify.stale_include_archived"))
79+
now = now or datetime.now(timezone.utc)
80+
cutoff = now - timedelta(days=limit_days)
81+
repos = session.query(Repository).order_by(Repository.full_name).all()
82+
applies = tagging.effective_many(session, repos)
83+
providers = {c.id: (c.provider or "github") for c in session.query(Connection)}
84+
85+
stale_repos, stale_branches = [], []
86+
active = unknown = undated = skipped_archived = 0
87+
for repo in repos:
88+
if repo.archived and not include_archived:
89+
skipped_archived += 1
90+
continue
91+
try:
92+
dates = json.loads(repo.branch_dates or "{}") or {}
93+
except ValueError:
94+
dates = {}
95+
branch_times = {name: datetime.fromtimestamp(int(ts), timezone.utc)
96+
for name, ts in dates.items() if isinstance(ts, (int, float))}
97+
pushed = _aware(repo.pushed_at)
98+
default_commit = _aware(repo.default_committed_at)
99+
known = [t for t in [pushed, default_commit, *branch_times.values()] if t]
100+
if not known:
101+
# Nothing says when it last moved. Not stale on no evidence.
102+
unknown += 1
103+
continue
104+
last = max(known)
105+
said = applies.get(repo.id) or {}
106+
base = {
107+
"organization": repo.organization,
108+
"repository": repo.full_name,
109+
"default_branch": repo.default_branch or "",
110+
"application": (said.get("application_name") or {}).get("value", ""),
111+
"application_owner": (said.get("application_owner") or {}).get("value", ""),
112+
"technical_owner": (said.get("technical_owner") or {}).get("value", ""),
113+
"archived": bool(repo.archived),
114+
}
115+
try:
116+
present = json.loads(repo.branch_names_present or "[]") or []
117+
except ValueError:
118+
present = []
119+
if last < cutoff:
120+
stale_repos.append(dict(
121+
base, kind="repository", branch="", last_activity=_date(last),
122+
days_idle=(now - last).days, last_push=_date(pushed),
123+
last_default_commit=_date(default_commit),
124+
branches=len(present) or len(branch_times), url=repo.html_url or ""))
125+
continue
126+
active += 1
127+
if not branch_times:
128+
undated += 1
129+
continue
130+
provider = providers.get(repo.connection_id, "github")
131+
for name, when in sorted(branch_times.items()):
132+
if name == repo.default_branch or when >= cutoff:
133+
continue
134+
stale_branches.append(dict(
135+
base, kind="branch", branch=name, last_activity=_date(when),
136+
days_idle=(now - when).days, last_push=_date(pushed),
137+
last_default_commit=_date(default_commit), branches="",
138+
url=_branch_url(repo.html_url or "", provider, name)))
139+
stale_repos.sort(key=lambda r: (-r["days_idle"], r["repository"]))
140+
stale_branches.sort(key=lambda r: (r["repository"], -r["days_idle"], r["branch"]))
141+
return {"days": limit_days, "cutoff": _date(cutoff),
142+
"generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
143+
"include_archived": include_archived,
144+
"repositories": stale_repos, "branches": stale_branches,
145+
"counts": {"repositories": len(stale_repos), "branches": len(stale_branches),
146+
"active_repositories": active, "no_dates": unknown,
147+
"no_branch_dates": undated, "archived_left_out": skipped_archived}}
148+
149+
150+
def csv_text(found: dict) -> str:
151+
from app.core.reports import _quote_formula
152+
153+
out = io.StringIO()
154+
writer = csv.writer(out)
155+
writer.writerow(COLUMNS)
156+
for row in found["repositories"] + found["branches"]:
157+
writer.writerow([_quote_formula(str("yes" if c == "archived" and row[c] else
158+
"" if c == "archived" else row.get(c, "")))
159+
for c in COLUMNS])
160+
return out.getvalue()
161+
162+
163+
def write(session, reports_dir=None) -> dict:
164+
"""staleness.csv and staleness.json in the report folder. -> the counts"""
165+
from pathlib import Path
166+
167+
from app.core.reports import _write_atomic, output_dir
168+
169+
reports_dir = Path(reports_dir) if reports_dir else output_dir(session)
170+
found = find(session)
171+
_write_atomic(reports_dir / CSV_NAME, csv_text(found))
172+
_write_atomic(reports_dir / JSON_NAME, json.dumps(found, indent=2) + "\n")
173+
return found["counts"]
174+
175+
176+
# ------------------------------------------------------------------ digest
177+
178+
def recipients(session) -> list:
179+
from app.core import notify
180+
181+
own = notify._split(store.get(session, "notify.stale_to"))
182+
return own or notify.recipients(session)
183+
184+
185+
class _When:
186+
"""A schedule shaped the way external_schedule.next_run reads one."""
187+
188+
def __init__(self, session):
189+
freq = str(store.get(session, "notify.stale_frequency") or "weekly")
190+
self.frequency = freq if freq in ("weekly", "monthly") else "weekly"
191+
self.time_of_day = str(store.get(session, "notify.stale_time") or "08:00")
192+
self.weekday = int(store.get(session, "notify.stale_weekday") or 0)
193+
self.day_of_month = int(store.get(session, "notify.stale_day") or 1)
194+
self.run_on = ""
195+
196+
197+
def next_due(session, after: datetime = None):
198+
from app.core import external_schedule
199+
200+
return external_schedule.next_run(_When(session), after or datetime.now(timezone.utc))
201+
202+
203+
def describe_schedule(session) -> str:
204+
from app.core import external_schedule
205+
206+
return external_schedule.describe(_When(session))
207+
208+
209+
def message(found: dict) -> tuple:
210+
"""-> (title, lines)"""
211+
c = found["counts"]
212+
title = (f"Staleness: {c['repositories']} repositor{'ies' if c['repositories'] != 1 else 'y'}"
213+
f" and {c['branches']} branch{'es' if c['branches'] != 1 else ''} with no commit"
214+
f" in {found['days']} days")
215+
lines = []
216+
by_org = {}
217+
for row in found["repositories"]:
218+
by_org[row["organization"]] = by_org.get(row["organization"], 0) + 1
219+
if by_org:
220+
lines.append("Stale repositories by organization: " + ", ".join(
221+
f"{org} {n}" for org, n in sorted(by_org.items(), key=lambda x: (-x[1], x[0]))))
222+
for row in found["repositories"][:DIGEST_LINES]:
223+
owner = row["technical_owner"] or row["application_owner"]
224+
lines.append(f"{row['repository']}: last commit {row['last_activity']}, "
225+
f"{row['days_idle']} days ago" + (f", owner {owner}" if owner else ""))
226+
if len(found["repositories"]) > DIGEST_LINES:
227+
lines.append(f"and {len(found['repositories']) - DIGEST_LINES} more repositories")
228+
branch_repos = len({b["repository"] for b in found["branches"]})
229+
if found["branches"]:
230+
lines.append(f"{c['branches']} stale branches in {branch_repos} active "
231+
f"repositor{'ies' if branch_repos != 1 else 'y'}.")
232+
if c["no_branch_dates"]:
233+
lines.append(f"{c['no_branch_dates']} active repositories have not been scanned since "
234+
f"branch dates were recorded, so their branches are not in this yet.")
235+
lines.append("Every row is in the attached staleness.csv, and in the report files.")
236+
return title, lines
237+
238+
239+
def send(session, found: dict = None) -> dict:
240+
"""Mail the digest now. -> {channel: sent}"""
241+
from app.core import notify
242+
243+
found = found or find(session)
244+
to = recipients(session)
245+
if not to:
246+
return {}
247+
title, lines = message(found)
248+
stamp = found["generated_at"][:10]
249+
sheet = {"name": f"staleness-{stamp}.csv", "text": csv_text(found)}
250+
root = notify.base_url(session)
251+
link = f"{root}/reports" if root else ""
252+
body = notify.text_message(session, title, lines, None, link, "")
253+
html = (notify.html_message(session, title, lines, None, link, "")
254+
if notify.mail_format(session) == "html" else "")
255+
if (store.get(session, "notify.mail_transport") or "smtp").lower() == "graph":
256+
return {"microsoft 365": notify.send_graph(session, title, body, html, to, sheet)}
257+
return {"email": notify.send_email(session, title, body, html, to, sheet)}
258+
259+
260+
def tick(now: datetime = None) -> dict:
261+
"""The scheduler's check, every few minutes. Sends when the digest is due."""
262+
from app.core.audit import record
263+
from app.core.db import session_scope
264+
265+
now = now or datetime.now(timezone.utc)
266+
with session_scope() as db:
267+
if not store.get(db, "notify.stale_enabled"):
268+
return {}
269+
due_text = store.get(db, "notify.stale_next") or ""
270+
try:
271+
due = _aware(datetime.fromisoformat(due_text)) if due_text else None
272+
except ValueError:
273+
due = None
274+
if due is None:
275+
store.set_value(db, "notify.stale_next", next_due(db, now).isoformat(), "scheduler")
276+
return {"armed": True}
277+
if due > now:
278+
return {}
279+
try:
280+
found = find(db, now=now)
281+
sent = send(db, found)
282+
except Exception as exc: # noqa: BLE001 - a background job must not take the scheduler down
283+
log.warning("staleness digest failed: %s", exc)
284+
sent, found = {}, {"counts": {}}
285+
ok = bool(sent) and all(sent.values())
286+
# Moved on either way, so a mail server that is down is tried again at
287+
# the next due time rather than every few minutes all day.
288+
store.set_value(db, "notify.stale_next", next_due(db, now).isoformat(), "scheduler")
289+
store.set_value(db, "notify.stale_last", {
290+
"at": now.isoformat(), "ok": ok, "counts": found.get("counts", {}),
291+
"sent": sent}, "scheduler")
292+
record(db, "notifications.staleness", actor="scheduler",
293+
detail="%s; %s" % (json.dumps(found.get("counts", {})),
294+
", ".join(f"{k}: {'ok' if v else 'failed'}"
295+
for k, v in sorted(sent.items()))
296+
or "no recipients, nothing sent"),
297+
outcome="ok" if ok else "warn")
298+
return {"sent": sent}

0 commit comments

Comments
 (0)