|
| 1 | +"""Repositories and branches nobody has touched lately. |
| 2 | +
|
| 3 | +A repository is stale when nothing landed in it for the number of days set on |
| 4 | +the Notifications page, 180 by default: no push the host knows of, no commit |
| 5 | +on its default branch, and no commit on any branch. A branch is stale when |
| 6 | +its newest commit is that old. |
| 7 | +
|
| 8 | +Branches are listed only in repositories that are not stale themselves. A |
| 9 | +stale repository's branches are stale by definition, and naming each one |
| 10 | +would bury the few branches worth a look under thousands of rows. The |
| 11 | +repository row says how many branches it holds instead. |
| 12 | +
|
| 13 | +Branch dates come from the scanner, which reads them off the clone. A |
| 14 | +repository not scanned since this release has none yet, and the report says |
| 15 | +how many those are rather than calling their branches fresh. |
| 16 | +
|
| 17 | +Written to the report folder with the other files, and mailed on the schedule |
| 18 | +set on the Staleness tab of the Notifications page. |
| 19 | +""" |
| 20 | + |
| 21 | +from __future__ import annotations |
| 22 | + |
| 23 | +import csv |
| 24 | +import io |
| 25 | +import json |
| 26 | +import logging |
| 27 | +from datetime import datetime, timedelta, timezone |
| 28 | +from urllib.parse import quote |
| 29 | + |
| 30 | +from app.core import settings_store as store |
| 31 | + |
| 32 | +log = logging.getLogger("gcr.staleness") |
| 33 | + |
| 34 | +DEFAULT_DAYS = 180 |
| 35 | +CSV_NAME = "staleness.csv" |
| 36 | +JSON_NAME = "staleness.json" |
| 37 | +COLUMNS = ["kind", "organization", "repository", "branch", "default_branch", |
| 38 | + "last_activity", "days_idle", "last_push", "last_default_commit", |
| 39 | + "branches", "archived", "application", "application_owner", |
| 40 | + "technical_owner", "url"] |
| 41 | +# How many repositories the digest names before pointing at the attachment. |
| 42 | +DIGEST_LINES = 40 |
| 43 | + |
| 44 | + |
| 45 | +def days(session) -> int: |
| 46 | + try: |
| 47 | + value = int(store.get(session, "notify.stale_days") or DEFAULT_DAYS) |
| 48 | + except (TypeError, ValueError): |
| 49 | + value = DEFAULT_DAYS |
| 50 | + return min(max(value, 1), 3650) |
| 51 | + |
| 52 | + |
| 53 | +def _aware(value): |
| 54 | + if value is None: |
| 55 | + return None |
| 56 | + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) |
| 57 | + |
| 58 | + |
| 59 | +def _date(value) -> str: |
| 60 | + return value.strftime("%Y-%m-%d") if value else "" |
| 61 | + |
| 62 | + |
| 63 | +def _branch_url(base: str, provider: str, branch: str) -> str: |
| 64 | + if not base: |
| 65 | + return "" |
| 66 | + middle = "/-/tree/" if provider == "gitlab" else "/tree/" |
| 67 | + return base.rstrip("/") + middle + quote(branch, safe="") |
| 68 | + |
| 69 | + |
| 70 | +def find(session, limit_days: int = None, now: datetime = None, |
| 71 | + include_archived: bool = None) -> dict: |
| 72 | + """Everything stale as of now. -> {days, cutoff, repositories, branches, ...}""" |
| 73 | + from app.core import tagging |
| 74 | + from app.core.models import Connection, Repository |
| 75 | + |
| 76 | + limit_days = limit_days or days(session) |
| 77 | + if include_archived is None: |
| 78 | + include_archived = bool(store.get(session, "notify.stale_include_archived")) |
| 79 | + now = now or datetime.now(timezone.utc) |
| 80 | + cutoff = now - timedelta(days=limit_days) |
| 81 | + repos = session.query(Repository).order_by(Repository.full_name).all() |
| 82 | + applies = tagging.effective_many(session, repos) |
| 83 | + providers = {c.id: (c.provider or "github") for c in session.query(Connection)} |
| 84 | + |
| 85 | + stale_repos, stale_branches = [], [] |
| 86 | + active = unknown = undated = skipped_archived = 0 |
| 87 | + for repo in repos: |
| 88 | + if repo.archived and not include_archived: |
| 89 | + skipped_archived += 1 |
| 90 | + continue |
| 91 | + try: |
| 92 | + dates = json.loads(repo.branch_dates or "{}") or {} |
| 93 | + except ValueError: |
| 94 | + dates = {} |
| 95 | + branch_times = {name: datetime.fromtimestamp(int(ts), timezone.utc) |
| 96 | + for name, ts in dates.items() if isinstance(ts, (int, float))} |
| 97 | + pushed = _aware(repo.pushed_at) |
| 98 | + default_commit = _aware(repo.default_committed_at) |
| 99 | + known = [t for t in [pushed, default_commit, *branch_times.values()] if t] |
| 100 | + if not known: |
| 101 | + # Nothing says when it last moved. Not stale on no evidence. |
| 102 | + unknown += 1 |
| 103 | + continue |
| 104 | + last = max(known) |
| 105 | + said = applies.get(repo.id) or {} |
| 106 | + base = { |
| 107 | + "organization": repo.organization, |
| 108 | + "repository": repo.full_name, |
| 109 | + "default_branch": repo.default_branch or "", |
| 110 | + "application": (said.get("application_name") or {}).get("value", ""), |
| 111 | + "application_owner": (said.get("application_owner") or {}).get("value", ""), |
| 112 | + "technical_owner": (said.get("technical_owner") or {}).get("value", ""), |
| 113 | + "archived": bool(repo.archived), |
| 114 | + } |
| 115 | + try: |
| 116 | + present = json.loads(repo.branch_names_present or "[]") or [] |
| 117 | + except ValueError: |
| 118 | + present = [] |
| 119 | + if last < cutoff: |
| 120 | + stale_repos.append(dict( |
| 121 | + base, kind="repository", branch="", last_activity=_date(last), |
| 122 | + days_idle=(now - last).days, last_push=_date(pushed), |
| 123 | + last_default_commit=_date(default_commit), |
| 124 | + branches=len(present) or len(branch_times), url=repo.html_url or "")) |
| 125 | + continue |
| 126 | + active += 1 |
| 127 | + if not branch_times: |
| 128 | + undated += 1 |
| 129 | + continue |
| 130 | + provider = providers.get(repo.connection_id, "github") |
| 131 | + for name, when in sorted(branch_times.items()): |
| 132 | + if name == repo.default_branch or when >= cutoff: |
| 133 | + continue |
| 134 | + stale_branches.append(dict( |
| 135 | + base, kind="branch", branch=name, last_activity=_date(when), |
| 136 | + days_idle=(now - when).days, last_push=_date(pushed), |
| 137 | + last_default_commit=_date(default_commit), branches="", |
| 138 | + url=_branch_url(repo.html_url or "", provider, name))) |
| 139 | + stale_repos.sort(key=lambda r: (-r["days_idle"], r["repository"])) |
| 140 | + stale_branches.sort(key=lambda r: (r["repository"], -r["days_idle"], r["branch"])) |
| 141 | + return {"days": limit_days, "cutoff": _date(cutoff), |
| 142 | + "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), |
| 143 | + "include_archived": include_archived, |
| 144 | + "repositories": stale_repos, "branches": stale_branches, |
| 145 | + "counts": {"repositories": len(stale_repos), "branches": len(stale_branches), |
| 146 | + "active_repositories": active, "no_dates": unknown, |
| 147 | + "no_branch_dates": undated, "archived_left_out": skipped_archived}} |
| 148 | + |
| 149 | + |
| 150 | +def csv_text(found: dict) -> str: |
| 151 | + from app.core.reports import _quote_formula |
| 152 | + |
| 153 | + out = io.StringIO() |
| 154 | + writer = csv.writer(out) |
| 155 | + writer.writerow(COLUMNS) |
| 156 | + for row in found["repositories"] + found["branches"]: |
| 157 | + writer.writerow([_quote_formula(str("yes" if c == "archived" and row[c] else |
| 158 | + "" if c == "archived" else row.get(c, ""))) |
| 159 | + for c in COLUMNS]) |
| 160 | + return out.getvalue() |
| 161 | + |
| 162 | + |
| 163 | +def write(session, reports_dir=None) -> dict: |
| 164 | + """staleness.csv and staleness.json in the report folder. -> the counts""" |
| 165 | + from pathlib import Path |
| 166 | + |
| 167 | + from app.core.reports import _write_atomic, output_dir |
| 168 | + |
| 169 | + reports_dir = Path(reports_dir) if reports_dir else output_dir(session) |
| 170 | + found = find(session) |
| 171 | + _write_atomic(reports_dir / CSV_NAME, csv_text(found)) |
| 172 | + _write_atomic(reports_dir / JSON_NAME, json.dumps(found, indent=2) + "\n") |
| 173 | + return found["counts"] |
| 174 | + |
| 175 | + |
| 176 | +# ------------------------------------------------------------------ digest |
| 177 | + |
| 178 | +def recipients(session) -> list: |
| 179 | + from app.core import notify |
| 180 | + |
| 181 | + own = notify._split(store.get(session, "notify.stale_to")) |
| 182 | + return own or notify.recipients(session) |
| 183 | + |
| 184 | + |
| 185 | +class _When: |
| 186 | + """A schedule shaped the way external_schedule.next_run reads one.""" |
| 187 | + |
| 188 | + def __init__(self, session): |
| 189 | + freq = str(store.get(session, "notify.stale_frequency") or "weekly") |
| 190 | + self.frequency = freq if freq in ("weekly", "monthly") else "weekly" |
| 191 | + self.time_of_day = str(store.get(session, "notify.stale_time") or "08:00") |
| 192 | + self.weekday = int(store.get(session, "notify.stale_weekday") or 0) |
| 193 | + self.day_of_month = int(store.get(session, "notify.stale_day") or 1) |
| 194 | + self.run_on = "" |
| 195 | + |
| 196 | + |
| 197 | +def next_due(session, after: datetime = None): |
| 198 | + from app.core import external_schedule |
| 199 | + |
| 200 | + return external_schedule.next_run(_When(session), after or datetime.now(timezone.utc)) |
| 201 | + |
| 202 | + |
| 203 | +def describe_schedule(session) -> str: |
| 204 | + from app.core import external_schedule |
| 205 | + |
| 206 | + return external_schedule.describe(_When(session)) |
| 207 | + |
| 208 | + |
| 209 | +def message(found: dict) -> tuple: |
| 210 | + """-> (title, lines)""" |
| 211 | + c = found["counts"] |
| 212 | + title = (f"Staleness: {c['repositories']} repositor{'ies' if c['repositories'] != 1 else 'y'}" |
| 213 | + f" and {c['branches']} branch{'es' if c['branches'] != 1 else ''} with no commit" |
| 214 | + f" in {found['days']} days") |
| 215 | + lines = [] |
| 216 | + by_org = {} |
| 217 | + for row in found["repositories"]: |
| 218 | + by_org[row["organization"]] = by_org.get(row["organization"], 0) + 1 |
| 219 | + if by_org: |
| 220 | + lines.append("Stale repositories by organization: " + ", ".join( |
| 221 | + f"{org} {n}" for org, n in sorted(by_org.items(), key=lambda x: (-x[1], x[0])))) |
| 222 | + for row in found["repositories"][:DIGEST_LINES]: |
| 223 | + owner = row["technical_owner"] or row["application_owner"] |
| 224 | + lines.append(f"{row['repository']}: last commit {row['last_activity']}, " |
| 225 | + f"{row['days_idle']} days ago" + (f", owner {owner}" if owner else "")) |
| 226 | + if len(found["repositories"]) > DIGEST_LINES: |
| 227 | + lines.append(f"and {len(found['repositories']) - DIGEST_LINES} more repositories") |
| 228 | + branch_repos = len({b["repository"] for b in found["branches"]}) |
| 229 | + if found["branches"]: |
| 230 | + lines.append(f"{c['branches']} stale branches in {branch_repos} active " |
| 231 | + f"repositor{'ies' if branch_repos != 1 else 'y'}.") |
| 232 | + if c["no_branch_dates"]: |
| 233 | + lines.append(f"{c['no_branch_dates']} active repositories have not been scanned since " |
| 234 | + f"branch dates were recorded, so their branches are not in this yet.") |
| 235 | + lines.append("Every row is in the attached staleness.csv, and in the report files.") |
| 236 | + return title, lines |
| 237 | + |
| 238 | + |
| 239 | +def send(session, found: dict = None) -> dict: |
| 240 | + """Mail the digest now. -> {channel: sent}""" |
| 241 | + from app.core import notify |
| 242 | + |
| 243 | + found = found or find(session) |
| 244 | + to = recipients(session) |
| 245 | + if not to: |
| 246 | + return {} |
| 247 | + title, lines = message(found) |
| 248 | + stamp = found["generated_at"][:10] |
| 249 | + sheet = {"name": f"staleness-{stamp}.csv", "text": csv_text(found)} |
| 250 | + root = notify.base_url(session) |
| 251 | + link = f"{root}/reports" if root else "" |
| 252 | + body = notify.text_message(session, title, lines, None, link, "") |
| 253 | + html = (notify.html_message(session, title, lines, None, link, "") |
| 254 | + if notify.mail_format(session) == "html" else "") |
| 255 | + if (store.get(session, "notify.mail_transport") or "smtp").lower() == "graph": |
| 256 | + return {"microsoft 365": notify.send_graph(session, title, body, html, to, sheet)} |
| 257 | + return {"email": notify.send_email(session, title, body, html, to, sheet)} |
| 258 | + |
| 259 | + |
| 260 | +def tick(now: datetime = None) -> dict: |
| 261 | + """The scheduler's check, every few minutes. Sends when the digest is due.""" |
| 262 | + from app.core.audit import record |
| 263 | + from app.core.db import session_scope |
| 264 | + |
| 265 | + now = now or datetime.now(timezone.utc) |
| 266 | + with session_scope() as db: |
| 267 | + if not store.get(db, "notify.stale_enabled"): |
| 268 | + return {} |
| 269 | + due_text = store.get(db, "notify.stale_next") or "" |
| 270 | + try: |
| 271 | + due = _aware(datetime.fromisoformat(due_text)) if due_text else None |
| 272 | + except ValueError: |
| 273 | + due = None |
| 274 | + if due is None: |
| 275 | + store.set_value(db, "notify.stale_next", next_due(db, now).isoformat(), "scheduler") |
| 276 | + return {"armed": True} |
| 277 | + if due > now: |
| 278 | + return {} |
| 279 | + try: |
| 280 | + found = find(db, now=now) |
| 281 | + sent = send(db, found) |
| 282 | + except Exception as exc: # noqa: BLE001 - a background job must not take the scheduler down |
| 283 | + log.warning("staleness digest failed: %s", exc) |
| 284 | + sent, found = {}, {"counts": {}} |
| 285 | + ok = bool(sent) and all(sent.values()) |
| 286 | + # Moved on either way, so a mail server that is down is tried again at |
| 287 | + # the next due time rather than every few minutes all day. |
| 288 | + store.set_value(db, "notify.stale_next", next_due(db, now).isoformat(), "scheduler") |
| 289 | + store.set_value(db, "notify.stale_last", { |
| 290 | + "at": now.isoformat(), "ok": ok, "counts": found.get("counts", {}), |
| 291 | + "sent": sent}, "scheduler") |
| 292 | + record(db, "notifications.staleness", actor="scheduler", |
| 293 | + detail="%s; %s" % (json.dumps(found.get("counts", {})), |
| 294 | + ", ".join(f"{k}: {'ok' if v else 'failed'}" |
| 295 | + for k, v in sorted(sent.items())) |
| 296 | + or "no recipients, nothing sent"), |
| 297 | + outcome="ok" if ok else "warn") |
| 298 | + return {"sent": sent} |
0 commit comments