From 7cf533bab2ac0e2db3d1b6d551cc67d51fea56f1 Mon Sep 17 00:00:00 2001 From: Edgar Date: Wed, 10 Jun 2026 08:53:46 +0200 Subject: [PATCH 1/3] feat: port ethrex-devnets ops toolkit into devnets_monitor/ Literal copy of lambdaclass/ethrex-devnets. Per-devnet incident history is gitignored (only _template.md kept); audited clean of secrets. --- .../.claude/skills/devnet-ops/SKILL.md | 66 + devnets_monitor/.gitignore | 33 + devnets_monitor/CLAUDE.md | 142 +++ devnets_monitor/README.md | 82 ++ devnets_monitor/config/devnets.yaml | 48 + devnets_monitor/config/devnets/.gitkeep | 0 .../config/devnets/glamsterdam-devnet-5.yaml | 52 + devnets_monitor/config/eips.json | 80 ++ devnets_monitor/config/schema.md | 68 + devnets_monitor/data/.gitkeep | 0 devnets_monitor/devnets/__init__.py | 1 + devnets_monitor/devnets/analyze.py | 61 + devnets_monitor/devnets/assertoor.py | 172 +++ devnets_monitor/devnets/bal.py | 222 ++++ devnets_monitor/devnets/blobtrack.py | 286 +++++ devnets_monitor/devnets/cli.py | 416 ++++++ devnets_monitor/devnets/collect.py | 94 ++ devnets_monitor/devnets/config.py | 190 +++ devnets_monitor/devnets/deploytl.py | 340 +++++ devnets_monitor/devnets/detect.py | 1113 +++++++++++++++++ devnets_monitor/devnets/discover.py | 266 ++++ devnets_monitor/devnets/dora.py | 295 +++++ devnets_monitor/devnets/eiptrack.py | 124 ++ devnets_monitor/devnets/epbs.py | 338 +++++ devnets_monitor/devnets/exectime.py | 182 +++ devnets_monitor/devnets/forks.py | 112 ++ devnets_monitor/devnets/forkview.py | 217 ++++ devnets_monitor/devnets/health.py | 89 ++ devnets_monitor/devnets/hive.py | 206 +++ devnets_monitor/devnets/network.py | 500 ++++++++ devnets_monitor/devnets/peers.py | 77 ++ devnets_monitor/devnets/proposals.py | 154 +++ devnets_monitor/devnets/remote.py | 231 ++++ devnets_monitor/devnets/spamoor.py | 194 +++ devnets_monitor/devnets/ssh.py | 55 + devnets_monitor/devnets/status.py | 124 ++ devnets_monitor/devnets/store.py | 253 ++++ devnets_monitor/devnets/wipe.py | 77 ++ devnets_monitor/docs/architecture.md | 134 ++ devnets_monitor/docs/cron-example.md | 67 + devnets_monitor/docs/devnet-ops-skill.md | 48 + devnets_monitor/docs/devnet-ops.md | 186 +++ devnets_monitor/docs/history/_template.md | 41 + devnets_monitor/pyproject.toml | 25 + devnets_monitor/uv.lock | 704 +++++++++++ devnets_monitor/web/__init__.py | 1 + devnets_monitor/web/aggregate.py | 251 ++++ devnets_monitor/web/app.py | 928 ++++++++++++++ devnets_monitor/web/templates/assertoor.html | 90 ++ devnets_monitor/web/templates/bal.html | 61 + devnets_monitor/web/templates/base.html | 305 +++++ devnets_monitor/web/templates/blobs.html | 183 +++ devnets_monitor/web/templates/clients.html | 95 ++ .../web/templates/command_center.html | 312 +++++ devnets_monitor/web/templates/deploy.html | 136 ++ devnets_monitor/web/templates/eiptrack.html | 78 ++ devnets_monitor/web/templates/epbs.html | 77 ++ devnets_monitor/web/templates/events.html | 134 ++ devnets_monitor/web/templates/exectime.html | 164 +++ devnets_monitor/web/templates/forks.html | 101 ++ devnets_monitor/web/templates/hive.html | 54 + devnets_monitor/web/templates/incidents.html | 17 + devnets_monitor/web/templates/index.html | 135 ++ devnets_monitor/web/templates/network.html | 130 ++ devnets_monitor/web/templates/node.html | 314 +++++ devnets_monitor/web/templates/proposals.html | 134 ++ devnets_monitor/web/templates/spamoor.html | 96 ++ 67 files changed, 11961 insertions(+) create mode 100644 devnets_monitor/.claude/skills/devnet-ops/SKILL.md create mode 100644 devnets_monitor/.gitignore create mode 100644 devnets_monitor/CLAUDE.md create mode 100644 devnets_monitor/README.md create mode 100644 devnets_monitor/config/devnets.yaml create mode 100644 devnets_monitor/config/devnets/.gitkeep create mode 100644 devnets_monitor/config/devnets/glamsterdam-devnet-5.yaml create mode 100644 devnets_monitor/config/eips.json create mode 100644 devnets_monitor/config/schema.md create mode 100644 devnets_monitor/data/.gitkeep create mode 100644 devnets_monitor/devnets/__init__.py create mode 100644 devnets_monitor/devnets/analyze.py create mode 100644 devnets_monitor/devnets/assertoor.py create mode 100644 devnets_monitor/devnets/bal.py create mode 100644 devnets_monitor/devnets/blobtrack.py create mode 100644 devnets_monitor/devnets/cli.py create mode 100644 devnets_monitor/devnets/collect.py create mode 100644 devnets_monitor/devnets/config.py create mode 100644 devnets_monitor/devnets/deploytl.py create mode 100644 devnets_monitor/devnets/detect.py create mode 100644 devnets_monitor/devnets/discover.py create mode 100644 devnets_monitor/devnets/dora.py create mode 100644 devnets_monitor/devnets/eiptrack.py create mode 100644 devnets_monitor/devnets/epbs.py create mode 100644 devnets_monitor/devnets/exectime.py create mode 100644 devnets_monitor/devnets/forks.py create mode 100644 devnets_monitor/devnets/forkview.py create mode 100644 devnets_monitor/devnets/health.py create mode 100644 devnets_monitor/devnets/hive.py create mode 100644 devnets_monitor/devnets/network.py create mode 100644 devnets_monitor/devnets/peers.py create mode 100644 devnets_monitor/devnets/proposals.py create mode 100644 devnets_monitor/devnets/remote.py create mode 100644 devnets_monitor/devnets/spamoor.py create mode 100644 devnets_monitor/devnets/ssh.py create mode 100644 devnets_monitor/devnets/status.py create mode 100644 devnets_monitor/devnets/store.py create mode 100644 devnets_monitor/devnets/wipe.py create mode 100644 devnets_monitor/docs/architecture.md create mode 100644 devnets_monitor/docs/cron-example.md create mode 100644 devnets_monitor/docs/devnet-ops-skill.md create mode 100644 devnets_monitor/docs/devnet-ops.md create mode 100644 devnets_monitor/docs/history/_template.md create mode 100644 devnets_monitor/pyproject.toml create mode 100644 devnets_monitor/uv.lock create mode 100644 devnets_monitor/web/__init__.py create mode 100644 devnets_monitor/web/aggregate.py create mode 100644 devnets_monitor/web/app.py create mode 100644 devnets_monitor/web/templates/assertoor.html create mode 100644 devnets_monitor/web/templates/bal.html create mode 100644 devnets_monitor/web/templates/base.html create mode 100644 devnets_monitor/web/templates/blobs.html create mode 100644 devnets_monitor/web/templates/clients.html create mode 100644 devnets_monitor/web/templates/command_center.html create mode 100644 devnets_monitor/web/templates/deploy.html create mode 100644 devnets_monitor/web/templates/eiptrack.html create mode 100644 devnets_monitor/web/templates/epbs.html create mode 100644 devnets_monitor/web/templates/events.html create mode 100644 devnets_monitor/web/templates/exectime.html create mode 100644 devnets_monitor/web/templates/forks.html create mode 100644 devnets_monitor/web/templates/hive.html create mode 100644 devnets_monitor/web/templates/incidents.html create mode 100644 devnets_monitor/web/templates/index.html create mode 100644 devnets_monitor/web/templates/network.html create mode 100644 devnets_monitor/web/templates/node.html create mode 100644 devnets_monitor/web/templates/proposals.html create mode 100644 devnets_monitor/web/templates/spamoor.html diff --git a/devnets_monitor/.claude/skills/devnet-ops/SKILL.md b/devnets_monitor/.claude/skills/devnet-ops/SKILL.md new file mode 100644 index 0000000..d766e87 --- /dev/null +++ b/devnets_monitor/.claude/skills/devnet-ops/SKILL.md @@ -0,0 +1,66 @@ +--- +name: devnet-ops +description: Operate, inspect, monitor, and debug ethrex EL nodes on ethpandaops devnets (status sweeps, peers/logs, blob & fork tracking, Hive results, wipe/resync, incident history). Use whenever the user asks to check/inspect a devnet node, ssh into one, read logs, investigate a devnet error, track blob inclusion or fork schedule, see Hive conformance, deploy/wipe a node, or ask about past devnet problems. +--- + +# devnet-ops + +This repo (`ethrex-devnets`) is the single home for ethrex devnet ops + monitoring. +Procedures, per-devnet history, and the `dv` CLI all live here. Paths below are +relative to the repo root. + +Before any devnet operation or answering a question about a node/incident, READ: + +1. `CLAUDE.md` — the working agreement (golden rules, exact data sources, the wipe + sequence). +2. `docs/devnet-ops.md` — generic access & inspection procedures (SSH, inventory, + container layout, build & deploy, debug logging, wipe & resync, Dora API). + Substitute `` with the target network. +3. `docs/history/.md` — per-devnet facts and incident history (roster, fork + schedule, commit map, known issues with root cause + recovery). For + glamsterdam-devnet-5 this is `docs/history/glamsterdam-devnet-5.md`. If a devnet + has no history file, create one from `docs/history/_template.md` as you learn facts. + +## The `dv` CLI + +Run from the repo root via `uv run dv ...`. Read-only by default; only `dv wipe` +mutates (gated behind `--yes`). Target devnet resolves: explicit arg > `$DEVNET` +env > `config/devnets.yaml` `default`. + +``` +uv run dv discover # refresh roster/forks/image from the ethpandaops repo (gh) +uv run dv status [node] # EL build/head/peers/state@head + CL + watchtower (--json) +uv run dv peers # peer count, inbound/outbound, client mix, body-serving fails +uv run dv logs [--since 2m] +uv run dv cl [--since 3m] +uv run dv collect [all|blobs|health|hive|forks] # into data/ethrex-devnets.sqlite +uv run dv blob # blob inclusion per proposer + ethrex-vs-others (decay lens) +uv run dv fork # fork schedule + EIPs + countdown +uv run dv hive # Hive conformance summary (groups from config/devnets.yaml) +uv run dv serve # read-only dashboard at http://127.0.0.1:8099 +uv run dv wipe --yes # MUTATING: recover a wedged EL +``` + +## Workflow + +1. Read `docs/devnet-ops.md` for HOW (procedures). +2. Read `docs/history/.md` for WHAT/WHY — check whether the current symptom + matches a known issue before investigating from scratch; many recur. +3. ALWAYS verify the live host before trusting a node name. `dv status` reads the + live `docker inspect execution` image; a `*-ethrex-*` node may have been swapped + to another client. +4. Default to read-only. `dv wipe` and any deploy/recreate are mutating; confirm + with the user first. + +## Adding a devnet + +1. Add an entry under `devnets:` in `config/devnets.yaml` (see `config/schema.md`). +2. `uv run dv discover ` to populate `config/devnets/.yaml`. +3. Create `docs/history/.md` from `docs/history/_template.md`. + +## Maintenance + +When you discover a new incident, divergence, or devnet fact, append a dated entry +to `docs/history/.md` (and `docs/devnet-ops.md` if a procedure changed) and +commit it. The fork -> EIP map is `config/eips.json` (sourced via eipmcp; re-run +`get_hardfork` to refresh). diff --git a/devnets_monitor/.gitignore b/devnets_monitor/.gitignore new file mode 100644 index 0000000..67e880a --- /dev/null +++ b/devnets_monitor/.gitignore @@ -0,0 +1,33 @@ +# No secrets, ever. The entries below guard by filename; a pre-commit content +# audit (rg for xatu/jwt/Bearer/password/secret/BEGIN KEY) is the real gate. +# SSH keys, JWTs, and Xatu credentials live only on hosts, never in this repo. + +# Data store + caches (regenerable) +data/*.sqlite +data/*.sqlite-* +data/*.parquet +.cache/ +tmp/ + +# Secrets / captured run commands (may embed tokens or host internals) +*.secret +*.jwt +*xatu*cred* +*credentials* +runlike-*.txt +run-*.sh +run-execution*.txt + +# Per-devnet incident history (may embed host internals); keep only the template +docs/history/*.md +!docs/history/_template.md + +# Python +__pycache__/ +*.pyc +.venv/ +.uv/ + +# Editor / OS +.DS_Store +*.swp diff --git a/devnets_monitor/CLAUDE.md b/devnets_monitor/CLAUDE.md new file mode 100644 index 0000000..d02ed72 --- /dev/null +++ b/devnets_monitor/CLAUDE.md @@ -0,0 +1,142 @@ +# CLAUDE.md — ethrex-devnets + +Working agreement for any agent operating in this repo. Read this first, then +`docs/architecture.md` for the component map and `docs/devnet-ops.md` for the +operational procedures. + +## What this repo is + +A standalone ops + monitoring toolkit for ethrex on ethpandaops devnets. NOT part +of the ethrex codebase; it operates ethrex nodes from the outside (SSH + docker + +JSON-RPC) and pulls public devnet data (Dora, Hive, the ethpandaops devnet repo). +The owner uses it to track ethrex behavior across forks ("forks news"). + +It replaces git-excluded notes that used to live scattered inside the ethrex +working copies. This is now the single home for the devnet ops runbook and +per-devnet incident history. + +## Golden rules + +1. **Read-only by default.** Inspection (status/peers/logs/curl/collect) never + mutates. Every mutating action (wipe, deploy, recreate) MUST be gated behind an + explicit flag (`--yes`) and refuse to run without it. When in doubt, do not mutate. +2. **No secrets in git, ever.** SSH keys, JWTs, and Xatu/ClickHouse credentials + live only on hosts. Before any `git add` of new/edited bash or docs, run: + `rg -n 'xatu|jwt|Bearer|password|secret|-----BEGIN' lib/bash/ bin/ docs/ config/` + and require empty output. The `.gitignore` guards by filename; this audit + guards by content. Captured `runlike`/`run-*.sh` files (which embed host + internals) are gitignored, never commit them. +3. **Generic over devnets.** Nothing is hardcoded to one devnet. New devnet = + one entry in `config/devnets.yaml` + `dv discover `. Never add a + `NODES=()`-style hardcoded roster; read it from `config/devnets/.yaml`. +4. **Verify the live host before trusting a name.** Inventory is INTENDED config; + a `*-ethrex-*` node may have been manually swapped to another client. The + status path always reads the live `docker inspect execution` image. +5. **Preserve the incident-tested remote sequences.** The shell snippets that run + ON the devnet host (the wipe sequence, the status/peers probes) are generalized + from a working, incident-tested helper. They run on the host over SSH and stay + shell; the `wipe` sequence in particular is load-bearing (see "wipe" below). + Change them only with a clear reason and preserve every step. + +## Architecture in one paragraph + +One codebase: **Python**, run via `uv`, with a single `dv` console entry point. +Python does everything local: config/YAML/JSON parsing, SSH orchestration +(subprocess to the system `ssh`), the SQLite store, analysis, and the dashboard. +The only shell involved is the snippets that execute ON the devnet host (docker, +runlike, curl, the wipe sequence); those are sent over `ssh ... bash -s` as +heredoc strings because they run on the host regardless of local language. A host +status probe emits JSON that Python parses in-process; there is no local-bash +intermediary and no cross-language file seam. Earlier drafts used a separate bash +CLI layer; it was collapsed into Python to remove the two-language seam and the +shell-injection / fragile-parsing surface. Full detail in `docs/architecture.md`. + +## Conventions + +- **Commits:** conventional commits, short. Types: `feat`, `fix`, `docs`, `chore`, + `refactor`. No Co-Authored-By lines. Run the secret audit before committing. +- **Prose / user-facing text:** use `;` or `,`, never the double-hyphen dash. +- **Python:** run via `uv` (`uv run dv ...`); stdlib first; deps minimal + (`pyyaml`, `requests`, and `fastapi`/`uvicorn`/`jinja2` only for the dashboard). + No premature abstraction. One module per concern (`config`, `ssh`, `remote`, + `status`, `discover`, `wipe`, `store`, `dora`, `hive`, `forks`, `collect`, + `blobtrack`, `forkview`). Idempotent, re-runnable collectors (upsert on primary + key, watermark incremental fetch). +- **Remote shell snippets:** keep them in `devnets/remote.py` as named string + constants; send via `ssh ... bash -s`. Validate any value interpolated into a + remote command (durations, node names) before sending; never f-string raw user + input into a shell command. Prefer passing values as positional args to + `bash -s` over interpolation. +- **Mutations:** only `dv wipe` mutates, gated behind `--yes`. Everything else is + read-only. +- **Simplicity over complexity.** This is a personal/small-team ops tool, not a + platform. Prefer the smallest thing that works. + +## Data sources (exact) + +- **SSH:** `devops@.srv..ethpandaops.io`. Per-node docker + containers: `execution` (ethrex), `beacon` (CL), `validator`, `snooper-engine` + (engine proxy CL<->EL, logs FCU/newPayload), `xatu-sentry`, `vector`, + `ethereum-node-docker-watchtower`, `prometheus`, `node_exporter`. +- **EL RPC** on node: `localhost:8545`, namespaces `eth,net,web3,admin,debug` + (NO `txpool`). Metrics `localhost:6060/metrics` (Prometheus; cumulative + counters, no current-pool-size gauge). +- **Dora** (blob inclusion over time, the practical source): + `/api/v1/slots?limit=N&with_missing=1&with_orphaned=1` -> + `data.slots[]` each with `slot`, `proposer_name`, `blob_count`, + `eth_block_number`, `status`, `time`, `epoch`, `gas_used`, `execution_times[]`; + paginate via `data.next_page` (also supports `min_slot`/`max_slot` range params). +- **ethpandaops config API:** `/api/v1/nodes/inventory` (enodes/ENRs). +- **Devnet repo (source of truth)** via `gh api`: + `repos//contents/ansible/inventories//inventory.ini` + (group `[ethrex:children]` = roster), `.../group_vars/all/images.yaml` + (`default_ethereum_client_images.ethrex` = image tag), + `network-configs//metadata/genesis.json` (chainId, fork timestamps, + blobSchedule). +- **Hive:** `https://hive.ethpandaops.io` group listings; per-devnet groups in + `devnets.yaml` `hive_groups`. Fetch the API directly; do not depend on any + external/Claude-only wrapper script. +- **eipmcp** (MCP tool, not HTTP): EIP/fork data for `dv fork` enrichment; + `dv eips-refresh` regenerates `config/eips.json`. Not auto-fetched at runtime. +- **Xatu / ClickHouse (FUTURE, credential-gated):** each node's `xatu-sentry` + ships CL events (incl. `blob_sidecar`) via gRPC/TLS to + `server.xatu-experimental.ethpandaops.io:443`. The public query endpoint + `clickhouse.xatu.ethpandaops.io` needs credentials we do NOT have, and devnet + data may live only in the experimental instance. Treat as optional; the core + must NOT depend on it. Dora is the available substitute. + +## The `wipe` sequence (do not break) + +Recover a wedged EL. The datadir is owned by uid 1004, so wipe via a root +container. The full, incident-tested sequence: +pause watchtower -> `runlike execution` capture (ABORT if capture is not a valid +`docker run`) -> note if `--nat.extip` already present -> `docker pull` -> +`docker rm -f execution` -> root-container `rm -rf` of the datadir -> recreate +with `-d` (runlike omits detach) -> **`docker restart snooper-engine`** (REQUIRED: +the proxy holds a stale connection to the old EL; without this the CL can't drive +the fresh EL and it logs "No messages from the consensus layer") -> unpause +watchtower -> print post-status. If a node talks to the EL directly without a +snooper, restart `beacon` instead. Gated behind `--yes`. + +## Adding a devnet + +1. Add an entry under `devnets:` in `config/devnets.yaml` (see `config/schema.md`). +2. `dv discover ` to populate `config/devnets/.yaml` from the repo. +3. Optionally `dv collect all` to start a history. +4. Create `docs/history/.md` from `docs/history/_template.md` and log + facts/incidents there as you learn them (dated entries; what / why / recovery). + +## Incident history is part of the job + +`docs/history/.md` is hard-won knowledge: root causes and recovery steps +for real wedges (blob decay, snap-leftover state wedges, 0-inbound-peers, etc.). +When you discover a new incident or devnet fact, append a dated entry. Check the +history before investigating a symptom from scratch; many recur. + +## Where to look + +- `docs/architecture.md` — components, the bash/Python seam, data flow. +- `docs/devnet-ops.md` — operational procedures (SSH, inspection, build/deploy, + debug logging, wipe/resync, Dora API). +- `docs/history/.md` — per-devnet facts + dated incident log. +- `config/schema.md` — config file field reference. diff --git a/devnets_monitor/README.md b/devnets_monitor/README.md new file mode 100644 index 0000000..61c8c3e --- /dev/null +++ b/devnets_monitor/README.md @@ -0,0 +1,82 @@ +# ethrex-devnets + +Ops + monitoring toolkit for [ethrex](https://github.com/lambdaclass/ethrex) on +[ethpandaops](https://ethpandaops.io) devnets. Single home for the devnet ops +runbook, per-devnet incident history, and the tooling that watches ethrex as new +forks (glamsterdam, BAL, fusaka, ...) roll out. + +Generic across devnets: everything is parameterized by devnet name +(`glamsterdam-devnet-5`, `bal-devnet-3`, ...). Read-only by default; every +mutating action (wipe, deploy) is gated behind an explicit flag. + +## What it does + +Four capability areas, one front door (`dv`): + +1. **Node health monitoring** — multi-node, multi-client status sweeps (EL + build/commit/head/peers/sync/state-at-head, CL sync line, watchtower); + peer mix; log tails. +2. **Blob & fork tracking** — blob inclusion per proposer over time, ethrex vs + other clients; fork schedule + EIP-per-fork; next-fork countdown. +3. **Hive / conformance** — pull and summarize Hive group runs (bal, bal-quick, + future fork groups) and pass rates for ethrex. +4. **Ops automation** — wipe/resync, debug-log capture, watchtower control, + wrapped as safe (mutation-gated) commands. + +## Layout + +``` +devnets/ # the Python package: CLI, SSH orchestration, collectors, analysis, store + cli.py # `dv` argparse dispatcher (the daily driver) + remote.py # shell snippets that run ON the devnet host (sent over ssh bash -s) +config/ # devnets.yaml (registry) + devnets/.yaml (discovered cache) + eips.json +docs/ # devnet-ops runbook, per-devnet history, architecture/agent guide +data/ # SQLite store (gitignored, regenerable) +web/ # FastAPI dashboard (read-only, localhost) +pyproject.toml # uv-managed; exposes the `dv` console script +``` + +## Quick start + +Everything runs through `uv run dv` (or just `dv` if the venv is on PATH): + +```bash +# discover a devnet's roster + fork schedule from the ethpandaops repo +uv run dv discover glamsterdam-devnet-5 + +# live health sweep across all ethrex nodes +uv run dv status glamsterdam-devnet-5 + +# collect historical data into SQLite, then analyze +uv run dv collect glamsterdam-devnet-5 all +uv run dv blob glamsterdam-devnet-5 # blob inclusion per proposer over time +uv run dv fork glamsterdam-devnet-5 # fork schedule + EIPs + countdown + +# pull Hive conformance results +uv run dv hive glamsterdam-devnet-5 + +# read-only dashboard at http://127.0.0.1:8099 +uv run dv serve + +uv run dv --help # full subcommand list (read-only vs mutating) +``` + +The default devnet (`config/devnets.yaml` `default:`) is used when no `` +argument is given. Override per-invocation with `DEVNET=`. + +## Requirements + +- `uv` (drives the whole CLI; manages the Python env) +- `ssh` (access to `devops@.srv..ethpandaops.io`) +- `gh` authenticated with read access to the ethpandaops devnet repos (for `dv discover`) +- On the devnet hosts: `docker`, `runlike` (for the wipe path), `curl` + +## Conventions + +- Read-only by default. Mutating commands (`dv wipe`) refuse to run without `--yes`. +- No secrets in git. SSH keys, JWTs, Xatu credentials live only on hosts; a + content audit gates commits (see CLAUDE.md). +- User-facing text uses `;` or `,`, not the double-hyphen dash. + +See `CLAUDE.md` for the full working agreement and `docs/architecture.md` for how +the pieces fit together. diff --git a/devnets_monitor/config/devnets.yaml b/devnets_monitor/config/devnets.yaml new file mode 100644 index 0000000..088e9c3 --- /dev/null +++ b/devnets_monitor/config/devnets.yaml @@ -0,0 +1,48 @@ +# ethrex-devnets: registry of ethpandaops devnets this toolkit operates on. +# +# This file is the static, hand-maintained entry point. Per-devnet LIVE facts +# (node roster, fork schedule, image tag) are auto-discovered into +# config/devnets/.yaml by `dv discover ` and read from there. +# Keep secrets OUT of this file; it is committed. +# +# Field reference: see config/schema.md + +# Default devnet used when a command is given no explicit argument +# and the DEVNET env var is unset. +default: glamsterdam-devnet-5 + +devnets: + glamsterdam-devnet-5: + # GitHub repo holding ansible inventory + network-configs (source of truth + # for roster, image tags, genesis/fork schedule). Used by `dv discover`. + devnets_repo: ethpandaops/glamsterdam-devnets + # Sub-path inside that repo for this devnet (ansible inventory dir name and + # network-configs dir name). For glamsterdam the dirs are "devnet-5". + repo_path: devnet-5 + # Public service base URLs (no trailing slash). + dora_base: https://dora.glamsterdam-devnet-5.ethpandaops.io + config_base: https://config.glamsterdam-devnet-5.ethpandaops.io + # ethrex devnet branch + image builder target (informational). + branch: glamsterdam-devnet-5 + builder_target: lambdaclass/ethrex@glamsterdam-devnet-5 + # SSH host pattern. Optional; defaults to the ethpandaops scheme below. + # Placeholders: {user}, {node}, {devnet}. Override only if a devnet differs. + # ssh_user: devops + # ssh_host_template: "{user}@{node}.srv.{devnet}.ethpandaops.io" + # Hive groups to pull conformance results from for this devnet. + hive_groups: + - bal + - bal-quick + + # ---- Template for adding another devnet (uncomment + adjust, then run + # ---- `dv discover `). repo_path is the dir name inside the repo. + # + # bal-devnet-3: + # devnets_repo: ethpandaops/bal-devnets + # repo_path: devnet-3 + # dora_base: https://dora.bal-devnet-3.ethpandaops.io + # config_base: https://config.bal-devnet-3.ethpandaops.io + # branch: bal-devnet-3 + # builder_target: lambdaclass/ethrex@bal-devnet-3 + # hive_groups: + # - bal diff --git a/devnets_monitor/config/devnets/.gitkeep b/devnets_monitor/config/devnets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/devnets_monitor/config/devnets/glamsterdam-devnet-5.yaml b/devnets_monitor/config/devnets/glamsterdam-devnet-5.yaml new file mode 100644 index 0000000..3c38365 --- /dev/null +++ b/devnets_monitor/config/devnets/glamsterdam-devnet-5.yaml @@ -0,0 +1,52 @@ +# Auto-generated by: dv discover glamsterdam-devnet-5 +# Regenerate at any time with: dv discover glamsterdam-devnet-5 + +discovered_at: 1780998248 +image_tag: ethpandaops/ethrex:glamsterdam-devnet-5 +roster: +- name: buildoor-prysm-ethrex-1 + verified: false +- name: grandine-ethrex-1 + verified: false +- name: lighthouse-ethrex-1 + verified: false +- name: lodestar-ethrex-1 + verified: false +- name: lodestar-ethrex-2 + verified: false +- name: nimbus-ethrex-1 + verified: false +- name: prysm-ethrex-1 + verified: false +- name: prysm-ethrex-2 + verified: false +- name: teku-ethrex-1 + verified: false +fork_schedule: + shanghai: + activation_ts: 0 + cancun: + activation_ts: 0 + blob_target: 3 + blob_max: 6 + prague: + activation_ts: 0 + blob_target: 6 + blob_max: 9 + osaka: + activation_ts: 0 + blob_target: 6 + blob_max: 9 + bpo1: + activation_ts: 1780578384 + blob_target: 10 + blob_max: 15 + bpo2: + activation_ts: 1780578768 + blob_target: 14 + blob_max: 21 + amsterdam: + activation_ts: 1780589520 + blob_target: 14 + blob_max: 21 +chain_id: 7095321190 diff --git a/devnets_monitor/config/eips.json b/devnets_monitor/config/eips.json new file mode 100644 index 0000000..88c897c --- /dev/null +++ b/devnets_monitor/config/eips.json @@ -0,0 +1,80 @@ +{ + "_comment": "Fork -> EIP map consumed by `dv fork` / the dashboard (loaded into fork_eips by `dv collect forks`). Sourced via eipmcp. Settled forks (cancun/shanghai/prague/osaka) list their final shipped EIP sets (no stage). glamsterdam (amsterdam) is pre-freeze: entries carry an inclusion `stage` per the EIP-7773 meta (SFI=Scheduled, CFI=Considered, PFI=Proposed). EIPs Declined for Inclusion (e.g. FOCIL 7805, EOF 7692, SSZ tx/receipts 6404/6466) are intentionally EXCLUDED. Stages shift until fork freeze; re-derive from get_eip(7773) body. bpo1/bpo2 are Blob-Parameter-Only forks (EIP-7892): no feature EIPs. The `status` field tracks ethrex implementation progress: done|in_progress|missing|n/a; absent/unknown means not yet assessed.", + "_source": "eipmcp get_hardfork + get_eip(7773) body (queried 2026-06-09)", + "forks": { + "cancun": [ + {"eip": 1153, "title": "Transient storage opcodes"}, + {"eip": 4788, "title": "Beacon block root in the EVM"}, + {"eip": 4844, "title": "Shard Blob Transactions"}, + {"eip": 5656, "title": "MCOPY - Memory copying instruction"}, + {"eip": 6780, "title": "SELFDESTRUCT only in same transaction"}, + {"eip": 7516, "title": "BLOBBASEFEE opcode"} + ], + "shanghai": [ + {"eip": 3651, "title": "Warm COINBASE"}, + {"eip": 3855, "title": "PUSH0 instruction"}, + {"eip": 3860, "title": "Limit and meter initcode"}, + {"eip": 4895, "title": "Beacon chain push withdrawals as operations"}, + {"eip": 6049, "title": "Deprecate SELFDESTRUCT"} + ], + "prague": [ + {"eip": 2537, "title": "Precompile for BLS12-381 curve operations"}, + {"eip": 2935, "title": "Serve historical block hashes from state"}, + {"eip": 6110, "title": "Supply validator deposits on chain"}, + {"eip": 7002, "title": "Execution layer triggerable withdrawals"}, + {"eip": 7251, "title": "Increase the MAX_EFFECTIVE_BALANCE"}, + {"eip": 7549, "title": "Move committee index outside Attestation"}, + {"eip": 7623, "title": "Increase calldata cost"}, + {"eip": 7642, "title": "eth/69 - history expiry and simpler receipts"}, + {"eip": 7685, "title": "General purpose execution layer requests"}, + {"eip": 7691, "title": "Blob throughput increase"}, + {"eip": 7702, "title": "Set Code for EOAs"}, + {"eip": 7840, "title": "Add blob schedule to EL config files"} + ], + "osaka": [ + {"eip": 7594, "title": "PeerDAS - Peer Data Availability Sampling"}, + {"eip": 7642, "title": "eth/69 - history expiry and simpler receipts"}, + {"eip": 7823, "title": "Set upper bounds for MODEXP"}, + {"eip": 7825, "title": "Transaction Gas Limit Cap"}, + {"eip": 7883, "title": "ModExp Gas Cost Increase"}, + {"eip": 7892, "title": "Blob Parameter Only Hardforks"}, + {"eip": 7910, "title": "eth_config JSON-RPC Method"}, + {"eip": 7917, "title": "Deterministic proposer lookahead"}, + {"eip": 7918, "title": "Blob base fee bounded by execution cost"}, + {"eip": 7934, "title": "RLP Execution Block Size Limit"}, + {"eip": 7935, "title": "Set default gas limit to 60M"}, + {"eip": 7939, "title": "Count leading zeros (CLZ) opcode"}, + {"eip": 7951, "title": "Precompile for secp256r1 Curve Support"} + ], + "bpo1": [], + "bpo2": [], + "amsterdam": [ + {"eip": 7708, "title": "ETH transfers emit a log", "stage": "SFI", "status": "done"}, + {"eip": 7732, "title": "Enshrined Proposer-Builder Separation", "stage": "SFI", "status": "missing"}, + {"eip": 7778, "title": "Block Gas Accounting without Refunds", "stage": "SFI", "status": "done"}, + {"eip": 7843, "title": "SLOTNUM opcode", "stage": "SFI", "status": "done"}, + {"eip": 7928, "title": "Block-Level Access Lists", "stage": "SFI", "status": "done"}, + {"eip": 7954, "title": "Increase Maximum Contract Size", "stage": "SFI", "status": "done"}, + {"eip": 7976, "title": "Increase Calldata Floor Cost", "stage": "SFI", "status": "done"}, + {"eip": 7981, "title": "Increase Access List Cost", "stage": "SFI", "status": "done"}, + {"eip": 8024, "title": "Backward compatible SWAPN, DUPN, EXCHANGE", "stage": "SFI", "status": "done"}, + {"eip": 8037, "title": "State Creation Gas Cost Increase", "stage": "SFI", "status": "done"}, + {"eip": 2780, "title": "Reduce intrinsic transaction gas", "stage": "CFI", "status": "missing"}, + {"eip": 7688, "title": "Forward compatible consensus data structures", "stage": "CFI", "status": "n/a"}, + {"eip": 7904, "title": "General Repricing", "stage": "CFI", "status": "missing"}, + {"eip": 7975, "title": "eth/70 - partial block receipt lists", "stage": "CFI", "status": "done"}, + {"eip": 7997, "title": "Deterministic Factory Predeploy", "stage": "CFI", "status": "missing"}, + {"eip": 8038, "title": "State-access gas cost increase", "stage": "CFI", "status": "missing"}, + {"eip": 8045, "title": "Exclude slashed validators from proposing", "stage": "CFI", "status": "n/a"}, + {"eip": 8061, "title": "Increase exit and consolidation churn", "stage": "CFI", "status": "n/a"}, + {"eip": 8070, "title": "eth/72 - Sparse Blobpool", "stage": "CFI", "status": "done"}, + {"eip": 8080, "title": "Let exits use the consolidation queue", "stage": "CFI", "status": "n/a"}, + {"eip": 8136, "title": "Cell-Level Deltas for Data Column Broadcast", "stage": "CFI", "status": "n/a"}, + {"eip": 8159, "title": "eth/71 - Block Access List Exchange", "stage": "CFI", "status": "done"}, + {"eip": 8246, "title": "Remove SELFDESTRUCT Burn", "stage": "CFI", "status": "done"}, + {"eip": 7610, "title": "Revert creation in case of non-empty storage", "stage": "PFI", "status": "done"}, + {"eip": 7979, "title": "Call and Return Opcodes for the EVM", "stage": "PFI", "status": "missing"}, + {"eip": 8163, "title": "Reserve EXTENSION (0xae) opcode", "stage": "PFI", "status": "missing"} + ] + } +} diff --git a/devnets_monitor/config/schema.md b/devnets_monitor/config/schema.md new file mode 100644 index 0000000..2adb068 --- /dev/null +++ b/devnets_monitor/config/schema.md @@ -0,0 +1,68 @@ +# Config schema + +Two layers of config. `devnets.yaml` is hand-maintained and committed. +`devnets/.yaml` is auto-generated by `dv discover` and is a cache of live +facts; it is committed too (so the CLI works offline / against dead devnets), +but it is always regenerable. + +## `config/devnets.yaml` (static registry) + +Top level: + +| field | type | meaning | +|----------|--------|---------| +| `default` | string | devnet name used when no `` arg and no `DEVNET` env is given | +| `devnets` | map | name -> devnet entry | + +Each `devnets.` entry (all hand-maintained): + +| field | required | meaning | +|------------------|----------|---------| +| `devnets_repo` | yes | `org/repo` holding ansible inventory + network-configs (source of truth) | +| `repo_path` | yes | dir name inside that repo for this devnet (e.g. `devnet-5`) | +| `dora_base` | yes | Dora explorer base URL, no trailing slash | +| `config_base` | yes | ethpandaops config API base URL, no trailing slash | +| `branch` | no | ethrex devnet branch name (informational) | +| `builder_target` | no | image builder target (informational) | +| `hive_groups` | no | list of Hive group names to pull conformance results from | +| `ssh_user` | no | SSH user; default `devops` | +| `ssh_host_template` | no | SSH host pattern with `{user}`/`{node}`/`{devnet}` placeholders; default `{user}@{node}.srv.{devnet}.ethpandaops.io` | + +## `config/devnets/.yaml` (discovered cache) + +Written by `dv discover ` from `gh api` against `devnets_repo`. Read by +both the bash CLI (roster) and the Python layer (forks). + +| field | type | source | meaning | +|------------------|--------|--------|---------| +| `discovered_at` | int | local clock | unix ts of last discover; `dv status` warns if > 24h old | +| `image_tag` | string | `group_vars/all/images.yaml` `default_ethereum_client_images.ethrex` | deployed ethrex image | +| `roster` | list | `inventory.ini` group `[ethrex:children]` | one entry per ethrex node | +| `roster[].name` | string | inventory | node short name, e.g. `lighthouse-ethrex-1` | +| `roster[].verified` | bool | live `docker inspect` | `true` only once a live check confirmed the node still runs ethrex (inventory is INTENDED config; a `*-ethrex-*` node may be swapped to another client) | +| `fork_schedule` | map | `network-configs//metadata/genesis.json` | fork name -> `{activation_ts, blob_target, blob_max}` | +| `chain_id` | int | genesis.json | chain id | + +### roster `verified` semantics + +`dv discover` writes roster entries with `verified: false`. A node only flips to +`verified: true` after a live `docker inspect execution` confirms the image is +`*ethrex*`. The bash CLI uses the full roster for `dv status` (it inspects live +anyway and prints the real client), but commands that assume ethrex specifics +should respect the flag. Re-run `dv discover` to refresh. + +## `config/eips.json` (fork -> EIP enrichment) + +Hand-refreshable via `dv eips-refresh` (which uses the eipmcp data). Maps fork +name -> list of EIP entries. Consumed by `dv fork`, `dv eip-track`, and the +dashboard. May go stale; the refresh command is the source of truth, regenerate +when a fork's EIP set changes. + +Each EIP entry supports these fields: + +| field | required | meaning | +|----------|----------|---------| +| `eip` | yes | EIP number (integer) | +| `title` | yes | short title string | +| `stage` | no | fork inclusion stage: `SFI` (Scheduled), `CFI` (Considered), `PFI` (Proposed); absent for settled forks | +| `status` | no | ethrex implementation status; operator-maintained. Values: `done`, `in_progress`, `missing`, `n/a`, `unknown` (default when absent). Settled forks (cancun/prague/etc.) omit this field since they are fully shipped. For upcoming forks (amsterdam), fill this in as work progresses. | diff --git a/devnets_monitor/data/.gitkeep b/devnets_monitor/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/devnets_monitor/devnets/__init__.py b/devnets_monitor/devnets/__init__.py new file mode 100644 index 0000000..d737dd4 --- /dev/null +++ b/devnets_monitor/devnets/__init__.py @@ -0,0 +1 @@ +# devnets package diff --git a/devnets_monitor/devnets/analyze.py b/devnets_monitor/devnets/analyze.py new file mode 100644 index 0000000..5f17d5c --- /dev/null +++ b/devnets_monitor/devnets/analyze.py @@ -0,0 +1,61 @@ +"""Comparative / baseline math helpers. + +Pure Python, no dependencies. Used by detectors and any relative analysis +that needs peer ratios, self-regression ratios, or z-scores. +""" + +from __future__ import annotations + +import math + + +def median(xs: list[float]) -> float | None: + """Return the median of xs, or None if xs is empty.""" + if not xs: + return None + s = sorted(xs) + n = len(s) + mid = n // 2 + if n % 2 == 1: + return float(s[mid]) + return (s[mid - 1] + s[mid]) / 2.0 + + +def peer_ratio(target: float, peers: list[float]) -> float | None: + """Return target / median(peers). + + None if peers is empty or median is 0. + """ + m = median(peers) + if m is None or m == 0: + return None + return target / m + + +def baseline_shift(recent: list[float], prior: list[float]) -> float | None: + """Return mean(recent) / mean(prior): the self-regression ratio. + + None if prior is empty or mean(prior) is 0. + """ + if not prior or not recent: + return None + prior_mean = sum(prior) / len(prior) + if prior_mean == 0: + return None + return (sum(recent) / len(recent)) / prior_mean + + +def zscore(value: float, sample: list[float]) -> float | None: + """Return (value - mean(sample)) / stdev(sample). + + None if fewer than 2 samples or stdev is 0. + """ + if len(sample) < 2: + return None + n = len(sample) + mean = sum(sample) / n + variance = sum((x - mean) ** 2 for x in sample) / (n - 1) + std = math.sqrt(variance) + if std == 0: + return None + return (value - mean) / std diff --git a/devnets_monitor/devnets/assertoor.py b/devnets_monitor/devnets/assertoor.py new file mode 100644 index 0000000..61045fc --- /dev/null +++ b/devnets_monitor/devnets/assertoor.py @@ -0,0 +1,172 @@ +"""Assertoor test run collector and detector. + +API shape verified against live glamsterdam-devnet-5: + GET /api/v1/test_runs + Response: {"status": "OK", "data": [ + {"run_id": int, "test_id": str, "name": str, "status": str, + "start_time": int, "stop_time": int}, ... + ]} + status values observed: "success", "failure", "running", "pending" +""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime, timezone +from typing import Any + +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + + +def _fetch_test_runs(devnet: str) -> list[dict[str, Any]] | None: + """ + GET /api/v1/test_runs from the Assertoor instance for this devnet. + Returns the list of run objects, or None on failure/absence. + Degrades gracefully on 404 (assertoor not deployed for this devnet). + """ + import requests + + url = f"https://assertoor.{devnet}.ethpandaops.io/api/v1/test_runs" + for attempt in range(2): + try: + resp = requests.get(url, timeout=20) + if resp.status_code == 200: + data = resp.json() + if data.get("status") == "OK": + return data.get("data") or [] + logger.warning("assertoor: unexpected response for %s: %s", devnet, data.get("status")) + return None + if resp.status_code == 404: + logger.info("assertoor: 404 for %s (not deployed)", devnet) + return None + logger.warning( + "assertoor fetch %s -> HTTP %d (attempt %d)", + devnet, resp.status_code, attempt, + ) + except Exception as exc: + logger.warning("assertoor fetch error for %s (attempt %d): %s", devnet, attempt, exc) + if attempt == 0: + time.sleep(1) + return None + + +def collect_assertoor(devnet: str) -> None: + """ + Fetch Assertoor test runs and upsert into assertoor_runs. + Degrades gracefully if the API is absent or returns 404. + """ + runs = _fetch_test_runs(devnet) + if runs is None: + print(f"collect_assertoor({devnet}): API unavailable, skipped (link-out only)") + return + + conn = connect() + migrate(conn) + + web_base = f"https://assertoor.{devnet}.ethpandaops.io" + inserted = 0 + + for r in runs: + run_id = r.get("run_id") + if run_id is None: + continue + row: dict[str, Any] = { + "devnet": devnet, + "run_id": int(run_id), + "test_id": str(r.get("test_id") or "")[:200], + "name": str(r.get("name") or "")[:300], + "status": str(r.get("status") or "")[:64], + "started_at": r.get("start_time"), + "stopped_at": r.get("stop_time"), + "web_url": f"{web_base}", + } + upsert(conn, "assertoor_runs", row) + inserted += 1 + + conn.commit() + conn.close() + print(f"collect_assertoor({devnet}): {inserted} run rows upserted") + + +def get_assertoor_data(devnet: str) -> dict[str, Any] | None: + """ + Return assertoor data for template rendering. + Returns None if no data is available (link-out-only mode). + """ + conn = connect() + migrate(conn) + + cnt = conn.execute( + "SELECT COUNT(*) AS c FROM assertoor_runs WHERE devnet=?", (devnet,) + ).fetchone() + if not cnt or cnt["c"] == 0: + conn.close() + return None + + rows = conn.execute( + """ + SELECT run_id, test_id, name, status, started_at, stopped_at, web_url + FROM assertoor_runs + WHERE devnet=? + ORDER BY run_id DESC + """, + (devnet,), + ).fetchall() + conn.close() + + def _fmt(ts: int | None) -> str: + if ts is None: + return "" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + except Exception: + return str(ts) + + result = [] + for r in rows: + result.append({ + "run_id": r["run_id"], + "test_id": r["test_id"] or "", + "name": r["name"] or "", + "status": r["status"] or "", + "started_at": _fmt(r["started_at"]), + "stopped_at": _fmt(r["stopped_at"]), + "web_url": r["web_url"] or f"https://assertoor.{devnet}.ethpandaops.io", + }) + + assertoor_url = f"https://assertoor.{devnet}.ethpandaops.io" + return { + "runs": result, + "assertoor_url": assertoor_url, + } + + +def show_assertoor(devnet: str) -> None: + """Print Assertoor test run summary to stdout.""" + data = get_assertoor_data(devnet) + if data is None: + print(f"assertoor({devnet}): no data. Run: dv collect {devnet} assertoor") + print(f" Link: https://assertoor.{devnet}.ethpandaops.io") + return + + runs = data["runs"] + print(f"\nAssertoor test runs for {devnet} ({len(runs)} total)\n") + print(f" {'ID':>5} {'STATUS':<10} {'STARTED':<20} NAME") + print(" " + "-" * 80) + for r in runs: + status = r["status"] + started = r["started_at"][:16] if r["started_at"] else "-" + name = r["name"][:50] + print(f" {r['run_id']:>5} {status:<10} {started:<20} {name}") + + print() + print(f" Assertoor UI: {data['assertoor_url']}") + print() + + +# Note: the `detect_assertoor_fail` detector lives in detect.py (queries the +# assertoor_runs table directly) so all detectors register uniformly via +# @_register without cross-module import-order fragility. diff --git a/devnets_monitor/devnets/bal.py b/devnets_monitor/devnets/bal.py new file mode 100644 index 0000000..178cb9c --- /dev/null +++ b/devnets_monitor/devnets/bal.py @@ -0,0 +1,222 @@ +"""BAL (Block-Level Access List, EIP-7928) inspector. + +Fetches /v1/slot/{slot}/block_access_list for ethrex-built canonical slots +and stores the access_count in the bal_inspect table. +""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime, timezone +from typing import Any + +from .config import devnet_entry +from .dora import _get_with_backoff +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + +_DEFAULT_LIMIT = 50 + + +def _max_bal_slot(conn: Any, devnet: str) -> int | None: + """Return the highest slot already in bal_inspect for this devnet.""" + row = conn.execute( + "SELECT MAX(slot) FROM bal_inspect WHERE devnet = ?", (devnet,) + ).fetchone() + if row and row[0] is not None: + return int(row[0]) + return None + + +def collect_bal(devnet: str, limit: int = _DEFAULT_LIMIT) -> None: + """ + Collect BAL data for recent ethrex-built canonical slots. + + Strategy: query the local slots table for ethrex proposer slots above the + current watermark (max bal slot), then fetch + /v1/slot/{slot}/block_access_list for each one not yet stored. + Uses per-fetch try/except so one failure does not abort the chain. + """ + entry = devnet_entry(devnet) + dora_base = entry.get("dora_base", "").rstrip("/") + if not dora_base: + logger.error("collect_bal: dora_base missing for devnet %s", devnet) + return + + conn = connect() + migrate(conn) + + watermark = _max_bal_slot(conn, devnet) + + # Find ethrex canonical slots above the watermark from the local slots table. + # proposer_name contains "ethrex" for ethrex-built slots. + query = """ + SELECT slot, proposer_name, eth_block_number + FROM slots + WHERE devnet = ? + AND status = 'Canonical' + AND proposer_name LIKE '%ethrex%' + """ + params: list[Any] = [devnet] + if watermark is not None: + query += " AND slot > ?" + params.append(watermark) + query += " ORDER BY slot DESC LIMIT ?" + params.append(limit) + + candidate_rows = conn.execute(query, params).fetchall() + + if not candidate_rows: + print(f"collect_bal({devnet}): no new ethrex canonical slots above watermark") + conn.close() + return + + fetched = 0 + errors = 0 + now = int(time.time()) + + for row in candidate_rows: + slot = row["slot"] + proposer_name = row["proposer_name"] or "" + eth_block_number = row["eth_block_number"] + + url = f"{dora_base}/api/v1/slot/{slot}/block_access_list" + try: + resp = _get_with_backoff(url, {}) + if resp is None: + logger.warning("collect_bal: no response for slot %d", slot) + errors += 1 + continue + + if resp.status_code != 200: + logger.warning( + "collect_bal: HTTP %d for slot %d", resp.status_code, slot + ) + errors += 1 + continue + + data = resp.json().get("data", {}) + block_root = data.get("block_root", "") + access_count = data.get("count", 0) or 0 + + bal_row: dict[str, Any] = { + "devnet": devnet, + "slot": slot, + "block_root": block_root, + "proposer_name": proposer_name, + "access_count": access_count, + "eth_block_number": eth_block_number, + "fetched_at": now, + } + upsert(conn, "bal_inspect", bal_row) + fetched += 1 + + except Exception as exc: + logger.warning("collect_bal: error on slot %d: %s", slot, exc) + errors += 1 + continue + + conn.commit() + conn.close() + print( + f"collect_bal({devnet}): {fetched} slots fetched, " + f"{errors} errors, watermark was {watermark}" + ) + + +def get_bal_data(devnet: str, limit: int = 200) -> dict[str, Any] | None: + """ + Return BAL inspection data for dashboard rendering. + + Keys: + rows list[dict] recent bal_inspect rows (newest first) + access_counts list[int] distribution of access_count values + zero_count int slots with access_count == 0 + total int total rows + """ + conn = connect() + migrate(conn) + + rows = conn.execute( + """ + SELECT slot, block_root, proposer_name, access_count, + eth_block_number, fetched_at + FROM bal_inspect + WHERE devnet = ? + ORDER BY slot DESC + LIMIT ? + """, + (devnet, limit), + ).fetchall() + conn.close() + + if not rows: + return None + + def _fmt(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + return str(ts) + + out_rows = [] + access_counts = [] + zero_count = 0 + for r in rows: + ac = r["access_count"] if r["access_count"] is not None else 0 + access_counts.append(ac) + if ac == 0: + zero_count += 1 + out_rows.append({ + "slot": r["slot"], + "block_root": r["block_root"] or "", + "block_root_short": (r["block_root"] or "")[:14] + "...", + "proposer_name": r["proposer_name"] or "", + "access_count": ac, + "eth_block_number": r["eth_block_number"], + "fetched_at": _fmt(r["fetched_at"]), + }) + + avg_ac = sum(access_counts) / len(access_counts) if access_counts else 0.0 + + return { + "rows": out_rows, + "access_counts": access_counts, + "zero_count": zero_count, + "total": len(out_rows), + "avg_access_count": round(avg_ac, 1), + } + + +def show_bal(devnet: str) -> None: + """Print BAL inspection summary to stdout.""" + data = get_bal_data(devnet) + if data is None: + print( + f"bal({devnet}): no BAL data. " + f"Run: dv collect {devnet} slow" + ) + return + + print(f"\nBAL inspection ({devnet})") + print(f"Total entries: {data['total']}, zero-access slots: {data['zero_count']}, " + f"avg access_count: {data['avg_access_count']}\n") + + print( + f"{'SLOT':>8} {'ETH_BLOCK':>10} {'ACCESS_COUNT':>13} " + f"{'PROPOSER':<30} {'FETCHED'}" + ) + print("-" * 90) + for r in data["rows"][:40]: + print( + f"{r['slot']:>8} {str(r['eth_block_number'] or ''):>10} " + f"{r['access_count']:>13} " + f"{r['proposer_name']:<30} {r['fetched_at']}" + ) + print() diff --git a/devnets_monitor/devnets/blobtrack.py b/devnets_monitor/devnets/blobtrack.py new file mode 100644 index 0000000..7c2e3db --- /dev/null +++ b/devnets_monitor/devnets/blobtrack.py @@ -0,0 +1,286 @@ +"""Blob inclusion analysis: per-proposer and ethrex-vs-others comparison.""" + +from __future__ import annotations + +import time +from typing import Any + +from .store import connect, migrate + + +# --------------------------------------------------------------------------- +# Data-returning helpers (used by the dashboard) +# --------------------------------------------------------------------------- + + +def get_blob_data( + devnet: str, + proposer: str | None = None, + since: str | None = None, +) -> dict[str, Any] | None: + """ + Return blob inclusion data as a dict suitable for template rendering. + + Returns None if there is no slot data. Dict keys: + window_label str + min_slot int + max_slot int + total_slots int + proposers list[dict] sorted by avg_blobs desc + keys: name, count, avg_blobs, sparkline + clients list[dict] sorted by avg_blobs desc + keys: client, slots, avg_blobs, total_blobs, is_ethrex + """ + conn = connect() + migrate(conn) + + slot_count = _resolve_since(since) + + base_query = "SELECT slot, proposer_name, blob_count, status FROM slots WHERE devnet = ?" + params: list[Any] = [devnet] + + if slot_count is not None: + max_row = conn.execute( + "SELECT MAX(slot) FROM slots WHERE devnet = ?", (devnet,) + ).fetchone() + if max_row and max_row[0] is not None: + cutoff = max_row[0] - slot_count + base_query += " AND slot >= ?" + params.append(cutoff) + + if proposer: + base_query += " AND proposer_name LIKE ?" + params.append(f"%{proposer}%") + + base_query += " ORDER BY slot ASC" + + rows = conn.execute(base_query, params).fetchall() + conn.close() + + if not rows: + return None + + proposer_data: dict[str, list[int]] = {} + for r in rows: + name = r["proposer_name"] or "unknown" + bc = r["blob_count"] if r["blob_count"] is not None else 0 + proposer_data.setdefault(name, []).append(bc) + + def mean(vals: list[int]) -> float: + return sum(vals) / len(vals) if vals else 0.0 + + sorted_proposers = sorted(proposer_data.items(), key=lambda kv: mean(kv[1]), reverse=True) + + total_slots = len(rows) + max_slot_num = max(r["slot"] for r in rows) + min_slot_num = min(r["slot"] for r in rows) + window_label = f"slots {min_slot_num}-{max_slot_num} ({total_slots} slots)" + + proposers_out = [] + for name, vals in sorted_proposers: + proposers_out.append({ + "name": name, + "count": len(vals), + "avg_blobs": round(mean(vals), 2), + "sparkline": _sparkline(vals), + }) + + client_data: dict[str, list[int]] = {} + for name, vals in proposer_data.items(): + client = _client_from_proposer(name) + client_data.setdefault(client, []).extend(vals) + + sorted_clients = sorted(client_data.items(), key=lambda kv: mean(kv[1]), reverse=True) + + clients_out = [] + for client, vals in sorted_clients: + clients_out.append({ + "client": client, + "slots": len(vals), + "avg_blobs": round(mean(vals), 2), + "total_blobs": sum(vals), + "is_ethrex": client == "ethrex", + }) + + # Per-slot blob counts grouped by client for the time-series chart + slot_series: dict[str, list[tuple[int, int]]] = {} + for r in rows: + name = r["proposer_name"] or "unknown" + client = _client_from_proposer(name) + bc = r["blob_count"] if r["blob_count"] is not None else 0 + slot_series.setdefault(client, []).append((r["slot"], bc)) + + return { + "window_label": window_label, + "min_slot": min_slot_num, + "max_slot": max_slot_num, + "total_slots": total_slots, + "proposers": proposers_out, + "clients": clients_out, + "slot_series": slot_series, + } + + +def _client_from_proposer(proposer_name: str) -> str: + """ + Extract EL client name from proposer_name by taking the SECOND dash-delimited + token. Examples: + lighthouse-ethrex-1 -> ethrex + grandine-erigon-1 -> erigon + teku-nethermind-2 -> nethermind + Falls back to the full name if the pattern does not match. + """ + parts = proposer_name.split("-") + if len(parts) >= 2: + return parts[1] + return proposer_name + + +def _sparkline(values: list[float]) -> str: + """Build an 8-character sparkline from a list of values using Unicode blocks.""" + blocks = " ▁▂▃▄▅▆▇█" + if not values: + return "" + lo = min(values) + hi = max(values) + span = hi - lo + result = [] + for v in values[-16:]: # Last 16 slots for the sparkline + if span == 0: + idx = 4 + else: + idx = int((v - lo) / span * 8) + idx = max(0, min(8, idx)) + result.append(blocks[idx]) + return "".join(result) + + +def _resolve_since(since: str | None) -> int | None: + """ + Parse --since as either a slot count (integer string) or a duration string + (e.g. '1h', '30m', '3600s'). Returns a slot count cutoff or None (no filter). + """ + if since is None: + return None + s = since.strip() + # Duration + if s.endswith("h"): + try: + return int(float(s[:-1]) * 225) # ~12 sec/slot + except ValueError: + pass + if s.endswith("m"): + try: + return int(float(s[:-1]) * 3.75) + except ValueError: + pass + if s.endswith("s"): + try: + return max(1, int(float(s[:-1]) / 12)) + except ValueError: + pass + # Plain integer = slot count + try: + return int(s) + except ValueError: + return None + + +def show_blobs( + devnet: str, + proposer: str | None = None, + since: str | None = None, +) -> None: + """ + Query the slots table and print: + 1. Per-proposer blob inclusion: recent avg blob_count and sparkline trend. + 2. Ethrex-vs-others: group proposer_name by EL client (second token), show + mean blob_count per client -- the blob-decay lens. + + Arguments: + proposer: filter to a single proposer_name substring + since: slot count or duration (e.g. '500', '2h', '30m') limiting the + window to the most recent N slots + """ + conn = connect() + migrate(conn) + + # Determine the slot window + slot_count = _resolve_since(since) + + base_query = "SELECT slot, proposer_name, blob_count, status FROM slots WHERE devnet = ?" + params: list[Any] = [devnet] + + if slot_count is not None: + max_row = conn.execute( + "SELECT MAX(slot) FROM slots WHERE devnet = ?", (devnet,) + ).fetchone() + if max_row and max_row[0] is not None: + cutoff = max_row[0] - slot_count + base_query += " AND slot >= ?" + params.append(cutoff) + + if proposer: + base_query += " AND proposer_name LIKE ?" + params.append(f"%{proposer}%") + + base_query += " ORDER BY slot ASC" + + rows = conn.execute(base_query, params).fetchall() + conn.close() + + if not rows: + print(f"blob({devnet}): no slot data. Run: dv collect {devnet} blobs") + return + + # Build per-proposer data + proposer_data: dict[str, list[int]] = {} + for r in rows: + name = r["proposer_name"] or "unknown" + bc = r["blob_count"] if r["blob_count"] is not None else 0 + proposer_data.setdefault(name, []).append(bc) + + # Sort by mean blob_count descending + def mean(vals: list[int]) -> float: + return sum(vals) / len(vals) if vals else 0.0 + + sorted_proposers = sorted(proposer_data.items(), key=lambda kv: mean(kv[1]), reverse=True) + + total_slots = len(rows) + max_slot_num = max(r["slot"] for r in rows) + min_slot_num = min(r["slot"] for r in rows) + window_label = f"slots {min_slot_num}-{max_slot_num} ({total_slots} slots)" + + print(f"\nBlob inclusion per proposer ({devnet})") + print(f"Window: {window_label}\n") + + col_w = max(len(p) for p, _ in sorted_proposers) + 2 if sorted_proposers else 20 + header = f"{'PROPOSER':<{col_w}} {'COUNT':>6} {'AVG_BLOBS':>10} SPARKLINE" + print(header) + print("-" * len(header)) + + for name, vals in sorted_proposers: + avg = mean(vals) + spark = _sparkline(vals) + print(f"{name:<{col_w}} {len(vals):>6} {avg:>10.2f} {spark}") + + # --- ethrex vs others --- + print(f"\nEL client comparison ({devnet})\n") + + client_data: dict[str, list[int]] = {} + for name, vals in proposer_data.items(): + client = _client_from_proposer(name) + client_data.setdefault(client, []).extend(vals) + + sorted_clients = sorted(client_data.items(), key=lambda kv: mean(kv[1]), reverse=True) + + print(f"{'CLIENT':<16} {'SLOTS':>7} {'AVG_BLOBS':>10} {'TOTAL_BLOBS':>12}") + print("-" * 50) + for client, vals in sorted_clients: + avg = mean(vals) + total = sum(vals) + marker = " <-- ethrex" if client == "ethrex" else "" + print( + f"{client:<16} {len(vals):>7} {avg:>10.2f} {total:>12}{marker}" + ) + print() diff --git a/devnets_monitor/devnets/cli.py b/devnets_monitor/devnets/cli.py new file mode 100644 index 0000000..8fa2ebb --- /dev/null +++ b/devnets_monitor/devnets/cli.py @@ -0,0 +1,416 @@ +""" +dv -- ethrex devnet ops CLI. + +Usage: uv run dv [devnet] [args] + +Devnet resolution priority: explicit arg > $DEVNET env > config/devnets.yaml default. +""" + +from __future__ import annotations + +import argparse +import re +import sys + + +_SINCE_RE = re.compile(r"^\d+[smh]$") + + +def _validate_since(value: str) -> str: + """Validate --since matches \\d+[smh]; reject otherwise.""" + if not _SINCE_RE.match(value): + raise argparse.ArgumentTypeError( + f"invalid --since value '{value}': must match \\d+[smh] (e.g. 2m, 30s, 1h)" + ) + return value + + +def _not_yet(name: str): + """Return a handler that prints 'not yet implemented (later phase)'.""" + def _handler(args: argparse.Namespace) -> None: + print(f"dv {name}: not yet implemented (later phase)") + sys.exit(0) + return _handler + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="dv", + description="ethrex devnet ops CLI. Read-only by default; 'wipe' is MUTATING.", + ) + sub = parser.add_subparsers(dest="command", metavar="subcommand") + sub.required = True + + # --- status --- + p_status = sub.add_parser( + "status", + help="per-node EL build/head/peers/sync/state@head + CL sync line + watchtower", + ) + p_status.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + p_status.add_argument("node", nargs="?", default=None, help="specific node name (default: all)") + p_status.add_argument("--json", dest="as_json", action="store_true", help="emit one JSON object per node") + + # --- peers --- + p_peers = sub.add_parser( + "peers", + help="peer count, inbound/outbound, client mix, body-serving failures", + ) + p_peers.add_argument("devnet", help="devnet name") + p_peers.add_argument("node", help="node name") + + # --- logs --- + p_logs = sub.add_parser( + "logs", + help="tail execution container WARN/ERROR lines", + ) + p_logs.add_argument("devnet", help="devnet name") + p_logs.add_argument("node", help="node name") + p_logs.add_argument( + "--since", + default="2m", + type=_validate_since, + help="docker logs --since value (default: 2m); must match \\d+[smh]", + ) + + # --- cl --- + p_cl = sub.add_parser( + "cl", + help="tail beacon sync lines", + ) + p_cl.add_argument("devnet", help="devnet name") + p_cl.add_argument("node", help="node name") + p_cl.add_argument( + "--since", + default="3m", + type=_validate_since, + help="docker logs --since value (default: 3m); must match \\d+[smh]", + ) + + # --- discover --- + p_discover = sub.add_parser( + "discover", + help="refresh config/devnets/.yaml roster/forks/image from the ethpandaops repo", + ) + p_discover.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- wipe (MUTATING) --- + p_wipe = sub.add_parser( + "wipe", + help="[MUTATING] recover a wedged EL node; requires --yes", + ) + p_wipe.add_argument("devnet", help="devnet name") + p_wipe.add_argument("node", help="node name") + p_wipe.add_argument( + "--yes", + action="store_true", + help="confirm the MUTATING wipe operation (required)", + ) + + # --- collect --- + p_collect = sub.add_parser( + "collect", + help="pull Dora/Hive/health/forks/network into SQLite", + ) + p_collect.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + p_collect.add_argument( + "what", + nargs="?", + default="all", + choices=[ + "blobs", "health", "hive", "forks", "network", "events", + "clients", "spamoor", "assertoor", + "bal", "epbs", "eiptrack", "deploygap", "slow", "all", + ], + help=( + "which collector to run (default: all); " + "'slow' runs bal + epbs + deploygap" + ), + ) + + # --- blob --- + p_blob = sub.add_parser( + "blob", + help="blob inclusion per proposer over time; ethrex vs others", + ) + p_blob.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + p_blob.add_argument("--proposer", default=None, help="filter to proposer_name substring") + p_blob.add_argument("--since", default=None, help="slot count or duration (e.g. 500, 2h, 30m)") + + # --- fork --- + p_fork = sub.add_parser( + "fork", + help="fork schedule with human times, blob target/max, EIP-per-fork, countdown", + ) + p_fork.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- hive --- + p_hive = sub.add_parser( + "hive", + help="summarize Hive group runs for the devnet", + ) + p_hive.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- events --- + p_events = sub.add_parser( + "events", + help="show detected events (anomalies, wedges, splits, etc.)", + ) + p_events.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + p_events.add_argument("--kind", default=None, help="filter by event kind") + p_events.add_argument("--severity", default=None, help="filter by severity (crit/warn/info)") + p_events.add_argument("--active", dest="active_only", action="store_true", help="show only active (unresolved) events") + p_events.add_argument("--all", dest="include_resolved", action="store_true", default=True, help="include resolved events (default)") + + # --- proposals --- + p_proposals = sub.add_parser( + "proposals", + help="per-proposer canonical/missed/orphaned slot summary", + ) + p_proposals.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + p_proposals.add_argument("--since", default=None, help="slot count or duration (e.g. 500, 2h, 30m)") + + # --- backfill --- + p_backfill = sub.add_parser( + "backfill", + help="range-collect slots [--from .. --to ..] into the slots table", + ) + p_backfill.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + p_backfill.add_argument("--from", dest="from_slot", type=int, required=True, metavar="SLOT", help="first slot (inclusive)") + p_backfill.add_argument("--to", dest="to_slot", type=int, required=True, metavar="SLOT", help="last slot (inclusive)") + + # --- bal --- + p_bal = sub.add_parser( + "bal", + help="show BAL (EIP-7928) access-count inspection for ethrex slots", + ) + p_bal.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- epbs --- + p_epbs = sub.add_parser( + "epbs", + help="show ePBS (EIP-7732) bid and PTC vote data per slot", + ) + p_epbs.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- eip-track --- + p_eiptrack = sub.add_parser( + "eip-track", + help="show EIP implementation-status summary for the devnet fork", + ) + p_eiptrack.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- clients --- + p_clients = sub.add_parser( + "clients", + help="EL+CL client diversity, ethrex versions live, fork agreement", + ) + p_clients.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- spamoor --- + p_spamoor = sub.add_parser( + "spamoor", + help="spamoor status: active spammers and blob load state", + ) + p_spamoor.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- deploy --- + p_deploy = sub.add_parser( + "deploy", + help="deploy timeline per node and GitHub gap vs ethrex main", + ) + p_deploy.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- exectime --- + p_exectime = sub.add_parser( + "exectime", + help="per-client execution-time comparison table and verdict", + ) + p_exectime.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- assertoor --- + p_assertoor = sub.add_parser( + "assertoor", + help="assertoor test run results", + ) + p_assertoor.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- eips-refresh --- + p_eips = sub.add_parser( + "eips-refresh", + help="regenerate config/eips.json from eipmcp data", + ) + p_eips.add_argument("devnet", nargs="?", default=None, help="devnet name (default: from config/env)") + + # --- serve --- + p_serve = sub.add_parser( + "serve", + help=( + "read-only FastAPI dashboard on 127.0.0.1 (localhost only, no auth, no write endpoints). " + "Requires `dv collect` to have been run first." + ), + ) + p_serve.add_argument( + "--port", + type=int, + default=8099, + metavar="N", + help="port to listen on (default: 8099)", + ) + p_serve.add_argument( + "--host", + default="127.0.0.1", + metavar="ADDR", + help="bind address (default: 127.0.0.1 -- localhost only)", + ) + + args = parser.parse_args() + + # Dispatch stubs + if hasattr(args, "_handler"): + args._handler(args) + return + + from .config import resolve_devnet + + if args.command == "status": + devnet = resolve_devnet(args.devnet) + from .status import status + status(devnet, args.node, args.as_json) + + elif args.command == "peers": + devnet = resolve_devnet(args.devnet) + from .peers import peers + peers(devnet, args.node) + + elif args.command == "logs": + devnet = resolve_devnet(args.devnet) + from .config import host_of + from .remote import LOGS_TAIL + from .ssh import run_remote + host = host_of(devnet, args.node) + result = run_remote(host, LOGS_TAIL, args=[args.since], timeout=30) + if result.stdout: + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr.strip(), file=sys.stderr) + sys.exit(result.returncode) + + elif args.command == "cl": + devnet = resolve_devnet(args.devnet) + from .config import host_of + from .remote import CL_TAIL + from .ssh import run_remote + host = host_of(devnet, args.node) + result = run_remote(host, CL_TAIL, args=[args.since], timeout=30) + if result.stdout: + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr.strip(), file=sys.stderr) + sys.exit(result.returncode) + + elif args.command == "discover": + devnet = resolve_devnet(args.devnet) + from .discover import discover + discover(devnet) + + elif args.command == "wipe": + devnet = resolve_devnet(args.devnet) + from .wipe import wipe + wipe(devnet, args.node, args.yes) + + elif args.command == "collect": + devnet = resolve_devnet(args.devnet) + from .collect import collect + collect(devnet, args.what) + + elif args.command == "blob": + devnet = resolve_devnet(args.devnet) + from .blobtrack import show_blobs + show_blobs(devnet, proposer=args.proposer, since=args.since) + + elif args.command == "fork": + devnet = resolve_devnet(args.devnet) + from .forkview import show_forks + show_forks(devnet) + + elif args.command == "hive": + devnet = resolve_devnet(args.devnet) + from .hive import show_hive + show_hive(devnet) + + elif args.command == "events": + devnet = resolve_devnet(args.devnet) + from .detect import show_events + show_events( + devnet, + kind=args.kind, + severity=args.severity, + active_only=args.active_only, + include_resolved=args.include_resolved, + ) + + elif args.command == "proposals": + devnet = resolve_devnet(args.devnet) + from .proposals import show_proposals + show_proposals(devnet, since=args.since) + + elif args.command == "backfill": + devnet = resolve_devnet(args.devnet) + from .dora import backfill + backfill(devnet, args.from_slot, args.to_slot) + + elif args.command == "bal": + devnet = resolve_devnet(args.devnet) + from .bal import show_bal + show_bal(devnet) + + elif args.command == "epbs": + devnet = resolve_devnet(args.devnet) + from .epbs import show_epbs + show_epbs(devnet) + + elif args.command == "eip-track": + devnet = resolve_devnet(args.devnet) + from .eiptrack import show_eiptrack + show_eiptrack(devnet) + + elif args.command == "clients": + devnet = resolve_devnet(args.devnet) + from .network import show_clients + show_clients(devnet) + + elif args.command == "spamoor": + devnet = resolve_devnet(args.devnet) + from .spamoor import show_spamoor + show_spamoor(devnet) + + elif args.command == "deploy": + devnet = resolve_devnet(args.devnet) + from .deploytl import show_deploy + show_deploy(devnet) + + elif args.command == "exectime": + devnet = resolve_devnet(args.devnet) + from .exectime import show_exectime + show_exectime(devnet) + + elif args.command == "assertoor": + devnet = resolve_devnet(args.devnet) + from .assertoor import show_assertoor + show_assertoor(devnet) + + elif args.command == "eips-refresh": + devnet = resolve_devnet(args.devnet) + from .collect import collect + collect(devnet, "forks") + + elif args.command == "serve": + import uvicorn + host = args.host + port = args.port + print(f"Dashboard: http://{host}:{port}/ (read-only, localhost-only, no auth)") + uvicorn.run("web.app:app", host=host, port=port) + + +if __name__ == "__main__": + main() diff --git a/devnets_monitor/devnets/collect.py b/devnets_monitor/devnets/collect.py new file mode 100644 index 0000000..f230e64 --- /dev/null +++ b/devnets_monitor/devnets/collect.py @@ -0,0 +1,94 @@ +"""Orchestrator for dv collect: runs data collectors in order.""" + +from __future__ import annotations + +import sys + +from .store import connect, migrate + +VALID_WHAT = ( + "blobs", "health", "hive", "forks", "network", "events", + "clients", "spamoor", "assertoor", + "bal", "epbs", "eiptrack", "deploygap", "slow", "all", +) + + +def collect(devnet: str, what: str) -> None: + """ + Run data collectors for the given devnet. + + what: one of blobs | health | hive | forks | network | events | + clients | spamoor | assertoor | + bal | epbs | eiptrack | deploygap | slow | all + + 'all' runs the fast set: forks, blobs, hive, health, network, + clients, spamoor, assertoor, then run_detectors last. + Excludes slow collectors (bal, epbs, deploygap). + 'slow' runs the expensive collectors: bal + epbs + deploygap. + 'eiptrack' reloads eips.json status into fork_eips; cheap, included in + 'forks' (and therefore in 'all'). + + The DB schema is always migrated before any collector runs. + """ + if what not in VALID_WHAT: + print( + f"collect: unknown target '{what}'. " + f"Valid: {', '.join(VALID_WHAT)}", + file=sys.stderr, + ) + sys.exit(1) + + # Ensure schema is up to date before any collector writes + conn = connect() + migrate(conn) + conn.close() + + if what in ("forks", "all", "eiptrack"): + from .forks import collect_forks + collect_forks(devnet) + + if what in ("blobs", "all"): + from .dora import collect_blobs + collect_blobs(devnet) + + if what in ("hive", "all"): + from .hive import collect_hive + collect_hive(devnet) + + if what in ("health", "all"): + from .health import collect_health + collect_health(devnet) + + if what in ("network", "all"): + from .network import collect_network + collect_network(devnet) + + if what in ("clients", "all"): + from .network import collect_clients + collect_clients(devnet) + + if what in ("spamoor", "all"): + from .spamoor import collect_spamoor + collect_spamoor(devnet) + + if what in ("assertoor", "all"): + from .assertoor import collect_assertoor + collect_assertoor(devnet) + + if what in ("bal", "slow"): + from .bal import collect_bal + collect_bal(devnet) + + if what in ("epbs", "slow"): + from .epbs import collect_epbs + collect_epbs(devnet) + + if what in ("deploygap", "slow"): + from .deploytl import collect_deploygap + collect_deploygap(devnet) + + # run_detectors last: reads whatever data exists (including bal/epbs if + # a prior 'slow' run populated those tables). + if what in ("events", "all"): + from .detect import run_detectors + run_detectors(devnet) diff --git a/devnets_monitor/devnets/config.py b/devnets_monitor/devnets/config.py new file mode 100644 index 0000000..58f98da --- /dev/null +++ b/devnets_monitor/devnets/config.py @@ -0,0 +1,190 @@ +"""Config loading: static registry (devnets.yaml) and per-devnet discovered cache.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Any + +import yaml + + +def repo_root() -> Path: + """Return the repo root: the directory containing config/ and devnets/.""" + # This file lives at /devnets/config.py + return Path(__file__).parent.parent.resolve() + + +def load_registry() -> dict[str, Any]: + """Parse config/devnets.yaml and return it as a dict.""" + path = repo_root() / "config" / "devnets.yaml" + if not path.exists(): + print(f"error: registry not found at {path}", file=sys.stderr) + sys.exit(1) + with path.open() as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + print("error: config/devnets.yaml is malformed", file=sys.stderr) + sys.exit(1) + return data + + +def resolve_devnet(name: str | None) -> str: + """ + Return the target devnet name. + Priority: explicit arg > DEVNET env var > registry default. + Exits with an error if nothing resolves. + """ + if name and name.strip(): + return name.strip() + env_val = os.environ.get("DEVNET", "").strip() + if env_val: + return env_val + registry = load_registry() + default = registry.get("default", "") + if default: + return str(default) + print( + "error: no devnet specified; pass a devnet name, set DEVNET, " + "or set 'default' in config/devnets.yaml", + file=sys.stderr, + ) + sys.exit(1) + + +def devnet_entry(name: str) -> dict[str, Any]: + """ + Return the registry entry for the given devnet name. + Exits with an error if the devnet is not in the registry. + """ + registry = load_registry() + devnets = registry.get("devnets", {}) + if name not in devnets: + known = ", ".join(devnets.keys()) if devnets else "(none)" + print( + f"error: devnet '{name}' not found in config/devnets.yaml. " + f"Known devnets: {known}", + file=sys.stderr, + ) + sys.exit(1) + return dict(devnets[name]) + + +def load_cache(name: str) -> dict[str, Any] | None: + """ + Parse config/devnets/.yaml and return it as a dict, or None if absent. + """ + path = repo_root() / "config" / "devnets" / f"{name}.yaml" + if not path.exists(): + return None + with path.open() as f: + data = yaml.safe_load(f) + return data if isinstance(data, dict) else None + + +def write_cache(name: str, data: dict[str, Any]) -> None: + """Dump data to config/devnets/.yaml (YAML format).""" + dir_path = repo_root() / "config" / "devnets" + dir_path.mkdir(parents=True, exist_ok=True) + path = dir_path / f"{name}.yaml" + header = ( + f"# Auto-generated by: dv discover {name}\n" + f"# Regenerate at any time with: dv discover {name}\n\n" + ) + with path.open("w") as f: + f.write(header) + yaml.dump(data, f, default_flow_style=False, allow_unicode=True, sort_keys=False) + + +def roster(name: str) -> list[str]: + """ + Return the list of node names for the given devnet. + + Sources (in priority order): + 1. cache roster[].name entries + 2. 'roster' list in the registry entry (if present) + 3. Error telling the user to run 'dv discover ' + """ + cache = load_cache(name) + if cache and cache.get("roster"): + nodes = cache["roster"] + result = [] + for entry in nodes: + if isinstance(entry, dict): + n = entry.get("name", "") + else: + n = str(entry) + if n: + result.append(n) + if result: + return result + + # Fallback: registry entry may have a 'roster' list + try: + entry = devnet_entry(name) + reg_roster = entry.get("roster", []) + if reg_roster: + return [str(n) for n in reg_roster] + except SystemExit: + pass + + print( + f"error: no roster found for '{name}'. " + f"Run 'dv discover {name}' to populate config/devnets/{name}.yaml", + file=sys.stderr, + ) + sys.exit(1) + + +def node_list(name: str, node_arg: str | None) -> list[str]: + """ + Return [node_arg] if a specific node is given, else the full roster (for all/None). + """ + if node_arg and node_arg.lower() != "all": + return [node_arg] + return roster(name) + + +# Default ethpandaops SSH host pattern. Placeholders: {user}, {node}, {devnet}. +# Override per devnet via the `ssh_host_template` field in config/devnets.yaml. +DEFAULT_SSH_HOST_TEMPLATE = "{user}@{node}.srv.{devnet}.ethpandaops.io" +DEFAULT_SSH_USER = "devops" + + +def is_ethrex_node(conn: Any, devnet: str, node: str) -> bool: + """ + Return True if the node runs ethrex. + + Checks the latest node_health.image for that node (golden rule 4: prefer + the live image). Falls back to checking whether "ethrex" appears in the + node name when no health row is available. + """ + try: + row = conn.execute( + """SELECT image FROM node_health + WHERE devnet=? AND node=? + ORDER BY ts DESC LIMIT 1""", + (devnet, node), + ).fetchone() + if row and row["image"]: + return "ethrex" in row["image"].lower() + except Exception: + pass + return "ethrex" in node.lower() + + +def host_of(name: str, node: str) -> str: + """ + Return the SSH host string for a node in a devnet. + + Uses the per-devnet `ssh_host_template` / `ssh_user` from config/devnets.yaml + if present, else the ethpandaops default `devops@.srv..ethpandaops.io`. + """ + try: + entry = devnet_entry(name) + except Exception: + entry = {} + template = entry.get("ssh_host_template", DEFAULT_SSH_HOST_TEMPLATE) + user = entry.get("ssh_user", DEFAULT_SSH_USER) + return template.format(user=user, node=node, devnet=name) diff --git a/devnets_monitor/devnets/deploytl.py b/devnets_monitor/devnets/deploytl.py new file mode 100644 index 0000000..41b65e7 --- /dev/null +++ b/devnets_monitor/devnets/deploytl.py @@ -0,0 +1,340 @@ +"""Deploy timeline + GitHub gap collector.""" + +from __future__ import annotations + +import logging +import subprocess +import time +from datetime import datetime, timezone +from typing import Any + +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + +# Cache TTL for the GitHub gap check: 6 hours +_GH_CACHE_TTL = 6 * 3600 + + +def _gh_api_json(path: str, jq: str | None = None) -> Any: + """ + Run 'gh api ' (optionally with --jq) and return parsed JSON. + Returns None on error. + """ + cmd = ["gh", "api", path] + if jq: + cmd += ["--jq", jq] + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if result.returncode != 0: + logger.warning("gh api %s failed: %s", path, result.stderr.strip()[:200]) + return None + text = result.stdout.strip() + if not text: + return None + import json + if jq: + # jq output may be a bare value (int, string) or JSON + try: + return json.loads(text) + except Exception: + return text + return json.loads(text) + except FileNotFoundError: + logger.error("'gh' CLI not found; cannot query GitHub") + return None + except subprocess.TimeoutExpired: + logger.warning("gh api %s timed out", path) + return None + except Exception as exc: + logger.warning("gh api %s error: %s", path, exc) + return None + + +def collect_deploygap(devnet: str) -> None: + """ + For each node in node_health, compare the deployed commit against + lambdaclass/ethrex main HEAD and store the gap into deploy_gap. + + Uses a gh_cache entry (key='ethrex_main_commit') with a 6h TTL to avoid + hammering the GitHub API on every slow run. + + Only re-queries gh if the cache is absent or older than 6h. + """ + conn = connect() + migrate(conn) + now = int(time.time()) + + # Fetch latest deployed commit per node from node_health + node_commits = conn.execute( + """ + SELECT nh.node, nh."commit" + FROM node_health nh + INNER JOIN ( + SELECT node, MAX(ts) AS max_ts + FROM node_health + WHERE devnet = ? + GROUP BY node + ) latest ON nh.node = latest.node AND nh.ts = latest.max_ts + WHERE nh.devnet = ? + """, + (devnet, devnet), + ).fetchall() + + if not node_commits: + conn.close() + print(f"collect_deploygap({devnet}): no node_health data; run dv collect {devnet} health first") + return + + # Check / refresh the main HEAD commit from cache + cache_row = conn.execute( + "SELECT value, fetched_at FROM gh_cache WHERE key='ethrex_main_commit'", + ).fetchone() + + main_commit: str | None = None + if cache_row and cache_row["fetched_at"] and (now - cache_row["fetched_at"]) < _GH_CACHE_TTL: + main_commit = cache_row["value"] + logger.info("collect_deploygap: using cached main commit %s", main_commit) + else: + # Fetch from GitHub + sha = _gh_api_json("repos/lambdaclass/ethrex/commits/main", jq=".sha") + if sha and isinstance(sha, str): + main_commit = sha.strip() + upsert(conn, "gh_cache", {"key": "ethrex_main_commit", "value": main_commit, "fetched_at": now}) + conn.commit() + logger.info("collect_deploygap: fetched main commit %s", main_commit) + else: + logger.warning("collect_deploygap: could not fetch main commit from GitHub") + + inserted = 0 + for nc in node_commits: + node = nc["node"] + deployed = (nc["commit"] or "").strip() + if not deployed: + continue + + commits_behind: int | None = None + if main_commit and deployed != main_commit: + # Only query compare if not cache-TTL blocked + # Check if we have a recent deploy_gap entry for this node + existing = conn.execute( + "SELECT commits_behind, checked_at FROM deploy_gap WHERE devnet=? AND node=?", + (devnet, node), + ).fetchone() + if ( + existing + and existing["checked_at"] + and (now - existing["checked_at"]) < _GH_CACHE_TTL + and existing["commits_behind"] is not None + ): + # Use cached gap value + commits_behind = existing["commits_behind"] + else: + result = _gh_api_json( + f"repos/lambdaclass/ethrex/compare/{deployed}...main", + jq=".ahead_by", + ) + if result is not None: + try: + commits_behind = int(result) + except (ValueError, TypeError): + pass + elif main_commit and deployed == main_commit: + commits_behind = 0 + + row: dict[str, Any] = { + "devnet": devnet, + "node": node, + "deployed_commit": deployed or None, + "main_commit": main_commit, + "commits_behind": commits_behind, + "checked_at": now, + } + upsert(conn, "deploy_gap", row) + inserted += 1 + + conn.commit() + conn.close() + print(f"collect_deploygap({devnet}): {inserted} nodes processed at ts={now}") + + +def get_deploy_data(devnet: str) -> dict[str, Any] | None: + """ + Return deploy timeline + gap data for template rendering. + + Timeline: per-node version-over-time from existing node_health rows + (groups by node, shows commit/buildnum/image changes over ts). + Gap: current deploy_gap table rows. + Events: events table rows for this devnet ordered by ts, overlaid. + + Returns None if no node_health data exists. + """ + conn = connect() + migrate(conn) + + # Check data exists + cnt = conn.execute( + "SELECT COUNT(*) AS c FROM node_health WHERE devnet=?", (devnet,) + ).fetchone() + if not cnt or cnt["c"] == 0: + conn.close() + return None + + # --- Timeline: per-node rows ordered by ts --- + # We return all node_health rows; the template groups by node and shows + # only rows where commit/buildnum/image changed (version transitions). + timeline_rows = conn.execute( + """ + SELECT node, ts, image, buildnum, "commit", head, peers + FROM node_health + WHERE devnet=? + ORDER BY node, ts DESC + """, + (devnet,), + ).fetchall() + + # Build per-node version transitions (collapse consecutive identical + # versions). Rows arrive newest-first (ts DESC), so each node's list stays + # newest-first by appending and comparing against the last appended entry. + from datetime import datetime, timezone + + def _ts_str(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + except Exception: + return str(ts) + + nodes: dict[str, list[dict]] = {} + for r in timeline_rows: + node = r["node"] + entry = { + "ts": r["ts"], + "ts_str": _ts_str(r["ts"]), + "image": r["image"], + "buildnum": r["buildnum"], + "commit": (r["commit"] or "")[:12], + "commit_full": r["commit"] or "", + "head": r["head"], + "peers": r["peers"], + } + if node not in nodes: + nodes[node] = [entry] + else: + last = nodes[node][-1] # oldest-so-far == previous (older) version + if ( + entry["commit_full"] != last["commit_full"] + or entry["buildnum"] != last["buildnum"] + or entry["image"] != last["image"] + ): + nodes[node].append(entry) + + # --- Deploy gap --- + gap_rows = conn.execute( + """ + SELECT node, deployed_commit, main_commit, commits_behind, checked_at + FROM deploy_gap + WHERE devnet=? + ORDER BY node + """, + (devnet,), + ).fetchall() + + gap_data = [] + for r in gap_rows: + checked_str = "" + if r["checked_at"]: + try: + checked_str = datetime.fromtimestamp(r["checked_at"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + checked_str = str(r["checked_at"]) + gap_data.append({ + "node": r["node"], + "deployed_commit": (r["deployed_commit"] or "")[:12], + "deployed_commit_full": r["deployed_commit"] or "", + "main_commit": (r["main_commit"] or "")[:12], + "commits_behind": r["commits_behind"], + "checked_at": checked_str, + }) + + # --- Events overlay (last 50 events for this devnet, ordered by last_seen) --- + event_rows = conn.execute( + """ + SELECT kind, severity, node, message, first_seen, last_seen + FROM events + WHERE devnet=? + ORDER BY last_seen DESC + LIMIT 50 + """, + (devnet,), + ).fetchall() + + events = [] + for e in event_rows: + ts_str = "" + if e["last_seen"]: + try: + ts_str = datetime.fromtimestamp(e["last_seen"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + ts_str = str(e["last_seen"]) + events.append({ + "kind": e["kind"], + "severity": e["severity"], + "node": e["node"] or "", + "message": e["message"], + "ts_str": ts_str, + }) + + conn.close() + + return { + "nodes": nodes, + "gap": gap_data, + "events": events, + "gh_repo": "https://github.com/lambdaclass/ethrex", + } + + +def show_deploy(devnet: str) -> None: + """Print deploy timeline and GitHub gap summary to stdout.""" + data = get_deploy_data(devnet) + if data is None: + print(f"deploy({devnet}): no data. Run: dv collect {devnet} health") + return + + print(f"\nDeploy timeline for {devnet}\n") + + for node, versions in sorted(data["nodes"].items()): + print(f" {node}:") + for v in versions[:5]: + ts_str = "" + if v["ts"]: + try: + ts_str = datetime.fromtimestamp(v["ts"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M" + ) + except Exception: + ts_str = str(v["ts"]) + print( + f" [{ts_str}] commit={v['commit']} build={v.get('buildnum') or '?'}" + ) + + if data["gap"]: + print("\nGitHub gap (deployed vs main):\n") + print(f" {'NODE':<36} {'DEPLOYED':>12} {'BEHIND':>8} {'CHECKED'}") + print(" " + "-" * 72) + for g in data["gap"]: + behind = g["commits_behind"] + behind_str = str(behind) if behind is not None else "?" + print( + f" {g['node']:<36} {g['deployed_commit']:>12} {behind_str:>8} {g['checked_at']}" + ) + else: + print("\nGitHub gap: no data. Run: dv collect slow") + + print() diff --git a/devnets_monitor/devnets/detect.py b/devnets_monitor/devnets/detect.py new file mode 100644 index 0000000..b6b31e6 --- /dev/null +++ b/devnets_monitor/devnets/detect.py @@ -0,0 +1,1113 @@ +"""Event detection backbone: record, resolve, and run detectors.""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Callable + +from .store import connect, migrate + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Dedup key helper +# --------------------------------------------------------------------------- + + +def _dedup_key(kind: str, node: str | None, discriminator: str) -> str: + return f"{kind}:{node or '-'}:{discriminator}" + + +# --------------------------------------------------------------------------- +# record_event / resolve_stale +# --------------------------------------------------------------------------- + + +def record_event( + conn: Any, + devnet: str, + kind: str, + severity: str, + node: str | None, + message: str, + details: dict, + now: int, + discriminator: str = "", +) -> str: + """ + Upsert an event row using (devnet, dedup_key) as the identity. + + - Absent: INSERT with first_seen=last_seen=now, count=1. + - Present, active (resolved_at IS NULL): UPDATE last_seen, count+1, message, details. + - Present, resolved: REOPEN (resolved_at=NULL, first_seen=now, count=1). + + Returns the dedup_key. + """ + key = _dedup_key(kind, node, discriminator) + details_json = json.dumps(details) + + row = conn.execute( + "SELECT count, resolved_at FROM events WHERE devnet=? AND dedup_key=?", + (devnet, key), + ).fetchone() + + if row is None: + conn.execute( + """INSERT INTO events + (devnet, dedup_key, kind, severity, node, message, details, + first_seen, last_seen, resolved_at, count) + VALUES (?,?,?,?,?,?,?,?,?,NULL,1)""", + (devnet, key, kind, severity, node, message, details_json, now, now), + ) + elif row["resolved_at"] is not None: + # Reopen: treat as a fresh occurrence + conn.execute( + """UPDATE events SET resolved_at=NULL, first_seen=?, last_seen=?, + count=1, severity=?, message=?, details=? + WHERE devnet=? AND dedup_key=?""", + (now, now, severity, message, details_json, devnet, key), + ) + else: + conn.execute( + """UPDATE events SET last_seen=?, count=count+1, severity=?, + message=?, details=? + WHERE devnet=? AND dedup_key=?""", + (now, severity, message, details_json, devnet, key), + ) + return key + + +def resolve_stale(conn: Any, devnet: str, seen_keys: set, now: int) -> None: + """ + Resolve all active events for this devnet whose dedup_key is NOT in seen_keys. + Only touches rows for this specific devnet. + """ + if not seen_keys: + # Resolve everything active for this devnet + conn.execute( + "UPDATE events SET resolved_at=? WHERE devnet=? AND resolved_at IS NULL", + (now, devnet), + ) + return + + placeholders = ",".join("?" for _ in seen_keys) + conn.execute( + f"UPDATE events SET resolved_at=? WHERE devnet=? AND resolved_at IS NULL " + f"AND dedup_key NOT IN ({placeholders})", + (now, devnet, *seen_keys), + ) + + +# --------------------------------------------------------------------------- +# Detector registry +# --------------------------------------------------------------------------- + +# Each detector: fn(conn, devnet, now, seen) -> None +# Detectors call record_event and add each returned key to seen. +DETECTORS: list[Callable] = [] + + +def _register(fn: Callable) -> Callable: + DETECTORS.append(fn) + return fn + + +# --------------------------------------------------------------------------- +# run_detectors +# --------------------------------------------------------------------------- + + +def run_detectors(devnet: str) -> None: + """Open DB, run every registered detector, resolve stale events, commit.""" + conn = connect() + migrate(conn) + now = int(time.time()) + seen: set[str] = set() + + errors = 0 + for fn in DETECTORS: + try: + fn(conn, devnet, now, seen) + except Exception as exc: + logger.warning("detector %s failed for %s: %s", fn.__name__, devnet, exc) + errors += 1 + + # If every detector crashed (e.g. schema/import problem), do NOT resolve + # active events -- an empty `seen` would otherwise wipe legitimate events. + if DETECTORS and errors == len(DETECTORS): + logger.error("all detectors failed for %s; skipping resolve_stale", devnet) + conn.commit() + conn.close() + print(f"run_detectors({devnet}): ALL {errors} detectors failed; events untouched") + return + + resolve_stale(conn, devnet, seen, now) + conn.commit() + conn.close() + print( + f"run_detectors({devnet}): {len(DETECTORS)} detectors, " + f"{len(seen)} active events" + ) + + +# --------------------------------------------------------------------------- +# get_events_data / show_events (CLI + web seam) +# --------------------------------------------------------------------------- + + +def get_events_data( + devnet: str, + kind: str | None = None, + severity: str | None = None, + active_only: bool = False, + include_resolved: bool = True, +) -> list[dict[str, Any]]: + """ + Query events for a devnet. Returns a list of dicts ordered by: + active first (severity crit > warn > info), then last_seen desc; resolved after. + + Annotates rows where count>2 AND last_seen-first_seen < 7200 as 'flapping'. + """ + conn = connect() + migrate(conn) + + clauses = ["devnet = ?"] + params: list[Any] = [devnet] + + if kind: + clauses.append("kind = ?") + params.append(kind) + if severity: + clauses.append("severity = ?") + params.append(severity) + if active_only: + clauses.append("resolved_at IS NULL") + elif not include_resolved: + clauses.append("resolved_at IS NULL") + + where = " AND ".join(clauses) + rows = conn.execute( + f"SELECT * FROM events WHERE {where} ORDER BY last_seen DESC", + params, + ).fetchall() + conn.close() + + _sev_order = {"crit": 0, "warn": 1, "info": 2} + + from datetime import datetime, timezone + + def _fmt(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + except Exception: + return str(ts) + + result = [] + for r in rows: + det = {} + if r["details"]: + try: + det = json.loads(r["details"]) + except Exception: + det = {"raw": r["details"]} + # Flapping = many re-observations in a short span (genuine churn), not just + # a steadily-persistent condition. At a 15-min cadence, >5 counts in <1h + # means it has been resolving and reopening. + flapping = ( + (r["count"] or 0) > 5 + and (r["last_seen"] - r["first_seen"]) < 3600 + ) + result.append({ + "devnet": r["devnet"], + "dedup_key": r["dedup_key"], + "kind": r["kind"], + "severity": r["severity"], + "node": r["node"], + "message": r["message"], + "details": det, + "first_seen": r["first_seen"], + "last_seen": r["last_seen"], + "first_seen_str": _fmt(r["first_seen"]), + "last_seen_str": _fmt(r["last_seen"]), + "resolved_at": r["resolved_at"], + "resolved_at_str": _fmt(r["resolved_at"]), + "count": r["count"], + "active": r["resolved_at"] is None, + "flapping": flapping, + }) + + # Sort: active first by severity then last_seen desc; resolved after by last_seen desc + def _sort_key(e: dict) -> tuple: + active_rank = 0 if e["active"] else 1 + sev_rank = _sev_order.get(e["severity"], 9) + return (active_rank, sev_rank, -e["last_seen"]) + + result.sort(key=_sort_key) + return result + + +def show_events( + devnet: str, + kind: str | None = None, + severity: str | None = None, + active_only: bool = False, + include_resolved: bool = True, +) -> None: + """Print events table to stdout.""" + from datetime import datetime, timezone + + def _ts(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + except Exception: + return str(ts) + + rows = get_events_data( + devnet, + kind=kind, + severity=severity, + active_only=active_only, + include_resolved=include_resolved, + ) + if not rows: + print(f"events({devnet}): no events found") + return + + print(f"\nEvents for {devnet} ({len(rows)} total)\n") + print( + f"{'SEV':<5} {'KIND':<18} {'NODE':<28} {'MSG':<45} {'COUNT':>5} " + f"{'LAST SEEN':>16} {'STATUS':<12}" + ) + print("-" * 135) + for e in rows: + status = "ACTIVE" if e["active"] else "resolved" + if e["flapping"]: + status += " FLAPPING" + node_col = (e["node"] or "-")[:27] + msg_col = e["message"][:44] + print( + f"{e['severity']:<5} {e['kind']:<18} {node_col:<28} {msg_col:<45} " + f"{e['count']:>5} {_ts(e['last_seen']):>16} {status:<12}" + ) + print() + + +# --------------------------------------------------------------------------- +# Detector implementations. All detectors live in this module and register via +# @_register at import time, so DETECTORS is fully populated on import (no +# lazy-load hack, no double-registration risk, no cross-module import order). +# Detectors only query the SQLite tables, so they need no collector modules. +# --------------------------------------------------------------------------- + + +@_register +def detect_version_change(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Detect an ethrex version/commit change on a node (deploy event). + + Compares each ethrex node's two most recent DISTINCT commits in node_health. + Emits an info event for the latest transition, keyed by the new commit so the + current deploy stays active and older transitions resolve as new ones land. + """ + from .config import is_ethrex_node + + node_rows = conn.execute( + "SELECT DISTINCT node FROM node_health WHERE devnet=?", (devnet,) + ).fetchall() + + for nr in node_rows: + node = nr["node"] + if not is_ethrex_node(conn, devnet, node): + continue + # Distinct commit values in time order (newest first), ignoring rows with + # no commit (e.g. a probe-error snapshot). + rows = conn.execute( + """SELECT "commit", buildnum, ts FROM node_health + WHERE devnet=? AND node=? AND "commit" IS NOT NULL AND "commit" != '' + ORDER BY ts DESC LIMIT 40""", + (devnet, node), + ).fetchall() + # Collapse consecutive duplicates into distinct version points. + distinct: list[dict] = [] + for r in rows: + c = r["commit"] + if not distinct or distinct[-1]["commit"] != c: + distinct.append({"commit": c, "buildnum": r["buildnum"], "ts": r["ts"]}) + if len(distinct) < 2: + continue # only ever one version seen -> no change + new, old = distinct[0], distinct[1] + # Node is its own column; buildnums are in the message. Keep details to + # just the two commits (rendered as clickable GitHub links) so the row + # isn't a verbatim restatement of the message. + details = { + "from_commit": (old["commit"] or "")[:12], + "to_commit": (new["commit"] or "")[:12], + } + key = record_event( + conn, devnet, + kind="version_change", + severity="info", + node=node, + message=( + f"ethrex upgraded {(old['commit'] or '')[:8]} -> " + f"{(new['commit'] or '')[:8]} (bn {old['buildnum']} -> {new['buildnum']})" + ), + details=details, + now=now, + discriminator=(new["commit"] or "")[:12], + ) + seen.add(key) + + +@_register +def detect_assertoor_fail(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Detect failed Assertoor test runs (warn), one event per failed run_id. + + Only considers runs that stopped within the last 3 days so old failures + (or pre-reset runs) age out and resolve instead of alerting forever. + """ + rows = conn.execute( + """ + SELECT run_id, name, test_id, status, started_at, stopped_at + FROM assertoor_runs + WHERE devnet=? AND status='failure' + AND (stopped_at IS NULL OR stopped_at >= ?) + ORDER BY run_id DESC + LIMIT 20 + """, + (devnet, now - 3 * 86400), + ).fetchall() + + for r in rows: + details = { + "run_id": r["run_id"], + "test_id": r["test_id"] or "", + "started_at": r["started_at"], + } + key = record_event( + conn, devnet, + kind="assertoor_fail", + severity="warn", + node=None, + message=f"Assertoor test failed: {r['name'] or r['test_id']} (run_id={r['run_id']})", + details=details, + now=now, + discriminator=str(r["run_id"]), + ) + seen.add(key) + + +# --------------------------------------------------------------------------- +# Phase 1 detectors (registered at module import via @_register) +# --------------------------------------------------------------------------- + + +@_register +def detect_chain_split(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Detect ethrex nodes on a minority head root or different fork than majority. + Requires >= 3 reporting nodes to avoid false positives on tiny devnets. + """ + from .network import _latest_network_rows + + splits = _latest_network_rows(conn, devnet) + if not splits: + return + + # Count total reporting nodes across all forks + total_nodes = sum(s.get("head_count", 0) or 0 for s in splits) + if total_nodes < 3: + return + + # Majority head = strictly > 50% of reporting nodes + majority = None + for s in splits: + cnt = s.get("head_count", 0) or 0 + if cnt > total_nodes / 2: + majority = s + break + + if majority is None: + # No majority: fire crit for every ethrex node that is not on the biggest fork + majority = max(splits, key=lambda s: s.get("head_count", 0) or 0) + + majority_root = majority.get("head_root", "") + + from .config import is_ethrex_node + + # Each split row is one fork (head_root) carrying that fork's clients_json. + # An ethrex node listed in a NON-majority fork is on the wrong head -> fire. + for fork in splits: + fork_root = fork.get("head_root", "") or "" + if fork_root == majority_root: + continue + try: + clients = json.loads(fork.get("clients_json") or "[]") + except Exception: + continue + for client in clients: + node_name = client.get("name", "") or "" + if not is_ethrex_node(conn, devnet, node_name): + continue + details = { + "node": node_name, + "node_head_root": fork_root, + "node_head_slot": fork.get("head_slot"), + "majority_head_root": majority_root, + } + key = record_event( + conn, devnet, + kind="chain_split", + severity="crit", + node=node_name, + message=f"{node_name} on minority fork (head {fork_root[:12]}..., majority {majority_root[:12]}...)", + details=details, + now=now, + discriminator="minority_fork", + ) + seen.add(key) + + +@_register +def detect_orphan_spike(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Over last 256 slots, compare ethrex orphan rate vs other clients. + Fire warn if ethrex_rate > 2 * others_rate AND ethrex_rate > 0.05. + """ + from .blobtrack import _client_from_proposer + + max_row = conn.execute( + "SELECT MAX(slot) FROM slots WHERE devnet=?", (devnet,) + ).fetchone() + if not max_row or max_row[0] is None: + return + cutoff = max_row[0] - 256 + + rows = conn.execute( + "SELECT proposer_name, status FROM slots WHERE devnet=? AND slot>=?", + (devnet, cutoff), + ).fetchall() + if not rows: + return + + ethrex_total = ethrex_orphaned = 0 + other_total = other_orphaned = 0 + for r in rows: + client = _client_from_proposer(r["proposer_name"] or "unknown") + orphaned = (r["status"] or "").lower() == "orphaned" + if client == "ethrex": + ethrex_total += 1 + if orphaned: + ethrex_orphaned += 1 + else: + other_total += 1 + if orphaned: + other_orphaned += 1 + + if ethrex_total == 0: + return + + ethrex_rate = ethrex_orphaned / ethrex_total + others_rate = other_orphaned / other_total if other_total > 0 else 0.0 + + if ethrex_rate > 2 * others_rate and ethrex_rate > 0.05: + details = { + "ethrex_orphaned": ethrex_orphaned, + "ethrex_total": ethrex_total, + "ethrex_rate": round(ethrex_rate, 4), + "others_orphaned": other_orphaned, + "others_total": other_total, + "others_rate": round(others_rate, 4), + "window": "last 256 slots", + } + key = record_event( + conn, devnet, + kind="orphan_spike", + severity="warn", + node=None, + message=( + f"ethrex orphan rate {ethrex_rate:.1%} vs others " + f"{others_rate:.1%} over last 256 slots" + ), + details=details, + now=now, + discriminator="", + ) + seen.add(key) + + +@_register +def detect_wedge(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + For each ethrex node, compare two most recent node_health rows. + Fire crit on: head not advancing, state_at_head != 'yes', restart climbing, peers == 0. + """ + from .config import is_ethrex_node + + # Get all ethrex nodes with at least 2 health rows + node_rows = conn.execute( + "SELECT DISTINCT node FROM node_health WHERE devnet=?", (devnet,) + ).fetchall() + + for nr in node_rows: + node = nr["node"] + if not is_ethrex_node(conn, devnet, node): + continue + + rows = conn.execute( + """SELECT head, peers, state_at_head, restart, syncing + FROM node_health WHERE devnet=? AND node=? + ORDER BY ts DESC LIMIT 2""", + (devnet, node), + ).fetchall() + + if not rows: + continue + + latest = rows[0] + + # head not advancing (requires 2 rows) + if len(rows) == 2: + prev = rows[1] + if ( + latest["head"] is not None + and prev["head"] is not None + and latest["head"] == prev["head"] + and latest["head"] > 0 + ): + details = {"head": latest["head"], "node": node} + key = record_event( + conn, devnet, + kind="wedge", + severity="crit", + node=node, + message=f"{node}: head stuck at {latest['head']}", + details=details, + now=now, + discriminator="head_stuck", + ) + seen.add(key) + + # restart climbing + if ( + latest["restart"] is not None + and prev["restart"] is not None + and latest["restart"] > prev["restart"] + ): + details = { + "node": node, + "restart_prev": prev["restart"], + "restart_now": latest["restart"], + } + key = record_event( + conn, devnet, + kind="wedge", + severity="crit", + node=node, + message=f"{node}: restart count climbing ({prev['restart']} -> {latest['restart']})", + details=details, + now=now, + discriminator="restart_climb", + ) + seen.add(key) + + # state_at_head not "yes" -- but a node that is still syncing legitimately + # has no state at head yet, so only flag when NOT syncing. + state = (latest["state_at_head"] or "").lower() + syncing_raw = (latest["syncing"] or "").lower() + is_syncing = syncing_raw.startswith("cur=") or syncing_raw in ("true", "1", "yes") + if state and state != "yes" and not is_syncing: + details = {"node": node, "state_at_head": latest["state_at_head"]} + key = record_event( + conn, devnet, + kind="wedge", + severity="crit", + node=node, + message=f"{node}: state_at_head={latest['state_at_head']}", + details=details, + now=now, + discriminator="state_at_head", + ) + seen.add(key) + + # peers == 0 + if latest["peers"] is not None and latest["peers"] == 0: + details = {"node": node, "peers": 0} + key = record_event( + conn, devnet, + kind="wedge", + severity="crit", + node=node, + message=f"{node}: 0 peers", + details=details, + now=now, + discriminator="no_peers", + ) + seen.add(key) + + +@_register +def detect_node_unreachable(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Latest node_health row with a probe error (syncing field contains the error) -> crit. + Health stores probe errors in the syncing column (see health.py). + """ + from .config import is_ethrex_node + + rows = conn.execute( + """SELECT node, syncing, ts + FROM node_health + WHERE devnet=? AND ts = ( + SELECT MAX(ts) FROM node_health nh2 + WHERE nh2.devnet = node_health.devnet AND nh2.node = node_health.node + )""", + (devnet,), + ).fetchall() + + for r in rows: + node = r["node"] + if not is_ethrex_node(conn, devnet, node): + continue + syncing_val = r["syncing"] or "" + # Probe errors are stored verbatim in the syncing column by health.py. + # Reachable values produced by remote.py: + # "synced(false)" / "synced(true)" -- synced + # "cur=N->hi=M" -- actively syncing (reachable!) + # "unknown" -- rpc responded, odd shape (reachable) + # "true"/"false"/"yes"/"no"/"1"/"0" + # Real errors: "Command ..." (timeout), "ssh exit N: ..." + sv_lower = syncing_val.lower() + is_error = bool(syncing_val) and ( + sv_lower.startswith("command ") + or sv_lower.startswith("ssh exit") + ) + if is_error: + # Turn the raw subprocess/ssh error into a concise human message; + # keep the raw text in details for debugging. + if "timed out" in sv_lower: + msg = f"{node}: ssh probe timed out (node unreachable)" + elif sv_lower.startswith("ssh exit"): + # "ssh exit N: " -> keep the tail, trimmed + msg = f"{node}: ssh failed ({syncing_val.split(':', 1)[-1].strip()[:80] or 'connection error'})" + else: + msg = f"{node}: probe failed" + details = {"node": node, "raw_error": syncing_val[:300]} + key = record_event( + conn, devnet, + kind="node_unreachable", + severity="crit", + node=node, + message=msg, + details=details, + now=now, + discriminator="probe_error", + ) + seen.add(key) + + +@_register +def detect_blob_decay(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Ethrex proposer avg blob_count over a window trending to ~0 -> warn. + Uses last 64 slots as the window. + + Spamoor guard: if the latest spamoor snapshot shows no active blob spammer + (scenario contains 'blob' AND status==1), skip blob_decay and instead record + a blob_load_off info event explaining the gap. If spamoor data is absent + entirely, fall through to the existing relative check. + """ + from .blobtrack import _client_from_proposer + from .spamoor import _blob_spammer_active + + # --- Spamoor guard --- + blob_active = _blob_spammer_active(conn, devnet) + if blob_active is False: + # No blob spammer running; a 0-blob window is expected. Record info event. + details = {"reason": "no active blob spammer (spamoor status: stopped)"} + key = record_event( + conn, devnet, + kind="blob_load_off", + severity="info", + node=None, + message="blob load off: no active blob spammer (spamoor confirms)", + details=details, + now=now, + discriminator="", + ) + seen.add(key) + return + # blob_active is None: no spamoor data; fall through to relative check. + # blob_active is True: spammer is running; proceed with decay check. + + max_row = conn.execute( + "SELECT MAX(slot) FROM slots WHERE devnet=?", (devnet,) + ).fetchone() + if not max_row or max_row[0] is None: + return + cutoff = max_row[0] - 64 + + rows = conn.execute( + """SELECT proposer_name, blob_count FROM slots + WHERE devnet=? AND slot>=? AND status='Canonical'""", + (devnet, cutoff), + ).fetchall() + if not rows: + return + + ethrex_blobs: list[int] = [] + other_blobs: list[int] = [] + for r in rows: + client = _client_from_proposer(r["proposer_name"] or "unknown") + bc = r["blob_count"] if r["blob_count"] is not None else 0 + if client == "ethrex": + ethrex_blobs.append(bc) + else: + other_blobs.append(bc) + + # Need enough ethrex samples AND a peer baseline to make a relative call. + if len(ethrex_blobs) < 4 or len(other_blobs) < 4: + return + + ethrex_avg = sum(ethrex_blobs) / len(ethrex_blobs) + others_avg = sum(other_blobs) / len(other_blobs) + + # Decay is RELATIVE: only fire when peers are including blobs (so the spammer + # is clearly active) but ethrex is not. If everyone is ~0 (spammer off or a + # network-wide issue), this is NOT ethrex decay; stay quiet. + if others_avg >= 0.5 and ethrex_avg < 0.2 * others_avg: + details = { + "ethrex_avg_blobs": round(ethrex_avg, 3), + "others_avg_blobs": round(others_avg, 3), + "ethrex_slots": len(ethrex_blobs), + "other_slots": len(other_blobs), + "window": "last 64 canonical slots", + } + key = record_event( + conn, devnet, + kind="blob_decay", + severity="warn", + node=None, + message=( + f"ethrex avg blobs {ethrex_avg:.2f} vs peers {others_avg:.2f} " + f"(last 64 canonical slots)" + ), + details=details, + now=now, + discriminator="", + ) + seen.add(key) + + +@_register +def detect_bal_anomaly(conn: Any, devnet: str, now: int, seen: set) -> None: + """ + Detect a run of recent ethrex BAL entries where access_count == 0. + + A single slot with zero BAL entries can be a legitimately empty block, + so this fires only when 3 or more of the last 10 ethrex BAL entries have + access_count == 0. That avoids per-slot noise while still catching a + systematic BAL bug. Discriminator is fixed ("recent_zero_bal") so the + event updates in-place rather than spawning one event per slot. + """ + rows = conn.execute( + """ + SELECT slot, access_count + FROM bal_inspect + WHERE devnet = ? + ORDER BY slot DESC + LIMIT 10 + """, + (devnet,), + ).fetchall() + + # Require a meaningful sample before concluding anything; 3-of-3 on a brand + # new devnet is not signal. + if len(rows) < 5: + return + + zero_slots = [r["slot"] for r in rows if (r["access_count"] or 0) == 0] + total = len(rows) + + if len(zero_slots) < 3: + return # Threshold not met; stay quiet. + + details = { + "zero_access_slots": zero_slots, + "zero_count": len(zero_slots), + "window": f"last {total} ethrex BAL entries", + } + key = record_event( + conn, devnet, + kind="bal_anomaly", + severity="warn", + node=None, + message=( + f"{len(zero_slots)}/{total} recent ethrex slots have BAL access_count=0 " + f"(possible EIP-7928 bug)" + ), + details=details, + now=now, + discriminator="recent_zero_bal", + ) + seen.add(key) + + +# --------------------------------------------------------------------------- +# T3 detectors: exec_regression, finality_stall, head_lag +# --------------------------------------------------------------------------- + + +@_register +def detect_exec_regression(conn: Any, devnet: str, now: int, seen: set) -> None: + """Detect ethrex execution-time regression vs peers and vs its own prior baseline. + + Over the last ~200 canonical slots, compare: + - ethrex recent mean vs peer median (non-ethrex clients) + - ethrex recent mean vs ethrex prior-window baseline + + Fires warn only when: + ethrex_mean > 50 ms AND + (ethrex_mean > 1.5 * peer_median OR ethrex_mean > 1.5 * ethrex_baseline) + + Requires >= 20 ethrex samples and >= 1 peer with samples; stays quiet otherwise. + """ + from .analyze import median as _median, peer_ratio as _peer_ratio, baseline_shift as _baseline_shift + + WINDOW = 200 + HALF = WINDOW // 2 + + max_row = conn.execute( + "SELECT MAX(slot) FROM slot_exec_times WHERE devnet=?", (devnet,) + ).fetchone() + if not max_row or max_row[0] is None: + return + max_slot = int(max_row[0]) + cutoff_recent = max_slot - HALF + cutoff_prior = max_slot - WINDOW + + # Fetch recent window (second half of the 200-slot span) + recent_rows = conn.execute( + """SELECT client_type, avg_time FROM slot_exec_times + WHERE devnet=? AND slot > ? AND avg_time IS NOT NULL""", + (devnet, cutoff_recent), + ).fetchall() + # Fetch prior window (first half) + prior_rows = conn.execute( + """SELECT client_type, avg_time FROM slot_exec_times + WHERE devnet=? AND slot > ? AND slot <= ? AND avg_time IS NOT NULL""", + (devnet, cutoff_prior, cutoff_recent), + ).fetchall() + + ethrex_recent: list[float] = [] + ethrex_prior: list[float] = [] + peer_recent: dict[str, list[float]] = {} + + for r in recent_rows: + ct = r["client_type"] or "unknown" + if ct == "ethrex": + ethrex_recent.append(r["avg_time"]) + else: + peer_recent.setdefault(ct, []).append(r["avg_time"]) + + for r in prior_rows: + ct = r["client_type"] or "unknown" + if ct == "ethrex": + ethrex_prior.append(r["avg_time"]) + + # Guard: thin data -> skip + if len(ethrex_recent) < 20: + return + if not peer_recent: + return + + ethrex_mean = sum(ethrex_recent) / len(ethrex_recent) + + # Must exceed 50 ms to even consider firing + if ethrex_mean <= 50.0: + return + + # Peer median: compute median per-client, then median of those medians. + # This keeps one slow outlier (erigon) from suppressing a real ethrex regression. + per_peer_medians = [_median(v) for v in peer_recent.values() if v] + valid_peer_meds = [m for m in per_peer_medians if m is not None] + peer_med = _median(valid_peer_meds) + + ratio_vs_peers = _peer_ratio(ethrex_mean, valid_peer_meds) + ratio_vs_baseline = _baseline_shift(ethrex_recent, ethrex_prior) + + fires = False + if ratio_vs_peers is not None and ratio_vs_peers > 1.5: + fires = True + if ratio_vs_baseline is not None and ratio_vs_baseline > 1.5: + fires = True + + if not fires: + return + + baseline_ms = (sum(ethrex_prior) / len(ethrex_prior)) if ethrex_prior else None + details = { + "ethrex_ms": round(ethrex_mean, 1), + "peer_median_ms": round(peer_med, 1) if peer_med is not None else None, + "baseline_ms": round(baseline_ms, 1) if baseline_ms is not None else None, + "ratio_vs_peers": round(ratio_vs_peers, 2) if ratio_vs_peers is not None else None, + "ratio_vs_baseline": round(ratio_vs_baseline, 2) if ratio_vs_baseline is not None else None, + "window": f"last {WINDOW} slots (recent/prior halves of {HALF} each)", + } + peer_med_str = f"{peer_med:.1f}" if peer_med is not None else "n/a" + baseline_str = f"{baseline_ms:.1f}" if baseline_ms is not None else "n/a" + key = record_event( + conn, devnet, + kind="exec_regression", + severity="warn", + node=None, + message=( + f"ethrex exec time elevated: {ethrex_mean:.1f} ms avg " + f"(peer median {peer_med_str} ms, baseline {baseline_str} ms)" + ), + details=details, + now=now, + discriminator="exec_regression", + ) + seen.add(key) + + +@_register +def detect_finality_stall(conn: Any, devnet: str, now: int, seen: set) -> None: + """Detect finality not advancing across a span of >= 20 minutes. + + Requires >= 3 snapshots in network_overview. Fires crit if the latest + finalized_epoch equals the finalized_epoch from the earliest snapshot + in a window of at least ~30 min, and the non-advancing span is >= 20 min. + """ + # Finality advances every ~2 epochs (~12.8 min); use a generous span so a + # Dora collection gap that recovers with same-epoch rows doesn't false-fire. + WINDOW_SECS = 60 * 60 # 60-minute look-back + MIN_SPAN_SECS = 40 * 60 # stall must span >= 40 minutes + + cutoff = now - WINDOW_SECS + + rows = conn.execute( + """SELECT ts, finalized_epoch FROM network_overview + WHERE devnet=? AND ts >= ? AND finalized_epoch IS NOT NULL + ORDER BY ts ASC""", + (devnet, cutoff), + ).fetchall() + + if len(rows) < 3: + return + + earliest = rows[0] + latest = rows[-1] + span = latest["ts"] - earliest["ts"] + + if span < MIN_SPAN_SECS: + return + + if latest["finalized_epoch"] != earliest["finalized_epoch"]: + return + + epoch = latest["finalized_epoch"] + span_minutes = span / 60 + + details = { + "finalized_epoch": epoch, + "span_minutes": round(span_minutes, 1), + "snapshots": len(rows), + } + key = record_event( + conn, devnet, + kind="finality_stall", + severity="warn", + node=None, + message=f"finality stalled at epoch {epoch} (~{span_minutes:.0f} min, no advance)", + details=details, + now=now, + discriminator="finality_stall", + ) + seen.add(key) + + +@_register +def detect_head_lag(conn: Any, devnet: str, now: int, seen: set) -> None: + """Detect ethrex nodes whose EL head is > 30 blocks behind the fleet max. + + Fleet max is computed across all nodes (ethrex + others). Nodes that are + actively syncing are excluded to avoid false positives during catch-up. + """ + from .config import is_ethrex_node + + LAG_THRESHOLD = 30 + STALE_SECS = 1800 # a snapshot older than this means the node is unreachable + + # Latest snapshot per node (correlated-subquery pattern from existing detectors) + rows = conn.execute( + """SELECT node, head, syncing, ts FROM node_health + WHERE devnet=? AND ts = ( + SELECT MAX(ts) FROM node_health nh2 + WHERE nh2.devnet = node_health.devnet AND nh2.node = node_health.node + )""", + (devnet,), + ).fetchall() + + if not rows: + return + + # Fleet max head across all nodes with a valid (non-None, > 0) head + heads = [r["head"] for r in rows if r["head"] is not None and r["head"] > 0] + if not heads: + return + + fleet_max = max(heads) + + for r in rows: + node = r["node"] + if not is_ethrex_node(conn, devnet, node): + continue + head = r["head"] + if head is None: + continue + + # Skip nodes whose latest snapshot is stale (unreachable) -- node_unreachable + # already covers those; an old head would look falsely far behind. + if r["ts"] is not None and r["ts"] < now - STALE_SECS: + continue + + # Skip nodes that are actively syncing + syncing_raw = (r["syncing"] or "").lower() + is_syncing = ( + syncing_raw.startswith("cur=") + or syncing_raw in ("yes", "true", "1") + ) + if is_syncing: + continue + + lag = fleet_max - head + if lag > LAG_THRESHOLD: + details = { + "node": node, + "head": head, + "fleet_max_head": fleet_max, + "lag_blocks": lag, + } + key = record_event( + conn, devnet, + kind="head_lag", + severity="warn", + node=node, + message=( + f"{node} is {lag} blocks behind fleet head {fleet_max} " + f"(node head: {head})" + ), + details=details, + now=now, + discriminator="head_lag", + ) + seen.add(key) diff --git a/devnets_monitor/devnets/discover.py b/devnets_monitor/devnets/discover.py new file mode 100644 index 0000000..1f26827 --- /dev/null +++ b/devnets_monitor/devnets/discover.py @@ -0,0 +1,266 @@ +""" +Discover devnet roster, image tag, and fork schedule from the ethpandaops devnet repo. +Writes config/devnets/.yaml via config.write_cache. +Uses 'gh api' (subprocess) to fetch file contents from GitHub. +""" + +from __future__ import annotations + +import base64 +import configparser +import io +import json +import subprocess +import sys +import time +from typing import Any + +from .config import devnet_entry, write_cache + + +def _gh_api_content(repo: str, path: str) -> bytes | None: + """ + Fetch file contents from GitHub via 'gh api repos//contents/'. + Returns the decoded bytes, or None on 404/error (with a warning printed). + """ + api_path = f"repos/{repo}/contents/{path}" + try: + result = subprocess.run( + ["gh", "api", api_path, "--jq", ".content"], + capture_output=True, + text=True, + timeout=30, + ) + except FileNotFoundError: + print("error: 'gh' CLI not found; install GitHub CLI to use 'dv discover'", file=sys.stderr) + sys.exit(1) + except subprocess.TimeoutExpired: + print(f"warning: gh api timeout fetching {api_path}", file=sys.stderr) + return None + + if result.returncode != 0: + stderr = result.stderr.strip() + if "404" in stderr or "Not Found" in stderr: + print(f"warning: 404 fetching {api_path}", file=sys.stderr) + else: + print(f"warning: gh api error for {api_path}: {stderr[:200]}", file=sys.stderr) + return None + + # GitHub API returns base64-encoded content (with newlines); decode it + encoded = result.stdout.strip() + # Remove embedded newlines that GitHub adds to the base64 output + encoded_clean = encoded.replace("\\n", "").replace("\n", "") + try: + return base64.b64decode(encoded_clean) + except Exception as exc: + print(f"warning: failed to decode base64 for {api_path}: {exc}", file=sys.stderr) + return None + + +def _parse_inventory(ini_bytes: bytes) -> list[str]: + """ + Parse an Ansible inventory.ini and extract ethrex node names. + + Looks for [ethrex:children] group to get child group names, then collects + host entries from each child group. + """ + text = ini_bytes.decode("utf-8", errors="replace") + + # configparser doesn't handle Ansible's [group:children] sections well, + # so we parse manually. + children_groups: list[str] = [] + current_section: str | None = None + group_members: dict[str, list[str]] = {} + + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line or line.startswith(";") or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1] + if current_section not in group_members: + group_members[current_section] = [] + continue + if current_section is not None: + # For [ethrex:children], each entry is a child group name + if current_section == "ethrex:children": + # line may be just a group name (possibly with ansible vars) + group_name = line.split()[0] + children_groups.append(group_name) + else: + # For regular groups, each entry is a hostname (optionally with vars) + host = line.split()[0] + group_members.setdefault(current_section, []).append(host) + + # Collect all hosts from the child groups + nodes: list[str] = [] + seen: set[str] = set() + for grp in children_groups: + for host in group_members.get(grp, []): + if host not in seen: + seen.add(host) + nodes.append(host) + + return nodes + + +def _parse_images_yaml(yaml_bytes: bytes) -> str | None: + """ + Extract the ethrex image tag from images.yaml. + Looks for the 'ethrex:' key under default_ethereum_client_images. + Returns the tag string or None. + """ + text = yaml_bytes.decode("utf-8", errors="replace") + # Simple line-based search: find a line containing 'ethrex:' (with optional spaces) + # after stripping quotes + in_client_images = False + for line in text.splitlines(): + stripped = line.strip() + if "default_ethereum_client_images" in stripped: + in_client_images = True + continue + if in_client_images: + # If we hit another top-level key (no leading spaces), stop + if stripped and not line.startswith(" ") and not line.startswith("\t"): + in_client_images = False + continue + if stripped.startswith("ethrex:"): + value = stripped[len("ethrex:"):].strip().strip('"').strip("'") + if value: + return value + return None + + +def _parse_fork_schedule(genesis_bytes: bytes) -> tuple[int | None, dict[str, dict]]: + """ + Parse genesis.json to extract chainId and fork schedule. + + Returns (chain_id, fork_schedule) where fork_schedule maps fork name to + {activation_ts, blob_target, blob_max}. + """ + text = genesis_bytes.decode("utf-8", errors="replace") + try: + genesis = json.loads(text) + except json.JSONDecodeError as exc: + print(f"warning: failed to parse genesis.json: {exc}", file=sys.stderr) + return None, {} + + config = genesis.get("config", {}) + + # chain_id + chain_id = config.get("chainId") or genesis.get("chainId") + if chain_id is not None: + chain_id = int(chain_id) + + # Fork timestamps: config keys ending in "Time" (excluding terminalTotalDifficultyPassed etc.) + fork_ts: dict[str, int] = {} + for key, val in config.items(): + if key.endswith("Time") and isinstance(val, (int, str)): + fork_name = key[: -len("Time")] + # Normalize to lower camel -> lowercase (e.g. cancunTime -> cancun) + # Just lowercase the first char for simple cases + name = fork_name[0].lower() + fork_name[1:] if fork_name else fork_name + try: + fork_ts[name] = int(val) + except (ValueError, TypeError): + pass + + # blobSchedule: maps fork name to {target, max} + blob_schedule: dict[str, dict[str, int]] = {} + raw_blob = config.get("blobSchedule", {}) + for fork, vals in raw_blob.items(): + blob_schedule[fork] = { + "target": int(vals.get("target", 0)), + "max": int(vals.get("max", 0)), + } + + # Build fork_schedule dict + fork_schedule: dict[str, dict] = {} + for fork, ts in fork_ts.items(): + entry: dict[str, Any] = {"activation_ts": ts} + if fork in blob_schedule: + entry["blob_target"] = blob_schedule[fork]["target"] + entry["blob_max"] = blob_schedule[fork]["max"] + fork_schedule[fork] = entry + + # Add any blob-schedule forks that didn't have a *Time key + for fork, blob in blob_schedule.items(): + if fork not in fork_schedule: + fork_schedule[fork] = { + "activation_ts": 0, + "blob_target": blob["target"], + "blob_max": blob["max"], + } + + return chain_id, fork_schedule + + +def discover(devnet: str) -> None: + """ + Fetch roster, image tag, and fork schedule from the ethpandaops devnet repo. + Writes config/devnets/.yaml. + """ + entry = devnet_entry(devnet) + devnets_repo: str = entry["devnets_repo"] + repo_path: str = entry["repo_path"] + + print(f"Discovering {devnet} from {devnets_repo} (path: {repo_path})...") + + # --- Fetch inventory.ini --- + ini_path = f"ansible/inventories/{repo_path}/inventory.ini" + ini_bytes = _gh_api_content(devnets_repo, ini_path) + + nodes: list[str] = [] + if ini_bytes is None: + # Try with full devnet name as fallback + ini_path_fallback = f"ansible/inventories/{devnet}/inventory.ini" + print(f" Trying fallback path: {ini_path_fallback}", file=sys.stderr) + ini_bytes = _gh_api_content(devnets_repo, ini_path_fallback) + + if ini_bytes: + nodes = _parse_inventory(ini_bytes) + print(f" Found {len(nodes)} ethrex nodes: {', '.join(nodes)}") + else: + print("warning: could not fetch inventory.ini; roster will be empty", file=sys.stderr) + + # --- Fetch images.yaml --- + images_path = f"ansible/inventories/{repo_path}/group_vars/all/images.yaml" + images_bytes = _gh_api_content(devnets_repo, images_path) + + image_tag: str = "" + if images_bytes: + image_tag = _parse_images_yaml(images_bytes) or "" + if image_tag: + print(f" Image tag: {image_tag}") + else: + print("warning: ethrex image tag not found in images.yaml", file=sys.stderr) + else: + print("warning: could not fetch images.yaml; image_tag will be empty", file=sys.stderr) + + # --- Fetch genesis.json --- + genesis_path = f"network-configs/{repo_path}/metadata/genesis.json" + genesis_bytes = _gh_api_content(devnets_repo, genesis_path) + + chain_id: int | None = None + fork_schedule: dict[str, dict] = {} + if genesis_bytes: + chain_id, fork_schedule = _parse_fork_schedule(genesis_bytes) + if chain_id is not None: + print(f" Chain ID: {chain_id}") + forks_found = list(fork_schedule.keys()) + print(f" Fork schedule keys: {', '.join(forks_found)}") + else: + print("warning: could not fetch genesis.json; fork_schedule will be empty", file=sys.stderr) + + # --- Build cache data --- + cache: dict[str, Any] = { + "discovered_at": int(time.time()), + "image_tag": image_tag, + "roster": [{"name": n, "verified": False} for n in nodes], + "fork_schedule": fork_schedule, + } + if chain_id is not None: + cache["chain_id"] = chain_id + + write_cache(devnet, cache) + print(f"Written: config/devnets/{devnet}.yaml") diff --git a/devnets_monitor/devnets/dora.py b/devnets_monitor/devnets/dora.py new file mode 100644 index 0000000..e650440 --- /dev/null +++ b/devnets_monitor/devnets/dora.py @@ -0,0 +1,295 @@ +"""Dora slots collector: fetch slot data and persist into SQLite.""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime, timezone +from typing import Any + +import requests + +from .config import devnet_entry +from .store import connect, max_slot, migrate, upsert + +logger = logging.getLogger(__name__) + +_PAGE_SIZE = 100 +_MAX_RETRIES = 6 +_BASE_BACKOFF = 2.0 # seconds; doubled each retry, capped + + +def _get_with_backoff(url: str, params: dict[str, Any]) -> requests.Response | None: + """ + GET with retry/backoff for HTTP 429 and transient 5xx. Honors a Retry-After + header when present, else exponential backoff (2,4,8,... capped at 60s). + Returns the Response on success, None if all retries are exhausted. + """ + for attempt in range(_MAX_RETRIES): + try: + resp = requests.get(url, params=params, timeout=30) + except requests.RequestException as exc: + wait = min(_BASE_BACKOFF * (2 ** attempt), 60.0) + logger.warning("dora request error (%s); retry in %.0fs", exc, wait) + time.sleep(wait) + continue + if resp.status_code == 429 or resp.status_code >= 500: + retry_after = resp.headers.get("Retry-After") + if retry_after and retry_after.isdigit(): + wait = float(retry_after) + else: + wait = min(_BASE_BACKOFF * (2 ** attempt), 60.0) + logger.warning( + "dora HTTP %d (rate-limited?); backing off %.0fs", + resp.status_code, + wait, + ) + time.sleep(wait) + continue + return resp + logger.error("dora: giving up after %d retries", _MAX_RETRIES) + return None + + +def _parse_time(value: Any) -> int | None: + """Convert ISO-8601 or unix-int time to a unix int. Returns None on failure.""" + if value is None: + return None + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, str): + try: + # Handle ISO-8601 with Z suffix + s = value.rstrip("Z") + dt = datetime.fromisoformat(s).replace(tzinfo=timezone.utc) + return int(dt.timestamp()) + except ValueError: + pass + return None + + +def _status_str(value: Any) -> str: + """Normalise slot status to a string.""" + if isinstance(value, int): + mapping = {0: "Missing", 1: "Canonical", 2: "Orphaned"} + return mapping.get(value, str(value)) + return str(value) if value is not None else "Unknown" + + +def _upsert_slot(conn: Any, devnet: str, s: dict[str, Any]) -> int | None: + """ + Upsert one raw Dora slot dict into slots + slot_exec_times. + Returns the slot number on success, None if slot is absent. + """ + slot_num = s.get("slot") + if slot_num is None: + return None + + row: dict[str, Any] = { + "devnet": devnet, + "slot": slot_num, + "epoch": s.get("epoch"), + "time": _parse_time(s.get("time")), + "proposer": str(s["proposer"]) if s.get("proposer") is not None else None, + "proposer_name": s.get("proposer_name"), + "status": _status_str(s.get("status")), + "blob_count": s.get("blob_count"), + "eth_block_number": s.get("eth_block_number"), + "gas_used": s.get("gas_used"), + } + upsert(conn, "slots", row) + + for et in s.get("execution_times") or []: + ct = et.get("client_type") + if not ct: + continue + et_row: dict[str, Any] = { + "devnet": devnet, + "slot": slot_num, + "client_type": ct, + "count": et.get("count"), + "avg_time": et.get("avg_time"), + "min_time": et.get("min_time"), + "max_time": et.get("max_time"), + } + upsert(conn, "slot_exec_times", et_row) + + return slot_num + + +def collect_blobs( + devnet: str, + since_slot: int | None = None, + min_slot: int | None = None, + max_slot_param: int | None = None, + limit_pages: int | None = None, +) -> None: + """ + Page through Dora slots and upsert into the `slots` and `slot_exec_times` tables. + + Modes: + - Incremental (default, no args): stops once slot numbers fall to or below + the stored watermark (max_slot). since_slot overrides the watermark. + - Bounded range: if min_slot and/or max_slot_param are given, passes them + as min_slot/max_slot query params to Dora and pages through that range + without applying the incremental watermark. + + Resilient to missing fields in the Dora response. + """ + entry = devnet_entry(devnet) + dora_base = entry.get("dora_base", "").rstrip("/") + if not dora_base: + logger.error("dv collect blobs: dora_base missing for devnet %s", devnet) + return + + conn = connect() + migrate(conn) + + bounded = min_slot is not None or max_slot_param is not None + if bounded: + watermark = None # range mode: no incremental stop + else: + watermark = since_slot if since_slot is not None else max_slot(conn, devnet) + + url = f"{dora_base}/api/v1/slots" + params: dict[str, Any] = { + "limit": _PAGE_SIZE, + "with_missing": 1, + "with_orphaned": 1, + } + if min_slot is not None: + params["min_slot"] = min_slot + if max_slot_param is not None: + params["max_slot"] = max_slot_param + + new_slots = 0 + min_seen: int | None = None + max_seen: int | None = None + page = 0 + + while True: + if limit_pages is not None and page >= limit_pages: + break + + params["page"] = page + resp = _get_with_backoff(url, params) + if resp is None: + logger.error("dora fetch failed at page %d; stopping", page) + break + + data = resp.json().get("data", {}) + slots = data.get("slots") or [] + if not slots: + break + + stop = False + for s in slots: + slot_num = s.get("slot") + if slot_num is None: + continue + + # Incremental stop (only in non-bounded mode) + if not bounded and watermark is not None and slot_num <= watermark: + stop = True + break + + inserted = _upsert_slot(conn, devnet, s) + if inserted is not None: + new_slots += 1 + if min_seen is None or inserted < min_seen: + min_seen = inserted + if max_seen is None or inserted > max_seen: + max_seen = inserted + + conn.commit() + + if stop: + break + + next_page = data.get("next_page") + if next_page is None or not slots: + break + + page = int(next_page) + + conn.close() + + slot_range = f"{min_seen}-{max_seen}" if min_seen is not None else "none" + logger.info( + "collect_blobs(%s): %d new slots inserted, slot range %s", + devnet, + new_slots, + slot_range, + ) + print( + f"collect_blobs({devnet}): {new_slots} new slots, " + f"slot range {slot_range}" + ) + + +def backfill(devnet: str, from_slot: int, to_slot: int) -> None: + """ + Range-collect slots [from_slot, to_slot] into the slots + slot_exec_times + tables using min_slot/max_slot paging. Prints progress. + """ + entry = devnet_entry(devnet) + dora_base = entry.get("dora_base", "").rstrip("/") + if not dora_base: + logger.error("backfill: dora_base missing for devnet %s", devnet) + return + + conn = connect() + migrate(conn) + + url = f"{dora_base}/api/v1/slots" + params: dict[str, Any] = { + "limit": _PAGE_SIZE, + "with_missing": 1, + "with_orphaned": 1, + "min_slot": from_slot, + "max_slot": to_slot, + } + + new_slots = 0 + min_seen: int | None = None + max_seen: int | None = None + page = 0 + + print(f"backfill({devnet}): fetching slots {from_slot}-{to_slot} ...") + + while True: + params["page"] = page + resp = _get_with_backoff(url, params) + if resp is None: + logger.error("backfill: dora fetch failed at page %d; stopping", page) + break + + data = resp.json().get("data", {}) + slots = data.get("slots") or [] + if not slots: + break + + for s in slots: + inserted = _upsert_slot(conn, devnet, s) + if inserted is not None: + new_slots += 1 + if min_seen is None or inserted < min_seen: + min_seen = inserted + if max_seen is None or inserted > max_seen: + max_seen = inserted + + conn.commit() + print(f" page {page}: {len(slots)} slots (total so far: {new_slots})") + + next_page = data.get("next_page") + if next_page is None or not slots: + break + + page = int(next_page) + + conn.close() + + slot_range = f"{min_seen}-{max_seen}" if min_seen is not None else "none" + print( + f"backfill({devnet}): done. {new_slots} slots inserted, range {slot_range}" + ) diff --git a/devnets_monitor/devnets/eiptrack.py b/devnets_monitor/devnets/eiptrack.py new file mode 100644 index 0000000..4f32bde --- /dev/null +++ b/devnets_monitor/devnets/eiptrack.py @@ -0,0 +1,124 @@ +"""EIP implementation-status tracker. + +Reads fork_eips (eip, title, stage, status) and groups by implementation status +to give a quick summary of ethrex's coverage for an upcoming fork. + +Status values (set by operator in config/eips.json): + done - implemented and passing tests + in_progress - work underway + missing - not yet started + n/a - not applicable to the EL (e.g. pure CL EIPs) + unknown - not yet assessed (default when absent) +""" + +from __future__ import annotations + +from typing import Any + +from .store import connect, migrate + + +_STATUS_ORDER = ["done", "in_progress", "missing", "n/a", "unknown"] +_STATUS_LABELS = { + "done": "Done", + "in_progress": "In Progress", + "missing": "Missing", + "n/a": "N/A - CL-only", + "unknown": "Unknown", +} + + +def get_eiptrack_data( + devnet: str, fork: str = "amsterdam" +) -> dict[str, Any] | None: + """ + Return EIP implementation-status data for dashboard rendering. + + Keys: + fork str fork name queried + groups list[dict] one entry per status value, ordered by _STATUS_ORDER + status str status key + label str display label + count int number of EIPs with this status + eips list[dict] {eip, title, stage, status} + total int total EIPs for this fork + stage_counts dict {SFI: N, CFI: N, PFI: N, None: N} + """ + conn = connect() + migrate(conn) + + rows = conn.execute( + """ + SELECT eip, title, stage, status + FROM fork_eips + WHERE devnet = ? AND fork = ? + ORDER BY eip ASC + """, + (devnet, fork), + ).fetchall() + conn.close() + + if not rows: + return None + + # Group by status + groups_map: dict[str, list[dict]] = {s: [] for s in _STATUS_ORDER} + stage_counts: dict[str | None, int] = {} + + for r in rows: + status = r["status"] or "unknown" + if status not in groups_map: + groups_map[status] = [] + groups_map[status].append({ + "eip": r["eip"], + "title": r["title"] or "", + "stage": r["stage"] or "", + "status": status, + }) + stage = r["stage"] or "none" + stage_counts[stage] = stage_counts.get(stage, 0) + 1 + + groups = [] + for status in _STATUS_ORDER: + eips = groups_map.get(status, []) + groups.append({ + "status": status, + "label": _STATUS_LABELS.get(status, status), + "count": len(eips), + "eips": eips, + }) + + return { + "fork": fork, + "groups": groups, + "total": len(rows), + "stage_counts": stage_counts, + } + + +def show_eiptrack(devnet: str, fork: str = "amsterdam") -> None: + """Print EIP status summary to stdout.""" + data = get_eiptrack_data(devnet, fork=fork) + if data is None: + print( + f"eip-track({devnet}): no fork_eips data for fork '{fork}'. " + f"Run: dv collect {devnet} forks" + ) + return + + print(f"\nEIP implementation status -- {devnet} / {data['fork']}") + print(f"Total EIPs: {data['total']}") + + stage_parts = [f"{s}={n}" for s, n in sorted(data["stage_counts"].items())] + print(f"Stages: {', '.join(stage_parts)}\n") + + for group in data["groups"]: + if not group["eips"] and group["status"] == "unknown": + # Show unknown even if empty so the table is always complete + pass + count_label = f"{group['label']} ({group['count']})" + print(f" {count_label}") + for e in group["eips"]: + stage = f"[{e['stage']}]" if e["stage"] else "" + print(f" EIP-{e['eip']:5d} {stage:<6} {e['title']}") + print() diff --git a/devnets_monitor/devnets/epbs.py b/devnets_monitor/devnets/epbs.py new file mode 100644 index 0000000..ae04ec6 --- /dev/null +++ b/devnets_monitor/devnets/epbs.py @@ -0,0 +1,338 @@ +"""ePBS (EIP-7732) panel: bid count and PTC vote data per slot. + +Fetches /v1/slot/{slot}/bids and /v1/slot/{slot}/ptc_votes for recent slots +and persists into the epbs_slot table. + +Confirmed field names from live glamsterdam-devnet-5 (slot 35537): + /bids -> data.{slot, block_root, count, bids[{is_self_built, is_winning, ...}]} + /ptc_votes -> data.{slot, block_root, total_ptc_size, vote_count, + non_voter_count, non_voter_percent, + aggregates[{payload_present, vote_count}]} +""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime, timezone +from typing import Any + +from .config import devnet_entry +from .dora import _get_with_backoff +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + +_DEFAULT_LIMIT = 50 + + +def _max_epbs_slot(conn: Any, devnet: str) -> int | None: + """Return the highest slot already in epbs_slot for this devnet.""" + row = conn.execute( + "SELECT MAX(slot) FROM epbs_slot WHERE devnet = ?", (devnet,) + ).fetchone() + if row and row[0] is not None: + return int(row[0]) + return None + + +def _payload_revealed(bids: list[dict]) -> int | None: + """ + Derive whether a payload was revealed. + + A bid with is_winning=true that is NOT is_self_built indicates an external + builder revealed a payload. If all winning bids are self-built (the local + proposer built the block), ePBS payload reveal may not have been triggered. + Returns 1 if an external winning bid exists, 0 if only self-built, None if + no bids at all. + """ + if not bids: + return None + for bid in bids: + if bid.get("is_winning") and not bid.get("is_self_built"): + return 1 + # No external winning bid; check if there is any winning bid at all + for bid in bids: + if bid.get("is_winning"): + return 0 + return None + + +def collect_epbs(devnet: str, limit: int = _DEFAULT_LIMIT) -> None: + """ + Collect ePBS bid and PTC vote data for recent slots. + + Fetches for ethrex-proposed canonical slots above the watermark, plus a + sample of other-client slots for comparison (up to half the limit). + Per-fetch try/except ensures one failure does not abort the chain. + """ + entry = devnet_entry(devnet) + dora_base = entry.get("dora_base", "").rstrip("/") + if not dora_base: + logger.error("collect_epbs: dora_base missing for devnet %s", devnet) + return + + conn = connect() + migrate(conn) + + watermark = _max_epbs_slot(conn, devnet) + + # Fetch ethrex canonical slots above the watermark + query_ethrex = """ + SELECT slot, proposer_name + FROM slots + WHERE devnet = ? + AND status = 'Canonical' + AND proposer_name LIKE '%ethrex%' + """ + params_ethrex: list[Any] = [devnet] + if watermark is not None: + query_ethrex += " AND slot > ?" + params_ethrex.append(watermark) + query_ethrex += " ORDER BY slot DESC LIMIT ?" + params_ethrex.append(limit) + + ethrex_rows = conn.execute(query_ethrex, params_ethrex).fetchall() + + # Also pull a comparison sample of non-ethrex slots (up to limit // 2) + comparison_limit = max(1, limit // 2) + query_other = """ + SELECT slot, proposer_name + FROM slots + WHERE devnet = ? + AND status = 'Canonical' + AND proposer_name NOT LIKE '%ethrex%' + """ + params_other: list[Any] = [devnet] + if watermark is not None: + query_other += " AND slot > ?" + params_other.append(watermark) + query_other += " ORDER BY slot DESC LIMIT ?" + params_other.append(comparison_limit) + + other_rows = conn.execute(query_other, params_other).fetchall() + + all_rows = list(ethrex_rows) + list(other_rows) + + if not all_rows: + print(f"collect_epbs({devnet}): no new slots above watermark") + conn.close() + return + + fetched = 0 + errors = 0 + now = int(time.time()) + + for row in all_rows: + slot = row["slot"] + proposer_name = row["proposer_name"] or "" + + bids_url = f"{dora_base}/api/v1/slot/{slot}/bids" + ptc_url = f"{dora_base}/api/v1/slot/{slot}/ptc_votes" + + bid_count: int | None = None + ptc_size: int | None = None + ptc_vote_count: int | None = None + ptc_nonvoter_pct: float | None = None + payload_rev: int | None = None + block_root: str = "" + bids_ok = False + ptc_ok = False + + try: + resp = _get_with_backoff(bids_url, {}) + if resp is not None and resp.status_code == 200: + bdata = resp.json().get("data", {}) + block_root = bdata.get("block_root", "") or "" + bid_count = bdata.get("count", 0) or 0 + bids_list = bdata.get("bids") or [] + payload_rev = _payload_revealed(bids_list) + bids_ok = True + else: + logger.warning( + "collect_epbs: bids HTTP %s for slot %d", + resp.status_code if resp is not None else "None", + slot, + ) + errors += 1 + except Exception as exc: + logger.warning("collect_epbs: bids error on slot %d: %s", slot, exc) + errors += 1 + + try: + resp2 = _get_with_backoff(ptc_url, {}) + if resp2 is not None and resp2.status_code == 200: + pdata = resp2.json().get("data", {}) + if not block_root: + block_root = pdata.get("block_root", "") or "" + ptc_size = pdata.get("total_ptc_size") + ptc_vote_count = pdata.get("vote_count") + ptc_nonvoter_pct = pdata.get("non_voter_percent") + ptc_ok = True + else: + logger.warning( + "collect_epbs: ptc_votes HTTP %s for slot %d", + resp2.status_code if resp2 is not None else "None", + slot, + ) + errors += 1 + except Exception as exc: + logger.warning("collect_epbs: ptc_votes error on slot %d: %s", slot, exc) + errors += 1 + + # Don't persist (or advance past) a slot we got nothing for; leave it + # below the watermark so the next run retries it. + if not bids_ok and not ptc_ok: + continue + + epbs_row: dict[str, Any] = { + "devnet": devnet, + "slot": slot, + "block_root": block_root, + "proposer_name": proposer_name, + "bid_count": bid_count, + "ptc_size": ptc_size, + "ptc_vote_count": ptc_vote_count, + "ptc_nonvoter_pct": ptc_nonvoter_pct, + "payload_revealed": payload_rev, + "fetched_at": now, + } + upsert(conn, "epbs_slot", epbs_row) + fetched += 1 + + conn.commit() + conn.close() + print( + f"collect_epbs({devnet}): {fetched} slots fetched, " + f"{errors} errors, watermark was {watermark}" + ) + + +def get_epbs_data(devnet: str, limit: int = 200) -> dict[str, Any] | None: + """ + Return ePBS slot data for dashboard rendering. + + Keys: + rows list[dict] recent epbs_slot rows (newest first) + total int total rows + ethrex_rows list[dict] ethrex-proposed rows only + avg_ptc_size float + avg_vote_count float + avg_nonvoter_pct float + """ + conn = connect() + migrate(conn) + + rows = conn.execute( + """ + SELECT slot, block_root, proposer_name, bid_count, + ptc_size, ptc_vote_count, ptc_nonvoter_pct, + payload_revealed, fetched_at + FROM epbs_slot + WHERE devnet = ? + ORDER BY slot DESC + LIMIT ? + """, + (devnet, limit), + ).fetchall() + conn.close() + + if not rows: + return None + + def _fmt(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + return str(ts) + + out_rows = [] + ptc_sizes = [] + vote_counts = [] + nonvoter_pcts = [] + + for r in rows: + is_ethrex = "ethrex" in (r["proposer_name"] or "") + ps = r["ptc_size"] + vc = r["ptc_vote_count"] + nvp = r["ptc_nonvoter_pct"] + if ps is not None: + ptc_sizes.append(ps) + if vc is not None: + vote_counts.append(vc) + if nvp is not None: + nonvoter_pcts.append(nvp) + + payload_rev = r["payload_revealed"] + if payload_rev is None: + payload_label = "unknown" + elif payload_rev: + payload_label = "yes" + else: + payload_label = "no (self-built)" + + out_rows.append({ + "slot": r["slot"], + "block_root": r["block_root"] or "", + "block_root_short": (r["block_root"] or "")[:14] + "...", + "proposer_name": r["proposer_name"] or "", + "bid_count": r["bid_count"] if r["bid_count"] is not None else 0, + "ptc_size": ps, + "ptc_vote_count": vc, + "ptc_nonvoter_pct": round(nvp, 1) if nvp is not None else None, + "payload_revealed": payload_label, + "fetched_at": _fmt(r["fetched_at"]), + "is_ethrex": is_ethrex, + }) + + def _avg(lst: list) -> float: + return round(sum(lst) / len(lst), 1) if lst else 0.0 + + return { + "rows": out_rows, + "total": len(out_rows), + "ethrex_rows": [r for r in out_rows if r["is_ethrex"]], + "avg_ptc_size": _avg(ptc_sizes), + "avg_vote_count": _avg(vote_counts), + "avg_nonvoter_pct": _avg(nonvoter_pcts), + } + + +def show_epbs(devnet: str) -> None: + """Print ePBS slot summary to stdout.""" + data = get_epbs_data(devnet) + if data is None: + print( + f"epbs({devnet}): no ePBS data. " + f"Run: dv collect {devnet} slow" + ) + return + + print(f"\nePBS inspection ({devnet})") + print( + f"Total entries: {data['total']}, " + f"avg PTC size: {data['avg_ptc_size']}, " + f"avg vote count: {data['avg_vote_count']}, " + f"avg non-voter%: {data['avg_nonvoter_pct']}\n" + ) + + print( + f"{'SLOT':>8} {'BIDS':>5} {'PTC_SZ':>7} {'VOTES':>6} " + f"{'NONVOTE%':>9} {'PAYLOAD':>14} {'PROPOSER':<28} {'ETHREX'}" + ) + print("-" * 100) + for r in data["rows"][:40]: + ethrex_flag = "<--" if r["is_ethrex"] else "" + ptc_sz = str(r["ptc_size"]) if r["ptc_size"] is not None else "-" + votes = str(r["ptc_vote_count"]) if r["ptc_vote_count"] is not None else "-" + nvpct = f"{r['ptc_nonvoter_pct']:.1f}" if r["ptc_nonvoter_pct"] is not None else "-" + print( + f"{r['slot']:>8} {r['bid_count']:>5} {ptc_sz:>7} {votes:>6} " + f"{nvpct:>9} {r['payload_revealed']:>14} " + f"{r['proposer_name']:<28} {ethrex_flag}" + ) + print() diff --git a/devnets_monitor/devnets/exectime.py b/devnets_monitor/devnets/exectime.py new file mode 100644 index 0000000..ade04c4 --- /dev/null +++ b/devnets_monitor/devnets/exectime.py @@ -0,0 +1,182 @@ +"""Execution-time analysis: per-client comparison and trend chart data. + +Reads slot_exec_times (already collected by dora.py); no new collection. +""" + +from __future__ import annotations + +from typing import Any + +from .store import connect, migrate + + +# Number of chart buckets (mirrors blobs.html) +_CHART_BUCKETS = 60 + + +def get_exectime_data(devnet: str, window_slots: int = 1500) -> dict[str, Any] | None: + """Return per-client exec-time summary and a binned time-series for charting. + + Queries slot_exec_times for the most recent window_slots slots. + + Returned dict: + window_label str + clients list[dict] per client: client_type, samples, avg_ms, min_ms, + max_ms, is_ethrex + series dict[str, list[[bucket_index, avg_ms]]] binned per client + min_slot int + max_slot int + """ + conn = connect() + migrate(conn) + + max_row = conn.execute( + "SELECT MAX(slot) FROM slot_exec_times WHERE devnet=?", (devnet,) + ).fetchone() + if not max_row or max_row[0] is None: + conn.close() + return None + + max_slot = int(max_row[0]) + min_slot = max_slot - window_slots + + rows = conn.execute( + """SELECT slot, client_type, count, avg_time, min_time, max_time + FROM slot_exec_times + WHERE devnet=? AND slot > ? + ORDER BY slot ASC""", + (devnet, min_slot), + ).fetchall() + conn.close() + + if not rows: + return None + + # Per-client aggregation + client_agg: dict[str, dict[str, Any]] = {} + # Per-client per-slot raw points for the time-series + client_pts: dict[str, list[tuple[int, float]]] = {} + + actual_min = min(r["slot"] for r in rows) + actual_max = max(r["slot"] for r in rows) + + for r in rows: + ct = r["client_type"] or "unknown" + if ct not in client_agg: + client_agg[ct] = { + "sum_avg": 0.0, + "count": 0, + "min_ms": float("inf"), + "max_ms": float("-inf"), + } + if r["avg_time"] is not None and r["count"] and r["count"] > 0: + client_agg[ct]["sum_avg"] += r["avg_time"] + client_agg[ct]["count"] += r["count"] + if r["min_time"] is not None: + client_agg[ct]["min_ms"] = min(client_agg[ct]["min_ms"], r["min_time"]) + if r["max_time"] is not None: + client_agg[ct]["max_ms"] = max(client_agg[ct]["max_ms"], r["max_time"]) + client_pts.setdefault(ct, []).append((r["slot"], r["avg_time"])) + + if not client_agg: + return None + + clients_out: list[dict[str, Any]] = [] + for ct, agg in sorted(client_agg.items()): + n = agg["count"] + if n == 0: + continue + # avg_ms = mean of per-slot avg_time values (not weighted by count, + # matching the "avg of avg_time" spec) + pts = client_pts.get(ct, []) + avg_ms = sum(p[1] for p in pts) / len(pts) if pts else 0.0 + min_ms = agg["min_ms"] if agg["min_ms"] != float("inf") else None + max_ms = agg["max_ms"] if agg["max_ms"] != float("-inf") else None + clients_out.append({ + "client_type": ct, + "samples": n, + "avg_ms": round(avg_ms, 1), + "min_ms": round(min_ms, 1) if min_ms is not None else None, + "max_ms": round(max_ms, 1) if max_ms is not None else None, + "is_ethrex": ct == "ethrex", + }) + + # Sort: ethrex first, then by avg_ms ascending + clients_out.sort(key=lambda c: (0 if c["is_ethrex"] else 1, c["avg_ms"])) + + # Binned time-series per client (~60 buckets) + span = (actual_max - actual_min) or 1 + series: dict[str, list[list]] = {} + for ct, pts in client_pts.items(): + bucket_sum: list[float] = [0.0] * _CHART_BUCKETS + bucket_cnt: list[int] = [0] * _CHART_BUCKETS + for slot, avg_t in pts: + b = min(_CHART_BUCKETS - 1, int((slot - actual_min) / span * _CHART_BUCKETS)) + bucket_sum[b] += avg_t + bucket_cnt[b] += 1 + binned = [] + for i in range(_CHART_BUCKETS): + if bucket_cnt[i] > 0: + binned.append([i, round(bucket_sum[i] / bucket_cnt[i], 1)]) + series[ct] = binned + + window_label = f"slots {actual_min}-{actual_max} (last {window_slots} slots)" + + return { + "window_label": window_label, + "clients": clients_out, + "series": series, + "min_slot": actual_min, + "max_slot": actual_max, + } + + +def show_exectime(devnet: str) -> None: + """Print per-client exec-time comparison table and a one-line verdict.""" + from .analyze import peer_ratio + + data = get_exectime_data(devnet) + if data is None: + print(f"exectime({devnet}): no data in slot_exec_times. Run: dv collect {devnet} blobs") + return + + clients = data["clients"] + print(f"\nExec time per client -- {devnet}") + print(f"Window: {data['window_label']}\n") + + col_w = max(len(c["client_type"]) for c in clients) + 2 if clients else 16 + print( + f"{'CLIENT':<{col_w}} {'SAMPLES':>8} {'AVG ms':>8} {'MIN ms':>8} {'MAX ms':>8}" + ) + print("-" * (col_w + 36)) + for c in clients: + marker = " <-- ethrex" if c["is_ethrex"] else "" + min_s = f"{c['min_ms']:.1f}" if c["min_ms"] is not None else "-" + max_s = f"{c['max_ms']:.1f}" if c["max_ms"] is not None else "-" + print( + f"{c['client_type']:<{col_w}} {c['samples']:>8} {c['avg_ms']:>8.1f} " + f"{min_s:>8} {max_s:>8}{marker}" + ) + + # Verdict: ethrex vs peer median + ethrex_row = next((c for c in clients if c["is_ethrex"]), None) + peer_avgs = [c["avg_ms"] for c in clients if not c["is_ethrex"]] + print() + if ethrex_row is None: + print("verdict: no ethrex data in this window") + elif not peer_avgs: + print(f"verdict: ethrex avg {ethrex_row['avg_ms']:.1f} ms (no peer baseline)") + else: + ratio = peer_ratio(ethrex_row["avg_ms"], peer_avgs) + from .analyze import median as _median + peer_med = _median(peer_avgs) + if ratio is None: + print(f"verdict: ethrex avg {ethrex_row['avg_ms']:.1f} ms, peer median unavailable") + else: + direction = "faster" if ratio < 1.0 else "slower" + print( + f"verdict: ethrex {ethrex_row['avg_ms']:.1f} ms avg; " + f"peer median {peer_med:.1f} ms; " + f"ratio {ratio:.2f}x ({direction})" + ) + print() diff --git a/devnets_monitor/devnets/forks.py b/devnets_monitor/devnets/forks.py new file mode 100644 index 0000000..04e4d47 --- /dev/null +++ b/devnets_monitor/devnets/forks.py @@ -0,0 +1,112 @@ +"""Fork schedule + EIP collector: read from discovered cache and eips.json.""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +from .config import load_cache, repo_root +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + + +def collect_forks(devnet: str) -> None: + """ + Read the fork_schedule from the discovered cache (config/devnets/.yaml) + and upsert into the fork_schedule table. + + If config/eips.json exists, also load fork -> EIP entries into fork_eips. + + eipmcp is an MCP tool (not an HTTP API reachable from Python). Therefore + eips-refresh cannot call it at runtime. Instead, eips.json is a + hand/agent-maintained file. If it is absent, collect_forks still loads the + fork schedule; only the EIP enrichment is skipped. Run `dv eips-refresh` to + populate eips.json from eipmcp data. + """ + cache = load_cache(devnet) + if not cache: + print( + f"collect_forks({devnet}): no discovered cache found. " + f"Run: dv discover {devnet}" + ) + return + + fork_schedule: dict[str, Any] = cache.get("fork_schedule") or {} + if not fork_schedule: + print(f"collect_forks({devnet}): no fork_schedule in cache") + return + + conn = connect() + migrate(conn) + + inserted = 0 + for fork_name, fork_data in fork_schedule.items(): + if not isinstance(fork_data, dict): + continue + row: dict[str, Any] = { + "devnet": devnet, + "fork": fork_name, + "activation_ts": fork_data.get("activation_ts"), + "blob_target": fork_data.get("blob_target"), + "blob_max": fork_data.get("blob_max"), + } + upsert(conn, "fork_schedule", row) + inserted += 1 + + conn.commit() + print(f"collect_forks({devnet}): {inserted} forks upserted from cache") + + # Load EIP enrichment if available + eips_path = repo_root() / "config" / "eips.json" + if eips_path.exists(): + _load_eips(conn, devnet, eips_path) + else: + print( + f"collect_forks({devnet}): config/eips.json not found; " + "EIP enrichment skipped. Run: dv eips-refresh" + ) + + conn.close() + + +def _load_eips(conn: Any, devnet: str, eips_path: Path) -> None: + """Load fork->EIP mapping from eips.json into fork_eips table.""" + try: + with eips_path.open() as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as exc: + logger.error("Failed to parse config/eips.json: %s", exc) + print(f"collect_forks: failed to parse config/eips.json: {exc}") + return + + forks_data = data.get("forks") or {} + # Clean reload: eips.json is the full source of truth for this devnet, so drop + # existing rows first (otherwise EIPs removed from the file, e.g. now-Declined + # ones, would linger as stale rows since upsert never deletes). + conn.execute("DELETE FROM fork_eips WHERE devnet = ?", (devnet,)) + inserted = 0 + for fork_name, eip_list in forks_data.items(): + if not isinstance(eip_list, list): + continue + for item in eip_list: + if not isinstance(item, dict): + continue + eip_num = item.get("eip") + if eip_num is None: + continue + row: dict[str, Any] = { + "devnet": devnet, + "fork": fork_name, + "eip": int(eip_num), + "title": item.get("title"), + "stage": item.get("stage"), + "status": item.get("status"), + } + upsert(conn, "fork_eips", row) + inserted += 1 + + conn.commit() + print(f"collect_forks({devnet}): {inserted} EIP entries loaded from eips.json") diff --git a/devnets_monitor/devnets/forkview.py b/devnets_monitor/devnets/forkview.py new file mode 100644 index 0000000..194c735 --- /dev/null +++ b/devnets_monitor/devnets/forkview.py @@ -0,0 +1,217 @@ +"""Fork schedule viewer: human-readable table with countdown and EIP list.""" + +from __future__ import annotations + +import time +from datetime import datetime, timezone +from typing import Any + +from .store import connect, migrate + + +# --------------------------------------------------------------------------- +# Data-returning helpers (used by the dashboard) +# --------------------------------------------------------------------------- + + +def get_fork_data(devnet: str) -> dict[str, Any] | None: + """ + Return fork schedule data as a dict suitable for template rendering. + + Returns None if there is no fork data. Dict keys: + forks list[dict] sorted by activation_ts + keys: fork, activation_ts, time_str, blob_target, blob_max, + status (active/in Xd Yh/in Xh Ym), is_next (bool), + eips (list[dict] with keys eip, title) + next_fork str | None name of the upcoming fork (for the countdown) + """ + conn = connect() + migrate(conn) + + forks = conn.execute( + """ + SELECT fork, activation_ts, blob_target, blob_max + FROM fork_schedule + WHERE devnet = ? + ORDER BY COALESCE(activation_ts, 0) + """, + (devnet,), + ).fetchall() + + if not forks: + conn.close() + return None + + eip_rows = conn.execute( + """ + SELECT fork, eip, title, stage + FROM fork_eips + WHERE devnet = ? + ORDER BY fork, eip + """, + (devnet,), + ).fetchall() + conn.close() + + eips_by_fork: dict[str, list[dict[str, Any]]] = {} + for r in eip_rows: + eips_by_fork.setdefault(r["fork"], []).append( + {"eip": r["eip"], "title": r["title"] or "", "stage": r["stage"]} + ) + + now = time.time() + future_forks = [ + f["fork"] + for f in forks + if f["activation_ts"] is not None and f["activation_ts"] > now + ] + next_fork_name = future_forks[0] if future_forks else None + + result = [] + for row in forks: + fork = row["fork"] + ts = row["activation_ts"] + result.append({ + "fork": fork, + "activation_ts": ts, + "time_str": _format_ts(ts), + "blob_target": row["blob_target"], + "blob_max": row["blob_max"], + "status": _countdown(ts, now), + "is_next": fork == next_fork_name, + "eips": eips_by_fork.get(fork, []), + }) + + return {"forks": result, "next_fork": next_fork_name} + + +def _format_ts(ts: int | None) -> str: + """Convert unix timestamp to a human-readable UTC string.""" + if ts is None: + return "genesis" + if ts == 0: + return "genesis (ts=0)" + try: + dt = datetime.fromtimestamp(ts, tz=timezone.utc) + return dt.strftime("%Y-%m-%d %H:%M UTC") + except (OSError, OverflowError, ValueError): + return str(ts) + + +def _countdown(ts: int | None, now: float) -> str: + """ + Return a human-readable label for a fork's activation relative to now. + Past/genesis forks are marked 'active'. The first future fork gets a + countdown string. + """ + if ts is None or ts == 0: + return "active" + diff = ts - now + if diff <= 0: + return "active" + days = int(diff // 86400) + hours = int((diff % 86400) // 3600) + minutes = int((diff % 3600) // 60) + if days > 0: + return f"in {days}d {hours}h" + if hours > 0: + return f"in {hours}h {minutes}m" + return f"in {minutes}m" + + +def show_forks(devnet: str) -> None: + """ + Print the fork schedule from the DB, sorted by activation_ts. + For each fork: human-readable activation time (UTC), blob target/max, + countdown to next activation, and EIP list if available in fork_eips. + """ + conn = connect() + migrate(conn) + + forks = conn.execute( + """ + SELECT fork, activation_ts, blob_target, blob_max + FROM fork_schedule + WHERE devnet = ? + ORDER BY COALESCE(activation_ts, 0) + """, + (devnet,), + ).fetchall() + + if not forks: + conn.close() + print( + f"fork({devnet}): no data. Run: dv collect {devnet} forks" + ) + return + + # Load EIPs per fork + eip_rows = conn.execute( + """ + SELECT fork, eip, title, stage + FROM fork_eips + WHERE devnet = ? + ORDER BY fork, eip + """, + (devnet,), + ).fetchall() + conn.close() + + eips_by_fork: dict[str, list[tuple[int, str, str]]] = {} + for r in eip_rows: + eips_by_fork.setdefault(r["fork"], []).append( + (r["eip"], r["title"] or "", r["stage"] or "") + ) + + now = time.time() + + # Find the first future fork for the countdown marker + future_forks = [ + f["fork"] + for f in forks + if f["activation_ts"] is not None and f["activation_ts"] > now + ] + next_fork = future_forks[0] if future_forks else None + + print(f"\nFork schedule for {devnet}\n") + + for row in forks: + fork = row["fork"] + ts = row["activation_ts"] + blob_target = row["blob_target"] + blob_max = row["blob_max"] + + time_str = _format_ts(ts) + status = _countdown(ts, now) + + # Marker for the next upcoming fork + marker = " <-- NEXT" if fork == next_fork else "" + + blob_str = "" + if blob_target is not None and blob_max is not None: + blob_str = f" blobs: target={blob_target}, max={blob_max}" + elif blob_target is not None: + blob_str = f" blobs: target={blob_target}" + + print(f" {fork:<12} {time_str:<25} [{status}]{marker}{blob_str}") + + # EIP list (grouped by inclusion stage where present) + eips = eips_by_fork.get(fork, []) + if eips and any(stage for _, _, stage in eips): + from collections import defaultdict + by_stage: dict[str, list[tuple[int, str]]] = defaultdict(list) + for eip_num, title, stage in eips: + by_stage[stage or "Other"].append((eip_num, title)) + stage_order = ["SFI", "CFI", "PFI"] + ordered = sorted(by_stage, key=lambda s: (stage_order.index(s) if s in stage_order else 99, s)) + for stage in ordered: + print(f" [{stage}]") + for eip_num, title in by_stage[stage]: + title_str = f" - {title}" if title else "" + print(f" EIP-{eip_num}{title_str}") + else: + for eip_num, title, _ in eips: + title_str = f" - {title}" if title else "" + print(f" EIP-{eip_num}{title_str}") + + print() diff --git a/devnets_monitor/devnets/health.py b/devnets_monitor/devnets/health.py new file mode 100644 index 0000000..778bb43 --- /dev/null +++ b/devnets_monitor/devnets/health.py @@ -0,0 +1,89 @@ +"""Node health collector: snapshot all nodes into node_health table.""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + + +def _safe_int(value: Any) -> int | None: + """Convert a value to int, returning None on failure.""" + if value is None: + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def collect_health(devnet: str) -> None: + """ + Probe every node in the devnet via SSH and insert a snapshot row into + node_health for each. One row per node per invocation (ts = now). + + Calls devnets.status.gather() directly rather than scraping stdout, + keeping a clean in-process seam. + """ + from .status import gather + + conn = connect() + migrate(conn) + + ts = int(time.time()) + results = gather(devnet, None) + + inserted = 0 + errors = 0 + for d in results: + node = d.get("node", "unknown") + if "_error" in d: + logger.warning("health probe error on %s: %s", node, d["_error"]) + errors += 1 + # Still store the error row so gaps are visible + row: dict[str, Any] = { + "devnet": devnet, + "node": node, + "ts": ts, + "image": None, + "buildnum": None, + "commit": None, + "restart": None, + "head": None, + "peers": None, + "syncing": d["_error"][:200], + "state_at_head": None, + "watchtower": None, + "cl_line": None, + } + else: + row = { + "devnet": devnet, + "node": node, + "ts": ts, + "image": d.get("image"), + "buildnum": d.get("buildnum"), + "commit": d.get("commit"), + "restart": _safe_int(d.get("restart")), + "head": _safe_int(d.get("head")), + "peers": _safe_int(d.get("peers")), + "syncing": d.get("syncing"), + "state_at_head": d.get("state_at_head"), + "watchtower": d.get("watchtower"), + "cl_line": d.get("cl_line"), + } + + upsert(conn, "node_health", row) + inserted += 1 + + conn.commit() + conn.close() + + print( + f"collect_health({devnet}): {inserted} nodes snapshotted " + f"({errors} errors) at ts={ts}" + ) diff --git a/devnets_monitor/devnets/hive.py b/devnets_monitor/devnets/hive.py new file mode 100644 index 0000000..733bbf5 --- /dev/null +++ b/devnets_monitor/devnets/hive.py @@ -0,0 +1,206 @@ +"""Hive conformance run collector and summary view.""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import Any + +import requests + +from .config import devnet_entry +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + +_HIVE_BASE = "https://hive.ethpandaops.io" + + +def _parse_iso(value: str | None) -> int | None: + """Convert ISO-8601 string to unix int. Returns None on failure.""" + if not value: + return None + try: + s = value.rstrip("Z") + dt = datetime.fromisoformat(s).replace(tzinfo=timezone.utc) + return int(dt.timestamp()) + except ValueError: + return None + + +def _extract_ethrex_version(versions: dict[str, str]) -> str | None: + """Extract ethrex version string from the versions dict. Returns None if absent.""" + for key, val in versions.items(): + if "ethrex" in key.lower(): + # Version string looks like "ethrex/v13.0.0-bal-devnet-7-/..." + # Return the raw version string, truncated at 80 chars + return val.strip()[:80] + return None + + +def _suite_id(entry: dict[str, Any]) -> str: + """ + Derive a stable suite_id from the run. Use `fileName` (the JSON log filename) + as it encodes a timestamp + hash and is unique per run. + """ + fname = entry.get("fileName", "") + if fname: + # Strip .json extension for cleanliness + return fname.replace(".json", "") + # Fallback: use start timestamp + name hash + return f"{entry.get('start', '')}_{entry.get('name', '')}" + + +def _is_ethrex_run(entry: dict[str, Any]) -> bool: + """Return True if this run involves ethrex as the execution client.""" + clients = entry.get("clients") or [] + for c in clients: + if "ethrex" in str(c).lower(): + return True + versions = entry.get("versions") or {} + for k in versions: + if "ethrex" in k.lower(): + return True + return False + + +def collect_hive(devnet: str) -> None: + """ + For each group in the devnet's hive_groups, fetch listing.jsonl from + hive.ethpandaops.io and upsert ethrex suite runs into hive_runs. + + listing.jsonl is ordered newest-first. We parse all lines and upsert the + latest ethrex run per (group, suite name) combination. + + Real endpoint shape (confirmed): + {"name": "eels/consume-engine", "ntests": 38208, "passes": 38208, + "fails": 0, "timeout": false, "clients": ["ethrex_default"], + "versions": {"ethrex_default": "ethrex/v13.0.0-..."}, + "start": "2026-06-08T15:18:27.36983134Z", + "fileName": "1780938802-1b35c1f406f2494248d3edc0039df8de.json", + "size": 136901252, + "simLog": "..."} + """ + entry = devnet_entry(devnet) + hive_groups: list[str] = entry.get("hive_groups") or [] + if not hive_groups: + print(f"collect_hive({devnet}): no hive_groups configured") + return + + conn = connect() + migrate(conn) + + total_inserted = 0 + + for group in hive_groups: + url = f"{_HIVE_BASE}/{group}/listing.jsonl" + try: + resp = requests.get(url, timeout=30) + resp.raise_for_status() + except requests.RequestException as exc: + logger.error("hive fetch error for group %s: %s", group, exc) + print(f"collect_hive({devnet}): error fetching {url}: {exc}") + continue + + inserted = 0 + for line in resp.text.splitlines(): + line = line.strip() + if not line: + continue + try: + import json + run = json.loads(line) + except Exception: + continue + + if not _is_ethrex_run(run): + continue + + suite_id = _suite_id(run) + versions = run.get("versions") or {} + ethrex_ver = _extract_ethrex_version(versions) + started_at = _parse_iso(run.get("start")) + + web_url = f"{_HIVE_BASE}/{group}/{run.get('fileName', '')}" + + row: dict[str, Any] = { + "devnet": devnet, + "group_name": group, + "suite_id": suite_id, + "ethrex_version": ethrex_ver, + "fork_filter": run.get("name"), + "ntests": run.get("ntests"), + "passes": run.get("passes"), + "fails": run.get("fails"), + "started_at": started_at, + "web_url": web_url, + } + upsert(conn, "hive_runs", row) + inserted += 1 + + conn.commit() + total_inserted += inserted + print( + f"collect_hive({devnet}): group={group}, {inserted} ethrex runs upserted" + ) + + conn.close() + print(f"collect_hive({devnet}): total {total_inserted} runs across all groups") + + +def show_hive(devnet: str) -> None: + """ + Print a summary table of the most recent ethrex Hive run per + (group_name, fork_filter) pair from hive_runs. + + Columns: group | suite | version | passes/fails/ntests | started_at | url + """ + conn = connect() + migrate(conn) + + rows = conn.execute( + """ + SELECT group_name, fork_filter, ethrex_version, + passes, fails, ntests, started_at, web_url + FROM hive_runs + WHERE devnet = ? + ORDER BY group_name, fork_filter, started_at DESC + """, + (devnet,), + ).fetchall() + conn.close() + + if not rows: + print(f"hive({devnet}): no runs found. Run: dv collect {devnet} hive") + return + + # Deduplicate: keep the newest run per (group, suite) + seen: set[tuple[str, str]] = set() + deduped = [] + for r in rows: + key = (r["group_name"], r["fork_filter"] or "") + if key not in seen: + seen.add(key) + deduped.append(r) + + print(f"\nHive runs for {devnet} (most recent ethrex run per suite)\n") + header = f"{'GROUP':<12} {'SUITE':<30} {'PASS/FAIL/TOTAL':<18} {'STARTED':<22} URL" + print(header) + print("-" * len(header)) + + for r in deduped: + group = r["group_name"] or "" + suite = (r["fork_filter"] or "")[:28] + passes = r["passes"] if r["passes"] is not None else "?" + fails = r["fails"] if r["fails"] is not None else "?" + ntests = r["ntests"] if r["ntests"] is not None else "?" + pft = f"{passes}/{fails}/{ntests}" + + started = "" + if r["started_at"]: + started = datetime.fromtimestamp(r["started_at"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + + url = r["web_url"] or "" + print(f"{group:<12} {suite:<30} {pft:<18} {started:<22} {url}") diff --git a/devnets_monitor/devnets/network.py b/devnets_monitor/devnets/network.py new file mode 100644 index 0000000..dd96fe8 --- /dev/null +++ b/devnets_monitor/devnets/network.py @@ -0,0 +1,500 @@ +"""Network splits, client distribution, and overview collector.""" + +from __future__ import annotations + +import json +import logging +import time +from datetime import datetime, timezone +from typing import Any + +from .config import devnet_entry +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + +# Real Dora API shapes (verified against live glamsterdam-devnet-5): +# +# GET /api/v1/network/splits +# data.splits[]: fork_id, head_slot, head_root, head_block_hash, +# head_execution_number, total_chain_weight, last_epoch_votes[], +# last_epoch_participation[], is_canonical +# +# GET /api/v1/network/client_head_forks +# data.forks[]: head_slot, head_root, client_count, +# clients[]: index, name, version, status, head_slot, distance, last_refresh +# data.fork_count (int) +# +# GET /api/v1/network/overview +# data.network_info: network_name, genesis_time, ... +# data.current_state: current_slot, current_epoch, current_epoch_progress, ... +# data.checkpoints: finalized_epoch, finalized_root, justified_epoch, ... +# data.forks[]: name, version, epoch, active, scheduled, time, type, fork_digest +# +# GET /api/v1/clients/execution +# clients[]: client_name, node_id, enode, ip, port, version, status, last_update +# +# GET /api/v1/clients/consensus +# clients[]: client_name, client_type, version, peer_id, head_slot, head_root, +# status, peer_count, peers_inbound, peers_outbound, last_refresh + + +def _fetch(url: str) -> dict | None: + """GET url with a single retry on transient failure. Returns parsed JSON or None.""" + import requests + + for attempt in range(2): + try: + resp = requests.get(url, timeout=20) + if resp.status_code == 200: + return resp.json() + logger.warning("network fetch %s -> HTTP %d", url, resp.status_code) + except Exception as exc: + logger.warning("network fetch %s error (attempt %d): %s", url, attempt, exc) + return None + + +def collect_network(devnet: str) -> None: + """ + Fetch network splits and client head forks from Dora, upsert into SQLite. + Each fetch is wrapped in its own try/except so a partial failure does not abort. + """ + entry = devnet_entry(devnet) + dora_base = entry.get("dora_base", "").rstrip("/") + if not dora_base: + logger.error("collect_network: dora_base missing for devnet %s", devnet) + print(f"collect_network({devnet}): dora_base missing, skipped") + return + + conn = connect() + migrate(conn) + ts = int(time.time()) + + splits_inserted = 0 + forks_processed = 0 + dist_inserted = 0 + + # --- Fetch splits --- + try: + data = _fetch(f"{dora_base}/api/v1/network/splits") + if data and data.get("status") == "OK": + splits = (data.get("data") or {}).get("splits") or [] + for s in splits: + head_root = s.get("head_root", "") or "" + row: dict[str, Any] = { + "devnet": devnet, + "ts": ts, + "head_root": head_root, + "head_slot": s.get("head_slot"), + "head_count": None, # filled from client_head_forks + "is_canonical": 1 if s.get("is_canonical") else 0, + "clients_json": None, + "fork_id": str(s.get("fork_id", "")), + } + upsert(conn, "network_splits", row) + splits_inserted += 1 + except Exception as exc: + logger.warning("collect_network splits error for %s: %s", devnet, exc) + + # --- Fetch client_head_forks (includes per-client data + head counts) --- + try: + data = _fetch(f"{dora_base}/api/v1/network/client_head_forks") + if data and data.get("status") == "OK": + forks = (data.get("data") or {}).get("forks") or [] + for fork in forks: + head_root = fork.get("head_root", "") or "" + clients = fork.get("clients") or [] + clients_json = json.dumps(clients) + head_count = fork.get("client_count") or len(clients) + + # Merge into the fork row from the splits fetch WITHOUT clobbering + # is_canonical / fork_id (a full upsert would overwrite them with + # NULL). Targeted insert-or-update of only the client-side fields. + conn.execute( + """INSERT INTO network_splits + (devnet, ts, head_root, head_slot, head_count, clients_json, + is_canonical, fork_id) + VALUES (?,?,?,?,?,?,NULL,NULL) + ON CONFLICT(devnet, ts, head_root) DO UPDATE SET + head_count=excluded.head_count, + clients_json=excluded.clients_json, + head_slot=COALESCE(excluded.head_slot, network_splits.head_slot)""", + (devnet, ts, head_root, fork.get("head_slot"), head_count, clients_json), + ) + + # Extract client distribution from version strings + cl_counts: dict[str, int] = {} + for c in clients: + version = c.get("version") or "" + cl_name = _parse_cl_client(version) + cl_counts[cl_name] = cl_counts.get(cl_name, 0) + 1 + + for cl_name, cnt in cl_counts.items(): + dist_row: dict[str, Any] = { + "devnet": devnet, + "ts": ts, + "layer": "cl", + "client": cl_name, + "version": "", + "count": cnt, + } + upsert(conn, "client_dist", dist_row) + dist_inserted += 1 + + forks_processed += 1 + except Exception as exc: + logger.warning("collect_network client_head_forks error for %s: %s", devnet, exc) + + conn.commit() + conn.close() + print( + f"collect_network({devnet}): {splits_inserted} split rows, " + f"{forks_processed} fork entries, {dist_inserted} client_dist rows at ts={ts}" + ) + + +def _parse_cl_client(version: str) -> str: + """ + Extract CL client name from version string. + Examples: 'Lighthouse/v8.1.3-...' -> 'lighthouse', + 'Prysm/v7.1.3-...' -> 'prysm', + 'Grandine/2.0.4-...' -> 'grandine'. + """ + if not version: + return "unknown" + name = version.split("/")[0].split(" ")[0].lower() + return name or "unknown" + + +def _latest_network_rows(conn: Any, devnet: str) -> list[dict]: + """ + Return the most recent network_splits rows for a devnet (all forks at latest ts). + Used by detectors. + """ + ts_row = conn.execute( + "SELECT MAX(ts) AS max_ts FROM network_splits WHERE devnet=?", (devnet,) + ).fetchone() + if not ts_row or ts_row["max_ts"] is None: + return [] + latest_ts = ts_row["max_ts"] + rows = conn.execute( + "SELECT * FROM network_splits WHERE devnet=? AND ts=?", + (devnet, latest_ts), + ).fetchall() + return [dict(r) for r in rows] + + +def get_network_data(devnet: str) -> dict[str, Any] | None: + """ + Return network data suitable for template rendering. + Returns None if no data is available. + """ + conn = connect() + migrate(conn) + + splits = _latest_network_rows(conn, devnet) + if not splits: + conn.close() + return None + + latest_ts = splits[0]["ts"] if splits else None + + # Load client distribution at same ts + dist_rows = [] + if latest_ts: + dist_rows = conn.execute( + """SELECT layer, client, version, count FROM client_dist + WHERE devnet=? AND ts=? ORDER BY layer, count DESC""", + (devnet, latest_ts), + ).fetchall() + + # Parse clients_json for the canonical fork + canonical_fork = None + for s in splits: + if s.get("is_canonical") or s.get("clients_json"): + # Prefer the row with clients_json + if s.get("clients_json"): + canonical_fork = s + break + if canonical_fork is None and splits: + canonical_fork = splits[0] + + clients_list = [] + if canonical_fork and canonical_fork.get("clients_json"): + try: + clients_list = json.loads(canonical_fork["clients_json"]) + except Exception: + pass + + conn.close() + + return { + "ts": latest_ts, + "splits": splits, + "clients": clients_list, + "client_dist": [dict(r) for r in dist_rows], + "fork_count": len(splits), + } + + +def show_network(devnet: str) -> None: + """Print network split and client distribution summary.""" + from datetime import datetime, timezone + + data = get_network_data(devnet) + if data is None: + print(f"network({devnet}): no data. Run: dv collect {devnet} network") + return + + ts_str = "" + if data["ts"]: + try: + ts_str = datetime.fromtimestamp(data["ts"], tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + except Exception: + ts_str = str(data["ts"]) + + print(f"\nNetwork status for {devnet} (at {ts_str})\n") + + print(f"Head forks: {data['fork_count']}") + for s in data["splits"]: + canonical = " [canonical]" if s.get("is_canonical") else "" + count = s.get("head_count", "?") + print( + f" fork_id={s.get('fork_id', '?')} " + f"head_slot={s.get('head_slot', '?')} " + f"head_root={str(s.get('head_root', ''))[:12]}... " + f"clients={count}{canonical}" + ) + + if data["clients"]: + print(f"\nClients on canonical head ({len(data['clients'])}):\n") + print(f" {'NAME':<32} {'STATUS':<12} {'DISTANCE':>8} {'HEAD_SLOT':>10}") + print(" " + "-" * 68) + for c in data["clients"]: + name = (c.get("name") or "")[:31] + status = (c.get("status") or "")[:11] + distance = c.get("distance", 0) + head_slot = c.get("head_slot", "?") + print(f" {name:<32} {status:<12} {distance:>8} {head_slot:>10}") + + if data["client_dist"]: + print("\nCL client distribution:\n") + for r in data["client_dist"]: + print(f" {r['client']:<16} {r['count']:>4}") + + print() + + +# --------------------------------------------------------------------------- +# collect_clients / get_clients_data / show_clients (T1) +# --------------------------------------------------------------------------- + + +def _parse_el_client(version: str) -> tuple[str, str]: + """ + Extract EL client name and short version from version string. + Examples: + 'ethrex/v15.0.0-glamsterdam...' -> ('ethrex', 'v15.0.0-glamsterdam...') + 'Nethermind/v1.39.0-...' -> ('nethermind', 'v1.39.0-...') + 'erigon/v3.6.0-...' -> ('erigon', 'v3.6.0-...') + """ + if not version: + return "unknown", "" + parts = version.split("/", 1) + name = parts[0].lower() + ver = parts[1].split("/")[0] if len(parts) > 1 else "" + return name or "unknown", ver + + +def collect_clients(devnet: str) -> None: + """ + Fetch EL+CL client lists and network overview from Dora. + Upserts EL distribution into client_dist (layer='el'). + Upserts overview summary into network_overview table. + Each fetch is wrapped in its own try/except. + """ + entry = devnet_entry(devnet) + dora_base = entry.get("dora_base", "").rstrip("/") + if not dora_base: + logger.error("collect_clients: dora_base missing for devnet %s", devnet) + print(f"collect_clients({devnet}): dora_base missing, skipped") + return + + conn = connect() + migrate(conn) + ts = int(time.time()) + + el_dist_inserted = 0 + overview_stored = 0 + + # --- EL clients --- + try: + data = _fetch(f"{dora_base}/api/v1/clients/execution") + if data is not None: + clients = data.get("clients") or [] + # Aggregate by (client_name, version) + el_counts: dict[tuple[str, str], int] = {} + for c in clients: + name, ver = _parse_el_client(c.get("version") or "") + # Truncate version to short tag to avoid unbounded cardinality + ver_short = ver[:64] + el_counts[(name, ver_short)] = el_counts.get((name, ver_short), 0) + 1 + + for (name, ver_short), cnt in el_counts.items(): + dist_row: dict[str, Any] = { + "devnet": devnet, + "ts": ts, + "layer": "el", + "client": name, + "version": ver_short, + "count": cnt, + } + upsert(conn, "client_dist", dist_row) + el_dist_inserted += 1 + except Exception as exc: + logger.warning("collect_clients EL error for %s: %s", devnet, exc) + + # --- Network overview --- + try: + data = _fetch(f"{dora_base}/api/v1/network/overview") + if data and data.get("status") == "OK": + payload = data.get("data") or {} + cur = payload.get("current_state") or {} + chk = payload.get("checkpoints") or {} + overview_row: dict[str, Any] = { + "devnet": devnet, + "ts": ts, + "current_slot": cur.get("current_slot"), + "current_epoch": cur.get("current_epoch"), + "finalized_epoch": chk.get("finalized_epoch"), + "justified_epoch": chk.get("justified_epoch"), + "json": json.dumps(payload), + } + upsert(conn, "network_overview", overview_row) + overview_stored = 1 + except Exception as exc: + logger.warning("collect_clients overview error for %s: %s", devnet, exc) + + conn.commit() + conn.close() + print( + f"collect_clients({devnet}): {el_dist_inserted} EL dist rows, " + f"{overview_stored} overview rows at ts={ts}" + ) + + +def get_clients_data(devnet: str) -> dict[str, Any] | None: + """ + Return clients data suitable for template rendering. + Includes EL+CL distribution, ethrex versions live, fork agreement, overview. + Returns None if no data is available. + """ + conn = connect() + migrate(conn) + + # Latest ts that has any client_dist row for this devnet + ts_row = conn.execute( + "SELECT MAX(ts) AS max_ts FROM client_dist WHERE devnet=?", (devnet,) + ).fetchone() + if not ts_row or ts_row["max_ts"] is None: + conn.close() + return None + + latest_ts = ts_row["max_ts"] + + dist_rows = conn.execute( + """SELECT layer, client, version, count FROM client_dist + WHERE devnet=? AND ts=? ORDER BY layer, count DESC""", + (devnet, latest_ts), + ).fetchall() + + # ethrex-specific: gather versions from client_dist (layer='el', client='ethrex') + ethrex_versions = [ + {"version": r["version"], "count": r["count"]} + for r in dist_rows + if r["layer"] == "el" and r["client"] == "ethrex" + ] + + # Latest network_overview for is-finalized status + overview = None + ov_row = conn.execute( + """SELECT current_slot, current_epoch, finalized_epoch, justified_epoch, json + FROM network_overview WHERE devnet=? ORDER BY ts DESC LIMIT 1""", + (devnet,), + ).fetchone() + if ov_row: + overview = dict(ov_row) + try: + overview["parsed"] = json.loads(ov_row["json"] or "{}") + except Exception: + overview["parsed"] = {} + + # Fork agreement from network_splits (most recent) + ts_ns = conn.execute( + "SELECT MAX(ts) AS max_ts FROM network_splits WHERE devnet=?", (devnet,) + ).fetchone() + fork_count = 0 + if ts_ns and ts_ns["max_ts"] is not None: + fc = conn.execute( + "SELECT COUNT(*) AS c FROM network_splits WHERE devnet=? AND ts=?", + (devnet, ts_ns["max_ts"]), + ).fetchone() + fork_count = fc["c"] if fc else 0 + + conn.close() + + ts_str = "" + try: + ts_str = datetime.fromtimestamp(latest_ts, tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + ts_str = str(latest_ts) + + return { + "ts": latest_ts, + "ts_str": ts_str, + "client_dist": [dict(r) for r in dist_rows], + "ethrex_versions": ethrex_versions, + "fork_count": fork_count, + "overview": overview, + } + + +def show_clients(devnet: str) -> None: + """Print EL+CL client diversity and ethrex version summary.""" + data = get_clients_data(devnet) + if data is None: + print(f"clients({devnet}): no data. Run: dv collect {devnet} clients") + return + + print(f"\nClients for {devnet} (at {data['ts_str']})\n") + + if data["ethrex_versions"]: + print("ethrex versions live:") + for ev in data["ethrex_versions"]: + print(f" {ev['version'][:80]} (count={ev['count']})") + print() + + if data["client_dist"]: + print(f"{'LAYER':<6} {'CLIENT':<16} {'COUNT':>5} VERSION") + print("-" * 60) + for r in data["client_dist"]: + ver = (r["version"] or "")[:40] + print(f" {r['layer']:<4} {r['client']:<16} {r['count']:>5} {ver}") + print() + + ov = data.get("overview") + if ov: + print( + f"Network: slot={ov.get('current_slot')}, epoch={ov.get('current_epoch')}, " + f"finalized={ov.get('finalized_epoch')}" + ) + print() + + print(f"Head forks (splits): {data['fork_count']}") + print( + f"Forkmon: https://forkmon.{devnet}.ethpandaops.io " + f"(no probed API, link-out only)" + ) + print() diff --git a/devnets_monitor/devnets/peers.py b/devnets_monitor/devnets/peers.py new file mode 100644 index 0000000..6e8d49b --- /dev/null +++ b/devnets_monitor/devnets/peers.py @@ -0,0 +1,77 @@ +"""Peer inspection: peer count, inbound/outbound split, client mix, body-serving failures.""" + +from __future__ import annotations + +import sys + +from .config import host_of +from .remote import PEERS_PROBE +from .ssh import run_remote + + +def _parse_peers_tsv(output: str) -> dict: + """ + Parse PEERS_PROBE TSV output into a structured dict. + + Expected lines: + peercount + total + inbound + outbound + client (zero or more) + bodyfail + """ + data: dict = { + "peercount": "?", + "total": "?", + "inbound": "?", + "outbound": "?", + "clients": {}, + "bodyfail": "?", + } + for line in output.splitlines(): + parts = line.split("\t") + if not parts: + continue + key = parts[0].strip() + if key == "client" and len(parts) >= 3: + name = parts[1] + count = parts[2].strip() + data["clients"][name] = count + elif key in ("peercount", "total", "inbound", "outbound", "bodyfail") and len(parts) >= 2: + data[key] = parts[1].strip() + return data + + +def peers(devnet: str, node: str) -> None: + """Run PEERS_PROBE on a specific node and print the peer report.""" + host = host_of(devnet, node) + try: + result = run_remote(host, PEERS_PROBE, timeout=30) + except Exception as exc: + print(f"error: SSH to {host} failed: {exc}", file=sys.stderr) + sys.exit(1) + + if result.returncode != 0: + print( + f"error: peers probe failed (exit {result.returncode}):\n{result.stderr.strip()}", + file=sys.stderr, + ) + sys.exit(1) + + d = _parse_peers_tsv(result.stdout) + + print(f"### {node}") + print(f" peercount (eth_peerCount): {d['peercount']}") + print(f" admin_peers total: {d['total']}") + print(f" inbound: {d['inbound']}") + print(f" outbound: {d['outbound']}") + print() + if d["clients"]: + print(" Client mix:") + for name, count in sorted(d["clients"].items(), key=lambda kv: -int(kv[1]) if kv[1].isdigit() else 0): + print(f" {count:>4} {name}") + else: + print(" Client mix: (no admin_peers data)") + print() + print(f" Body-serving failures (last 60s): {d['bodyfail']}") diff --git a/devnets_monitor/devnets/proposals.py b/devnets_monitor/devnets/proposals.py new file mode 100644 index 0000000..17bc669 --- /dev/null +++ b/devnets_monitor/devnets/proposals.py @@ -0,0 +1,154 @@ +"""Proposals analysis: per-proposer canonical/missed/orphaned summary.""" + +from __future__ import annotations + +from typing import Any + +from .blobtrack import _client_from_proposer, _resolve_since +from .store import connect, migrate + + +def get_proposals_data(devnet: str, since: str | None = None) -> dict[str, Any] | None: + """ + Return per-proposer canonical/missed/orphaned counts and orphan rates. + Also aggregates by client (ethrex vs others). + Returns None if no slot data exists. + """ + conn = connect() + migrate(conn) + + slot_count = _resolve_since(since) + + base_query = "SELECT slot, proposer_name, status FROM slots WHERE devnet = ?" + params: list[Any] = [devnet] + + if slot_count is not None: + max_row = conn.execute( + "SELECT MAX(slot) FROM slots WHERE devnet = ?", (devnet,) + ).fetchone() + if max_row and max_row[0] is not None: + cutoff = max_row[0] - slot_count + base_query += " AND slot >= ?" + params.append(cutoff) + + base_query += " ORDER BY slot ASC" + + rows = conn.execute(base_query, params).fetchall() + conn.close() + + if not rows: + return None + + # Per-proposer counts + proposer_stats: dict[str, dict[str, int]] = {} + for r in rows: + name = r["proposer_name"] or "unknown" + status = (r["status"] or "unknown").lower() + if name not in proposer_stats: + proposer_stats[name] = {"canonical": 0, "missed": 0, "orphaned": 0, "total": 0} + proposer_stats[name]["total"] += 1 + if status == "canonical": + proposer_stats[name]["canonical"] += 1 + elif status in ("missing", "missed"): + proposer_stats[name]["missed"] += 1 + elif status == "orphaned": + proposer_stats[name]["orphaned"] += 1 + + # Per-client aggregation + client_stats: dict[str, dict[str, int]] = {} + for name, stats in proposer_stats.items(): + client = _client_from_proposer(name) + if client not in client_stats: + client_stats[client] = {"canonical": 0, "missed": 0, "orphaned": 0, "total": 0} + for k in ("canonical", "missed", "orphaned", "total"): + client_stats[client][k] += stats[k] + + # Build output rows with orphan rate + proposers_out = [] + for name, stats in sorted( + proposer_stats.items(), + key=lambda kv: kv[1]["orphaned"] / kv[1]["total"] if kv[1]["total"] else 0, + reverse=True, + ): + total = stats["total"] + orphan_rate = stats["orphaned"] / total if total else 0.0 + client = _client_from_proposer(name) + proposers_out.append({ + "name": name, + "client": client, + "is_ethrex": client == "ethrex", + "canonical": stats["canonical"], + "missed": stats["missed"], + "orphaned": stats["orphaned"], + "total": total, + "orphan_rate": round(orphan_rate, 4), + }) + + clients_out = [] + for client, stats in sorted( + client_stats.items(), + key=lambda kv: kv[1]["orphaned"] / kv[1]["total"] if kv[1]["total"] else 0, + reverse=True, + ): + total = stats["total"] + orphan_rate = stats["orphaned"] / total if total else 0.0 + clients_out.append({ + "client": client, + "is_ethrex": client == "ethrex", + "canonical": stats["canonical"], + "missed": stats["missed"], + "orphaned": stats["orphaned"], + "total": total, + "orphan_rate": round(orphan_rate, 4), + }) + + total_slots = len(rows) + max_slot_num = max(r["slot"] for r in rows) + min_slot_num = min(r["slot"] for r in rows) + window_label = f"slots {min_slot_num}-{max_slot_num} ({total_slots} slots)" + + return { + "window_label": window_label, + "min_slot": min_slot_num, + "max_slot": max_slot_num, + "total_slots": total_slots, + "proposers": proposers_out, + "clients": clients_out, + } + + +def show_proposals(devnet: str, since: str | None = None) -> None: + """Print per-proposer and per-client orphan summary.""" + data = get_proposals_data(devnet, since=since) + if data is None: + print(f"proposals({devnet}): no slot data. Run: dv collect {devnet} blobs") + return + + print(f"\nProposals summary ({devnet})") + print(f"Window: {data['window_label']}\n") + + # Client comparison + print("EL client comparison:\n") + print(f"{'CLIENT':<16} {'TOTAL':>7} {'CANONICAL':>10} {'MISSED':>7} {'ORPHANED':>9} {'ORPHAN%':>9}") + print("-" * 65) + for c in data["clients"]: + marker = " <-- ethrex" if c["is_ethrex"] else "" + print( + f"{c['client']:<16} {c['total']:>7} {c['canonical']:>10} " + f"{c['missed']:>7} {c['orphaned']:>9} {c['orphan_rate']:>8.1%}{marker}" + ) + + print(f"\nPer-proposer breakdown ({len(data['proposers'])} proposers):\n") + col_w = max((len(p["name"]) for p in data["proposers"]), default=20) + 2 + print( + f"{'PROPOSER':<{col_w}} {'CLIENT':<12} {'TOTAL':>6} " + f"{'CANONICAL':>10} {'MISSED':>7} {'ORPHANED':>9} {'ORPHAN%':>9}" + ) + print("-" * (col_w + 65)) + for p in data["proposers"]: + print( + f"{p['name']:<{col_w}} {p['client']:<12} {p['total']:>6} " + f"{p['canonical']:>10} {p['missed']:>7} {p['orphaned']:>9} " + f"{p['orphan_rate']:>8.1%}" + ) + print() diff --git a/devnets_monitor/devnets/remote.py b/devnets_monitor/devnets/remote.py new file mode 100644 index 0000000..15867e5 --- /dev/null +++ b/devnets_monitor/devnets/remote.py @@ -0,0 +1,231 @@ +""" +Host-side shell snippets, sent over 'ssh bash -s'. + +LINE PROTOCOL (TSV): + STATUS_PROBE and PEERS_PROBE emit lines of the form: + keyvalue + One key-value pair per line. Python assembles the dict in-process. + This avoids hand-rolled JSON quoting bugs (cl_line can contain quotes/ANSI). + + STATUS_PROBE keys: image, status, buildnum, commit, restart, head, peers, + syncing, state_at_head, watchtower, cl_line + + PEERS_PROBE keys: peercount, total, inbound, outbound, + then zero or more: client + then: bodyfail + +WIPE_SEQUENCE takes: $1 = image tag (full tag, e.g. ethpandaops/ethrex:devnet-5) + +LOGS_TAIL takes: $1 = since value (e.g. 2m, 30s), pre-validated by Python +CL_TAIL takes: $1 = since value, pre-validated by Python +""" + +# --------------------------------------------------------------------------- +# STATUS_PROBE +# --------------------------------------------------------------------------- +# Emits TSV lines: keyvalue +# cl_line is truncated to 200 chars, tabs and newlines stripped. +STATUS_PROBE = r""" +set -uo pipefail + +rpc(){ curl -s --max-time 4 localhost:8545 -H 'Content-Type: application/json' -d "$1"; } + +hx(){ + v=$(grep -oE '0x[0-9a-f]+' <<<"$1" | head -1) + [ -n "$v" ] && printf '%d' "$v" || echo "null" +} + +img=$(docker inspect execution --format '{{.Config.Image}}' 2>/dev/null || echo "") +status=$(docker inspect execution --format '{{.State.Status}}' 2>/dev/null || echo "no-EL") +restart=$(docker inspect execution --format '{{.RestartCount}}' 2>/dev/null || echo 0) +buildnum=$(docker inspect execution --format '{{index .Config.Labels "buildnum"}}' 2>/dev/null || echo "") +commit=$(docker inspect execution --format '{{index .Config.Labels "commit"}}' 2>/dev/null || echo "") + +bn_raw=$(rpc '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}') +head=$(hx "$bn_raw") + +pc_raw=$(rpc '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}') +peers=$(hx "$pc_raw") + +sy_raw=$(rpc '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}') + +if grep -q '"result":false' <<<"$sy_raw"; then + syncing="synced(false)" +else + cur=$(grep -oE '"currentBlock":"0x[0-9a-f]+"' <<<"$sy_raw" | grep -oE '0x[0-9a-f]+' | head -1) + hi=$(grep -oE '"highestBlock":"0x[0-9a-f]+"' <<<"$sy_raw" | grep -oE '0x[0-9a-f]+' | head -1) + if [ -n "$cur" ] && [ -n "$hi" ]; then + cur_d=$(printf '%d' "$cur") + hi_d=$(printf '%d' "$hi") + syncing="cur=${cur_d}->hi=${hi_d}" + else + syncing="unknown" + fi +fi + +gb_raw=$(rpc '{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x0000000000000000000000000000000000000000","latest"],"id":1}') +if grep -q '"result"' <<<"$gb_raw"; then + state_at_head="yes" +else + state_at_head="no" +fi + +wt=$(docker inspect ethereum-node-docker-watchtower --format '{{.State.Status}}' 2>/dev/null || echo "?") + +# Bound by line count, not time window: a chatty CL (e.g. nimbus) can emit +# thousands of lines in 4m, making `docker logs --since 4m` take 30s+ and blow +# the probe timeout. `--tail` is O(N lines), fast regardless of verbosity. +cl_raw=$(docker logs --tail 400 beacon 2>&1 | grep -iE "Synced|Syncing|Slot start|head slot|exec-block|descendant|empty slots" | tail -1 || echo "") +# Strip tabs, newlines, and ANSI escape sequences; truncate to 200 chars +cl_line=$(printf '%s' "$cl_raw" | sed 's/\x1b\[[0-9;]*[mGKHF]//g' | tr '\t\n\r' ' ' | cut -c1-200) + +printf 'image\t%s\n' "$img" +printf 'status\t%s\n' "$status" +printf 'buildnum\t%s\n' "$buildnum" +printf 'commit\t%s\n' "$commit" +printf 'restart\t%s\n' "$restart" +printf 'head\t%s\n' "$head" +printf 'peers\t%s\n' "$peers" +printf 'syncing\t%s\n' "$syncing" +printf 'state_at_head\t%s\n' "$state_at_head" +printf 'watchtower\t%s\n' "$wt" +printf 'cl_line\t%s\n' "$cl_line" +""" + +# --------------------------------------------------------------------------- +# PEERS_PROBE +# --------------------------------------------------------------------------- +# Emits TSV lines: +# peercount +# total +# inbound +# outbound +# client (one per distinct client name, zero or more) +# bodyfail +# +# Uses mktemp for a safe temp file; cleans up on exit. +PEERS_PROBE = r""" +set -uo pipefail + +rpc(){ curl -s --max-time 4 localhost:8545 -H 'Content-Type: application/json' -d "$1"; } + +tmpfile=$(mktemp) +trap 'rm -f "$tmpfile"' EXIT + +pc_raw=$(rpc '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}') +peercount=$(grep -oE '0x[0-9a-f]+' <<<"$pc_raw" | head -1 | xargs -I{} printf '%d' {} 2>/dev/null || echo 0) + +rpc '{"jsonrpc":"2.0","method":"admin_peers","params":[],"id":1}' > "$tmpfile" + +total=$(grep -oE '"enode"' "$tmpfile" | wc -l) +inbound=$(grep -c '"inbound":true' "$tmpfile" || true) +outbound=$(grep -c '"inbound":false' "$tmpfile" || true) + +printf 'peercount\t%s\n' "$peercount" +printf 'total\t%s\n' "$total" +printf 'inbound\t%s\n' "$inbound" +printf 'outbound\t%s\n' "$outbound" + +# Client name histogram: extract "name":"" fields +grep -oE '"name":"[^"]*"' "$tmpfile" | grep -oE '"[^"]*"$' | tr -d '"' | sort | uniq -c | \ + awk '{count=$1; $1=""; name=substr($0,2); printf "client\t%s\t%s\n", name, count}' + +# Body-serving failures in the last 60s +bodyfail=$(docker logs --since 60s execution 2>&1 | grep -c "Didn.t receive block bodies" || true) +printf 'bodyfail\t%s\n' "$bodyfail" +""" + +# --------------------------------------------------------------------------- +# WIPE_SEQUENCE +# --------------------------------------------------------------------------- +# Args: $1 = image tag (full, e.g. ethpandaops/ethrex:devnet-5) +# Incident-tested sequence; do not break. +WIPE_SEQUENCE = r""" +set -euo pipefail + +IMAGE="$1" + +if [ -z "$IMAGE" ]; then + echo "ABORT: no image tag supplied as \$1" >&2 + exit 1 +fi + +echo "==> Pausing watchtower..." +docker pause ethereum-node-docker-watchtower || true + +echo "==> Capturing current execution container config with runlike..." +runfile=$(mktemp) +trap 'rm -f "$runfile"' EXIT + +runlike execution > "$runfile" 2>&1 +captured=$(cat "$runfile") + +if ! grep -q '^docker run ' "$runfile"; then + echo "ABORT: runlike capture does not start with 'docker run'. Contents:" >&2 + cat "$runfile" >&2 + echo "==> Unpausing watchtower due to abort..." + docker unpause ethereum-node-docker-watchtower || true + exit 1 +fi + +if grep -q '\-\-nat\.extip' "$runfile"; then + echo "(note: --nat.extip is present in the run command)" +fi + +echo "==> Pulling image: $IMAGE ..." +docker pull "$IMAGE" + +echo "==> Removing execution container..." +docker rm -f execution + +echo "==> Wiping datadir via root container..." +docker run --rm --user 0 --entrypoint sh -v /data/ethrex:/d "$IMAGE" \ + -c 'rm -rf /d/* /d/.[!.]*' + +echo "==> Recreating execution container (detached)..." +recreate=$(sed 's#^docker run #docker run -d #' "$runfile") +if ! echo "$recreate" | grep -q '^docker run -d '; then + echo "ABORT: recreate command does not start with 'docker run -d '" >&2 + echo "==> Unpausing watchtower due to abort..." + docker unpause ethereum-node-docker-watchtower || true + exit 1 +fi +bash -c "$recreate" + +echo "==> Restarting snooper-engine (REQUIRED: reconnects engine proxy to new EL)..." +docker restart snooper-engine || echo "(no snooper-engine; restart beacon manually if needed)" + +echo "==> Unpausing watchtower..." +docker unpause ethereum-node-docker-watchtower || true + +echo "==> Waiting 8s for EL to start..." +sleep 8 + +echo "==> Post-wipe status:" +curl -s --max-time 4 localhost:8545 -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' || true +echo "" +docker logs --since 12s execution 2>&1 | \ + grep -iE "Storing genesis|Unknown state|SYNCING|Sync target" | tail -2 || true +""" + +# --------------------------------------------------------------------------- +# LOGS_TAIL +# --------------------------------------------------------------------------- +# Args: $1 = since value (e.g. 2m, 30s); pre-validated by Python before sending. +LOGS_TAIL = r""" +set -uo pipefail +SINCE="$1" +docker logs --since "$SINCE" execution 2>&1 | grep -iE 'WARN|ERROR' | tail -30 +""" + +# --------------------------------------------------------------------------- +# CL_TAIL +# --------------------------------------------------------------------------- +# Args: $1 = since value (e.g. 3m); pre-validated by Python before sending. +CL_TAIL = r""" +set -uo pipefail +SINCE="$1" +docker logs --since "$SINCE" beacon 2>&1 | \ + grep -iE 'Synced|Syncing|Slot start|head|exec-block|descendant|finaliz' | tail -15 +""" diff --git a/devnets_monitor/devnets/spamoor.py b/devnets_monitor/devnets/spamoor.py new file mode 100644 index 0000000..c23a679 --- /dev/null +++ b/devnets_monitor/devnets/spamoor.py @@ -0,0 +1,194 @@ +"""Spamoor status collector: fetch active spammers and persist into SQLite.""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime, timezone +from typing import Any + +from .store import connect, migrate, upsert + +logger = logging.getLogger(__name__) + +# Real Spamoor API shape (verified against live glamsterdam-devnet-5): +# +# GET https://spamoor..ethpandaops.io/api/spammers +# Returns a JSON ARRAY (no envelope) of: +# id, name, description, scenario, status (1=running, 0=stopped), +# created_at, is_group, group_id, +# member_config: { weight, enabled, sort_order } +# +# Blob spammer: id=3, scenario="blob-combined", status=1 (confirmed live). + + +def _fetch_spammers(devnet: str) -> list[dict[str, Any]] | None: + """GET /api/spammers from the Spamoor instance for this devnet.""" + import requests + + url = f"https://spamoor.{devnet}.ethpandaops.io/api/spammers" + for attempt in range(2): + try: + resp = requests.get(url, timeout=20) + if resp.status_code == 200: + data = resp.json() + if isinstance(data, list): + return data + logger.warning("spamoor: unexpected response shape for %s", devnet) + return None + if resp.status_code == 404: + logger.info("spamoor: 404 for %s (not deployed on this devnet)", devnet) + return None + logger.warning( + "spamoor fetch %s -> HTTP %d (attempt %d)", + devnet, resp.status_code, attempt, + ) + except Exception as exc: + logger.warning("spamoor fetch error for %s (attempt %d): %s", devnet, attempt, exc) + if attempt == 0: + time.sleep(1) + return None + + +def collect_spamoor(devnet: str) -> None: + """ + Fetch Spamoor spammer list and upsert into spamoor_status. + Degrades gracefully if the API is absent or returns 404. + """ + spammers = _fetch_spammers(devnet) + if spammers is None: + print(f"collect_spamoor({devnet}): API unavailable, skipped (link-out only)") + return + + conn = connect() + migrate(conn) + ts = int(time.time()) + + inserted = 0 + for s in spammers: + spammer_id = s.get("id") + if spammer_id is None: + continue + member_config = s.get("member_config") or {} + enabled_val = member_config.get("enabled") + row: dict[str, Any] = { + "devnet": devnet, + "ts": ts, + "spammer_id": int(spammer_id), + "name": str(s.get("name") or "")[:200], + "scenario": str(s.get("scenario") or "")[:200], + "status": int(s.get("status") or 0), + "enabled": 1 if enabled_val else 0, + } + upsert(conn, "spamoor_status", row) + inserted += 1 + + conn.commit() + conn.close() + print(f"collect_spamoor({devnet}): {inserted} spammer rows at ts={ts}") + + +def _blob_spammer_active(conn: Any, devnet: str) -> bool | None: + """ + Return True if a blob spammer is currently active (latest snapshot has + scenario containing 'blob' AND status==1). Return False if data exists + but no blob spammer is active. Return None if no spamoor data at all. + """ + ts_row = conn.execute( + "SELECT MAX(ts) AS max_ts FROM spamoor_status WHERE devnet=?", (devnet,) + ).fetchone() + if not ts_row or ts_row["max_ts"] is None: + return None + + latest_ts = ts_row["max_ts"] + rows = conn.execute( + "SELECT scenario, status FROM spamoor_status WHERE devnet=? AND ts=?", + (devnet, latest_ts), + ).fetchall() + + for r in rows: + scenario = (r["scenario"] or "").lower() + if "blob" in scenario and r["status"] == 1: + return True + return False + + +def get_spamoor_data(devnet: str) -> dict[str, Any] | None: + """ + Return spamoor data suitable for template rendering. + Returns None if no data is available. + """ + conn = connect() + migrate(conn) + + ts_row = conn.execute( + "SELECT MAX(ts) AS max_ts FROM spamoor_status WHERE devnet=?", (devnet,) + ).fetchone() + if not ts_row or ts_row["max_ts"] is None: + conn.close() + return None + + latest_ts = ts_row["max_ts"] + rows = conn.execute( + """SELECT spammer_id, name, scenario, status, enabled + FROM spamoor_status WHERE devnet=? AND ts=? + ORDER BY spammer_id""", + (devnet, latest_ts), + ).fetchall() + + blob_active = _blob_spammer_active(conn, devnet) + conn.close() + + ts_str = "" + try: + ts_str = datetime.fromtimestamp(latest_ts, tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + ts_str = str(latest_ts) + + spamoor_url = f"https://spamoor.{devnet}.ethpandaops.io" + + return { + "ts": latest_ts, + "ts_str": ts_str, + "spammers": [dict(r) for r in rows], + "blob_active": blob_active, + "spamoor_url": spamoor_url, + } + + +def show_spamoor(devnet: str) -> None: + """Print spamoor status summary to stdout.""" + data = get_spamoor_data(devnet) + if data is None: + print(f"spamoor({devnet}): no data. Run: dv collect {devnet} spamoor") + print(f" Link: https://spamoor.{devnet}.ethpandaops.io") + return + + print(f"\nSpamoor status for {devnet} (at {data['ts_str']})\n") + + blob_active = data["blob_active"] + if blob_active: + print(" BLOB LOAD: ON (blob spammer active)") + elif blob_active is False: + print(" BLOB LOAD: OFF (no active blob spammer)") + else: + print(" BLOB LOAD: unknown (no spamoor data)") + + print() + print(f" {'ID':>4} {'NAME':<36} {'SCENARIO':<24} {'STATUS':<8} {'ENABLED'}") + print(" " + "-" * 84) + for s in data["spammers"]: + status_str = "running" if s["status"] == 1 else "stopped" + enabled_str = "yes" if s["enabled"] else "no" + name = (s["name"] or "")[:35] + scenario = (s["scenario"] or "")[:23] + print( + f" {s['spammer_id']:>4} {name:<36} {scenario:<24} " + f"{status_str:<8} {enabled_str}" + ) + + print() + print(f" Spamoor UI: {data['spamoor_url']}") + print() diff --git a/devnets_monitor/devnets/ssh.py b/devnets_monitor/devnets/ssh.py new file mode 100644 index 0000000..cf91683 --- /dev/null +++ b/devnets_monitor/devnets/ssh.py @@ -0,0 +1,55 @@ +"""SSH helpers: run host-side shell snippets via 'ssh bash -s'.""" + +from __future__ import annotations + +import subprocess +from typing import Sequence + + +SSH_OPTS: list[str] = [ + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=5", + # Trip a dead/hung connection fast (server-alive probes; ~6s total) rather + # than waiting on the subprocess timeout. + "-o", "ServerAliveInterval=3", + "-o", "ServerAliveCountMax=2", +] + + +def run_remote( + host: str, + script: str, + args: Sequence[str] | None = None, + timeout: int = 60, +) -> subprocess.CompletedProcess[str]: + """ + Run a shell script on a remote host via 'ssh bash -s [args...]', + feeding the script on stdin. Positional args become $1, $2, ... on the remote. + + Values are passed as positional args, never interpolated into the script string. + Returns a CompletedProcess with stdout/stderr captured as text. + """ + cmd: list[str] = ["ssh"] + SSH_OPTS + [host, "bash", "-s"] + if args: + cmd.extend(args) + return subprocess.run( + cmd, + input=script, + capture_output=True, + text=True, + timeout=timeout, + ) + + +def run_remote_checked( + host: str, + script: str, + args: Sequence[str] | None = None, + timeout: int = 60, +) -> subprocess.CompletedProcess[str]: + """ + Same as run_remote but raises subprocess.CalledProcessError on nonzero exit. + """ + result = run_remote(host, script, args=args, timeout=timeout) + result.check_returncode() + return result diff --git a/devnets_monitor/devnets/status.py b/devnets_monitor/devnets/status.py new file mode 100644 index 0000000..379f566 --- /dev/null +++ b/devnets_monitor/devnets/status.py @@ -0,0 +1,124 @@ +"""Status sweep: run STATUS_PROBE per node, parse TSV, format output.""" + +from __future__ import annotations + +import json +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any + +from .config import host_of, node_list +from .remote import STATUS_PROBE +from .ssh import run_remote + + +def _parse_tsv(output: str) -> dict[str, str]: + """Parse TSV keyvalue lines into a dict.""" + result: dict[str, str] = {} + for line in output.splitlines(): + if "\t" in line: + key, _, value = line.partition("\t") + result[key.strip()] = value + return result + + +# Per-node probe timeout. A healthy probe (ssh + docker inspect + RPC + log tail) +# is ~1.5s; 5s leaves headroom while failing a wedged node fast. +PROBE_TIMEOUT = 5 + + +def _probe_node(devnet: str, node: str) -> dict[str, Any]: + """Run STATUS_PROBE on one node and return a parsed dict (with 'node' added).""" + host = host_of(devnet, node) + try: + result = run_remote(host, STATUS_PROBE, timeout=PROBE_TIMEOUT) + if result.returncode != 0: + return { + "node": node, + "_error": f"ssh exit {result.returncode}: {result.stderr.strip()[:200]}", + } + data = _parse_tsv(result.stdout) + data["node"] = node + return data + except Exception as exc: # includes TimeoutExpired, OSError + return {"node": node, "_error": str(exc)[:200]} + + +def _human_line(d: dict[str, Any]) -> str: + """Format a probe result as the human-readable status line.""" + node = d.get("node", "?") + if "_error" in d: + return f"### {node}\n ERROR: {d['_error']}" + + image = d.get("image", "") + status = d.get("status", "?") + restart = d.get("restart", "0") + buildnum = d.get("buildnum", "") + commit = d.get("commit", "") + head = d.get("head", "?") + peers = d.get("peers", "?") + syncing = d.get("syncing", "?") + state_at_head = d.get("state_at_head", "?") + watchtower = d.get("watchtower", "?") + cl_line = d.get("cl_line", "") + + # Warn if the live image is not ethrex + image_note = "" + if image and "ethrex" not in image.lower(): + image_note = f" [WARNING: image is not ethrex: {image}]\n" + + build_str = f"bn{buildnum}/{commit}" if buildnum or commit else "" + el_line = ( + f" EL: {status}/r{restart} {build_str} " + f"head={head} peers={peers} " + f"state@head={state_at_head} {syncing} wt={watchtower}" + ) + cl_disp = f" CL: {cl_line}" if cl_line else " CL: (no recent sync lines)" + + return f"### {node}\n{image_note}{el_line}\n{cl_disp}" + + +def gather(devnet: str, node_arg: str | None) -> list[dict[str, Any]]: + """ + Run the status probe across nodes and return a list of per-node dicts. + Does not print anything; callers use the returned data directly. + This is the clean seam for the health collector. + """ + nodes = node_list(devnet, node_arg) + results: list[dict[str, Any]] = [None] * len(nodes) # type: ignore[list-item] + + # Fire the whole roster at once (capped at 16) so one wedged node can't + # serialize behind a worker slot; the slow node's timeout no longer gates + # nodes waiting for a free worker. + with ThreadPoolExecutor(max_workers=min(16, max(1, len(nodes)))) as pool: + futures = {pool.submit(_probe_node, devnet, n): i for i, n in enumerate(nodes)} + for fut in as_completed(futures): + idx = futures[fut] + results[idx] = fut.result() + + return results + + +def status(devnet: str, node_arg: str | None, as_json: bool) -> None: + """Run status probe across nodes and print results.""" + # Check cache age + from .config import load_cache + cache = load_cache(devnet) + if cache: + discovered_at = cache.get("discovered_at", 0) + age_hours = (time.time() - discovered_at) / 3600 + if age_hours > 24: + print( + f"warning: cache for '{devnet}' is {age_hours:.0f}h old; " + f"run 'dv discover {devnet}' to refresh", + file=sys.stderr, + ) + + results = gather(devnet, node_arg) + + for d in results: + if as_json: + print(json.dumps(d)) + else: + print(_human_line(d)) diff --git a/devnets_monitor/devnets/store.py b/devnets_monitor/devnets/store.py new file mode 100644 index 0000000..1743d90 --- /dev/null +++ b/devnets_monitor/devnets/store.py @@ -0,0 +1,253 @@ +"""SQLite store: open, migrate, and helpers for the data collectors.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import Any + +from .config import repo_root + + +def db_path() -> Path: + """Return path to the SQLite database; ensures data/ directory exists.""" + data_dir = repo_root() / "data" + data_dir.mkdir(exist_ok=True) + return data_dir / "ethrex-devnets.sqlite" + + +def connect() -> sqlite3.Connection: + """Open (and create if absent) the SQLite database with sensible defaults.""" + conn = sqlite3.connect(str(db_path())) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA journal_mode = WAL") + return conn + + +_CREATE_STATEMENTS = [ + """CREATE TABLE IF NOT EXISTS slots ( + devnet TEXT NOT NULL, + slot INTEGER NOT NULL, + epoch INTEGER, + time INTEGER, + proposer TEXT, + proposer_name TEXT, + status TEXT, + blob_count INTEGER, + eth_block_number INTEGER, + gas_used INTEGER, + PRIMARY KEY (devnet, slot) + )""", + """CREATE TABLE IF NOT EXISTS slot_exec_times ( + devnet TEXT NOT NULL, + slot INTEGER NOT NULL, + client_type TEXT NOT NULL, + count INTEGER, + avg_time REAL, + min_time REAL, + max_time REAL, + PRIMARY KEY (devnet, slot, client_type) + )""", + """CREATE TABLE IF NOT EXISTS node_health ( + devnet TEXT NOT NULL, + node TEXT NOT NULL, + ts INTEGER NOT NULL, + image TEXT, + buildnum TEXT, + "commit" TEXT, + restart INTEGER, + head INTEGER, + peers INTEGER, + syncing TEXT, + state_at_head TEXT, + watchtower TEXT, + cl_line TEXT, + PRIMARY KEY (devnet, node, ts) + )""", + """CREATE TABLE IF NOT EXISTS hive_runs ( + devnet TEXT NOT NULL, + group_name TEXT NOT NULL, + suite_id TEXT NOT NULL, + ethrex_version TEXT, + fork_filter TEXT, + ntests INTEGER, + passes INTEGER, + fails INTEGER, + started_at INTEGER, + web_url TEXT, + PRIMARY KEY (devnet, group_name, suite_id) + )""", + """CREATE TABLE IF NOT EXISTS fork_schedule ( + devnet TEXT NOT NULL, + fork TEXT NOT NULL, + activation_ts INTEGER, + blob_target INTEGER, + blob_max INTEGER, + PRIMARY KEY (devnet, fork) + )""", + """CREATE TABLE IF NOT EXISTS fork_eips ( + devnet TEXT NOT NULL, + fork TEXT NOT NULL, + eip INTEGER NOT NULL, + title TEXT, + stage TEXT, + status TEXT, + PRIMARY KEY (devnet, fork, eip) + )""", + """CREATE TABLE IF NOT EXISTS events ( + devnet TEXT NOT NULL, + dedup_key TEXT NOT NULL, + kind TEXT NOT NULL, + severity TEXT NOT NULL, + node TEXT, + message TEXT NOT NULL, + details TEXT, + first_seen INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + resolved_at INTEGER, + count INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (devnet, dedup_key) + )""", + """CREATE INDEX IF NOT EXISTS idx_events_devnet_lastseen + ON events(devnet, last_seen DESC)""", + """CREATE TABLE IF NOT EXISTS network_splits ( + devnet TEXT NOT NULL, + ts INTEGER NOT NULL, + head_root TEXT NOT NULL, + head_slot INTEGER, + head_count INTEGER, + is_canonical INTEGER, + clients_json TEXT, + fork_id TEXT, + PRIMARY KEY (devnet, ts, head_root) + )""", + """CREATE TABLE IF NOT EXISTS client_dist ( + devnet TEXT NOT NULL, + ts INTEGER NOT NULL, + layer TEXT NOT NULL, + client TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (devnet, ts, layer, client, version) + )""", + """CREATE TABLE IF NOT EXISTS bal_inspect ( + devnet TEXT NOT NULL, + slot INTEGER NOT NULL, + block_root TEXT, + proposer_name TEXT, + access_count INTEGER, + eth_block_number INTEGER, + fetched_at INTEGER, + PRIMARY KEY (devnet, slot) + )""", + """CREATE TABLE IF NOT EXISTS epbs_slot ( + devnet TEXT NOT NULL, + slot INTEGER NOT NULL, + block_root TEXT, + proposer_name TEXT, + bid_count INTEGER, + ptc_size INTEGER, + ptc_vote_count INTEGER, + ptc_nonvoter_pct REAL, + payload_revealed INTEGER, + fetched_at INTEGER, + PRIMARY KEY (devnet, slot) + )""", + """CREATE TABLE IF NOT EXISTS network_overview ( + devnet TEXT NOT NULL, + ts INTEGER NOT NULL, + current_slot INTEGER, + current_epoch INTEGER, + finalized_epoch INTEGER, + justified_epoch INTEGER, + json TEXT, + PRIMARY KEY (devnet, ts) + )""", + """CREATE TABLE IF NOT EXISTS spamoor_status ( + devnet TEXT NOT NULL, + ts INTEGER NOT NULL, + spammer_id INTEGER NOT NULL, + name TEXT, + scenario TEXT, + status INTEGER, + enabled INTEGER, + PRIMARY KEY (devnet, ts, spammer_id) + )""", + """CREATE TABLE IF NOT EXISTS assertoor_runs ( + devnet TEXT NOT NULL, + run_id INTEGER NOT NULL, + test_id TEXT, + name TEXT, + status TEXT, + started_at INTEGER, + stopped_at INTEGER, + web_url TEXT, + PRIMARY KEY (devnet, run_id) + )""", + """CREATE TABLE IF NOT EXISTS deploy_gap ( + devnet TEXT NOT NULL, + node TEXT NOT NULL, + deployed_commit TEXT, + main_commit TEXT, + commits_behind INTEGER, + checked_at INTEGER, + PRIMARY KEY (devnet, node) + )""", + """CREATE TABLE IF NOT EXISTS gh_cache ( + key TEXT PRIMARY KEY, + value TEXT, + fetched_at INTEGER + )""", +] + + +_migrated = False + + +def migrate(conn: sqlite3.Connection) -> None: + """ + Create all tables if they do not exist. Idempotent, and runs the DDL at most + once per process (a long-running dashboard calls this on many requests; the + guard avoids re-issuing CREATE/ALTER + commit on every call). + """ + global _migrated + if _migrated: + return + for stmt in _CREATE_STATEMENTS: + conn.execute(stmt) + # Additive migrations: add columns to tables created before these fields existed. + fork_cols = {r[1] for r in conn.execute("PRAGMA table_info(fork_eips)")} + if "stage" not in fork_cols: + conn.execute("ALTER TABLE fork_eips ADD COLUMN stage TEXT") + if "status" not in fork_cols: + conn.execute("ALTER TABLE fork_eips ADD COLUMN status TEXT") + conn.commit() + _migrated = True + + +def upsert(conn: sqlite3.Connection, table: str, row: dict[str, Any]) -> None: + """ + Insert or update a row identified by its primary key columns. + Uses INSERT ... ON CONFLICT DO UPDATE SET for all non-PK columns. + Column names are double-quoted to handle reserved words (e.g. 'commit'). + """ + cols = list(row.keys()) + placeholders = ", ".join("?" for _ in cols) + col_names = ", ".join(f'"{c}"' for c in cols) + updates = ", ".join(f'"{c}" = excluded."{c}"' for c in cols) + sql = ( + f"INSERT INTO {table} ({col_names}) VALUES ({placeholders}) " + f"ON CONFLICT DO UPDATE SET {updates}" + ) + conn.execute(sql, list(row.values())) + + +def max_slot(conn: sqlite3.Connection, devnet: str) -> int | None: + """Return the highest stored slot for a devnet, or None if the table is empty.""" + row = conn.execute( + "SELECT MAX(slot) FROM slots WHERE devnet = ?", (devnet,) + ).fetchone() + if row and row[0] is not None: + return int(row[0]) + return None diff --git a/devnets_monitor/devnets/wipe.py b/devnets_monitor/devnets/wipe.py new file mode 100644 index 0000000..c0c6d0f --- /dev/null +++ b/devnets_monitor/devnets/wipe.py @@ -0,0 +1,77 @@ +""" +Wipe: recover a wedged EL node. MUTATING; gated behind --yes. +Sends WIPE_SEQUENCE to the host with the image tag as $1. +""" + +from __future__ import annotations + +import subprocess +import sys + +from .config import host_of, load_cache +from .remote import WIPE_SEQUENCE +from .ssh import SSH_OPTS + + +def wipe(devnet: str, node: str, yes: bool) -> None: + """ + Run the incident-tested wipe sequence on the given node. + Requires yes=True; refuses and exits nonzero otherwise. + """ + if not yes: + print( + "error: 'dv wipe' is a MUTATING operation and requires --yes to confirm.\n" + " Usage: dv wipe [devnet] --yes", + file=sys.stderr, + ) + sys.exit(1) + + cache = load_cache(devnet) + if not cache: + print( + f"error: no cache found for '{devnet}'. " + f"Run 'dv discover {devnet}' first to populate config/devnets/{devnet}.yaml", + file=sys.stderr, + ) + sys.exit(1) + + image_tag: str = cache.get("image_tag", "") + if not image_tag: + print( + f"error: no image_tag in cache for '{devnet}'. " + f"Run 'dv discover {devnet}' to refresh the cache.", + file=sys.stderr, + ) + sys.exit(1) + + host = host_of(devnet, node) + print(f"==> Wiping {node} ({host}) with image {image_tag}") + print(f"==> Sending wipe sequence via SSH...") + + # Stream output: use Popen so we can print in real time + cmd = ["ssh"] + SSH_OPTS + [host, "bash", "-s", image_tag] + try: + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + assert proc.stdin is not None + assert proc.stdout is not None + proc.stdin.write(WIPE_SEQUENCE) + proc.stdin.close() + for line in proc.stdout: + print(line, end="") + proc.wait() + except Exception as exc: + print(f"error: SSH wipe failed: {exc}", file=sys.stderr) + sys.exit(1) + + if proc.returncode != 0: + print( + f"\nerror: wipe sequence exited with code {proc.returncode}", + file=sys.stderr, + ) + sys.exit(proc.returncode) diff --git a/devnets_monitor/docs/architecture.md b/devnets_monitor/docs/architecture.md new file mode 100644 index 0000000..3d5891c --- /dev/null +++ b/devnets_monitor/docs/architecture.md @@ -0,0 +1,134 @@ +# Architecture + +How the pieces fit. For the working agreement and rules, see `CLAUDE.md`. + +## Design goals + +- One home for devnet ops + monitoring + incident history. +- Generic across ethpandaops devnets, parameterized by name. +- Read-only by default; mutations explicitly gated. +- Preserve the incident-tested remote sequences (the host-side shell). +- Minimal dependencies; the smallest thing that works. + +## One codebase: Python + +Everything local is Python, run via `uv`, behind a single `dv` console entry +point. There is no separate bash CLI layer (an earlier draft had one; it was +collapsed into Python to drop the two-language seam and the shell-injection / +fragile-parsing surface that came with it). + +``` + dv (argparse dispatcher in devnets/cli.py) + | + +------------------------------+------------------------------+ + | | | + host-touching data collection analysis / view + status/peers/logs/ collect (dora/hive/ blob/fork + serve + cl/discover/wipe health/forks) (FastAPI dashboard) + | | | + | subprocess: ssh / gh | requests: Dora/Hive/gh | reads SQLite + v v v + devnet hosts (over ssh) data/ethrex-devnets.sqlite data/ + config/ + | + | the ONLY shell: snippets that run ON the host + v + ssh bash -s << remote.py constant (docker / runlike / curl) +``` + +### The only shell is host-side + +The commands that operate a node (docker inspect, runlike, the datadir wipe, RPC +curls) run ON the devnet host, so they are shell no matter what drives them +locally. They live in `devnets/remote.py` as named string constants and are sent +over `ssh bash -s`, with values passed as positional args (not +interpolated) and any free-form input validated first. A host status probe emits +JSON on stdout; Python parses it in-process. So there is no local-bash +intermediary and no cross-language file seam; the JSON is produced on the host and +consumed directly. + +### Why subprocess-ssh (not paramiko/fabric) + +`subprocess` to the system `ssh` reuses the user's existing ssh config, keys, and +`known_hosts`, adds zero dependencies, and keeps the remote snippets identical to +what an operator would run by hand. A library would add a dependency and an auth +surface for no gain here. + +## The `dv` CLI + +`dv [devnet] [args]` (via `uv run dv ...`). It resolves the target +devnet (explicit arg > `$DEVNET` > `config/devnets.yaml` `default`), then dispatches: + +| subcommand | mutating | what | +|----------------|----------|------| +| `status` | no | per-node EL build/head/peers/state@head + CL line + watchtower (or `--json`) | +| `peers` | no | peer count, inbound/outbound, client mix, body-serving failures | +| `logs` | no | tail execution WARN/ERROR (validated `--since`) | +| `cl` | no | tail beacon sync lines (validated `--since`) | +| `discover` | no | refresh `config/devnets/.yaml` roster/forks/image from the repo (via `gh`) | +| `wipe` | YES | recover a wedged EL; requires `--yes` | +| `collect` | no | pull Dora/Hive/health/forks into SQLite | +| `blob` | no | blob inclusion per proposer over time; ethrex vs others | +| `fork` | no | fork schedule -> human time, blob target/max, EIP-per-fork, countdown | +| `hive` | no | summarize Hive group runs for the devnet | +| `eips-refresh` | no | regenerate `config/eips.json` from eipmcp data | +| `serve` | no | read-only FastAPI dashboard on 127.0.0.1 | + +## Config flow + +`config/devnets.yaml` (static, hand-maintained) lists each devnet and its repo + +service URLs. `dv discover ` reads the ethpandaops devnet repo via `gh` +(subprocess) and writes `config/devnets/.yaml` (roster, fork schedule, image +tag, `discovered_at`) parsed with `pyyaml` / `json`. Both the CLI and the data +layer read these files; neither hardcodes a roster. The cache lets the CLI work +offline and against shut-down devnets. See `config/schema.md` for fields. + +## Data store + +Single SQLite file `data/ethrex-devnets.sqlite`, every table keyed by a `devnet` +column so one store covers all devnets. Collectors are idempotent (upsert on +primary key) and incremental (watermark on max stored slot). Tables: + +- `slots(devnet, slot, ...)` — per-slot proposer, blob_count, block number, status +- `slot_exec_times(devnet, slot, client_type, ...)` — per-client execution timing +- `node_health(devnet, node, ts, ...)` — point-in-time health snapshots +- `hive_runs(devnet, group_name, suite_id, ...)` — Hive pass/fail per run +- `fork_schedule(devnet, fork, activation_ts, blob_target, blob_max)` +- `fork_eips(devnet, fork, eip, title)` — EIP-per-fork enrichment + +Incident history lives in `docs/history/.md` (markdown), not the DB. + +## Dashboard + +`web/app.py` is a read-only FastAPI app over the SQLite store, bound to +`127.0.0.1`, no auth, no write endpoints. It is a view over already-collected +data; run `dv collect` first or it shows empty. Routes: `/`, `/blobs/`, +`/forks/`, `/hive/`, `/incidents/`. The index surfaces the +ethpandaops portal Services URLs (Dora, Forkmon, Assertoor, Checkpoint Sync, +Tracoor, Syncoor, Spamoor, Buildoor) per devnet. + +## What lives where + +``` +devnets/cli.py argparse dispatcher + `dv` console entry (the front door) +devnets/config.py load registry + discovered cache; resolve target devnet +devnets/ssh.py run a remote snippet via subprocess ssh (bash -s, positional args) +devnets/remote.py host-side shell snippets as constants (status probe, peers probe, wipe) +devnets/status.py status sweep: ssh-run the probe per node, parse JSON, format/store +devnets/peers.py peer inspection +devnets/discover.py gh-api roster/fork discovery -> config cache +devnets/wipe.py mutating recovery driver (--yes gated; sends the wipe snippet) +devnets/store.py sqlite open + schema migrations +devnets/dora.py Dora slots fetch -> store +devnets/hive.py Hive API fetch -> hive_runs +devnets/forks.py fork schedule + eips.json -> store +devnets/collect.py orchestrator for `dv collect` +devnets/blobtrack.py `dv blob` analysis +devnets/forkview.py `dv fork` analysis +config/devnets.yaml static registry +config/devnets/.yaml discovered cache (committed, regenerable) +config/eips.json fork -> EIP map (dv eips-refresh) +docs/devnet-ops.md operational runbook +docs/history/.md per-devnet facts + incident log +web/app.py FastAPI dashboard +pyproject.toml uv project; `dv` console script -> devnets.cli:main +``` diff --git a/devnets_monitor/docs/cron-example.md b/devnets_monitor/docs/cron-example.md new file mode 100644 index 0000000..94bfb78 --- /dev/null +++ b/devnets_monitor/docs/cron-example.md @@ -0,0 +1,67 @@ +# Scheduling periodic data collection + +Run `dv collect all` on a schedule to keep the SQLite store current. +The working directory **must** be the repo root so that `config/` and `data/` +resolve correctly. + +## crontab (every 15 minutes) + +```cron +*/15 * * * * cd /path/to/ethrex-devnets && uv run dv collect glamsterdam-devnet-5 all >> /tmp/dv-collect.log 2>&1 +``` + +Replace `/path/to/ethrex-devnets` with the absolute path to this repo. + +## systemd user timer + +Create `~/.config/systemd/user/dv-collect.service`: + +```ini +[Unit] +Description=ethrex-devnets data collection + +[Service] +Type=oneshot +WorkingDirectory=/path/to/ethrex-devnets +ExecStart=uv run dv collect glamsterdam-devnet-5 all +StandardOutput=journal +StandardError=journal +``` + +Create `~/.config/systemd/user/dv-collect.timer`: + +```ini +[Unit] +Description=Run ethrex-devnets collection every 15 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15min +Persistent=true + +[Install] +WantedBy=timers.target +``` + +Enable and start: + +```bash +systemctl --user daemon-reload +systemctl --user enable --now dv-collect.timer +systemctl --user status dv-collect.timer +``` + +Check recent runs: + +```bash +journalctl --user -u dv-collect.service -n 50 +``` + +## Notes + +- The timer uses `Persistent=true` so a missed run (e.g. laptop was suspended) + executes once on the next boot or wakeup. +- SSH connectivity to devnet nodes is required for the `health` sub-collector. + The other collectors (`forks`, `blobs`, `hive`) use only HTTP and will succeed + without SSH access. +- To collect only blobs (no SSH), use: `uv run dv collect blobs`. diff --git a/devnets_monitor/docs/devnet-ops-skill.md b/devnets_monitor/docs/devnet-ops-skill.md new file mode 100644 index 0000000..5649deb --- /dev/null +++ b/devnets_monitor/docs/devnet-ops-skill.md @@ -0,0 +1,48 @@ +--- +name: devnet-ops +description: Operate, inspect, and debug ethrex EL nodes on ethpandaops devnets (SSH access, finding nodes, reading logs, deploy, wipe/resync, incident history). Use whenever the user asks to ssh into a devnet node, check/inspect a devnet node, find or read logs on glamsterdam/bal/any ethpandaops devnet, investigate a devnet error report, deploy or wipe a node, or asks about past devnet problems. +--- + +# devnet-ops + +Before doing any ethrex devnet operations or answering questions about a devnet node/incident, READ both +reference files: + +1. `docs/devnet-ops.md` — generic access & inspection procedures (SSH, finding nodes via inventory, + container layout, inspection curl/docker commands, build & deploy, debug logging, wipe & resync, Dora + API). Substitute `` with the target network (e.g. `glamsterdam-devnet-5`). + +2. `docs/history/.md` — per-devnet facts and incident history (node roster, fork schedule, commit + map, known issues with root cause and recovery). For glamsterdam-devnet-5 this is + `docs/history/glamsterdam-devnet-5.md`. If a file for the requested devnet doesn't exist yet, use the + generic doc and create a new `docs/history/.md` from `docs/history/_template.md` once you learn + devnet-specific facts. + +## CLI + +The `dv` dispatcher wraps all common SSH/curl/docker checks (read-only by default; `wipe` is mutating and +gated behind `--yes`). Prefer it over hand-typed one-liners: +``` +dv status [devnet] [node|all] # EL build/head/peers/state@head + CL sync line + watchtower +dv peers [devnet] # peer count, inbound/outbound, client mix, body-serving failures +dv logs [devnet] [since] # tail execution WARN/ERROR (default since 2m) +dv cl [devnet] [since] # tail beacon sync lines (default since 3m) +dv wipe [devnet] --yes # MUTATING: full wipe + resync sequence +dv discover [devnet] # refresh config/devnets/.yaml from the devnet repo +``` +Details (subcommands, the mandatory snooper-engine restart on wipe) are in `docs/devnet-ops.md`. + +## Workflow + +1. Read `docs/devnet-ops.md` for HOW (procedures, commands). +2. Read `docs/history/.md` for WHAT/WHY (known issues, divergences, facts) — check whether the + current symptom matches a known issue before investigating from scratch. +3. ALWAYS verify the live host before trusting a node name: `docker inspect execution --format "{{.Config.Image}}"`. + A `*-ethrex-*` node may have been manually swapped to another client. +4. Default to read-only (logs/inspect/curl). Recreate/wipe/deploy are mutating — confirm with the user first. + +## Maintenance + +When you discover a new incident, divergence, or devnet fact during a session, append it to +`docs/history/.md` (and `docs/devnet-ops.md` if a procedure changed). Keep entries dated. +These files are committed in this repo and form the long-term ops record. diff --git a/devnets_monitor/docs/devnet-ops.md b/devnets_monitor/docs/devnet-ops.md new file mode 100644 index 0000000..d99324a --- /dev/null +++ b/devnets_monitor/docs/devnet-ops.md @@ -0,0 +1,186 @@ +# ethrex devnet ops — access & inspection runbook (generic) + +Generic procedures for operating/inspecting ethrex EL nodes on any ethpandaops devnet. Substitute +`` with the network name, e.g. `glamsterdam-devnet-5`. Per-devnet facts (fork schedule, node +roster, incidents) live in `docs/history/.md`. + +Read-only by default (logs/inspect/curl). Recreate/wipe are the only mutating actions. + +## CLI: `dv` + +The `dv` dispatcher wraps all SSH+curl+docker patterns. Read-only by default; `wipe` is the only +mutating subcommand, gated behind `--yes`. Set `DEVNET` env to target another network, or pass it +as the first argument (default taken from `config/devnets.yaml`). +``` +dv status [devnet] [node|all] # EL build/head/peers/syncing/state@head + CL sync line + watchtower +dv peers [devnet] # peer count, inbound/outbound, client mix, body-serving failures +dv logs [devnet] [since] # tail execution WARN/ERROR (default since 2m) +dv cl [devnet] [since] # tail beacon sync lines (default since 3m) +dv wipe [devnet] --yes # MUTATING: pause wt->runlike->rm->wipe DB->recreate->snooper restart->unpause +dv discover [devnet] # refresh config/devnets/.yaml from the devnet repo +``` +`wipe` bakes in the full procedure below INCLUDING the mandatory `docker restart snooper-engine` (see "Wipe +& resync"). `status`'s `state@head` uses `eth_getBalance(@latest)` (yes = canonical head has state on disk; +distinguishes a real stateless wedge from a node that simply trails its CL). Prefer this over hand-typed +one-liners; if the node roster changes, run `dv discover ` to refresh `config/devnets/.yaml`. + +## Find the nodes (inventory / instances) + +The ethpandaops instances page (`https://ethpandaops.io/networks//?tab=instances`) is a +client-rendered SPA; it 404s to any plain fetcher/WebFetch. Source of truth = the devnet repo: +``` +# repo varies per devnet family; e.g. ethpandaops/glamsterdam-devnets for glamsterdam-devnet-N +gh api repos/ethpandaops//contents/ansible/inventories//inventory.ini --jq .content | base64 -d +gh api repos/ethpandaops//contents/ansible/inventories//group_vars/all/images.yaml --jq .content | base64 -d +``` +- `inventory.ini` groups nodes by `[_]`; the `[ethrex:children]` group lists all ethrex nodes. +- `images.yaml` `default_ethereum_client_images.ethrex` = the deployed ethrex image tag. + +CAUTION: inventory = INTENDED config, not live. A node named `*-ethrex-*` may have been manually +swapped to another client. ALWAYS verify on the host: +``` +docker inspect execution --format "{{.Config.Image}}" +``` + +## SSH access + +``` +ssh -o StrictHostKeyChecking=accept-new devops@.srv..ethpandaops.io +``` +Node naming: `--` and `buildoor---` (local block builder nodes). +All hosts accept the same `devops@` key. + +## Containers per node + +- `execution` → ethrex (EL). Entrypoint `./ethrex`, workdir `/usr/local/bin`. +- `beacon` → the CL (lighthouse/grandine/prysm/...). +- `validator` → validator client. +- `snooper-engine` → engine-API proxy between CL and EL (`http://snooper-engine:8561`); logs FCU/newPayload. +- `ethereum-node-docker-watchtower` → auto-updates containers on new image. + +ethrex container details: +- datadir bind mount: host `/data/ethrex` → container `/data` (owned by uid **1004**) +- JWT: `/data/execution-auth.secret` → `/execution-auth.jwt` +- network config: `/data/ethereum-network-config/metadata` → `/network-config` +- ports: 30303 tcp/udp (p2p, public), `127.0.0.1:8545` (http rpc), `127.0.0.1:8551` (authrpc), `127.0.0.1:6060` (metrics) +- network: docker network `shared`; restart=always; log-opt max-size=500m max-file=8 +- image: `ethpandaops/ethrex:`; labels `buildnum`, `commit` + +## Common inspection (on a node) + +``` +# what build/commit is deployed + uptime + which client image +docker inspect execution --format "image={{.Config.Image}} buildnum={{index .Config.Labels \"buildnum\"}} commit={{index .Config.Labels \"commit\"}} started={{.State.StartedAt}}" + +# head / peers / syncing +curl -s localhost:8545 -H content-type:application/json -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' +curl -s localhost:8545 -H content-type:application/json -d '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' +curl -s localhost:8545 -H content-type:application/json -d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' +# blob base fee + ratio +curl -s localhost:8545 -H content-type:application/json -d '{"jsonrpc":"2.0","method":"eth_feeHistory","params":["0xa","latest",[]],"id":1}' + +# metrics (Prometheus): tx tracker by type, p2p msg counts, getBlobs, tx errors +curl -s localhost:6060/metrics | grep -E "transactions_tracker|ethrex_p2p_(in|out)going_messages|getBlobsV3.+count|transaction_errors" + +# CL-not-driving-EL symptom +docker logs --since 90s execution 2>&1 | grep -c "No messages from the consensus" + +# sync-loop health (full-sync stalls) +docker logs --since 30m execution 2>&1 | grep -iE "Sync cycle|No bodies|state root missing|fetch headers|penalizing" + +# beacon peers (CL p2p) — watch for peers=[0/200] (isolated) or headSlot<&1 | grep -iE "peers=|headSlot|forkchoice|execution" + +# engine traffic (FCU/newPayload + VALID/INVALID/SYNCING) +docker logs --since 3m snooper-engine 2>&1 | grep -iE "forkchoiceUpdated|newPayload|SYNCING|VALID|INVALID" +``` + +Note: metrics counters are cumulative (since restart); there is NO current-pool-size-by-type gauge, and +`txpool` RPC namespace is not enabled (only `eth,net,web3,admin,debug`). So you cannot read current +blob-mempool count without debug logs. + +## Build & deploy workflow + +1. Commit + push to the devnet branch (`git push origin `). +2. Image build (NOT automatic on commit — must be triggered): GitHub Actions on + `ethpandaops/eth-client-docker-image-builder`, target `lambdaclass/ethrex@`. + - check runs: `gh run list --repo ethpandaops/eth-client-docker-image-builder --limit 10` + - it builds branch HEAD at checkout time → push BEFORE triggering, or later commits won't be included. + - run's `headSha` is the BUILDER repo's commit, not ethrex's — identify the built ethrex commit by + timing / buildnum bump. + - Rust build takes ~15-20 min (amd64 + arm64 + manifest). +3. Deploy = `ethereum-node-docker-watchtower` (15-min poll, `--include-restarting`) auto-pulls the new tag + and recreates `execution` fleet-wide. + - watchtower recreate PRESERVES the running container's config (env/cmd) — a manual config change carries + forward until the next image. + - all ethrex nodes' watchtowers fire ~same cycle → simultaneous fleet restart (transient missed slots). + +To deploy a specific build with custom config (e.g. info/debug logging), pull + recreate manually instead +of waiting for watchtower (see below), and pause watchtower so it doesn't fight you. + +## Debug logging (requires container recreate) + +ethrex log level: `--log.level ` / env `ETHREX_LOG_LEVEL`, OR `RUST_LOG`. +On these devnets `RUST_LOG=info` is set as env. To enable debug you must recreate the container. + +Safe recreate using `runlike` (installed at `/usr/local/bin/runlike`): +``` +docker pause ethereum-node-docker-watchtower # so it can't bounce mid-capture +runlike execution > /tmp/run-execution-orig.txt # capture exact run cmd +# edit: add `-d` (runlike omits detach!), set RUST_LOG to debug filter: +# RUST_LOG=info,ethrex_blockchain=debug,ethrex_p2p=debug,ethrex_common=debug,ethrex_vm=debug,ethrex_levm=debug +sed -e 's#--env=RUST_LOG=info#--env=RUST_LOG=#' -e 's#^docker run #docker run -d #' /tmp/run-execution-orig.txt > /tmp/run-debug.sh +docker rm -f execution && bash /tmp/run-debug.sh +# ... capture ... +# revert: recreate with RUST_LOG=info (orig cmd + -d), then: +docker unpause ethereum-node-docker-watchtower +``` + +Build-path debug lines (in `ethrex_blockchain`): `Creating a new payload`, `fails 2D inclusion check` +(`payload.rs` 2D skip), `Failed to execute transaction` (apply fail), `No more blob gas`, `max data blobs`, +`Adding transaction: X to payload`. + +CAUTION: a restart resets the in-memory mempool (clears any decay state) AND can break an in-progress +snap sync. So debug-capturing a slow-accumulating bug means letting it RE-decay with debug on. + +## Wipe & resync (recover a wedged EL) + +datadir is owned by uid 1004, so `devops` can't `rm` it; wipe via a root container: +``` +docker pause ethereum-node-docker-watchtower +runlike execution > /tmp/run.txt # before removing +docker rm -f execution +docker run --rm --user 0 --entrypoint sh -v /data/ethrex:/d ethpandaops/ethrex: \ + -c 'rm -rf /d/* /d/.[!.]*' +sed 's#^docker run #docker run -d #' /tmp/run.txt | bash # fresh start from genesis +docker restart snooper-engine # REQUIRED: reconnect engine proxy to the new EL +# let sync COMPLETE uninterrupted, confirm head reaches tip, then: +docker unpause ethereum-node-docker-watchtower +``` +IMPORTANT: after `docker rm -f execution` + recreate, the `snooper-engine` proxy still holds a stale +connection to the OLD execution container, so the CL can't drive the fresh EL — the EL logs `No messages +from the consensus layer` and `eth_syncing` stays at currentBlock=0/highestBlock=0 with peers but no FCU. +Fix: `docker restart snooper-engine`. After that the CL's FCU/newPayload reach the EL and full sync starts +(EL logs `[SYNCING] N% of batch processed`, currentBlock climbs toward the CL's exec head). Observed +2026-06-08 wiping lodestar-ethrex-2. (If a node talks to the EL directly without a snooper, restart the +beacon instead.) +For a wedged CL (e.g. grandine halted) a plain CL restart (`docker restart beacon`) is often enough — no +EL wipe needed. + +## Dora API (devnet explorer) + +Base: `https://dora..ethpandaops.io/api` (swagger at `/api/swagger/doc.json`) +``` +# recent slots; filter by proposer_name = "--"; fields: blob_count, eth_block_number, status, time +curl -s "https://dora..ethpandaops.io/api/v1/slots?limit=400&with_missing=1&with_orphaned=1" +# blob inclusion per ethrex pair (post a time): split proposer_name on '-' +# status: Canonical / Missing / Orphaned. chain tip = latest slot's eth_block_number +``` + +## Find genesis / fork schedule (per-devnet) + +``` +gh api repos/ethpandaops//contents/network-configs//metadata/genesis.json --jq .content | base64 -d +``` +Fields: chainId, fork activation timestamps (cancun/prague/osaka/bpoN/amsterdam @ ts), `blobSchedule` +(target/max per fork). Record the concrete values for a given devnet in `docs/history/.md`. diff --git a/devnets_monitor/docs/history/_template.md b/devnets_monitor/docs/history/_template.md new file mode 100644 index 0000000..ccbde2a --- /dev/null +++ b/devnets_monitor/docs/history/_template.md @@ -0,0 +1,41 @@ +# — history & facts + +Devnet-specific facts and the running log of problems (what / why / when / how recovered). Generic +access/inspection procedures live in `docs/devnet-ops.md`. + +Devnet branch: ``. Builder target: `lambdaclass/ethrex@`. +Devnet repo: `ethpandaops/`. ethrex image: `ethpandaops/ethrex:`. + +## Node roster (from inventory, group `[ethrex:children]`) + +All intended as `ethpandaops/ethrex:`. Populate from `dv discover ` output. + +- + +Live divergences from inventory (verify before trusting a name): +- + +## Genesis / fork schedule + +From `ethpandaops/`, `network-configs//metadata/genesis.json`. Populate +from `dv discover ` output (also written to `config/devnets/.yaml`). + +- chainId: +- Fork activations: cancun/prague/osaka @0; +- blobSchedule (target/max): +- + +## Commit map + +Notable commits / PRs that affected this devnet: + +- + +## Known issues / learnings + +Numbered, dated entries. Format: symptom description, root cause, recovery steps. + +1. diff --git a/devnets_monitor/pyproject.toml b/devnets_monitor/pyproject.toml new file mode 100644 index 0000000..c5c8fdb --- /dev/null +++ b/devnets_monitor/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "ethrex-devnets" +version = "0.1.0" +description = "Ops and monitoring toolkit for ethrex on ethpandaops devnets" +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0", + "requests>=2.31", + "fastapi>=0.110", + "uvicorn[standard]>=0.29", + "jinja2>=3.1", + "markdown>=3.5", +] + +[project.scripts] +dv = "devnets.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["devnets", "web"] + +[tool.uv] diff --git a/devnets_monitor/uv.lock b/devnets_monitor/uv.lock new file mode 100644 index 0000000..40f3268 --- /dev/null +++ b/devnets_monitor/uv.lock @@ -0,0 +1,704 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "ethrex-devnets" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "fastapi" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = ">=0.110" }, + { name = "jinja2", specifier = ">=3.1" }, + { name = "markdown", specifier = ">=3.5" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "requests", specifier = ">=2.31" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.29" }, +] + +[[package]] +name = "fastapi" +version = "0.136.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httptools" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/88/1d21a36da8f5cb0fa49eafd4b169eba5608d57e75bbcf61845cbc6243216/httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d", size = 208247, upload-time = "2026-05-25T22:17:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5", size = 113064, upload-time = "2026-05-25T22:17:09.136Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" }, + { url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" }, + { url = "https://files.pythonhosted.org/packages/05/0b/4240efeb672751ee5b9b380cb0e3fdc050bc05f68adc7a8aefc4fcd9a69a/httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150", size = 90918, upload-time = "2026-05-25T22:17:15.155Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e5/8cfcabc5546e8022f168be28bcdaa128a240a0befdd03b59d558b4f18bd6/httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8", size = 205148, upload-time = "2026-05-25T22:17:16.333Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0e/0fb14848c19a686c8062ff9067c1a48793e3224b47bc5b201535b6036fce/httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c", size = 111368, upload-time = "2026-05-25T22:17:17.586Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/46f1cecf06b9bbde8e4b8c88034ac7908989e5ff7a3a388ef38392949c1f/httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7", size = 486447, upload-time = "2026-05-25T22:17:18.564Z" }, + { url = "https://files.pythonhosted.org/packages/77/00/258bfc0837221f81d9725c45f9b948a6a6b2994a147a4fb66e85100c668f/httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d", size = 482448, upload-time = "2026-05-25T22:17:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/04/ab/d1cef3b5523f4d272a70f42a776c3169a2dddfe3a54de4b2ce4a36341528/httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681", size = 464460, upload-time = "2026-05-25T22:17:20.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5d1d072442277bb2b3434e0e60690b8e8c23840ef7de8b6ea54040a536d3/httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683", size = 471312, upload-time = "2026-05-25T22:17:22.085Z" }, + { url = "https://files.pythonhosted.org/packages/0d/66/b96623b27e51a68199ef4efdda0613cced9233fe3062ac74e50749c5ad37/httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1", size = 90117, upload-time = "2026-05-25T22:17:23.074Z" }, + { url = "https://files.pythonhosted.org/packages/1a/12/fa3fbf5f9517b273edea2dc982aa82a8c634091e67c590792b729017bc6f/httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6", size = 206183, upload-time = "2026-05-25T22:17:24.004Z" }, + { url = "https://files.pythonhosted.org/packages/30/fc/5e7c4cb443370f2090a3aba0453a07384d29ff66b7435bb90e77e1037599/httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b", size = 112079, upload-time = "2026-05-25T22:17:25.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" }, + { url = "https://files.pythonhosted.org/packages/62/42/94e15bc68ce3d423243c45d7f1b0c7561f13844f97dc52ae23182fb65628/httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e", size = 480865, upload-time = "2026-05-25T22:17:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7c/fe2980fc03723272e30f135b62360b075f513dfe7cc73aef36c7f04012bd/httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b", size = 463189, upload-time = "2026-05-25T22:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/47fc5fff68acd1bfa20b4734059c9a06cadb88119dcd5258b5b0d21d91c8/httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0", size = 466610, upload-time = "2026-05-25T22:17:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/60/bd/07b13c93ffd9bec9546e0d43f8e19378dd696dbd278511406bc07371ef1f/httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527", size = 92705, upload-time = "2026-05-25T22:17:31.133Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c4/121648f68ce066d7bd762d6b6d97e620847642d38d54f3d90ff11d947629/httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568", size = 215023, upload-time = "2026-05-25T22:17:32.401Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b0/312a062ae741ae3e8baa8c8bf20be81b2e67337b259ab4349bebc7b6142e/httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b", size = 117405, upload-time = "2026-05-25T22:17:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/fc/37/fccd705f795386bb05bf413012fecff2a33e5aa8c2f069096de3e9fd8702/httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca", size = 558497, upload-time = "2026-05-25T22:17:34.732Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/f172e8003576de35f5ba77ff417cf0e34429d35dc014deef15afa337a72c/httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f", size = 571585, upload-time = "2026-05-25T22:17:35.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/f5564760af99f3dbbf3f9104dc00e5da27e96cf433c6bdcf77617f70bf3f/httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d", size = 543297, upload-time = "2026-05-25T22:17:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/99/67/8d9f2c313618e161b82f3873188e7196126da1d6e29688df40eb3997c77a/httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081", size = 539535, upload-time = "2026-05-25T22:17:38.032Z" }, + { url = "https://files.pythonhosted.org/packages/48/63/b906c01e53f50d432c0defe43ce52764a111dc1bdd028bafbeb54dcfd008/httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77", size = 108209, upload-time = "2026-05-25T22:17:39.473Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "markdown" +version = "3.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "starlette" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", size = 2701854, upload-time = "2026-05-31T01:07:51.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.49.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", size = 91284, upload-time = "2026-06-03T22:01:30.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", size = 71376, upload-time = "2026-06-03T22:01:29.037Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, + { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, + { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, + { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, + { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, + { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, + { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, + { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/70a7feced9f87e2ff26dba42667290f41694fc64646c67261fbb8cab5d5c/watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98", size = 399730, upload-time = "2026-05-18T04:31:38.162Z" }, + { url = "https://files.pythonhosted.org/packages/31/3a/0da302f2307aee316922806ebd5726c542cbd787c938271cf14a074c7daf/watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44", size = 392842, upload-time = "2026-05-18T04:30:27.051Z" }, + { url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" }, + { url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" }, + { url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" }, + { url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/71/72/4508db1856d1d87fcbb3b63f4839bab1b5682cb0e8d224d122263c09654a/watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799", size = 275183, upload-time = "2026-05-18T04:30:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/f9/36/14b76ca57652e5cc5fd1c11f32a261292c08a0d19a00351013c2549cbfb2/watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9", size = 288059, upload-time = "2026-05-18T04:32:07.937Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8d/0a85e395398d8d20fadfe5c5d32c726eee17a519e78fb356f2cf7531bffe/watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077", size = 280186, upload-time = "2026-05-18T04:31:54.484Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/36db056f1fdcc5f07302f56e631774d6835bcd6fa3ace402304621d5f9e5/watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08", size = 399031, upload-time = "2026-05-18T04:30:44.576Z" }, + { url = "https://files.pythonhosted.org/packages/c1/64/01a9d6f66a82a5c101ce939274106cc72759d62427e153f01edd2b9f87c2/watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9", size = 391205, upload-time = "2026-05-18T04:30:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" }, + { url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" }, + { url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" }, + { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, + { url = "https://files.pythonhosted.org/packages/e7/54/a9c7ea9a82a4ac65e7004c0a03920b5cdd2f9c3b678757d9cd425aa51d53/watchfiles-1.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8c8358484d5fa12ef34f05b7f4168eaf1932f408725ff6d023c33ec17bd79d4", size = 400205, upload-time = "2026-05-18T04:32:05.153Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5d/c9ab3534374a4a67450696905d6ef16a04405448b8dc52bd752ae50423d4/watchfiles-1.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f04b092229ad2c50126dd3c922c8822e51e605993764a33058d4a791ab42281", size = 392508, upload-time = "2026-05-18T04:30:54.849Z" }, + { url = "https://files.pythonhosted.org/packages/26/ca/1ad30103535cf0cecd7b993e8d50edc5351b1820e38f2d22e3df58962feb/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d", size = 452448, upload-time = "2026-05-18T04:30:53.727Z" }, + { url = "https://files.pythonhosted.org/packages/37/a1/ceee2cdf2afbd715fa07758d39c9859513eae411b23196f7fd039e5feedd/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b9909cc2b48468b575eefa944919e1fe8a36c5849d5c7c168f80a8c1db69398e", size = 459605, upload-time = "2026-05-18T04:30:23.312Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f6/421e30fd1cb3907a84ed92ab3f1983e37ba2dca015e9a894a048418417a2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a37faaed405c67e28e6be45a1fa4f206ef5a2860f27c237db9fa30704c38242", size = 490757, upload-time = "2026-05-18T04:30:47.358Z" }, + { url = "https://files.pythonhosted.org/packages/41/b0/55ed1b97ed08be7bba6f9a541cac15f2a858e1d74d2b07b6da70a82aab00/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9649193aa27bd9ff2e80ff29bfaa93085496c7a3a377592823cc58b77ee88add", size = 568672, upload-time = "2026-05-18T04:30:38.915Z" }, + { url = "https://files.pythonhosted.org/packages/d1/cf/d8ae8a80dd7bafab395ea7681c10237311bbf34d37704a8c744e7cf31fc7/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e4ff8e37f99cf1da89e255e07c9c4b37c214038c4283707bdec308cb1b0ea1f", size = 464197, upload-time = "2026-05-18T04:30:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/3076c496ca8dafe0e8cd03fcebdfc47be4b1174b4e5b24ff6e396e6b3af2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:054dc20fd2e3132b4c3883b4a00d72fd6e1f56fdaf89fccd12e8057d74cd74d7", size = 453181, upload-time = "2026-05-18T04:30:14.829Z" }, + { url = "https://files.pythonhosted.org/packages/e5/10/9745e17c98e7b8a86454df0a3c7b5686bd650383f1e9f26e4ebcbd6cc0c0/watchfiles-1.2.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e140ed30ebde76796b686e67c182cff10ea2fbab186fafd1560f74bb5a473a6e", size = 465109, upload-time = "2026-05-18T04:30:28.123Z" }, + { url = "https://files.pythonhosted.org/packages/8f/95/8ef4a95481d3e0cb52d62a06fa6e972e81424be2d9698b91a2fecca9904c/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:bb7e52ecf68ba46d22df23467b87cffeb2146908aa523ebfe803019618cfda06", size = 630653, upload-time = "2026-05-18T04:31:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e4/3b3bf36b0f829b50c6ebcb8d031583863c59f923d6a6af3d485e470d0fac/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:23282a321c8baf9b3a3c4afff673f9fe65eb7fdc2338d765ccad9d3d1916a5ba", size = 657838, upload-time = "2026-05-18T04:31:06.497Z" }, + { url = "https://files.pythonhosted.org/packages/21/b1/6cbbb50c1f3002ab568777d44aa21206dfb8807a840990c4037523b51812/watchfiles-1.2.0-cp314-cp314-win32.whl", hash = "sha256:c0db965c5f79aa49fe672d297cf1febc5ad149b658594944f49a54a2b96270a7", size = 275108, upload-time = "2026-05-18T04:30:06.891Z" }, + { url = "https://files.pythonhosted.org/packages/92/45/190ce6db8dcb4536682cf75d3889ff1a27182a58cb519d343cb6d9ea63d8/watchfiles-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:71283b39fd17e5408eb123bd37aeecfd9d54c81fc184421943208aadb879d103", size = 288441, upload-time = "2026-05-18T04:32:12.901Z" }, + { url = "https://files.pythonhosted.org/packages/74/0d/3eae1c2313ab08378431d907c3f8095ecca00f3eda33111cf4f0f2591799/watchfiles-1.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:c5c19526f4e54a00f2666a6c0e9e40d582c09e865055ea7378bf0009aab857b3", size = 280684, upload-time = "2026-05-18T04:31:26.902Z" }, + { url = "https://files.pythonhosted.org/packages/b1/75/fb64e6c25d6b5ca636d03df34ffb1c6e9873303e76d27967e045f8df088f/watchfiles-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d73a585accffa5ae39c17264c36ec3166d2fad7000c780f5ef83b2722afb9dd2", size = 398857, upload-time = "2026-05-18T04:32:17.108Z" }, + { url = "https://files.pythonhosted.org/packages/73/4e/9f7adf01754cbf81843722ccfec169d8f26c69778281a302855cecd2ee08/watchfiles-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae99b14c5f21e026e0e9d96f40e07d8570ebee6cafd9d8fc318354606daa7a28", size = 392413, upload-time = "2026-05-18T04:31:07.911Z" }, + { url = "https://files.pythonhosted.org/packages/47/c8/bec626bcc2d69f44b9acb24ce7d60ed7b16b73628eea747fcbd169d8edda/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4429f3b105524a10b72c3a819b091c495d2811d419c1e1e8df773a5a5974f831", size = 452409, upload-time = "2026-05-18T04:31:20.142Z" }, + { url = "https://files.pythonhosted.org/packages/00/b7/b6362068e81e7c556d155a34c35d40ac3ef42d747b06d7f6e5bf58e359c2/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43d818978d06062d9b22c4fab2ebe44cf5213d42dc8e62bda8c2760cfa2eeb33", size = 458827, upload-time = "2026-05-18T04:32:06.219Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/9a813fa42afb1e0b4625e75f0479826644d3ee8dc287e093799bc01f390c/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f732dc58b2dbe69e464ccf8fff7a03b0dd0be439da4c0720d3558527d3d6b4", size = 490104, upload-time = "2026-05-18T04:31:56.034Z" }, + { url = "https://files.pythonhosted.org/packages/2f/bf/27dfb6094ca4c9aad21298b5525b6c53cb36121ee454331d05161e58d130/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f200104103feb097de4cab8fe4f5dd18a2026934c7dea98c55a2f5fd6d5a33b", size = 571360, upload-time = "2026-05-18T04:31:57.133Z" }, + { url = "https://files.pythonhosted.org/packages/fb/39/44a096d67270ea93df91d33877dbe91fbda3aa4f8ec2edf799d93eda8736/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ac26eefbf4af1741247d6fb68b11c49a25b2f7413fbd318a83a12aaa9cf666", size = 464644, upload-time = "2026-05-18T04:30:57.33Z" }, + { url = "https://files.pythonhosted.org/packages/0e/80/c7472203bad6268e3ef1ad260739704847898938ad7ea8b63a5131f46b50/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c4997d4e4a55f0d02b6cde327322daf3a0400e5df6c6b15948994bf72497925", size = 454771, upload-time = "2026-05-18T04:30:48.736Z" }, + { url = "https://files.pythonhosted.org/packages/51/cf/3b10b268b4b7f0fc26e9debb5eef1998b515887840f444cd3ec80c688755/watchfiles-1.2.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4c887eba18b7945ac73067a8b4a66f21cd46c2539b2bc68588f7be6c7eb6d26b", size = 463494, upload-time = "2026-05-18T04:31:33.826Z" }, + { url = "https://files.pythonhosted.org/packages/3d/3e/a4302545cd589262a0dc7d140e86f7688eba3f9c72776c27f7e23b8864c4/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3416ff151bb6b5a8d8d11664974fbef4d9305b9b2957839ab5a270468fd8df30", size = 629383, upload-time = "2026-05-18T04:31:15.596Z" }, + { url = "https://files.pythonhosted.org/packages/db/99/d5649df0a9a410d45b7c882304d0b790903ac9b6e8f2cfd12114e0c6b9f2/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:0e831a271c035d89789cffc386b6aa1375f39f1cd25eb7ca0997e4970d152fc5", size = 656093, upload-time = "2026-05-18T04:31:58.707Z" }, + { url = "https://files.pythonhosted.org/packages/92/b9/362702539275019a54dd2e94511b31a9b89c5f9e6a21966de7eb692549fc/watchfiles-1.2.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:37a6721cdf3f65dbb13aa9503510ccb4451603ac837e44d265d7992a597e1374", size = 400109, upload-time = "2026-05-18T04:31:16.879Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/71d5ba62db781e5587bded1d944c675374bc4aa37ff33d5018d98e8b6538/watchfiles-1.2.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2b37d10b5a63bd4d87e18472d80fa525bd670586fae62e5dd580452764879b65", size = 392167, upload-time = "2026-05-18T04:31:28.058Z" }, + { url = "https://files.pythonhosted.org/packages/3c/01/c66dd95d0423fe30d31820e2d1d5bda773764131bbb6ac0cb1cf303ac328/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a105bc2283f67e8fbec74253ec2d94925de92ed72c0393f1206bf326b7b7b69", size = 452372, upload-time = "2026-05-18T04:31:00.836Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/2fe99557e72f85627c6a8eed50d889e8d101623e060a22ad75b875cb932d/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5327989a465505f05cfe06f04fa9d0c2fd5432bb243e10e6f012b1bdca3c8579", size = 459596, upload-time = "2026-05-18T04:31:34.96Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/d4acfa0023367428ed48351b3b9b267893037b6cadae55620c61c24bcfd4/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecb47f183a8025b2aa18b546725c3657e542112ae9c0613a2af79b4fa8d04ad7", size = 490869, upload-time = "2026-05-18T04:31:59.923Z" }, + { url = "https://files.pythonhosted.org/packages/a4/5f/3164cbdce06c9fb95c4f7b9e2f9760b5e2797af43a9ecc317ef42a23a278/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8520a4ab0e37f770afc34459c4f8f7019e153f9124dc101c15538365875d1ab2", size = 571641, upload-time = "2026-05-18T04:32:00.948Z" }, + { url = "https://files.pythonhosted.org/packages/41/e6/85d3731c55e65cd7690f3f803d24c139588aaf863e4bf2148fe7a7fa1a19/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71cd71740ed2c15211ebb237ced4e39a1cdf6f80566e5fe95428da1626f4fde6", size = 464444, upload-time = "2026-05-18T04:30:34.298Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7d/562641012b8b09872742c3b8adf9629ec479fd78f8d68ae4a0c13da8add6/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f88af53d6ddaf72179ef613ddc905e6f4785f712b49b80b3bef9f3525e6194b4", size = 453593, upload-time = "2026-05-18T04:31:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/56/fe/cb8ef3d6f929d14158fdaaad9925985b7310abc9384dcd4d82dd0016fb59/watchfiles-1.2.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:cee9d5efd929efdac5f7e58f72b3376f676b64050a91c5b99a7094c5b2317488", size = 465096, upload-time = "2026-05-18T04:31:30.384Z" }, + { url = "https://files.pythonhosted.org/packages/25/91/80908e835e100527a9267147b08c0eee1fa6ab0ffec15edc04d1d44885f7/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:b718bf356bbc15e559bd8ef41782b573b8ae0e3f177ab244b440568d7ea02cfb", size = 630638, upload-time = "2026-05-18T04:30:49.89Z" }, + { url = "https://files.pythonhosted.org/packages/46/4b/95ab2f256bb4af3cb2eb23b9317bda984ee6e0f11733a5c004a6c95b06e3/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377", size = 657684, upload-time = "2026-05-18T04:31:32.027Z" }, +] + +[[package]] +name = "websockets" +version = "16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365, upload-time = "2026-01-10T09:22:46.787Z" }, + { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038, upload-time = "2026-01-10T09:22:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328, upload-time = "2026-01-10T09:22:49.809Z" }, + { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915, upload-time = "2026-01-10T09:22:51.071Z" }, + { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152, upload-time = "2026-01-10T09:22:52.224Z" }, + { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583, upload-time = "2026-01-10T09:22:53.443Z" }, + { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880, upload-time = "2026-01-10T09:22:55.033Z" }, + { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261, upload-time = "2026-01-10T09:22:56.251Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693, upload-time = "2026-01-10T09:22:57.478Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364, upload-time = "2026-01-10T09:22:59.333Z" }, + { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039, upload-time = "2026-01-10T09:23:01.171Z" }, + { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323, upload-time = "2026-01-10T09:23:02.341Z" }, + { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975, upload-time = "2026-01-10T09:23:03.756Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203, upload-time = "2026-01-10T09:23:05.01Z" }, + { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653, upload-time = "2026-01-10T09:23:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920, upload-time = "2026-01-10T09:23:07.492Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255, upload-time = "2026-01-10T09:23:09.245Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689, upload-time = "2026-01-10T09:23:10.483Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, + { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, + { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, + { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, + { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, + { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, +] diff --git a/devnets_monitor/web/__init__.py b/devnets_monitor/web/__init__.py new file mode 100644 index 0000000..fc1d492 --- /dev/null +++ b/devnets_monitor/web/__init__.py @@ -0,0 +1 @@ +# web package: read-only FastAPI dashboard diff --git a/devnets_monitor/web/aggregate.py b/devnets_monitor/web/aggregate.py new file mode 100644 index 0000000..0ae9370 --- /dev/null +++ b/devnets_monitor/web/aggregate.py @@ -0,0 +1,251 @@ +"""Command-center data aggregator. + +All queries are DB-only (read from SQLite). No SSH, no HTTP, no subprocess calls +happen in these functions; they are safe to call from FastAPI request handlers. +""" + +from __future__ import annotations + +import sqlite3 +import time +from datetime import datetime, timezone +from typing import Any + + +def _fmt_ts(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + except Exception: + return str(ts) + + +def _node_rollup(health: dict[str, Any], node_events: list[dict[str, Any]]) -> str: + """ + Compute red/amber/green rollup for a single node health row. + + red = active wedge or node_unreachable event for this node + amber = syncing == "yes" (or syncing starts with cur=) OR snapshot is stale (>1h) + green = synced, state_at_head="yes", recent snapshot + """ + # Check for red events on this node + node_name = health.get("node", "") + for ev in node_events: + if ev.get("active") and ev.get("node") == node_name: + if ev.get("kind") in ("wedge", "node_unreachable"): + return "red" + + syncing_raw = (health.get("syncing") or "").lower() + is_syncing = ( + syncing_raw == "yes" + or syncing_raw.startswith("cur=") + or syncing_raw in ("true", "1") + ) + + state_ok = (health.get("state_at_head") or "").lower() == "yes" + + # Stale snapshot: ts older than 3600s + ts = health.get("ts_raw") + stale = False + if ts is not None: + now = time.time() + stale = (now - ts) > 3600 + + if is_syncing or not state_ok or stale: + return "amber" + + return "green" + + +def _get_finality(conn: sqlite3.Connection, devnet: str) -> dict[str, Any] | None: + """Return the latest network_overview row for the devnet, or None.""" + try: + row = conn.execute( + """SELECT current_slot, current_epoch, finalized_epoch, justified_epoch + FROM network_overview WHERE devnet=? ORDER BY ts DESC LIMIT 1""", + (devnet,), + ).fetchone() + if row is None: + return None + return { + "current_slot": row["current_slot"], + "current_epoch": row["current_epoch"], + "finalized_epoch": row["finalized_epoch"], + "justified_epoch": row["justified_epoch"], + } + except Exception: + return None + + +def _get_blob_flow(conn: sqlite3.Connection, devnet: str) -> str | None: + """ + Return "ON", "OFF", or None (no data). + Uses the latest spamoor_status snapshot. + """ + try: + ts_row = conn.execute( + "SELECT MAX(ts) AS max_ts FROM spamoor_status WHERE devnet=?", (devnet,) + ).fetchone() + if not ts_row or ts_row["max_ts"] is None: + return None + latest_ts = ts_row["max_ts"] + rows = conn.execute( + "SELECT scenario, status FROM spamoor_status WHERE devnet=? AND ts=?", + (devnet, latest_ts), + ).fetchall() + for r in rows: + scenario = (r["scenario"] or "").lower() + if "blob" in scenario and r["status"] == 1: + return "ON" + return "OFF" + except Exception: + return None + + +def _get_next_fork(conn: sqlite3.Connection, devnet: str) -> dict[str, Any] | None: + """Return the next upcoming fork name + countdown string, or None.""" + try: + now = time.time() + row = conn.execute( + """SELECT fork, activation_ts FROM fork_schedule + WHERE devnet=? AND activation_ts IS NOT NULL AND activation_ts > ? + ORDER BY activation_ts ASC LIMIT 1""", + (devnet, now), + ).fetchone() + if row is None: + return None + ts = row["activation_ts"] + diff = ts - now + days = int(diff // 86400) + hours = int((diff % 86400) // 3600) + minutes = int((diff % 3600) // 60) + if days > 0: + countdown = f"in {days}d {hours}h" + elif hours > 0: + countdown = f"in {hours}h {minutes}m" + else: + countdown = f"in {minutes}m" + time_str = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + return {"fork": row["fork"], "countdown": countdown, "time_str": time_str} + except Exception: + return None + + +def _get_latest_health_with_raw_ts(conn: sqlite3.Connection, devnet: str) -> list[dict[str, Any]]: + """Like app._latest_health but also includes raw ts for rollup calculations.""" + try: + rows = conn.execute( + """ + SELECT nh.* + FROM node_health nh + INNER JOIN ( + SELECT node, MAX(ts) AS max_ts + FROM node_health + WHERE devnet = ? + GROUP BY node + ) latest ON nh.node = latest.node AND nh.ts = latest.max_ts + WHERE nh.devnet = ? + ORDER BY nh.node + """, + (devnet, devnet), + ).fetchall() + except Exception: + return [] + + result = [] + for r in rows: + syncing = r["syncing"] + if isinstance(syncing, str): + syncing_disp = "yes" if syncing.lower() in ("true", "1", "yes") else "no" + else: + syncing_disp = "yes" if syncing else "no" + ts_raw = r["ts"] + ts_disp = _fmt_ts(ts_raw) + result.append({ + "node": r["node"], + "head": r["head"], + "peers": r["peers"], + "state_at_head": r["state_at_head"], + "syncing": syncing_disp, + "syncing_raw": r["syncing"], + "buildnum": r["buildnum"], + "commit": r["commit"], + "ts": ts_disp, + "ts_raw": ts_raw, + }) + return result + + +def command_center_data(conn: sqlite3.Connection, devnet: str) -> dict[str, Any]: + """ + Assemble the command-center panel data for one devnet from the DB only. + + Returns a dict with keys: + nodes list[dict] -- health tiles with rollup color + finality dict|None -- current_slot, current_epoch, finalized_epoch + next_fork dict|None -- fork, countdown, time_str + blob_flow str|None -- "ON" | "OFF" | None + events list[dict] -- top 8 active events sorted crit>warn>info + """ + result: dict[str, Any] = { + "nodes": [], + "finality": None, + "next_fork": None, + "blob_flow": None, + "events": [], + } + + # --- Events (needed first so rollup can check by node) --- + active_events: list[dict[str, Any]] = [] # full set, for the node rollup + try: + _sev_order = {"crit": 0, "warn": 1, "info": 2} + ev_rows = conn.execute( + """SELECT kind, severity, node, message, last_seen, resolved_at, count, + first_seen + FROM events WHERE devnet=? ORDER BY last_seen DESC""", + (devnet,), + ).fetchall() + all_events = [] + for e in ev_rows: + all_events.append({ + "kind": e["kind"], + "severity": e["severity"], + "node": e["node"], + "message": e["message"], + "last_seen": e["last_seen"], + "last_seen_str": _fmt_ts(e["last_seen"]), + "first_seen_str": _fmt_ts(e["first_seen"]), + "count": e["count"], + "active": e["resolved_at"] is None, + }) + all_events.sort( + key=lambda e: (0 if e["active"] else 1, _sev_order.get(e["severity"], 9), -e["last_seen"]) + ) + active_events = [e for e in all_events if e["active"]] + result["events"] = active_events[:8] # display cap + except Exception: + pass + + # --- Node tiles --- + try: + health_rows = _get_latest_health_with_raw_ts(conn, devnet) + for h in health_rows: + # Pass ALL active events (not the display-capped 8) so a red event on + # the 9th+ node still colors that node's tile red. + rollup = _node_rollup(h, active_events) + h["rollup"] = rollup + result["nodes"] = health_rows + except Exception: + pass + + # --- Finality --- + result["finality"] = _get_finality(conn, devnet) + + # --- Next fork --- + result["next_fork"] = _get_next_fork(conn, devnet) + + # --- Blob flow --- + result["blob_flow"] = _get_blob_flow(conn, devnet) + + return result diff --git a/devnets_monitor/web/app.py b/devnets_monitor/web/app.py new file mode 100644 index 0000000..bba4799 --- /dev/null +++ b/devnets_monitor/web/app.py @@ -0,0 +1,928 @@ +"""Read-only FastAPI dashboard over the SQLite store. + +All routes are GET only; the SQLite connection is opened read-only. +Bound to 127.0.0.1 by default (set in cli.py / uvicorn.run call). +No authentication, no write endpoints. +""" + +from __future__ import annotations + +import sqlite3 +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import markdown +from fastapi import FastAPI, Request +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates + +# --------------------------------------------------------------------------- +# App setup +# --------------------------------------------------------------------------- + +_HERE = Path(__file__).parent +_TEMPLATES_DIR = _HERE / "templates" + +app = FastAPI(title="ethrex-devnets dashboard", docs_url=None, redoc_url=None) +templates = Jinja2Templates(directory=str(_TEMPLATES_DIR)) + + +def _ethrex_commit_url(commit: str | None) -> str: + """GitHub URL for an ethrex commit (short hashes resolve). Empty if no commit.""" + c = (commit or "").strip() + return f"https://github.com/lambdaclass/ethrex/commit/{c}" if c else "" + + +# Expose to all templates so any commit can render as a link to ethrex source. +templates.env.globals["ethrex_commit_url"] = _ethrex_commit_url + + +def _humanize(s: Any) -> str: + """snake_case / kebab-case -> 'Sentence case' for display (e.g. + version_change -> 'Version change', from_commit -> 'From commit').""" + return str(s or "").replace("_", " ").replace("-", " ").strip().capitalize() + + +templates.env.filters["humanize"] = _humanize + +# --------------------------------------------------------------------------- +# Read-only DB helpers +# --------------------------------------------------------------------------- + + +def _db_path() -> Path: + """Locate the SQLite database file.""" + return _HERE.parent / "data" / "ethrex-devnets.sqlite" + + +def _connect_ro() -> sqlite3.Connection | None: + """Open the SQLite DB read-only. Returns None if the file does not exist.""" + path = _db_path() + if not path.exists(): + return None + uri = f"file:{path}?mode=ro" + conn = sqlite3.connect(uri, uri=True, timeout=5) + conn.row_factory = sqlite3.Row + return conn + + +# --------------------------------------------------------------------------- +# Config helpers (no side effects -- registry read only) +# --------------------------------------------------------------------------- + + +def _load_registry() -> dict[str, Any]: + import yaml + reg_path = _HERE.parent / "config" / "devnets.yaml" + if not reg_path.exists(): + return {} + with reg_path.open() as f: + data = yaml.safe_load(f) + return data if isinstance(data, dict) else {} + + +def _devnet_names() -> list[str]: + reg = _load_registry() + devnets = reg.get("devnets", {}) + return list(devnets.keys()) if isinstance(devnets, dict) else [] + + +def _devnet_entry(name: str) -> dict[str, Any]: + reg = _load_registry() + devnets = reg.get("devnets", {}) or {} + return dict(devnets.get(name, {})) + + +# --------------------------------------------------------------------------- +# URL derivation helpers +# --------------------------------------------------------------------------- + +_SERVICES = [ + ("Dora", "dora"), + ("Forkmon", "forkmon"), + ("Assertoor", "assertoor"), + ("Checkpoint Sync", "checkpoint-sync"), + ("Tracoor", "tracoor"), + ("Syncoor", "syncoor"), + ("Spamoor", "spamoor"), + ("Buildoor", "buildoor"), + ("JSON RPC", "rpc"), + ("Beacon RPC", "beacon"), +] + + +def _service_urls(devnet: str, entry: dict[str, Any]) -> list[dict[str, str]]: + """ + Build the list of ethpandaops service URLs for a devnet. + Use dora_base / config_base from the registry where available; + derive the rest from the ..ethpandaops.io pattern. + """ + dora_base = entry.get("dora_base", "") + urls = [] + for label, slug in _SERVICES: + if slug == "dora" and dora_base: + url = dora_base.rstrip("/") + else: + url = f"https://{slug}.{devnet}.ethpandaops.io" + urls.append({"label": label, "url": url}) + return urls + + +# --------------------------------------------------------------------------- +# Index data helper +# --------------------------------------------------------------------------- + + +def _latest_health(conn: sqlite3.Connection, devnet: str) -> list[dict[str, Any]]: + """Return the most recent node_health row per node for a devnet.""" + rows = conn.execute( + """ + SELECT nh.* + FROM node_health nh + INNER JOIN ( + SELECT node, MAX(ts) AS max_ts + FROM node_health + WHERE devnet = ? + GROUP BY node + ) latest ON nh.node = latest.node AND nh.ts = latest.max_ts + WHERE nh.devnet = ? + ORDER BY nh.node + """, + (devnet, devnet), + ).fetchall() + result = [] + for r in rows: + syncing = r["syncing"] + if isinstance(syncing, str): + syncing_disp = "yes" if syncing.lower() in ("true", "1", "yes") else "no" + else: + syncing_disp = "yes" if syncing else "no" + ts_disp = "" + if r["ts"]: + try: + ts_disp = datetime.fromtimestamp(r["ts"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + ts_disp = str(r["ts"]) + result.append({ + "node": r["node"], + "head": r["head"], + "peers": r["peers"], + "state_at_head": r["state_at_head"], + "syncing": syncing_disp, + "buildnum": r["buildnum"], + "commit": r["commit"], + "ts": ts_disp, + }) + return result + + +def _latest_hive_per_group(conn: sqlite3.Connection, devnet: str) -> list[dict[str, Any]]: + """Return the most recent hive run per group_name for a devnet.""" + rows = conn.execute( + """ + SELECT group_name, passes, fails, ntests, started_at, web_url + FROM hive_runs + WHERE devnet = ? + ORDER BY group_name, started_at DESC + """, + (devnet,), + ).fetchall() + seen: set[str] = set() + result = [] + for r in rows: + g = r["group_name"] + if g in seen: + continue + seen.add(g) + started = "" + if r["started_at"]: + try: + started = datetime.fromtimestamp(r["started_at"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + started = str(r["started_at"]) + passes = r["passes"] if r["passes"] is not None else "?" + fails = r["fails"] if r["fails"] is not None else "?" + ntests = r["ntests"] if r["ntests"] is not None else "?" + result.append({ + "group": g, + "passes": passes, + "fails": fails, + "ntests": ntests, + "started": started, + "web_url": r["web_url"] or "", + }) + return result + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@app.get("/", response_class=HTMLResponse) +async def index(request: Request) -> HTMLResponse: + from web.aggregate import command_center_data + + devnet_names = _devnet_names() + devnets_data: list[dict[str, Any]] = [] + conn = _connect_ro() + + for name in devnet_names: + entry = _devnet_entry(name) + cc: dict[str, Any] = {} + hive: list[dict[str, Any]] = [] + if conn is not None: + try: + cc = command_center_data(conn, name) + except Exception: + cc = {"nodes": [], "finality": None, "next_fork": None, "blob_flow": None, "events": []} + hive = _latest_hive_per_group(conn, name) + devnets_data.append({ + "name": name, + "cc": cc, + "hive": hive, + "services": _service_urls(name, entry), + }) + + if conn is not None: + conn.close() + + return templates.TemplateResponse( + request, + "command_center.html", + {"devnets": devnets_data, "devnet_names": devnet_names}, + ) + + +@app.get("/blobs/{devnet}", response_class=HTMLResponse) +async def blobs(request: Request, devnet: str) -> HTMLResponse: + import json as _json + + blob_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + # Use the data-returning helper from blobtrack + from devnets.blobtrack import get_blob_data + blob_data = get_blob_data(devnet) + if blob_data is None: + error = f"No slot data for {devnet}. Run: dv collect {devnet} blobs" + + # Serialize slot_series for inline JS chart + chart_json = "{}" + if blob_data and blob_data.get("slot_series"): + chart_json = _json.dumps(blob_data["slot_series"]) + + return templates.TemplateResponse( + request, + "blobs.html", + { + "devnet": devnet, + "blob_data": blob_data, + "chart_json": chart_json, + "error": error, + }, + ) + + +@app.get("/forks/{devnet}", response_class=HTMLResponse) +async def forks(request: Request, devnet: str) -> HTMLResponse: + fork_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.forkview import get_fork_data + fork_data = get_fork_data(devnet) + if fork_data is None: + error = f"No fork data for {devnet}. Run: dv collect {devnet} forks" + + return templates.TemplateResponse( + request, + "forks.html", + { + "devnet": devnet, + "fork_data": fork_data, + "error": error, + }, + ) + + +@app.get("/hive/{devnet}", response_class=HTMLResponse) +async def hive(request: Request, devnet: str) -> HTMLResponse: + rows: list[dict[str, Any]] = [] + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + raw = conn.execute( + """ + SELECT group_name, fork_filter, ethrex_version, + passes, fails, ntests, started_at, web_url + FROM hive_runs + WHERE devnet = ? + ORDER BY group_name, fork_filter, started_at DESC + """, + (devnet,), + ).fetchall() + conn.close() + + seen: set[tuple[str, str]] = set() + for r in raw: + key = (r["group_name"], r["fork_filter"] or "") + if key in seen: + continue + seen.add(key) + started = "" + if r["started_at"]: + try: + started = datetime.fromtimestamp(r["started_at"], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M UTC" + ) + except Exception: + started = str(r["started_at"]) + rows.append({ + "group": r["group_name"] or "", + "suite": r["fork_filter"] or "", + "version": r["ethrex_version"] or "", + "passes": r["passes"] if r["passes"] is not None else "?", + "fails": r["fails"] if r["fails"] is not None else "?", + "ntests": r["ntests"] if r["ntests"] is not None else "?", + "started": started, + "web_url": r["web_url"] or "", + }) + + if not rows: + error = f"No Hive runs for {devnet}. Run: dv collect {devnet} hive" + + return templates.TemplateResponse( + request, + "hive.html", + { + "devnet": devnet, + "rows": rows, + "error": error, + }, + ) + + +@app.get("/events/{devnet}", response_class=HTMLResponse) +async def events(request: Request, devnet: str) -> HTMLResponse: + events_data: list[dict[str, Any]] = [] + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.detect import get_events_data + events_data = get_events_data(devnet, include_resolved=True) + if not events_data: + error = f"No events for {devnet}. Run: dv collect {devnet} events" + + active = [e for e in events_data if e.get("active")] + resolved = [e for e in events_data if not e.get("active")] + + return templates.TemplateResponse( + request, + "events.html", + { + "devnet": devnet, + "active_events": active, + "resolved_events": resolved, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/network/{devnet}", response_class=HTMLResponse) +async def network(request: Request, devnet: str) -> HTMLResponse: + network_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.network import get_network_data + network_data = get_network_data(devnet) + if network_data is None: + error = f"No network data for {devnet}. Run: dv collect {devnet} network" + + return templates.TemplateResponse( + request, + "network.html", + { + "devnet": devnet, + "network_data": network_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/proposals/{devnet}", response_class=HTMLResponse) +async def proposals(request: Request, devnet: str) -> HTMLResponse: + proposals_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.proposals import get_proposals_data + proposals_data = get_proposals_data(devnet, since=None) + if proposals_data is None: + error = f"No slot data for {devnet}. Run: dv collect {devnet} blobs" + + return templates.TemplateResponse( + request, + "proposals.html", + { + "devnet": devnet, + "proposals_data": proposals_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/bal/{devnet}", response_class=HTMLResponse) +async def bal(request: Request, devnet: str) -> HTMLResponse: + bal_data = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.bal import get_bal_data + bal_data = get_bal_data(devnet) + if bal_data is None: + error = f"No BAL data for {devnet}. Run: dv collect {devnet} slow" + + return templates.TemplateResponse( + request, + "bal.html", + { + "devnet": devnet, + "bal_data": bal_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/epbs/{devnet}", response_class=HTMLResponse) +async def epbs(request: Request, devnet: str) -> HTMLResponse: + epbs_data = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.epbs import get_epbs_data + epbs_data = get_epbs_data(devnet) + if epbs_data is None: + error = f"No ePBS data for {devnet}. Run: dv collect {devnet} slow" + + return templates.TemplateResponse( + request, + "epbs.html", + { + "devnet": devnet, + "epbs_data": epbs_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/eips/{devnet}", response_class=HTMLResponse) +async def eips(request: Request, devnet: str) -> HTMLResponse: + eiptrack_data = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.eiptrack import get_eiptrack_data + eiptrack_data = get_eiptrack_data(devnet) + if eiptrack_data is None: + error = ( + f"No EIP-track data for {devnet}. " + f"Run: dv collect {devnet} forks" + ) + + return templates.TemplateResponse( + request, + "eiptrack.html", + { + "devnet": devnet, + "eiptrack_data": eiptrack_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/clients/{devnet}", response_class=HTMLResponse) +async def clients(request: Request, devnet: str) -> HTMLResponse: + clients_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.network import get_clients_data + clients_data = get_clients_data(devnet) + if clients_data is None: + error = f"No client data for {devnet}. Run: dv collect {devnet} clients" + + forkmon_url = f"https://forkmon.{devnet}.ethpandaops.io" + + return templates.TemplateResponse( + request, + "clients.html", + { + "devnet": devnet, + "clients_data": clients_data, + "forkmon_url": forkmon_url, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/spamoor/{devnet}", response_class=HTMLResponse) +async def spamoor(request: Request, devnet: str) -> HTMLResponse: + spamoor_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.spamoor import get_spamoor_data + spamoor_data = get_spamoor_data(devnet) + if spamoor_data is None: + error = f"No spamoor data for {devnet}. Run: dv collect {devnet} spamoor" + + spamoor_url = f"https://spamoor.{devnet}.ethpandaops.io" + + return templates.TemplateResponse( + request, + "spamoor.html", + { + "devnet": devnet, + "spamoor_data": spamoor_data, + "spamoor_url": spamoor_url, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/deploy/{devnet}", response_class=HTMLResponse) +async def deploy(request: Request, devnet: str) -> HTMLResponse: + deploy_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.deploytl import get_deploy_data + deploy_data = get_deploy_data(devnet) + if deploy_data is None: + error = f"No deploy data for {devnet}. Run: dv collect {devnet} health" + + return templates.TemplateResponse( + request, + "deploy.html", + { + "devnet": devnet, + "deploy_data": deploy_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/assertoor/{devnet}", response_class=HTMLResponse) +async def assertoor(request: Request, devnet: str) -> HTMLResponse: + assertoor_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.assertoor import get_assertoor_data + assertoor_data = get_assertoor_data(devnet) + if assertoor_data is None: + error = f"No assertoor data for {devnet}. Run: dv collect {devnet} assertoor" + + assertoor_url = f"https://assertoor.{devnet}.ethpandaops.io" + + return templates.TemplateResponse( + request, + "assertoor.html", + { + "devnet": devnet, + "assertoor_data": assertoor_data, + "assertoor_url": assertoor_url, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/node/{devnet}/{node}", response_class=HTMLResponse) +async def node_drilldown(request: Request, devnet: str, node: str) -> HTMLResponse: + """Per-node drill-down: health history, version history, events, proposals. + DB-only: no SSH, no HTTP, no subprocess calls in this handler. + """ + from datetime import datetime, timezone as _tz + + node_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + try: + def _ts_str(ts: int | None) -> str: + if ts is None: + return "-" + try: + return datetime.fromtimestamp(ts, tz=_tz.utc).strftime("%Y-%m-%d %H:%M UTC") + except Exception: + return str(ts) + + # --- Latest snapshot --- + latest_row = conn.execute( + """SELECT * FROM node_health WHERE devnet=? AND node=? + ORDER BY ts DESC LIMIT 1""", + (devnet, node), + ).fetchone() + + # --- Health history (last 20 snapshots) --- + history_rows = conn.execute( + """SELECT ts, head, peers, state_at_head, syncing, buildnum, "commit" + FROM node_health WHERE devnet=? AND node=? + ORDER BY ts DESC LIMIT 20""", + (devnet, node), + ).fetchall() + health_history = [] + for r in history_rows: + syncing_raw = r["syncing"] or "" + if isinstance(syncing_raw, str): + syncing_disp = "yes" if syncing_raw.lower() in ("true", "1", "yes") else "no" + else: + syncing_disp = "yes" if syncing_raw else "no" + health_history.append({ + "ts": _ts_str(r["ts"]), + "head": r["head"], + "peers": r["peers"], + "state_at_head": r["state_at_head"], + "syncing": syncing_disp, + "buildnum": r["buildnum"], + "commit": r["commit"] or "", + }) + + # --- Version history: distinct commit/buildnum transitions --- + all_rows = conn.execute( + """SELECT ts, "commit", buildnum, image, head, peers + FROM node_health WHERE devnet=? AND node=? + AND "commit" IS NOT NULL AND "commit" != '' + ORDER BY ts DESC LIMIT 500""", + (devnet, node), + ).fetchall() + version_history: list[dict[str, Any]] = [] + for r in all_rows: + entry = { + "ts_str": _ts_str(r["ts"]), + "commit": (r["commit"] or "")[:12], + "commit_full": r["commit"] or "", + "buildnum": r["buildnum"], + "image": r["image"], + "head": r["head"], + "peers": r["peers"], + } + if not version_history: + version_history.append(entry) + else: + last = version_history[-1] + if ( + entry["commit_full"] != last["commit_full"] + or entry["buildnum"] != last["buildnum"] + or entry["image"] != last["image"] + ): + version_history.append(entry) + + # --- This node's events (active + recently resolved) --- + _sev_order = {"crit": 0, "warn": 1, "info": 2} + ev_rows = conn.execute( + """SELECT kind, severity, node, message, details, + first_seen, last_seen, resolved_at, count + FROM events WHERE devnet=? AND (node=? OR node IS NULL) + ORDER BY last_seen DESC LIMIT 40""", + (devnet, node), + ).fetchall() + import json as _json + node_events = [] + for e in ev_rows: + details: dict = {} + if e["details"]: + try: + details = _json.loads(e["details"]) + except Exception: + details = {"raw": e["details"]} + # include only if node matches or no node (network-wide events) + node_events.append({ + "kind": e["kind"], + "severity": e["severity"], + "node": e["node"] or "", + "message": e["message"], + "details": details, + "first_seen_str": _ts_str(e["first_seen"]), + "last_seen_str": _ts_str(e["last_seen"]), + "resolved_at_str": _ts_str(e["resolved_at"]), + "count": e["count"], + "active": e["resolved_at"] is None, + }) + node_events.sort( + key=lambda e: (0 if e["active"] else 1, _sev_order.get(e["severity"], 9)) + ) + + # --- This node's proposals --- + counts_row = conn.execute( + """SELECT + SUM(CASE WHEN LOWER(status)='canonical' THEN 1 ELSE 0 END) AS canonical, + SUM(CASE WHEN LOWER(status) IN ('missing','missed') THEN 1 ELSE 0 END) AS missed, + SUM(CASE WHEN LOWER(status)='orphaned' THEN 1 ELSE 0 END) AS orphaned, + COUNT(*) AS total + FROM slots WHERE devnet=? AND proposer_name=?""", + (devnet, node), + ).fetchone() + recent_slots_rows = conn.execute( + """SELECT slot, status, blob_count, time FROM slots + WHERE devnet=? AND proposer_name=? + ORDER BY slot DESC LIMIT 20""", + (devnet, node), + ).fetchall() + proposals = { + "canonical": (counts_row["canonical"] or 0) if counts_row else 0, + "missed": (counts_row["missed"] or 0) if counts_row else 0, + "orphaned": (counts_row["orphaned"] or 0) if counts_row else 0, + "total": (counts_row["total"] or 0) if counts_row else 0, + "recent": [ + { + "slot": r["slot"], + "status": r["status"], + "blob_count": r["blob_count"], + "time": _ts_str(r["time"]), + } + for r in recent_slots_rows + ], + } + + # Format latest snapshot for display + latest: dict[str, Any] | None = None + if latest_row: + syncing_raw = latest_row["syncing"] or "" + if isinstance(syncing_raw, str): + syncing_disp = "yes" if syncing_raw.lower() in ("true", "1", "yes") else "no" + else: + syncing_disp = "yes" if syncing_raw else "no" + latest = { + "ts": _ts_str(latest_row["ts"]), + "head": latest_row["head"], + "peers": latest_row["peers"], + "state_at_head": latest_row["state_at_head"], + "syncing": syncing_disp, + "buildnum": latest_row["buildnum"], + "commit": latest_row["commit"] or "", + "image": latest_row["image"] or "", + "restart": latest_row["restart"], + "cl_line": latest_row["cl_line"] or "", + } + + node_data = { + "node": node, + "devnet": devnet, + "latest": latest, + "health_history": health_history, + "version_history": version_history, + "events": node_events, + "proposals": proposals, + } + except Exception as exc: + error = f"Error loading node data: {exc}" + finally: + conn.close() + + return templates.TemplateResponse( + request, + "node.html", + { + "devnet": devnet, + "node": node, + "node_data": node_data, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/exectime/{devnet}", response_class=HTMLResponse) +async def exectime(request: Request, devnet: str) -> HTMLResponse: + import json as _json + + exectime_data: dict[str, Any] | None = None + error: str | None = None + + conn = _connect_ro() + if conn is None: + error = "No data yet. Run: dv collect" + else: + conn.close() + from devnets.exectime import get_exectime_data + exectime_data = get_exectime_data(devnet) + if exectime_data is None: + error = f"No exec-time data for {devnet}. Run: dv collect {devnet} blobs" + + series_json = "{}" + if exectime_data and exectime_data.get("series"): + series_json = _json.dumps(exectime_data["series"]) + + return templates.TemplateResponse( + request, + "exectime.html", + { + "devnet": devnet, + "exectime_data": exectime_data, + "series_json": series_json, + "error": error, + "current_devnet": devnet, + }, + ) + + +@app.get("/incidents/{devnet}", response_class=HTMLResponse) +async def incidents(request: Request, devnet: str) -> HTMLResponse: + history_path = _HERE.parent / "docs" / "history" / f"{devnet}.md" + content: str | None = None + error: str | None = None + + content_html: str | None = None + if history_path.exists(): + content = history_path.read_text(encoding="utf-8") + # History docs are trusted (we author them), so rendering their HTML is safe. + content_html = markdown.markdown( + content, + extensions=["fenced_code", "tables", "sane_lists", "nl2br"], + ) + else: + error = f"No incident history found for {devnet} at docs/history/{devnet}.md" + + return templates.TemplateResponse( + request, + "incidents.html", + { + "devnet": devnet, + "content_html": content_html, + "error": error, + }, + ) diff --git a/devnets_monitor/web/templates/assertoor.html b/devnets_monitor/web/templates/assertoor.html new file mode 100644 index 0000000..eb03aa8 --- /dev/null +++ b/devnets_monitor/web/templates/assertoor.html @@ -0,0 +1,90 @@ +{% extends "base.html" %} + +{% block title %}Assertoor — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Assertoor — {{ devnet }}

+ +{% if error and not assertoor_data %} +
{{ error }}
+
+ Assertoor UI: {{ assertoor_url }} +
+{% elif assertoor_data %} + +{% set runs = assertoor_data.runs %} + + +{% set failed = runs | selectattr("status", "equalto", "failure") | list %} +{% set running = runs | selectattr("status", "equalto", "running") | list %} +
+ {% if failed %} + {{ failed | length }} FAILED test run{{ "s" if failed | length != 1 }} + {% elif running %} + {{ running | length }} test run{{ "s" if running | length != 1 }} running + {% else %} + All tests passed ({{ runs | length }} runs) + {% endif %} +
+ +{% if runs %} +

Test runs

+
+ + + + + + + + + + + + + {% for r in runs %} + + + + + + + + + {% endfor %} + +
Run IDStatusNameTest IDStartedStopped
{{ r.run_id }} + {% if r.status == "success" %} + success + {% elif r.status == "failure" %} + failure + {% elif r.status == "running" %} + running + {% else %} + {{ r.status }} + {% endif %} + {{ r.name or "-" }}{{ r.test_id or "-" }}{{ r.started_at or "-" }}{{ r.stopped_at or "-" }}
+
+{% endif %} + +
+ Assertoor UI: {{ assertoor_data.assertoor_url }} +
+ +{% else %} +
No assertoor data. Run: dv collect {{ devnet }} assertoor +
Assertoor UI: {{ assertoor_url }} +
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/bal.html b/devnets_monitor/web/templates/bal.html new file mode 100644 index 0000000..6f72f95 --- /dev/null +++ b/devnets_monitor/web/templates/bal.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{% block title %}BAL -- {{ devnet }}{% endblock %} + +{% block content %} +

BAL Inspection (EIP-7928) -- {{ devnet }}

+

+ Block-Level Access Lists for ethrex-proposed canonical slots. + Collected via dv collect {{ devnet }} slow. +

+ +{% if error %} +
{{ error }}
+{% elif bal_data %} + +
+

Summary

+ + + + +
Total entries{{ bal_data.total }}
Zero-access slots{{ bal_data.zero_count }}
Avg access_count{{ bal_data.avg_access_count }}
+
+ +
+

Recent ethrex BAL entries (newest first)

+
+ + + + + + + + + + + + {% for r in bal_data.rows %} + + + + + + + + {% endfor %} + +
SlotEL blockaccess_countProposerFetched
{{ r.slot }}{{ r.eth_block_number if r.eth_block_number is not none else "-" }} + {% if r.access_count == 0 %} + 0 + {% else %} + {{ r.access_count }} + {% endif %} + {{ r.proposer_name }}{{ r.fetched_at }}
+
+
+ +{% else %} +
No BAL data. Run: dv collect {{ devnet }} slow
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/base.html b/devnets_monitor/web/templates/base.html new file mode 100644 index 0000000..c2dd1fe --- /dev/null +++ b/devnets_monitor/web/templates/base.html @@ -0,0 +1,305 @@ + + + + + + + {% block title %}ethrex devnets{% endblock %} + + {% block head_extra %}{% endblock %} + + +
+
ethrex-devnets
+ + {% if devnet_names is defined and devnet_names %} +
+ + +
+ {% endif %} +
+
+ {% block content %}{% endblock %} +
+
ethrex-devnets dashboard — read-only — localhost only — no auth
+ + diff --git a/devnets_monitor/web/templates/blobs.html b/devnets_monitor/web/templates/blobs.html new file mode 100644 index 0000000..fae2f64 --- /dev/null +++ b/devnets_monitor/web/templates/blobs.html @@ -0,0 +1,183 @@ +{% extends "base.html" %} + +{% block title %}Blobs — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Blob inclusion — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif blob_data %} + +

Window: {{ blob_data.window_label }}

+ + +

EL client comparison

+
+ + + + + + + + + + + {% for c in blob_data.clients %} + + + + + + + {% endfor %} + +
ClientSlotsAvg blobsTotal blobs
+ {% if c.is_ethrex %} + {{ c.client }} + {% else %} + {{ c.client }} + {% endif %} + {{ c.slots }}{{ "%.2f" | format(c.avg_blobs) }}{{ c.total_blobs }}
+
+ + +

Per-proposer

+
+ + + + + + + + + + + {% for p in blob_data.proposers %} + + + + + + + {% endfor %} + +
ProposerSlotsAvg blobsTrend
{{ p.name }}{{ p.count }}{{ "%.2f" | format(p.avg_blobs) }}{{ p.sparkline }}
+
+ + +

Avg blobs per block over time (per client)

+
+
+ + +
+ + + +{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/clients.html b/devnets_monitor/web/templates/clients.html new file mode 100644 index 0000000..bbe5379 --- /dev/null +++ b/devnets_monitor/web/templates/clients.html @@ -0,0 +1,95 @@ +{% extends "base.html" %} + +{% block title %}Clients — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Clients — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif clients_data %} + +

Snapshot at {{ clients_data.ts_str }}

+ + +{% if clients_data.ethrex_versions %} +

ethrex versions live

+
+ + + + + + {% for ev in clients_data.ethrex_versions %} + + + + + {% endfor %} + +
VersionCount
{{ ev.version }}{{ ev.count }}
+
+{% endif %} + + +{% if clients_data.overview %} +{% set ov = clients_data.overview %} +

Network overview

+
+ slot={{ ov.current_slot if ov.current_slot is not none else "?" }}, + epoch={{ ov.current_epoch if ov.current_epoch is not none else "?" }}, + finalized={{ ov.finalized_epoch if ov.finalized_epoch is not none else "?" }}, + justified={{ ov.justified_epoch if ov.justified_epoch is not none else "?" }} +
+{% endif %} + + +

Fork agreement

+
+ Head forks (splits): {{ clients_data.fork_count }} + {% if clients_data.fork_count == 1 %} + — consensus (all nodes on same head) + {% elif clients_data.fork_count > 1 %} + — SPLIT (nodes on different heads) + {% endif %} +   |   + Network detail +   |   + Forkmon: {{ forkmon_url }} (link-out only, no API) +
+ + +{% if clients_data.client_dist %} +

Client distribution (EL + CL)

+
+ + + + + + {% for r in clients_data.client_dist %} + + + + + + + {% endfor %} + +
LayerClientCountVersion
{{ r.layer }} + {% if r.client == "ethrex" %} + {{ r.client }} + {% else %} + {{ r.client }} + {% endif %} + {{ r.count }}{{ (r.version or "")[:80] }}
+
+{% endif %} + +{% else %} +
No client data. Run: dv collect {{ devnet }} clients
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/command_center.html b/devnets_monitor/web/templates/command_center.html new file mode 100644 index 0000000..b1dd5d2 --- /dev/null +++ b/devnets_monitor/web/templates/command_center.html @@ -0,0 +1,312 @@ +{% extends "base.html" %} + +{% block title %}Command Center — ethrex devnets{% endblock %} + +{# T3: 30-second auto-refresh on this page only #} +{% block head_extra %} + + +{% endblock %} + +{% block content %} +
+

Command Center

+ auto-refreshing every 30s +
+ +{% set devnet_names = devnets | map(attribute='name') | list %} + +{% for dv_entry in devnets %} +{% set cc = dv_entry.cc %} +
+
+ {{ dv_entry.name }} +
+ + {# --- Summary bar: finality, fork countdown, blob flow --- #} +
+ {% if cc.finality %} +
+ slot + {{ cc.finality.current_slot if cc.finality.current_slot is not none else "?" }} +
+
+ epoch + {{ cc.finality.current_epoch if cc.finality.current_epoch is not none else "?" }} +
+
+ finalized + {% if cc.finality.finalized_epoch is not none %} + {{ cc.finality.finalized_epoch }} + {% else %} + ? + {% endif %} +
+ {% if cc.finality.justified_epoch is not none %} +
+ justified + {{ cc.finality.justified_epoch }} +
+ {% endif %} + | + {% endif %} + {% if cc.next_fork %} +
+ next fork + {{ cc.next_fork.fork }} + {{ cc.next_fork.countdown }} +
+ | + {% endif %} +
+ blob load + {% if cc.blob_flow == "ON" %} + ON + {% elif cc.blob_flow == "OFF" %} + OFF + {% else %} + unknown + {% endif %} +
+
+ + {# --- Node status tiles --- #} + {% if cc.nodes %} +
+ {% for h in cc.nodes %} + {% set rollup = h.rollup if h.rollup else "amber" %} +
+
+ {{ h.node }} +   + {% if rollup == "green" %} + OK + {% elif rollup == "amber" %} + warn + {% else %} + crit + {% endif %} +
+
+ head + {{ h.head if h.head is not none else "-" }} +   + peers + {{ h.peers if h.peers is not none else "-" }} +
+
+ state@head + {% if (h.state_at_head or "") | lower == "yes" %} + yes + {% else %} + {{ h.state_at_head or "?" }} + {% endif %} +   + sync + {% if h.syncing == "yes" %} + syncing + {% else %} + synced + {% endif %} +
+
+ commit + {% if h.commit %} + + {{ h.commit[:7] }} + + {% else %} + - + {% endif %} + {% if h.buildnum %} +  bn{{ h.buildnum }} + {% endif %} +
+
{{ h.ts }}
+
+ {% endfor %} +
+ {% else %} +
No node health data. Run: dv collect {{ dv_entry.name }} health
+ {% endif %} + + {# --- Latest events --- #} + {% if cc.events %} +

Active events ({{ cc.events | length }}{% if cc.events | length == 8 %}+{% endif %})

+
+ {% for e in cc.events %} +
+ {% if e.severity == "crit" %} + crit + {% elif e.severity == "warn" %} + warn + {% else %} + {{ e.severity }} + {% endif %} + {{ e.kind | humanize }} + {% if e.node %} + {{ e.node }} + {% endif %} + {{ e.message }} + {{ e.last_seen_str }} +
+ {% endfor %} +
+

+ All events → +

+ {% else %} +

No active events.

+ {% endif %} + + {# --- Hive summary --- #} + {% if dv_entry.hive %} +

Hive (latest run per group)

+
+ + + + + + + + + + + + + {% for h in dv_entry.hive %} + + + + + + + + + {% endfor %} + +
GroupPassFailTotalStartedLink
{{ h.group }}{{ h.passes }} + {% if h.fails != "?" and h.fails | int > 0 %} + {{ h.fails }} + {% else %} + {{ h.fails }} + {% endif %} + {{ h.ntests }}{{ h.started if h.started else "-" }}{% if h.web_url %}view{% else %}-{% endif %}
+
+ {% endif %} + + {# --- Services table --- #} +

Services

+
+ + + + + + {% for svc in dv_entry.services %} + + + + + {% endfor %} + +
ServiceURL
{{ svc.label }}{{ svc.url }}
+
+ +
+{% endfor %} + +{% if not devnets %} +
No devnets configured in config/devnets.yaml.
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/deploy.html b/devnets_monitor/web/templates/deploy.html new file mode 100644 index 0000000..6bdda0b --- /dev/null +++ b/devnets_monitor/web/templates/deploy.html @@ -0,0 +1,136 @@ +{% extends "base.html" %} + +{% block title %}Deploy — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Deploy — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif deploy_data %} + + +{% if deploy_data.gap %} +

GitHub gap (deployed vs lambdaclass/ethrex main)

+
+ + + + + + + + + + + + {% for g in deploy_data.gap %} + + + + + + + + {% endfor %} + +
NodeDeployed commitMain commitCommits behindChecked at
{{ g.node }} + {{ g.deployed_commit }} + {{ g.main_commit }} + {% if g.commits_behind is not none %} + {% if g.commits_behind == 0 %} + up to date + {% elif g.commits_behind <= 5 %} + {{ g.commits_behind }} + {% else %} + {{ g.commits_behind }} + {% endif %} + {% else %} + ? (run dv collect slow) + {% endif %} + {{ g.checked_at or "-" }}
+
+{% else %} +
No GitHub gap data. Run: dv collect {{ devnet }} slow
+{% endif %} + + +{% if deploy_data.nodes %} +

Version timeline (from health snapshots)

+{% for node, versions in deploy_data.nodes.items() | sort %} +

{{ node }}

+
+ + + + + + + + + + + + {% for v in versions %} + + + + + + + + {% endfor %} + +
TimestampCommitBuildHeadPeers
{{ v.ts_str }} + {% if v.commit_full %} + {{ v.commit }} + {% else %}{{ v.commit }}{% endif %} + {{ v.buildnum or "-" }}{{ v.head if v.head is not none else "-" }}{{ v.peers if v.peers is not none else "-" }}
+
+{% endfor %} +{% endif %} + + +{% if deploy_data.events %} +

Recent events

+
+ + + + + + + + + + + + {% for e in deploy_data.events %} + + + + + + + + {% endfor %} + +
SeverityKindNodeMessageLast seen
+ {% if e.severity == "crit" %} + crit + {% elif e.severity == "warn" %} + warn + {% else %} + {{ e.severity }} + {% endif %} + {{ e.kind }}{{ e.node or "-" }}{{ e.message }}{{ e.ts_str }}
+
+{% endif %} + +{% else %} +
No deploy data. Run: dv collect {{ devnet }} health
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/eiptrack.html b/devnets_monitor/web/templates/eiptrack.html new file mode 100644 index 0000000..07de8db --- /dev/null +++ b/devnets_monitor/web/templates/eiptrack.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% block title %}EIP tracker -- {{ devnet }}{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block content %} +

EIP Tracker -- {{ devnet }}

+

+ ethrex implementation status for upcoming fork EIPs. Set the + status field in config/eips.json and re-run + dv collect {{ devnet }} forks to update. + Values: done, in_progress, missing, + n/a, unknown. +

+ +{% if error %} +
{{ error }}
+{% elif eiptrack_data %} + +
+

Fork: {{ eiptrack_data.fork }} -- {{ eiptrack_data.total }} EIPs

+

+ {% for s, n in eiptrack_data.stage_counts.items() %} + {{ s }}: {{ n }}{% if not loop.last %}  |  {% endif %} + {% endfor %} +

+ +
+ {% for g in eiptrack_data.groups %} + + {{ g.label }}: {{ g.count }} + + {% endfor %} +
+ + {% for g in eiptrack_data.groups %} + {% if g.eips %} +
+

+ {{ g.label }} + ({{ g.count }}) +

+
+ {% for e in g.eips %} +
+ EIP-{{ e.eip }} + {% if e.stage %} + {{ e.stage }} + {% endif %} + {{ e.title }} +
+ {% endfor %} +
+
+ {% endif %} + {% endfor %} +
+ +{% else %} +
+ No EIP-track data. Run: dv collect {{ devnet }} forks +
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/epbs.html b/devnets_monitor/web/templates/epbs.html new file mode 100644 index 0000000..922c6a1 --- /dev/null +++ b/devnets_monitor/web/templates/epbs.html @@ -0,0 +1,77 @@ +{% extends "base.html" %} +{% block title %}ePBS -- {{ devnet }}{% endblock %} + +{% block content %} +

ePBS Panel (EIP-7732) -- {{ devnet }}

+

+ Bid count and PTC vote data per slot. + Collected via dv collect {{ devnet }} slow. + Ethrex-proposed slots are highlighted. +

+ +{% if error %} +
{{ error }}
+{% elif epbs_data %} + +
+

Summary

+ + + + + + +
Total entries{{ epbs_data.total }}
Ethrex entries{{ epbs_data.ethrex_rows | length }}
Avg PTC size{{ epbs_data.avg_ptc_size }}
Avg PTC vote count{{ epbs_data.avg_vote_count }}
Avg non-voter %{{ epbs_data.avg_nonvoter_pct }}
+
+ +
+

Recent slots (newest first)

+
+ + + + + + + + + + + + + + + {% for r in epbs_data.rows %} + + + + + + + + + + + {% endfor %} + +
SlotBidsPTC sizePTC votesNon-voter %Payload revealedProposerFetched
+ {{ r.slot }} + {% if r.is_ethrex %}ethrex{% endif %} + {{ r.bid_count }}{{ r.ptc_size if r.ptc_size is not none else "-" }}{{ r.ptc_vote_count if r.ptc_vote_count is not none else "-" }} + {% if r.ptc_nonvoter_pct is not none %} + {% if r.ptc_nonvoter_pct > 10 %} + {{ r.ptc_nonvoter_pct }}% + {% else %} + {{ r.ptc_nonvoter_pct }}% + {% endif %} + {% else %} + - + {% endif %} + {{ r.payload_revealed }}{{ r.proposer_name }}{{ r.fetched_at }}
+
+
+ +{% else %} +
No ePBS data. Run: dv collect {{ devnet }} slow
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/events.html b/devnets_monitor/web/templates/events.html new file mode 100644 index 0000000..6b67029 --- /dev/null +++ b/devnets_monitor/web/templates/events.html @@ -0,0 +1,134 @@ +{% extends "base.html" %} + +{% block title %}Events — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Events — {{ devnet }}

+ +{% if error and not active_events and not resolved_events %} +
{{ error }}
+{% else %} + + +

Active events ({{ active_events | length }})

+{% if active_events %} +
+ + + + + + + + + + + + + + + {% for e in active_events %} + + + + + + + + + + + {% if e.details %} + + + + {% endif %} + {% endfor %} + +
SeverityKindNodeMessageCountFirst seenLast seenFlags
+ {% if e.severity == 'crit' %} + crit + {% elif e.severity == 'warn' %} + warn + {% else %} + {{ e.severity }} + {% endif %} + {{ e.kind | humanize }}{% if e.node %}{{ e.node }}{% else %}-{% endif %}{{ e.message }}{{ e.count }}{{ e.first_seen_str or "-" }}{{ e.last_seen_str or "-" }} + {% if e.flapping %} + FLAPPING + {% endif %} +
+ {% for k, v in e.details.items() %} + {{ k | humanize }}: + {% if 'commit' in k|lower and v %}{{ v }} + {% elif v is string and v|length > 60 %}{{ v }} + {% else %}{{ v }}{% endif %} + + {% endfor %} +
+
+{% else %} +
No active events. Run: dv collect {{ devnet }} events to detect.
+{% endif %} + + +{% if resolved_events %} +

+ Recently resolved ({{ resolved_events | length }}) + +

+ + +{% endif %} + +{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/exectime.html b/devnets_monitor/web/templates/exectime.html new file mode 100644 index 0000000..5dd3bb5 --- /dev/null +++ b/devnets_monitor/web/templates/exectime.html @@ -0,0 +1,164 @@ +{% extends "base.html" %} + +{% block title %}Exec time — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Execution time per client — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif exectime_data %} + +

Window: {{ exectime_data.window_label }}

+ + +

Client comparison

+
+ + + + + + + + + + + + {% for c in exectime_data.clients %} + + + + + + + + {% endfor %} + +
ClientSamplesAvg msMin msMax ms
+ {% if c.is_ethrex %} + {{ c.client_type }} + {% else %} + {{ c.client_type }} + {% endif %} + {{ c.samples }}{{ "%.1f" | format(c.avg_ms) }}{{ "%.1f" | format(c.min_ms) if c.min_ms is not none else "-" }}{{ "%.1f" | format(c.max_ms) if c.max_ms is not none else "-" }}
+
+ + +

Avg exec ms over slots (per client)

+
+
+ + +
+ + + +{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/forks.html b/devnets_monitor/web/templates/forks.html new file mode 100644 index 0000000..010fabe --- /dev/null +++ b/devnets_monitor/web/templates/forks.html @@ -0,0 +1,101 @@ +{% extends "base.html" %} + +{% block title %}Forks — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Fork schedule — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif fork_data %} + +{% if fork_data.next_fork %} +

+ Next fork: {{ fork_data.next_fork }} + {% for f in fork_data.forks %} + {% if f.fork == fork_data.next_fork and f.status != 'active' %} + — {{ f.status }} + {% endif %} + {% endfor %} +

+{% endif %} + +
+ + + + + + + + + + + + + {% for f in fork_data.forks %} + + + + + + + + + {% endfor %} + +
ForkActivation (UTC)StatusBlob targetBlob maxEIPs
+ {% if f.is_next %} + {{ f.fork }} + {% else %} + {{ f.fork }} + {% endif %} + {{ f.time_str }} + {% if f.status == "active" %} + active + {% else %} + {{ f.status }} + {% endif %} + {{ f.blob_target if f.blob_target is not none else "-" }}{{ f.blob_max if f.blob_max is not none else "-" }} + {% if f.eips %} + {% set staged = f.eips | selectattr('stage') | list %} + {% if staged %} + {# Pre-freeze fork: group by inclusion stage (SFI/CFI/PFI). #} + {% for stage, label, cls in [('SFI','Scheduled','stage-sfi'), ('CFI','Considered','stage-cfi'), ('PFI','Proposed','stage-pfi')] %} + {% set group = f.eips | selectattr('stage','equalto',stage) | list %} + {% if group %} +
+ {{ stage }} +
+ {% for eip in group %} + EIP-{{ eip.eip }} + {% endfor %} +
+
+ {% endif %} + {% endfor %} + {{ f.eips | length }} EIPs (SFI/CFI/PFI; Declined excluded) + {% else %} + {# Settled fork: flat shipped set. #} +
+ {% for eip in f.eips %} + EIP-{{ eip.eip }} + {% endfor %} +
+ {{ f.eips | length }} EIP{{ "s" if f.eips | length != 1 }} + {% endif %} + {% else %} + — + {% endif %} +
+
+ +{% else %} +
No fork data for {{ devnet }}. Run: dv collect {{ devnet }} forks
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/hive.html b/devnets_monitor/web/templates/hive.html new file mode 100644 index 0000000..3c7f3d4 --- /dev/null +++ b/devnets_monitor/web/templates/hive.html @@ -0,0 +1,54 @@ +{% extends "base.html" %} + +{% block title %}Hive — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Hive runs — {{ devnet }}

+

Most recent ethrex run per (group, suite). Run: dv collect {{ devnet }} hive to refresh.

+ +{% if error %} +
{{ error }}
+{% elif rows %} + + + + + + + + + + + + + + + {% for r in rows %} + + + + + + + + + + + {% endfor %} + +
GroupSuitePassFailTotalVersionStartedLink
{{ r.group }}{{ r.suite }}{{ r.passes }} + {% if r.fails != "?" and r.fails | int > 0 %} + {{ r.fails }} + {% else %} + {{ r.fails }} + {% endif %} + {{ r.ntests }} + {{ r.version[:60] }}{% if r.version | length > 60 %}...{% endif %} + {{ r.started if r.started else "-" }}{% if r.web_url %}view{% else %}-{% endif %}
+{% else %} +
No Hive runs for {{ devnet }}. Run: dv collect {{ devnet }} hive
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/incidents.html b/devnets_monitor/web/templates/incidents.html new file mode 100644 index 0000000..96ccf98 --- /dev/null +++ b/devnets_monitor/web/templates/incidents.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} + +{% block title %}Incidents — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Incident history — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif content_html %} +

Source: docs/history/{{ devnet }}.md

+
{{ content_html | safe }}
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/index.html b/devnets_monitor/web/templates/index.html new file mode 100644 index 0000000..3a24e13 --- /dev/null +++ b/devnets_monitor/web/templates/index.html @@ -0,0 +1,135 @@ +{% extends "base.html" %} + +{% block title %}ethrex devnets{% endblock %} + +{% block content %} +

ethrex devnets

+ +{% set devnet_names = devnets | map(attribute='name') | list %} + +{% for devnet_entry in devnets %} +
+

+ {{ devnet_entry.name }} +

+ + + + +

Node health (latest snapshot)

+ {% if devnet_entry.health %} +
+ + + + + + + + + + + + + + + {% for h in devnet_entry.health %} + + + + + + + + + + + {% endfor %} + +
NodeHeadPeersState@HeadSyncingBuildCommitSnapshot at
{{ h.node }}{{ h.head if h.head is not none else "-" }}{{ h.peers if h.peers is not none else "-" }}{{ h.state_at_head if h.state_at_head else "-" }} + {% if h.syncing == "yes" %} + syncing + {% else %} + synced + {% endif %} + {{ h.buildnum if h.buildnum else "-" }} + {% if h.commit %} + {{ h.commit[:7] }} + {% else %}-{% endif %} + {{ h.ts if h.ts else "-" }}
+
+ {% else %} +
No node health data. Run: dv collect {{ devnet_entry.name }} health
+ {% endif %} + + +

Hive (latest run per group)

+ {% if devnet_entry.hive %} +
+ + + + + + + + + + + + + {% for h in devnet_entry.hive %} + + + + + + + + + {% endfor %} + +
GroupPassFailTotalStartedLink
{{ h.group }}{{ h.passes }} + {% if h.fails != "?" and h.fails | int > 0 %} + {{ h.fails }} + {% else %} + {{ h.fails }} + {% endif %} + {{ h.ntests }}{{ h.started if h.started else "-" }}{% if h.web_url %}view{% else %}-{% endif %}
+
+ {% else %} +
No Hive data. Run: dv collect {{ devnet_entry.name }} hive
+ {% endif %} + + +

Services

+
+ + + + + + {% for svc in devnet_entry.services %} + + + + + {% endfor %} + +
ServiceURL
{{ svc.label }}{{ svc.url }}
+
+
+{% endfor %} + +{% if not devnets %} +
No devnets configured in config/devnets.yaml.
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/network.html b/devnets_monitor/web/templates/network.html new file mode 100644 index 0000000..6b58aff --- /dev/null +++ b/devnets_monitor/web/templates/network.html @@ -0,0 +1,130 @@ +{% extends "base.html" %} + +{% block title %}Network — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Network — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif network_data %} + +

Snapshot at ts={{ network_data.ts }}

+ + +

Head forks ({{ network_data.fork_count }})

+
+ + + + + + + + + + + + {% for s in network_data.splits %} + + + + + + + + {% endfor %} + +
Fork IDHead slotHead rootClientsCanonical
{{ s.fork_id or "-" }}{{ s.head_slot if s.head_slot is not none else "-" }}{{ (s.head_root or "")[:16] }}{% if s.head_root %}...{% endif %}{{ s.head_count if s.head_count is not none else "-" }} + {% if s.is_canonical %} + yes + {% else %} + no + {% endif %} +
+
+ + +{% if network_data.clients %} +

Clients on head ({{ network_data.clients | length }})

+
+ + + + + + + + + + + + + {% for c in network_data.clients %} + {% set parts = (c.name or "").split("-") %} + {% set cl_client = parts[0] if parts | length >= 1 else "" %} + {% set el_client = parts[1] if parts | length >= 2 else "" %} + + + + + + + + + {% endfor %} + +
NameCL clientEL clientStatusHead slotDistance
+ {% if el_client == "ethrex" %} + + {{ c.name }} + + {% else %} + {{ c.name }} + {% endif %} + {{ cl_client }}{{ el_client }} + {% if c.status == "online" %} + online + {% elif c.status == "optimistic" %} + optimistic + {% else %} + {{ c.status or "?" }} + {% endif %} + {{ c.head_slot if c.head_slot is not none else "-" }} + {% if c.distance is not none and c.distance > 0 %} + +{{ c.distance }} + {% else %} + 0 + {% endif %} +
+
+{% endif %} + + +{% if network_data.client_dist %} +

CL client distribution

+
+ + + + + + {% for d in network_data.client_dist %} + + + + + + {% endfor %} + +
LayerClientCount
{{ d.layer }}{{ d.client }}{{ d.count }}
+
+{% endif %} + +{% else %} +
No network data. Run: dv collect {{ devnet }} network
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/node.html b/devnets_monitor/web/templates/node.html new file mode 100644 index 0000000..00f3848 --- /dev/null +++ b/devnets_monitor/web/templates/node.html @@ -0,0 +1,314 @@ +{% extends "base.html" %} + +{% block title %}Node {{ node }} — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

+ + Command Center / + {{ devnet }} / + + {{ node }} +

+ +{% if error %} +
{{ error }}
+{% elif node_data %} + +{# --- Latest snapshot --- #} +{% if node_data.latest %} +{% set lat = node_data.latest %} +

Latest snapshot

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {% if lat.restart is not none %} + + + + + {% endif %} + {% if lat.cl_line %} + + + + + {% endif %} + +
snapshot at{{ lat.ts }}
head block{{ lat.head if lat.head is not none else "-" }}
peers{{ lat.peers if lat.peers is not none else "-" }}
state@head + {% if (lat.state_at_head or "") | lower == "yes" %} + yes + {% else %} + {{ lat.state_at_head or "?" }} + {% endif %} +
syncing + {% if lat.syncing == "yes" %} + syncing + {% else %} + synced + {% endif %} +
commit + {% if lat.commit %} + {{ lat.commit[:12] }} + {% else %}-{% endif %} +
buildnum{{ lat.buildnum or "-" }}
image{{ lat.image or "-" }}
restarts{{ lat.restart }}
CL{{ lat.cl_line }}
+
+{% else %} +
No health snapshot found for this node. Run: dv collect {{ devnet }} health
+{% endif %} + +{# --- Events --- #} +

Events for this node

+{% if node_data.events %} +
+ + + + + + + + + + + + + + + {% for e in node_data.events %} + + + + + + + + + + + {% endfor %} + +
StatusSeverityKindMessageCountFirst seenLast seenResolved
+ {% if e.active %} + active + {% else %} + resolved + {% endif %} + + {% if e.severity == "crit" %} + crit + {% elif e.severity == "warn" %} + warn + {% else %} + {{ e.severity }} + {% endif %} + {{ e.kind | humanize }}{{ e.message }}{{ e.count }}{{ e.first_seen_str }}{{ e.last_seen_str }}{{ e.resolved_at_str if not e.active else "-" }}
+
+{% else %} +
No events for this node.
+{% endif %} + +{# --- Version history --- #} +

Version history

+{% if node_data.version_history %} +
+ + + + + + + + + + + + + {% for v in node_data.version_history %} + + + + + + + + + {% endfor %} + +
TimestampCommitBuildImageHeadPeers
{{ v.ts_str }} + {% if v.commit_full %} + {{ v.commit }} + {% else %}{{ v.commit }}{% endif %} + {{ v.buildnum or "-" }}{{ v.image or "-" }}{{ v.head if v.head is not none else "-" }}{{ v.peers if v.peers is not none else "-" }}
+
+{% else %} +
No version history (no commits recorded). Run: dv collect {{ devnet }} health
+{% endif %} + +{# --- Health history --- #} +

Health history (last 20 snapshots)

+{% if node_data.health_history %} +
+ + + + + + + + + + + + + + {% for h in node_data.health_history %} + + + + + + + + + + {% endfor %} + +
TimestampHeadPeersState@HeadSyncingBuildCommit
{{ h.ts }}{{ h.head if h.head is not none else "-" }}{{ h.peers if h.peers is not none else "-" }} + {% if (h.state_at_head or "") | lower == "yes" %} + yes + {% elif h.state_at_head %} + {{ h.state_at_head }} + {% else %}-{% endif %} + + {% if h.syncing == "yes" %} + syncing + {% else %} + no + {% endif %} + {{ h.buildnum or "-" }} + {% if h.commit %} + {{ h.commit[:7] }} + {% else %}-{% endif %} +
+
+{% else %} +
No health snapshots yet.
+{% endif %} + +{# --- Proposals --- #} +

Proposals

+{% if node_data.proposals and node_data.proposals.total > 0 %} +{% set prop = node_data.proposals %} +
+ Total: {{ prop.total }}  |  + Canonical: {{ prop.canonical }}  |  + Missed: + {% if prop.missed > 0 %} + {{ prop.missed }} + {% else %}{{ prop.missed }}{% endif %} +  |  + Orphaned: + {% if prop.orphaned > 0 %} + {{ prop.orphaned }} + {% else %}{{ prop.orphaned }}{% endif %} + {% if prop.total > 0 %} +  |  + Orphan rate: + {% set orphan_pct = (prop.orphaned / prop.total * 100) %} + {% if orphan_pct > 10 %} + {{ "%.1f%%" | format(orphan_pct) }} + {% elif orphan_pct > 5 %} + {{ "%.1f%%" | format(orphan_pct) }} + {% else %} + {{ "%.1f%%" | format(orphan_pct) }} + {% endif %} + {% endif %} +
+{% if prop.recent %} +

Recent slots (last {{ prop.recent | length }})

+
+ + + + + + + + + + + {% for s in prop.recent %} + + + + + + + {% endfor %} + +
SlotStatusBlobsTime
{{ s.slot }} + {% set st = (s.status or "") | lower %} + {% if st == "canonical" %} + canonical + {% elif st in ("missing", "missed") %} + missed + {% elif st == "orphaned" %} + orphaned + {% else %} + {{ s.status or "?" }} + {% endif %} + {{ s.blob_count if s.blob_count is not none else "-" }}{{ s.time }}
+
+{% endif %} +{% else %} +
No proposals found for this node in the database.
+{% endif %} + +{# --- Live data note --- #} +
+ Live data (not shown here; DB-only view): + use dv logs {{ devnet }} {{ node }} for execution logs, + dv peers {{ devnet }} for live peer info, + dv status {{ devnet }} for live health over SSH. +
+ +{% else %} +
No data for node {{ node }} in devnet {{ devnet }}.
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/proposals.html b/devnets_monitor/web/templates/proposals.html new file mode 100644 index 0000000..59741fb --- /dev/null +++ b/devnets_monitor/web/templates/proposals.html @@ -0,0 +1,134 @@ +{% extends "base.html" %} + +{% block title %}Proposals — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Proposals — {{ devnet }}

+ +{% if error %} +
{{ error }}
+{% elif proposals_data %} + +

Window: {{ proposals_data.window_label }}

+ + +

EL client comparison

+
+ + + + + + + + + + + + + {% for c in proposals_data.clients %} + + + + + + + + + {% endfor %} + +
ClientTotalCanonicalMissedOrphanedOrphan rate
+ {% if c.is_ethrex %} + {{ c.client }} + {% else %} + {{ c.client }} + {% endif %} + {{ c.total }}{{ c.canonical }} + {% if c.missed > 0 %} + {{ c.missed }} + {% else %} + {{ c.missed }} + {% endif %} + + {% if c.orphaned > 0 %} + {{ c.orphaned }} + {% else %} + {{ c.orphaned }} + {% endif %} + + {% if c.orphan_rate > 0.1 %} + {{ "%.1f%%" | format(c.orphan_rate * 100) }} + {% elif c.orphan_rate > 0.05 %} + {{ "%.1f%%" | format(c.orphan_rate * 100) }} + {% else %} + {{ "%.1f%%" | format(c.orphan_rate * 100) }} + {% endif %} +
+
+ + +

Per-proposer breakdown

+
+ + + + + + + + + + + + + + {% for p in proposals_data.proposers %} + + + + + + + + + + {% endfor %} + +
ProposerClientTotalCanonicalMissedOrphanedOrphan rate
+ {% if p.is_ethrex %} + + {{ p.name }} + + {% else %} + {{ p.name }} + {% endif %} + {{ p.client }}{{ p.total }}{{ p.canonical }} + {% if p.missed > 0 %} + {{ p.missed }} + {% else %} + {{ p.missed }} + {% endif %} + + {% if p.orphaned > 0 %} + {{ p.orphaned }} + {% else %} + {{ p.orphaned }} + {% endif %} + + {% if p.orphan_rate > 0.1 %} + {{ "%.1f%%" | format(p.orphan_rate * 100) }} + {% elif p.orphan_rate > 0.05 %} + {{ "%.1f%%" | format(p.orphan_rate * 100) }} + {% else %} + {{ "%.1f%%" | format(p.orphan_rate * 100) }} + {% endif %} +
+
+ +{% else %} +
No proposal data. Run: dv collect {{ devnet }} blobs
+{% endif %} +{% endblock %} diff --git a/devnets_monitor/web/templates/spamoor.html b/devnets_monitor/web/templates/spamoor.html new file mode 100644 index 0000000..68cdf54 --- /dev/null +++ b/devnets_monitor/web/templates/spamoor.html @@ -0,0 +1,96 @@ +{% extends "base.html" %} + +{% block title %}Spamoor — {{ devnet }}{% endblock %} + +{% set current_devnet = devnet %} +{% set devnet_names = [devnet] %} + +{% block content %} +

Spamoor — {{ devnet }}

+ +{% if error and not spamoor_data %} +
{{ error }}
+
+ Spamoor UI: {{ spamoor_url }} +
+{% elif spamoor_data %} + +

Snapshot at {{ spamoor_data.ts_str }}

+ + +
+ {% if spamoor_data.blob_active %} + BLOB LOAD: ON (blob spammer active) + {% elif spamoor_data.blob_active is sameas false %} + BLOB LOAD: OFF (no active blob spammer) + {% else %} + BLOB LOAD: unknown + {% endif %} +
+ + +{% if spamoor_data.spammers %} +

Spammers

+
+ + + + + + + + + + + + {% for s in spamoor_data.spammers %} + {% set is_blob = "blob" in (s.scenario or "").lower() %} + + + + + + + + {% endfor %} + +
IDNameScenarioStatusEnabled
{{ s.spammer_id }} + {% if is_blob %} + {{ s.name or "-" }} + {% else %} + {{ s.name or "-" }} + {% endif %} + {{ s.scenario or "-" }} + {% if s.status == 1 %} + running + {% else %} + stopped + {% endif %} + + {% if s.enabled %} + yes + {% else %} + no + {% endif %} +
+
+{% endif %} + +
+ Spamoor UI: {{ spamoor_data.spamoor_url }} +
+ +{% else %} +
No spamoor data. Run: dv collect {{ devnet }} spamoor +
Spamoor UI: {{ spamoor_url }} +
+{% endif %} +{% endblock %} From cc2b252314244913ba1560208968b967549c1802 Mon Sep 17 00:00:00 2001 From: Edgar Date: Wed, 10 Jun 2026 08:56:42 +0200 Subject: [PATCH 2/3] docs: adapt devnet-ops skill + README to devnets_monitor/ subdir --- .../.claude/skills/devnet-ops/SKILL.md | 23 +++++++++++-------- devnets_monitor/README.md | 12 ++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/devnets_monitor/.claude/skills/devnet-ops/SKILL.md b/devnets_monitor/.claude/skills/devnet-ops/SKILL.md index d766e87..71c263a 100644 --- a/devnets_monitor/.claude/skills/devnet-ops/SKILL.md +++ b/devnets_monitor/.claude/skills/devnet-ops/SKILL.md @@ -5,9 +5,11 @@ description: Operate, inspect, monitor, and debug ethrex EL nodes on ethpandaops # devnet-ops -This repo (`ethrex-devnets`) is the single home for ethrex devnet ops + monitoring. -Procedures, per-devnet history, and the `dv` CLI all live here. Paths below are -relative to the repo root. +This ethrex devnet ops + monitoring toolkit lives in the `devnets_monitor/` +directory of the `ethrex-tooling` repo (ported from the standalone `ethrex-devnets` +repo). It is self-contained: procedures, per-devnet history, and the `dv` CLI all +live here. Run every command from `devnets_monitor/` (`cd devnets_monitor` first); +all paths below are relative to that directory. Before any devnet operation or answering a question about a node/incident, READ: @@ -17,13 +19,14 @@ Before any devnet operation or answering a question about a node/incident, READ: container layout, build & deploy, debug logging, wipe & resync, Dora API). Substitute `` with the target network. 3. `docs/history/.md` — per-devnet facts and incident history (roster, fork - schedule, commit map, known issues with root cause + recovery). For - glamsterdam-devnet-5 this is `docs/history/glamsterdam-devnet-5.md`. If a devnet - has no history file, create one from `docs/history/_template.md` as you learn facts. + schedule, commit map, known issues with root cause + recovery). These files are + gitignored (local-only; they embed host internals), so a freshly-cloned tree has + none. If a devnet has no history file, create one from + `docs/history/_template.md` as you learn facts. ## The `dv` CLI -Run from the repo root via `uv run dv ...`. Read-only by default; only `dv wipe` +Run from `devnets_monitor/` via `uv run dv ...`. Read-only by default; only `dv wipe` mutates (gated behind `--yes`). Target devnet resolves: explicit arg > `$DEVNET` env > `config/devnets.yaml` `default`. @@ -61,6 +64,6 @@ uv run dv wipe --yes # MUTATING: recover a wedged EL ## Maintenance When you discover a new incident, divergence, or devnet fact, append a dated entry -to `docs/history/.md` (and `docs/devnet-ops.md` if a procedure changed) and -commit it. The fork -> EIP map is `config/eips.json` (sourced via eipmcp; re-run -`get_hardfork` to refresh). +to `docs/history/.md` (local-only, gitignored; do not commit it). If a +procedure changed, update `docs/devnet-ops.md` and commit that. The fork -> EIP map +is `config/eips.json` (sourced via eipmcp; re-run `get_hardfork` to refresh). diff --git a/devnets_monitor/README.md b/devnets_monitor/README.md index 61c8c3e..2047729 100644 --- a/devnets_monitor/README.md +++ b/devnets_monitor/README.md @@ -1,9 +1,13 @@ -# ethrex-devnets +# devnets_monitor Ops + monitoring toolkit for [ethrex](https://github.com/lambdaclass/ethrex) on -[ethpandaops](https://ethpandaops.io) devnets. Single home for the devnet ops -runbook, per-devnet incident history, and the tooling that watches ethrex as new -forks (glamsterdam, BAL, fusaka, ...) roll out. +[ethpandaops](https://ethpandaops.io) devnets, living in the `devnets_monitor/` +directory of `ethrex-tooling` (ported from the standalone `ethrex-devnets` repo). +Single home for the devnet ops runbook, per-devnet incident history, and the +tooling that watches ethrex as new forks (glamsterdam, BAL, fusaka, ...) roll out. + +Self-contained: it has its own Python package (`uv`-managed) and `dv` CLI; run all +commands from this directory (`cd devnets_monitor`). Generic across devnets: everything is parameterized by devnet name (`glamsterdam-devnet-5`, `bal-devnet-3`, ...). Read-only by default; every From f13b0f04ff866c7deb255821bdf2a47d1c60de2b Mon Sep 17 00:00:00 2001 From: Edgar Date: Wed, 10 Jun 2026 08:57:07 +0200 Subject: [PATCH 3/3] docs: prettify README title --- devnets_monitor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devnets_monitor/README.md b/devnets_monitor/README.md index 2047729..2bbdd3b 100644 --- a/devnets_monitor/README.md +++ b/devnets_monitor/README.md @@ -1,4 +1,4 @@ -# devnets_monitor +# Devnets Monitor Ops + monitoring toolkit for [ethrex](https://github.com/lambdaclass/ethrex) on [ethpandaops](https://ethpandaops.io) devnets, living in the `devnets_monitor/`