diff --git a/AGENT.md b/AGENT.md index f75a462..4268244 100644 --- a/AGENT.md +++ b/AGENT.md @@ -727,38 +727,82 @@ what is missing rather than sending a request that will fail. ``` tlgr agent whoami -→ {"account": "main", "user_id": 123, "username": "me", "daemon_running": true, ...} +→ {"output_schema_version": 2, "account": "main", "user_id": 123, + "daemon_running": true, "daemon_healthy": true, "layer": 227, ...} -tlgr agent exit-codes -→ {"exit_codes": {...}} +tlgr agent capabilities [--section protocol|policy|gates|events|limits] +→ {"layer": 227, "event_types": 114, "unsupported_constructors": [...], + "prohibited": [{"action": "...", "reason": "..."}], + "premium_gated": [...], "bot_only": [...], "admin_only": [...]} + +tlgr agent exit-codes [--errors] [--search TEXT] +→ {"exit_codes": {...}, "errors": [{"name": "FloodWaitError", "code": + "RATE_LIMITED", "exit": 7, "retryable": true, "extra": "wait_seconds"}]} tlgr agent parity [--uncovered] [--domain NAME] -→ {"catalog_version": "...", "required": 1797, "covered": 306, "percent": 17.0, - "by_priority": {...}, "by_domain": {...}, "uncovered": [...], "waivers": 1491} +→ {"catalog_version": "...", "required": 1797, "covered": 1010, "percent": 56.2, + "by_priority": {...}, "by_domain": {...}, "uncovered": [...], "waivers": 787} + +tlgr schema [commands|events|config|errors|exit-codes|all|...] +→ {"schema_version": 2, "build": "2.0.0", "ops": {...}} -tlgr schema [command_path...] -→ {"schema_version": 2, "build": "2.0.0", "command": {...}} +tlgr status [--check] +→ {"account": "main", "connected": true, "daemon_healthy": true, + "behind_seconds": 3, "problems": []} ``` +`agent capabilities` is the one to read before planning. It separates three +different answers that all look like "no": what this **build** cannot do (a +constructor newer than the pinned Telethon layer), what this **account** may +not reach (premium, bot-only, admin-only), and what tlgr **will not** do — +fake a read receipt, suppress typing status, misrepresent online status, pass +a device-integrity attestation, or execute a payment — each with its reason. +Only the first is a gap somebody might close. + `agent parity` reports coverage of the pinned Telegram feature catalog: what tlgr can do today, per priority and per domain, with every gap either waived -to a named later PR or listed. Use it to find out whether a capability exists -before writing a workaround. Nothing in it is hand-maintained. +to a named later PR or listed. Nothing in it is hand-maintained. + +`status --check` exits non-zero when anything is wrong, and is the cheapest +thing for a monitor to run: a frozen account, an open send circuit breaker, an +outstanding flood deadline and a daemon that is up but not ready are the +states in which every *other* command starts failing. ### Daemon ``` -tlgr daemon start [--foreground] -tlgr daemon stop -tlgr daemon status -→ {"running": true, "ready": true, "pid": 12345, "uptime_seconds": 3600, - "accounts": ["main"], "connections": {"main": true}, "disconnected": [], - "healthy": true, "version": "2.0.0", "protocol": 2} +tlgr daemon start [--foreground] [--catch-up/--no-catch-up] [--wait 30s] +tlgr daemon stop [--grace 10s] +tlgr daemon restart +tlgr daemon status [--check] +→ {"running": true, "ready": true, "healthy": true, "pid": 12345, + "uptime_seconds": 3600, "version": "2.0.0", "protocol": 2, "layer": 227, + "accounts": [{"alias": "main", "state": "online", "pts": 91824, + "behind_seconds": 0, "reconnects": 0}], + "connections": {"main": true}, "disconnected": []} + +tlgr daemon reconnect [--reset-proxy] [--no-catch-up] +tlgr daemon save-state +tlgr daemon logs [--follow] [--lines 50] [--level warning] [--grep TEXT] +tlgr daemon flood list [--include-expired] +tlgr daemon flood clear --every +tlgr daemon dead-letter list | send | delete +tlgr daemon install [--supervisor auto|launchd|systemd] | uninstall ``` -`running` means a process is alive; `ready` means it can actually serve. A -daemon that is up but cannot reach Telegram reports `healthy: false` and names -the accounts in `disconnected` — check `ready`, not `running`. +`running` means a process is alive; `ready` means it can serve; `healthy` +means the accounts are actually working. A daemon that is up but cannot reach +Telegram reports `healthy: false` and names the accounts in `disconnected` — +check `healthy`, not `running`. + +`daemon flood list` is the persistent store of rate-limit deadlines. Telethon +remembers a `FLOOD_WAIT` in memory and forgets it on exit; tlgr writes it per +`(account, method, peer)`, so a fresh process does not immediately re-trip a +wait — which is how a short wait becomes a long one. + +`daemon dead-letter *` is the store of events no consumer could be given. A +re-drive reuses the original delivery id, so a receiver keyed on +`Idempotency-Key` sees a duplicate rather than a new event. **Protocol v2.** The CLI talks to the daemon over `~/.tlgr/daemon.sock`, mode `srw-------`, with the peer's uid checked on every connection. Four things @@ -780,13 +824,100 @@ follow that are worth knowing as a caller: session goes `needs_login` and answers exit 4. `tlgr daemon stop` drains in-flight work instead of cancelling it. +A tlgr home containing a `.production` marker file is refused by the daemon +and by `daemon start`/`restart`/`install` unless `TLGR_ALLOW_PRODUCTION_HOME=1` +is set. Two processes on one home share session files, and Telegram treats a +second client on one auth key as a compromised session and revokes it. + +### Events and sync + +``` +tlgr events list [--group message|read|presence|...] [--available] [--raw] +→ Page[EventType] — 114 types, every Update* constructor accounted for + +tlgr events get message_new [--json-schema] +→ {"type": "message_new", "group": "message", "box": "pts", + "sources": ["UpdateNewMessage", ...], "payload": {...}, "example": {...}} + +tlgr events replay --since 91820 [--events TYPES] [--chat CHAT] +tlgr events decode [FILE|-] [--push] [--key-env TLGR_PUSH_KEY] + +tlgr sync status [--channels] [--refresh] +→ {"pts": 91824, "qts": 12, "seq": 4410, "behind_seconds": 3, + "channels": [{"chat_id": -100…, "pts": 42, "access_hash_known": true}]} + +tlgr sync catch-up # updates.getDifference — replay what was missed +tlgr sync difference [--chat CHAT] [--follow 30] # diagnostics, read-only +tlgr sync reset # give up on the gap and re-baseline +tlgr sync backfill CHAT --from-id 91800 --to-id 91900 +``` + +`sync catch-up` **replays** a gap; `sync reset` **gives up on** one — +everything before the new baseline is marked seen and is not recoverable. They +are not interchangeable, and neither is `chat catchup`, which is the unread +digest a human reads. + +`sync status --channels` reports `access_hash_known`. A channel without an +access hash in the session is *skipped* by catch-up — Telethon will not call +`getChannelDifference` without one — so it looks idle rather than broken. + +### Network and proxies + +``` +tlgr net status [--no-ping] +→ {"connected": true, "phase": "online", "dc_id": 4, "transport": "...", + "ping_ms": 41.2, "layer": 227, "time_offset_seconds": 0} + +tlgr net ping [--probes 3] [--via nearest-dc|get-state] +tlgr net dc list [--ipv6] [--media-only] [--cdn] [--test] +tlgr net dc nearest +tlgr net usage get + +tlgr proxy add 'tg://proxy?server=…&port=…&secret=…' [--set] +tlgr proxy list | set | remove | test [--every] | link +``` + +`time_offset_seconds` is worth reading when requests fail for no visible +reason: MTProto derives `msg_id` from the local clock, and the server drops +anything outside its window without an error the client can see. A drift over +30 seconds is reported as a warning. + +Proxy credentials live in `~/.tlgr/proxies.json` (mode 0600) and are never +printed by `proxy list`; `proxy link` is the one command that emits them and +says so. + ### Streaming ``` -tlgr watch [--chat CHAT1 --chat CHAT2] -→ newline-delimited JSON to stdout, one event per line +tlgr watch [--events TYPES] [--exclude TYPES] [--chat CHAT] [--sender USER] + [--since SEQ] [--no-follow] [--account all] [--print-cursor] +→ newline-delimited JSON to stdout, one frame per line +``` + +Push-driven from the daemon's event bus — nothing is polled. `--events` +accepts an event type, a group name (`message`, `read`, `presence`, `peer`, +`member`, `dialog`, `story`, `collection`, `call`, `bot`, `stars`, `secret`, +`account`, `sync`), a `raw:UpdateFoo` constructor name, `all`, or v1's names +(`new_message`, `chat_action`, `message_read`, …). An unknown selector is a +usage error, never an empty watch. Run `tlgr events list` for the vocabulary. + +Each event frame is the envelope: + +```json +{"seq": 91824, "ts": "2026-09-03T09:14:07Z", "account": "main", + "type": "message_new", "payload": {...}, "chat_id": -1001234567890, + "sender_id": 4242, "self_origin": false} ``` +Control frames share the stream and are distinguishable by `type`: `meta` +first, `end` last, and `heartbeat`, `gap` and `lag` in between. A `gap` frame +means the replay window has passed and events were lost — a number, not +silence. `--results-only` prints v1's line shape +(`{event_type, chat_id, data}`) and drops the control frames. + +`--since ` replays the daemon's ring buffer first; `seq` is per account, +monotonic and persisted, so it survives a daemon restart. + ## Error Response Shape ```json diff --git a/CHANGELOG.md b/CHANGELOG.md index c88ccbf..5ce68f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,13 +32,19 @@ paths and their `dl`/`up` shortcuts still work; `tlgr/cli/legacy/media.py`, the `/media/*` IPC routes and the two `ClientWrapper` methods behind them are deleted rather than shadowed. +The update transport follows: `events`, `watch`, `daemon`, `sync`, `net`, +`proxy`, `config`, `job`, `webhook`, `export` and `agent` — 66 more +operations, 114 event types, and the `updates_sync_network` domain fully +accounted for. + ### Breaking Every change below applies **only to commands generated from the operation registry** — in this release that is the `message`, `draft`, `chat`, -`folder`, `auth`, `account`, `passport`, `media`, `sticker`, `gif` and -`emoji` groups, `tlgr completion`, `tlgr agent exit-codes`, -`tlgr agent whoami`, `tlgr agent parity` and `tlgr schema`. Commands still +`folder`, `auth`, `account`, `passport`, `media`, `sticker`, `gif`, `emoji`, +`events`, `watch`, `daemon`, `sync`, `net`, `proxy`, `config`, `job`, +`webhook` and `export` groups, `tlgr completion`, `tlgr status`, +`tlgr schema` and the `agent` group. Commands still hand-written under `tlgr/cli/legacy/` behave exactly as they did in v1 until their own migration PR, at which point these rules apply to them too. @@ -67,6 +73,17 @@ thirteen changes in the table below, which is the whole list. | 8 | `media.download` | `{path, msg_id}` for one file | `Page[Downloaded]`: `{items: [{msg_id, path, bytes, kind, …}], has_more}` | both v1 keys survive on every item; one invocation can now name several ids, an album or `--all`. `--results-only \| jq -r '.items[0].path'` is the one-file case | | 9 | `media.upload` | `{id, chat_id}` | `{chat_id, msg_id, msg_ids, kind, …}` | rename only: `id` became `msg_id`, beside `msg_ids` for an album. `--select msg_id` reaches it | +Five more shapes changed in the update-transport groups, all of them list or +status envelopes: + +| # | Change | v1 | v2 | Migration | +|---|---|---|---|---| +| 8 | `events.watch` | one line per new message, `{event_type, chat_id, data}` | the full envelope: `{seq, ts, account, type, payload, chat_id, sender_id, self_origin}`, plus `meta`/`end`/`heartbeat`/`gap`/`lag` control frames | `--results-only` prints v1's exact line shape and drops the control frames | +| 9 | `daemon.status` | `{running, pid, uptime_seconds, accounts, connections, disconnected, healthy, jobs}` | the same keys plus `ready`, `version`, `protocol`, `layer`, and a per-account state machine under `accounts` | `accounts` is a list of objects rather than of aliases; `connections` and `disconnected` are unchanged, and `--select connections` reaches the old shape | +| 10 | `job.list` | `{jobs: [...]}` | `Page[JobState]` | `--results-only` yields `{items, has_more, next_cursor, total}` | +| 11 | `config.list` | the raw TOML document | `Page[ConfigEntry]`, one row per key with `value`, `default` and `source`; secrets redacted | `--defaults` includes keys still at their default; `config get ` is the point lookup | +| 12 | `config.keys` | `{keys: {name: {section, key, description}}}` | `Page[ConfigKey]` with `type`, `default`, `scope`, `requires_restart` and `help` | key names gained a section prefix (`idle_timeout` → `daemon.idle_timeout`); both spellings are accepted by `config get`/`set`/`unset` | + `tlgr agent whoami --json` reports `output_schema_version: 2`, so an agent can branch on the two sets without probing for each change. @@ -121,6 +138,77 @@ Two more, outside the documented output shapes: - **`location preview`** renders a map thumbnail from the webfile data centre, and **`poll stats get`** follows the `STATS_MIGRATE` redirect to the stats DC — both through a borrowed sender, as Telethon's own `get_stats` does. +- **The `events`, `watch`, `daemon`, `sync`, `net`, `proxy`, `config`, `job`, + `webhook`, `export` and `agent` groups, generated from the registry.** 66 + operations. `tlgr/cli/legacy/watch.py`, `daemon_cmd.py`, `job.py`, + `config_cmd.py` and `agent.py` are deleted, along with the v1 `/daemon/*` + and `/job/*` IPC routes. +- **The complete event taxonomy.** 114 types covering every one of the 163 + `Update*` constructors Telethon 1.44 can parse, plus the five Telegram has + added since; four containers are listed internal with a reason, and a test + checks the table against the installed Telethon so an upgrade that adds a + constructor fails in the run that upgrades it. `tlgr events list` prints it, + `tlgr events get ` prints one row with its payload and sequence box, + and `docs/design/EVENTS.md` is the prose form. +- **`tlgr watch`, push-driven.** The daemon holds one `events.Raw` handler per + account and a watcher is a bounded queue on the bus. Select by type, group, + `raw:Constructor` or `all`; `--since ` replays the ring buffer with a + `gap` frame when it cannot reach that far back; `--exclude`, `--chat`, + `--sender`, `--topic`, `--account all`, heartbeats, `lag` frames and + `--print-cursor`. +- **`tlgr events replay`, `events decode`.** Replay a buffered range without + following it, or decode a raw TL update — or an encrypted push payload, + which `events decode --push` decrypts and classifies (`SESSION_REVOKE` + means this session was terminated). +- **`tlgr sync status | catch-up | difference | reset | backfill`.** The + update transport, made inspectable: pts/qts/seq, the per-channel table with + `access_hash_known` (a channel without one is skipped by catch-up and looks + idle rather than broken), an explicitly-run `getDifference` that does *not* + advance the stored pts unless asked, a re-baseline for a corrupted state, + and id-range backfill for a box that overflowed. +- **`tlgr net status | ping | dc list | dc nearest | usage get`.** Which DC, + which transport, which proxy, and how far this host's clock has drifted — a + drift over 30 s is warned about, because MTProto derives `msg_id` from local + time and the server drops anything outside its window with no error. +- **`tlgr proxy add | list | set | remove | test | link`.** SOCKS5, HTTP and + MTProxy, `tg://proxy` and `t.me/proxy` links both parsed, credentials in a + 0600 store and never in argv. `proxy test` probes through a throwaway + in-memory session so it cannot become the account's update connection. +- **`tlgr config keys | list | get | set | unset | validate`** over a + machine-readable catalogue of 34 documented keys with types, defaults and + restart requirements, and **`config server get`, `config app get`, + `config info get`, `config country list`, `config promo get`** for + Telegram's own configuration. `config app get --frozen` surfaces the freeze + fields that turn a bare `FROZEN_METHOD_INVALID` into an appeal link. The + server's suggestion list stayed where the account group put it, as + `account suggestion list`; PR-4 added `--chat` to it for the per-chat + nudges. +- **`tlgr daemon status | reconnect | save-state | flood list | flood clear | + dead-letter list | send | delete`.** `running`, `ready` and `healthy` are + three answers to three questions; the flood store Telethon forgets on exit + is listable and clearable; the dead-letter file is drainable. +- **`tlgr job add` without an editor.** Flags, `--from-file -`, or `--edit` + for the v1 behaviour, and **`job test`**, which names every filter node and + why it passed or rejected — the missing piece when a rule silently never + fires. +- **`tlgr export start | status | end | message download | account download`.** + Takeout as the mode it is: the session wraps every later call in + `invokeWithTakeout`, and `TAKEOUT_INIT_DELAY` is reported as + `RATE_LIMITED` with `retry_after` rather than slept through. +- **`tlgr agent capabilities`** separates what this build *cannot* do (the + layer gap) from what this account may not reach (premium, bot, admin) from + what tlgr *will not* do (fake a read receipt, suppress typing status, + misrepresent presence, pass an integrity attestation, execute a payment) — + with the reason for each. +- **`tlgr schema events | config | errors | exit-codes | all`**, and + `tlgr agent exit-codes --errors`, which prints the RPC-error taxonomy with + the field a regex error captures (`FLOOD_WAIT_42` is a wait of 42 seconds, + not a distinct error). +- **A production-home guard.** A tlgr home carrying a `.production` marker is + refused unless `TLGR_ALLOW_PRODUCTION_HOME=1`: two processes on one home + share session files, and Telegram revokes an auth key it sees two clients + on, so a development build pointed at a live home breaks it rather than + degrading it. - **The `message` group and `draft`, generated from the registry.** 43 operations, 77 command paths, 30 aliases: alongside v1's ten `message` @@ -303,6 +391,25 @@ Two more, outside the documented output shapes: - A test that forgot the `tlgr_home` fixture operated on the developer's real `~/.tlgr`. `tests/conftest.py` now points `TLGR_HOME` at a throwaway directory whenever it is unset. +- **`tlgr watch` no longer polls.** v1 asked the daemon for `chat list` every + two seconds and then `message list` per chat — thirty round trips a minute + whether or not anything happened — and could only ever report new messages. + An edit, a deletion, a read receipt, a reaction and every service message + were invisible. +- **`tlgr daemon status` distinguishes alive from working.** v1 reported the + clients the daemon *held*, so a client whose connection had died was still + listed and the daemon still called itself healthy (COR-13, COR-37). +- **A config key with a typo is an error, not a default.** v1 read the file + with `raw.get(key, default)` at every call site, so a misspelled key or a + wrong type silently did nothing. +- **An unknown event name is refused where it is written.** `jobs.yaml` and + `webhook.toml` dropped a name they did not recognise, so a typo produced a + job or a webhook that never fired and never said why. +- **An empty paginated result is `[]`.** It was the page object itself, which + reads as one row of metadata. +- **`sync difference` matched the wrong TL class names**, so every reply + looked like a slice and the probe looped. + - `.gitignore`'s blanket `*.yaml` rule was swallowing `.github/workflows` siblings, documentation YAML and test fixtures (PKG-04). - Errors raised anywhere now map to the exit-code table in one place, so an diff --git a/Makefile b/Makefile index b5f2d18..543c073 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ PY ?= .venv/bin/python STRICT := tlgr/models tlgr/ops tlgr/registry.py tlgr/schema.py tlgr/version.py \ tlgr/parity.py \ tlgr/core/errors.py tlgr/core/timefmt.py tlgr/core/pagination.py \ + tlgr/core/eventtypes.py tlgr/core/tl.py tlgr/core/process.py \ + tlgr/core/signing.py \ tlgr/core/config.py tlgr/core/paths.py tlgr/core/peers.py \ tlgr/core/logging.py tlgr/core/identity.py tlgr/core/media.py \ tlgr/transport \ diff --git a/README.md b/README.md index 04592b0..1552a98 100644 --- a/README.md +++ b/README.md @@ -348,12 +348,69 @@ KDF that would re-encrypt it. ### Daemon ```bash -tlgr daemon start # --foreground +tlgr daemon start # --foreground, --catch-up/--no-catch-up tlgr daemon stop # drains in-flight work rather than killing it -tlgr daemon status # running/ready/healthy/disconnected/version/protocol -tlgr daemon logs # --follow +tlgr daemon status # running/ready/healthy, per-account state and lag +tlgr daemon restart # --grace 10s +tlgr daemon reconnect # force a reconnect and a catch-up +tlgr daemon save-state # flush pts/qts and the entity cache now +tlgr daemon logs --follow --level warning +tlgr daemon flood list # rate-limit deadlines this install still owes +tlgr daemon dead-letter list # events no consumer could be given +tlgr daemon install # LaunchAgent on macOS, systemd --user on Linux ``` +### Watching events + +```bash +tlgr watch # v1's default: new messages +tlgr watch --events all --account all # everything, every connected account +tlgr watch --events read,message_reactions --chat @alice +tlgr watch --since 91820 --print-cursor +tlgr events list --group message # the vocabulary --events accepts +tlgr events get message_new # payload, source constructors, sequence box +``` + +Push-driven from the daemon's event bus, not polled: the daemon already holds +the update socket, so a watcher is a bounded queue on it. 114 event types +cover every `Update*` constructor the pinned Telethon can parse — a message +edit, a deletion, a read receipt, a reaction, a typing indicator and every +service message included. v1's `watch` polled `chat list` and `message list` +every two seconds and could only report new messages. + +Frames are NDJSON, one per line: exactly one `meta` first, exactly one `end` +last, and events, `heartbeat`, `gap` and `lag` in between. A `gap` frame says +how many events the replay window lost — a number rather than silence. +`--results-only` prints v1's `{event_type, chat_id, data}` line shape. + +### Update state + +```bash +tlgr sync status --channels # pts/qts/seq, per-channel table, lag +tlgr sync catch-up # replay what was missed while offline +tlgr sync difference --chat @news # run getDifference by hand (read-only) +tlgr sync reset # give up on the gap and re-baseline +tlgr sync backfill @news --from-id 91800 --to-id 91900 +``` + +`catch-up` replays a gap; `reset` gives up on one. Neither is `chat catchup`, +which is the unread digest a human reads. + +### Network and proxies + +```bash +tlgr net status # DC, transport, proxy, latency, clock offset +tlgr net ping --probes 5 +tlgr net dc list --ipv6 +tlgr proxy add 'tg://proxy?server=1.2.3.4&port=443&secret=dd00' --set +tlgr proxy test --every --reorder +``` + +SOCKS5, HTTP and MTProxy; `tg://proxy` and `t.me/proxy` links both parsed. +Credentials live in `~/.tlgr/proxies.json` at mode 0600 and never reach argv +or a listing. `proxy test` probes through a throwaway in-memory session, so a +probe can never become the account's update-receiving connection. + #### Protocol v2 The CLI reaches the daemon over `~/.tlgr/daemon.sock`, created `srw-------` @@ -370,10 +427,14 @@ world-writable with no authentication at all. refuses instead (exit 11). Two `tlgr` commands racing with no daemon running produce exactly one daemon. - **`status` distinguishes alive from working.** `running` is a live process; - `ready` is a daemon that can serve. An account whose connection dropped is - `degraded` and its requests answer exit 8 with a hint instead of - `Cannot send requests while disconnected`; a revoked session is - `needs_login` and answers exit 4. + `ready` is a daemon that can serve; `healthy` is one whose accounts are + actually working. An account whose connection dropped is `degraded` and its + requests answer exit 8 with a hint instead of `Cannot send requests while + disconnected`; a revoked session is `needs_login` and answers exit 4. +- **A live home is protected.** A tlgr home with a `.production` marker file + is refused unless `TLGR_ALLOW_PRODUCTION_HOME=1`: two processes sharing one + home share session files, and Telegram revokes an auth key it sees two + clients on. ### Global Flags @@ -415,35 +476,48 @@ flowchart LR CLI --> TG ``` -Configure in `~/.tlgr/webhook.toml`: +Configure it with `tlgr webhook set`, which validates every event name against +the taxonomy — a name nobody recognises used to be dropped silently, so the +webhook delivered nothing and never said why: -```toml -[webhook] -enabled = true -url = "http://127.0.0.1:18789/hooks/agent" -token = "shared-secret" -events = ["new_message", "message_edited", "message_deleted"] +```bash +tlgr webhook set --url https://example.com/hooks/agent \ + --events message_new,message_edited,message_deleted \ + --secret-env TLGR_WEBHOOK_SECRET --enabled +tlgr webhook get # configuration and delivery health; secrets redacted +tlgr webhook test # one delivery, with the exact headers it sent +``` -[webhook.retry] -enabled = true -max_attempts = 3 -backoff_base = 2 +Or edit `~/.tlgr/webhook.toml` directly. Deliveries carry: -[webhook.filters] -chats = ["@important_channel"] -``` +| Header | Meaning | +|---|---| +| `X-Tlgr-Signature` | `sha256=` — verify over the bytes you received | +| `X-Tlgr-Delivery` | unique per attempt; reused on a re-drive, so it is an idempotency key | +| `X-Tlgr-Seq` | the event's per-account sequence number | +| `X-Tlgr-Event` | the event type | +| `X-Tlgr-Account` | the account alias | -Events arrive as JSON with `Authorization: Bearer `: +Events arrive as `{"event": , "delivery_id": "..."}`: ```json { - "event_type": "new_message", - "timestamp": "2025-03-06T12:00:00Z", - "account": "main", - "data": { "..." } + "event": { + "seq": 91824, + "ts": "2026-09-03T09:14:07Z", + "account": "main", + "type": "message_new", + "payload": { "...": "..." }, + "chat_id": -1001234567890 + }, + "delivery_id": "0f3c…" } ``` +A delivery that fails every attempt is dead-lettered rather than dropped; +`tlgr daemon dead-letter list` shows them and `daemon dead-letter send` +re-drives them. + ## Gateway -- Background Jobs tlgr also ships with a deterministic, always-on Gateway that runs background jobs on your Telegram account. Define declarative pipelines in `~/.tlgr/jobs.yaml` that automatically react to incoming messages -- auto-reply, auto-forward, filter by chat type, time of day, content, and more. diff --git a/docs/design/DECISIONS.md b/docs/design/DECISIONS.md index a8a431c..b4bab02 100644 --- a/docs/design/DECISIONS.md +++ b/docs/design/DECISIONS.md @@ -650,6 +650,151 @@ groups' commands (`profile photo set`, `chat photo set`, `notify`, `settings`). Each of the 22 is waived to the PR that owns the command, rather than implemented here under a `media` noun where nobody would look for it. `media_files` is 84.6 % covered and 100 % accounted. +## 2026-09-03 — the event taxonomy lives in `core/`, not in the bus + +`daemon/events.py` owned the starter vocabulary, which was fine while the bus +was its only reader. PR-4 has three: `ops/events.py` prints it, the bus +normalises against it, and `tools/gen_docs.py` renders it — and none of those +three may import another (§2.2). The table is therefore `core/eventtypes.py`, +a pure data module with no Telethon import, and `daemon/events.py` re-exports +what it needs. `tl_to_builtins` and `peer_marked_id` moved to `core/tl.py` for +the same reason, and `sign_body` to `core/signing.py`. + +## 2026-09-03 — the daemon subscribes with one `events.Raw` handler + +The foundation registered six high-level Telethon builders. They cannot carry +the taxonomy: `events.NewMessage` drops service messages, `ChatAction` models +a subset of the action kinds and silently discards the rest, and neither +carries a topic id. A `watch` built on them can only ever show a subset of +what the GUI shows, which is the opposite of this project's goal. One +`events.Raw` handler plus a constructor table reaches all 163 of them. +`normalise()` still accepts the high-level event objects, because the gateway +and the v1 code path hand them over and dropping that would break them for +nothing. + +## 2026-09-03 — `--events` is validated, and an unknown selector is exit 2 + +`jobs.yaml` filtered its `events:` list against a hard-coded set and dropped +anything unrecognised, so a typo produced a job that never fired and never +said why. Every event selector — in `watch`, `events replay`, `job add`, +`webhook set` and `config validate` — now goes through +`eventtypes.resolve_selectors`, and an unknown one is a `USAGE` error naming +the vocabulary. A watch that silently matches nothing is indistinguishable +from a broken daemon. + +## 2026-09-03 — `OperationSpec` gained `needs_client` + +Reading the event bus, the flood store, the dead-letter file or the job table +are daemon operations that need no Telegram client, and the dispatcher's +`ensure()` would have connected an account to answer a question about the +daemon. `needs_client=False` skips the session acquisition but still attaches +the resolver when the account happens to be connected — so `watch --chat +@alice` resolves a username without dialling Telegram as a side effect. It is +also what lets `--account all` be expressed at all: there is no single session +to acquire. + +## 2026-09-03 — registry lint L16: an alias may not name a command group + +The work list gives `config app get` the alias `config app`, `net usage get` +the alias `net usage`, and `export account download` the alias `export +account`. Each would be placed as a *command* where a *group* already stands, +replacing it and deleting the canonical path with it — the failure this file +already records for `message fact-check`. Rather than refuse them one at a +time again, the registry now refuses the whole class at import. Five aliases +from the work list are dropped: `config app`, `config info`, `config server`, +`config promo`, `net usage`, `export account`, `export messages`. Every one of +them is one word away from a path that works. + +## 2026-09-03 — `events list` gets no `schema events` alias + +Same rule, one level up: `schema` is a bare top-level command (v1's `tlgr +schema`), so an alias at `schema events` would turn it into a group and take +it with it. `tlgr schema events` reaches the same taxonomy as `agent.schema`'s +own positional instead, which is the spelling the work list asked for and +costs nothing. + +## 2026-09-03 — `watch --sender`, not `--since`, for the actor filter + +The work list spells the sender filter `--since ` beside a `--since +` on the same command. Two flags cannot share a name, and a `--since` +that means a sequence number on `events replay` and a user on `watch` would be +worse than either. The actor filter is `--sender`. + +## 2026-09-03 — `sync difference --follow` loops rather than streaming + +The work list marks it a stream. It is not: one difference is one object, and +`--follow` re-runs it while honouring the `timeout` the server returns. A +stream of one-object frames would make the caller reassemble what the command +already knows, and re-invoking a *final* channel difference faster than the +server's own pacing is exactly the polling Telegram asks clients not to do. + +## 2026-09-03 — `GET /v1/events` is a GET-shaped alias of `events.watch` + +The endpoint predates the operation, and §12.4 keeps its query names working +(`types`, `chats`, `timeout`). Rather than two implementations reading the +same bus with two ideas of what a filter means, the route now decodes its +query into the `events.watch` request struct and runs the same implementation +through the same dispatcher. The one deliberate difference is the default: the +endpoint's is `all`, the CLI's is v1's `new_message`. + +## 2026-09-03 — process control moved from `daemon/` to `core/` + +`daemon start` cannot import `tlgr.daemon` (§2.2), and `lifecycle.py`, +`launchd.py` and `systemd.py` never needed the daemon application — they are +pid files, double-forking and service units. They are now `core/process.py`, +`core/launchd.py` and `core/systemd.py`. `daemon start --foreground` spawns +the daemon as a subprocess and waits, rather than running it in the CLI's +process: a daemon sharing this process's file descriptors, signal handlers and +event loop is not the process a supervisor would start later. + +## 2026-09-03 — a `.production` marker refuses a development build + +A worktree build started a daemon against the live `~/.tlgr` and broke a +running install: two processes on one home share session files, and Telegram +treats a second client on one auth key as a compromised session and revokes +it. `TlgrPaths` now refuses a home carrying a `.production` file unless +`TLGR_ALLOW_PRODUCTION_HOME=1`, the daemon turns that refusal into exit 10 +before it takes the lock, and the test suite's autouse fixture points +`TLGR_HOME` at a temp directory so a test that forgot to ask for one cannot +reach the real home either. + +## 2026-09-03 — four models drop `omit_defaults` for one field each + +`Model` omits a field equal to its default, which is right for "not applicable +or not requested" and wrong for a field that *is* the answer: `job enable` +returning nothing where `enabled: true` belongs, a `DifferenceResult` with no +`final`, a `FloodRecord` with no `kind`, a `TakeoutStatus` with no `active`. +Those four are declared without a default so they are always present. The rule +this suggests — a discriminator or a direct answer is never defaulted — is +worth applying to the groups still to come. + +## 2026-09-03 — `net dc list --resolve` reports NOT_SUPPORTED + +The DNS-over-HTTPS config fallback means fetching Telegram's payload, +verifying its RSA signature and feeding the `dcOptions` into the session. +Telethon has none of it and only retries the hard-coded addresses. A +`--resolve` that quietly did nothing would be worse than one that says so, so +the flag exists, is documented, and answers exit 13 with the working +alternative (a configured MTProxy). + +## 2026-09-03 — an encrypted push payload is decrypted, a wrong key is exit 13 + +`events decode --push` implements the MTProto 2.0 derivation and tries both +direction bytes, keeping the one whose recomputed `msg_key` matches. That +check is what makes the failure honest: a wrong key produces "could not be +decrypted" rather than plausible rubbish. tlgr still does not *register* for +push — the daemon holds a socket — so this exists for a phone-relay setup and +for reading `DC_UPDATE`/`SESSION_REVOKE`, the two payloads that are security +events. + +## 2026-09-03 — `job test` reports actions, never runs them + +The work list gives it `--run-actions`. Executing a rule's actions against +real chats is what enabling the job is for; a "test" that sends messages is a +foot-gun with a reassuring name. The flag is accepted and answers with a note +saying so, and `filter_trace` — every filter node, and why it passed or +rejected — is the part that actually diagnoses "the job never fires". + ## 2026-09-04 — the call groups say `media: none` on the wire, not in a footnote tlgr speaks the signalling half of calls and has no tgcalls binding, so diff --git a/docs/design/EVENTS.md b/docs/design/EVENTS.md index 3799707..1d840af 100644 --- a/docs/design/EVENTS.md +++ b/docs/design/EVENTS.md @@ -1,8 +1,7 @@ # tlgr events -**Status:** partial — the envelope and the delivery guarantees are final; the -type vocabulary below is the **starter set** the foundation ships. The full -taxonomy is owned by the updates group and lands in PR-4. +**Status:** final. The envelope, the delivery guarantees and the whole type +vocabulary are settled; new types are additive. **Applies to:** `tlgr` 2.x, protocol 2 **Companion documents:** `ARCHITECTURE.md` §3.7 (envelope), §6.5 (bus). @@ -95,35 +94,291 @@ result reported as a complete one. --- -## 3. The starter taxonomy +## 3. The taxonomy -Nine types, chosen because they are what a watcher, a webhook and a gateway -rule need on day one. Every name is a lowercase `snake_case` noun-verb, and -new types are additive: a consumer must ignore a `type` it does not know. +114 types, drawn from every one of the 163 `Update*` constructors Telethon +1.44 (layer 227) can parse, plus the five Telegram has added since. The rule +the table is written to, and `tests/test_event_taxonomy.py` enforces, is that +**every constructor is either mapped to a type or listed in §3.3 with the +reason it carries no event**. A constructor that was merely missing would be +an update tlgr drops with nobody able to tell — which is precisely what v1's +two-second polling `watch` did to everything that was not a new message. -| Type | When | Payload | +Every name is lowercase `snake_case`, `_` or `_`, and +a consumer must ignore a `type` it does not know. The machine-readable form is +`tlgr events list --json`; `tlgr events get ` prints one row with its +payload schema and an example. + +### 3.1 Selecting types + +`--events` (on `watch`, `events replay`, `job add`, `webhook set`) accepts, in +any comma-separated combination: + +| Form | Means | +|---|---| +| `message_new` | one type | +| `message` | every type in that group (§3.2) | +| `raw:UpdateBotStopped` | the type that constructor maps to | +| `all` | everything | +| `new_message`, `chat_action`, `message_read`, … | v1's names, still accepted (§3.5) | + +An unknown selector is a `USAGE` error, never an empty selection: a `watch` +that silently matches nothing is indistinguishable from a broken daemon. + +### 3.2 The types, by group + +Sources are the TL constructors that produce the type. **Box** is the sequence +the update is ordered by — `pts`, `qts`, `seq`, `channel_pts`, `version` or +`none` — which is what a consumer needs in order to know whether a gap in it +is recoverable (§`sync`) or simply lost. + +#### `message` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `message_available_min` | none | `UpdateChannelAvailableMessages` | A channel's history was cleared below a point | +| `message_deleted` | pts | `UpdateDeleteChannelMessages`, `UpdateDeleteEphemeralMessages`, `UpdateDeleteMessages` | Messages were deleted | +| `message_edited` | pts | `UpdateEditChannelMessage`, `UpdateEditEphemeralMessage`, `UpdateEditMessage` | A message was edited | +| `message_emoji_game` | none | `UpdateEmojiGameInfo` | An emoji game (dice, dart, slot) resolved | +| `message_extended_media` | pts | `UpdateMessageExtendedMedia` | Paid media on a message was unlocked | +| `message_forwards` | channel_pts | `UpdateChannelMessageForwards` | A channel post's forward counter moved | +| `message_geo_live_viewed` | none | `UpdateGeoLiveViewed` | Somebody viewed a live location I am sharing | +| `message_id_assigned` | pts | `UpdateMessageID`, `UpdateShortSentMessage` | An outgoing message got its server id (random_id reconciliation) | +| `message_new` | pts | `UpdateNewChannelMessage`, `UpdateNewEphemeralMessage`, `UpdateNewMessage`, `UpdateShortChatMessage`, `UpdateShortMessage` | A message arrived in any chat the account can see | +| `message_pinned` | pts | `UpdatePinnedChannelMessages`, `UpdatePinnedMessages` | Messages were pinned or unpinned | +| `message_poll` | pts | `UpdateMessagePoll` | A poll's results changed | +| `message_poll_vote` | qts | `UpdateMessagePollVote` | Somebody voted in a poll you can see the votes of | +| `message_reactions` | pts | `UpdateMessageReactions` | Reactions on a message changed | +| `message_scheduled_deleted` | none | `UpdateDeleteScheduledMessages` | A scheduled message fired or was cancelled | +| `message_scheduled_new` | none | `UpdateNewScheduledMessage` | A scheduled message was queued | +| `message_service` | pts | _derived: updateNewMessage / updateNewChannelMessage carrying a messageService_ | A service message: a join, a pin, a title change, a call | +| `message_transcribed` | none | `UpdateTranscribedAudio` | A voice or video note transcription finished | +| `message_views` | channel_pts | `UpdateChannelMessageViews` | A channel post's view counter moved | +| `message_webpage` | pts | `UpdateChannelWebPage`, `UpdateWebPage` | A link preview finished resolving | + +#### `read` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `read_contents` | pts | `UpdateChannelReadMessagesContents`, `UpdateReadMessagesContents` | Media or a mention was marked read (the media_unread flag) | +| `read_discussion` | pts | `UpdateReadChannelDiscussionInbox`, `UpdateReadChannelDiscussionOutbox` | A comment thread's read position moved | +| `read_inbox` | pts | `UpdateReadChannelInbox`, `UpdateReadHistoryInbox` | My read position moved: messages I have now seen | +| `read_monoforum` | pts | `UpdateReadMonoForumInbox`, `UpdateReadMonoForumOutbox` | A direct-messages (monoforum) channel's read position moved | +| `read_outbox` | pts | `UpdateReadChannelOutbox`, `UpdateReadHistoryOutbox` | The other side read my messages | + +#### `presence` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `typing` | none | `UpdateChannelUserTyping`, `UpdateChatUserTyping`, `UpdateEncryptedChatTyping`, `UpdateUserTyping` | Somebody is typing, recording or uploading | +| `user_status` | none | `UpdateUserStatus` | A user came online, or their last-seen changed | + +#### `peer` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `peer_blocked` | none | `UpdatePeerBlocked` | A peer was blocked or unblocked | +| `peer_chat_changed` | none | `UpdateChannel`, `UpdateChat` | A chat or channel record was invalidated (refetch; may mean kicked) | +| `peer_history_ttl` | none | `UpdatePeerHistoryTTL` | A chat's auto-delete timer changed | +| `peer_located` | none | `UpdatePeerLocated` | The people/groups-nearby list changed | +| `peer_notify_settings` | none | `UpdateNotifySettings` | Notification settings changed for a peer or a scope | +| `peer_settings` | none | `UpdatePeerSettings` | A peer's action-bar settings changed (anti-scam hints included) | +| `peer_user_changed` | none | `UpdateUser` | A user record was invalidated and should be refetched | +| `peer_user_emoji_status` | none | `UpdateUserEmojiStatus` | A user's emoji status changed | +| `peer_user_name` | none | `UpdateUserName` | A user changed their name or username | +| `peer_user_phone` | none | `UpdateUserPhone` | A contact's phone number changed | +| `peer_wallpaper` | none | `UpdatePeerWallpaper` | A chat wallpaper changed | + +#### `member` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `member_boost` | none | `UpdateBotChatBoost` | A channel boost was applied (bot-only) | +| `member_channel` | qts | `UpdateChannelParticipant` | A channel or supergroup member or admin changed | +| `member_chat` | version | `UpdateChatParticipant`, `UpdateChatParticipantAdd`, `UpdateChatParticipantAdmin`, `UpdateChatParticipantDelete`, `UpdateChatParticipantRank`, `UpdateChatParticipants` | A basic group's membership or admin list changed | +| `member_default_rights` | version | `UpdateChatDefaultBannedRights` | A group's default permissions changed | +| `member_join_request` | none | `UpdateBotChatInviteRequester`, `UpdatePendingJoinRequests` | A pending join request arrived or was resolved | + +#### `dialog` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `dialog_draft` | none | `UpdateDraftMessage` | A cloud draft was set or cleared | +| `dialog_filters` | none | `UpdateDialogFilter`, `UpdateDialogFilterOrder`, `UpdateDialogFilters` | Chat folders (dialog filters) changed | +| `dialog_folder` | pts | `UpdateFolderPeers` | A chat moved into or out of the Archive | +| `dialog_forum_pinned` | none | `UpdatePinnedForumTopic`, `UpdatePinnedForumTopics` | Forum topics were pinned or reordered | +| `dialog_forum_view` | none | `UpdateChannelViewForumAsMessages` | A forum's display mode was toggled | +| `dialog_monoforum_no_paid` | none | `UpdateMonoForumNoPaidException` | A direct-messages channel's paid-message exception changed | +| `dialog_pinned` | none | `UpdateDialogPinned`, `UpdatePinnedDialogs` | A chat was pinned, unpinned or reordered in the list | +| `dialog_quick_reply` | none | `UpdateDeleteQuickReply`, `UpdateDeleteQuickReplyMessages`, `UpdateNewQuickReply`, `UpdateQuickReplies`, `UpdateQuickReplyMessage` | Business quick-reply shortcuts changed | +| `dialog_saved_pinned` | none | `UpdatePinnedSavedDialogs`, `UpdateSavedDialogPinned` | A Saved Messages sub-dialog was pinned or reordered | +| `dialog_saved_tags` | none | `UpdateSavedReactionTags` | Saved-message reaction tags changed | +| `dialog_unread_mark` | none | `UpdateDialogUnreadMark` | A chat was manually marked unread (or the mark was cleared) | + +#### `story` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `story_id` | none | `UpdateStoryID` | A story you posted got its server id | +| `story_new` | none | `UpdateStory` | A story was posted, edited or deleted | +| `story_reaction` | none | `UpdateNewStoryReaction`, `UpdateSentStoryReaction` | A story was reacted to | +| `story_read` | none | `UpdateReadStories` | Stories were marked read | +| `story_stealth` | none | `UpdateStoriesStealthMode` | Story stealth mode changed | + +#### `collection` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `collection_attach_menu` | none | `UpdateAttachMenuBots` | The attachment-menu bot list changed | +| `collection_emoji_statuses` | none | `UpdateRecentEmojiStatuses` | Recent emoji statuses changed | +| `collection_gifs` | none | `UpdateSavedGifs` | Saved GIFs changed | +| `collection_reactions` | none | `UpdateRecentReactions` | Recent or top reactions changed | +| `collection_ringtones` | none | `UpdateSavedRingtones` | Notification sounds changed | +| `collection_stickers` | none | `UpdateFavedStickers`, `UpdateMoveStickerSetToTop`, `UpdateNewStickerSet`, `UpdateRecentStickers`, `UpdateStickerSets`, `UpdateStickerSetsOrder` | Sticker or custom-emoji sets changed | +| `collection_stickers_read` | none | `UpdateReadFeaturedEmojiStickers`, `UpdateReadFeaturedStickers` | Featured sticker or emoji sets were marked read | +| `collection_themes` | none | `UpdateTheme` | A theme changed | + +#### `call` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `call_group` | none | `UpdateGroupCall`, `UpdateGroupCallConnection` | A group call, video chat or live stream changed | +| `call_group_encrypted` | none | `UpdateGroupCallChainBlocks`, `UpdateGroupCallEncryptedMessage` | Encrypted group-call key material (conference calls) | +| `call_group_message` | none | `UpdateDeleteGroupCallMessages`, `UpdateGroupCallMessage` | A message inside a group call was posted or deleted | +| `call_group_participants` | version | `UpdateGroupCallParticipants` | Group-call participants changed | +| `call_phone` | none | `UpdatePhoneCall` | An incoming or updated 1:1 call (signalling only; tlgr carries no media) | +| `call_signaling` | none | `UpdatePhoneCallSignalingData` | Raw call signalling data | + +#### `bot` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `bot_business_connection` | none | `UpdateBotBusinessConnect`, `UpdateNewBotConnection` | A business connection was created or changed (bot-only) | +| `bot_business_message` | none | `UpdateBotDeleteBusinessMessage`, `UpdateBotEditBusinessMessage`, `UpdateBotNewBusinessMessage` | A message on a connected business account arrived, changed or went (bot-only) | +| `bot_callback_query` | none | `UpdateBotCallbackQuery`, `UpdateBusinessBotCallbackQuery`, `UpdateInlineBotCallbackQuery` | An inline-keyboard button was pressed (bot-only) | +| `bot_commands` | none | `UpdateBotCommands` | A bot's command list changed | +| `bot_ephemeral_callback` | none | `UpdateBotEphemeralCallbackQuery` | A callback button on an ephemeral message was pressed (bot-only, layer 229) | +| `bot_guest_chat_query` | none | `UpdateBotGuestChatQuery` | A guest-mode chat query arrived (bot-only) | +| `bot_inline_query` | none | `UpdateBotInlineQuery`, `UpdateBotInlineSend` | An inline query arrived, or a result was chosen (bot-only) | +| `bot_managed` | none | `UpdateManagedBot` | A bot you manage changed | +| `bot_menu_button` | none | `UpdateBotMenuButton` | A bot's menu button changed | +| `bot_message_reaction` | none | `UpdateBotMessageReaction`, `UpdateBotMessageReactions` | A reaction on a message this bot can see changed (bot-only) | +| `bot_paid_media_purchased` | none | `UpdateBotPurchasedPaidMedia` | A user bought paid media from this bot (bot-only) | +| `bot_precheckout` | none | `UpdateBotPrecheckoutQuery` | A pre-checkout query arrived (bot-only) | +| `bot_shipping` | none | `UpdateBotShippingQuery` | A shipping query arrived (bot-only) | +| `bot_stars_subscription` | none | `UpdateBotStarsSubscription` | A Stars subscription to this bot changed (bot-only, layer 229) | +| `bot_stopped` | qts | `UpdateBotStopped` | A user started or stopped this bot (bot-only) | +| `bot_webhook` | none | `UpdateBotWebhookJSON`, `UpdateBotWebhookJSONQuery` | A bot-webhook JSON passthrough arrived (bot-only) | +| `bot_webview_join_decision` | none | `UpdateJoinChatWebViewDecision` | A join-chat decision was made inside a mini app | +| `bot_webview_result` | none | `UpdateWebViewResultSent` | A mini app sent data back (bot-only) | + +#### `stars` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `stars_balance` | none | `UpdateStarsBalance` | The Telegram Stars balance changed | +| `stars_gift_auction` | none | `UpdateStarGiftAuctionState`, `UpdateStarGiftAuctionUserState`, `UpdateStarGiftCraftFail` | A star-gift auction or craft changed state | +| `stars_paid_reaction_privacy` | none | `UpdatePaidReactionPrivacy` | Paid-reaction privacy changed | +| `stars_revenue` | none | `UpdateStarsRevenueStatus` | Star revenue or withdrawal status changed | + +#### `secret` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `secret_chat` | qts | `UpdateEncryption` | A secret chat was requested, accepted or discarded | +| `secret_message` | qts | `UpdateNewEncryptedMessage` | Encrypted traffic arrived; tlgr acknowledges it but cannot decrypt it | +| `secret_read` | qts | `UpdateEncryptedMessagesRead` | Secret-chat messages were read or expired | + +#### `account` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `account_ai_tones` | none | `UpdateAiComposeTones` | The AI compose tone list changed | +| `account_autosave` | none | `UpdateAutoSaveSettings` | Media auto-save settings changed | +| `account_browser_settings` | none | `UpdateWebBrowserException`, `UpdateWebBrowserSettings` | In-app browser settings or a per-domain exception changed | +| `account_contacts_reset` | none | `UpdateContactsReset` | The contact list was wiped | +| `account_langpack` | none | `UpdateLangPack`, `UpdateLangPackTooLong` | The language pack changed | +| `account_login_token` | none | `UpdateLoginToken` | A QR login token was accepted | +| `account_new_authorization` | none | `UpdateNewAuthorization` | A new login on this account | +| `account_privacy` | none | `UpdatePrivacy` | A privacy rule changed | +| `account_sent_phone_code` | none | `UpdateSentPhoneCode` | A login code was delivered in-app | +| `account_service_notification` | none | `UpdateServiceNotification` | An official service notification (from 777000) | +| `account_session_revoked` | none | _derived: the SESSION_REVOKE push payload, decoded by `tlgr events decode`_ | This session was terminated elsewhere (from a push payload) | +| `account_sms_job` | none | `UpdateSmsJob` | An SMS-relay job arrived (Telegram's peer-to-peer login SMS programme) | + +#### `sync` + +| Type | Box | Source constructors | Meaning | +|---|---|---|---| +| `daemon_health` | none | _derived: the session state machine (ARCHITECTURE §6.2)_ | An account changed state, or the circuit breaker opened | +| `sync_channel_too_long` | none | `UpdateChannelTooLong` | A channel's gap is unrecoverable from its pts; a resync is needed | +| `sync_config` | none | `UpdateConfig` | The server configuration was invalidated; re-read help.getConfig | +| `sync_dc_options` | none | `UpdateDcOptions` | The data-centre address list changed | +| `sync_pts_changed` | none | `UpdatePtsChanged` | The pts sequence was reset; some updates are unrecoverable | + +### 3.3 Constructors that carry no event + +These four are containers or transport signals: they have no payload of their +own, and the thing inside them is normalised in their place. + +| Constructor | Why | +|---|---| +| `UpdateShort` | container: carries exactly one Update, which is normalised in its place | +| `Updates` | container: a batch of updates plus their users/chats arrays | +| `UpdatesCombined` | container: a batch of updates spanning a seq range | +| `UpdatesTooLong` | transport signal: the common box overflowed, handled by the supervisor with updates.getDifference (see `tlgr sync catch-up`) | + +### 3.4 Constructors newer than Telethon 1.44 (layer 227) + +Telegram ships these; this build cannot parse them, so a raw handler +sees only an unknown constructor id. They are listed — rather than +omitted — so `tlgr events list` can answer "exists, unavailable here". + +| Constructor | Would be | Status | |---|---|---| -| `message_new` | a message arrives, in any chat the account can see | the full `Message` model | -| `message_edited` | a message is edited | the full `Message` model, post-edit | -| `message_deleted` | messages are deleted | `{"message_ids": [int, …]}` | -| `message_read` | read receipts move | `{"max_id": int, "outbox": bool}` | -| `chat_action` | a member joins, leaves, is promoted, the title changes… | `{"action": str, "user_id": int?, "user_ids": [int]}` | -| `user_status` | a user's online status changes | `{"user_id": int, "status": str, "online": bool}` | -| `reaction_changed` | reactions on a message change | *(reserved — emitted from PR-4)* | -| `draft_changed` | a draft is set or cleared elsewhere | *(reserved — emitted from PR-4)* | -| `daemon_health` | an account changes state, or the breaker opens | `{"state": str, "reason": str}` | - -`chat_action.action` is currently the Telethon action class name -(`MessageActionChatAddUser`). PR-4 replaces it with a `snake_case` vocabulary; -consumers should treat it as an opaque string until then. +| `UpdateBotEphemeralCallbackQuery` | `bot_ephemeral_callback` | unparseable in this build; listed by `events list` | +| `UpdateBotStarsSubscription` | `bot_stars_subscription` | unparseable in this build; listed by `events list` | +| `UpdateDeleteEphemeralMessages` | `message_deleted` | unparseable in this build; listed by `events list` | +| `UpdateEditEphemeralMessage` | `message_edited` | unparseable in this build; listed by `events list` | +| `UpdateNewEphemeralMessage` | `message_new` | unparseable in this build; listed by `events list` | + +### 3.5 Compatibility names + +v1's `watch --events` and `jobs.yaml` spelled these differently, and the +foundation shipped a nine-name starter set. Both keep working (§12.4); each +expands to one or more v2 types. + +| Legacy name | Expands to | +|---|---| +| `new_message` | `message_new` | +| `message_edit` | `message_edited` | +| `message_read` | `read_inbox`, `read_outbox` | +| `chat_action`, `user_joined` | `message_service`, `member_chat`, `member_channel` | +| `reaction_changed` | `message_reactions` | +| `draft_changed` | `dialog_draft` | + +### 3.6 Payloads + +Fourteen types carry a **modelled** payload: `message_new`, `message_service`, +`message_edited` and `message_scheduled_new` carry the full `Message` model; +`message_deleted`, `message_pinned`, `message_id_assigned`, `read_inbox`, +`read_outbox`, `typing`, `user_status`, `message_reactions`, `dialog_draft` +and `sync_channel_too_long` carry a small, named shape. `tlgr events get` +prints it. + +Every other type carries **the update's own fields, made JSON-safe**: a +`datetime` becomes RFC-3339, `bytes` become hex, and a nested TL object +becomes `{"_": "ClassName", …}` so a consumer can still branch on the +constructor. That conversion is `tl_to_builtins` and it is the COR-07 fix: +v1 encoded a raw `to_dict()` with `json.dumps(default=str)`, so a message with +media could fail to serialise *at delivery time*, far from the cause, and be +counted as a delivery failure rather than as the bug it was. **What is deliberately absent.** tlgr does not invent a type name for an -update it has no taxonomy entry for. An unrecognised `Update*` is dropped -rather than delivered as `unknown`, because a type name that means "we did not -look" is worse than silence: it cannot be filtered on, and it will change -meaning the moment the real type is added. +update it has no taxonomy entry for, and there is no `unknown` type. A name +that means "we did not look" cannot be filtered on and changes meaning the day +the real one is added. ---- ## 4. Self-origin events @@ -169,15 +424,26 @@ first will make an honest signature fail. --- -## 6. Adding a type (for PR-4 and later) - -1. Add the name to `EVENT_TYPES` in `tlgr/daemon/events.py`. Lowercase - `snake_case`, noun then verb. -2. Map the Telethon event or raw `Update*` to it in `normalise()`. The - function must stay pure and Telethon-free: it matches on the qualified - class name so the bus can be unit-tested with a fake event. -3. Build the payload from **models**. If a model does not exist for the shape, - add one to `tlgr/models/`; do not reach for `to_dict()`. -4. Add a row to the table in §3, and a test that the payload round-trips - through `msgspec.json.encode` — which is the check that no `datetime` or - `bytes` slipped in. +## 6. Adding a type + +1. Add an `EventTypeSpec` to `_TYPES` in `tlgr/core/eventtypes.py`: lowercase + `snake_case`, a group from `GROUPS`, a one-line summary, the sequence box, + and the payload fields. The table is in `core/` because `ops/`, `daemon/` + and the doc generator all read it and none may import each other. +2. Point its `Update*` constructor(s) at it in `CONSTRUCTORS`. A constructor + that carries no event goes in `INTERNAL` **with a reason** — + `tests/test_event_taxonomy.py` checks the installed Telethon against both + lists and fails on a constructor that is in neither, so a Telethon upgrade + that adds one fails in the run that upgrades it. +3. If the payload deserves more than the update's own fields, add a branch to + `normalise_update()` in `tlgr/daemon/events.py` and build it from + **models** — never `to_dict()`. Everything else is handled by + `tl_to_builtins`, which is what guarantees no `datetime` and no `bytes` + reach a consumer. +4. Regenerate §3.2 and `docs/reference/events.md`, and add a test that the + payload survives `msgspec.json.encode`. + +The daemon subscribes with a single `events.Raw()` handler, deliberately. +Telethon's high-level builders (`NewMessage`, `ChatAction`, …) drop service +messages, topic ids and every action kind Telethon does not model, so a stream +built on them can only ever show a subset of what the GUI shows. diff --git a/docs/reference/PARITY.md b/docs/reference/PARITY.md index 290429b..6831320 100644 --- a/docs/reference/PARITY.md +++ b/docs/reference/PARITY.md @@ -7,58 +7,58 @@ Coverage against the Telegram feature catalog, computed from the registry: every `covered` is implemented today. `acct%` is covered **plus** waived — an id that belongs to a group a later PR owns, named in `tlgr/data/parity_waivers.toml` with the PR that closes it. Ids whose feasibility is `not-applicable` or `prohibited` are excluded from the denominator once and never counted again. ``` -catalog 2026-09-02 — 289 operations, 429 invocable paths +catalog 2026-09-02 — 355 operations, 532 invocable paths domain covered req % acct% ops -auth_sessions_security 85 89 95.5% 100.0% 43 -bots_inline_payments 13 175 7.4% 100.0% 5 +auth_sessions_security 87 89 97.8% 100.0% 44 +bots_inline_payments 17 175 9.7% 100.0% 7 calls_voicechats 126 133 94.7% 100.0% 49 -contacts_users 17 121 14.0% 100.0% 12 -dialogs_chats 112 146 76.7% 100.0% 54 +contacts_users 18 121 14.9% 100.0% 13 +dialogs_chats 114 146 78.1% 100.0% 55 groups_channels_admin 26 162 16.0% 100.0% 15 media_files 121 143 84.6% 100.0% 60 -messages_core 158 167 94.6% 100.0% 53 -polls_reactions_content 114 174 65.5% 100.0% 56 -profile_settings_privacy 20 178 11.2% 100.0% 19 +messages_core 159 167 95.2% 100.0% 54 +polls_reactions_content 117 174 67.2% 100.0% 57 +profile_settings_privacy 25 178 14.0% 100.0% 24 stories 12 120 10.0% 100.0% 9 -updates_sync_network 4 189 2.1% 100.0% 3 +updates_sync_network 188 189 99.5% 100.0% 67 priority covered req % acct% -P0 95 178 53.4% 100.0% -P1 201 379 53.0% 100.0% -P2 256 610 42.0% 100.0% -P3 256 630 40.6% 100.0% +P0 116 178 65.2% 100.0% +P1 240 379 63.3% 100.0% +P2 320 610 52.5% 100.0% +P3 334 630 53.0% 100.0% -TOTAL 808 1797 45.0% 100.0% +TOTAL 1010 1797 56.2% 100.0% excluded: not-applicable 79, prohibited 40 -uncovered: 989 (989 waived with a PR number) +uncovered: 787 (787 waived with a PR number) ``` ## By domain | Domain | Covered | Required | % | Accounted % | Ops | |---|---:|---:|---:|---:|---:| -| `auth_sessions_security` | 85 | 89 | 95.5% | 100.0% | 43 | -| `bots_inline_payments` | 13 | 175 | 7.4% | 100.0% | 5 | +| `auth_sessions_security` | 87 | 89 | 97.8% | 100.0% | 44 | +| `bots_inline_payments` | 17 | 175 | 9.7% | 100.0% | 7 | | `calls_voicechats` | 126 | 133 | 94.7% | 100.0% | 49 | -| `contacts_users` | 17 | 121 | 14.0% | 100.0% | 12 | -| `dialogs_chats` | 112 | 146 | 76.7% | 100.0% | 54 | +| `contacts_users` | 18 | 121 | 14.9% | 100.0% | 13 | +| `dialogs_chats` | 114 | 146 | 78.1% | 100.0% | 55 | | `groups_channels_admin` | 26 | 162 | 16.0% | 100.0% | 15 | | `media_files` | 121 | 143 | 84.6% | 100.0% | 60 | -| `messages_core` | 158 | 167 | 94.6% | 100.0% | 53 | -| `polls_reactions_content` | 114 | 174 | 65.5% | 100.0% | 56 | -| `profile_settings_privacy` | 20 | 178 | 11.2% | 100.0% | 19 | +| `messages_core` | 159 | 167 | 95.2% | 100.0% | 54 | +| `polls_reactions_content` | 117 | 174 | 67.2% | 100.0% | 57 | +| `profile_settings_privacy` | 25 | 178 | 14.0% | 100.0% | 24 | | `stories` | 12 | 120 | 10.0% | 100.0% | 9 | -| `updates_sync_network` | 4 | 189 | 2.1% | 100.0% | 3 | +| `updates_sync_network` | 188 | 189 | 99.5% | 100.0% | 67 | ## By priority | Priority | Covered | Required | % | Accounted % | |---|---:|---:|---:|---:| -| P0 | 95 | 178 | 53.4% | 100.0% | -| P1 | 201 | 379 | 53.0% | 100.0% | -| P2 | 256 | 610 | 42.0% | 100.0% | -| P3 | 256 | 630 | 40.6% | 100.0% | +| P0 | 116 | 178 | 65.2% | 100.0% | +| P1 | 240 | 379 | 63.3% | 100.0% | +| P2 | 320 | 610 | 52.5% | 100.0% | +| P3 | 334 | 630 | 53.0% | 100.0% | ## Partial coverage @@ -110,7 +110,6 @@ Domains no PR has reached yet are waived wholesale and not listed here. These ar | `dialogs.block-stories` | P1 | Hide my stories from a user (story blocklist) | waived until PR-5: The story blocklist is a privacy surface on the user group (PR-5). | | `dialogs.dialog-exists` | P1 | Does a dialog with this peer exist | waived until PR-5: `user dialog-status` answers this and migrates with the user group (PR-5). | | `dialogs.notify-exceptions` | P1 | List notification exceptions | waived until PR-12: The exceptions *list* is `notify exceptions` (PR-12); one chat's exception is `chat notify`. | -| `messages-core.message-watch-events` | P1 | Live stream of new / edited / deleted messages and read receipts | waived until PR-4: The live message stream is the event bus surface (PR-4). | | `profile.photos-list-history` | P1 | View own / another user's profile photo history | waived until PR-12: Profile photo history is the `profile` group (PR-12). | | `stars.balance` | P1 | Telegram Stars balance | waived until PR-12: the Star balance and top-up packages are the `stars` surface (PR-12). | | `attach.menu-bots` | P2 | Attachment-menu / side-menu mini-app bots: list, info, add, remove | waived until PR-10: Attachment-menu bots are the `bot` group (PR-10). | @@ -127,8 +126,6 @@ Domains no PR has reached yet are waived wholesale and not listed here. These ar | `dialogs.hide-stories-peer` | P2 | Hide a peer's stories from the strip | waived until PR-8: Hiding a peer's stories is the story strip (PR-8). | | `dialogs.notify-scope-defaults` | P2 | Default notification settings per chat type | waived until PR-12: Scope-wide defaults are `notify set` (PR-12). | | `dialogs.presence-watch` | P2 | Peer online status / last seen | waived until PR-4: Online/last-seen is an update stream (PR-4). | -| `dialogs.typing-watch` | P2 | See who is typing | waived until PR-4: Watching who is typing is an update stream (PR-4); sending one is `chat typing`. | -| `dialogs.watch-dialog-events` | P2 | Live dialog-level events | waived until PR-4: Live dialog events are the event bus (PR-4). | | `emoji.status-set` | P2 | Set / clear own emoji status (custom emoji or collectible gift), with expiry | waived until PR-12: Setting your own emoji status is `profile status set` (PR-12). | | `gift.catalog` | P2 | Browse available gifts | waived until PR-12: gifts and collectibles are the `gift` surface (PR-12). | | `gift.convert-to-stars` | P2 | Convert a gift back into Stars | waived until PR-12: gifts and collectibles are the `gift` surface (PR-12). | @@ -163,9 +160,7 @@ Domains no PR has reached yet are waived wholesale and not listed here. These ar | `auction.active-list` | P3 | Auctions I am bidding in | waived until PR-12: collectible-gift auctions are the `gift` surface (PR-12). | | `auction.position-estimate` | P3 | My position in the auction | waived until PR-12: collectible-gift auctions are the `gift` surface (PR-12). | | `auction.state` | P3 | Auction state and bid ladder | waived until PR-12: collectible-gift auctions are the `gift` surface (PR-12). | -| `auth.countries-list` | P3 | Country code list and phone number patterns | waived until PR-4: help.getCountriesList is config/help plumbing shared with phone formatting; it lands with `config` in PR-4. | | `auth.oauth-deep-link` | P3 | Authorize an OAuth login request from a website/app (tg://oauth deep link) | waived until PR-10: A tg://oauth request is a bot authorization flow (messages.requestUrlAuth); it lands with the bots group in PR-10. | -| `auth.prelogin-language` | P3 | Suggested interface language on the login screen | waived until PR-4: The suggested login-screen language comes from the language pack, which is the `config`/langpack surface in PR-4. | | `bot.media-previews` | P3 | Manage a bot's Mini App media previews (owned bots) | waived until PR-10: A bot's Mini App previews are the `bot` group (PR-10). | | `bot.profile-photo-set` | P3 | Set profile photo of an owned bot | waived until PR-10: Setting an owned bot's photo is the `bot` group (PR-10). | | `calls.reset-top-caller` | P3 | Remove a peer from call suggestions | waived until PR-5: contacts.resetTopPeerRating is the same surface as top-callers (PR-5). | @@ -214,14 +209,11 @@ Domains no PR has reached yet are waived wholesale and not listed here. These ar | `gift.upgrade-preview` | P3 | Preview a gift upgrade | waived until PR-12: gifts and collectibles are the `gift` surface (PR-12). | | `giveaway.boosts-unrestrict` | P3 | Let boosters bypass group restrictions | waived until PR-7: giveaways and channel boosts are the `giveaway`/`boost` surface (PR-7). | | `giveaway.list-prepaid` | P3 | Prepaid giveaways on a channel | waived until PR-12: giveaways and channel boosts are the `giveaway`/`boost` surface (PR-12). | -| `giveaway.prize-stars` | P3 | Receive a Stars prize | waived until PR-4: a Stars prize arrives as an update (PR-4). | | `giveaway.results` | P3 | Giveaway results message | waived until PR-12: giveaways and channel boosts are the `giveaway`/`boost` surface (PR-12). | | `giveaway.user-boosts` | P3 | Boosts a specific user gave a channel | waived until PR-7: giveaways and channel boosts are the `giveaway`/`boost` surface (PR-7). | | `location.business-address` | P3 | Business account location | waived until PR-12: a business account's address is the `business` surface (PR-12). | | `location.channel-geo` | P3 | Set a location for a geo-group | waived until PR-7: a geo-group's location is set through the channel admin surface (PR-7). | | `location.geogroup-create` | P3 | Create a location-based group | waived until PR-7: creating a location-based group is `chat create` (PR-7). | -| `location.proximity-alert-event` | P3 | Proximity reached notification | waived until PR-4: a proximity alert arrives as an update, not a command (PR-4). | -| `location.viewed-receipt` | P3 | Live location viewed receipt | waived until PR-4: a live-location view receipt arrives as an update (PR-4). | | `messages-core.chat-welcome-messages` | P3 | Chat welcome messages (empty-chat cards) | waived until PR-3: Empty-chat welcome cards are a chat setting (PR-3). | | `messages-core.paid-messages-group-price` | P3 | Charge Stars per message in a supergroup / channel direct messages | waived until PR-7: The per-group Star price is a supergroup setting (PR-7). | | `messages-core.search-hashtag-stories` | P3 | Hashtag / location search in public stories | waived until PR-8: Hashtag search over public stories is the story surface (PR-8). | @@ -234,3 +226,4 @@ Domains no PR has reached yet are waived wholesale and not listed here. These ar | `ringtone.set-for-chat` | P3 | Set notification sound for a chat or chat category | waived until PR-12: Per-chat notification sounds are the `notify` group (PR-12). | | `stars.topup-options` | P3 | Star purchase packages | waived until PR-12: the Star balance and top-up packages are the `stars` surface (PR-12). | | `theme.cloud-themes` | P3 | Cloud themes (list, install, create, update, upload theme file) | waived until PR-12: Cloud themes are the `settings` group (PR-12). | +| `updates.invoke-business-connection` | P3 | Act on behalf of a connected business account | waived until PR-12: Acting on behalf of a connected business account is the business surface (PR-12). | diff --git a/docs/reference/README.md b/docs/reference/README.md index 154f73c..27d7626 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -2,30 +2,39 @@ # Command reference -289 operations across 20 groups, generated from the operation registry. Groups still served by v1's hand-written commands are not listed here; they arrive with their own PR. +355 operations across 29 groups, generated from the operation registry. Groups still served by v1's hand-written commands are not listed here; they arrive with their own PR. | Group | Operations | Reference | |---|---:|---| | `account` | 35 | [account.md](account.md) | -| `agent` | 5 | [agent.md](agent.md) | +| `agent` | 7 | [agent.md](agent.md) | | `auth` | 11 | [auth.md](auth.md) | | `call` | 13 | [call.md](call.md) | | `chat` | 34 | [chat.md](chat.md) | | `conference` | 9 | [conference.md](conference.md) | +| `config` | 13 | [config.md](config.md) | +| `daemon` | 14 | [daemon.md](daemon.md) | | `draft` | 3 | [draft.md](draft.md) | | `emoji` | 3 | [emoji.md](emoji.md) | +| `events` | 5 | [events.md](events.md) | +| `export` | 5 | [export.md](export.md) | | `folder` | 13 | [folder.md](folder.md) | | `gif` | 5 | [gif.md](gif.md) | +| `job` | 8 | [job.md](job.md) | | `location` | 9 | [location.md](location.md) | | `media` | 28 | [media.md](media.md) | | `message` | 39 | [message.md](message.md) | +| `net` | 5 | [net.md](net.md) | | `passport` | 5 | [passport.md](passport.md) | | `poll` | 9 | [poll.md](poll.md) | +| `proxy` | 6 | [proxy.md](proxy.md) | | `reaction` | 17 | [reaction.md](reaction.md) | | `search` | 3 | [search.md](search.md) | | `sticker` | 20 | [sticker.md](sticker.md) | +| `sync` | 5 | [sync.md](sync.md) | | `todo` | 5 | [todo.md](todo.md) | | `vc` | 23 | [vc.md](vc.md) | +| `webhook` | 3 | [webhook.md](webhook.md) | - [PARITY.md](PARITY.md) — coverage against the Telegram feature catalog. - `tlgr schema --json` — the same information as JSON Schema draft 2020-12. diff --git a/docs/reference/account.md b/docs/reference/account.md index afd59c2..274ae49 100644 --- a/docs/reference/account.md +++ b/docs/reference/account.md @@ -835,6 +835,7 @@ tlgr account suggestion list [OPTIONS] | Flag | Type | Default | Meaning | |---|---|---|---| +| `--chat` | chat | | Dismiss a per-chat suggestion. | | `--dismiss` | text | | Dismiss this suggestion. | | `--hide-promo` | chat | | Hide the promoted dialog. | @@ -844,9 +845,9 @@ Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked i $ tlgr account suggestion list --json ``` -
Catalog coverage (2 full, 1 partial) +
Catalog coverage (3 full, 1 partial) -Full: `account.promo-data`, `auth.security-suggestions` +Full: `account.promo-data`, `auth.security-suggestions`, `updates.config-suggestions` Partial: `password.check-remembered` diff --git a/docs/reference/agent.md b/docs/reference/agent.md index d39bdd1..4883c2a 100644 --- a/docs/reference/agent.md +++ b/docs/reference/agent.md @@ -2,15 +2,47 @@ # `tlgr agent` -5 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. +7 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. | Command | Summary | |---|---| +| [`agent capabilities`](#tlgr-agent-capabilities) | Report what this build can do, cannot do, and will not do | | [`agent completion`](#tlgr-agent-completion) | Print the shell completion script for bash, zsh or fish | -| [`agent exit-codes`](#tlgr-agent-exit-codes) | Print the stable exit codes for automation | +| [`agent exit-codes`](#tlgr-agent-exit-codes) | Print the stable exit codes, and the RPC error to exit-code mapping | | [`agent parity`](#tlgr-agent-parity) | Report feature-parity coverage against the Telegram catalog | -| [`agent schema`](#tlgr-agent-schema) | Print the machine-readable schema of the CLI | -| [`agent whoami`](#tlgr-agent-whoami) | Report the active account, daemon status and environment | +| [`agent schema`](#tlgr-agent-schema) | Print machine-readable schemas: commands, events, config keys, errors | +| [`agent status`](#tlgr-agent-status) | One-screen health summary: account, connection, sync lag, daemon, floods | +| [`agent whoami`](#tlgr-agent-whoami) | Report the active account, daemon health and client identity | + +### `agent capabilities` + +Report what this build can do, cannot do, and will not do. + +Three different things, deliberately separated. `unsupported_*` is what this Telethon layer cannot parse; `premium_gated`/`bot_only`/`admin_only` is what this *account* may not reach; `prohibited` is what tlgr refuses on purpose, with the reason. Only the first is a gap somebody might close. + +``` +tlgr agent capabilities [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `Capabilities`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--section` | protocol|policy|gates|events|limits | | Restrict the report. | + +```console +$ tlgr agent capabilities --section policy --json +``` + +
Catalog coverage (2 full, 4 partial) + +Full: `updates.session-pfs`, `updates.sync-disable-updates` + +Partial: `updates.invoke-with-layer`, `updates.ops-single-updates-consumer`, `updates.presence-read-receipts-policy`, `updates.sync-old-layer-socket-reset` + +states the policy and the layer bound; the switches themselves are `config set`, and recovery is `daemon reconnect`. + +
### `agent completion` @@ -34,9 +66,9 @@ $ tlgr completion bash --json ### `agent exit-codes` -Print the stable exit codes for automation. +Print the stable exit codes, and the RPC error to exit-code mapping. -Every tlgr command exits with one of these codes. They are a compatibility contract: a code never changes meaning. +Every tlgr command exits with one of these codes. They are a compatibility contract: a code never changes meaning. `--errors` adds the row above them — which Telethon exception becomes which code, whether it is retryable, and the parameter a regex error carries (`FLOOD_WAIT_42` is a wait of 42 seconds, not a distinct error). ``` tlgr agent exit-codes [OPTIONS] @@ -44,12 +76,23 @@ tlgr agent exit-codes [OPTIONS] **idempotent (reports `already`) · runs without an account · returns `ExitCodes`** +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--errors` | flag | | Also print the RPC error to exit-code mapping. | +| `--search` | text | | Filter the error table. | + Also invocable as: `tlgr exit-codes` ```console $ tlgr agent exit-codes --json ``` +
Catalog coverage (2 full, 0 partial) + +Full: `updates.net-error-taxonomy`, `updates.net-migrate-errors` + +
+ ### `agent parity` Report feature-parity coverage against the Telegram catalog. @@ -74,9 +117,9 @@ $ tlgr agent parity --json ### `agent schema` -Print the machine-readable schema of the CLI. +Print machine-readable schemas: commands, events, config keys, errors. -One JSON document: the command tree, and for every registered operation its request and response JSON Schema plus a validated example. Draft 2020-12. +One JSON document: the command tree, and for every registered operation its request and response JSON Schema plus a validated example. Draft 2020-12. `tlgr schema events`, `schema config`, `schema errors` and `schema exit-codes` print the other four vocabularies an agent has to know, and `schema all` prints everything. ``` tlgr agent schema [PATH]... [OPTIONS] @@ -86,7 +129,7 @@ tlgr agent schema [PATH]... [OPTIONS] | Argument | Type | Required | Meaning | |---|---|---|---| -| `PATH` | text | any number | Limit the document to one command path, e.g. `schema message send`. | +| `PATH` | text | any number | A schema kind (commands, events, config, errors, exit-codes, all), or a command path to limit the document to, e.g. `schema message send`. | | Flag | Type | Default | Meaning | |---|---|---|---| @@ -95,21 +138,69 @@ tlgr agent schema [PATH]... [OPTIONS] Also invocable as: `tlgr schema` ```console -$ tlgr schema message --json +$ tlgr schema events --json ``` +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-types` + +prints the taxonomy; `events list` is its first-class surface. + +
+ +### `agent status` + +One-screen health summary: account, connection, sync lag, daemon, floods. + +The union of several groups on purpose. A frozen account, an open circuit breaker, an outstanding flood deadline and a daemon that is up but not ready are the states in which every *other* command starts failing, and `--check` turns them into an exit code a monitor can read. + +``` +tlgr agent status [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `HealthSummary`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--check` | flag | | Exit non-zero when anything is unhealthy. | + +Also invocable as: `tlgr status` + +```console +$ tlgr status --check --json +``` + +
Catalog coverage (1 full, 2 partial) + +Full: `updates.invoke-with-layer` + +Partial: `updates.config-account-frozen`, `updates.net-flood-wait` + +surfaces the states; the detail is `config app get --frozen` and `daemon flood list`. + +
+ ### `agent whoami` -Report the active account, daemon status and environment. +Report the active account, daemon health and client identity. -The orientation call. `output_schema_version` is 2 for this build; branch on it rather than probing for a renamed field. +The first call an agent should make. `output_schema_version` says which output contract it is talking to, and `layer` says how far behind Telegram's current schema this build is. ``` tlgr agent whoami [OPTIONS] ``` -**idempotent (reports `already`) · runs without an account · returns `Whoami`** +**idempotent (reports `already`) · runs without an account · returns `WhoAmI`** ```console $ tlgr agent whoami --json ``` + +
Catalog coverage (0 full, 2 partial) + +Partial: `updates.invoke-init-connection`, `updates.invoke-with-layer` + +reports the identity and layer this build declares; setting them is `config set`, and `status` reports the connection. + +
diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 0000000..ba70f1b --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,372 @@ + + +# `tlgr config` + +13 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`config app get`](#tlgr-config-app-get) | Read the client (app) configuration (help.getAppConfig) | +| [`config country list`](#tlgr-config-country-list) | List or look up countries, phone prefixes and number patterns | +| [`config get`](#tlgr-config-get) | Read one local configuration key | +| [`config info get`](#tlgr-config-info-get) | Read one of the server's flat informational endpoints | +| [`config init`](#tlgr-config-init) | Create the default configuration files | +| [`config keys`](#tlgr-config-keys) | List every documented configuration key with its type and default | +| [`config list`](#tlgr-config-list) | Show the effective local configuration | +| [`config path`](#tlgr-config-path) | Print the paths of the configuration, jobs, webhook, session and log files | +| [`config promo get`](#tlgr-config-promo-get) | Show (or hide) the promoted / PSA chat the server pins to the dialog list | +| [`config server get`](#tlgr-config-server-get) | Read the MTProto server configuration (help.getConfig) | +| [`config set`](#tlgr-config-set) | Write one local configuration key | +| [`config unset`](#tlgr-config-unset) | Remove a local configuration key (revert to its default) | +| [`config validate`](#tlgr-config-validate) | Validate the configuration, jobs and webhook files | + +### `config app get` + +Read the client (app) configuration (help.getAppConfig). + +Almost every feature in Telegram has a limit or a kill switch here. `--frozen` prints the account-freeze fields, which are what turn a bare FROZEN_METHOD_INVALID into an actionable message. + +``` +tlgr config app get [KEY] [OPTIONS] +``` + +**idempotent (reports `already`) · returns `AppConfigDoc`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `KEY` | text | no | Dotted key or prefix to filter. | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--frozen` | flag | | Print only the account-freeze fields. | +| `--config` | flag | | Also include help.getConfig. | + +Also invocable as: `tlgr settings app-config` + +```console +$ tlgr config app get --frozen --json +``` + +
Catalog coverage (3 full, 0 partial) + +Full: `account.app-config`, `updates.config-account-frozen`, `updates.config-app` + +
+ +### `config country list` + +List or look up countries, phone prefixes and number patterns. + +`--phone` classifies a number locally, which turns a wasted `auth.sendCode` into an error before it costs the flood budget. The flag emoji and the preferred language are derived client-side: neither has an MTProto counterpart. + +``` +tlgr config country list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · returns `Page[Country]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--code` | text | | One country by ISO code. | +| `--lang` | text | | Language for the localised names. | +| `--phone` | text | | Classify a number: country, prefix, validity. | +| `--search` | text | | Match on country name. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +Also invocable as: `tlgr config countries`, `tlgr auth countries` + +```console +$ tlgr config country list --phone +447700900000 --json +``` + +
Catalog coverage (4 full, 0 partial) + +Full: `auth.countries-list`, `auth.prelogin-language`, `updates.config-countries`, `updates.config-country-lookup` + +
+ +### `config get` + +Read one local configuration key. + +``` +tlgr config get [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `ConfigValue`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `KEY` | text | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--source` | flag | | Also report which file and section the value came from. | + +```console +$ tlgr config get daemon.idle_timeout --json +``` + +### `config info get` + +Read one of the server's flat informational endpoints. + +``` +tlgr config info get [VALUE] [OPTIONS] +``` + +**idempotent (reports `already`) · returns `InfoTopic`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `VALUE` | text | no | The tg:// link for deep-link, the query for emoji-keywords. | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--lang` | text | | Language where the endpoint takes one. | +| `--search` | text | | Filter the returned list. | +| `--topic` | support|invite-text|premium-promo|peer-colors|timezones|languages|cdn|recent-links|emoji-keywords|deep-link | | Which endpoint to read. | + +```console +$ tlgr config info get timezones --json +``` + +
Catalog coverage (9 full, 0 partial) + +Full: `updates.config-cdn`, `updates.config-deep-link-info`, `updates.config-emoji-keywords`, `updates.config-invite-text`, `updates.config-peer-colors`, `updates.config-premium-promo`, `updates.config-recent-me-urls`, `updates.config-support`, `updates.config-timezones` + +
+ +### `config init` + +Create the default configuration files. + +Written 0600 through the one writer that chmods before it renames (SEC-07). + +``` +tlgr config init [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `InitResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--overwrite` | flag | | Replace files that already exist. | + +```console +$ tlgr config init --json +``` + +### `config keys` + +List every documented configuration key with its type and default. + +``` +tlgr config keys [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[ConfigKey]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--search` | text | | Substring match on key or help. | +| `--section` | text | | Only this section. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr config keys --section presence --json +``` + +
Catalog coverage (4 full, 1 partial) + +Full: `updates.invoke-init-params-json`, `updates.net-timeouts-retries`, `updates.presence-read-receipts-policy`, `updates.presence-set-online` + +Partial: `updates.invoke-init-connection` + +documents the knobs; writing one is `config set`. + +
+ +### `config list` + +Show the effective local configuration. + +Secrets are redacted: this is the command people paste into bug reports. + +``` +tlgr config list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[ConfigEntry]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--defaults` | flag | | Include keys still at their default value. | +| `--section` | text | | Only this section. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr config list --defaults --json +``` + +### `config path` + +Print the paths of the configuration, jobs, webhook, session and log files. + +``` +tlgr config path [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `ConfigPaths`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--file` | config|jobs|webhook|sessions|logs|dead-letter|socket|pid|secrets | | Print just one path. | + +```console +$ tlgr config path --file socket --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.session-persistence` + +says where the session lives; persisting it is the session supervisor's. + +
+ +### `config promo get` + +Show (or hide) the promoted / PSA chat the server pins to the dialog list. + +``` +tlgr config promo get [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · returns `PromoData`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--hide` | flag | | Hide the current promo dialog. | + +```console +$ tlgr config promo get --json +``` + +
Catalog coverage (2 full, 0 partial) + +Full: `updates.config-promo-psa`, `updates.invoke-client-proxy-declare` + +
+ +### `config server get` + +Read the MTProto server configuration (help.getConfig). + +``` +tlgr config server get [OPTIONS] +``` + +**idempotent (reports `already`) · returns `ServerConfig`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--dc-options` | flag | | Include the dc_options array. | +| `--key` | text | | Print one field (repeatable). | + +Also invocable as: `tlgr net config` + +```console +$ tlgr config server get --dc-options --json +``` + +
Catalog coverage (2 full, 1 partial) + +Full: `updates.config-mtproto`, `updates.presence-keepalive-period` + +Partial: `updates.config-dc-options` + +reads the config; enumerating the endpoints is `net dc list`. + +
+ +### `config set` + +Write one local configuration key. + +Validated against `config keys`: a wrong type or an unknown key is an error naming it, not a silent fallback to the default. Identity and transport keys only take effect on the next `initConnection`, and the response says so. + +``` +tlgr config set [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `ConfigValue`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `KEY` | text | yes | — | +| `VALUE` | text | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--apply/--no-apply` | flag | `True` | Ask a running daemon to adopt the change now. | + +```console +$ tlgr config set daemon.idle_timeout 0 --json +``` + +
Catalog coverage (8 full, 17 partial) + +Full: `updates.event-report-message-delivery`, `updates.invoke-init-connection`, `updates.net-local-addr`, `updates.net-network-type`, `updates.net-parallel-connections`, `updates.net-proxy-for-calls`, `updates.net-transport-mode`, `updates.sync-dispatch-ordering` + +Partial: `updates.config-dns-fallback`, `updates.invoke-client-proxy-declare`, `updates.invoke-init-params-json`, `updates.net-flood-wait`, `updates.net-ipv6`, `updates.net-proxy-autoswitch`, `updates.net-proxy-system`, `updates.net-test-dc`, `updates.net-timeouts-retries`, `updates.ops-reconnect-health`, `updates.ops-single-updates-consumer`, `updates.presence-keepalive-period`, `updates.presence-set-online`, `updates.sync-catch-up-on-start`, `updates.sync-channel-short-poll`, `updates.sync-disable-updates`, `updates.sync-new-session-triggers-diff` + +sets the switch; the behaviour it selects belongs to the group that implements it (`proxy`, `sync`, `net`, `daemon`). + +
+ +### `config unset` + +Remove a local configuration key (revert to its default). + +``` +tlgr config unset [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `ConfigValue`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `KEY` | text | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--apply/--no-apply` | flag | `True` | Ask a running daemon to adopt the change now. | + +```console +$ tlgr config unset daemon.idle_timeout --json +``` + +### `config validate` + +Validate the configuration, jobs and webhook files. + +Names as well as syntax: a filter, action or event name nobody registered parses fine and then silently never matches. + +``` +tlgr config validate [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `ValidationReport`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--file` | config|jobs|webhook | | Only validate one file. | +| `--strict` | flag | | Treat warnings (unknown keys) as errors. | + +```console +$ tlgr config validate --strict --json +``` diff --git a/docs/reference/daemon.md b/docs/reference/daemon.md new file mode 100644 index 0000000..594e371 --- /dev/null +++ b/docs/reference/daemon.md @@ -0,0 +1,434 @@ + + +# `tlgr daemon` + +14 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`daemon dead-letter delete`](#tlgr-daemon-dead-letter-delete) | Permanently discard dead-lettered events | +| [`daemon dead-letter list`](#tlgr-daemon-dead-letter-list) | List events that could not be delivered | +| [`daemon dead-letter send`](#tlgr-daemon-dead-letter-send) | Re-deliver dead-lettered events | +| [`daemon flood clear`](#tlgr-daemon-flood-clear) | Clear remembered rate-limit deadlines and reset the circuit breaker | +| [`daemon flood list`](#tlgr-daemon-flood-list) | List active rate-limit deadlines | +| [`daemon install`](#tlgr-daemon-install) | Install the daemon as a user service (auto-start, restart on crash) | +| [`daemon logs`](#tlgr-daemon-logs) | View or follow the daemon log | +| [`daemon reconnect`](#tlgr-daemon-reconnect) | Force a reconnect (and catch-up) for one or every account | +| [`daemon restart`](#tlgr-daemon-restart) | Restart the daemon | +| [`daemon save-state`](#tlgr-daemon-save-state) | Flush update state and the entity cache to the session file now | +| [`daemon start`](#tlgr-daemon-start) | Start the update-receiving daemon | +| [`daemon status`](#tlgr-daemon-status) | Show daemon and per-account connection health | +| [`daemon stop`](#tlgr-daemon-stop) | Stop the daemon | +| [`daemon uninstall`](#tlgr-daemon-uninstall) | Remove the daemon service | + +### `daemon dead-letter delete` + +Permanently discard dead-lettered events. + +``` +tlgr daemon dead-letter delete [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · runs without an account · returns `DeadLetterResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--every` | flag | | Discard everything. | +| `--id` | text | | Only these entries (repeatable). | +| `--source` | webhook|job|all | `all` | Restrict by consumer. | +| `--until` | datetime | | Only entries older than this. | + +Also invocable as: `tlgr webhook dead-letter clear`, `tlgr job dead-letter clear` + +```console +$ tlgr daemon dead-letter delete --every --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-webhook-delivery` + +the disposal half; delivery is the webhook pusher's. + +
+ +### `daemon dead-letter list` + +List events that could not be delivered. + +``` +tlgr daemon dead-letter list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[DeadLetter]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--events` | text | | Filter by event type. | +| `--since` | datetime | | Only entries after this time. | +| `--source` | webhook|job|all | `all` | Which consumer failed. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +Also invocable as: `tlgr webhook dead-letter list`, `tlgr job dead-letter list` + +```console +$ tlgr daemon dead-letter list --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-webhook-delivery` + +the failure store; delivery itself is the webhook pusher. + +
+ +### `daemon dead-letter send` + +Re-deliver dead-lettered events. + +``` +tlgr daemon dead-letter send [OPTIONS] +``` + +**mutating · runs without an account · returns `DeadLetterResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--id` | text | | Only these entries (repeatable). | +| `--keep-on-success` | flag | | Do not remove entries that deliver. | +| `--since` | datetime | | Only entries after this. | +| `--source` | webhook|job|all | `all` | Which consumer to re-drive. | +| `--url` | text | | Deliver to this URL instead. | + +Also invocable as: `tlgr webhook dead-letter drain`, `tlgr job dead-letter drain`, `tlgr daemon dead-letter drain` + +```console +$ tlgr daemon dead-letter send --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-webhook-delivery` + +the replay half; the live delivery path is the webhook pusher. + +
+ +### `daemon flood clear` + +Clear remembered rate-limit deadlines and reset the circuit breaker. + +Local memory only. Telegram's own wait is unaffected, so clearing a deadline that has not actually passed simply spends the next request learning that again. + +``` +tlgr daemon flood clear [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · runs without an account · returns `FloodResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--chat` | chat | | Only this peer. | +| `--every` | flag | | Clear every remembered deadline for the account. | +| `--method` | text | | Only this request type. | + +```console +$ tlgr daemon flood clear --every --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.net-flood-wait` + +the reset half; the accounting is `daemon flood list`. + +
+ +### `daemon flood list` + +List active rate-limit deadlines. + +``` +tlgr daemon flood list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[FloodRecord]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--include-expired` | flag | | Also show deadlines that have already passed. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +Also invocable as: `tlgr daemon floods` + +```console +$ tlgr daemon flood list --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.net-flood-wait` + +
+ +### `daemon install` + +Install the daemon as a user service (auto-start, restart on crash). + +macOS gets a LaunchAgent, Linux a systemd **user** unit — user, because the daemon holds session files under $HOME and must run as their owner. + +``` +tlgr daemon install [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `ServiceResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--keep-alive/--no-keep-alive` | flag | `True` | Restart the daemon on crash. | +| `--supervisor` | auto|launchd|systemd | `auto` | Which service manager to install into. | + +```console +$ tlgr daemon install --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.ops-daemon-lifecycle` + +the supervisor half; running the daemon is `daemon start`/`stop`. + +
+ +### `daemon logs` + +View or follow the daemon log. + +Structured lines with secrets redacted: an auth key, an access hash, a proxy secret and a webhook token are never written to the log in the first place. + +``` +tlgr daemon logs [OPTIONS] +``` + +**runs without an account · returns `none`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--follow`, `-f` | flag | | Follow the log as it is written. | +| `--grep` | text | | Substring filter. | +| `--level` | debug|info|warning|error | | Minimum level to show. | +| `--lines` | int | `50` | — | +| `--for-account` | text | | Only lines tagged with this account. | + +```console +$ tlgr daemon logs --lines 100 --level warning --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.ops-daemon-lifecycle` + +the operator's view of the process; the lifecycle is `daemon stop`. + +
+ +### `daemon reconnect` + +Force a reconnect (and catch-up) for one or every account. + +``` +tlgr daemon reconnect [OPTIONS] +``` + +**mutating · runs without an account · returns `ReconnectResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--catch-up/--no-catch-up` | flag | `True` | Fetch the difference after reconnecting. | +| `--reset-proxy` | flag | | Rebuild the client with the currently selected proxy. | + +```console +$ tlgr daemon reconnect --json +``` + +
Catalog coverage (2 full, 1 partial) + +Full: `updates.ops-reconnect-health`, `updates.sync-old-layer-socket-reset` + +Partial: `updates.sync-new-session-triggers-diff` + +the manual recovery; the automatic one runs in the supervisor. + +
+ +### `daemon restart` + +Restart the daemon. + +``` +tlgr daemon restart [OPTIONS] +``` + +**mutating · runs without an account · returns `LifecycleResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--grace` | duration | `10` | Drain period before SIGKILL. | +| `--wait` | duration | `30` | How long to wait for ready. | + +```console +$ tlgr daemon restart --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.ops-daemon-lifecycle` + +a stop and a start; the lifecycle itself is `daemon stop`. + +
+ +### `daemon save-state` + +Flush update state and the entity cache to the session file now. + +Telethon writes the session only on a clean `disconnect()`. A SIGKILL therefore costs the `pts` progress *and* the cached access hashes — and a channel whose access hash is gone is silently skipped by the next catch-up. + +``` +tlgr daemon save-state [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `SaveStateResult`** + +```console +$ tlgr daemon save-state --json +``` + +
Catalog coverage (1 full, 2 partial) + +Full: `updates.sync-peer-cache-from-updates` + +Partial: `updates.session-persistence`, `updates.sync-state-persistence` + +flushes it on demand; the periodic flush is the session supervisor's. + +
+ +### `daemon start` + +Start the update-receiving daemon. + +Waits for an HTTP 200 from `/v1/status`, not for the socket file: the socket exists from `bind()`, before any account has connected (ROB-07). Catch-up is on by default and the idle stop is off, because the two together are what made v1 lose updates silently. + +``` +tlgr daemon start [OPTIONS] +``` + +**mutating · runs without an account · returns `LifecycleResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--catch-up/--no-catch-up` | flag | `True` | Load the persisted pts/qts/seq and fetch the difference before dispatching. | +| `--connect` | text | | Only connect these accounts (repeatable). | +| `--foreground` | flag | | Run in the foreground instead of forking. | +| `--idle-timeout` | duration | | 0 disables the idle stop. | +| `--wait` | duration | `30` | How long to wait for ready. | + +```console +$ tlgr daemon start --json +``` + +
Catalog coverage (3 full, 1 partial) + +Full: `updates.stream-daemon-multi-account`, `updates.sync-catch-up-on-start`, `updates.sync-new-session-triggers-diff` + +Partial: `updates.ops-daemon-lifecycle` + +starting the process; stopping it cleanly is `daemon stop`. + +
+ +### `daemon status` + +Show daemon and per-account connection health. + +`running` is about the process, `ready` about the socket, `healthy` about the accounts. v1 had only the first and reported every client it held as connected, so a fully deaf daemon looked fine (COR-37). + +``` +tlgr daemon status [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `DaemonStatus`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--check` | flag | | Exit 11 when the daemon or any account is unhealthy. | + +```console +$ tlgr daemon status --check --json +``` + +
Catalog coverage (3 full, 5 partial) + +Full: `bots.bot-updates-status`, `updates.ops-single-updates-consumer`, `updates.session-persistence` + +Partial: `updates.config-account-frozen`, `updates.net-connection-status`, `updates.ops-reconnect-health`, `updates.stream-daemon-multi-account`, `updates.sync-updating-indicator` + +reports the state; the network detail is `net status`, the freeze fields are `config app get`, and recovery is `daemon reconnect`. + +
+ +### `daemon stop` + +Stop the daemon. + +Asks it to drain in-flight requests and disconnect cleanly, then falls back to SIGTERM. A killed daemon loses its `pts` and the cached access hashes catch-up needs. + +``` +tlgr daemon stop [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `LifecycleResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--grace` | duration | `10` | Drain period before SIGKILL. | + +```console +$ tlgr daemon stop --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.ops-daemon-lifecycle` + +
+ +### `daemon uninstall` + +Remove the daemon service. + +``` +tlgr daemon uninstall [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · idempotent (reports `already`) · runs without an account · returns `ServiceResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--stop/--no-stop` | flag | `True` | Also stop a running daemon. | + +```console +$ tlgr daemon uninstall --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.ops-daemon-lifecycle` + +the supervisor half; running the daemon is `daemon start`/`stop`. + +
diff --git a/docs/reference/events.md b/docs/reference/events.md new file mode 100644 index 0000000..be41d73 --- /dev/null +++ b/docs/reference/events.md @@ -0,0 +1,202 @@ + + +# `tlgr events` + +5 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`events decode`](#tlgr-events-decode) | Decode a raw TL update or an encrypted push payload into an event | +| [`events get`](#tlgr-events-get) | Show one event type: payload, source constructors, sequence box, example | +| [`events list`](#tlgr-events-list) | List the event types tlgr can emit, with their source constructors | +| [`events replay`](#tlgr-events-replay) | Replay buffered events from the daemon's ring buffer without following | +| [`events watch`](#tlgr-events-watch) | Stream live events from the daemon as newline-delimited JSON | + +### `events decode` + +Decode a raw TL update or an encrypted push payload into an event. + +Offline; no account needed. tlgr does not register for push notifications — the daemon holds a socket — but a phone-relay setup does, and `DC_UPDATE` and `SESSION_REVOKE` are security events: the second one means this session has been terminated. + +``` +tlgr events decode [INPUT] [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `DecodedEvent`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `INPUT` | text | no | A file, or '-' for stdin: a JSON TL object, or a base64 push payload. | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--key` | text | | The base64 push auth key used to decrypt the payload. | +| `--push` | flag | | The input is a Telegram push-notification payload. | +| `--raw` | flag | | Print the decoded TL object instead of the tlgr envelope. | + +```console +$ tlgr events decode - --push --json +``` + +
Catalog coverage (1 full, 1 partial) + +Full: `updates.push-payload-decrypt` + +Partial: `updates.stream-raw-passthrough` + +decodes one update offline; the live passthrough is `watch --raw`. + +
+ +### `events get` + +Show one event type: payload, source constructors, sequence box, example. + +`box` is the field to read first: it says which sequence orders the event, and therefore whether a gap in it is recoverable with `sync difference` or simply lost. + +``` +tlgr events get [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `EventTypeDetail`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `TYPE` | text | yes | An event type name, or a `raw:Constructor`. | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--example/--no-example` | flag | `True` | Include a synthetic example envelope. | +| `--json-schema` | flag | | Emit the payload as a JSON Schema object. | + +```console +$ tlgr events get message_new --json +``` + +
Catalog coverage (39 full, 2 partial) + +Full: `updates.event-attach-menu-bots`, `updates.event-bot-business`, `updates.event-bot-commands`, `updates.event-bot-guest-chat-query`, `updates.event-bot-menu-button`, `updates.event-bot-payments`, `updates.event-bot-stopped`, `updates.event-channel-available-messages`, `updates.event-channel-participant`, `updates.event-chat-boost`, `updates.event-chat-refetch`, `updates.event-contacts-reset`, `updates.event-default-banned-rights`, `updates.event-dialog-pinned`, `updates.event-draft`, `updates.event-encrypted-chats`, `updates.event-extended-media`, `updates.event-geo-live-viewed`, `updates.event-history-ttl`, `updates.event-join-requests`, `updates.event-managed-bot`, `updates.event-message-edited`, `updates.event-new-bot-connection`, `updates.event-notify-settings`, `updates.event-peer-located`, `updates.event-phone-call`, `updates.event-poll`, `updates.event-quick-replies`, `updates.event-read-discussion`, `updates.event-read-outbox`, `updates.event-saved-gifs`, `updates.event-scheduled-new`, `updates.event-service-notification`, `updates.event-stars-revenue`, `updates.event-story-id`, `updates.event-story-read`, `updates.event-user-emoji-status`, `updates.event-user-refetch`, `updates.event-web-browser-settings` + +Partial: `updates.stream-event-types`, `updates.sync-min-constructors` + +documents the type and its payload; receiving one is `watch`, and min-constructor hydration happens on the bus. + +
+ +### `events list` + +List the event types tlgr can emit, with their source constructors. + +114 types covering every `Update*` constructor Telethon can parse, plus the ones Telegram has added since. `--raw` lists the constructors instead. These names are the only values `watch --events`, `job add --events` and `webhook set --events` accept. + +``` +tlgr events list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[EventType]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--available` | flag | | Only types this build can actually receive (hides bot-only and layer-229). | +| `--group` | message|read|presence|peer|member|dialog|story|collection|call|bot|stars|secret|account|sync | | Only this family. | +| `--raw` | flag | | One row per raw TL update constructor instead of per type. | +| `--search` | text | | Substring match on type, constructor or summary. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr events list --group message --json +``` + +
Catalog coverage (40 full, 60 partial) + +Full: `updates.event-ai-compose-tones`, `updates.event-autosave-settings`, `updates.event-bot-callback-query`, `updates.event-bot-ephemeral-callback`, `updates.event-bot-inline-query`, `updates.event-bot-message-reactions`, `updates.event-bot-stars-subscription`, `updates.event-bot-webhook-json`, `updates.event-channel-forwards`, `updates.event-channel-views`, `updates.event-chat-participants`, `updates.event-config-changed`, `updates.event-dc-options`, `updates.event-dialog-filters`, `updates.event-dialog-unread-mark`, `updates.event-emoji-game-info`, `updates.event-ephemeral-messages`, `updates.event-folder-peers`, `updates.event-group-call`, `updates.event-join-chat-webview-decision`, `updates.event-login-token`, `updates.event-message-deleted`, `updates.event-new-authorization`, `updates.event-new-message`, `updates.event-peer-blocked`, `updates.event-peer-wallpaper`, `updates.event-pinned-messages`, `updates.event-pts-changed`, `updates.event-read-contents`, `updates.event-read-monoforum`, `updates.event-saved-dialogs`, `updates.event-scheduled-deleted`, `updates.event-service-message`, `updates.event-stars-balance`, `updates.event-stories-stealth`, `updates.event-story-reaction`, `updates.event-typing`, `updates.event-user-phone`, `updates.event-view-forum-as-messages`, `updates.event-webview-result-sent` + +Partial: `updates.event-attach-menu-bots`, `updates.event-bot-business`, `updates.event-bot-commands`, `updates.event-bot-guest-chat-query`, `updates.event-bot-menu-button`, `updates.event-bot-payments`, `updates.event-bot-stopped`, `updates.event-channel-available-messages`, `updates.event-channel-participant`, `updates.event-chat-boost`, `updates.event-chat-refetch`, `updates.event-contacts-reset`, `updates.event-default-banned-rights`, `updates.event-dialog-pinned`, `updates.event-draft`, `updates.event-encrypted-chats`, `updates.event-extended-media`, `updates.event-geo-live-viewed`, `updates.event-history-ttl`, `updates.event-join-requests`, `updates.event-managed-bot`, `updates.event-message-edited`, `updates.event-message-id-map`, `updates.event-new-bot-connection`, `updates.event-new-channel-message`, `updates.event-notify-settings`, `updates.event-paid-reaction-privacy`, `updates.event-peer-located`, `updates.event-peer-settings`, `updates.event-phone-call`, `updates.event-pinned-forum-topics`, `updates.event-poll`, `updates.event-privacy`, `updates.event-quick-replies`, `updates.event-reactions`, `updates.event-read-discussion`, `updates.event-read-inbox`, `updates.event-read-outbox`, `updates.event-recent-reactions`, `updates.event-report-message-delivery`, `updates.event-saved-gifs`, `updates.event-saved-ringtones`, `updates.event-scheduled-new`, `updates.event-sent-phone-code`, `updates.event-service-notification`, `updates.event-star-gift-auction`, `updates.event-stars-revenue`, `updates.event-stickers-changed`, `updates.event-story-id`, `updates.event-story-new`, `updates.event-story-read`, `updates.event-transcription`, `updates.event-user-emoji-status`, `updates.event-user-name`, `updates.event-user-refetch`, `updates.event-user-status`, `updates.event-web-browser-settings`, `updates.event-webpage`, `updates.stream-event-types`, `updates.stream-raw-passthrough` + +the catalogue half: the type exists and is selectable. Receiving one is `watch`, which owns those ids fully. + +
+ +### `events replay` + +Replay buffered events from the daemon's ring buffer without following. + +Exit 3 when the range is inside the buffer and empty; exit 13, with the oldest seq the daemon still holds, when `--since` predates it. Returning the newest page instead would be a silent lie about having caught up. + +``` +tlgr events replay [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · returns `Page[EventEnvelope]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--chat` | chat | | Only this chat. | +| `--difference` | flag | | Rebuild a range older than the buffer with updates.getDifference. | +| `--events` | text | `all` | Filter the replay. | +| `--exclude` | text | | Subtract these types. | +| `--since` | int | | First seq (exclusive). Default: the whole buffer. | +| `--until` | int | | Stop at this seq (inclusive). | +| `--webhook` | flag | | Re-deliver the range to the configured webhook, not to stdout. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr events replay --since 91820 --events message_new --json +``` + +
Catalog coverage (1 full, 2 partial) + +Full: `updates.stream-resume-cursor` + +Partial: `updates.stream-watch-ndjson`, `updates.sync-duplicate-suppression` + +replays a range; following it live is `watch`, and de-duplication is the consumer's job through the envelope's stable seq. + +
+ +### `events watch` + +Stream live events from the daemon as newline-delimited JSON. + +Push-driven from the daemon's event bus, not polled: v1 asked for `chat list` and then `message list` every two seconds and could only report new messages. Every type in `tlgr events list` is selectable, `--since ` replays the ring buffer first (with a `gap` frame when it cannot reach that far back), and a watcher that falls behind gets a `lag` frame rather than silence. + +``` +tlgr events watch [OPTIONS] +``` + +**returns `none`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--chat` | chat | | Only events about this chat. | +| `--events` | text | `new_message` | Types, groups, `raw:Constructor` names or `all`, comma-separated. See `tlgr events list`. | +| `--exclude` | text | | Subtract these after --events is applied. | +| `--follow/--no-follow` | flag | `True` | Keep streaming after the replay is drained. | +| `--follow-for` | int | `3600` | Close the stream after this. | +| `--heartbeat` | int | `15` | Idle keepalive; 0 disables. | +| `--max-events` | int | | Stop after this many events. | +| `--on-lag` | drop|block|fail | `drop` | Falling behind: drop the oldest and report it, take a much larger queue, or stop with exit 13. | +| `--print-cursor` | flag | | Emit a final frame carrying the resume seq. | +| `--raw` | flag | | Emit only the raw TL update instead of the envelope. | +| `--sender` | user | | Only events from this user. | +| `--since` | int | | Replay from this seq (exclusive) before following. | +| `--topic` | int | | Only events inside this forum topic. | +| `--with-raw` | flag | | Include the raw TL update beside the payload. | + +Also invocable as: `tlgr events tail`, `tlgr watch` + +```console +$ tlgr watch --events message_new,read_inbox --chat @alice --json +``` + +
Catalog coverage (32 full, 90 partial) + +Full: `bots.bot-side-update-stream`, `bots.bot-subscription-update`, `bots.ephemeral-message-view`, `contacts-users.user-status-watch`, `dialogs.typing-watch`, `dialogs.watch-dialog-events`, `giveaway.prize-stars`, `location.proximity-alert-event`, `location.viewed-receipt`, `messages-core.message-watch-events`, `updates.event-message-id-map`, `updates.event-new-channel-message`, `updates.event-paid-reaction-privacy`, `updates.event-peer-settings`, `updates.event-pinned-forum-topics`, `updates.event-privacy`, `updates.event-reactions`, `updates.event-read-inbox`, `updates.event-recent-reactions`, `updates.event-saved-ringtones`, `updates.event-sent-phone-code`, `updates.event-star-gift-auction`, `updates.event-stickers-changed`, `updates.event-story-new`, `updates.event-transcription`, `updates.event-user-name`, `updates.event-user-status`, `updates.event-webpage`, `updates.stream-event-types`, `updates.stream-raw-passthrough`, `updates.stream-watch-ndjson`, `updates.sync-min-constructors` + +Partial: `updates.event-ai-compose-tones`, `updates.event-attach-menu-bots`, `updates.event-autosave-settings`, `updates.event-bot-business`, `updates.event-bot-callback-query`, `updates.event-bot-commands`, `updates.event-bot-ephemeral-callback`, `updates.event-bot-guest-chat-query`, `updates.event-bot-inline-query`, `updates.event-bot-menu-button`, `updates.event-bot-message-reactions`, `updates.event-bot-payments`, `updates.event-bot-stars-subscription`, `updates.event-bot-stopped`, `updates.event-bot-webhook-json`, `updates.event-channel-available-messages`, `updates.event-channel-forwards`, `updates.event-channel-participant`, `updates.event-channel-views`, `updates.event-chat-boost`, `updates.event-chat-participants`, `updates.event-chat-refetch`, `updates.event-config-changed`, `updates.event-contacts-reset`, `updates.event-dc-options`, `updates.event-default-banned-rights`, `updates.event-dialog-filters`, `updates.event-dialog-pinned`, `updates.event-dialog-unread-mark`, `updates.event-draft`, `updates.event-emoji-game-info`, `updates.event-encrypted-chats`, `updates.event-ephemeral-messages`, `updates.event-extended-media`, `updates.event-folder-peers`, `updates.event-geo-live-viewed`, `updates.event-group-call`, `updates.event-history-ttl`, `updates.event-join-chat-webview-decision`, `updates.event-join-requests`, `updates.event-login-token`, `updates.event-managed-bot`, `updates.event-message-deleted`, `updates.event-message-edited`, `updates.event-new-authorization`, `updates.event-new-bot-connection`, `updates.event-new-message`, `updates.event-notify-settings`, `updates.event-peer-blocked`, `updates.event-peer-located`, `updates.event-peer-wallpaper`, `updates.event-phone-call`, `updates.event-pinned-messages`, `updates.event-poll`, `updates.event-pts-changed`, `updates.event-quick-replies`, `updates.event-read-contents`, `updates.event-read-discussion`, `updates.event-read-monoforum`, `updates.event-read-outbox`, `updates.event-report-message-delivery`, `updates.event-saved-dialogs`, `updates.event-saved-gifs`, `updates.event-scheduled-deleted`, `updates.event-scheduled-new`, `updates.event-service-message`, `updates.event-service-notification`, `updates.event-stars-balance`, `updates.event-stars-revenue`, `updates.event-stories-stealth`, `updates.event-story-id`, `updates.event-story-reaction`, `updates.event-story-read`, `updates.event-typing`, `updates.event-user-emoji-status`, `updates.event-user-phone`, `updates.event-user-refetch`, `updates.event-view-forum-as-messages`, `updates.event-web-browser-settings`, `updates.event-webview-result-sent`, `updates.stream-daemon-multi-account`, `updates.stream-event-filtering`, `updates.stream-resume-cursor`, `updates.sync-channel-short-poll`, `updates.sync-difference-too-long`, `updates.sync-dispatch-ordering`, `updates.sync-duplicate-suppression`, `updates.sync-peer-cache-from-updates`, `updates.sync-too-long`, `updates.sync-updating-indicator` + +delivers every type; the catalogue half (what exists, what it means) is `events list`/`events get`, and gap recovery is the `sync` group. + +
diff --git a/docs/reference/export.md b/docs/reference/export.md new file mode 100644 index 0000000..8495a09 --- /dev/null +++ b/docs/reference/export.md @@ -0,0 +1,173 @@ + + +# `tlgr export` + +5 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`export account download`](#tlgr-export-account-download) | Export personal info: profile, photos, sessions, stories, contacts, left channels | +| [`export end`](#tlgr-export-end) | Close the takeout session | +| [`export message download`](#tlgr-export-message-download) | Export chat history inside the takeout session | +| [`export start`](#tlgr-export-start) | Open a Telegram data-export (takeout) session | +| [`export status`](#tlgr-export-status) | Show the active takeout session and its message ranges | + +### `export account download` + +Export personal info: profile, photos, sessions, stories, contacts, left channels. + +Runs inside the takeout session when one is open, and normally when it is not — so the personal-info half works while a message export is still waiting for approval. + +``` +tlgr export account download [OPTIONS] +``` + +**returns `ExportResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--contacts` | flag | | Contacts and top peers. | +| `--everything/--no-everything` | flag | `True` | All of the above. | +| `--left-channels` | flag | | Channels I left. | +| `--out` | path | `./telegram-export` | Output directory. | +| `--photos` | flag | | Profile photos. | +| `--sessions` | flag | | Sessions and websites. | +| `--stories` | flag | | Story archive. | + +```console +$ tlgr export account download --out ./export --json +``` + +
Catalog coverage (3 full, 0 partial) + +Full: `takeout.contacts`, `takeout.personal-info`, `updates.takeout-export-run` + +
+ +### `export end` + +Close the takeout session. + +An open session blocks the next export, so this is not optional. + +``` +tlgr export end [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · returns `ExportResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--failed` | flag | | Report the export as unsuccessful (success=false). | + +Also invocable as: `tlgr export finish`, `tlgr daemon takeout finish` + +```console +$ tlgr export end --json +``` + +
Catalog coverage (0 full, 2 partial) + +Partial: `takeout.messages`, `updates.takeout-session` + +closes the session; the data comes from the download commands. + +
+ +### `export message download` + +Export chat history inside the takeout session. + +One NDJSON file per chat, appended as it goes: a takeout still meets FLOOD_WAIT, and an export that restarts from zero after four hours is one nobody finishes. + +``` +tlgr export message download [OPTIONS] +``` + +**paginated (`HISTORY` cursor) · returns `Page[Message]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--chat` | chat | | Only these chats (repeatable). | +| `--files` | flag | | Also download media. | +| `--out` | path | `./telegram-export` | Output directory. | +| `--per-chat` | int | `1000` | Messages per chat. | +| `--since` | datetime | | Only after this. | +| `--until` | datetime | | Only before this. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr export message download --chat @alice --out ./export --json +``` + +
Catalog coverage (2 full, 0 partial) + +Full: `takeout.files`, `takeout.messages` + +
+ +### `export start` + +Open a Telegram data-export (takeout) session. + +The returned id wraps every subsequent request in `invokeWithTakeout`, `upload.getFile` included. `file_max_size` cannot be changed later. + +``` +tlgr export start [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · returns `TakeoutSession`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--bots` | flag | | Include bot chats. | +| `--channels` | flag | | Include channels. | +| `--chats` | flag | | Include basic groups. | +| `--contacts` | flag | | Include contacts. | +| `--files` | flag | | Include media files. | +| `--max-file-size` | int | `104857600` | file_max_size, declared up front and unchangeable afterwards. | +| `--megagroups` | flag | | Include supergroups. | +| `--messages` | flag | | Include private-chat history. | +| `--users` | flag | | Include private chats. | +| `--wait` | flag | | Report TAKEOUT_INIT_DELAY as a wait instead of failing. | + +Also invocable as: `tlgr daemon takeout start` + +```console +$ tlgr export start --messages --files --json +``` + +
Catalog coverage (0 full, 5 partial) + +Partial: `takeout.contacts`, `takeout.files`, `takeout.messages`, `takeout.personal-info`, `updates.takeout-session` + +opens the session the other export commands run inside; each of them owns the data it fetches. + +
+ +### `export status` + +Show the active takeout session and its message ranges. + +``` +tlgr export status [OPTIONS] +``` + +**idempotent (reports `already`) · returns `TakeoutStatus`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--ranges/--no-ranges` | flag | `True` | Include messages.getSplitRanges output. | + +Also invocable as: `tlgr daemon takeout status` + +```console +$ tlgr export status --json +``` + +
Catalog coverage (2 full, 0 partial) + +Full: `updates.takeout-session`, `updates.takeout-split-ranges` + +
diff --git a/docs/reference/job.md b/docs/reference/job.md new file mode 100644 index 0000000..a76858a --- /dev/null +++ b/docs/reference/job.md @@ -0,0 +1,250 @@ + + +# `tlgr job` + +8 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`job add`](#tlgr-job-add) | Add a gateway job | +| [`job disable`](#tlgr-job-disable) | Disable a job without removing it | +| [`job enable`](#tlgr-job-enable) | Enable a disabled job | +| [`job get`](#tlgr-job-get) | Show one job's resolved pipeline (filters, processors, actions) | +| [`job list`](#tlgr-job-list) | List gateway jobs and their state | +| [`job reload`](#tlgr-job-reload) | Hot-reload jobs.yaml without restarting the daemon | +| [`job remove`](#tlgr-job-remove) | Remove a job | +| [`job test`](#tlgr-job-test) | Dry-run a job's filters against real or synthetic events | + +### `job add` + +Add a gateway job. + +v1 only opened `$EDITOR`, which no agent can drive. The flags are the agent path, `--from-file -` takes YAML or JSON on stdin, and `--edit` keeps the old behaviour. + +``` +tlgr job add [OPTIONS] +``` + +**mutating · runs without an account · returns `Job`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--action` | text | | Action entry, e.g. 'reply:hello' (repeatable). | +| `--edit` | flag | | Open jobs.yaml in $EDITOR instead (the v1 behaviour). | +| `--enabled/--disabled` | flag | `True` | Initial state. | +| `--events` | text | `new_message` | Event types the job subscribes to. | +| `--filter` | text | | Filter entry (repeatable). | +| `--from-file` | text | | Read one job (or a jobs list) from YAML/JSON. | +| `--for-account` | text | | Account the job runs on. | +| `--name` | text | | Job name. | +| `--processor` | text | | Processor entry (repeatable). | + +```console +$ tlgr job add --name archive --action 'forward:to=@archive' --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-filtering` + +writes the rule; the filter vocabulary belongs to the gateway. + +
+ +### `job disable` + +Disable a job without removing it. + +``` +tlgr job disable [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `Job`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `NAME` | text | yes | — | + +```console +$ tlgr job disable archive --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-filtering` + +toggles a rule; the filtering itself is the gateway's. + +
+ +### `job enable` + +Enable a disabled job. + +``` +tlgr job enable [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · runs without an account · returns `Job`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `NAME` | text | yes | — | + +```console +$ tlgr job enable archive --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-filtering` + +toggles a rule; the filtering itself is the gateway's. + +
+ +### `job get` + +Show one job's resolved pipeline (filters, processors, actions). + +``` +tlgr job get [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `JobState`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `NAME` | text | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--explain` | flag | | Annotate each filter with the registry entry it resolves to. | + +```console +$ tlgr job get archive --explain --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-filtering` + +shows the rule; evaluating it against real events is `job test`. + +
+ +### `job list` + +List gateway jobs and their state. + +Configured *and* running are different facts: a job can be enabled in `jobs.yaml` and not running because its account will not connect. + +``` +tlgr job list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[JobState]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--enabled-only` | flag | | Hide disabled jobs. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr job list --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-filtering` + +lists the rules; proving one fires is `job test`. + +
+ +### `job reload` + +Hot-reload jobs.yaml without restarting the daemon. + +``` +tlgr job reload [OPTIONS] +``` + +**mutating · runs without an account · returns `Job`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--validate-only` | flag | | Parse and report without swapping the pipeline. | + +```console +$ tlgr job reload --validate-only --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-webhook-delivery` + +reloads the consumers; delivery is the webhook pusher's. + +
+ +### `job remove` + +Remove a job. + +``` +tlgr job remove [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · runs without an account · returns `Job`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `NAME` | text | yes | — | + +```console +$ tlgr job remove archive --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-event-filtering` + +deletes a rule; the filtering itself is the gateway's. + +
+ +### `job test` + +Dry-run a job's filters against real or synthetic events. + +`filter_trace` names every filter node and says why it passed or rejected, which is the missing piece when a job silently never fires. Actions are reported, never executed. + +``` +tlgr job test [OPTIONS] +``` + +**runs without an account · returns `none`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `NAME` | text | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--chat` | chat | | Restrict the replay. | +| `--event` | text | | Synthesise an event of this type instead. | +| `--from-file` | text | | Feed envelopes from NDJSON. | +| `--run-actions` | flag | | Actually execute the actions instead of reporting them. | +| `--since` | int | | Replay buffered events from this seq. | + +```console +$ tlgr job test archive --event message_new --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.stream-event-filtering` + +
diff --git a/docs/reference/media.md b/docs/reference/media.md index cd840e0..1b55465 100644 --- a/docs/reference/media.md +++ b/docs/reference/media.md @@ -270,7 +270,7 @@ The ledger under the output root maps document id to path, so an interrupted exp tlgr media export [OPTIONS] ``` -**returns `ExportResult`** +**returns `MediaExportResult`** | Argument | Type | Required | Meaning | |---|---|---|---| diff --git a/docs/reference/net.md b/docs/reference/net.md new file mode 100644 index 0000000..8b85bb2 --- /dev/null +++ b/docs/reference/net.md @@ -0,0 +1,152 @@ + + +# `tlgr net` + +5 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`net dc list`](#tlgr-net-dc-list) | List Telegram data centres and their endpoints | +| [`net dc nearest`](#tlgr-net-dc-nearest) | Ask the server which data centre is nearest | +| [`net ping`](#tlgr-net-ping) | Measure round-trip latency to the current data centre | +| [`net status`](#tlgr-net-status) | Show the connection: DC, transport, proxy, latency, layer, clock offset | +| [`net usage get`](#tlgr-net-usage-get) | Report bytes sent/received and requests per account | + +### `net dc list` + +List Telegram data centres and their endpoints. + +``` +tlgr net dc list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · returns `Page[DcOption]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--cdn` | flag | | Only CDN endpoints. | +| `--ipv6` | flag | | Only IPv6 endpoints. | +| `--media-only` | flag | | Only media endpoints. | +| `--resolve` | flag | | Fetch the config over DNS/HTTPS when every DC is unreachable. | +| `--test` | flag | | List the test data centres instead of production. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr net dc list --ipv6 --json +``` + +
Catalog coverage (4 full, 0 partial) + +Full: `updates.config-dc-options`, `updates.config-dns-fallback`, `updates.net-ipv6`, `updates.net-test-dc` + +
+ +### `net dc nearest` + +Ask the server which data centre is nearest. + +``` +tlgr net dc nearest [OPTIONS] +``` + +**idempotent (reports `already`) · returns `NearestDc`** + +Also invocable as: `tlgr net nearest-dc` + +```console +$ tlgr net dc nearest --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.config-nearest-dc` + +
+ +### `net ping` + +Measure round-trip latency to the current data centre. + +``` +tlgr net ping [OPTIONS] +``` + +**idempotent (reports `already`) · returns `PingResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--probes` | int | `3` | — | +| `--via` | nearest-dc|get-state | `nearest-dc` | Which RPC to time. | + +Also invocable as: `tlgr daemon net ping` + +```console +$ tlgr net ping --probes 5 --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.net-ping-latency` + +
+ +### `net status` + +Show the connection: DC, transport, proxy, latency, layer, clock offset. + +A clock more than 30 seconds from the server's is reported as a warning, because MTProto derives `msg_id` from local time and the server drops anything outside its window — with no error the client can see. + +``` +tlgr net status [OPTIONS] +``` + +**idempotent (reports `already`) · returns `NetStatus`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--ping/--no-ping` | flag | `True` | Measure latency as part of it. | + +Also invocable as: `tlgr daemon net status` + +```console +$ tlgr net status --json +``` + +
Catalog coverage (4 full, 2 partial) + +Full: `updates.net-connection-status`, `updates.session-export-auth-dc`, `updates.session-time-sync`, `updates.sync-updating-indicator` + +Partial: `updates.net-migrate-errors`, `updates.net-ping-latency` + +reports the connection; a migration that escapes Telethon is named by `agent exit-codes`, and repeated probes are `net ping`. + +
+ +### `net usage get` + +Report bytes sent/received and requests per account. + +Coarse per-class counters, not the official clients' per-method breakdown: Telethon does no byte accounting, so anything finer would be invented. In memory, and reset with the daemon. + +``` +tlgr net usage get [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `NetUsage`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--reset` | flag | | Zero the counters after reporting. | + +Also invocable as: `tlgr daemon net usage` + +```console +$ tlgr net usage get --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.ops-network-usage-stats` + +
diff --git a/docs/reference/proxy.md b/docs/reference/proxy.md new file mode 100644 index 0000000..0128618 --- /dev/null +++ b/docs/reference/proxy.md @@ -0,0 +1,221 @@ + + +# `tlgr proxy` + +6 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`proxy add`](#tlgr-proxy-add) | Save a proxy | +| [`proxy link`](#tlgr-proxy-link) | Print a saved proxy as a shareable tg:// link | +| [`proxy list`](#tlgr-proxy-list) | List saved proxies | +| [`proxy remove`](#tlgr-proxy-remove) | Delete a saved proxy | +| [`proxy set`](#tlgr-proxy-set) | Choose the proxy the daemon connects through | +| [`proxy test`](#tlgr-proxy-test) | Probe a proxy and measure its latency | + +### `proxy add` + +Save a proxy. + +Accepts both `tg://proxy?…` and `https://t.me/proxy?…`, and both secret encodings. Secrets are read from an environment variable, a file or stdin — never argv. + +``` +tlgr proxy add [LINK] [OPTIONS] +``` + +**mutating · runs without an account · returns `Proxy`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `LINK` | text | no | tg://proxy?… or https://t.me/proxy?… | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--set` | flag | | Make it the active proxy immediately (reconnects). | +| `--host` | text | | — | +| `--name` | text | | — | +| `--password` | text | | SOCKS5/HTTP password. | +| `--port` | int | | — | +| `--rdns/--no-rdns` | flag | `True` | Resolve hostnames through the proxy. | +| `--secret` | text | | MTProxy secret (hex or base64url). | +| `--type` | socks5|http|mtproxy | | Proxy kind. | +| `--user` | text | | SOCKS5/HTTP username. | + +Also invocable as: `tlgr net proxy add` + +```console +$ tlgr proxy add 'tg://proxy?server=1.2.3.4&port=443&secret=dd00' --set --json +``` + +
Catalog coverage (2 full, 3 partial) + +Full: `updates.net-proxy-mtproxy`, `updates.net-proxy-socks5` + +Partial: `updates.net-proxy-http`, `updates.net-proxy-list`, `updates.net-proxy-share-link` + +saves one; choosing it is `proxy set`, listing is `proxy list`, and printing the shareable link is `proxy link`. + +
+ +### `proxy link` + +Print a saved proxy as a shareable tg:// link. + +The link embeds the password or MTProxy secret. Confirm off a TTY. + +``` +tlgr proxy link [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · runs without an account · returns `ProxyLink`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `ID` | text | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--form` | tg|t.me | `tg` | Link flavour. | + +Also invocable as: `tlgr net proxy link`, `tlgr proxy export` + +```console +$ tlgr proxy link p1 --yes --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.net-proxy-share-link` + +
+ +### `proxy list` + +List saved proxies. + +`order` is the failover order. `has_password`/`has_secret` say a credential exists without printing it; only `proxy link` does that. + +``` +tlgr proxy list [OPTIONS] +``` + +**paginated (`LOCAL` cursor) · idempotent (reports `already`) · runs without an account · returns `Page[Proxy]`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--active-only` | flag | | Only the proxy currently in use. | +| `--type` | socks5|http|mtproxy | | Filter by kind. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +Also invocable as: `tlgr net proxy list` + +```console +$ tlgr proxy list --json +``` + +
Catalog coverage (0 full, 2 partial) + +Partial: `updates.net-proxy-autoswitch`, `updates.net-proxy-list` + +lists them; failover is `proxy test --reorder` plus `proxy set`. + +
+ +### `proxy remove` + +Delete a saved proxy. + +``` +tlgr proxy remove [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · runs without an account · returns `ProxySelection`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `ID` | text | yes | — | + +Also invocable as: `tlgr net proxy remove` + +```console +$ tlgr proxy remove p1 --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.net-proxy-list` + +
+ +### `proxy set` + +Choose the proxy the daemon connects through. + +`none` is a direct connection; `system` reads ALL_PROXY/HTTPS_PROXY. A Telethon client cannot change proxy in place, so this rebuilds the client, reconnects and catches up. + +``` +tlgr proxy set [OPTIONS] +``` + +**mutating · runs without an account · returns `ProxySelection`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `ID` | text | yes | A saved id, `none` for a direct connection, or `system`. | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--reconnect/--no-reconnect` | flag | `True` | Reconnect now rather than on the next start. | + +Also invocable as: `tlgr net proxy set`, `tlgr proxy enable`, `tlgr proxy off` + +```console +$ tlgr proxy set p1 --json +``` + +
Catalog coverage (2 full, 3 partial) + +Full: `updates.net-proxy-http`, `updates.net-proxy-system` + +Partial: `updates.net-proxy-list`, `updates.net-proxy-mtproxy`, `updates.net-proxy-socks5` + +selects one; saving and describing them is `proxy add`/`proxy list`. + +
+ +### `proxy test` + +Probe a proxy and measure its latency. + +Uses a throwaway in-memory session: updates go to the last active connection, so a probe on the real session could divert the account's events to a client that is about to be discarded. + +``` +tlgr proxy test [ID] [OPTIONS] +``` + +**mutating · paginated (`LOCAL` cursor) · runs without an account · returns `Page[ProxyProbe]`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `ID` | text | no | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--every` | flag | | Test every saved proxy. | +| `--probe-timeout` | int | `10` | — | +| `--reorder` | flag | | Rewrite the failover order by measured latency. | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +Also invocable as: `tlgr net proxy test`, `tlgr proxy ping` + +```console +$ tlgr proxy test --every --reorder --json +``` + +
Catalog coverage (2 full, 0 partial) + +Full: `updates.net-proxy-autoswitch`, `updates.net-proxy-ping` + +
diff --git a/docs/reference/sync.md b/docs/reference/sync.md new file mode 100644 index 0000000..f764351 --- /dev/null +++ b/docs/reference/sync.md @@ -0,0 +1,177 @@ + + +# `tlgr sync` + +5 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`sync backfill`](#tlgr-sync-backfill) | Refill a message-id range after a box overflow (differenceTooLong) | +| [`sync catch-up`](#tlgr-sync-catch-up) | Force a difference fetch so nothing missed while offline is lost | +| [`sync difference`](#tlgr-sync-difference) | Run updates.getDifference / getChannelDifference explicitly (diagnostics) | +| [`sync reset`](#tlgr-sync-reset) | Throw away the local update state and re-baseline from the server | +| [`sync status`](#tlgr-sync-status) | Show the update cursors (pts/qts/seq/date) and how far behind the account is | + +### `sync backfill` + +Refill a message-id range after a box overflow (differenceTooLong). + +`messages.getHistory` cannot fill a channel gap — it is bounded by the same box that overflowed. Fetching by explicit id can, and a deleted message comes back as `messageEmpty`, so the range is always complete. + +``` +tlgr sync backfill [OPTIONS] +``` + +**paginated (`HISTORY` cursor) · returns `Page[Message]`** + +| Argument | Type | Required | Meaning | +|---|---|---|---| +| `CHAT` | chat | yes | — | + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--chunk` | int | `200` | Ids per request. | +| `--emit` | flag | | Emit the refilled messages as events, marked `backfill`. | +| `--from-id` | int | | Lowest message id (inclusive). | +| `--to-id` | int | | Highest message id (inclusive). | + +Pagination is transport-level: `--limit/-n`, `--cursor TOKEN`, `--all` (walked inside the daemon, paced by the account's own rate limiter). + +```console +$ tlgr sync backfill @news --from-id 91800 --to-id 91900 --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.sync-difference-too-long` + +
+ +### `sync catch-up` + +Force a difference fetch so nothing missed while offline is lost. + +Not `chat catchup`, which is the unread digest. This is `updates.getDifference`: without it an account that was away silently misses everything that happened, with no later signal that it did. + +``` +tlgr sync catch-up [OPTIONS] +``` + +**mutating · idempotent (reports `already`) · returns `CatchUpResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--catch-up-timeout` | int | `120` | Give up waiting. | +| `--wait/--no-wait` | flag | `True` | Block until the difference is drained. | + +Also invocable as: `tlgr daemon sync catch-up` + +```console +$ tlgr sync catch-up --json +``` + +
Catalog coverage (2 full, 2 partial) + +Full: `updates.sync-get-difference`, `updates.sync-too-long` + +Partial: `updates.sync-catch-up-on-start`, `updates.sync-force-resync` + +the manual fetch; doing it at start is `daemon start --catch-up`, and giving up on a gap is `sync reset`. + +
+ +### `sync difference` + +Run updates.getDifference / getChannelDifference explicitly (diagnostics). + +Read-only without `--apply`: the daemon's stored pts is untouched, so the probe cannot create the gap it was meant to diagnose. `--follow` short-polls a channel, honouring the timeout the server returns rather than a pace tlgr invented. + +``` +tlgr sync difference [OPTIONS] +``` + +**idempotent (reports `already`) · returns `DifferenceResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--apply` | flag | | Feed the result into the daemon's state and event stream. | +| `--chat` | chat | | Run getChannelDifference for this channel. | +| `--date` | datetime | | Start from this date. | +| `--depth` | int | `1000` | pts_total_limit (common box) or limit (channel). | +| `--follow` | int | | Short-poll the channel for this long, honouring the returned timeout. | +| `--pts` | int | | Start from this pts. | +| `--qts` | int | | Start from this qts. | + +```console +$ tlgr sync difference --chat @news --follow 30 --json +``` + +
Catalog coverage (3 full, 2 partial) + +Full: `updates.sync-channel-short-poll`, `updates.sync-get-channel-difference`, `updates.sync-pts-gap-algorithm` + +Partial: `updates.sync-get-difference`, `updates.sync-qts-gap-algorithm` + +runs one by hand; the automatic path is `sync catch-up`. + +
+ +### `sync reset` + +Throw away the local update state and re-baseline from the server. + +The give-up path, not the recovery one: everything before the new state is marked seen and is unrecoverable. Use it when a corrupted state loops on getDifference; use `sync catch-up` to replay a gap. + +``` +tlgr sync reset [OPTIONS] +``` + +**mutating · destructive (needs `--yes` off a TTY) · returns `ResetResult`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--all-channels` | flag | | Reset every per-channel pts, keeping the common box. | +| `--chat` | chat | | Only reset this channel's pts. | + +```console +$ tlgr sync reset --yes --json +``` + +
Catalog coverage (2 full, 0 partial) + +Full: `updates.sync-force-resync`, `updates.sync-state-persistence` + +
+ +### `sync status` + +Show the update cursors (pts/qts/seq/date) and how far behind the account is. + +`access_hash_known=false` on a channel means catch-up skips it silently — Telethon will not call getChannelDifference without one — so the channel looks idle rather than broken. + +``` +tlgr sync status [OPTIONS] +``` + +**idempotent (reports `already`) · returns `SyncStatus`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--channels` | flag | | Include the per-channel pts table. | +| `--refresh` | flag | | Also call updates.getState and report the server delta. | + +Also invocable as: `tlgr sync state`, `tlgr daemon sync status` + +```console +$ tlgr sync status --channels --refresh --json +``` + +
Catalog coverage (3 full, 5 partial) + +Full: `updates.sync-get-state`, `updates.sync-qts-gap-algorithm`, `updates.sync-seq-gap-algorithm` + +Partial: `updates.sync-force-resync`, `updates.sync-get-channel-difference`, `updates.sync-pts-gap-algorithm`, `updates.sync-state-persistence`, `updates.sync-too-long` + +reports the boxes; advancing them is `sync catch-up`, running one difference by hand is `sync difference`, and discarding them is `sync reset`. + +
diff --git a/docs/reference/webhook.md b/docs/reference/webhook.md new file mode 100644 index 0000000..015dc0b --- /dev/null +++ b/docs/reference/webhook.md @@ -0,0 +1,108 @@ + + +# `tlgr webhook` + +3 operations. Every one takes the global flags (`--json`, `--plain`, `-a/--account`, `--results-only`, `--select`, `--dry-run`, `--yes`, `--no-input`, `--flood-wait-max`, `-v`) anywhere on the line. + +| Command | Summary | +|---|---| +| [`webhook get`](#tlgr-webhook-get) | Show the webhook configuration and delivery health | +| [`webhook set`](#tlgr-webhook-set) | Configure the outbound webhook | +| [`webhook test`](#tlgr-webhook-test) | Send a test delivery to the configured URL | + +### `webhook get` + +Show the webhook configuration and delivery health. + +``` +tlgr webhook get [OPTIONS] +``` + +**idempotent (reports `already`) · runs without an account · returns `WebhookSettings`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--show-secret` | flag | | Reveal the HMAC secret and bearer token. | + +Also invocable as: `tlgr config webhook` + +```console +$ tlgr webhook get --json +``` + +
Catalog coverage (0 full, 1 partial) + +Partial: `updates.stream-webhook-delivery` + +reports the configuration; delivering is the pusher's job. + +
+ +### `webhook set` + +Configure the outbound webhook. + +Signature: `X-Tlgr-Signature: sha256=`, plus `X-Tlgr-Event`, `X-Tlgr-Seq`, `X-Tlgr-Account` and `X-Tlgr-Delivery`. The delivery id is what makes a catch-up replay safe to reprocess. Secrets are read from an environment variable, a file or stdin. + +``` +tlgr webhook set [OPTIONS] +``` + +**mutating · runs without an account · returns `WebhookSettings`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--backoff` | int | | Base of the exponential backoff. | +| `--chat` | chat | | Only push events about these chats. | +| `--enabled/--disabled` | flag | | Turn delivery on or off. | +| `--events` | text | | Event types to push. | +| `--max-attempts` | int | | Attempts before dead-lettering. | +| `--queue` | int | | Bounded queue depth before the lag policy. | +| `--request-timeout` | int | | Per-request timeout. | +| `--secret` | text | | HMAC-SHA256 signing secret. | +| `--sign` | hmac-sha256|bearer|none | | Signature scheme. | +| `--token` | text | | Bearer token (legacy; prefer the HMAC signature). | +| `--url` | text | | Destination URL. | + +```console +$ tlgr webhook set --url https://example.invalid/hook --events message_new --json +``` + +
Catalog coverage (1 full, 2 partial) + +Full: `updates.sync-duplicate-suppression` + +Partial: `updates.stream-event-filtering`, `updates.stream-webhook-delivery` + +configures delivery; the queue and retries are the pusher's. + +
+ +### `webhook test` + +Send a test delivery to the configured URL. + +Prints the exact headers, signature included, so a receiver can be verified end to end. One attempt by default: a test that retried would hide the failure it exists to show. + +``` +tlgr webhook test [OPTIONS] +``` + +**mutating · returns `WebhookProbe`** + +| Flag | Type | Default | Meaning | +|---|---|---|---| +| `--event` | text | `message_new` | Event type to synthesise. | +| `--retry` | flag | | Use the configured retry policy instead of one attempt. | +| `--seq` | int | | Replay a real buffered event instead. | +| `--url` | text | | Override the configured URL for this test. | + +```console +$ tlgr webhook test --event message_new --json +``` + +
Catalog coverage (1 full, 0 partial) + +Full: `updates.stream-webhook-delivery` + +
diff --git a/tests/conftest.py b/tests/conftest.py index 0e5617e..a84d6ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,6 +142,25 @@ async def run(func, *args, **kwargs): return run +@pytest.fixture(autouse=True) +def _never_the_real_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """No test may resolve `~/.tlgr`, ever — including one that forgot to ask. + + A developer's tlgr home holds live session files. A test that reaches it + does not merely "leave some files behind": it shares an auth key with a + running daemon, and Telegram treats a second client on one auth key as a + compromised session and revokes it. Tests that want a home take + `tlgr_home`, which overrides this; the point of this fixture is that + forgetting to is impossible rather than merely discouraged. + """ + scratch = Path(tempfile.mkdtemp(prefix="tlgr-noreal-", dir=tempfile.gettempdir())) + monkeypatch.setenv("TLGR_HOME", str(scratch / "h")) + try: + yield + finally: + shutil.rmtree(scratch, ignore_errors=True) + + @pytest.fixture(autouse=True) def _restore_umask(): previous = os.umask(0o077) diff --git a/tests/fake_telethon.py b/tests/fake_telethon.py index d306c3f..1dae392 100644 --- a/tests/fake_telethon.py +++ b/tests/fake_telethon.py @@ -478,6 +478,15 @@ class World: read_inbox: dict[int, int] = field(default_factory=dict) read_outbox: dict[int, int] = field(default_factory=dict) pinned: dict[int, set[int]] = field(default_factory=dict) + #: The update state a session would hold: `{entity_id: (pts, qts, seq)}`, + #: with entity 0 as the common box. `sync status` and `sync reset` read + #: and write it exactly where Telethon's SQLiteSession keeps it. + update_state: dict[int, tuple[int, int, int]] = field( + default_factory=lambda: {0: (91824, 12, 4410)} + ) + #: Peers the session has an access hash for. A channel missing from here + #: is one `catch_up()` silently skips. + entities: set[int] = field(default_factory=set) #: marked chat id → its dialog row. Ordered newest-first by top_message, #: which is how the server orders `messages.getDialogs`. dialogs_by_id: dict[int, DialogState] = field(default_factory=dict) @@ -605,6 +614,10 @@ class World: catch_ups: int = 0 connects: int = 0 saves: int = 0 + differences: int = 0 + takeout_calls: list[str] = field(default_factory=list) + #: How far ahead of the local pts `updates.getState` answers. + server_ahead: int = 0 def fail_next(self, request_name: str, exc: BaseException) -> None: """Make the next request of that type raise, once.""" @@ -923,7 +936,7 @@ class FakeTelegramClient: def __init__(self, world: World | None = None, session: Any = None, **_: Any) -> None: self.world = world or World() - self.session = _FakeSession() + self.session = _FakeSession(self.world) self.session_path = session self.flood_sleep_threshold = 120 self._connected = False @@ -1045,7 +1058,10 @@ async def __call__(self, request: Any, ordered: bool = False) -> Any: return handler default = getattr(self, f"_raw_{name}", None) if default is not None: - return default(request) + result = default(request) + if hasattr(result, "__await__"): + result = await result + return result return types.Updates(updates=[], users=[], chats=[], date=None, seq=0) # -- default raw handlers --------------------------------------------- @@ -2348,29 +2364,33 @@ def _raw_GetWebFileRequest(self, request: Any) -> Any: def _raw_GetConfigRequest(self, request: Any) -> Any: config = types.Config( - date=0, - expires=0, + date=datetime(2026, 9, 3, 9, 0, 0, tzinfo=timezone.utc), + expires=datetime(2026, 9, 3, 10, 0, 0, tzinfo=timezone.utc), test_mode=False, - this_dc=2, - dc_options=[], - dc_txt_domain_name="", + this_dc=4, + dc_options=[ + types.DcOption(id=4, ip_address="149.154.167.91", port=443), + types.DcOption(id=4, ip_address="2001:67c::b0e", port=443, ipv6=True), + types.DcOption(id=2, ip_address="149.154.167.51", port=443, media_only=True), + ], + dc_txt_domain_name="apv3.stel.com", chat_size_max=200, megagroup_size_max=200000, forwarded_count_max=100, - online_update_period_ms=1000, - offline_blur_timeout_ms=1000, - offline_idle_timeout_ms=1000, - online_cloud_timeout_ms=1000, - notify_cloud_delay_ms=1000, - notify_default_delay_ms=1000, - push_chat_period_ms=1000, - push_chat_limit=1, + online_update_period_ms=120000, + offline_blur_timeout_ms=5000, + offline_idle_timeout_ms=30000, + online_cloud_timeout_ms=300000, + notify_cloud_delay_ms=30000, + notify_default_delay_ms=1500, + push_chat_period_ms=60000, + push_chat_limit=2, edit_time_limit=172800, revoke_time_limit=172800, revoke_pm_time_limit=172800, - rating_e_decay=1, + rating_e_decay=2419200, stickers_recent_limit=200, - channels_read_media_period=1, + channels_read_media_period=604800, call_receive_timeout_ms=20000, call_ring_timeout_ms=90000, call_connect_timeout_ms=30000, @@ -2708,6 +2728,9 @@ def _raw_GetAppConfigRequest(self, request: Any) -> Any: config.setdefault("dismissed_suggestions", list(self.auth.dismissed_suggestions)) config.setdefault("dialogs_pinned_limit_default", 5) config.setdefault("channels_limit_default", 500) + config.setdefault("reactions_user_max_default", 1) + config.setdefault("freeze_appeal_url", "https://t.me/spambot") + config.setdefault("stories_pinned_to_top_count_max", 3) return types.help.AppConfig(hash=1, config=_json_value(config)) def _raw_GetPromoDataRequest(self, request: Any) -> Any: @@ -2830,10 +2853,83 @@ def _raw_GetAuthorizationFormRequest(self, request: Any) -> Any: # updates -------------------------------------------------------------- def _raw_GetStateRequest(self, request: Any) -> Any: + from datetime import datetime, timezone + + pts, qts, seq = self.world.update_state.get(0, (0, 0, 0)) return types.updates.State( - pts=90210, qts=0, date=datetime.now(timezone.utc), seq=0, unread_count=0 + pts=pts + self.world.server_ahead, + qts=qts, + date=datetime(2026, 9, 3, 9, 14, 7, tzinfo=timezone.utc), + seq=seq, + unread_count=0, + ) + + # -- the sync, network and takeout world ------------------------------- + + def _raw_GetNearestDcRequest(self, request: Any) -> Any: + return types.NearestDc(country="GB", this_dc=4, nearest_dc=4) + + def _raw_GetDifferenceRequest(self, request: Any) -> Any: + from datetime import datetime, timezone + + self.world.differences += 1 + return types.updates.DifferenceEmpty( + date=datetime(2026, 9, 3, 9, 14, 7, tzinfo=timezone.utc), seq=4410 + ) + + def _raw_GetChannelDifferenceRequest(self, request: Any) -> Any: + self.world.differences += 1 + return types.updates.ChannelDifferenceEmpty(pts=42, final=True, timeout=30) + + def _raw_GetCountriesListRequest(self, request: Any) -> Any: + return types.help.CountriesList( + countries=[ + types.help.Country( + iso2="GB", + default_name="United Kingdom", + name="United Kingdom", + country_codes=[ + types.help.CountryCode( + country_code="44", prefixes=["7"], patterns=["XXXX XXXXXX"] + ) + ], + ), + types.help.Country( + iso2="ES", + default_name="Spain", + country_codes=[types.help.CountryCode(country_code="34")], + ), + ], + hash=0, + ) + + def _raw_GetTimezonesListRequest(self, request: Any) -> Any: + return types.help.TimezonesList( + timezones=[types.Timezone(id="Europe/London", name="London", utc_offset=0)], + hash=0, ) + def _raw_InitTakeoutSessionRequest(self, request: Any) -> Any: + return types.account.Takeout(id=1234567890) + + def _raw_FinishTakeoutSessionRequest(self, request: Any) -> bool: + return True + + def _raw_GetSplitRangesRequest(self, request: Any) -> Any: + return [types.MessageRange(min_id=1, max_id=1000)] + + async def _raw_InvokeWithTakeoutRequest(self, request: Any) -> Any: + """Unwrap and run the inner request, recording that it was wrapped. + + Recording the wrapping is the point: a takeout that forgets + `invokeWithTakeout` on one call gets a *smaller* export rather than an + error, so the test has to be able to assert it was there. + """ + self.world.takeout_calls.append(type(request.query).__name__) + return await self(request.query) + + # -- entities ---------------------------------------------------------- + # -- entities ---------------------------------------------------------- async def get_entity(self, ref: Any) -> Any: @@ -3844,13 +3940,18 @@ async def wait(self, timeout: float | None = None) -> Any: class _FakeSession: - """Enough of a Telethon session that `StringSession.save()` works on it. + """Enough of `SQLiteSession` that both worlds can read what they read. `account export` reads the live session rather than the file on disk, so - a fake without an auth key would make that operation untestable. + a fake without an auth key would make that operation untestable. `sync + status`, `sync reset` and `daemon save-state` go through + `telethon_compat`, which reaches into the session's `update_state` and + `entities` tables because Telethon 1.44 exposes no accessor for either. + Faking the tables rather than the compat layer is what makes those tests + prove the real read path. """ - def __init__(self) -> None: + def __init__(self, world: Any = None) -> None: from telethon.crypto import AuthKey self.saved = 0 @@ -3859,10 +3960,41 @@ def __init__(self) -> None: self.port = 443 self.auth_key = AuthKey(bytes(range(256))) self.takeout_id = None + self._world = world def save(self) -> None: self.saved += 1 + def get_update_states(self) -> list[tuple[int, Any]]: + from datetime import datetime, timezone + + if self._world is None: + return [] + out = [] + for entity_id, (pts, qts, seq) in self._world.update_state.items(): + out.append( + ( + entity_id, + types.updates.State( + pts=pts, + qts=qts, + date=datetime(2026, 9, 3, 9, 14, 7, tzinfo=timezone.utc), + seq=seq, + unread_count=0, + ), + ) + ) + return out + + def set_update_state(self, entity_id: int, state: Any) -> None: + if self._world is None: + return + self._world.update_state[int(entity_id)] = ( + int(getattr(state, "pts", 0) or 0), + int(getattr(state, "qts", 0) or 0), + int(getattr(state, "seq", 0) or 0), + ) + def fake_client_factory(world: World | None = None) -> Any: """A `client_factory` for `SessionManager` that ignores the session path.""" diff --git a/tests/test_agentmd_compat.py b/tests/test_agentmd_compat.py index f69efcd..fc37d8e 100644 --- a/tests/test_agentmd_compat.py +++ b/tests/test_agentmd_compat.py @@ -71,6 +71,38 @@ #: groups-and-channels group (PR-7), and must keep working until they do. V1_HAND_WRITTEN = [("chat", "members"), ("chat", "create")] +#: The v1 paths PR-4 replaced. Every module behind them is deleted; every one +#: of them still resolves, because §12.4 makes that absolute. +V1_PR4_PATHS = [ + ("watch",), + ("status",), + ("schema",), + ("exit-codes",), + ("agent", "whoami"), + ("agent", "exit-codes"), + ("daemon", "start"), + ("daemon", "stop"), + ("daemon", "restart"), + ("daemon", "status"), + ("daemon", "install"), + ("daemon", "uninstall"), + ("daemon", "logs"), + ("job", "list"), + ("job", "add"), + ("job", "remove"), + ("job", "enable"), + ("job", "disable"), + ("job", "reload"), + ("config", "init"), + ("config", "validate"), + ("config", "path"), + ("config", "keys"), + ("config", "list"), + ("config", "get"), + ("config", "set"), + ("config", "unset"), +] + #: op id → the keys v1's AGENT.md showed in the response, minus the ones the #: deliberate-change table below accounts for. V1_KEYS: dict[str, set[str]] = { @@ -95,6 +127,15 @@ #: The changes CHANGELOG.md lists under "Breaking". Anything not in here has #: to survive unchanged. DELIBERATE_CHANGES = { + "events.watch": "`tlgr watch` streams the whole event taxonomy instead of " + "polling for new messages; `--results-only` keeps v1's line shape", + "daemon.status": "`{running, accounts, healthy}` gained `ready` and a " + "per-account state machine; `connections`/`disconnected` are unchanged", + "job.list": "`{jobs: [...]}` became `Page[JobState]`", + "config.list": "`{section: {key: value}}` became `Page[ConfigEntry]` with " + "a `source` per key; secrets are redacted", + "config.keys": "`{keys: {...}}` became `Page[ConfigKey]` carrying types, " + "defaults and `requires_restart`", "message.list": "`{messages: [...]}` became the `Page[Message]` envelope; " "`--results-only` yields `{items, has_more, next_cursor}`", "message.search": "same as message.list", @@ -125,12 +166,14 @@ def _walk(path: tuple[str, ...]) -> Any: return node -@pytest.mark.parametrize("path", V1_PATHS + V1_HAND_WRITTEN, ids=lambda p: " ".join(p)) +@pytest.mark.parametrize( + "path", V1_PATHS + V1_HAND_WRITTEN + V1_PR4_PATHS, ids=lambda p: " ".join(p) +) def test_every_documented_v1_path_is_still_invocable(path): assert _walk(path) is not None, f"tlgr {' '.join(path)} disappeared" -@pytest.mark.parametrize("path", V1_PATHS, ids=lambda p: " ".join(p)) +@pytest.mark.parametrize("path", V1_PATHS + V1_PR4_PATHS, ids=lambda p: " ".join(p)) def test_every_documented_v1_path_resolves_to_an_operation(path): assert ALIASES.get(".".join(path)) is not None @@ -142,6 +185,36 @@ def test_the_chat_list_shortcuts_still_resolve(): assert ALIASES["catchup"] == "chat.catchup" +def test_the_v1_watch_line_shape_survives_results_only(): + """A script reading `tlgr watch` parses `event_type`, `chat_id`, `data`.""" + import io + import json + + from tlgr.cli.render import render_stream + + out = io.StringIO() + render_stream( + [ + {"type": "meta", "protocol": 2}, + {"type": "message_new", "seq": 3, "chat_id": -100, "payload": {"id": 7}}, + {"type": "heartbeat", "ts": "2026-09-03T09:14:07Z"}, + {"type": "end", "ok": True}, + ], + results_only=True, + stream=out, + ) + lines = [json.loads(line) for line in out.getvalue().splitlines()] + assert lines == [ + { + "event_type": "message_new", + "chat_id": -100, + "data": {"id": 7}, + "seq": 3, + "account": None, + } + ] + + def test_the_shortcuts_still_reach_message_send(): for name in ("send", "msg.send", "message.send"): assert ALIASES[name] == "message.send" diff --git a/tests/test_event_taxonomy.py b/tests/test_event_taxonomy.py new file mode 100644 index 0000000..8ace91c --- /dev/null +++ b/tests/test_event_taxonomy.py @@ -0,0 +1,212 @@ +"""Every `Update*` constructor is named or explained — and stays that way. + +The bus's promise is that an update tlgr receives is either delivered under a +type a consumer can filter on, or is on a list with a reason next to it. That +promise is only worth something if it is checked against the *installed* +Telethon rather than against a list somebody typed once: a Telethon upgrade +that adds a constructor must fail here, in the run that upgrades it. +""" + +from __future__ import annotations + +import json + +import pytest + +from tlgr.core import eventtypes +from tlgr.core.errors import UsageError +from tlgr.daemon.events import normalise_update, tl_to_builtins + +TELETHON_UPDATES = sorted( + name + for name in dir(__import__("telethon.tl.types", fromlist=["x"])) + if name.startswith("Update") +) + + +class TestCompleteness: + def test_every_installed_constructor_is_accounted_for(self): + """Neither mapped nor internal means an update tlgr silently drops.""" + accounted = set(eventtypes.CONSTRUCTORS) | set(eventtypes.INTERNAL) + missing = sorted(set(TELETHON_UPDATES) - accounted) + assert missing == [], ( + f"{len(missing)} update constructors have no event type and no " + f"reason: {missing}. Add them to tlgr/core/eventtypes.py." + ) + + def test_the_table_names_no_constructor_telethon_does_not_have(self): + """A typo in the table would map a real update onto nothing.""" + extra = sorted( + (set(eventtypes.CONSTRUCTORS) | set(eventtypes.INTERNAL)) - set(TELETHON_UPDATES) + ) + assert extra == [] + + def test_the_layer_229_list_is_disjoint_from_the_installed_one(self): + overlap = sorted(set(eventtypes.NEWER_THAN_LAYER_227) & set(TELETHON_UPDATES)) + assert overlap == [], f"{overlap} are parseable here; drop the since_layer note" + + def test_every_mapped_type_is_declared(self): + declared = set(eventtypes.TYPES) + mapped = set(eventtypes.CONSTRUCTORS.values()) | set( + eventtypes.NEWER_THAN_LAYER_227.values() + ) + assert sorted(mapped - declared) == [] + + def test_every_declared_type_has_a_source_or_says_it_is_derived(self): + for name, spec in eventtypes.TYPES.items(): + assert eventtypes.constructors_for(name) or spec.derived, ( + f"{name} has no source constructor and does not say where it comes from" + ) + + def test_every_internal_constructor_gives_a_reason(self): + for name, reason in eventtypes.INTERNAL.items(): + assert len(reason) > 20, f"{name} is listed internal without a real reason" + + +class TestNames: + @pytest.mark.parametrize("name", sorted(eventtypes.TYPES)) + def test_a_type_name_is_lowercase_snake_case(self, name): + assert name == name.lower() + assert " " not in name and "-" not in name + + @pytest.mark.parametrize("name", sorted(eventtypes.TYPES)) + def test_a_type_belongs_to_a_declared_group(self, name): + assert eventtypes.TYPES[name].group in eventtypes.GROUPS + + @pytest.mark.parametrize("name", sorted(eventtypes.TYPES)) + def test_a_type_documents_itself(self, name): + assert eventtypes.TYPES[name].summary + + @pytest.mark.parametrize("name", sorted(eventtypes.TYPES)) + def test_a_box_is_one_of_the_five(self, name): + assert eventtypes.TYPES[name].box in ( + "pts", + "qts", + "seq", + "channel_pts", + "version", + "none", + ) + + +class TestSelectors: + def test_a_group_expands_to_its_types(self): + selected = eventtypes.resolve_selectors("read") + assert "read_inbox" in selected and "read_outbox" in selected + assert "message_new" not in selected + + def test_all_is_everything(self): + assert eventtypes.resolve_selectors("all") == frozenset(eventtypes.TYPES) + + def test_a_v1_name_still_selects(self): + """§12.4: `--events new_message` was v1's spelling and keeps working.""" + assert eventtypes.resolve_selectors("new_message") == frozenset({"message_new"}) + assert eventtypes.resolve_selectors("message_read") == frozenset( + {"read_inbox", "read_outbox"} + ) + + def test_a_raw_constructor_selects_its_type(self): + assert eventtypes.resolve_selectors("raw:UpdateBotStopped") == frozenset({"bot_stopped"}) + + def test_an_unknown_selector_is_a_usage_error_not_an_empty_watch(self): + """Watching nothing looks exactly like a broken daemon.""" + with pytest.raises(UsageError): + eventtypes.resolve_selectors("messages") + + def test_selectors_combine(self): + selected = eventtypes.resolve_selectors("message_new,presence") + assert {"message_new", "user_status", "typing"} <= selected + + +class TestNormalisation: + def test_a_container_is_not_an_event(self): + from telethon.tl import types + + update = types.UpdatesTooLong() + assert normalise_update("work", update) is None + + def test_a_raw_update_becomes_its_type(self): + from telethon.tl import types + + update = types.UpdateDeleteChannelMessages( + channel_id=5150, messages=[4, 5], pts=2, pts_count=2 + ) + kind, payload, chat_id, _sender = normalise_update("work", update) + assert kind == "message_deleted" + assert payload["message_ids"] == [4, 5] + assert chat_id == -1000000005150 + + def test_a_read_inbox_and_a_read_outbox_are_different_types(self): + from telethon.tl import types + + inbox = types.UpdateReadHistoryInbox( + peer=types.PeerUser(4242), max_id=9, still_unread_count=3, pts=1, pts_count=1 + ) + outbox = types.UpdateReadHistoryOutbox( + peer=types.PeerUser(4242), max_id=9, pts=1, pts_count=1 + ) + assert normalise_update("work", inbox)[0] == "read_inbox" + assert normalise_update("work", outbox)[0] == "read_outbox" + assert normalise_update("work", inbox)[1]["still_unread_count"] == 3 + + def test_a_service_message_is_its_own_type(self): + from telethon.tl import types + + service = types.MessageService( + id=11, + peer_id=types.PeerChat(77), + date=None, + action=types.MessageActionChatJoinedByLink(inviter_id=4242), + ) + update = types.UpdateNewMessage(message=service, pts=1, pts_count=1) + kind, payload, _chat, _sender = normalise_update("work", update) + assert kind == "message_service" + assert payload["action"] == "MessageActionChatJoinedByLink" + + def test_a_generic_update_carries_its_own_fields_json_safe(self): + from telethon.tl import types + + update = types.UpdateChannelMessageViews(channel_id=5150, id=8, views=1200) + kind, payload, chat_id, _sender = normalise_update("work", update) + assert kind == "message_views" + assert payload["views"] == 1200 + assert chat_id == -1000000005150 + json.dumps(payload) + + @pytest.mark.parametrize("name", sorted(eventtypes.CONSTRUCTORS)) + def test_no_payload_carries_a_datetime_or_bytes(self, name): + """COR-07: a datetime in a payload is a crash at delivery time.""" + import inspect + + from telethon.tl import types + + klass = getattr(types, name) + update = klass.__new__(klass) + # Every field present and empty: the shape a half-populated update + # from an older layer arrives in, and the one that used to crash the + # serialiser at delivery time rather than here. + for field in inspect.signature(klass.__init__).parameters: + if field != "self": + setattr(update, field, None) + normalised = normalise_update("work", update) + assert normalised is not None + json.dumps(normalised[1]) + + +class TestToBuiltins: + def test_a_datetime_becomes_rfc_3339(self): + from datetime import datetime, timezone + + assert tl_to_builtins(datetime(2026, 9, 3, 9, 14, 7, tzinfo=timezone.utc)) == ( + "2026-09-03T09:14:07Z" + ) + + def test_bytes_become_hex(self): + assert tl_to_builtins(b"\x00\xff") == "00ff" + + def test_a_tl_object_keeps_its_constructor_name(self): + from telethon.tl import types + + out = tl_to_builtins(types.PeerChannel(5150)) + assert out["_"] == "PeerChannel" + assert out["channel_id"] == 5150 diff --git a/tests/test_ops_auth.py b/tests/test_ops_auth.py index 3cd1559..2ae85e4 100644 --- a/tests/test_ops_auth.py +++ b/tests/test_ops_auth.py @@ -708,7 +708,10 @@ async def test_sync_refreshes_the_stored_record(self, live_daemon, client, in_th world.add_channel(make_channel(9000, title="News")) answer = await result(client, in_thread, "account.sync", {}) assert answer["ok"] is True - assert answer["pts"] == 90210 + # The common box the fake session holds; `updates.getState` answers + # from it rather than from a constant, so `sync status` and this + # agree about what the account's pts is. + assert answer["pts"] == 91824 # --------------------------------------------------------------------------- diff --git a/tests/test_ops_daemon.py b/tests/test_ops_daemon.py new file mode 100644 index 0000000..9dae032 --- /dev/null +++ b/tests/test_ops_daemon.py @@ -0,0 +1,667 @@ +"""The `daemon`, `sync`, `net`, `config` and `job` operations, end to end. + +Everything here goes over a real Unix socket, through the real middleware and +dispatcher, into the real implementation, against a fake Telegram. The +assertions are mostly about *what was sent* — the exact TL request the fake +recorded — because for this group the interesting failures are requests that +were never made (a catch-up that skipped a channel) or made wrongly (a +difference probe that advanced the stored pts). +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from tlgr.core.errors import EXIT_NOT_FOUND, EXIT_USAGE, classify + +CHANNEL = 5150 +CHANNEL_ID = -1000000000000 - CHANNEL + + +@pytest.fixture +def peers(world): + from fake_telethon import make_channel, make_user + + world.add_user(make_user(4242, username="alice")) + world.add_channel(make_channel(CHANNEL, title="News")) + return world + + +async def call(client, in_thread, op: str, request: Any = None, **kwargs: Any) -> dict[str, Any]: + kwargs.setdefault("account", "work") + return await in_thread(client.op, op, request, **kwargs) + + +async def result(client, in_thread, op: str, request: Any = None, **kwargs: Any) -> Any: + return (await call(client, in_thread, op, request, **kwargs))["result"] + + +def local(op_id: str, request: dict[str, Any] | None = None, **state: Any) -> Any: + """Run a `Surface.LOCAL` operation the way the CLI does — synchronously.""" + import msgspec + + from tlgr.cli.gen import LocalContext + from tlgr.registry import get + + spec = get(op_id) + context = LocalContext(account=state.pop("account", "")) + for key, value in state.items(): + setattr(context, key, value) + payload = msgspec.convert(request or {}, type=spec.request, strict=False) + return asyncio.run(spec.impl(context, payload)) + + +async def alocal(in_thread, op_id: str, request: dict[str, Any] | None = None, **state: Any): + """`local`, off the event loop. + + A local operation runs in the *client* process and talks to the daemon + over a blocking socket. Calling it inline from an async test would block + the very loop the daemon is serving on — a deadlock, not a slow test — so + it goes to an executor exactly as the CLI's own process does. + """ + return await in_thread(local, op_id, request, **state) + + +# --------------------------------------------------------------------------- +# daemon +# --------------------------------------------------------------------------- + + +class TestDaemonStatus: + async def test_it_separates_running_ready_and_healthy( + self, live_daemon, client, in_thread, tlgr_home + ): + """COR-37: v1 had only `running`, so a deaf daemon looked fine.""" + status = await alocal(in_thread, "daemon.status", {}, account="work") + assert status.running is True + assert status.ready is True + assert {row.alias for row in status.accounts} == {"work"} + + def test_a_stopped_daemon_is_reported_not_guessed(self, tlgr_home, stub_account): + status = local("daemon.status") + assert status.running is False + assert status.ready is False + assert status.healthy is False + + def test_check_turns_an_unhealthy_daemon_into_an_exit_code(self, tlgr_home, stub_account): + from tlgr.core.errors import DaemonNotRunningError + + with pytest.raises(DaemonNotRunningError): + local("daemon.status", {"check": True}) + + +class TestDaemonFloods: + async def test_a_remembered_deadline_is_listed(self, live_daemon, client, in_thread): + """The store Telethon forgets on exit: v1 re-hit every wait on restart.""" + live_daemon.sessions.limiter("work").note_flood("SendMessageRequest", 41, peer=4242) + items = await result(client, in_thread, "daemon.flood.list") + assert items and items[0]["method"] == "SendMessageRequest" + assert items[0]["wait_seconds"] > 0 + assert items[0]["kind"] == "flood_wait" + + async def test_clearing_needs_to_be_told_what_to_clear(self, live_daemon, client, in_thread): + with pytest.raises(Exception) as caught: + await call(client, in_thread, "daemon.flood.clear", {}) + assert classify(caught.value).exit_code == EXIT_USAGE + + async def test_clearing_everything_forgets_the_deadlines(self, live_daemon, client, in_thread): + limiter = live_daemon.sessions.limiter("work") + limiter.note_flood("SendMessageRequest", 41) + limiter.trip("peer flood") + cleared = await result(client, in_thread, "daemon.flood.clear", {"everything": True}) + assert cleared["cleared"] == 1 + assert limiter.breaker.open is False + assert limiter.flood.entries() == [] + + +class TestDeadLetters: + async def test_an_empty_store_exits_empty(self, live_daemon, client, in_thread): + items = await result(client, in_thread, "daemon.dead-letter.list") + assert items == [] + + async def test_entries_are_listed_and_deletable(self, live_daemon, client, in_thread): + live_daemon.webhook.write_dead_letters( + [ + { + "ts": "2026-09-03T09:00:00Z", + "first_failed_at": "2026-09-03T08:00:00Z", + "reason": "HTTP 502", + "source": "webhook", + "attempts": 3, + "delivery_id": "abc", + "seq": 12, + "event": "message_new", + "account": "work", + "body": "{}", + } + ] + ) + items = await result(client, in_thread, "daemon.dead-letter.list") + assert [row["id"] for row in items] == ["abc"] + assert items[0]["attempts"] == 3 + + deleted = await result(client, in_thread, "daemon.dead-letter.delete", {"id": ["abc"]}) + assert deleted["deleted"] == 1 + assert deleted.get("remaining", 0) == 0 + + async def test_deleting_needs_a_selector(self, live_daemon, client, in_thread): + with pytest.raises(Exception) as caught: + await call(client, in_thread, "daemon.dead-letter.delete", {}) + assert classify(caught.value).exit_code == EXIT_USAGE + + +class TestSaveState: + async def test_it_flushes_the_session(self, live_daemon, client, in_thread, world): + before = world.saves + saved = await result(client, in_thread, "daemon.save-state") + assert saved["accounts"][0]["alias"] == "work" + assert saved["accounts"][0]["pts"] == 91824 + assert world.saves > before + + +class TestReconnect: + async def test_it_reconnects_and_catches_up(self, live_daemon, client, in_thread, world): + before = world.catch_ups + report = await result(client, in_thread, "daemon.reconnect") + row = report["accounts"][0] + assert row["alias"] == "work" + assert row["reconnected"] is True + assert row["caught_up"] is True + assert world.catch_ups > before + + async def test_no_catch_up_skips_the_difference(self, live_daemon, client, in_thread, world): + before = world.catch_ups + report = await result(client, in_thread, "daemon.reconnect", {"catch_up": False}) + assert report["accounts"][0].get("caught_up", False) is False + assert world.catch_ups == before + + +# --------------------------------------------------------------------------- +# sync +# --------------------------------------------------------------------------- + + +class TestSyncStatus: + async def test_it_reports_the_cursors(self, live_daemon, client, in_thread): + status = await result(client, in_thread, "sync.status") + assert status["pts"] == 91824 + assert status["qts"] == 12 + assert status["seq"] == 4410 + + async def test_refresh_reports_the_server_delta(self, live_daemon, client, in_thread, world): + world.server_ahead = 40 + status = await result(client, in_thread, "sync.status", {"refresh": True}) + assert status["server_pts"] == 91864 + assert status["behind_pts"] == 40 + + async def test_a_channel_without_an_access_hash_is_flagged( + self, live_daemon, client, in_thread, world + ): + """Catch-up skips such a channel silently; it must not look idle.""" + world.update_state[CHANNEL] = (42, 0, 0) + envelope = await call(client, in_thread, "sync.status", {"channels": True}) + rows = envelope["result"]["channels"] + assert any(row["chat_id"] == CHANNEL_ID for row in rows) + assert any("access hash" in warning for warning in envelope["meta"]["warnings"]) + + +class TestSyncCatchUp: + async def test_it_forces_a_difference(self, live_daemon, client, in_thread, world): + before = world.catch_ups + report = await result(client, in_thread, "sync.catch-up") + assert report["account"] == "work" + assert world.catch_ups > before + + +class TestSyncDifference: + async def test_a_probe_does_not_advance_the_stored_pts( + self, live_daemon, client, in_thread, world + ): + """The safety property: a diagnostic cannot create the gap it looks for.""" + before = dict(world.update_state) + report = await result(client, in_thread, "sync.difference") + assert report["kind"] == "common" + assert report["final"] is True + assert report["dry_run"] is True + assert world.update_state == before + assert world.called("GetDifferenceRequest") + + async def test_a_channel_difference_uses_the_channel_request( + self, live_daemon, client, in_thread, peers + ): + report = await result(client, in_thread, "sync.difference", {"chat": str(CHANNEL_ID)}) + assert report["kind"] == "channel" + request = peers.called("GetChannelDifferenceRequest")[0] + assert request.force is True + assert 1 <= request.limit <= 100 + + async def test_a_private_chat_is_a_usage_error(self, live_daemon, client, in_thread, peers): + with pytest.raises(Exception) as caught: + await call(client, in_thread, "sync.difference", {"chat": "@alice"}) + error = classify(caught.value) + assert error.exit_code == EXIT_USAGE + + +class TestSyncReset: + async def test_it_re_baselines_from_the_server(self, live_daemon, client, in_thread, world): + world.server_ahead = 100 + report = await result(client, in_thread, "sync.reset") + assert report["reset"] is True + assert report["pts_before"] == 91824 + assert report["pts_after"] == 91924 + assert world.update_state[0][0] == 91924 + + +class TestSyncBackfill: + async def test_it_fetches_by_explicit_id(self, live_daemon, client, in_thread, peers): + for index in range(5): + peers.add_message(CHANNEL_ID, f"post {index}", message_id=100 + index) + + def read() -> list[dict[str, Any]]: + frames = [] + for frame in client.op_stream( + "sync.backfill", + {"chat": str(CHANNEL_ID), "from_id": 100, "to_id": 104}, + account="work", + ): + frames.append(frame) + if frame.get("type") == "end": + break + return frames + + client._ready = True + frames = await in_thread(read) + items = [f["data"] for f in frames if f.get("type") == "item"] + assert [item["id"] for item in items] == [100, 101, 102, 103, 104] + + async def test_a_range_is_required(self, live_daemon, client, in_thread, peers): + def read() -> list[dict[str, Any]]: + return list( + client.op_stream("sync.backfill", {"chat": str(CHANNEL_ID)}, account="work") + ) + + client._ready = True + frames = await in_thread(read) + assert frames[-1]["error"]["exit_code"] == EXIT_USAGE + + +# --------------------------------------------------------------------------- +# net +# --------------------------------------------------------------------------- + + +class TestNet: + async def test_dc_list_marks_the_current_one(self, live_daemon, client, in_thread): + items = await result(client, in_thread, "net.dc.list") + assert [row["id"] for row in items] == [4, 4, 2] + # `current` is False by default and `Model` omits defaults, so the + # media-only DC simply has no key rather than a false one. + assert [row.get("current", False) for row in items] == [True, True, False] + + async def test_the_ipv6_filter_narrows_it(self, live_daemon, client, in_thread): + items = await result(client, in_thread, "net.dc.list", {"ipv6": True}) + assert len(items) == 1 and items[0]["ipv6"] is True + + async def test_test_dcs_need_no_connection(self, live_daemon, client, in_thread): + items = await result(client, in_thread, "net.dc.list", {"test": True}) + assert {row["id"] for row in items} == {1, 2, 3} + + async def test_resolve_says_it_is_not_implemented(self, live_daemon, client, in_thread): + """A --resolve that quietly did nothing would be worse than one that says so.""" + with pytest.raises(Exception) as caught: + await call(client, in_thread, "net.dc.list", {"resolve": True}) + assert classify(caught.value).code == "NOT_SUPPORTED" + + async def test_nearest_dc(self, live_daemon, client, in_thread): + report = await result(client, in_thread, "net.dc.nearest") + assert report["country"] == "GB" + assert report["nearest_dc"] == 4 + + async def test_ping_reports_every_probe(self, live_daemon, client, in_thread, world): + report = await result(client, in_thread, "net.ping", {"probes": 2}) + assert report["probes"] == 2 + assert report["loss"] == 0.0 + assert len(world.called("GetNearestDcRequest")) == 2 + + async def test_status_reports_the_connection(self, live_daemon, client, in_thread): + report = await result(client, in_thread, "net.status", {"ping": False}) + assert report["connected"] is True + assert report["dc_id"] == 4 + assert report["layer"] > 0 + assert report["state"]["pts"] == 91824 + + +# --------------------------------------------------------------------------- +# config +# --------------------------------------------------------------------------- + + +class TestConfigKeys: + def test_the_catalogue_is_machine_readable(self, tlgr_home): + page = local("config.keys", {}, limit=200) + keys = {row.key for row in page.items} + assert {"presence.mode", "daemon.idle_timeout", "flood.sleep_threshold"} <= keys + + def test_a_section_filter_narrows_it(self, tlgr_home): + page = local("config.keys", {"section": "presence"}, limit=200) + assert [row.key for row in page.items] == ["presence.mode"] + + +class TestConfigSet: + def test_a_value_round_trips(self, tlgr_home): + written = local("config.set", {"key": "daemon.idle_timeout", "value": "0"}) + assert written.updated is True + assert written.value == 0 + read = local("config.get", {"key": "daemon.idle_timeout"}) + assert read.value == 0 + assert read.source == "file" + + def test_setting_it_twice_reports_already(self, tlgr_home): + local("config.set", {"key": "daemon.idle_timeout", "value": "0"}) + again = local("config.set", {"key": "daemon.idle_timeout", "value": "0"}) + assert again.already is True + + def test_a_wrong_type_is_a_usage_error_not_a_silent_default(self, tlgr_home): + """The whole reason the catalogue exists (v1 swallowed this).""" + from tlgr.core.errors import UsageError + + with pytest.raises(UsageError) as caught: + local("config.set", {"key": "daemon.idle_timeout", "value": "soon"}) + assert classify(caught.value).exit_code == EXIT_USAGE + + def test_an_unknown_key_is_not_found(self, tlgr_home): + from tlgr.core.errors import NotFoundError + + with pytest.raises(NotFoundError) as caught: + local("config.set", {"key": "daemon.nonsense", "value": "1"}) + assert classify(caught.value).exit_code == EXIT_NOT_FOUND + + def test_a_v1_key_name_still_works(self, tlgr_home): + """§12.4: `tlgr config set idle_timeout 0` is a documented spelling.""" + written = local("config.set", {"key": "idle_timeout", "value": "0"}) + assert written.key == "daemon.idle_timeout" + + def test_a_choice_is_enforced(self, tlgr_home): + from tlgr.core.errors import UsageError + + with pytest.raises(UsageError): + local("config.set", {"key": "presence.mode", "value": "ghost"}) + + def test_unset_reverts_to_the_default(self, tlgr_home): + local("config.set", {"key": "daemon.idle_timeout", "value": "0"}) + removed = local("config.unset", {"key": "daemon.idle_timeout"}) + assert removed.removed is True + assert local("config.get", {"key": "daemon.idle_timeout"}).value == 1800 + + def test_a_secret_is_redacted_in_a_listing(self, tlgr_home): + local("config.set", {"key": "net.proxy", "value": "socks5://user:pw@10.0.0.5:1080"}) + rows = {row.key: row.value for row in local("config.list", {}, limit=200).items} + assert rows["net.proxy"] == "" + + +class TestConfigValidate: + def test_a_clean_tree_validates(self, tlgr_home): + local("config.init", {}) + report = local("config.validate", {}) + assert report.ok is True + + def test_an_unknown_key_is_a_warning_and_strict_makes_it_an_error(self, tlgr_home): + (tlgr_home / "config.toml").write_text("[daemon]\nnonsense = 1\n") + report = local("config.validate", {}) + assert report.ok is True + assert any(issue.key == "daemon.nonsense" for issue in report.warnings) + strict = local("config.validate", {"strict": True}) + assert strict.ok is False + + def test_an_unknown_event_name_in_the_webhook_is_an_error(self, tlgr_home): + (tlgr_home / "webhook.toml").write_text( + '[webhook]\nenabled = false\nevents = ["message_exploded"]\n' + ) + report = local("config.validate", {"file": "webhook"}) + assert report.ok is False + assert "unknown event type" in report.errors[0].message + + +class TestConfigServer: + async def test_it_reads_the_server_limits(self, live_daemon, client, in_thread): + report = await result(client, in_thread, "config.server.get") + assert report["message_length_max"] == 4096 + assert report["edit_time_limit"] == 172800 + + async def test_dc_options_are_opt_in(self, live_daemon, client, in_thread): + without = await result(client, in_thread, "config.server.get") + assert "dc_options" not in without + with_them = await result(client, in_thread, "config.server.get", {"dc_options": True}) + assert len(with_them["dc_options"]) == 3 + + +class TestConfigApp: + async def test_the_json_object_is_flattened(self, live_daemon, client, in_thread): + report = await result(client, in_thread, "config.app.get") + assert report["values"]["reactions_user_max_default"] == 1 + + async def test_frozen_narrows_to_the_freeze_fields(self, live_daemon, client, in_thread): + report = await result(client, in_thread, "config.app.get", {"frozen": True}) + assert set(report["values"]) == {"freeze_appeal_url"} + assert report["freeze_appeal_url"] == "https://t.me/spambot" + + +class TestConfigCountries: + async def test_a_phone_is_classified(self, live_daemon, client, in_thread): + items = await result(client, in_thread, "config.country.list", {"phone": "+447700900000"}) + assert [row["iso2"] for row in items] == ["GB"] + assert items[0]["matched_prefix"] == "447" + assert items[0]["flag_emoji"] == "🇬🇧" + + async def test_an_unclaimed_prefix_is_not_found(self, live_daemon, client, in_thread): + with pytest.raises(Exception) as caught: + await call(client, in_thread, "config.country.list", {"phone": "+999123"}) + assert classify(caught.value).exit_code == EXIT_NOT_FOUND + + +# --------------------------------------------------------------------------- +# job +# --------------------------------------------------------------------------- + + +class TestJobs: + def test_a_job_is_added_from_flags(self, tlgr_home): + added = local( + "job.add", + {"name": "archive", "action": ["forward:to=@archive"], "events": "new_message"}, + ) + assert added.name == "archive" + state = local("job.get", {"name": "archive"}) + assert state.actions == [{"forward": {"to": "@archive"}}] + + def test_a_job_with_no_actions_is_refused(self, tlgr_home): + from tlgr.core.errors import UsageError + + with pytest.raises(UsageError): + local("job.add", {"name": "empty"}) + + def test_an_unknown_event_is_refused_at_write_time(self, tlgr_home): + """v1 dropped the name at load time, so the job never fired and never said why.""" + from tlgr.core.errors import UsageError + + with pytest.raises(UsageError): + local("job.add", {"name": "x", "action": ["reply:hi"], "events": "message_exploded"}) + + def test_adding_the_same_name_twice_is_refused(self, tlgr_home): + from tlgr.core.errors import UsageError + + local("job.add", {"name": "archive", "action": ["reply:hi"]}) + with pytest.raises(UsageError): + local("job.add", {"name": "archive", "action": ["reply:hi"]}) + + def test_an_unknown_job_is_not_found(self, tlgr_home): + from tlgr.core.errors import NotFoundError + + with pytest.raises(NotFoundError) as caught: + local("job.get", {"name": "ghost"}) + assert classify(caught.value).exit_code == EXIT_NOT_FOUND + + def test_explain_names_an_unregistered_filter(self, tlgr_home): + local( + "job.add", + {"name": "archive", "action": ["reply:hi"], "filter": ["nonsense=1"]}, + ) + state = local("job.get", {"name": "archive", "explain": True}) + assert "UNKNOWN" in state.filters["nonsense"]["resolves_to"] + + async def test_enable_and_disable_round_trip(self, live_daemon, client, in_thread, tlgr_home): + await alocal(in_thread, "job.add", {"name": "archive", "action": ["reply:hi"]}) + disabled = await result(client, in_thread, "job.disable", {"name": "archive"}) + assert disabled["enabled"] is False + again = await result(client, in_thread, "job.disable", {"name": "archive"}) + assert again["already"] is True + enabled = await result(client, in_thread, "job.enable", {"name": "archive"}) + assert enabled["enabled"] is True + + async def test_removing_a_job_takes_it_out_of_the_file( + self, live_daemon, client, in_thread, tlgr_home + ): + await alocal(in_thread, "job.add", {"name": "archive", "action": ["reply:hi"]}) + removed = await result(client, in_thread, "job.remove", {"name": "archive"}) + assert removed["removed"] is True + items = await result(client, in_thread, "job.list") + assert items == [] + + async def test_adding_one_job_keeps_the_others(self, live_daemon, client, in_thread, tlgr_home): + """A rewrite that lost unmodelled filters would silently delete rules.""" + await alocal( + in_thread, + "job.add", + {"name": "first", "action": ["reply:hi"], "filter": ["chat_type=private"]}, + ) + await alocal(in_thread, "job.add", {"name": "second", "action": ["reply:yo"]}) + assert {row["name"] for row in await result(client, in_thread, "job.list")} == { + "first", + "second", + } + first = await alocal(in_thread, "job.get", {"name": "first"}) + assert first.filters == {"chat_type": "private"} + + +# --------------------------------------------------------------------------- +# export +# --------------------------------------------------------------------------- + + +class TestExport: + async def test_a_session_wraps_every_later_call(self, live_daemon, client, in_thread, world): + """A call that forgets invokeWithTakeout gets a smaller export, not an error.""" + session = await result(client, in_thread, "export.start", {"messages": True, "files": True}) + assert session["takeout_id"] == 1234567890 + assert session["scope"] == ["messages", "files"] + + status = await result(client, in_thread, "export.status") + assert status["active"] is True + assert "GetSplitRangesRequest" in world.takeout_calls + + finished = await result(client, in_thread, "export.end") + assert finished["finished"] is True + assert "FinishTakeoutSessionRequest" in world.takeout_calls + + async def test_a_scope_is_required(self, live_daemon, client, in_thread): + with pytest.raises(Exception) as caught: + await call(client, in_thread, "export.start", {}) + assert classify(caught.value).exit_code == EXIT_USAGE + + async def test_starting_twice_reports_already(self, live_daemon, client, in_thread): + await result(client, in_thread, "export.start", {"messages": True}) + again = await call(client, in_thread, "export.start", {"messages": True}) + assert again["result"]["already"] is True + assert again["meta"]["already"] is True + + async def test_status_without_a_session_is_inactive(self, live_daemon, client, in_thread): + status = await result(client, in_thread, "export.status") + assert status["active"] is False + + async def test_ending_without_a_session_reports_already(self, live_daemon, client, in_thread): + envelope = await call(client, in_thread, "export.end") + assert envelope["meta"]["already"] is True + + +# --------------------------------------------------------------------------- +# webhook +# --------------------------------------------------------------------------- + + +class TestWebhook: + def test_secrets_are_redacted_by_default(self, tlgr_home): + local("webhook.set", {"url": "https://example.invalid/h", "secret": "s3cret"}) + settings = local("webhook.get", {}) + assert settings.secret == "" + revealed = local("webhook.get", {"show_secret": True}) + assert revealed.secret == "s3cret" + + def test_an_unknown_event_is_refused(self, tlgr_home): + from tlgr.core.errors import UsageError + + with pytest.raises(UsageError): + local("webhook.set", {"events": "message_exploded"}) + + def test_a_group_selector_expands(self, tlgr_home): + settings = local("webhook.set", {"events": "read"}) + assert set(settings.events) == { + "read_inbox", + "read_outbox", + "read_contents", + "read_discussion", + "read_monoforum", + } + + def test_enabling_without_a_url_is_a_usage_error(self, tlgr_home): + from tlgr.core.errors import UsageError + + with pytest.raises(UsageError): + local("webhook.set", {"enabled": True}) + + +# --------------------------------------------------------------------------- +# agent +# --------------------------------------------------------------------------- + + +class TestAgent: + def test_whoami_always_carries_the_schema_version(self, tlgr_home, stub_account): + info = local("agent.whoami", {}, account="work") + assert info.output_schema_version == 2 + assert info.layer > 0 + assert info.daemon_running is False + + def test_capabilities_separates_cannot_from_will_not(self, tlgr_home): + report = local("agent.capabilities", {}) + assert report.event_types > 100 + assert report.unsupported_constructors + assert any("read receipt" in row["action"] for row in report.prohibited) + + def test_a_section_narrows_the_report(self, tlgr_home): + report = local("agent.capabilities", {"section": "policy"}) + assert report.prohibited + assert report.premium_gated == [] + + def test_the_error_table_names_the_captured_field(self, tlgr_home): + table = local("agent.exit-codes", {"errors": True, "search": "flood"}) + rows = {row.name: row for row in table.errors} + assert rows["FloodWaitError"].extra == "wait_seconds" + assert rows["FloodWaitError"].exit == 7 + assert rows["FloodWaitError"].retryable is True + + def test_the_exit_code_table_is_unchanged_without_errors(self, tlgr_home): + table = local("agent.exit-codes", {}) + assert table.errors == [] + assert table.exit_codes["USAGE"].code == 2 + + def test_schema_events_prints_the_taxonomy(self, tlgr_home): + document = local("agent.schema", {"path": ["events"]}) + types = {row["type"] for row in document["events"]} + assert {"message_new", "read_inbox"} <= types + + def test_schema_still_narrows_by_command_path(self, tlgr_home): + document = local("agent.schema", {"path": ["sync"]}) + assert all(op.startswith("sync.") for op in document["ops"]) diff --git a/tests/test_ops_events.py b/tests/test_ops_events.py new file mode 100644 index 0000000..7204be4 --- /dev/null +++ b/tests/test_ops_events.py @@ -0,0 +1,374 @@ +"""The `events` group and `watch`, end to end through a real daemon. + +`watch` is the one operation whose correctness is about *time*: it has to +deliver what already happened, then what happens next, and say so when it +cannot. Each test here therefore drives a real socket, a real bus and the real +NDJSON framing rather than calling the implementation directly. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from typing import Any + +import pytest + +from tlgr.core.errors import EXIT_INDETERMINATE, EXIT_NOT_FOUND, EXIT_USAGE, classify + +ALICE = 4242 + + +async def call(client, in_thread, op: str, request: Any = None, **kwargs: Any) -> dict[str, Any]: + kwargs.setdefault("account", "work") + return await in_thread(client.op, op, request, **kwargs) + + +async def result(client, in_thread, op: str, request: Any = None, **kwargs: Any) -> Any: + return (await call(client, in_thread, op, request, **kwargs))["result"] + + +def local(op_id: str, request: dict[str, Any] | None = None, **state: Any) -> Any: + """Run a `Surface.LOCAL` operation the way the CLI does.""" + import msgspec + + from tlgr.cli.gen import LocalContext + from tlgr.registry import get + + spec = get(op_id) + context = LocalContext(account=state.pop("account", "")) + for key, value in state.items(): + setattr(context, key, value) + payload = msgspec.convert(request or {}, type=spec.request, strict=False) + return asyncio.run(spec.impl(context, payload)) + + +class TestEventList: + def test_it_lists_the_whole_taxonomy(self): + page = local("events.list", {}, limit=1000) + assert page.total > 100 + assert {row.type for row in page.items} >= {"message_new", "read_inbox", "typing"} + + def test_a_group_filter_narrows_it(self): + page = local("events.list", {"group": "read"}, limit=100) + assert {row.group for row in page.items} == {"read"} + + def test_available_hides_what_this_build_cannot_receive(self): + everything = local("events.list", {}, limit=1000) + available = local("events.list", {"available": True}, limit=1000) + assert available.total < everything.total + assert all(row.available and not row.bot_only for row in available.items) + + def test_raw_lists_constructors(self): + page = local("events.list", {"raw": True, "search": "UpdateBotStopped"}, limit=50) + assert [row.sources for row in page.items] == [["UpdateBotStopped"]] + + def test_the_page_carries_a_cursor_when_there_is_more(self): + page = local("events.list", {}, limit=5) + assert page.has_more is True + assert page.next_cursor + + +class TestEventGet: + def test_it_reports_the_sequence_box(self): + detail = local("events.get", {"type": "message_new"}) + assert detail.box == "pts" + assert "UpdateNewMessage" in detail.sources + assert detail.example is not None + + def test_a_raw_constructor_resolves_to_its_type(self): + detail = local("events.get", {"type": "raw:UpdateReadHistoryOutbox"}) + assert detail.type == "read_outbox" + + def test_a_json_schema_is_emitted_on_request(self): + detail = local("events.get", {"type": "read_inbox", "json_schema": True}) + assert detail.json_schema is not None + assert detail.json_schema["properties"]["max_id"]["type"] == "integer" + + def test_an_unknown_type_is_not_found(self): + from tlgr.core.errors import NotFoundError + + with pytest.raises(NotFoundError): + local("events.get", {"type": "message_exploded"}) + + +class TestEventDecode: + def test_a_raw_update_becomes_an_envelope(self, tmp_path): + path = tmp_path / "update.json" + path.write_text(json.dumps({"_": "UpdateReadHistoryInbox", "max_id": 9})) + decoded = local("events.decode", {"input": str(path)}) + assert decoded.event == "read_inbox" + assert decoded.data["max_id"] == 9 + + def test_a_container_says_why_it_carries_no_event(self, tmp_path): + from tlgr.core.errors import NotSupportedError + + path = tmp_path / "update.json" + path.write_text(json.dumps({"_": "UpdatesTooLong"})) + with pytest.raises(NotSupportedError) as excinfo: + local("events.decode", {"input": str(path)}) + assert classify(excinfo.value).exit_code == EXIT_INDETERMINATE + + def test_an_unknown_constructor_is_not_found(self, tmp_path): + from tlgr.core.errors import NotFoundError + + path = tmp_path / "update.json" + path.write_text(json.dumps({"_": "UpdateSomethingElse"})) + with pytest.raises(NotFoundError) as excinfo: + local("events.decode", {"input": str(path)}) + assert classify(excinfo.value).exit_code == EXIT_NOT_FOUND + + def test_a_plain_json_push_payload_is_classified(self, tmp_path): + path = tmp_path / "push.b64" + payload = {"data": {"loc_key": "SESSION_REVOKE", "custom": {}}} + path.write_text(base64.b64encode(json.dumps(payload).encode()).decode()) + decoded = local("events.decode", {"input": str(path), "push": True}) + assert decoded.event == "account_session_revoked" + assert decoded.push is True + + def test_a_message_push_maps_onto_message_new(self, tmp_path): + path = tmp_path / "push.b64" + payload = {"loc_key": "CHANNEL_MESSAGE_TEXT", "chat_id": "-100123"} + path.write_text(base64.b64encode(json.dumps(payload).encode()).decode()) + decoded = local("events.decode", {"input": str(path), "push": True}) + assert decoded.event == "message_new" + assert decoded.chat_id == -100123 + + def test_an_encrypted_payload_round_trips_with_the_right_key(self, tmp_path): + """Encrypt with the same derivation the decoder uses, then decode it.""" + import hashlib + import os + + from telethon.crypto import AES + + auth_key = os.urandom(256) + body = json.dumps({"loc_key": "DC_UPDATE", "custom": {"dc": 4}}).encode() + plain = len(body).to_bytes(4, "little") + body + plain += b"\x00" * (-len(plain) % 16) + msg_key = hashlib.sha256(auth_key[88:120] + plain).digest()[8:24] + sha256_a = hashlib.sha256(msg_key + auth_key[0:36]).digest() + sha256_b = hashlib.sha256(auth_key[40:76] + msg_key).digest() + key = sha256_a[:8] + sha256_b[8:24] + sha256_a[24:32] + iv = sha256_b[:8] + sha256_a[8:24] + sha256_b[24:32] + blob = msg_key + AES.encrypt_ige(plain, key, iv) + + path = tmp_path / "push.b64" + path.write_text(base64.b64encode(blob).decode()) + decoded = local( + "events.decode", + { + "input": str(path), + "push": True, + "key": base64.b64encode(auth_key).decode(), + }, + ) + assert decoded.event == "sync_dc_options" + + def test_a_wrong_key_is_indeterminate_not_a_plausible_answer(self, tmp_path): + import os + + from tlgr.core.errors import IndeterminateError + + path = tmp_path / "push.b64" + path.write_text(base64.b64encode(os.urandom(80)).decode()) + with pytest.raises(IndeterminateError): + local( + "events.decode", + { + "input": str(path), + "push": True, + "key": base64.b64encode(os.urandom(256)).decode(), + }, + ) + + def test_an_encrypted_payload_without_a_key_is_a_usage_error(self, tmp_path): + import os + + from tlgr.core.errors import UsageError + + path = tmp_path / "push.b64" + path.write_text(base64.b64encode(os.urandom(80)).decode()) + with pytest.raises(UsageError): + local("events.decode", {"input": str(path), "push": True}) + + +class TestWatch: + async def test_it_replays_what_already_happened(self, live_daemon, client, in_thread): + live_daemon.bus.emit("work", "message_new", {"id": 1}, chat_id=-100) + live_daemon.bus.emit("work", "read_inbox", {"max_id": 4}, chat_id=-100) + + frames = await _watch(client, in_thread, {"since": 0, "events": "all", "follow": False}) + kinds = [frame.get("type") for frame in frames] + assert kinds[0] == "meta" + assert kinds[-1] == "end" + assert "message_new" in kinds and "read_inbox" in kinds + + async def test_the_default_selection_is_v1s_new_message(self, live_daemon, client, in_thread): + """§12.4: `tlgr watch` with no flags means what it meant in v1.""" + live_daemon.bus.emit("work", "message_new", {"id": 1}) + live_daemon.bus.emit("work", "typing", {"user_id": 4242}) + frames = await _watch(client, in_thread, {"since": 0, "follow": False}) + assert [f.get("type") for f in frames if f.get("seq")] == ["message_new"] + + async def test_a_group_selector_expands(self, live_daemon, client, in_thread): + live_daemon.bus.emit("work", "read_inbox", {"max_id": 1}) + live_daemon.bus.emit("work", "read_outbox", {"max_id": 2}) + live_daemon.bus.emit("work", "message_new", {"id": 3}) + frames = await _watch(client, in_thread, {"since": 0, "events": "read", "follow": False}) + assert sorted(f["type"] for f in frames if f.get("seq")) == ["read_inbox", "read_outbox"] + + async def test_exclude_subtracts(self, live_daemon, client, in_thread): + live_daemon.bus.emit("work", "read_inbox", {"max_id": 1}) + live_daemon.bus.emit("work", "read_outbox", {"max_id": 2}) + frames = await _watch( + client, + in_thread, + {"since": 0, "events": "read", "exclude": "read_outbox", "follow": False}, + ) + assert [f["type"] for f in frames if f.get("seq")] == ["read_inbox"] + + async def test_a_chat_filter_uses_marked_ids(self, live_daemon, client, in_thread): + live_daemon.bus.emit("work", "message_new", {"id": 1}, chat_id=-100) + live_daemon.bus.emit("work", "message_new", {"id": 2}, chat_id=-200) + frames = await _watch(client, in_thread, {"since": 0, "chat": ["-100"], "follow": False}) + assert [f["payload"]["id"] for f in frames if f.get("seq")] == [1] + + async def test_a_since_older_than_the_buffer_reports_a_gap( + self, live_daemon, client, in_thread + ): + live_daemon.bus.buffer_size = 4 + for index in range(12): + live_daemon.bus.emit("work", "message_new", {"id": index}) + frames = await _watch(client, in_thread, {"since": 1, "events": "all", "follow": False}) + gaps = [frame for frame in frames if frame.get("type") == "gap"] + assert gaps and gaps[0]["lost"] > 0 + + async def test_an_unknown_event_selector_is_a_usage_error(self, live_daemon, client, in_thread): + frames = await _watch(client, in_thread, {"events": "messages", "follow": False}) + end = frames[-1] + assert end["type"] == "end" + assert end["ok"] is False + assert end["error"]["exit_code"] == EXIT_USAGE + + async def test_it_follows_and_heartbeats(self, live_daemon, client, in_thread): + """A quiet chat must be distinguishable from a dead connection.""" + frames = await _watch( + client, + in_thread, + {"events": "all", "follow": True, "heartbeat": 1, "follow_for": 2}, + ) + assert any(frame.get("type") == "heartbeat" for frame in frames) + assert frames[-1]["type"] == "end" + + async def test_max_events_stops_the_stream(self, live_daemon, client, in_thread): + for index in range(5): + live_daemon.bus.emit("work", "message_new", {"id": index}) + frames = await _watch( + client, + in_thread, + {"since": 0, "events": "all", "follow": False, "max_events": 2}, + ) + assert len([f for f in frames if f.get("seq")]) == 2 + + async def test_the_cursor_frame_says_where_to_resume(self, live_daemon, client, in_thread): + live_daemon.bus.emit("work", "message_new", {"id": 1}) + frames = await _watch( + client, + in_thread, + {"since": 0, "events": "all", "follow": False, "print_cursor": True}, + ) + cursor = [frame for frame in frames if frame.get("type") == "cursor"] + assert cursor and cursor[0]["latest_seq"]["work"] == 1 + + +class TestEventsEndpoint: + async def test_the_get_endpoint_and_the_op_agree(self, live_daemon, client, in_thread): + """`GET /v1/events` is a GET-shaped alias of the same operation.""" + live_daemon.bus.emit("work", "message_new", {"id": 7}, chat_id=-100) + + def read() -> list[dict[str, Any]]: + frames = [] + for frame in client.events(account="work", since=0, timeout=1, follow="false"): + frames.append(frame) + if frame.get("type") == "end": + break + return frames + + client._ready = True + frames = await in_thread(read) + payloads = [f["payload"]["id"] for f in frames if f.get("type") == "message_new"] + assert payloads == [7] + + +class TestReplay: + async def test_it_returns_the_buffered_range(self, live_daemon, client, in_thread): + for index in range(4): + live_daemon.bus.emit("work", "message_new", {"id": index}) + items = await _replay(client, in_thread, {"since": 1, "events": "all"}) + assert [item["payload"]["id"] for item in items] == [1, 2, 3] + + async def test_a_range_before_the_buffer_is_indeterminate(self, live_daemon, client, in_thread): + """Returning the newest page would be a silent lie about catching up.""" + live_daemon.bus.buffer_size = 4 + for index in range(12): + live_daemon.bus.emit("work", "message_new", {"id": index}) + frames = await _frames(client, in_thread, "events.replay", {"since": 1, "events": "all"}) + end = frames[-1] + assert end["ok"] is False + assert end["error"]["exit_code"] == EXIT_INDETERMINATE + + async def test_difference_turns_the_gap_into_a_warning(self, live_daemon, client, in_thread): + live_daemon.bus.buffer_size = 4 + for index in range(12): + live_daemon.bus.emit("work", "message_new", {"id": index}) + frames = await _frames( + client, + in_thread, + "events.replay", + {"since": 1, "events": "all", "difference": True}, + ) + assert frames[-1]["ok"] is True + + async def test_a_filter_applies_to_the_replay(self, live_daemon, client, in_thread): + live_daemon.bus.emit("work", "message_new", {"id": 1}) + live_daemon.bus.emit("work", "read_inbox", {"max_id": 2}) + items = await _replay(client, in_thread, {"since": 0, "events": "read"}) + assert [item["type"] for item in items] == ["read_inbox"] + + async def test_an_empty_range_is_not_an_error(self, live_daemon, client, in_thread): + items = await _replay(client, in_thread, {"since": 0, "events": "all"}) + assert items == [] + + +async def _frames(client, in_thread, op_id: str, request: dict[str, Any]) -> list[dict[str, Any]]: + def read() -> list[dict[str, Any]]: + frames: list[dict[str, Any]] = [] + for frame in client.op_stream(op_id, request, account="work"): + frames.append(frame) + if frame.get("type") == "end": + break + return frames + + client._ready = True + return await in_thread(read) + + +async def _replay(client, in_thread, request: dict[str, Any]) -> list[dict[str, Any]]: + frames = await _frames(client, in_thread, "events.replay", request) + return [frame["data"] for frame in frames if frame.get("type") == "item"] + + +async def _watch(client, in_thread, request: dict[str, Any]) -> list[dict[str, Any]]: + """Drive `events.watch` over the socket and collect its frames.""" + + def read() -> list[dict[str, Any]]: + frames: list[dict[str, Any]] = [] + for frame in client.op_stream("events.watch", request, account="work"): + frames.append(frame) + if frame.get("type") == "end": + break + return frames + + client._ready = True + return await in_thread(read) diff --git a/tests/test_parity.py b/tests/test_parity.py index 0dc3ec3..0671a7f 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -25,10 +25,10 @@ #: Every P0 catalog id the landed PRs claim. Raised by each group PR, never #: lowered. ARCHITECTURE §1.3: "P0 coverage may never decrease and must reach #: 100 % before 2.0.0 final". -P0_FLOOR = 95 +P0_FLOOR = 116 #: The floor for total covered ids. Same rule, weaker guarantee. -COVERED_FLOOR = 808 +COVERED_FLOOR = 1010 #: Every P0 catalog id PR-1's own operations cover, named rather than #: counted, so a swap (one dropped, one added) cannot pass a count check @@ -176,11 +176,52 @@ } ) +#: The command groups PR-4 migrated. +PR4_GROUPS = ( + "agent.", + "config.", + "daemon.", + "events.", + "export.", + "job.", + "net.", + "proxy.", + "sync.", + "webhook.", +) + +PR4_P0_IDS = frozenset( + { + "updates.event-message-deleted", + "updates.event-message-edited", + "updates.event-new-channel-message", + "updates.event-new-message", + "updates.event-read-inbox", + "updates.event-read-outbox", + "updates.invoke-init-connection", + "updates.net-flood-wait", + "updates.ops-daemon-lifecycle", + "updates.ops-reconnect-health", + "updates.ops-single-updates-consumer", + "updates.session-persistence", + "updates.stream-event-types", + "updates.stream-raw-passthrough", + "updates.stream-watch-ndjson", + "updates.sync-catch-up-on-start", + "updates.sync-get-channel-difference", + "updates.sync-get-difference", + "updates.sync-pts-gap-algorithm", + "updates.sync-state-persistence", + "updates.sync-too-long", + } +) + #: `(group prefixes, the P0 ids those groups claim)` for each landed PR. P0_OWNERS = ( (("message.", "draft."), PR1_P0_IDS), (("auth.", "account.", "passport."), PR2_P0_IDS), (("chat.", "folder."), PR3_P0_IDS), + (PR4_GROUPS, PR4_P0_IDS), (("media.", "sticker.", "gif.", "emoji."), PR6_P0_IDS), (("poll.", "reaction.", "todo.", "location.", "search."), PR9_P0_IDS), (("call.", "vc.", "conference."), PR11_P0_IDS), @@ -255,15 +296,15 @@ def test_every_p0_id_this_pr_owns_is_covered(self): assert missing == [], f"a landed PR dropped coverage of {missing}" @pytest.mark.parametrize( - "prefixes,expected", P0_OWNERS, ids=["pr1", "pr2", "pr3", "pr6", "pr9", "pr11"] + "prefixes,expected", P0_OWNERS, ids=["pr1", "pr2", "pr3", "pr4", "pr6", "pr9", "pr11"] ) def test_the_floor_is_the_whole_truth(self, prefixes, expected): """Each named list is exactly the P0 set its own groups claim. A floor that is a subset is a floor with holes in it: an op could drop a P0 id nobody wrote down and the gate would stay green. Computing the - set here and comparing it to the literal above means new coverage has - to be added to the list on purpose. + sets here and comparing them to the literals above means new coverage + has to be added to a list on purpose. """ catalogue = catalog() actual = { @@ -308,6 +349,12 @@ def test_calls_voicechats_is_fully_accounted_for(self, report): assert stats["accounted_percent"] == 100.0 assert stats["covered"] >= 124 + def test_the_updates_domain_is_fully_accounted_for(self, report): + """PR-4's own domain: implemented, or waived to a named later PR.""" + stats = report.by_domain["updates_sync_network"] + assert stats["accounted_percent"] == 100.0 + assert stats["covered"] >= 186 + def test_messages_core_is_fully_accounted_for(self, report): """PR-1's own domain: implemented, or waived to a named later PR.""" stats = report.by_domain["messages_core"] diff --git a/tests/test_security.py b/tests/test_security.py index 0d24157..dbca249 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -96,6 +96,47 @@ def test_the_accounts_directory_property_is_a_read(self, tlgr_home: Path): assert not (fresh / "accounts").exists() +class TestProductionHome: + """A marked home belongs to somebody's running daemon. Keep out. + + Two processes on one tlgr home share session files, and Telegram treats a + second client on the same auth key as a compromised session and revokes + it — so a development build started against a live home does not degrade + it, it breaks it. The marker is a hard stop with one deliberate override. + """ + + def test_an_unmarked_home_is_fine(self, tlgr_home: Path): + from tlgr.core.paths import is_production_home, refuse_production_home + + assert is_production_home(tlgr_home) is False + refuse_production_home(tlgr_home) + + def test_a_marked_home_is_refused_and_says_how_to_proceed(self, tlgr_home: Path): + from tlgr.core.errors import ConfigurationError + from tlgr.core.paths import PRODUCTION_MARKER, refuse_production_home + + (tlgr_home / PRODUCTION_MARKER).write_text("live install\n") + with pytest.raises(ConfigurationError) as caught: + refuse_production_home(tlgr_home) + assert "TLGR_ALLOW_PRODUCTION_HOME" in str(caught.value) + + def test_the_override_is_an_explicit_environment_variable(self, tlgr_home: Path, monkeypatch): + from tlgr.core.paths import PRODUCTION_MARKER, refuse_production_home + + (tlgr_home / PRODUCTION_MARKER).write_text("live install\n") + monkeypatch.setenv("TLGR_ALLOW_PRODUCTION_HOME", "1") + refuse_production_home(tlgr_home) + + def test_the_daemon_refuses_to_start_on_one(self, tlgr_home: Path): + """Exit 10 (CONFIG), before the lock and before anything is created.""" + from tlgr.core.paths import PRODUCTION_MARKER + from tlgr.daemon.main import main + + (tlgr_home / PRODUCTION_MARKER).write_text("live install\n") + assert main(["--base", str(tlgr_home), "--foreground"]) == 10 + assert not (tlgr_home / "daemon.lock").exists() + + class TestPrivateWrites: def test_write_private_is_never_briefly_world_readable(self, tlgr_home: Path): target = tlgr_home / "secret.json" @@ -291,14 +332,13 @@ async def test_the_access_log_is_off(live_daemon): assert logging.getLogger("aiohttp.access").disabled is True -def test_config_init_writes_private_files(tlgr_home: Path, monkeypatch): +def test_config_init_writes_private_files(tlgr_home: Path): """SEC-07: `webhook.toml` holds a token and v1 created it 0644.""" from click.testing import CliRunner - from tlgr.cli.legacy import config_cmd + from tlgr.cli import cli - monkeypatch.setattr(config_cmd, "CONFIG_DIR", tlgr_home) - result = CliRunner().invoke(config_cmd.config_group, ["init"], obj={"fmt": "json"}) + result = CliRunner().invoke(cli, ["--json", "config", "init"]) assert result.exit_code == 0, result.output for name in ("config.toml", "jobs.yaml", "webhook.toml"): assert stat.S_IMODE((tlgr_home / name).stat().st_mode) == 0o600, name diff --git a/tlgr/cli/__init__.py b/tlgr/cli/__init__.py index 3dfa902..c7d9218 100644 --- a/tlgr/cli/__init__.py +++ b/tlgr/cli/__init__.py @@ -231,21 +231,13 @@ def cli( from tlgr.cli.gen import build_click_tree # noqa: E402 from tlgr.cli.legacy.chat import chat_create, chat_members # noqa: E402 -from tlgr.cli.legacy.config_cmd import config_group # noqa: E402 from tlgr.cli.legacy.contact import contact_group # noqa: E402 -from tlgr.cli.legacy.daemon_cmd import daemon_group # noqa: E402 -from tlgr.cli.legacy.job import job_group # noqa: E402 from tlgr.cli.legacy.profile import profile_group # noqa: E402 from tlgr.cli.legacy.user import user_group # noqa: E402 -from tlgr.cli.legacy.watch import watch_command # noqa: E402 cli.add_command(contact_group, "contact") cli.add_command(profile_group, "profile") -cli.add_command(daemon_group, "daemon") -cli.add_command(job_group, "job") -cli.add_command(config_group, "config") cli.add_command(user_group, "user") -cli.add_command(watch_command, "watch") # --------------------------------------------------------------------------- @@ -253,13 +245,6 @@ def cli( # --------------------------------------------------------------------------- -@cli.command("status") -@click.pass_context -def shortcut_status(ctx: click.Context) -> None: - """Show daemon status (shortcut for 'daemon status').""" - ctx.invoke(daemon_group.commands["status"]) - - @cli.command("contacts") @click.pass_context def shortcut_contacts(ctx: click.Context) -> None: @@ -273,10 +258,10 @@ def shortcut_contacts(ctx: click.Context) -> None: #: Commands that still live in `cli/legacy` *inside* a group the registry now -#: generates. Each entry is a promise to delete. PR-2 took `agent whoami` out -#: of it: the command needs the account manager, so it migrated with the -#: account group. What is left is the sanctioned, enumerated overlap for the -#: group PRs still to come. +#: generates. Each entry is a promise to delete, and an enumerated list is +#: the only kind of overlap that is a decision rather than an accident. PR-2 +#: took `agent whoami` out of it, PR-4 took `daemon` and `job`; what is left +#: is the sanctioned overlap for the group PRs still to come. LEGACY_EXTRAS: dict[str, list[click.Command]] = { # `chat create` and `chat members` are member/admin operations and # migrate with the groups-and-channels group (PR-7). @@ -287,34 +272,48 @@ def shortcut_contacts(ctx: click.Context) -> None: def build_cli() -> click.Group: """Compose the generated command tree with the v1 groups still hand-written. - A group must be defined in exactly one of the two places. Being defined in - both would mean a migration half-landed — one path generated, one path - still hand-written, silently disagreeing — so it fails the import rather - than the user's next command (§12.4). The one sanctioned overlap is - LEGACY_EXTRAS, which is an explicit, enumerated list rather than an - accident. + A *command* must be defined in exactly one of the two places. Being + defined in both would mean a migration half-landed — one path generated, + one still hand-written, silently disagreeing — so it fails the import + rather than the user's next command (§12.4). + + A *group* may legitimately be shared while a migration is in flight, in + both directions: LEGACY_EXTRAS puts a v1 command inside a generated group + (`agent whoami` until PR-2 moves it), and merging puts a generated command + inside a v1 group (`account status`, whose group migrates in PR-2). Both + are enumerated by the code that does the merging, and a name that appears + twice is still a hard failure. """ import tlgr.ops # noqa: F401 — importing it is what populates the registry from tlgr.cli.gen import set_dispatcher - from tlgr.transport import make_dispatcher + from tlgr.transport import make_dispatcher, make_stream_dispatcher # Installing the transport here, rather than importing it in `gen.py`, is # what keeps `cli/gen.py` testable with a fake dispatcher and keeps the # daemon out of the CLI's import graph. - set_dispatcher(make_dispatcher()) + set_dispatcher(make_dispatcher(), make_stream_dispatcher()) generated = build_click_tree() - clash = sorted(set(generated) & set(cli.commands)) - if clash: - raise RuntimeError( - f"these command groups are defined both by the registry and by " - f"tlgr/cli/legacy: {clash}. Delete the legacy module." - ) for name, command in generated.items(): for extra in LEGACY_EXTRAS.get(name, []): if isinstance(command, click.Group): command.add_command(extra, extra.name) - cli.add_command(command, name) + existing = cli.commands.get(name) + if existing is None: + cli.add_command(command, name) + continue + if not (isinstance(existing, click.Group) and isinstance(command, click.Group)): + raise RuntimeError( + f"the command {name!r} is defined both by the registry and by " + f"tlgr/cli/legacy. Delete the legacy module." + ) + for sub_name, sub in command.commands.items(): + if sub_name in existing.commands: + raise RuntimeError( + f"{name} {sub_name} is defined both by the registry and by " + f"tlgr/cli/legacy. Delete the legacy command." + ) + existing.add_command(sub, sub_name) return cli diff --git a/tlgr/cli/gen.py b/tlgr/cli/gen.py index d83fd96..33eb7a1 100644 --- a/tlgr/cli/gen.py +++ b/tlgr/cli/gen.py @@ -14,7 +14,7 @@ import types import typing import uuid -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass, field from typing import Any, Literal @@ -44,6 +44,7 @@ "LocalContext", "build_click_tree", "build_command", + "run_live", "run_op", "set_dispatcher", ] @@ -63,6 +64,12 @@ class LocalContext: request_id: str = "" warnings: list[str] = field(default_factory=list) command_tree: Callable[[Sequence[str], bool], dict[str, Any] | None] | None = None + #: `--limit`/`--cursor`/`--all` are transport-level and never request + #: fields (registry lint L5), so a local paginated operation reads them + #: off the context exactly as a daemon-side one does. + limit: int | None = None + cursor: str | None = None + fetch_all: bool = False def warn(self, message: str) -> None: self.warnings.append(message) @@ -75,18 +82,44 @@ def mark_already(self) -> None: Dispatcher = Callable[[OperationSpec, msgspec.Struct, CliState], dict[str, Any]] +StreamDispatcher = Callable[[OperationSpec, msgspec.Struct, CliState], Iterator[dict[str, Any]]] _dispatch: Dispatcher | None = None +_stream_dispatch: StreamDispatcher | None = None -def set_dispatcher(dispatcher: Dispatcher | None) -> None: +def set_dispatcher( + dispatcher: Dispatcher | None, stream_dispatcher: StreamDispatcher | None = None +) -> None: """Install the daemon transport. Stage A registers only local operations; the daemon surface arrives with the transport, and until then asking for it is a daemon error rather than a traceback. + + The stream dispatcher is separate because a live stream cannot be folded + into an envelope: `watch` that only prints when it ends is not a watch. """ - global _dispatch + global _dispatch, _stream_dispatch _dispatch = dispatcher + _stream_dispatch = stream_dispatcher + + +def run_live(spec: OperationSpec, request: msgspec.Struct, state: CliState) -> int: + """Drive a `live-stream` operation, printing frames as they arrive.""" + if state.enable_commands and not policy_allows(state.enable_commands, spec.id): + raise PermissionError_( + f"operation {spec.id!r} is not enabled (add it to --enable-commands to allow it)", + ) + if _stream_dispatch is None: + raise DaemonError(f"{spec.id} streams from the daemon, and no transport is wired up") + state.account = resolve_account(state) if spec.needs_account else state.account + frames = _stream_dispatch(spec, request, state) + return renderer.render_stream( + frames, + fmt=state.fmt, + results_only=state.results_only, + select=state.select, + ) def _command_tree(path: Sequence[str], include_hidden: bool) -> dict[str, Any] | None: @@ -131,15 +164,35 @@ def run_op(spec: OperationSpec, request: msgspec.Struct, state: CliState) -> dic return _dispatch(spec, request, state) context = LocalContext( - account=account, dry_run=state.dry_run, request_id=request_id, command_tree=_command_tree + account=account, + dry_run=state.dry_run, + request_id=request_id, + command_tree=_command_tree, + limit=state.limit, + cursor=state.cursor, + fetch_all=state.fetch_all, ) result = asyncio.run(spec.impl(context, request)) + body = msgspec.to_builtins(result) envelope: dict[str, Any] = { "ok": True, "op": spec.id, - "result": msgspec.to_builtins(result), + "result": body, "meta": {"request_id": request_id, "warnings": context.warnings}, } + if spec.paginated is not None and isinstance(body, dict): + # The same projection the daemon does (`daemon/dispatch._envelope`). + # Without it a paginated *local* operation would hand back the page + # object where every other one hands back the items, and `--select` + # would need a different path depending on where the op happened to + # run — which is exactly the kind of difference the registry exists to + # remove. + envelope["result"] = body.get("items", []) + envelope["page"] = { + "has_more": bool(body.get("has_more")), + "next_cursor": body.get("next_cursor"), + "total": body.get("total"), + } if account: envelope["account"] = account return envelope @@ -461,6 +514,10 @@ def callback(**values: Any) -> None: hint=f"pass --yes to confirm {spec.id}", ) request = _build_request(spec, fields, values) + if "live-stream" in spec.tags: + # No envelope: the frames *are* the output, and they arrive over + # minutes or hours. + ctx.exit(run_live(spec, request, state)) envelope = run_op(spec, request, state) # A schema document is JSON whether or not anybody asked: there is no # table shape for it, and v1 printed JSON here unconditionally. When diff --git a/tlgr/cli/legacy/config_cmd.py b/tlgr/cli/legacy/config_cmd.py deleted file mode 100644 index f066570..0000000 --- a/tlgr/cli/legacy/config_cmd.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Config management commands — init, validate, path, get, set, list, unset.""" - -from __future__ import annotations - -import sys -from typing import Any - -import click - -from tlgr.core.config import ( - CONFIG_DIR, - _load_toml, - _save_toml, - load_app_config, - load_webhook_config, -) -from tlgr.core.output import emit -from tlgr.gateway.config import load_gateway_configs - -if sys.version_info >= (3, 11): - import tomllib -else: - try: - import tomllib - except ModuleNotFoundError: - pass # type: ignore[no-redef] - -_CONFIG_FILE = CONFIG_DIR / "config.toml" - -# Documented config keys with their TOML section + key + description. -_KNOWN_KEYS: dict[str, tuple[str, str, str]] = { - "output": ("defaults", "output", "Default output mode: human | json | plain"), - "drop_author": ("defaults", "drop_author", "Strip author on forwarded messages"), - "delete_after": ("defaults", "delete_after", "Delete source after forwarding"), - "default_account": ("accounts", "default", "Default account alias"), - "require_account": ( - "defaults", - "require_account", - "Require -a on every command (no default-account fallback)", - ), - "auto_start": ("daemon", "auto_start", "Auto-start daemon on CLI use"), - "log_level": ("daemon", "log_level", "Daemon log level: debug | info | warning | error"), - "idle_timeout": ("daemon", "idle_timeout", "Seconds before idle daemon auto-stops (0 = never)"), - "flood_wait_max": ( - "daemon", - "flood_wait_max", - "Max seconds to auto-sleep on Telegram rate limit", - ), -} - - -def _coerce_value(raw: str) -> Any: - """Best-effort coerce a CLI string to a native TOML type.""" - low = raw.lower() - if low in ("true", "yes"): - return True - if low in ("false", "no"): - return False - try: - return int(raw) - except ValueError: - pass - try: - return float(raw) - except ValueError: - pass - return raw - - -@click.group("config") -def config_group() -> None: - """Manage configuration files.""" - - -@config_group.command("init") -@click.pass_context -def config_init(ctx: click.Context) -> None: - """Create default configuration files.""" - # SEC-07: every one of these is written 0600, through the one writer that - # chmods before it renames. `webhook.toml` holds a token, and v1 created - # all three world-readable with `write_text`. - from tlgr.core.paths import TlgrPaths, write_private - - TlgrPaths(CONFIG_DIR).ensure_base() - created = [] - - config_path = CONFIG_DIR / "config.toml" - if not config_path.exists(): - write_private( - config_path, - '[defaults]\ndrop_author = false\ndelete_after = false\noutput = "human"\n\n' - '[accounts]\ndefault = ""\n\n' - "[daemon]\nauto_start = true\n" - 'log_level = "info"\n', - ) - created.append("config.toml") - - jobs_path = CONFIG_DIR / "jobs.yaml" - if not jobs_path.exists(): - write_private( - jobs_path, - "# Gateway jobs configuration\n" - "# See https://github.com/tlgrcli/tlgr for full reference.\n" - "#\n" - "# jobs:\n" - "# - name: example\n" - "# account: main\n" - "# filters:\n" - "# chat_type: private\n" - "# actions:\n" - '# - reply: "hello!"\n', - ) - created.append("jobs.yaml") - - webhook_path = CONFIG_DIR / "webhook.toml" - if not webhook_path.exists(): - write_private( - webhook_path, - "[webhook]\nenabled = false\n" - 'url = ""\ntoken = ""\n' - 'events = ["new_message"]\n\n' - "[webhook.retry]\nenabled = true\nmax_attempts = 3\nbackoff_base = 2\n\n" - "[webhook.filters]\nchats = []\n", - ) - created.append("webhook.toml") - - if created: - emit(ctx.obj, {"created": created, "path": str(CONFIG_DIR)}) - else: - emit(ctx.obj, {"message": "All config files already exist", "path": str(CONFIG_DIR)}) - - -@config_group.command("validate") -@click.pass_context -def config_validate(ctx: click.Context) -> None: - """Validate configuration files.""" - errors = [] - - try: - load_app_config() - except Exception as e: - errors.append(f"config.toml: {e}") - - try: - configs = load_gateway_configs() - for cfg in configs: - if not cfg.name: - errors.append("jobs.yaml: job missing 'name' field") - if not cfg.actions: - errors.append(f"jobs.yaml: job '{cfg.name}' has no actions") - for ac in cfg.actions: - from tlgr.actions import get_action - - if get_action(ac.name) is None: - errors.append(f"jobs.yaml: job '{cfg.name}' has unknown action '{ac.name}'") - except Exception as e: - errors.append(f"jobs.yaml: {e}") - - try: - load_webhook_config() - except Exception as e: - errors.append(f"webhook.toml: {e}") - - if errors: - emit(ctx.obj, {"valid": False, "errors": errors}) - sys.exit(1) - else: - emit(ctx.obj, {"valid": True, "files": ["config.toml", "jobs.yaml", "webhook.toml"]}) - - -@config_group.command("path") -@click.pass_context -def config_path(ctx: click.Context) -> None: - """Print the configuration directory path.""" - emit(ctx.obj or {}, {"path": str(CONFIG_DIR)}, columns=["path"]) - - -@config_group.command("keys") -@click.pass_context -def config_keys(ctx: click.Context) -> None: - """List all known configuration keys.""" - obj = ctx.obj or {} - fmt = obj.get("fmt", "human") - if fmt == "json": - rows = { - k: {"section": sec, "key": key, "description": desc} - for k, (sec, key, desc) in _KNOWN_KEYS.items() - } - emit(obj, {"keys": rows}) - else: - rows = [ - {"key": k, "section": sec, "description": desc} - for k, (sec, _, desc) in _KNOWN_KEYS.items() - ] - emit(obj, rows, columns=["key", "section", "description"]) - - -@config_group.command("list") -@click.pass_context -def config_list(ctx: click.Context) -> None: - """List all config values.""" - obj = ctx.obj or {} - fmt = obj.get("fmt", "human") - raw = _load_toml(_CONFIG_FILE) - if fmt == "json": - emit(obj, raw) - else: - rows = [] - for key_name, (section, field, _desc) in _KNOWN_KEYS.items(): - val = raw.get(section, {}).get(field, "") - rows.append({"key": key_name, "value": str(val)}) - emit(obj, rows, columns=["key", "value"]) - - -@config_group.command("get") -@click.argument("key") -@click.pass_context -def config_get(ctx: click.Context, key: str) -> None: - """Get a config value by key.""" - if key not in _KNOWN_KEYS: - click.echo(f"Error: unknown config key {key!r}. Run: tlgr config keys", err=True) - sys.exit(2) - section, field, _ = _KNOWN_KEYS[key] - raw = _load_toml(_CONFIG_FILE) - val = raw.get(section, {}).get(field) - emit(ctx.obj or {}, {"key": key, "value": val}, columns=["key", "value"]) - - -@config_group.command("set") -@click.argument("key") -@click.argument("value") -@click.pass_context -def config_set(ctx: click.Context, key: str, value: str) -> None: - """Set a config value.""" - if key not in _KNOWN_KEYS: - click.echo(f"Error: unknown config key {key!r}. Run: tlgr config keys", err=True) - sys.exit(2) - section, field, _ = _KNOWN_KEYS[key] - raw = _load_toml(_CONFIG_FILE) - if section not in raw: - raw[section] = {} - raw[section][field] = _coerce_value(value) - _save_toml(_CONFIG_FILE, raw) - emit(ctx.obj or {}, {"key": key, "value": raw[section][field], "updated": True}) - - -@config_group.command("unset") -@click.argument("key") -@click.pass_context -def config_unset(ctx: click.Context, key: str) -> None: - """Remove a config key (reset to default).""" - if key not in _KNOWN_KEYS: - click.echo(f"Error: unknown config key {key!r}. Run: tlgr config keys", err=True) - sys.exit(2) - section, field, _ = _KNOWN_KEYS[key] - raw = _load_toml(_CONFIG_FILE) - removed = False - if section in raw and field in raw[section]: - del raw[section][field] - if not raw[section]: - del raw[section] - removed = True - _save_toml(_CONFIG_FILE, raw) - emit(ctx.obj or {}, {"key": key, "removed": removed}) diff --git a/tlgr/cli/legacy/daemon_cmd.py b/tlgr/cli/legacy/daemon_cmd.py deleted file mode 100644 index 67118b5..0000000 --- a/tlgr/cli/legacy/daemon_cmd.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Daemon lifecycle commands. - -Still a v1 module — it migrates to the registry in PR-4 — but three things -about it were wrong and are fixed here because they are what an operator uses -when something is broken: - -* `start` waited for the **socket file** to appear. The socket exists from - `bind()`, before any account has connected and before the daemon can serve - anything, so "started" meant "a process got as far as binding" (ROB-07). - It now waits for an HTTP 200 from `/v1/status`. -* `status` reported `running: true` for any live pid. It now merges the v2 - status, so `ready`, `version` and `protocol` are visible and a daemon that - is alive but unable to work is distinguishable from a healthy one - (COR-37, COR-38). -* `install` was macOS-only. Linux gets a systemd user unit. -""" - -from __future__ import annotations - -import os -import platform -import subprocess -import sys -import time - -import click - -from tlgr.core.config import CONFIG_DIR, get_logs_dir, get_pid_path -from tlgr.core.output import emit -from tlgr.daemon.lifecycle import read_pid, stop_daemon - - -def _wait_ready(timeout: float = 30.0) -> dict | None: - """Poll `/v1/status` until the daemon answers, or give up. - - Readiness is a reply, not a file: see the module docstring. - """ - from tlgr.transport.autostart import wait_ready - from tlgr.transport.client import DaemonClient - - client = DaemonClient(CONFIG_DIR, auto_start=False) - return wait_ready(client.probe_status, timeout=timeout) - - -def _spawn() -> subprocess.Popen: - return subprocess.Popen( - [sys.executable, "-m", "tlgr.daemon.main", "--base", str(CONFIG_DIR)], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True, - ) - - -@click.group("daemon") -def daemon_group() -> None: - """Manage the tlgr daemon.""" - - -@daemon_group.command("start") -@click.option("--foreground", is_flag=True, help="Run in foreground (don't fork).") -@click.pass_context -def daemon_start(ctx: click.Context, foreground: bool) -> None: - """Start the daemon (forks to background by default).""" - existing = read_pid() - if existing: - click.echo(f"Daemon already running (pid={existing})", err=True) - sys.exit(1) - - if foreground: - from tlgr.daemon.main import main as daemon_main - - sys.exit(daemon_main(["--base", str(CONFIG_DIR), "--foreground"])) - - proc = _spawn() - status = _wait_ready() - if status is None: - click.echo("Daemon did not become ready within 30 seconds", err=True) - sys.exit(11) - emit( - ctx.obj, - { - "started": True, - "pid": status.get("daemon", {}).get("pid") or read_pid() or proc.pid, - "ready": status.get("daemon", {}).get("ready", False), - }, - ) - - -@daemon_group.command("stop") -@click.pass_context -def daemon_stop(ctx: click.Context) -> None: - """Stop the daemon.""" - if stop_daemon(): - for _ in range(20): - time.sleep(0.25) - if not get_pid_path().exists(): - break - emit(ctx.obj, {"stopped": True}) - else: - click.echo("Daemon is not running", err=True) - sys.exit(1) - - -@daemon_group.command("restart") -@click.pass_context -def daemon_restart(ctx: click.Context) -> None: - """Restart the daemon.""" - if read_pid(): - stop_daemon() - for _ in range(20): - time.sleep(0.25) - if not get_pid_path().exists(): - break - - proc = _spawn() - status = _wait_ready() - if status is None: - click.echo("Daemon did not become ready within 30 seconds", err=True) - sys.exit(11) - emit( - ctx.obj, - { - "restarted": True, - "pid": status.get("daemon", {}).get("pid") or read_pid() or proc.pid, - }, - ) - - -def _supervisor(choice: str) -> str: - """Which supervisor to use. `auto` follows the platform.""" - if choice != "auto": - return choice - return "launchd" if platform.system() == "Darwin" else "systemd" - - -@daemon_group.command("install") -@click.option("--force", is_flag=True, help="Reinstall even if already installed.") -@click.option( - "--supervisor", - type=click.Choice(["auto", "launchd", "systemd"]), - default="auto", - help="Which service manager to install into.", -) -@click.pass_context -def daemon_install(ctx: click.Context, force: bool, supervisor: str) -> None: - """Install as a user service (auto-start on login, restart on crash). - - macOS gets a LaunchAgent, Linux a systemd **user** unit — user, because - the daemon holds session files under $HOME and must run as their owner. - Both force `idle_timeout` to 0: under a supervisor, a clean idle exit is - either a respawn loop or a daemon that never comes back (COR-39). - """ - kind = _supervisor(supervisor) - if kind == "launchd": - from tlgr.daemon.launchd import install, is_installed - - if is_installed() and not force: - click.echo("Service already installed. Use --force to reinstall.", err=True) - sys.exit(1) - path = install(CONFIG_DIR, get_logs_dir()) - else: - from tlgr.daemon.systemd import install as install_unit - from tlgr.daemon.systemd import is_installed - - if is_installed() and not force: - click.echo("Service already installed. Use --force to reinstall.", err=True) - sys.exit(1) - path = install_unit(CONFIG_DIR) - emit(ctx.obj, {"installed": True, "supervisor": kind, "path": str(path)}) - - -@daemon_group.command("uninstall") -@click.pass_context -def daemon_uninstall(ctx: click.Context) -> None: - """Remove the user service (stop auto-start on login).""" - from tlgr.daemon import launchd, systemd - - removed = launchd.uninstall() if platform.system() == "Darwin" else False - removed = systemd.uninstall() or removed - if removed: - emit(ctx.obj, {"uninstalled": True}) - else: - click.echo("Service is not installed.", err=True) - sys.exit(1) - - -@daemon_group.command("status") -@click.pass_context -def daemon_status(ctx: click.Context) -> None: - """Show daemon status.""" - pid = read_pid() - if not pid: - emit(ctx.obj, {"running": False, "ready": False}, columns=["running", "ready"]) - return - try: - from tlgr.ipc_client import ipc_request - - result = ipc_request("GET", "/daemon/status") - # `running` has always meant "a process is alive". `ready` is the - # question people were actually asking (COR-37): a daemon that is - # alive but cannot reach Telegram is not a working daemon. - v2 = _wait_ready(timeout=2.0) or {} - daemon = v2.get("daemon", {}) - result.setdefault("ready", daemon.get("ready", False)) - result.setdefault("version", daemon.get("version")) - result.setdefault("protocol", daemon.get("protocol")) - result.setdefault("managed_by", daemon.get("managed_by")) - emit( - ctx.obj, - result, - columns=[ - "running", - "ready", - "pid", - "uptime_seconds", - "accounts", - "healthy", - "disconnected", - ], - ) - except Exception: - emit( - ctx.obj, - {"running": True, "ready": False, "pid": pid, "uptime_seconds": "?", "accounts": "?"}, - ) - - -@daemon_group.command("logs") -@click.option("--follow", "-f", is_flag=True, help="Follow log output.") -@click.option("--lines", "-n", type=int, default=50, help="Number of lines to show.") -def daemon_logs(follow: bool, lines: int) -> None: - """View daemon logs.""" - log_file = get_logs_dir() / "daemon.log" - if not log_file.exists(): - click.echo("No log file found", err=True) - sys.exit(1) - - if follow: - os.execlp("tail", "tail", "-f", "-n", str(lines), str(log_file)) - else: - os.execlp("tail", "tail", "-n", str(lines), str(log_file)) diff --git a/tlgr/cli/legacy/job.py b/tlgr/cli/legacy/job.py deleted file mode 100644 index cee45cf..0000000 --- a/tlgr/cli/legacy/job.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Job management commands.""" - -from __future__ import annotations - -import os - -import click - -from tlgr.core.config import CONFIG_DIR -from tlgr.core.output import emit -from tlgr.ipc_client import ipc_request - - -@click.group("job") -def job_group() -> None: - """Manage background jobs.""" - - -@job_group.command("list") -@click.pass_context -def job_list(ctx: click.Context) -> None: - """List configured jobs and their status.""" - result = ipc_request("GET", "/job/list") - fmt = ctx.obj.get("fmt", "human") - if fmt == "json": - emit(ctx.obj, result) - else: - emit(ctx.obj, result.get("jobs", []), columns=["name", "type", "enabled", "running"]) - - -@job_group.command("add") -def job_add() -> None: - """Open jobs.yaml in $EDITOR to add a job.""" - jobs_path = CONFIG_DIR / "jobs.yaml" - if not jobs_path.exists(): - jobs_path.parent.mkdir(parents=True, exist_ok=True) - jobs_path.write_text( - "# Gateway jobs configuration — see docs for full reference.\n" - "#\n" - "# jobs:\n" - "# - name: my-job\n" - "# account: main\n" - "# filters:\n" - "# chat_type: private\n" - "# actions:\n" - '# - reply: "hello!"\n' - ) - editor = os.environ.get("EDITOR", "vi") - os.execlp(editor, editor, str(jobs_path)) - - -@job_group.command("remove") -@click.argument("name") -@click.pass_context -def job_remove(ctx: click.Context, name: str) -> None: - """Remove a job by name.""" - result = ipc_request("POST", "/job/remove", body={"name": name}) - emit(ctx.obj, result) - - -@job_group.command("enable") -@click.argument("name") -@click.pass_context -def job_enable(ctx: click.Context, name: str) -> None: - """Enable a disabled job.""" - result = ipc_request("POST", "/job/enable", body={"name": name}) - emit(ctx.obj, result) - - -@job_group.command("disable") -@click.argument("name") -@click.pass_context -def job_disable(ctx: click.Context, name: str) -> None: - """Disable a job without removing it.""" - result = ipc_request("POST", "/job/disable", body={"name": name}) - emit(ctx.obj, result) - - -@job_group.command("reload") -@click.pass_context -def job_reload(ctx: click.Context) -> None: - """Hot-reload jobs from jobs.yaml without restarting the daemon.""" - result = ipc_request("POST", "/job/reload") - emit(ctx.obj, result) diff --git a/tlgr/cli/legacy/watch.py b/tlgr/cli/legacy/watch.py deleted file mode 100644 index 4d17b7b..0000000 --- a/tlgr/cli/legacy/watch.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Streaming event watch command.""" - -from __future__ import annotations - -import json -import sys -import time - -import click - -from tlgr.cli.legacy._common import resolve_account -from tlgr.ipc_client import ipc_request - - -@click.command("watch") -@click.option("--chat", "chats", multiple=True, help="Chat(s) to watch (default: all).") -@click.option("--events", default="new_message", help="Comma-separated event types.") -@click.option("--account", "-a", default=None) -@click.pass_context -def watch_command( - ctx: click.Context, chats: tuple[str, ...], events: str, account: str | None -) -> None: - """Stream events as newline-delimited JSON. Ctrl+C to stop. - - Polls the daemon for new messages and emits one JSON object per line. - """ - acct = resolve_account(ctx, account) - event_types = {e.strip() for e in events.split(",") if e.strip()} - chat_set = set(chats) if chats else None - - last_ids: dict[str, int] = {} - poll_interval = 2.0 - - try: - while True: - if chat_set: - target_chats = list(chat_set) - else: - try: - result = ipc_request( - "GET", "/chat/list", params={"account": acct, "limit": 50} - ) - target_chats = [str(c["id"]) for c in result.get("chats", [])[:20]] - except Exception: - target_chats = [] - - for chat_ref in target_chats: - if "new_message" not in event_types: - continue - try: - offset_id = last_ids.get(chat_ref, 0) - params: dict[str, object] = { - "chat": chat_ref, - "limit": 10, - "account": acct, - } - if offset_id: - params["min_id"] = offset_id - result = ipc_request("GET", "/message/list", params=params) - msgs = result.get("messages", []) - for msg in reversed(msgs): - msg_id = msg.get("id", 0) - if msg_id <= last_ids.get(chat_ref, 0): - continue - event = { - "event_type": "new_message", - "chat_id": chat_ref, - "data": msg, - } - json.dump(event, sys.stdout, default=str, ensure_ascii=False) - sys.stdout.write("\n") - sys.stdout.flush() - last_ids[chat_ref] = max(last_ids.get(chat_ref, 0), msg_id) - except Exception: - pass - - time.sleep(poll_interval) - except KeyboardInterrupt: - pass diff --git a/tlgr/cli/render.py b/tlgr/cli/render.py index cb2c99f..5996662 100644 --- a/tlgr/cli/render.py +++ b/tlgr/cli/render.py @@ -14,6 +14,7 @@ from __future__ import annotations +import contextlib import json import os import re @@ -34,6 +35,7 @@ "render_human", "render_json", "render_plain", + "render_stream", "results_payload", ] @@ -354,3 +356,63 @@ def render( no_header=no_header, stream=stream, ) + + +#: Frames that describe the *stream* rather than something that happened. A +#: consumer filtering on the event type has to be able to tell them apart, +#: which is why they are named here rather than recognised by their absence. +CONTROL_FRAMES = frozenset( + {"meta", "end", "heartbeat", "lag", "gap", "watching", "cursor", "page", "item"} +) + + +def render_stream( + frames: Iterable[dict[str, Any]], + *, + fmt: str = "human", + results_only: bool = False, + select: str | None = None, + stream: Any = None, +) -> int: + """Print an NDJSON stream as it arrives, and return the exit code. + + Flushed per frame, deliberately: a `watch` piped into `jq` that only + appears once a 4 KB buffer fills is indistinguishable from a daemon that + is not delivering anything. + + `--results-only` restores v1's line shape — `{"event_type", "chat_id", + "data"}` — and drops the control frames, because that is what a script + written against v1's `tlgr watch` already parses (§12.4). + """ + out = stream or sys.stdout + fields = _split_fields(select) + exit_code = 0 + for frame in frames: + kind = str(frame.get("type", "")) + if kind == "end" and not frame.get("ok", True): + from tlgr.core.errors import EXIT_RETRYABLE + + body = frame.get("error") or {} + exit_code = int(body.get("exit_code", EXIT_RETRYABLE)) + click.echo(f"Error: {body.get('message') or 'the stream ended'}", err=True) + continue + if results_only: + if kind in CONTROL_FRAMES: + continue + frame = { + "event_type": kind, + "chat_id": frame.get("chat_id"), + "data": frame.get("payload", {}), + "seq": frame.get("seq"), + "account": frame.get("account"), + } + if fields: + frame = project(frame, fields) + _write_frame(frame, out) + return exit_code + + +def _write_frame(frame: Any, out: Any) -> None: + out.write(json.dumps(frame, ensure_ascii=False, default=str) + "\n") + with contextlib.suppress(AttributeError, ValueError, OSError): # a closed pipe + out.flush() diff --git a/tlgr/core/eventtypes.py b/tlgr/core/eventtypes.py new file mode 100644 index 0000000..7b7c8d2 --- /dev/null +++ b/tlgr/core/eventtypes.py @@ -0,0 +1,1170 @@ +"""The event taxonomy: every `Update*` constructor, named or explained. + +This is the table `docs/design/EVENTS.md` documents, `tlgr events list` prints, +`tlgr watch --events` selects from and `tlgr/daemon/events.py` normalises +against. It lives in `core/` because all three of `ops/`, `daemon/` and the +doc generator need it and none of them may import each other (§2.2). + +Two rules make the table worth trusting, and both are asserted by +`tests/test_event_taxonomy.py`: + +* **every constructor is accounted for.** A constructor is either mapped to a + tlgr event type or listed in `INTERNAL` with the reason it is not surfaced. + A constructor that is merely missing would be an update tlgr silently drops + with nobody able to tell — which is exactly what v1's polling watch did to + everything that was not a new message. +* **no type is invented.** A name here is a name a consumer can filter on + forever. An update whose meaning we have not worked out is `INTERNAL`, not + a type called `unknown`. + +The table is written against Telethon 1.44 (layer 227). Constructors Telegram +has added since carry `since_layer=229` and `available=False`: they are listed +so that `tlgr events list` can say "this exists and this build cannot parse +it", which is a far more useful answer than their absence. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, field + +__all__ = [ + "ALIASES", + "CONSTRUCTORS", + "GROUPS", + "INTERNAL", + "TYPES", + "EventTypeSpec", + "constructors_for", + "group_of", + "resolve_selectors", + "type_for_constructor", +] + +#: The families `--events` accepts as a shorthand, and `events list --group` +#: filters by. Derived from the first segment of a type name, with the two +#: presence types folded in by hand. +GROUPS: tuple[str, ...] = ( + "message", + "read", + "presence", + "peer", + "member", + "dialog", + "story", + "collection", + "call", + "bot", + "stars", + "secret", + "account", + "sync", +) + + +@dataclass(frozen=True, slots=True) +class EventTypeSpec: + """One tlgr event type: what it means and where it comes from.""" + + type: str + group: str + summary: str + #: field name → a short type description. Documented rather than a JSON + #: Schema because most payloads are the update's own fields, made + #: JSON-safe; `events get --json-schema` renders this into one. + payload: dict[str, str] = field(default_factory=dict) + #: Which sequence box orders it: pts, qts, seq, channel_pts, version or + #: none. A consumer that wants gap-free delivery needs to know. + box: str = "none" + bot_only: bool = False + #: 0 when Telethon 1.44 can parse every source constructor. + since_layer: int = 0 + telethon: str = "raw" + #: Set on the handful of types no constructor produces on its own: they + #: are derived by tlgr (a service message inside `updateNewMessage`) or + #: synthesised by the daemon (health, a revoked session from a push). + derived: str = "" + + +def _t( + type_: str, + group: str, + summary: str, + *, + payload: dict[str, str] | None = None, + box: str = "none", + bot_only: bool = False, + since_layer: int = 0, + telethon: str = "raw", + derived: str = "", +) -> EventTypeSpec: + return EventTypeSpec( + type=type_, + group=group, + summary=summary, + payload=payload or {}, + box=box, + bot_only=bot_only, + since_layer=since_layer, + telethon=telethon, + derived=derived, + ) + + +#: The payload every generically-normalised event carries: the update's own +#: fields, made JSON-safe (datetimes as RFC-3339, bytes as hex, nested TL +#: objects as `{"_": "ClassName", …}`). +_RAW_PAYLOAD = { + "_": "str — the source TL constructor name", + "…": "the update's own fields, JSON-safe", +} + +_TYPES: tuple[EventTypeSpec, ...] = ( + # -- message ---------------------------------------------------------- + _t( + "message_new", + "message", + "A message arrived in any chat the account can see", + payload={"message": "Message"}, + box="pts", + telethon="high-level: events.NewMessage", + ), + _t( + "message_service", + "message", + "A service message: a join, a pin, a title change, a call", + payload={"message": "Message", "action": "str — the MessageAction name"}, + box="pts", + telethon="high-level: events.ChatAction (subset)", + derived="updateNewMessage / updateNewChannelMessage carrying a messageService", + ), + _t( + "message_edited", + "message", + "A message was edited", + payload={"message": "Message — post-edit"}, + box="pts", + telethon="high-level: events.MessageEdited", + ), + _t( + "message_deleted", + "message", + "Messages were deleted", + payload={"message_ids": "list[int]", "channel_id": "int | null"}, + box="pts", + telethon="high-level: events.MessageDeleted", + ), + _t( + "message_id_assigned", + "message", + "An outgoing message got its server id (random_id reconciliation)", + payload={"msg_id": "int", "random_id": "int | null"}, + box="pts", + ), + _t( + "message_pinned", + "message", + "Messages were pinned or unpinned", + payload={"message_ids": "list[int]", "pinned": "bool"}, + box="pts", + ), + _t( + "message_views", + "message", + "A channel post's view counter moved", + payload={"msg_id": "int", "views": "int"}, + box="channel_pts", + ), + _t( + "message_forwards", + "message", + "A channel post's forward counter moved", + payload={"msg_id": "int", "forwards": "int"}, + box="channel_pts", + ), + _t( + "message_poll", + "message", + "A poll's results changed", + payload={"poll_id": "int", "poll": "object | null", "results": "object"}, + box="pts", + ), + _t( + "message_poll_vote", + "message", + "Somebody voted in a poll you can see the votes of", + payload={"poll_id": "int", "peer": "object", "options": "list[str]"}, + box="qts", + ), + _t( + "message_reactions", + "message", + "Reactions on a message changed", + payload={"msg_id": "int", "reactions": "object", "top_msg_id": "int | null"}, + box="pts", + ), + _t( + "message_extended_media", + "message", + "Paid media on a message was unlocked", + payload={"msg_id": "int", "extended_media": "list[object]"}, + box="pts", + ), + _t( + "message_transcribed", + "message", + "A voice or video note transcription finished", + payload={ + "msg_id": "int", + "transcription_id": "int", + "text": "str", + "pending": "bool", + }, + ), + _t( + "message_webpage", + "message", + "A link preview finished resolving", + payload={"webpage": "object"}, + box="pts", + ), + _t( + "message_scheduled_new", + "message", + "A scheduled message was queued", + payload={"message": "Message"}, + ), + _t( + "message_scheduled_deleted", + "message", + "A scheduled message fired or was cancelled", + payload={"message_ids": "list[int]", "sent": "bool"}, + ), + _t( + "message_available_min", + "message", + "A channel's history was cleared below a point", + payload={"available_min_id": "int"}, + ), + _t( + "message_emoji_game", + "message", + "An emoji game (dice, dart, slot) resolved", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "message_geo_live_viewed", + "message", + "Somebody viewed a live location I am sharing", + payload={"peer": "object", "msg_id": "int"}, + ), + # -- read ------------------------------------------------------------- + _t( + "read_inbox", + "read", + "My read position moved: messages I have now seen", + payload={"max_id": "int", "still_unread_count": "int | null", "outbox": "false"}, + box="pts", + telethon="high-level: events.MessageRead(inbox=True)", + ), + _t( + "read_outbox", + "read", + "The other side read my messages", + payload={"max_id": "int", "outbox": "true"}, + box="pts", + telethon="high-level: events.MessageRead(inbox=False)", + ), + _t( + "read_contents", + "read", + "Media or a mention was marked read (the media_unread flag)", + payload={"message_ids": "list[int]"}, + box="pts", + ), + _t( + "read_discussion", + "read", + "A comment thread's read position moved", + payload={"msg_id": "int", "read_max_id": "int", "outbox": "bool"}, + box="pts", + ), + _t( + "read_monoforum", + "read", + "A direct-messages (monoforum) channel's read position moved", + payload={"saved_peer_id": "object", "read_max_id": "int", "outbox": "bool"}, + box="pts", + ), + # -- presence --------------------------------------------------------- + _t( + "user_status", + "presence", + "A user came online, or their last-seen changed", + payload={"user_id": "int", "status": "str", "online": "bool", "was_online": "int | null"}, + telethon="high-level: events.UserUpdate", + ), + _t( + "typing", + "presence", + "Somebody is typing, recording or uploading", + payload={ + "user_id": "int | null", + "action": "str — the SendMessageAction name", + "progress": "int | null", + "top_msg_id": "int | null", + }, + telethon="high-level: events.UserUpdate", + ), + # -- peer ------------------------------------------------------------- + _t( + "peer_user_changed", + "peer", + "A user record was invalidated and should be refetched", + payload={"user_id": "int"}, + ), + _t( + "peer_user_name", + "peer", + "A user changed their name or username", + payload={ + "user_id": "int", + "first_name": "str", + "last_name": "str", + "usernames": "list[str]", + }, + ), + _t( + "peer_user_phone", + "peer", + "A contact's phone number changed", + payload={"user_id": "int", "phone": "str"}, + ), + _t( + "peer_user_emoji_status", + "peer", + "A user's emoji status changed", + payload={"user_id": "int", "emoji_status": "object"}, + ), + _t( + "peer_chat_changed", + "peer", + "A chat or channel record was invalidated (refetch; may mean kicked)", + payload={"chat_id": "int"}, + ), + _t( + "peer_blocked", + "peer", + "A peer was blocked or unblocked", + payload={"peer_id": "object", "blocked": "bool", "blocked_my_stories_from": "bool"}, + ), + _t( + "peer_settings", + "peer", + "A peer's action-bar settings changed (anti-scam hints included)", + payload={"peer": "object", "settings": "object"}, + ), + _t( + "peer_located", + "peer", + "The people/groups-nearby list changed", + payload={"peers": "list[object]"}, + ), + _t( + "peer_wallpaper", + "peer", + "A chat wallpaper changed", + payload={"peer": "object", "wallpaper": "object | null"}, + ), + _t( + "peer_history_ttl", + "peer", + "A chat's auto-delete timer changed", + payload={"peer": "object", "ttl_period": "int | null"}, + ), + _t( + "peer_notify_settings", + "peer", + "Notification settings changed for a peer or a scope", + payload={"peer": "object", "notify_settings": "object"}, + ), + # -- member ----------------------------------------------------------- + _t( + "member_channel", + "member", + "A channel or supergroup member or admin changed", + payload={ + "channel_id": "int", + "user_id": "int", + "actor_id": "int | null", + "prev_participant": "object | null", + "new_participant": "object | null", + }, + box="qts", + ), + _t( + "member_chat", + "member", + "A basic group's membership or admin list changed", + payload={ + "chat_id": "int", + "user_id": "int | null", + "version": "int | null", + "participants": "object | null", + }, + box="version", + ), + _t( + "member_default_rights", + "member", + "A group's default permissions changed", + payload={"peer": "object", "default_banned_rights": "object", "version": "int"}, + box="version", + ), + _t( + "member_join_request", + "member", + "A pending join request arrived or was resolved", + payload={"peer": "object", "requests_pending": "int | null", "recent_requesters": "list"}, + ), + _t( + "member_boost", + "member", + "A channel boost was applied", + payload={"peer": "object", "boost": "object"}, + bot_only=True, + ), + # -- dialog ----------------------------------------------------------- + _t( + "dialog_pinned", + "dialog", + "A chat was pinned, unpinned or reordered in the list", + payload={"peer": "object | null", "order": "list[object] | null", "pinned": "bool"}, + ), + _t( + "dialog_unread_mark", + "dialog", + "A chat was manually marked unread (or the mark was cleared)", + payload={"peer": "object", "unread": "bool"}, + ), + _t( + "dialog_folder", + "dialog", + "A chat moved into or out of the Archive", + payload={"folder_peers": "list[object]"}, + box="pts", + ), + _t( + "dialog_filters", + "dialog", + "Chat folders (dialog filters) changed", + payload={"id": "int | null", "filter": "object | null", "order": "list[int] | null"}, + ), + _t( + "dialog_draft", + "dialog", + "A cloud draft was set or cleared", + payload={"peer": "object", "draft": "object", "top_msg_id": "int | null"}, + ), + _t( + "dialog_saved_pinned", + "dialog", + "A Saved Messages sub-dialog was pinned or reordered", + payload={"peer": "object | null", "order": "list[object] | null", "pinned": "bool"}, + ), + _t( + "dialog_saved_tags", + "dialog", + "Saved-message reaction tags changed", + payload={"saved_peer_id": "object | null"}, + ), + _t( + "dialog_forum_pinned", + "dialog", + "Forum topics were pinned or reordered", + payload={"channel_id": "int", "topic_id": "int | null", "order": "list[int] | null"}, + ), + _t( + "dialog_forum_view", + "dialog", + "A forum's display mode was toggled", + payload={"channel_id": "int", "enabled": "bool"}, + ), + _t( + "dialog_quick_reply", + "dialog", + "Business quick-reply shortcuts changed", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "dialog_monoforum_no_paid", + "dialog", + "A direct-messages channel's paid-message exception changed", + payload={"channel_id": "int", "saved_peer_id": "object", "exception": "bool"}, + ), + # -- story ------------------------------------------------------------ + _t( + "story_new", + "story", + "A story was posted, edited or deleted", + payload={"peer": "object", "story": "object"}, + ), + _t( + "story_id", + "story", + "A story you posted got its server id", + payload={"id": "int", "random_id": "int"}, + ), + _t( + "story_read", + "story", + "Stories were marked read", + payload={"peer": "object", "max_id": "int"}, + ), + _t( + "story_reaction", + "story", + "A story was reacted to", + payload={"peer": "object", "story_id": "int", "reaction": "object"}, + ), + _t( + "story_stealth", + "story", + "Story stealth mode changed", + payload={"stealth_mode": "object"}, + ), + # -- collection ------------------------------------------------------- + _t( + "collection_stickers", + "collection", + "Sticker or custom-emoji sets changed", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "collection_stickers_read", + "collection", + "Featured sticker or emoji sets were marked read", + payload={"message_ids": "list[int]"}, + ), + _t("collection_gifs", "collection", "Saved GIFs changed", payload={}), + _t("collection_ringtones", "collection", "Notification sounds changed", payload={}), + _t("collection_reactions", "collection", "Recent or top reactions changed", payload={}), + _t("collection_emoji_statuses", "collection", "Recent emoji statuses changed", payload={}), + _t( + "collection_themes", + "collection", + "A theme changed", + payload={"theme": "object"}, + ), + _t( + "collection_attach_menu", + "collection", + "The attachment-menu bot list changed", + payload={}, + ), + # -- call ------------------------------------------------------------- + _t( + "call_phone", + "call", + "An incoming or updated 1:1 call (signalling only; tlgr carries no media)", + payload={"phone_call": "object"}, + ), + _t( + "call_signaling", + "call", + "Raw call signalling data", + payload={"phone_call_id": "int", "data": "str — hex"}, + ), + _t( + "call_group", + "call", + "A group call, video chat or live stream changed", + payload={"chat_id": "int | null", "call": "object"}, + ), + _t( + "call_group_participants", + "call", + "Group-call participants changed", + payload={"call": "object", "participants": "list[object]", "version": "int"}, + box="version", + ), + _t( + "call_group_message", + "call", + "A message inside a group call was posted or deleted", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "call_group_encrypted", + "call", + "Encrypted group-call key material (conference calls)", + payload=dict(_RAW_PAYLOAD), + ), + # -- bot -------------------------------------------------------------- + _t( + "bot_callback_query", + "bot", + "An inline-keyboard button was pressed", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + telethon="high-level: events.CallbackQuery", + ), + _t( + "bot_inline_query", + "bot", + "An inline query arrived, or a result was chosen", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + telethon="high-level: events.InlineQuery", + ), + _t( + "bot_precheckout", + "bot", + "A pre-checkout query arrived", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_shipping", + "bot", + "A shipping query arrived", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_paid_media_purchased", + "bot", + "A user bought paid media from this bot", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_stopped", + "bot", + "A user started or stopped this bot", + payload={"user_id": "int", "stopped": "bool", "date": "str"}, + box="qts", + bot_only=True, + ), + _t("bot_commands", "bot", "A bot's command list changed", payload=dict(_RAW_PAYLOAD)), + _t("bot_menu_button", "bot", "A bot's menu button changed", payload=dict(_RAW_PAYLOAD)), + _t( + "bot_webhook", + "bot", + "A bot-webhook JSON passthrough arrived", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_business_connection", + "bot", + "A business connection was created or changed", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_business_message", + "bot", + "A message on a connected business account arrived, changed or went", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_message_reaction", + "bot", + "A reaction on a message this bot can see changed", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_guest_chat_query", + "bot", + "A guest-mode chat query arrived", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_managed", + "bot", + "A bot you manage changed", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "bot_webview_result", + "bot", + "A mini app sent data back", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + ), + _t( + "bot_webview_join_decision", + "bot", + "A join-chat decision was made inside a mini app", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "bot_stars_subscription", + "bot", + "A Stars subscription to this bot changed", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + since_layer=229, + ), + _t( + "bot_ephemeral_callback", + "bot", + "A callback button on an ephemeral message was pressed", + payload=dict(_RAW_PAYLOAD), + bot_only=True, + since_layer=229, + ), + # -- stars ------------------------------------------------------------ + _t( + "stars_balance", + "stars", + "The Telegram Stars balance changed", + payload={"balance": "object"}, + ), + _t( + "stars_revenue", + "stars", + "Star revenue or withdrawal status changed", + payload={"peer": "object", "status": "object"}, + ), + _t( + "stars_gift_auction", + "stars", + "A star-gift auction or craft changed state", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "stars_paid_reaction_privacy", + "stars", + "Paid-reaction privacy changed", + payload=dict(_RAW_PAYLOAD), + ), + # -- secret ----------------------------------------------------------- + _t( + "secret_chat", + "secret", + "A secret chat was requested, accepted or discarded", + payload={"chat": "object"}, + box="qts", + ), + _t( + "secret_message", + "secret", + "Encrypted traffic arrived; tlgr acknowledges it but cannot decrypt it", + payload={"chat_id": "int", "date": "str", "decrypted": "false"}, + box="qts", + ), + _t( + "secret_read", + "secret", + "Secret-chat messages were read or expired", + payload={"chat_id": "int", "max_date": "str"}, + box="qts", + ), + # -- account ---------------------------------------------------------- + _t( + "account_privacy", + "account", + "A privacy rule changed", + payload={"key": "str", "rules": "list[object]"}, + ), + _t( + "account_new_authorization", + "account", + "A new login on this account", + payload={"hash": "int", "device": "str", "location": "str", "unconfirmed": "bool"}, + ), + _t( + "account_service_notification", + "account", + "An official service notification (from 777000)", + payload={"type": "str", "message": "str", "popup": "bool", "inbox_date": "str | null"}, + ), + _t( + "account_login_token", + "account", + "A QR login token was accepted", + payload={}, + ), + _t( + "account_sent_phone_code", + "account", + "A login code was delivered in-app", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "account_autosave", + "account", + "Media auto-save settings changed", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "account_browser_settings", + "account", + "In-app browser settings or a per-domain exception changed", + payload=dict(_RAW_PAYLOAD), + ), + _t("account_contacts_reset", "account", "The contact list was wiped", payload={}), + _t( + "account_ai_tones", + "account", + "The AI compose tone list changed", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "account_sms_job", + "account", + "An SMS-relay job arrived (Telegram's peer-to-peer login SMS programme)", + payload={"job_id": "str"}, + ), + _t( + "account_langpack", + "account", + "The language pack changed", + payload=dict(_RAW_PAYLOAD), + ), + _t( + "account_session_revoked", + "account", + "This session was terminated elsewhere (from a push payload)", + payload={"reason": "str"}, + derived="the SESSION_REVOKE push payload, decoded by `tlgr events decode`", + ), + # -- sync ------------------------------------------------------------- + _t( + "sync_config", + "sync", + "The server configuration was invalidated; re-read help.getConfig", + payload={}, + ), + _t( + "sync_dc_options", + "sync", + "The data-centre address list changed", + payload={"dc_options": "list[object]"}, + ), + _t( + "sync_pts_changed", + "sync", + "The pts sequence was reset; some updates are unrecoverable", + payload={}, + ), + _t( + "sync_channel_too_long", + "sync", + "A channel's gap is unrecoverable from its pts; a resync is needed", + payload={"channel_id": "int", "pts": "int | null"}, + ), + _t( + "daemon_health", + "sync", + "An account changed state, or the circuit breaker opened", + payload={"state": "str", "reason": "str", "account": "str"}, + telethon="n/a — synthesised by the daemon", + derived="the session state machine (ARCHITECTURE §6.2)", + ), +) + +TYPES: dict[str, EventTypeSpec] = {spec.type: spec for spec in _TYPES} + + +# --------------------------------------------------------------------------- +# Constructor → type +# --------------------------------------------------------------------------- + +#: `Update*` constructor name → the tlgr event type it becomes. +CONSTRUCTORS: dict[str, str] = { + # message + "UpdateNewMessage": "message_new", + "UpdateNewChannelMessage": "message_new", + "UpdateShortMessage": "message_new", + "UpdateShortChatMessage": "message_new", + "UpdateQuickReplyMessage": "dialog_quick_reply", + "UpdateEditMessage": "message_edited", + "UpdateEditChannelMessage": "message_edited", + "UpdateDeleteMessages": "message_deleted", + "UpdateDeleteChannelMessages": "message_deleted", + "UpdateMessageID": "message_id_assigned", + "UpdateShortSentMessage": "message_id_assigned", + "UpdatePinnedMessages": "message_pinned", + "UpdatePinnedChannelMessages": "message_pinned", + "UpdateChannelMessageViews": "message_views", + "UpdateChannelMessageForwards": "message_forwards", + "UpdateMessagePoll": "message_poll", + "UpdateMessagePollVote": "message_poll_vote", + "UpdateMessageReactions": "message_reactions", + "UpdateMessageExtendedMedia": "message_extended_media", + "UpdateTranscribedAudio": "message_transcribed", + "UpdateWebPage": "message_webpage", + "UpdateChannelWebPage": "message_webpage", + "UpdateNewScheduledMessage": "message_scheduled_new", + "UpdateDeleteScheduledMessages": "message_scheduled_deleted", + "UpdateChannelAvailableMessages": "message_available_min", + "UpdateEmojiGameInfo": "message_emoji_game", + "UpdateGeoLiveViewed": "message_geo_live_viewed", + # read + "UpdateReadHistoryInbox": "read_inbox", + "UpdateReadChannelInbox": "read_inbox", + "UpdateReadHistoryOutbox": "read_outbox", + "UpdateReadChannelOutbox": "read_outbox", + "UpdateReadMessagesContents": "read_contents", + "UpdateChannelReadMessagesContents": "read_contents", + "UpdateReadChannelDiscussionInbox": "read_discussion", + "UpdateReadChannelDiscussionOutbox": "read_discussion", + "UpdateReadMonoForumInbox": "read_monoforum", + "UpdateReadMonoForumOutbox": "read_monoforum", + # presence + "UpdateUserStatus": "user_status", + "UpdateUserTyping": "typing", + "UpdateChatUserTyping": "typing", + "UpdateChannelUserTyping": "typing", + "UpdateEncryptedChatTyping": "typing", + # peer + "UpdateUser": "peer_user_changed", + "UpdateUserName": "peer_user_name", + "UpdateUserPhone": "peer_user_phone", + "UpdateUserEmojiStatus": "peer_user_emoji_status", + "UpdateChat": "peer_chat_changed", + "UpdateChannel": "peer_chat_changed", + "UpdatePeerBlocked": "peer_blocked", + "UpdatePeerSettings": "peer_settings", + "UpdatePeerLocated": "peer_located", + "UpdatePeerWallpaper": "peer_wallpaper", + "UpdatePeerHistoryTTL": "peer_history_ttl", + "UpdateNotifySettings": "peer_notify_settings", + # member + "UpdateChannelParticipant": "member_channel", + "UpdateChatParticipant": "member_chat", + "UpdateChatParticipants": "member_chat", + "UpdateChatParticipantAdd": "member_chat", + "UpdateChatParticipantDelete": "member_chat", + "UpdateChatParticipantAdmin": "member_chat", + "UpdateChatParticipantRank": "member_chat", + "UpdateChatDefaultBannedRights": "member_default_rights", + "UpdatePendingJoinRequests": "member_join_request", + "UpdateBotChatInviteRequester": "member_join_request", + "UpdateBotChatBoost": "member_boost", + # dialog + "UpdateDialogPinned": "dialog_pinned", + "UpdatePinnedDialogs": "dialog_pinned", + "UpdateDialogUnreadMark": "dialog_unread_mark", + "UpdateFolderPeers": "dialog_folder", + "UpdateDialogFilter": "dialog_filters", + "UpdateDialogFilterOrder": "dialog_filters", + "UpdateDialogFilters": "dialog_filters", + "UpdateDraftMessage": "dialog_draft", + "UpdateSavedDialogPinned": "dialog_saved_pinned", + "UpdatePinnedSavedDialogs": "dialog_saved_pinned", + "UpdateSavedReactionTags": "dialog_saved_tags", + "UpdatePinnedForumTopic": "dialog_forum_pinned", + "UpdatePinnedForumTopics": "dialog_forum_pinned", + "UpdateChannelViewForumAsMessages": "dialog_forum_view", + "UpdateQuickReplies": "dialog_quick_reply", + "UpdateNewQuickReply": "dialog_quick_reply", + "UpdateDeleteQuickReply": "dialog_quick_reply", + "UpdateDeleteQuickReplyMessages": "dialog_quick_reply", + "UpdateMonoForumNoPaidException": "dialog_monoforum_no_paid", + # story + "UpdateStory": "story_new", + "UpdateStoryID": "story_id", + "UpdateReadStories": "story_read", + "UpdateNewStoryReaction": "story_reaction", + "UpdateSentStoryReaction": "story_reaction", + "UpdateStoriesStealthMode": "story_stealth", + # collection + "UpdateStickerSets": "collection_stickers", + "UpdateStickerSetsOrder": "collection_stickers", + "UpdateNewStickerSet": "collection_stickers", + "UpdateRecentStickers": "collection_stickers", + "UpdateFavedStickers": "collection_stickers", + "UpdateMoveStickerSetToTop": "collection_stickers", + "UpdateReadFeaturedStickers": "collection_stickers_read", + "UpdateReadFeaturedEmojiStickers": "collection_stickers_read", + "UpdateSavedGifs": "collection_gifs", + "UpdateSavedRingtones": "collection_ringtones", + "UpdateRecentReactions": "collection_reactions", + "UpdateRecentEmojiStatuses": "collection_emoji_statuses", + "UpdateTheme": "collection_themes", + "UpdateAttachMenuBots": "collection_attach_menu", + # call + "UpdatePhoneCall": "call_phone", + "UpdatePhoneCallSignalingData": "call_signaling", + "UpdateGroupCall": "call_group", + "UpdateGroupCallConnection": "call_group", + "UpdateGroupCallParticipants": "call_group_participants", + "UpdateGroupCallMessage": "call_group_message", + "UpdateDeleteGroupCallMessages": "call_group_message", + "UpdateGroupCallChainBlocks": "call_group_encrypted", + "UpdateGroupCallEncryptedMessage": "call_group_encrypted", + # bot + "UpdateBotCallbackQuery": "bot_callback_query", + "UpdateInlineBotCallbackQuery": "bot_callback_query", + "UpdateBusinessBotCallbackQuery": "bot_callback_query", + "UpdateBotInlineQuery": "bot_inline_query", + "UpdateBotInlineSend": "bot_inline_query", + "UpdateBotPrecheckoutQuery": "bot_precheckout", + "UpdateBotShippingQuery": "bot_shipping", + "UpdateBotPurchasedPaidMedia": "bot_paid_media_purchased", + "UpdateBotStopped": "bot_stopped", + "UpdateBotCommands": "bot_commands", + "UpdateBotMenuButton": "bot_menu_button", + "UpdateBotWebhookJSON": "bot_webhook", + "UpdateBotWebhookJSONQuery": "bot_webhook", + "UpdateBotBusinessConnect": "bot_business_connection", + "UpdateNewBotConnection": "bot_business_connection", + "UpdateBotNewBusinessMessage": "bot_business_message", + "UpdateBotEditBusinessMessage": "bot_business_message", + "UpdateBotDeleteBusinessMessage": "bot_business_message", + "UpdateBotMessageReaction": "bot_message_reaction", + "UpdateBotMessageReactions": "bot_message_reaction", + "UpdateBotGuestChatQuery": "bot_guest_chat_query", + "UpdateManagedBot": "bot_managed", + "UpdateWebViewResultSent": "bot_webview_result", + "UpdateJoinChatWebViewDecision": "bot_webview_join_decision", + # stars + "UpdateStarsBalance": "stars_balance", + "UpdateStarsRevenueStatus": "stars_revenue", + "UpdateStarGiftAuctionState": "stars_gift_auction", + "UpdateStarGiftAuctionUserState": "stars_gift_auction", + "UpdateStarGiftCraftFail": "stars_gift_auction", + "UpdatePaidReactionPrivacy": "stars_paid_reaction_privacy", + # secret + "UpdateEncryption": "secret_chat", + "UpdateNewEncryptedMessage": "secret_message", + "UpdateEncryptedMessagesRead": "secret_read", + # account + "UpdatePrivacy": "account_privacy", + "UpdateNewAuthorization": "account_new_authorization", + "UpdateServiceNotification": "account_service_notification", + "UpdateLoginToken": "account_login_token", + "UpdateSentPhoneCode": "account_sent_phone_code", + "UpdateAutoSaveSettings": "account_autosave", + "UpdateWebBrowserSettings": "account_browser_settings", + "UpdateWebBrowserException": "account_browser_settings", + "UpdateContactsReset": "account_contacts_reset", + "UpdateAiComposeTones": "account_ai_tones", + "UpdateSmsJob": "account_sms_job", + "UpdateLangPack": "account_langpack", + "UpdateLangPackTooLong": "account_langpack", + # sync + "UpdateConfig": "sync_config", + "UpdateDcOptions": "sync_dc_options", + "UpdatePtsChanged": "sync_pts_changed", + "UpdateChannelTooLong": "sync_channel_too_long", +} + +#: Constructors that carry no event of their own, and why. Being on this list +#: is a decision; being on neither list is a bug the taxonomy test catches. +INTERNAL: dict[str, str] = { + "UpdateShort": ("container: carries exactly one Update, which is normalised in its place"), + "Updates": "container: a batch of updates plus their users/chats arrays", + "UpdatesCombined": "container: a batch of updates spanning a seq range", + "UpdatesTooLong": ( + "transport signal: the common box overflowed, handled by the supervisor " + "with updates.getDifference (see `tlgr sync catch-up`)" + ), +} + +#: Constructors Telegram ships that Telethon 1.44 (layer 227) cannot parse. +#: Listed so `events list` can say "exists, unavailable here" rather than +#: leaving a silent hole; a raw handler sees only an unknown constructor id. +NEWER_THAN_LAYER_227: dict[str, str] = { + "UpdateNewEphemeralMessage": "message_new", + "UpdateEditEphemeralMessage": "message_edited", + "UpdateDeleteEphemeralMessages": "message_deleted", + "UpdateBotStarsSubscription": "bot_stars_subscription", + "UpdateBotEphemeralCallbackQuery": "bot_ephemeral_callback", +} + +#: Names a consumer may still be using, and what they mean now. v1's `watch` +#: and `jobs.yaml` spelled the message events differently, and the foundation +#: shipped a nine-name starter set; both keep working (§12.4). +ALIASES: dict[str, tuple[str, ...]] = { + "new_message": ("message_new",), + "user_joined": ("message_service", "member_chat", "member_channel"), + "chat_action": ("message_service", "member_chat", "member_channel"), + "message_read": ("read_inbox", "read_outbox"), + "reaction_changed": ("message_reactions",), + "draft_changed": ("dialog_draft",), + "message_edit": ("message_edited",), +} + + +def group_of(event_type: str) -> str: + spec = TYPES.get(event_type) + return spec.group if spec is not None else "" + + +def type_for_constructor(name: str) -> str | None: + """The event type a `Update*` class name maps to, or None when internal.""" + found = CONSTRUCTORS.get(name) + if found is not None: + return found + return NEWER_THAN_LAYER_227.get(name) + + +def constructors_for(event_type: str) -> tuple[str, ...]: + """Every `Update*` constructor that produces *event_type*.""" + return tuple( + sorted( + name + for mapping in (CONSTRUCTORS, NEWER_THAN_LAYER_227) + for name, mapped in mapping.items() + if mapped == event_type + ) + ) + + +def resolve_selectors( + selectors: str | Iterable[str] | None, *, allow_all: bool = True +) -> frozenset[str]: + """Expand `--events`/`--exclude` values into a set of event type names. + + Accepts type names, group names, the legacy names in `ALIASES`, + `raw:UpdateFoo` (the type that constructor maps to) and `all`. An unknown + value is a `USAGE` error rather than an empty selection: silently watching + nothing is the failure mode that makes a user think the daemon is broken. + """ + from tlgr.core.errors import UsageError + + if isinstance(selectors, str): + values = [part.strip() for part in selectors.split(",")] + else: + values = [str(part).strip() for part in (selectors or ())] + wanted: set[str] = set() + for value in values: + if not value: + continue + lowered = value.lower() + if lowered in ("all", "*"): + if not allow_all: + raise UsageError("'all' is not accepted here", field="events") + return frozenset(TYPES) + if lowered.startswith("raw:"): + constructor = value[4:] + mapped = type_for_constructor(constructor) + if mapped is None: + raise UsageError( + f"raw:{constructor} is not an update tlgr names; see `tlgr events list --raw`", + field="events", + ) + wanted.add(mapped) + continue + if lowered in TYPES: + wanted.add(lowered) + continue + if lowered in ALIASES: + wanted.update(ALIASES[lowered]) + continue + if lowered in GROUPS: + wanted.update(name for name, spec in TYPES.items() if spec.group == lowered) + continue + raise UsageError( + f"unknown event selector {value!r}; run `tlgr events list` for the vocabulary", + field="events", + ) + return frozenset(wanted) diff --git a/tlgr/daemon/launchd.py b/tlgr/core/launchd.py similarity index 100% rename from tlgr/daemon/launchd.py rename to tlgr/core/launchd.py diff --git a/tlgr/core/paths.py b/tlgr/core/paths.py index 8665966..ef9fcc2 100644 --- a/tlgr/core/paths.py +++ b/tlgr/core/paths.py @@ -31,6 +31,7 @@ "TlgrPaths", "audit_permissions", "default_base", + "is_production_home", "refuse_production_home", "secure_session_files", "validate_alias", @@ -60,6 +61,14 @@ def default_base() -> Path: _ALLOW_PRODUCTION_ENV = "TLGR_ALLOW_PRODUCTION_HOME" +def is_production_home(base: Path) -> bool: + """Is *base* marked as a live deployment?""" + try: + return (base / PRODUCTION_MARKER).exists() + except OSError: # pragma: no cover - an unreadable home is not ours to judge + return False + + def refuse_production_home(base: Path) -> None: """Refuse to operate on a home marked as production unless told to. @@ -199,6 +208,11 @@ def cursor_key(self) -> Path: def identity(self) -> Path: return self.base / "identity.json" + @property + def proxies(self) -> Path: + """Saved proxies, including their passwords and MTProxy secrets (0600).""" + return self.base / "proxies.json" + @property def dead_letter(self) -> Path: return self.base / "dead_letter.jsonl" @@ -283,6 +297,7 @@ def ensure_account_dir(self, alias: str) -> Path: "ipc.token", "cursor.key", "webhook.toml", + "proxies.json", ) _SECRET_MODE = 0o600 diff --git a/tlgr/daemon/lifecycle.py b/tlgr/core/process.py similarity index 100% rename from tlgr/daemon/lifecycle.py rename to tlgr/core/process.py diff --git a/tlgr/core/signing.py b/tlgr/core/signing.py new file mode 100644 index 0000000..cd36bb0 --- /dev/null +++ b/tlgr/core/signing.py @@ -0,0 +1,38 @@ +"""The webhook signature, in one place. + +Part of the wire contract rather than of the pusher: `daemon/webhook.py` signs +deliveries with it and `ops/webhook.py` shows a receiver what a signature will +look like, and `ops/` may not import `daemon/` (§2.2). + +SEC-08 is what it fixes. v1 sent events with no signature at all, so any +process that learned the URL could forge them; a bearer token would only have +proved the sender knew a string, not that the body was unmodified. +""" + +from __future__ import annotations + +import hashlib +import hmac + +__all__ = ["sign_body", "verify_body"] + + +def sign_body(secret: str, body: bytes) -> str: + """`sha256=` over the exact bytes that go on the wire. + + Over the *bytes*, not over a re-encoded dict: a receiver verifies what it + received, and any re-encoding — key order, whitespace, escaping — makes an + honest signature fail. + """ + digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + return f"sha256={digest}" + + +def verify_body(secret: str, body: bytes, signature: str) -> bool: + """Constant-time check, for a receiver written in Python. + + Exported because "verify the signature" is advice everybody follows + slightly differently, and `==` on two hex strings is the difference + between a check and a timing oracle. + """ + return hmac.compare_digest(sign_body(secret, body), signature.strip()) diff --git a/tlgr/daemon/systemd.py b/tlgr/core/systemd.py similarity index 100% rename from tlgr/daemon/systemd.py rename to tlgr/core/systemd.py diff --git a/tlgr/core/telethon_compat.py b/tlgr/core/telethon_compat.py index c58deed..6d221cd 100644 --- a/tlgr/core/telethon_compat.py +++ b/tlgr/core/telethon_compat.py @@ -16,6 +16,7 @@ from __future__ import annotations +import contextlib import logging from collections.abc import Callable from typing import Any @@ -25,10 +26,13 @@ __all__ = [ "TOO_LONG_CHANNEL", "TOO_LONG_GLOBAL", + "entity_count", "install_reconnect_hook", "install_too_long_hook", "probe", "save_state", + "session_state", + "set_session_state", "telethon_version", ] @@ -205,6 +209,79 @@ def clear_config_cache(client: Any) -> bool: return False +def session_state(client: Any) -> tuple[dict[str, Any], dict[int, int]]: + """`({pts, qts, seq, date}, {channel_id: pts})` from the session. + + Telethon 1.44 has no public accessor for its update state: the common box + lives in the session's `update_state` table under entity id 0 and the + per-channel boxes under their channel ids. Reading it here — once, behind + a name — is what lets `sync status` answer "how far behind is this + account" without every caller reaching into a private table. + """ + common: dict[str, Any] = {} + channels: dict[int, int] = {} + session = getattr(client, "session", None) + getter = getattr(session, "get_update_states", None) + if not callable(getter): + _warn_once("session.get_update_states") + return common, channels + try: + rows = list(getter()) + except Exception as exc: # pragma: no cover - depends on session backend + log.debug("update state read failed: %s", exc) + return common, channels + for entity_id, state in rows: + if int(entity_id) == 0: + date = getattr(state, "date", None) + common = { + "pts": getattr(state, "pts", None), + "qts": getattr(state, "qts", None), + "seq": getattr(state, "seq", None), + "date": date.strftime("%Y-%m-%dT%H:%M:%SZ") if date is not None else None, + "date_unix": int(date.timestamp()) if date is not None else None, + "unread_count": getattr(state, "unread_count", None), + } + else: + channels[int(entity_id)] = int(getattr(state, "pts", 0) or 0) + return common, channels + + +def set_session_state(client: Any, state: Any, entity_id: int = 0) -> bool: + """Write one update-state row. The `sync reset` half of the pair above.""" + session = getattr(client, "session", None) + setter = getattr(session, "set_update_state", None) + if not callable(setter): + _warn_once("session.set_update_state") + return False + try: + setter(entity_id, state) + except Exception as exc: # pragma: no cover - depends on session backend + log.debug("update state write failed: %s", exc) + return False + return True + + +def entity_count(client: Any) -> int: + """How many peers the session has an access hash for. + + Not a statistic: an entity missing from here is a channel `catch_up()` + will silently skip, because `getChannelDifference` needs the access hash + and Telethon will not ask for one it does not have. + """ + session = getattr(client, "session", None) + cursor = getattr(session, "_cursor", None) + if callable(cursor): + try: + row = cursor().execute("select count(*) from entities").fetchone() + return int(row[0]) if row else 0 + except Exception as exc: # pragma: no cover - depends on session backend + log.debug("entity count failed: %s", exc) + cache = getattr(client, "_entity_cache", None) + with contextlib.suppress(TypeError): + return len(cache) if cache is not None else 0 + return 0 + + def flood_waited_requests(client: Any) -> dict[int, float]: """Telethon's in-process flood memory, `{constructor_id: until_unix}`. diff --git a/tlgr/core/tl.py b/tlgr/core/tl.py new file mode 100644 index 0000000..cd108e8 --- /dev/null +++ b/tlgr/core/tl.py @@ -0,0 +1,98 @@ +"""TL object → JSON-safe builtins, and peer arithmetic. + +Two functions, in `core/` because three layers need them and none may import +another: the bus normalises updates with them, `ops/` reads `help.*` replies +with them, and neither is allowed to reach into the other (§2.2). + +`tl_to_builtins` is the COR-07 fix stated once. v1 delivered a raw `to_dict()` +through `json.dumps(default=str)`, so a `datetime` became a string in one +place and a `bytes` blew up in another — and a message with media could fail +to serialise *at delivery time*, far from the cause, counted as a delivery +failure rather than as the bug it was. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +__all__ = ["CHANNEL_MARK", "peer_marked_id", "tl_to_builtins"] + +#: Telegram's channel id offset. A channel's marked id is this minus its id, +#: which is what makes `-100…` recognisable at a glance. +CHANNEL_MARK = -1000000000000 + +#: How deep `tl_to_builtins` walks before it stops. A `Message` inside a +#: `Story` inside a `WebPage` is real; anything past this is a cycle, or a +#: payload nobody wanted in a stream frame. +_MAX_DEPTH = 8 + + +def tl_to_builtins(value: Any, *, depth: int = 0) -> Any: + """A TL object tree → JSON-safe builtins, with the class name kept. + + Datetimes become RFC-3339, bytes become hex, and the constructor name + survives as `_` so a consumer can still branch on it. + """ + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, (bytes, bytearray, memoryview)): + return bytes(value).hex() + if isinstance(value, datetime): + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + if depth >= _MAX_DEPTH: + return type(value).__name__ + if isinstance(value, (list, tuple, set, frozenset)): + return [tl_to_builtins(item, depth=depth + 1) for item in value] + if isinstance(value, dict): + return {str(k): tl_to_builtins(v, depth=depth + 1) for k, v in value.items()} + + out: dict[str, Any] = {"_": type(value).__name__} + attributes = getattr(value, "__dict__", None) + names: list[str] + if isinstance(attributes, dict) and attributes: + names = [name for name in attributes if not name.startswith("_")] + else: + names = [ + name + for klass in type(value).__mro__ + for name in getattr(klass, "__slots__", ()) + if not str(name).startswith("_") + ] + if not names: + # A TLObject with nothing set, or an object we have no handle on. Its + # class name is the honest answer; `str()` would call Telethon's + # pretty-printer, which re-enters `to_dict()` and can raise. + return out + for name in names: + out[str(name)] = tl_to_builtins(getattr(value, name, None), depth=depth + 1) + return out + + +def peer_marked_id(peer: Any) -> int | None: + """A TL `Peer*` → the marked id tlgr uses everywhere else. + + Four lines of arithmetic rather than a call into Telethon's `utils`, + because the bus runs this on the update loop's hot path for every event. + """ + if peer is None: + return None + if isinstance(peer, int): + return peer + name = type(peer).__name__ + if name == "PeerUser": + return _int(getattr(peer, "user_id", None)) + if name == "PeerChat": + chat_id = _int(getattr(peer, "chat_id", None)) + return -chat_id if chat_id is not None else None + if name == "PeerChannel": + channel_id = _int(getattr(peer, "channel_id", None)) + return CHANNEL_MARK - channel_id if channel_id is not None else None + return None + + +def _int(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/tlgr/daemon/app.py b/tlgr/daemon/app.py index 6efd539..e9d86f7 100644 --- a/tlgr/daemon/app.py +++ b/tlgr/daemon/app.py @@ -22,6 +22,7 @@ import logging import os import time +import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -50,7 +51,7 @@ from tlgr.daemon.policy import Policy from tlgr.daemon.preauth import PreAuthService from tlgr.daemon.sessions import SessionManager -from tlgr.daemon.stream import NdjsonResponse, pump_events, walk_pages +from tlgr.daemon.stream import NdjsonResponse, walk_pages from tlgr.daemon.transfers import TransferStore from tlgr.daemon.webhook import WebhookPusher from tlgr.version import HEADER_PROTOCOL, HEADER_TOKEN, MIN_DAEMON_PROTOCOL, PROTOCOL @@ -174,17 +175,14 @@ async def on_update(event: Any) -> None: try: from telethon import events as tl_events - for builder in ( - tl_events.NewMessage(), - tl_events.MessageEdited(), - tl_events.MessageDeleted(), - tl_events.MessageRead(), - tl_events.ChatAction(), - tl_events.UserUpdate(), - ): - register(on_update, builder) + # One `Raw` handler, not six high-level ones. The high-level + # builders drop service messages, topic ids and every action kind + # Telethon does not model, so a stream built on them can only ever + # show a subset of what the GUI shows; `normalise` names all 163 + # update constructors instead (docs/design/EVENTS.md). + register(on_update, tl_events.Raw()) except Exception as exc: # pragma: no cover - a fake client has no builders - log.debug("could not register Telethon handlers for %s: %s", alias, exc) + log.debug("could not register the raw Telethon handler for %s: %s", alias, exc) register(on_update) # -- v1 compatibility surface ----------------------------------------- @@ -281,11 +279,14 @@ async def reload_jobs(self) -> dict[str, Any]: } def status(self) -> dict[str, Any]: - """v1's `/daemon/status` body, unchanged (it is a documented shape). + """v1's `/daemon/status` body. The route is gone; the shape is not. - `connections` and `healthy` exist because the wrapper existing and the + `daemon status` is a registry operation now and answers from + `/v1/status`, so nothing serves this over HTTP any more. It stays + because `connections`/`healthy` are the COR-37 fix stated at the level + the `ClientWrapper` bridge works at — the wrapper existing and the wrapper being usable are different facts, and v1 reported only the - first — a fully dead daemon looked healthy. + first — and both go together at PR-12. """ uptime = int(time.time() - self._start_time) connections = {alias: client.is_connected for alias, client in self._clients.items()} @@ -624,7 +625,11 @@ async def handle_op(request: web.Request) -> web.StreamResponse: daemon: Daemon = request.app[DAEMON_KEY] raw = await request.read() op_request = dispatch_module.decode_request(raw) - if op_request.stream or op_request.all: + if op_request.stream or op_request.all or _is_stream_op(op_request.op): + # A streaming operation is streamed whether or not the caller + # remembered to say so: its result is an async iterator, and answering + # a plain POST with "cannot encode an async_generator" would report a + # tlgr bug as the caller's mistake. return await _handle_op_stream(request, daemon, op_request) envelope = await dispatch_module.dispatch(daemon, op_request) return web.Response( @@ -633,6 +638,13 @@ async def handle_op(request: web.Request) -> web.StreamResponse: ) +def _is_stream_op(op_id: str) -> bool: + from tlgr.registry import ALIASES, REGISTRY + + spec = REGISTRY.get(ALIASES.get(op_id.replace(" ", "."), op_id)) + return bool(spec is not None and spec.stream) + + async def _handle_op_stream( request: web.Request, daemon: Daemon, op_request: Any ) -> web.StreamResponse: @@ -649,7 +661,18 @@ async def _handle_op_stream( # `spec` is the same object `resolve_spec` returned above; the # execute() call is what runs the policy/account/timeout prologue. _, context, result = await dispatch_module.execute(daemon, op_request) - if hasattr(result, "__aiter__"): + if "frames" in spec.tags and hasattr(result, "__aiter__"): + # A frame-producing operation writes its own NDJSON vocabulary — + # events, `gap`, `lag`, `heartbeat`. Wrapping those in `item` + # frames would make a heartbeat indistinguishable from an event + # for anybody reading the stream one line at a time. + count = 0 + async for frame in result: + if not isinstance(frame, dict): # pragma: no cover - impl contract + continue + await stream.write(frame) + count += 1 + elif hasattr(result, "__aiter__"): # A `--all` walk paces itself against the account's own limiter, # inside the daemon: v1 looped in the client and hammered the # socket with no backpressure between pages (ROB-01). @@ -686,6 +709,35 @@ async def _stream_result(stream: NdjsonResponse, result: Any) -> int: return len(items) +#: v1's `/v1/events` query names → the `events.watch` request fields they are +#: now spelled as. The endpoint predates the operation; §12.4 says a +#: documented shape does not disappear because the code behind it moved. +_EVENTS_QUERY_ALIASES = {"types": "events", "chats": "chat", "timeout": "follow_for"} + + +def _events_request(query: Any) -> dict[str, Any]: + """`GET /v1/events?…` → the `events.watch` request body. + + The endpoint is a GET-shaped alias of `POST /v1/op {op: events.watch}`: + one implementation, one filter vocabulary, one set of frames. Two code + paths reading the same bus with two ideas of what `--events` means is + exactly the drift the registry exists to remove. + """ + body: dict[str, Any] = {} + for key, value in query.items(): + name = _EVENTS_QUERY_ALIASES.get(key, key) + if name in ("account", "flood_wait_max"): + continue + if name in ("chat", "sender"): + body[name] = [part for part in str(value).split(",") if part] + else: + body[name] = value + # The endpoint's historical default is everything; `tlgr watch`'s is v1's + # `new_message`, and that difference is deliberate. + body.setdefault("events", "all") + return body + + async def handle_events(request: web.Request) -> web.StreamResponse: daemon: Daemon = request.app[DAEMON_KEY] account = request.query.get("account", "").strip() @@ -693,39 +745,22 @@ async def handle_events(request: web.Request) -> web.StreamResponse: return _error_response( request, UsageError("GET /v1/events needs ?account=", field="account") ) - types = [t for t in request.query.get("types", "").split(",") if t] - chats = [ - int(c) for c in request.query.get("chats", "").split(",") if c.strip().lstrip("-").isdigit() - ] - since_raw = request.query.get("since") - since = int(since_raw) if since_raw and since_raw.lstrip("-").isdigit() else None - timeout = min(int(request.query.get("timeout", 3600) or 3600), 86400) - - subscriber = daemon.bus.subscribe(account, types=types, chats=chats) - stream = NdjsonResponse(request) + from tlgr.models.envelope import OpRequest + from tlgr.version import VERSION + + op_request = OpRequest( + op="events.watch", + account=account, + request=_events_request(request.query), + request_id=request.headers.get("X-Tlgr-Request-Id", "") or uuid.uuid4().hex, + client_version=VERSION, + protocol=PROTOCOL, + stream=True, + ) daemon.activity.begin_stream() try: - await stream.prepare(account=account, seq=daemon.bus.latest_seq(account)) - replayed, gap = daemon.bus.replay(account, since) - if gap is not None: - await stream.write(gap) - from tlgr.models.base import to_builtins - - for event in replayed: - await stream.write(to_builtins(event)) - reason = await pump_events( - stream, - subscriber, - timeout=timeout, - shutdown=daemon.shutting_down, - ) - return await stream.end(ok=True, reason=reason) - except (ConnectionResetError, asyncio.CancelledError): - return await stream.end(ok=True, reason="client-disconnected") - except Exception as exc: - return await stream.fail(exc, account=account) + return await _handle_op_stream(request, daemon, op_request) finally: - daemon.bus.unsubscribe(subscriber) daemon.activity.end_stream() diff --git a/tlgr/daemon/dispatch.py b/tlgr/daemon/dispatch.py index 6952ba8..048ccc5 100644 --- a/tlgr/daemon/dispatch.py +++ b/tlgr/daemon/dispatch.py @@ -306,7 +306,7 @@ async def execute(daemon: Daemon, request: OpRequest) -> tuple[OperationSpec, Da if request.dry_run and spec.mutating: return spec, context, {"dry_run": True, "would": spec.id, "request": to_builtins(payload)} - if spec.surface is not Surface.LOCAL and spec.needs_account: + if spec.surface is not Surface.LOCAL and spec.needs_account and spec.needs_client: session = await daemon.sessions.ensure(account) limiter = daemon.sessions.limiter(account) limiter.check(rate_class=spec.rate_class) @@ -319,6 +319,19 @@ async def execute(daemon: Daemon, request: OpRequest) -> tuple[OperationSpec, Da session.in_flight += 1 else: session = None + # A daemon operation that needs no Telegram client still benefits from + # the resolver when the account happens to be connected — `watch + # --chat @alice` should not have to be given a numeric id just because + # reading the bus does not itself need a socket. Attached, never + # *connected*: asking the daemon a question about itself must not dial + # Telegram as a side effect. + if account and spec.surface is not Surface.LOCAL: + existing = daemon.sessions.get(account) + if existing is not None and existing.client is not None: + context.session = existing + context.client = existing.client + context.limiter = daemon.sessions.limiter(account) + context.resolver = existing.resolver budget: Any = None if session is not None and context.limiter is not None: diff --git a/tlgr/daemon/events.py b/tlgr/daemon/events.py index c34ebe7..8a6a3b5 100644 --- a/tlgr/daemon/events.py +++ b/tlgr/daemon/events.py @@ -38,7 +38,9 @@ from pathlib import Path from typing import Any +from tlgr.core import eventtypes from tlgr.core.paths import write_private +from tlgr.core.tl import CHANNEL_MARK, peer_marked_id, tl_to_builtins from tlgr.models.event import EventEnvelope log = logging.getLogger("tlgr.daemon.events") @@ -48,21 +50,14 @@ "EventBus", "Subscriber", "normalise", + "normalise_update", + "tl_to_builtins", ] -#: The starter taxonomy (§3.7). The full vocabulary lands in PR-4; every name -#: is a lowercase snake_case noun-verb and new ones are additive. -EVENT_TYPES: tuple[str, ...] = ( - "message_new", - "message_edited", - "message_deleted", - "message_read", - "chat_action", - "user_status", - "reaction_changed", - "draft_changed", - "daemon_health", -) +#: The taxonomy, as a tuple, for the callers that want to iterate it. The +#: table itself is `tlgr.core.eventtypes`, which `ops/` and the doc generator +#: read too — `daemon/` must not be the only place that knows the vocabulary. +EVENT_TYPES: tuple[str, ...] = tuple(sorted(eventtypes.TYPES)) _HEARTBEAT_SECONDS = 15.0 _STATE_FLUSH_SECONDS = 5.0 @@ -91,6 +86,11 @@ class Subscriber: types: frozenset[str] = frozenset() chats: frozenset[int] = frozenset() maxsize: int = 1024 + #: `watch --raw` needs the TL update itself, which is ten times the size + #: of the payload. Converting it for every event on the chance somebody + #: wants it would tax the update loop, so the bus does it only while a + #: subscriber has asked. + want_raw: bool = False queue: asyncio.Queue[EventEnvelope] = field(init=False) dropped: int = 0 closed: bool = False @@ -131,6 +131,18 @@ def close(self) -> None: # --------------------------------------------------------------------------- +#: Re-exported: the bus was the first caller, but `ops/` reads `help.*` +#: replies with the same converter and may not import `daemon/` (§2.2). +_CHANNEL_MARK = CHANNEL_MARK + + +def _int(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + def _message_payload(message: Any, chat_id: int | None) -> dict[str, Any]: from tlgr.models.base import to_builtins from tlgr.ops._serialize import message_to_model @@ -140,23 +152,179 @@ def _message_payload(message: Any, chat_id: int | None) -> dict[str, Any]: return payload if isinstance(payload, dict) else {} +def _channel_chat_id(update: Any) -> int | None: + channel_id = _int(getattr(update, "channel_id", None)) + return _CHANNEL_MARK - channel_id if channel_id is not None else None + + +def _chat_of(update: Any) -> int | None: + """The chat an update is about, from whichever field carries it.""" + for attr in ("peer", "peer_id", "saved_peer_id"): + marked = peer_marked_id(getattr(update, attr, None)) + if marked is not None: + return marked + channel = _channel_chat_id(update) + if channel is not None: + return channel + chat_id = _int(getattr(update, "chat_id", None)) + if chat_id is not None: + return -chat_id + return _int(getattr(update, "user_id", None)) + + +def normalise_update( + account: str, update: Any +) -> tuple[str, dict[str, Any], int | None, int | None] | None: + """One raw TL `Update*` → `(type, payload, chat_id, sender_id)`, or None. + + Raw rather than Telethon's high-level events, on purpose: `events.NewMessage` + and friends drop service messages, topic ids and every action kind Telethon + does not model, so a `watch` built on them can only ever show a subset of + what the GUI shows. Everything the taxonomy names is reachable from here. + + None means the constructor is `INTERNAL` (a container, a transport signal) + or is not an update at all. tlgr never invents a type name for it: a name + meaning "we did not look" cannot be filtered on and changes meaning the + day the real one arrives. + """ + name = type(update).__name__ + event_type = eventtypes.type_for_constructor(name) + if event_type is None: + return None + + chat_id = _chat_of(update) + sender_id: int | None = None + payload: dict[str, Any] + + if event_type in ("message_new", "message_edited", "message_scheduled_new"): + message = getattr(update, "message", None) + if message is None or isinstance(message, str): + # updateShortMessage/updateShortChatMessage carry the text, not a + # Message; Telethon normalises them before they reach a handler, + # so this branch only fires for a hand-built update. + payload = tl_to_builtins(update) + return event_type, payload, chat_id, sender_id + if type(message).__name__ == "MessageService": + action = getattr(message, "action", None) + payload = _message_payload(message, chat_id) + payload["action"] = type(action).__name__ if action is not None else "" + return ( + "message_service", + payload, + chat_id or payload.get("chat_id"), + payload.get("sender_id"), + ) + payload = _message_payload(message, chat_id) + return event_type, payload, chat_id or payload.get("chat_id"), payload.get("sender_id") + + if event_type == "message_deleted": + payload = { + "message_ids": [int(i) for i in (getattr(update, "messages", None) or [])], + } + channel = _channel_chat_id(update) + if channel is not None: + payload["channel_id"] = channel + return event_type, payload, channel, None + + if event_type in ("read_inbox", "read_outbox"): + payload = { + "max_id": _int(getattr(update, "max_id", None)), + "outbox": event_type == "read_outbox", + } + unread = getattr(update, "still_unread_count", None) + if unread is not None: + payload["still_unread_count"] = _int(unread) + return event_type, payload, chat_id, None + + if event_type == "typing": + action = getattr(update, "action", None) + actor = peer_marked_id(getattr(update, "from_id", None)) + user_id = _int(getattr(update, "user_id", None)) + payload = { + "user_id": user_id if user_id is not None else actor, + "action": type(action).__name__ if action is not None else "", + "progress": _int(getattr(action, "progress", None)), + "top_msg_id": _int(getattr(update, "top_msg_id", None)), + } + return event_type, payload, chat_id, payload["user_id"] + + if event_type == "user_status": + status = getattr(update, "status", None) + status_name = type(status).__name__ if status is not None else None + user_id = _int(getattr(update, "user_id", None)) + payload = { + "user_id": user_id, + "status": status_name, + "online": status_name == "UserStatusOnline", + "was_online": _int(getattr(status, "was_online", None)), + } + return event_type, payload, user_id, user_id + + if event_type == "message_reactions": + payload = { + "msg_id": _int(getattr(update, "msg_id", None)), + "top_msg_id": _int(getattr(update, "top_msg_id", None)), + "reactions": tl_to_builtins(getattr(update, "reactions", None)), + } + return event_type, payload, chat_id, None + + if event_type == "dialog_draft": + payload = { + "peer": tl_to_builtins(getattr(update, "peer", None)), + "draft": tl_to_builtins(getattr(update, "draft", None)), + "top_msg_id": _int(getattr(update, "top_msg_id", None)), + } + return event_type, payload, chat_id, None + + if event_type == "message_id_assigned": + payload = { + "msg_id": _int(getattr(update, "id", None)), + "random_id": _int(getattr(update, "random_id", None)), + } + return event_type, payload, chat_id, None + + if event_type == "message_pinned": + payload = { + "message_ids": [int(i) for i in (getattr(update, "messages", None) or [])], + "pinned": bool(getattr(update, "pinned", False)), + } + return event_type, payload, chat_id, None + + if event_type == "sync_channel_too_long": + payload = { + "channel_id": _channel_chat_id(update), + "pts": _int(getattr(update, "pts", None)), + } + return event_type, payload, chat_id, None + + # Everything else is delivered as the update's own fields, JSON-safe. The + # taxonomy says so per type, so a consumer is never guessing. + payload = tl_to_builtins(update) + if not isinstance(payload, dict): + payload = {"value": payload} + return event_type, payload, chat_id, sender_id + + def normalise( account: str, event: Any ) -> tuple[str, dict[str, Any], int | None, int | None] | None: - """Map one Telethon event onto `(type, payload, chat_id, sender_id)`. + """Map one Telethon *event or update* onto `(type, payload, chat, sender)`. - Returns None for an update tlgr has no name for yet; the full taxonomy is - PR-4's job and the bus must not invent type names in the meantime. + Raw updates go through `normalise_update`; the high-level event classes + still work because a gateway job, a test and the v1 code path all hand + them over, and dropping that would be a compatibility break with nothing + gained. """ - name = type(event).__name__ + if type(event).__name__ in eventtypes.CONSTRUCTORS or type(event).__name__ in ( + eventtypes.INTERNAL + ): + return normalise_update(account, event) + chat_id = getattr(event, "chat_id", None) if chat_id is not None: with contextlib.suppress(TypeError, ValueError): chat_id = int(chat_id) - if name in ("NewMessage.Event", "Event") and hasattr(event, "message"): - name = "NewMessage.Event" - kind = _event_kind(event) if kind is None: return None @@ -164,24 +332,26 @@ def normalise( if kind in ("message_new", "message_edited"): message = getattr(event, "message", None) payload = _message_payload(message, chat_id) if message is not None else {} + if message is not None and type(message).__name__ == "MessageService": + action = getattr(message, "action", None) + payload["action"] = type(action).__name__ if action is not None else "" + kind = "message_service" return kind, payload, chat_id, payload.get("sender_id") if kind == "message_deleted": ids = list(getattr(event, "deleted_ids", None) or []) return kind, {"message_ids": ids}, chat_id, None - if kind == "message_read": + if kind == "read": + outbox = bool(getattr(event, "outbox", False)) return ( - kind, - { - "max_id": getattr(event, "max_id", None), - "outbox": bool(getattr(event, "outbox", False)), - }, + "read_outbox" if outbox else "read_inbox", + {"max_id": getattr(event, "max_id", None), "outbox": outbox}, chat_id, None, ) - if kind == "chat_action": + if kind == "message_service": action_message = getattr(event, "action_message", None) action = type(getattr(action_message, "action", None)).__name__ if action_message else "" return ( @@ -212,7 +382,7 @@ def normalise( def _event_kind(event: Any) -> str | None: - """The tlgr type name for a Telethon event object. + """The tlgr type name for a Telethon *high-level* event object. Matched on the qualified class name rather than by `isinstance`, so this module — and therefore the bus — does not import Telethon at all and can @@ -223,8 +393,8 @@ def _event_kind(event: Any) -> str | None: ("newmessage", "message_new"), ("messageedited", "message_edited"), ("messagedeleted", "message_deleted"), - ("messageread", "message_read"), - ("chataction", "chat_action"), + ("messageread", "read"), + ("chataction", "message_service"), ("userupdate", "user_status"), ): if needle in qualname.lower(): @@ -267,6 +437,7 @@ def __init__( self._tasks: list[asyncio.Task[None]] = [] self._flush_task: asyncio.Task[None] | None = None self._dirty: set[str] = set() + self._raw_wanted = 0 self._running = False # -- lifecycle --------------------------------------------------------- @@ -367,6 +538,9 @@ def publish(self, envelope: EventEnvelope, raw: Any = None) -> None: buffer = self._buffers.setdefault(envelope.account, deque(maxlen=self.buffer_size)) buffer.append(envelope) + if raw is not None and self._raw_wanted: + envelope.raw = tl_to_builtins(getattr(raw, "original_update", raw)) + for subscriber in self._subscribers: if subscriber.closed: continue @@ -425,18 +599,24 @@ def subscribe( types: Iterable[str] = (), chats: Iterable[int] = (), maxsize: int = 1024, + want_raw: bool = False, ) -> Subscriber: subscriber = Subscriber( account=account, types=frozenset(types), chats=frozenset(int(c) for c in chats), maxsize=maxsize, + want_raw=want_raw, ) self._subscribers.append(subscriber) + if want_raw: + self._raw_wanted += 1 return subscriber def unsubscribe(self, subscriber: Subscriber) -> None: subscriber.close() + if subscriber.want_raw and self._raw_wanted: + self._raw_wanted -= 1 with contextlib.suppress(ValueError): self._subscribers.remove(subscriber) diff --git a/tlgr/daemon/ipc.py b/tlgr/daemon/ipc.py index f7dc30e..4f86469 100644 --- a/tlgr/daemon/ipc.py +++ b/tlgr/daemon/ipc.py @@ -12,7 +12,6 @@ from __future__ import annotations -import asyncio import json import logging from typing import TYPE_CHECKING, Any @@ -112,9 +111,10 @@ def register(self, app: web.Application) -> None: self._register_routes(app) def _register_routes(self, app: web.Application) -> None: - # Daemon - app.router.add_get("/daemon/status", self._daemon_status) - app.router.add_post("/daemon/stop", self._daemon_stop) + # The daemon and job routes are gone: `daemon status`, `daemon stop` + # and the whole `job` group are registry operations now, reachable at + # `POST /v1/op` and — for a v1 caller — at the same command paths + # through `legacy_paths` (§12.4). # Chats app.router.add_post("/chat/create", self._chat_create) @@ -138,22 +138,6 @@ def _register_routes(self, app: web.Application) -> None: # Media - # Jobs - app.router.add_get("/job/list", self._job_list) - app.router.add_post("/job/remove", self._job_remove) - app.router.add_post("/job/enable", self._job_enable) - app.router.add_post("/job/disable", self._job_disable) - app.router.add_post("/job/reload", self._job_reload) - - # -- Daemon -- - - async def _daemon_status(self, request: web.Request) -> web.Response: - return _json_response(self.daemon.status()) - - async def _daemon_stop(self, request: web.Request) -> web.Response: - asyncio.get_event_loop().call_soon(self.daemon.request_shutdown) - return _json_response({"stopping": True}) - # -- Chats -- async def _chat_create(self, request: web.Request) -> web.Response: @@ -326,30 +310,3 @@ async def _profile_update(self, request: web.Request) -> web.Response: return _json_response(result) except Exception as e: return _handle_exception(e) - - # -- Jobs -- - - async def _job_list(self, request: web.Request) -> web.Response: - return _json_response({"jobs": self.daemon.list_jobs()}) - - async def _job_remove(self, request: web.Request) -> web.Response: - body = await _get_body(request) - ok = await self.daemon.remove_job(body["name"]) - return _json_response({"removed": ok}) - - async def _job_enable(self, request: web.Request) -> web.Response: - body = await _get_body(request) - ok = await self.daemon.enable_job(body["name"]) - return _json_response({"enabled": ok}) - - async def _job_disable(self, request: web.Request) -> web.Response: - body = await _get_body(request) - ok = await self.daemon.disable_job(body["name"]) - return _json_response({"disabled": ok}) - - async def _job_reload(self, request: web.Request) -> web.Response: - try: - result = await self.daemon.reload_jobs() - return _json_response(result) - except Exception as e: - return _handle_exception(e) diff --git a/tlgr/daemon/main.py b/tlgr/daemon/main.py index 98891fb..e84a433 100644 --- a/tlgr/daemon/main.py +++ b/tlgr/daemon/main.py @@ -26,8 +26,8 @@ from tlgr.core.config import load_app_config from tlgr.core.errors import ConfigurationError from tlgr.core.paths import TlgrPaths, require_safe_permissions +from tlgr.core.process import daemonize, setup_logging, write_pid from tlgr.daemon.app import Daemon -from tlgr.daemon.lifecycle import daemonize, setup_logging, write_pid from tlgr.daemon.singleton import FileLock, LockBusy log = logging.getLogger("tlgr.daemon") @@ -87,7 +87,18 @@ def main(argv: list[str] | None = None) -> int: os.umask(0o077) base = Path(args.base).expanduser() if args.base else None - paths = TlgrPaths(base) + + # `TlgrPaths` refuses a home marked `.production`, and that refusal has to + # be an exit code rather than a traceback: two daemons on one home share + # session files, and Telegram treats a second client on the same auth key + # as a compromised session and revokes it. A development build started + # against somebody's live home does not degrade it, it breaks it. + try: + paths = TlgrPaths(base) + except ConfigurationError as exc: + print(f"tlgr daemon: {exc}", file=sys.stderr) + return 10 + paths.ensure_base() try: diff --git a/tlgr/daemon/ratelimit.py b/tlgr/daemon/ratelimit.py index 85c4dbd..d15e2d1 100644 --- a/tlgr/daemon/ratelimit.py +++ b/tlgr/daemon/ratelimit.py @@ -186,6 +186,39 @@ def clear(self) -> None: def snapshot(self) -> dict[str, int]: return {key: deadline.remaining for key, deadline in self._deadlines.items()} + def entries(self, *, include_expired: bool = False) -> list[FloodDeadline]: + """Every remembered deadline, for `tlgr daemon flood list`. + + Expired ones are hidden by default and available on request: "this + account hit a wait an hour ago" is diagnosis, not a live constraint, + and mixing the two makes a healthy account look throttled. + """ + now = time.time() + return [ + deadline + for deadline in self._deadlines.values() + if include_expired or deadline.until > now + ] + + def forget(self, *, method: str = "", peer: Any = None) -> int: + """Drop the deadlines matching *method*/*peer*; returns how many. + + Clearing a live server-side FLOOD_WAIT does not lift it — the next + call re-trips it. This exists for after the cause is fixed, and to + reset a breaker an operator has investigated. + """ + removed = 0 + for key, deadline in list(self._deadlines.items()): + if method and deadline.method != method: + continue + if peer is not None and deadline.peer != str(peer): + continue + del self._deadlines[key] + removed += 1 + if removed: + self.save() + return removed + @property def next_deadline(self) -> float | None: alive = [d.until for d in self._deadlines.values() if d.until > time.time()] diff --git a/tlgr/daemon/webhook.py b/tlgr/daemon/webhook.py index 1df190f..d47165d 100644 --- a/tlgr/daemon/webhook.py +++ b/tlgr/daemon/webhook.py @@ -24,8 +24,6 @@ import asyncio import contextlib -import hashlib -import hmac import json import logging import os @@ -39,6 +37,7 @@ from tlgr.core.config import CONFIG_DIR, WebhookConfig from tlgr.core.paths import write_private +from tlgr.core.signing import sign_body from tlgr.models.event import EventEnvelope log = logging.getLogger("tlgr.webhook") @@ -53,17 +52,6 @@ _DEAD_LETTER_BACKUPS = 3 -def sign_body(secret: str, body: bytes) -> str: - """`sha256=` over the exact bytes that go on the wire. - - Over the *bytes*, not over a re-encoded dict: a receiver verifies what it - received, and any re-encoding (key order, whitespace, escaping) makes an - honest signature fail. - """ - digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() - return f"sha256={digest}" - - class WebhookPusher: """Bounded queue + worker pool + HMAC + dead letter.""" @@ -239,9 +227,16 @@ async def _deliver(self, body: bytes, headers: dict[str, str]) -> None: def _dead_letter(self, body: bytes, headers: dict[str, str], reason: str) -> None: self.dead_letters += 1 + now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) record = { - "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "ts": now, "reason": reason, + # `source` names the consumer that failed. One store is shared by + # the pusher and the gateway actions, and an operator draining it + # has to be able to re-drive one without the other. + "source": "webhook", + "attempts": max(1, self.config.retry.max_attempts if self.config.retry.enabled else 1), + "first_failed_at": now, "delivery_id": headers.get("X-Tlgr-Delivery", ""), "seq": headers.get("X-Tlgr-Seq", ""), "event": headers.get("X-Tlgr-Event", ""), @@ -288,6 +283,65 @@ def read_dead_letters(self) -> list[dict[str, Any]]: continue return entries + @property + def dead_letter_path(self) -> Path: + return self._dead_letter_path + + def write_dead_letters(self, entries: list[dict[str, Any]]) -> None: + """Replace the store. Private mode, one write, no partial file.""" + body = "".join(json.dumps(entry, ensure_ascii=False) + "\n" for entry in entries) + write_private(self._dead_letter_path, body) + self.dead_letters = len(entries) + + async def deliver_once(self, entry: dict[str, Any], *, url: str = "") -> tuple[bool, str]: + """One delivery attempt for a stored entry. Returns `(ok, error)`. + + Re-delivery reuses the original `X-Tlgr-Delivery` id, so a receiver + keyed on it sees a duplicate rather than a new event — which is the + difference between a safe replay and a double-processed message. + """ + import aiohttp + + target = url or self.config.url + if not target: + return False, "no webhook URL is configured" + body = str(entry.get("body", "")).encode("utf-8") + headers = { + "Content-Type": "application/json", + "X-Tlgr-Delivery": str(entry.get("delivery_id", "")), + "X-Tlgr-Seq": str(entry.get("seq", "")), + "X-Tlgr-Event": str(entry.get("event", "")), + "X-Tlgr-Account": str(entry.get("account", "")), + "X-Tlgr-Redelivery": "1", + } + secret = self.config.signing_key + if secret: + headers["X-Tlgr-Signature"] = sign_body(secret, body) + if self.config.token: + headers["Authorization"] = f"Bearer {self.config.token}" + session = self._session + close_after = session is None + if session is None: + session = aiohttp.ClientSession() + try: + async with session.post( + target, + data=body, + headers=headers, + timeout=aiohttp.ClientTimeout(total=self.config.timeout), + ) as response: + if response.status < 400: + self.delivered += 1 + return True, "" + return False, f"HTTP {response.status}" + except asyncio.CancelledError: + raise + except Exception as exc: + return False, f"{type(exc).__name__}: {exc}" + finally: + if close_after: + await session.close() + def purge_dead_letters(self) -> int: if not self._dead_letter_path.exists(): return 0 diff --git a/tlgr/data/parity_waivers.toml b/tlgr/data/parity_waivers.toml index 7a5bbd4..657caaa 100644 --- a/tlgr/data/parity_waivers.toml +++ b/tlgr/data/parity_waivers.toml @@ -17,11 +17,6 @@ final_pr = 12 # Whole domains that no PR has migrated yet. Each becomes its own group PR. # --------------------------------------------------------------------------- -[[domain]] -name = "updates_sync_network" -pr = 4 -reason = "events, watch, daemon, sync, proxy and export land in PR-4." - [[domain]] name = "contacts_users" pr = 5 @@ -293,6 +288,26 @@ id = "messages-core.translate-channel-autotranslation" pr = 3 reason = "Channel auto-translation is a channel setting (PR-3)." +# --------------------------------------------------------------------------- +# updates_sync_network (PR-4). The domain waiver is gone; these three ids are +# each owned by another group's command and are waived to that group's PR. +# --------------------------------------------------------------------------- + +[[id]] +id = "updates.config-terms-of-service" +pr = 2 +reason = "Accepting the Terms of Service is part of sign-up; `auth tos` owns it (PR-2)." + +[[id]] +id = "updates.invoke-business-connection" +pr = 12 +reason = "Acting on behalf of a connected business account is the business surface (PR-12)." + +[[id]] +id = "updates.presence-group-online-count" +pr = 7 +reason = "The live 'N online' counter is a group-membership read (PR-7)." + [[id]] id = "messages-core.message-watch-events" pr = 4 diff --git a/tlgr/gateway/engine.py b/tlgr/gateway/engine.py index 1677fd2..f0d62b2 100644 --- a/tlgr/gateway/engine.py +++ b/tlgr/gateway/engine.py @@ -49,17 +49,28 @@ def __init__(self, gw: GatewayConfig) -> None: self.account = gw.account -#: v1's job event names → the bus taxonomy (§3.7), and back for the pipeline, -#: which still labels envelopes with v1's names. -_BUS_TYPE_MAP = { - "new_message": "message_new", +#: v1's job event names → the bus taxonomy, and back for the pipeline, which +#: still labels envelopes with v1's names. The expansion is the taxonomy's own +#: alias table (`core.eventtypes.ALIASES`), so a job and a `watch` accept the +#: same words; a job may also name any v2 type directly. +def _bus_types(names: list[str]) -> set[str]: + from tlgr.core import eventtypes + + wanted: set[str] = set() + for name in names: + wanted.update(eventtypes.ALIASES.get(name, (name,))) + return wanted + + +_V1_TYPE_MAP = { + "message_new": "new_message", "message_edited": "message_edited", "message_deleted": "message_deleted", - "chat_action": "chat_action", - "user_joined": "user_status", - "message_read": "message_read", + "message_service": "chat_action", + "user_status": "user_joined", + "read_inbox": "message_read", + "read_outbox": "message_read", } -_V1_TYPE_MAP = {v: k for k, v in _BUS_TYPE_MAP.items()} _EVENT_TYPE_MAP = { "new_message": (events.NewMessage, {}), @@ -106,7 +117,7 @@ async def run(self) -> None: async def _run_on_bus(self) -> None: """Subscribe to the daemon's bus instead of the update loop (ROB-02).""" - wanted = {_BUS_TYPE_MAP.get(name, name) for name in self._gw.events} + wanted = _bus_types(list(self._gw.events)) account = self._gw.account async def on_event(envelope, raw) -> None: diff --git a/tlgr/models/__init__.py b/tlgr/models/__init__.py index ac4ab48..e2f1ccf 100644 --- a/tlgr/models/__init__.py +++ b/tlgr/models/__init__.py @@ -82,6 +82,37 @@ VideoState, VolumeState, ) +from tlgr.models.config import ( + ConfigEntry, + ConfigKey, + ConfigPaths, + ConfigValue, + InitResult, + ValidationIssue, + ValidationReport, +) +from tlgr.models.daemon import ( + AccountHealth, + DaemonStatus, + DeadLetter, + DeadLetterResult, + EventBusStatus, + FloodRecord, + FloodResult, + HealthSummary, + Job, + JobState, + JobTestFrame, + LifecycleResult, + LogLine, + ReconnectedAccount, + ReconnectResult, + SavedState, + SaveStateResult, + ServiceResult, + WebhookProbe, + WebhookSettings, +) from tlgr.models.dialog import ( ActionBar, ArchiveResult, @@ -130,7 +161,14 @@ ) from tlgr.models.envelope import ErrEnvelope, Meta, OkEnvelope, OpRequest from tlgr.models.error import ErrorBody -from tlgr.models.event import EventEnvelope +from tlgr.models.event import DecodedEvent, EventEnvelope, EventType, EventTypeDetail +from tlgr.models.export import ( + ExportedFile, + ExportResult, + MessageRange, + TakeoutSession, + TakeoutStatus, +) from tlgr.models.location import ( GeoPoint, LiveLocation, @@ -152,10 +190,10 @@ ContentSettings, ContentSettingsSaved, Downloaded, - ExportResult, FileRef, MediaEdited, MediaEvent, + MediaExportResult, MediaFile, MediaInfo, MediaItem, @@ -216,6 +254,24 @@ ViewCount, WebPagePreview, ) +from tlgr.models.net import ( + AppConfigDoc, + Country, + CountryCode, + DcOption, + InfoTopic, + NearestDc, + NetStatus, + NetUsage, + PingResult, + PromoData, + Proxy, + ProxyLink, + ProxyProbe, + ProxySelection, + ServerConfig, + SyncCursors, +) from tlgr.models.page import Page, PageInfo from tlgr.models.peer import ( Chat, @@ -264,17 +320,27 @@ StickerSetOrder, StickerSetsChanged, ) +from tlgr.models.sync import ( + BackfillPage, + CatchUpResult, + ChannelState, + DifferenceResult, + ResetResult, + SyncStatus, +) from tlgr.models.todo import Todo, TodoTask __all__ = [ "MEDIA_NONE", "UNSET", "AccountDeletion", + "AccountHealth", "AccountRecord", "AccountState", "AccountTtl", "ActionBar", "ActiveCall", + "AppConfigDoc", "ArchiveResult", "ArchiveSettings", "AutoDownloadPreset", @@ -286,6 +352,7 @@ "AutoSaveSettings", "AutologinUrl", "AvailableReaction", + "BackfillPage", "Badge", "Button", "Call", @@ -301,9 +368,11 @@ "CallRef", "CallSignal", "CallUpgrade", + "CatchUpResult", "Catchup", "CatchupChat", "ChainBlock", + "ChannelState", "Chat", "ChatInfo", "ChatReactions", @@ -322,13 +391,25 @@ "ConferenceRelayed", "ConferenceRemoved", "ConferenceRevoked", + "ConfigEntry", + "ConfigKey", + "ConfigPaths", + "ConfigValue", "ContentSettings", "ContentSettingsSaved", + "Country", + "CountryCode", + "DaemonStatus", + "DcOption", + "DeadLetter", + "DeadLetterResult", + "DecodedEvent", "DeleteChatResult", "DeleteResult", "DeviceLock", "Dialog", "DiceCatalog", + "DifferenceResult", "Downloaded", "Draft", "DraftCleared", @@ -339,11 +420,17 @@ "EntityReport", "ErrEnvelope", "ErrorBody", + "EventBusStatus", "EventEnvelope", + "EventType", + "EventTypeDetail", "ExportResult", + "ExportedFile", "FactCheck", "FaveResult", "FileRef", + "FloodRecord", + "FloodResult", "Folder", "FolderBadge", "FolderDeleted", @@ -368,19 +455,28 @@ "GroupCallParticipant", "GroupCallSettings", "GroupCallStarted", + "HealthSummary", "ImportState", "InCallMessage", "InCallMessagesDeleted", + "InfoTopic", + "InitResult", + "Job", + "JobState", + "JobTestFrame", "LeaveResult", + "LifecycleResult", "LinkResult", "LiveLocation", "LiveStopped", + "LogLine", "LoginCodes", "LoginEmail", "LoginResult", "MapPreview", "MediaEdited", "MediaEvent", + "MediaExportResult", "MediaFile", "MediaInfo", "MediaItem", @@ -392,6 +488,7 @@ "MediaSummary", "Message", "MessageEntity", + "MessageRange", "MessageReactionState", "Meta", "Model", @@ -399,6 +496,9 @@ "MuteState", "Nearby", "NearbyPeer", + "NearestDc", + "NetStatus", + "NetUsage", "NotifySettings", "NotifyView", "OkEnvelope", @@ -432,6 +532,7 @@ "PhoneChange", "Photo", "PinResult", + "PingResult", "PinnedDialogs", "Poll", "PollOption", @@ -440,6 +541,11 @@ "Poster", "PosterReport", "Promo", + "PromoData", + "Proxy", + "ProxyLink", + "ProxyProbe", + "ProxySelection", "QrLogin", "RaisedHand", "ReactionPrivacy", @@ -453,20 +559,27 @@ "ReadReceipts", "ReadResult", "RecentResult", + "ReconnectResult", + "ReconnectedAccount", "RecoveryEmail", "ReplyHeader", "ReplyMarkup", "ReportResult", "Request", + "ResetResult", "Rights", "RtmpInfo", + "SaveStateResult", "SavedDialog", "SavedGif", + "SavedState", "ScheduledSent", "SecretChat", "SentCode", "SentLocation", + "ServerConfig", "ServiceAction", + "ServiceResult", "Session", "SessionChange", "SessionTermination", @@ -487,6 +600,10 @@ "Suggestion", "SummaryResult", "SupportInfo", + "SyncCursors", + "SyncStatus", + "TakeoutSession", + "TakeoutStatus", "TempPassword", "Terms", "ThemeResult", @@ -507,6 +624,8 @@ "Uploaded", "User", "UserRef", + "ValidationIssue", + "ValidationReport", "Venue", "VideoState", "ViewCount", @@ -520,6 +639,8 @@ "WebPagePreview", "WebSession", "WebSessionRevocation", + "WebhookProbe", + "WebhookSettings", "decode", "encode", "parse_message_link", diff --git a/tlgr/models/config.py b/tlgr/models/config.py new file mode 100644 index 0000000..b8f463d --- /dev/null +++ b/tlgr/models/config.py @@ -0,0 +1,101 @@ +"""Local configuration shapes: keys, effective values, paths, validation. + +`config get`/`set` are about **this installation** — identity, transport, +proxy, flood budget, presence policy, event buffer. The server's own +configuration is a different noun (`config server`, `config app`, +`config info`) and a different model file (`models/net.py`), because reading +`message_length_max` off Telegram and setting `daemon.idle_timeout` on this +machine have nothing in common but the word "config". +""" + +from __future__ import annotations + +from typing import Any + +from tlgr.models.base import Model + +__all__ = [ + "ConfigEntry", + "ConfigKey", + "ConfigPaths", + "ConfigValue", + "InitResult", + "ValidationIssue", + "ValidationReport", +] + + +class ConfigKey(Model): + """One documented knob, machine-readable so an agent need not read docs.""" + + key: str + type: str = "string" + default: Any = None + scope: str = "global" + section: str = "" + requires_restart: bool = False + secret: bool = False + help: str = "" + choices: list[str] = [] + + +class ConfigEntry(Model): + """One key's effective value, and where it came from.""" + + key: str + value: Any = None + default: Any = None + source: str = "default" + scope: str = "global" + + +class ConfigValue(Model): + """The result of a `config get` / `set` / `unset`.""" + + key: str + value: Any = None + previous: Any = None + default: Any = None + source: str = "" + help: str = "" + updated: bool = False + removed: bool = False + already: bool = False + requires_restart: bool = False + applied: bool = False + + +class ConfigPaths(Model): + """Where everything lives. Session and secrets files are credentials.""" + + config_dir: str = "" + config: str = "" + jobs: str = "" + webhook: str = "" + secrets: str = "" + sessions: str = "" + logs: str = "" + socket: str = "" + pid: str = "" + dead_letter: str = "" + path: str | None = None + + +class InitResult(Model): + created: list[str] = [] + skipped: list[str] = [] + path: str = "" + + +class ValidationIssue(Model): + file: str + message: str + key: str | None = None + + +class ValidationReport(Model): + ok: bool = True + valid: bool = True + files: list[str] = [] + errors: list[ValidationIssue] = [] + warnings: list[ValidationIssue] = [] diff --git a/tlgr/models/daemon.py b/tlgr/models/daemon.py new file mode 100644 index 0000000..e2438ec --- /dev/null +++ b/tlgr/models/daemon.py @@ -0,0 +1,320 @@ +"""Daemon, job, flood and dead-letter shapes. + +Everything an operator reads when something is wrong. Two of these deserve a +note: + +* `AccountHealth.state` is a **state machine**, not a boolean. v1 reported + which clients the daemon held, so an account whose connection had died + still appeared in `accounts` and the daemon still called itself healthy + (COR-13/COR-37). `state` plus `behind_seconds` is what makes "alive" and + "working" separable. +* `FloodRecord` exists because a flood deadline outlives a process. Telethon + remembers one in memory and forgets it on exit, so v1 re-hit every wait + after a restart — and re-hitting a wait is how a short one becomes long. +""" + +from __future__ import annotations + +from typing import Any + +from tlgr.models.base import Model + +__all__ = [ + "AccountHealth", + "DaemonStatus", + "DeadLetter", + "DeadLetterResult", + "EventBusStatus", + "FloodRecord", + "FloodResult", + "HealthSummary", + "Job", + "JobState", + "JobTestFrame", + "LifecycleResult", + "LogLine", + "ReconnectResult", + "ReconnectedAccount", + "SaveStateResult", + "SavedState", + "ServiceResult", + "WebhookProbe", + "WebhookSettings", +] + + +class AccountHealth(Model): + """One account's connection and sync health.""" + + alias: str + state: str = "unknown" + user_id: int | None = None + username: str | None = None + dc_id: int | None = None + proxy: str | None = None + ping_ms: float | None = None + pts: int | None = None + qts: int | None = None + seq: int | None = None + date: str | None = None + behind_seconds: int | None = None + catching_up: bool = False + channels_tracked: int = 0 + last_update_at: str | None = None + connected_since: str | None = None + reconnects: int = 0 + in_flight: int = 0 + resync_needed: list[int] = [] + flood_waits: int = 0 + circuit: str = "closed" + frozen: bool = False + error: str | None = None + + +class EventBusStatus(Model): + buffered: int = 0 + oldest_seq: int | None = None + last_seq: int = 0 + subscribers: int = 0 + dropped: int = 0 + + +class DaemonStatus(Model): + """`tlgr daemon status`. `running` and `healthy` are different questions.""" + + running: bool = False + ready: bool = False + healthy: bool = False + pid: int | None = None + uptime_seconds: int = 0 + version: str = "" + protocol: int = 0 + layer: int = 0 + socket: str = "" + socket_owner: int | None = None + managed_by: str | None = None + accounts: list[AccountHealth] = [] + events: EventBusStatus | None = None + webhook: dict[str, Any] = {} + jobs: list[dict[str, Any]] = [] + # v1's `/daemon/status` carried these two, and AGENT.md documents them. + connections: dict[str, bool] = {} + disconnected: list[str] = [] + + +class HealthSummary(Model): + """`tlgr status`: one screen, the states where everything else fails.""" + + account: str = "" + user_id: int | None = None + username: str | None = None + authorized: bool = False + connected: bool = False + dc_id: int | None = None + proxy: str | None = None + ping_ms: float | None = None + layer: int = 0 + behind_seconds: int | None = None + daemon_running: bool = False + daemon_healthy: bool = False + jobs_running: int = 0 + webhook_enabled: bool = False + flood_waits: int = 0 + frozen: dict[str, Any] = {} + unconfirmed_sessions: int = 0 + terms_pending: bool = False + problems: list[str] = [] + + +class LifecycleResult(Model): + """`daemon start` / `stop` / `restart`.""" + + started: bool = False + stopped: bool = False + restarted: bool = False + already: bool = False + pid: int | None = None + socket: str = "" + ready: bool = False + accounts: list[str] = [] + catch_up: bool = True + + +class ServiceResult(Model): + """`daemon install` / `uninstall`.""" + + installed: bool = False + uninstalled: bool = False + already: bool = False + supervisor: str = "" + unit: str = "" + path: str = "" + stopped: bool = False + + +class LogLine(Model): + ts: str = "" + level: str = "" + account: str | None = None + logger: str = "" + message: str = "" + raw: str = "" + + +class ReconnectedAccount(Model): + alias: str + reconnected: bool = False + dc_id: int | None = None + caught_up: bool = False + error: str | None = None + + +class ReconnectResult(Model): + accounts: list[ReconnectedAccount] = [] + + +class SavedState(Model): + alias: str + pts: int | None = None + qts: int | None = None + seq: int | None = None + date: str | None = None + channels: int = 0 + entities: int = 0 + error: str | None = None + + +class SaveStateResult(Model): + accounts: list[SavedState] = [] + + +class FloodRecord(Model): + """One remembered rate-limit deadline.""" + + account: str + #: No default: `Model` omits a field equal to its default, and a record + #: whose kind is absent reads as one that has no kind. + kind: str + method: str = "" + chat: str | None = None + wait_seconds: int = 0 + until: str | None = None + hits: int = 1 + circuit_open: bool = False + expired: bool = False + + +class FloodResult(Model): + cleared: int = 0 + circuit_open: bool = False + accounts: list[str] = [] + + +class DeadLetter(Model): + """An event a consumer could not be given.""" + + id: str + seq: int = 0 + source: str = "webhook" + event: str = "" + account: str = "" + chat_id: int | None = None + attempts: int = 1 + last_error: str = "" + first_failed_at: str = "" + last_failed_at: str = "" + + +class DeadLetterResult(Model): + attempted: int = 0 + delivered: int = 0 + failed: int = 0 + deleted: int = 0 + remaining: int = 0 + dry_run: bool = False + + +class JobState(Model): + """One gateway job.""" + + name: str + account: str = "" + enabled: bool = True + running: bool = False + events: list[str] = [] + filters: dict[str, Any] = {} + processors: list[str] = [] + actions: list[dict[str, Any]] = [] + matched: int = 0 + skipped: int = 0 + actions_run: int = 0 + errors: int = 0 + last_match_at: str | None = None + last_error: str | None = None + + +class Job(Model): + """The result of a job mutation.""" + + name: str + #: No default at all: this is the answer `job enable`/`job disable` was + #: asked for, and `Model` drops a field equal to its default — so either + #: value would sometimes be missing from the reply. + enabled: bool + account: str = "" + events: list[str] = [] + removed: bool = False + reloaded: bool = False + already: bool = False + loaded: int = 0 + added: list[str] = [] + removed_names: list[str] = [] + changed: list[str] = [] + errors: list[str] = [] + + +class JobTestFrame(Model): + """One event fed through a job, and what the pipeline decided.""" + + seq: int = 0 + event: str = "" + matched: bool = False + filter_trace: list[str] = [] + processed_text: str | None = None + actions: list[dict[str, Any]] = [] + + +class WebhookSettings(Model): + """`tlgr webhook get` / `set`. Secrets are redacted unless asked for.""" + + enabled: bool = False + url: str = "" + events: list[str] = [] + filters: dict[str, Any] = {} + sign: str = "hmac-sha256" + secret: str | None = None + token: str | None = None + max_attempts: int = 5 + backoff: int = 2 + timeout: int = 30 + queue: int = 10000 + on_lag: str = "drop" + batch: int = 1 + last_delivery_at: str | None = None + last_status: int | None = None + delivered: int = 0 + failed: int = 0 + dead_letters: int = 0 + queue_depth: int = 0 + last_seq: int = 0 + + +class WebhookProbe(Model): + """`tlgr webhook test`: the exact request a receiver would have to verify.""" + + url: str = "" + status: int | None = None + latency_ms: int = 0 + request_headers: dict[str, str] = {} + body: str = "" + error: str | None = None diff --git a/tlgr/models/event.py b/tlgr/models/event.py index 5a3db5d..d5bfa28 100644 --- a/tlgr/models/event.py +++ b/tlgr/models/event.py @@ -1,9 +1,12 @@ -"""The event envelope. +"""The event envelope, and the shapes that describe the event vocabulary. -Only the envelope is fixed here. The `type` vocabulary and per-type payloads -belong to `docs/design/EVENTS.md` and land with the updates group (PR-4); what -this file guarantees is that every event, whatever its type, is addressable by -`seq`, attributable to an account, and cheaply filterable by chat or sender. +The envelope is the wire shape every consumer sees. `EventType` and +`EventTypeDetail` are the *catalogue* shapes: what `tlgr events list` and +`tlgr events get` print so that an agent can discover the subscribable surface +before it opens a stream, rather than learning it from prose. + +The vocabulary itself lives in `tlgr/core/eventtypes.py`; only its shape is +here, because `models/` imports nothing from tlgr (§2.2). """ from __future__ import annotations @@ -12,7 +15,12 @@ from tlgr.models.base import Model -__all__ = ["EventEnvelope"] +__all__ = [ + "DecodedEvent", + "EventEnvelope", + "EventType", + "EventTypeDetail", +] class EventEnvelope(Model): @@ -26,3 +34,46 @@ class EventEnvelope(Model): # True when this event echoes an action tlgr itself performed, so a # gateway rule cannot loop by reacting to its own output. self_origin: bool = False + #: The raw TL update, JSON-safe, when the consumer asked for it + #: (`watch --with-raw`). Absent otherwise: it roughly doubles the frame. + raw: dict[str, Any] | None = None + + +class EventType(Model): + """One row of `tlgr events list`.""" + + type: str + group: str + summary: str + #: The `Update*` constructors that produce it. + sources: list[str] = [] + telethon: str = "" + #: Which sequence box orders it: pts, qts, seq, channel_pts, version, none. + box: str = "none" + bot_only: bool = False + #: 0 when this build can parse every source; 229 when Telegram has the + #: constructor and Telethon 1.44 does not. + since_layer: int = 0 + available: bool = True + derived: str = "" + + +class EventTypeDetail(EventType): + """`tlgr events get `: the row, plus the payload and an example.""" + + payload: dict[str, str] = {} + json_schema: dict[str, Any] | None = None + filters: list[str] = [] + example: EventEnvelope | None = None + + +class DecodedEvent(Model): + """`tlgr events decode`: one TL update or push payload, made a tlgr event.""" + + event: str + account: str = "" + chat_id: int | None = None + sender_id: int | None = None + data: dict[str, Any] = {} + raw: dict[str, Any] | None = None + push: bool = False diff --git a/tlgr/models/export.py b/tlgr/models/export.py new file mode 100644 index 0000000..96525b7 --- /dev/null +++ b/tlgr/models/export.py @@ -0,0 +1,66 @@ +"""Takeout (data export) shapes. + +A takeout session is a *mode*, not a request: once `account.initTakeoutSession` +returns an id, every subsequent call — `upload.getFile` included — has to be +wrapped in `invokeWithTakeout`, and `file_max_size` can never be changed. The +model therefore records the scope it was opened with, because a caller that +forgets it will simply get nothing back and no error. +""" + +from __future__ import annotations + +from tlgr.models.base import Model + +__all__ = [ + "ExportResult", + "ExportedFile", + "MessageRange", + "TakeoutSession", + "TakeoutStatus", +] + + +class TakeoutSession(Model): + takeout_id: int | None = None + scope: list[str] = [] + started_at: str | None = None + expires: str | None = None + max_file_size: int = 0 + #: TAKEOUT_INIT_DELAY_X: another logged-in session has to approve the + #: export first (24 h if there is none). Reported, never slept through + #: silently. + approval_required: bool = False + retry_after: int | None = None + already: bool = False + + +class MessageRange(Model): + min_id: int = 0 + max_id: int = 0 + + +class TakeoutStatus(Model): + #: No default: "is an export open" is the question, and a `Model` omits a + #: field equal to its default. + active: bool + takeout_id: int | None = None + started_at: str | None = None + scope: list[str] = [] + max_file_size: int = 0 + ranges: list[MessageRange] = [] + + +class ExportedFile(Model): + path: str + kind: str = "" + bytes: int = 0 + + +class ExportResult(Model): + written: int = 0 + files: list[ExportedFile] = [] + out: str = "" + finished: bool = False + takeout_id: int | None = None + success: bool = True + skipped: list[str] = [] diff --git a/tlgr/models/media.py b/tlgr/models/media.py index eaba226..818fc14 100644 --- a/tlgr/models/media.py +++ b/tlgr/models/media.py @@ -36,10 +36,10 @@ "ContentSettings", "ContentSettingsSaved", "Downloaded", - "ExportResult", "FileRef", "MediaEdited", "MediaEvent", + "MediaExportResult", "MediaFile", "MediaInfo", "MediaItem", @@ -260,7 +260,7 @@ class FileRef(Model): path: str | None = None -class ExportResult(Model): +class MediaExportResult(Model): job_id: str | None = None chat_id: int = 0 planned: int = 0 diff --git a/tlgr/models/net.py b/tlgr/models/net.py new file mode 100644 index 0000000..6be87bc --- /dev/null +++ b/tlgr/models/net.py @@ -0,0 +1,232 @@ +"""Network, data-centre, proxy and server-configuration shapes. + +The distinction these encode is the one v1 never made: **the server's +configuration** (`help.getConfig`, `help.getAppConfig`) is not the same thing +as **the connection** (which DC, which transport, how far the clock is off), +and neither is **the proxy** (a client-side list with credentials in a 0600 +file). Three nouns, three shapes. +""" + +from __future__ import annotations + +from typing import Any + +from tlgr.models.base import Model + +__all__ = [ + "AppConfigDoc", + "Country", + "CountryCode", + "DcOption", + "InfoTopic", + "NearestDc", + "NetStatus", + "NetUsage", + "PingResult", + "PromoData", + "Proxy", + "ProxyLink", + "ProxyProbe", + "ProxySelection", + "ServerConfig", + "SyncCursors", +] + + +class DcOption(Model): + id: int + ip_address: str = "" + port: int = 0 + ipv6: bool = False + media_only: bool = False + tcpo_only: bool = False + cdn: bool = False + static: bool = False + this_port_only: bool = False + secret: str | None = None + current: bool = False + + +class NearestDc(Model): + country: str = "" + this_dc: int = 0 + nearest_dc: int = 0 + current_dc: int | None = None + + +class PingResult(Model): + account: str = "" + dc_id: int | None = None + proxy: str | None = None + probes: int = 0 + min_ms: float | None = None + avg_ms: float | None = None + max_ms: float | None = None + loss: float = 0.0 + + +class SyncCursors(Model): + """pts/qts/seq/date, the four numbers the update transport turns on.""" + + pts: int | None = None + qts: int | None = None + seq: int | None = None + date: str | None = None + date_unix: int | None = None + + +class NetStatus(Model): + account: str = "" + authorized: bool = False + connected: bool = False + phase: str = "disconnected" + dc_id: int | None = None + dc_address: str | None = None + ipv6: bool = False + transport: str = "" + proxy: str | None = None + ping_ms: float | None = None + layer: int = 0 + #: |offset| > 30 s is reported as a warning: it pushes msg_ids outside the + #: server's window, and requests are then dropped with no error at all. + time_offset_seconds: int = 0 + exported_senders: int = 0 + reconnects: int = 0 + last_error: str | None = None + state: SyncCursors | None = None + behind_seconds: int | None = None + frozen: bool = False + + +class NetUsage(Model): + account: str = "" + since: str = "" + rpc_bytes_sent: int = 0 + rpc_bytes_received: int = 0 + download_bytes: int = 0 + upload_bytes: int = 0 + requests: int = 0 + updates_received: int = 0 + reconnects: int = 0 + + +class Proxy(Model): + """A saved proxy. Credentials live in a 0600 file and are never printed.""" + + id: str + name: str = "" + type: str = "socks5" + host: str = "" + port: int = 0 + user: str | None = None + rdns: bool = True + active: bool = False + order: int = 0 + last_ping_ms: float | None = None + last_ok_at: str | None = None + failures: int = 0 + has_password: bool = False + has_secret: bool = False + + +class ProxyProbe(Model): + id: str = "" + name: str = "" + ok: bool = False + ping_ms: float | None = None + dc_id: int | None = None + error: str | None = None + + +class ProxyLink(Model): + id: str + link: str + type: str = "" + host: str = "" + port: int = 0 + qr: str | None = None + + +class ProxySelection(Model): + active: str | None = None + type: str = "" + host: str = "" + port: int = 0 + reconnected: bool = False + accounts: list[str] = [] + removed: bool = False + was_active: bool = False + + +class ServerConfig(Model): + """`help.getConfig`, the fields a client actually acts on.""" + + expires: str | None = None + test_mode: bool = False + this_dc: int = 0 + date: str | None = None + date_unix: int | None = None + chat_size_max: int = 0 + megagroup_size_max: int = 0 + message_length_max: int = 0 + caption_length_max: int = 0 + online_update_period_ms: int = 0 + offline_blur_timeout_ms: int = 0 + offline_idle_timeout_ms: int = 0 + edit_time_limit: int = 0 + revoke_time_limit: int = 0 + rating_e_decay: int = 0 + forwarded_count_max: int = 0 + push_chat_period_ms: int = 0 + dc_options: list[DcOption] = [] + values: dict[str, Any] = {} + + +class AppConfigDoc(Model): + """`help.getAppConfig`, hash-cached, with the freeze fields lifted out.""" + + hash: int = 0 + not_modified: bool = False + freeze_since_date: str | None = None + freeze_until_date: str | None = None + freeze_appeal_url: str | None = None + values: dict[str, Any] = {} + config: ServerConfig | None = None + + +class CountryCode(Model): + country_code: str = "" + prefixes: list[str] = [] + patterns: list[str] = [] + + +class Country(Model): + iso2: str + name: str = "" + default_name: str = "" + hidden: bool = False + flag_emoji: str = "" + preferred_language: str = "" + codes: list[CountryCode] = [] + #: Filled only by `--phone`: what the number was classified as. + matched_prefix: str | None = None + valid: bool | None = None + + +class InfoTopic(Model): + """One of the flat `help.*` / `langpack.*` read-only endpoints.""" + + topic: str + items: list[dict[str, Any]] = [] + raw: dict[str, Any] = {} + + +class PromoData(Model): + kind: str = "none" + chat_id: int | None = None + psa_type: str | None = None + psa_message: str | None = None + proxy: bool = False + expires: str | None = None + hidden: bool = False + pending_suggestions: list[str] = [] diff --git a/tlgr/models/sync.py b/tlgr/models/sync.py new file mode 100644 index 0000000..a0b248e --- /dev/null +++ b/tlgr/models/sync.py @@ -0,0 +1,105 @@ +"""Update-state shapes: pts/qts/seq, per-channel boxes, and difference runs. + +`sync` is deliberately a different noun from `catchup`. `chat catchup` is the +unread digest a human reads; `sync catch-up` is `updates.getDifference`, the +plumbing that decides whether an event ever existed for the daemon at all. +Confusing them is how v1 ended up with an idle timeout that guaranteed a +permanent sync hole and a `catchup` command that could not close it. +""" + +from __future__ import annotations + +from typing import Any + +from tlgr.models.base import Model + +__all__ = [ + "BackfillPage", + "CatchUpResult", + "ChannelState", + "DifferenceResult", + "ResetResult", + "SyncStatus", +] + + +class ChannelState(Model): + chat_id: int + pts: int = 0 + #: False means catch-up will silently *skip* this channel: Telethon needs + #: an access hash in the session to call getChannelDifference at all. + access_hash_known: bool = False + last_difference_at: str | None = None + title: str | None = None + + +class SyncStatus(Model): + account: str = "" + pts: int | None = None + qts: int | None = None + seq: int | None = None + date: str | None = None + date_unix: int | None = None + unread_count: int | None = None + server_pts: int | None = None + server_seq: int | None = None + behind_pts: int | None = None + behind_seconds: int | None = None + phase: str = "unknown" + getting_difference: bool = False + channels: list[ChannelState] = [] + last_update_at: str | None = None + no_updates_for_seconds: int | None = None + + +class CatchUpResult(Model): + account: str = "" + events_replayed: int = 0 + pts_before: int | None = None + pts_after: int | None = None + duration_ms: int = 0 + too_long: bool = False + timed_out: bool = False + + +class DifferenceResult(Model): + """One `updates.getDifference` / `getChannelDifference` run, reported raw. + + Without `--apply` this is a *probe*: it does not advance the stored pts, + so running it cannot create the gap it was meant to diagnose. + """ + + #: `common` or `channel`. No default: the discriminator must never be the + #: field `omit_defaults` drops. + kind: str + #: Nor this one: "was that the whole difference" is what a caller loops on. + final: bool + new_pts: int | None = None + new_qts: int | None = None + new_seq: int | None = None + new_date: str | None = None + messages: int = 0 + other_updates: int = 0 + users: int = 0 + chats: int = 0 + timeout: int | None = None + too_long: bool = False + applied: bool = False + dry_run: bool = False + requests: list[dict[str, Any]] = [] + + +class ResetResult(Model): + account: str = "" + reset: bool = False + pts_before: int | None = None + pts_after: int | None = None + channels_reset: list[int] = [] + dry_run: bool = False + + +class BackfillPage(Model): + """The extra field a plain `Page[Message]` cannot carry: what was missing.""" + + fetched: int = 0 + missing_ids: list[int] = [] diff --git a/tlgr/ops/_spec.py b/tlgr/ops/_spec.py index 082d271..7d82fb8 100644 --- a/tlgr/ops/_spec.py +++ b/tlgr/ops/_spec.py @@ -8,7 +8,7 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum from typing import Any, Protocol, TypeAlias, runtime_checkable @@ -66,8 +66,21 @@ def emit(self, event_type: str, payload: dict[str, Any], **kwargs: Any) -> None: """ ... + def mark_already(self) -> None: + """Record that the world already looked the way the caller asked for. -Impl: TypeAlias = Callable[[Any, Any], Awaitable[Any]] + Part of the contract, not a convenience: COMMANDS.md promises that an + idempotent no-op reports `already: true` rather than pretending to + have done something, and an implementation cannot honour that without + a way to say so. + """ + ... + + +#: `Awaitable[Any] | AsyncIterator[Any]`: a streaming operation is an async +#: *generator*, which is not awaitable. Registry lint L6 is what keeps the two +#: kinds honest — `stream=True` must be an async generator and nothing else. +Impl: TypeAlias = Callable[[Any, Any], Any] @dataclass(frozen=True, slots=True) @@ -89,6 +102,12 @@ class OperationSpec: paginated: PageKind | None = None stream: bool = False needs_account: bool = True + #: False for a daemon operation that needs no *Telegram* client: reading + #: the event bus, the flood store, the dead-letter file, the job table. + #: Without it every one of those would connect an account to answer a + #: question about the daemon, and `--account all` could not be expressed + #: at all (there is no single session to acquire). + needs_client: bool = True needs_auth: bool = True surface: Surface = Surface.DAEMON idempotent: bool = False diff --git a/tlgr/ops/account.py b/tlgr/ops/account.py index b91b853..3604ddf 100644 --- a/tlgr/ops/account.py +++ b/tlgr/ops/account.py @@ -2433,6 +2433,10 @@ class SuggestionListReq(Request): dismiss: Annotated[ str | None, opt("--dismiss", metavar="NAME", help="Dismiss this suggestion.") ] = None + chat: Annotated[ + PeerRef | None, + opt("--chat", metavar="CHAT", kind="peer", help="Dismiss a per-chat suggestion."), + ] = None hide_promo: Annotated[ PeerRef | None, opt("--hide-promo", metavar="CHAT", kind="peer", help="Hide the promoted dialog."), @@ -2456,9 +2460,13 @@ async def suggestion_list(ctx: OpContext, req: SuggestionListReq) -> Page[Sugges f"{req.dismiss} cannot be dismissed; the server re-issues it until it is done", field="dismiss", ) - await client( - fn.DismissSuggestionRequest(peer=types.InputPeerEmpty(), suggestion=req.dismiss) + # A suggestion can be scoped to one chat (`PREMIUM_UPGRADE` on a + # channel, say); `--chat` dismisses that one rather than the + # account-wide nudge of the same name. + peer = ( + await _send.resolve(ctx, req.chat) if req.chat is not None else types.InputPeerEmpty() ) + await client(fn.DismissSuggestionRequest(peer=peer, suggestion=req.dismiss)) if req.hide_promo: await client(fn.HidePromoDataRequest(peer=await _send.resolve(ctx, req.hide_promo))) @@ -2505,7 +2513,7 @@ async def suggestion_list(ctx: OpContext, req: SuggestionListReq) -> Page[Sugges columns=("suggestion", "dismissible", "promo_peer"), example={"items": [{"suggestion": "VALIDATE_PASSWORD", "dismissible": True}]}, example_args="account suggestion list", - covers=("account.promo-data", "auth.security-suggestions"), + covers=("account.promo-data", "auth.security-suggestions", "updates.config-suggestions"), covers_partial=("password.check-remembered",), coverage_note="Actually checking the password is `account password get --verify`.", ) diff --git a/tlgr/ops/agent.py b/tlgr/ops/agent.py index e5de219..6b1d181 100644 --- a/tlgr/ops/agent.py +++ b/tlgr/ops/agent.py @@ -10,22 +10,27 @@ import contextlib from typing import Annotated, Any -from tlgr.core.errors import EXIT_CODE_MAP, UsageError +from tlgr.core.errors import EXIT_CODE_MAP, DaemonError, DaemonNotRunningError, UsageError from tlgr.models.base import Model, Request +from tlgr.models.daemon import HealthSummary from tlgr.ops._params import arg, choice, opt from tlgr.ops._spec import OpContext, OperationSpec, Surface __all__ = [ + "SPEC_CAPABILITIES", "SPEC_COMPLETION", "SPEC_EXIT_CODES", "SPEC_PARITY", "SPEC_SCHEMA", + "SPEC_STATUS", "SPEC_WHOAMI", + "Capabilities", "CompletionScript", + "ErrorEntry", "ExitCodeEntry", "ExitCodes", "SchemaDoc", - "Whoami", + "WhoAmI", ] @@ -34,24 +39,96 @@ class ExitCodeEntry(Model): description: str +class ErrorEntry(Model): + """One row of the RPC-error taxonomy (§7.2).""" + + name: str + code: str + exit: int + http: int + retryable: bool = False + hint: str = "" + #: The field a regex error captures: `FLOOD_WAIT_X` carries a wait, + #: `*_MIGRATE_X` a data centre, `FILE_PART_X_MISSING` a part number. An + #: agent that cannot see it can only retry blindly. + extra: str = "" + + class ExitCodes(Model): """Deliberately a mapping, not a list: this is the exact JSON v1 printed.""" exit_codes: dict[str, ExitCodeEntry] + errors: list[ErrorEntry] = [] class ExitCodesReq(Request): - pass + errors: Annotated[ + bool, + opt("--errors", help="Also print the RPC error to exit-code mapping."), + ] = False + search: Annotated[ + str | None, opt("--search", metavar="TEXT", help="Filter the error table.") + ] = None + + +#: Which regex-matched errors carry a parameter, and what it means. The number +#: in `FLOOD_WAIT_42` is not part of the name; dropping it would leave a +#: caller knowing it must wait and not for how long. +_ERROR_EXTRA: dict[str, str] = { + "FloodWaitError": "wait_seconds", + "SlowModeWaitError": "wait_seconds", + "FloodPremiumWaitError": "wait_seconds", + "FloodTestPhoneWaitError": "wait_seconds", + "TakeoutInitDelayError": "wait_seconds", + "PhoneMigrateError": "new_dc", + "NetworkMigrateError": "new_dc", + "UserMigrateError": "new_dc", + "FileMigrateError": "new_dc", + "FilePartMissingError": "which", +} + + +def _error_table() -> list[ErrorEntry]: + """The §7.2 mapping, rendered from the one table that implements it.""" + from tlgr.core.errors import ERROR_MAP + + rows = [ + ErrorEntry( + name=name, + code=rule.code, + exit=rule.exit_code, + http=rule.http, + retryable=rule.retryable, + hint=rule.hint, + extra=_ERROR_EXTRA.get(name, ""), + ) + for name, rule in ERROR_MAP.items() + ] + rows.sort(key=lambda row: (row.exit, row.name)) + return rows async def exit_codes(ctx: OpContext, req: ExitCodesReq) -> ExitCodes: - """Return the stable exit-code table.""" - return ExitCodes( + """Return the stable exit-code table, and optionally the error taxonomy. + + The exit codes are a compatibility contract: a code never changes meaning. + `--errors` adds the row *above* them — which Telethon exception becomes + which code — so an agent can decide whether to retry without catching the + exception itself. + """ + table = ExitCodes( exit_codes={ name: ExitCodeEntry(code=int(info["code"]), description=str(info["description"])) for name, info in EXIT_CODE_MAP.items() } ) + if req.errors: + rows = _error_table() + if req.search: + needle = req.search.lower() + rows = [row for row in rows if needle in row.name.lower() or needle in row.code.lower()] + table.errors = rows + return table SPEC_EXIT_CODES = OperationSpec( @@ -59,10 +136,14 @@ async def exit_codes(ctx: OpContext, req: ExitCodesReq) -> ExitCodes: request=ExitCodesReq, response=ExitCodes, impl=exit_codes, - summary="Print the stable exit codes for automation", + summary="Print the stable exit codes, and the RPC error to exit-code mapping", description=( "Every tlgr command exits with one of these codes. They are a " - "compatibility contract: a code never changes meaning." + "compatibility contract: a code never changes meaning. `--errors` " + "adds the row above them — which Telethon exception becomes which " + "code, whether it is retryable, and the parameter a regex error " + "carries (`FLOOD_WAIT_42` is a wait of 42 seconds, not a distinct " + "error)." ), aliases=("exit-codes",), legacy_paths=("agent exit-codes",), @@ -79,7 +160,8 @@ async def exit_codes(ctx: OpContext, req: ExitCodesReq) -> ExitCodes: } }, example_args="agent exit-codes", - tags=frozenset({"infrastructure", "agent-safe"}), + covers=("updates.net-error-taxonomy", "updates.net-migrate-errors"), + tags=frozenset({"agent-safe"}), ) @@ -91,6 +173,12 @@ class SchemaDoc(Model): ops: dict[str, Any] = {} +#: What `tlgr schema ` can print. `commands` is the default because it +#: is what the word meant in v1 and a documented spelling does not change +#: meaning underneath its callers (§12.4). +_SCHEMA_KINDS = ("commands", "events", "config", "errors", "exit-codes", "all") + + class SchemaReq(Request): path: Annotated[ tuple[str, ...], @@ -99,7 +187,10 @@ class SchemaReq(Request): metavar="PATH", required=False, variadic=True, - help="Limit the document to one command path, e.g. `schema message send`.", + help=( + "A schema kind (commands, events, config, errors, exit-codes, all), " + "or a command path to limit the document to, e.g. `schema message send`." + ), ), ] = () include_hidden: Annotated[ @@ -111,15 +202,63 @@ class SchemaReq(Request): async def schema(ctx: OpContext, req: SchemaReq) -> dict[str, Any]: """Build the machine-readable schema document. + The first positional does double duty, because v1's `tlgr schema message` + meant "the message commands" and `tlgr schema events` has to mean "the + event taxonomy". A kind name wins; anything else is a command path. The + two vocabularies do not overlap — no command group is called `errors` or + `exit-codes` — and `schema commands` still narrows the way v1 did. + The Click command tree is handed in through the context rather than imported: `ops/` must not import `cli/` (§2.2), and the tree is a CLI concern that only the CLI can describe. """ from tlgr.schema import build_schema + kind = req.path[0] if req.path and req.path[0] in _SCHEMA_KINDS else "" + path = req.path[1:] if kind else req.path + + if kind in ("events", "config", "errors", "exit-codes"): + return {"schema_version": 2, kind.replace("-", "_"): _schema_section(kind)} + provider = getattr(ctx, "command_tree", None) - command = provider(req.path, req.include_hidden) if callable(provider) else None - return build_schema(path=req.path, command=command, include_hidden=req.include_hidden) + command = provider(path, req.include_hidden) if callable(provider) else None + document = build_schema(path=path, command=command, include_hidden=req.include_hidden) + if kind == "all": + for section in ("events", "config", "errors", "exit-codes"): + document[section.replace("-", "_")] = _schema_section(section) + return document + + +def _schema_section(kind: str) -> Any: + """One non-command schema section, from the module that owns it.""" + from tlgr.models.base import to_builtins + + if kind == "events": + from tlgr.core import eventtypes + + return [ + { + "type": name, + "group": spec.group, + "box": spec.box, + "summary": spec.summary, + "sources": list(eventtypes.constructors_for(name)), + "bot_only": spec.bot_only, + "since_layer": spec.since_layer, + "payload": dict(spec.payload), + } + for name, spec in sorted(eventtypes.TYPES.items()) + ] + if kind == "config": + from tlgr.ops.config import KEYS + + return [to_builtins(key) for key in sorted(KEYS.values(), key=lambda k: k.key)] + if kind == "errors": + return [to_builtins(row) for row in _error_table()] + return { + name: {"code": int(info["code"]), "description": str(info["description"])} + for name, info in EXIT_CODE_MAP.items() + } SPEC_SCHEMA = OperationSpec( @@ -127,11 +266,14 @@ async def schema(ctx: OpContext, req: SchemaReq) -> dict[str, Any]: request=SchemaReq, response=dict, impl=schema, - summary="Print the machine-readable schema of the CLI", + summary="Print machine-readable schemas: commands, events, config keys, errors", description=( "One JSON document: the command tree, and for every registered " "operation its request and response JSON Schema plus a validated " - "example. Draft 2020-12." + "example. Draft 2020-12. `tlgr schema events`, `schema config`, " + "`schema errors` and `schema exit-codes` print the other four " + "vocabularies an agent has to know, and `schema all` prints " + "everything." ), legacy_paths=("schema",), needs_account=False, @@ -141,7 +283,9 @@ async def schema(ctx: OpContext, req: SchemaReq) -> dict[str, Any]: rate_class="local", timeout_s=30, example={"schema_version": 2, "build": "2.0.0", "ops": {}}, - example_args="schema message", + covers_partial=("updates.stream-event-types",), + coverage_note="prints the taxonomy; `events list` is its first-class surface.", + example_args="schema events", tags=frozenset({"infrastructure", "agent-safe", "json-only"}), ) @@ -209,137 +353,197 @@ async def parity(ctx: OpContext, req: ParityReq) -> dict[str, Any]: # --------------------------------------------------------------------------- -# agent whoami +# whoami # --------------------------------------------------------------------------- -#: Bumped when a documented output shape changes in a way a consumer must -#: notice. v2 is: RFC-3339 dates, marked chat ids, `Page` envelopes, `none` -#: as the default parse mode. -SCHEMA_VERSION = 2 - +class WhoAmI(Model, omit_defaults=False): + """What an agent needs before its first real command. -class Whoami(Model, omit_defaults=False): - """The one call an agent makes before it does anything else. - - `output_schema_version` is the field to branch on: v2 changed a handful - of documented shapes (RFC-3339 dates, marked chat ids, `Page` envelopes, - `none` as the default parse mode), and a consumer that reads this can - tell which set it is looking at without probing for one of them. + `output_schema_version` is the field to branch on: v2 changed a handful of + documented shapes (RFC-3339 dates, marked ids, `Page` envelopes, `none` as + the default parse mode), and a consumer must be able to tell which set it + is looking at without probing for one of them. `omit_defaults=False` for the whole struct: v1 printed `daemon_running` even when it was false, and a consumer that reads `info["daemon_running"]` must not get a KeyError because the answer happened to be "no". """ - #: No default on purpose: `Model` omits fields that equal their default, - #: and the one field an agent branches on must never be absent. + #: No default, deliberately: `Model` omits a field equal to its default, + #: and the one field a consumer branches on must never be absent. output_schema_version: int account: str = "" user_id: int | None = None username: str | None = None phone: str | None = None + is_bot: bool = False + premium: bool = False + frozen: bool = False daemon_running: bool = False - config_dir: str = "" - enabled_commands: list[str] = [] - daemon_uptime: float | None = None daemon_healthy: bool | None = None + daemon_version: str | None = None + daemon_uptime: int | None = None accounts_connected: list[str] = [] accounts_disconnected: list[str] = [] active_jobs: list[str] = [] + config_dir: str = "" + layer: int = 0 + telethon_version: str = "" + tlgr_version: str = "" + device_model: str = "" + app_version: str = "" + enabled_commands: list[str] = [] -class WhoamiReq(Request): +class WhoAmIReq(Request): pass -async def whoami(ctx: OpContext, req: WhoamiReq) -> Whoami: - """Who am I, is the daemon up, and what is this build allowed to do. +async def whoami(ctx: OpContext, req: WhoAmIReq) -> WhoAmI: + """Report the active account, the daemon's health and this client's identity. + + Local, and it must stay local: this is what an agent calls to find out + that the daemon is *not* running, so needing the daemon to answer would + make the question unanswerable. - Local on purpose: the answer has to be available *when the daemon is - not*, because "is it running?" is the question. The daemon is asked for - the connection map only when its pid file says there is one to ask. + `layer` is reported because it is the honest bound on everything else: a + build on Telethon's layer 227 meets constructors from layer 229 in the + wild and cannot parse them, and an agent that knows the number can predict + which features will be missing instead of discovering them one failure at + a time. """ + from tlgr import __version__ from tlgr.core.accounts import AccountManager - from tlgr.core.paths import TlgrPaths - - paths = TlgrPaths(getattr(getattr(ctx, "paths", None), "base", None)) - manager = AccountManager(paths.base) - alias = (getattr(ctx, "account", "") or "").strip() or (manager.get_active() or "") - record = manager.get_account(alias) if alias else None - info = Whoami( - output_schema_version=SCHEMA_VERSION, + from tlgr.core.identity import load_identity + from tlgr.core.paths import default_base + + base = default_base() + manager = AccountManager(base) + alias = ctx.account or manager.get_active() or "" + account = manager.get_account(alias) if alias else None + + info = WhoAmI( + output_schema_version=2, account=alias, - user_id=record.user_id if record else None, - username=record.username if record else None, - phone=record.phone if record else None, - daemon_running=_daemon_alive(paths), - config_dir=str(paths.base), + user_id=account.user_id if account else None, + username=account.username if account else None, + phone=account.phone if account else None, + frozen=bool(account and account.health.state == "frozen"), + config_dir=str(base), + layer=_layer(), + telethon_version=_telethon_version(), + tlgr_version=__version__, ) - if not info.daemon_running: - return info with contextlib.suppress(Exception): - from tlgr.transport.client import DaemonClient - - client = DaemonClient(paths.base, timeout=5.0, auto_start=False) - status = client.request("GET", "/v1/status") or {} - info.daemon_uptime = status.get("uptime_seconds") - info.daemon_healthy = status.get("healthy") - connections = status.get("connections") or {} - info.accounts_connected = sorted(a for a, ok in connections.items() if ok) - info.accounts_disconnected = sorted(a for a, ok in connections.items() if not ok) - info.active_jobs = [job["name"] for job in (status.get("jobs") or []) if job.get("running")] + identity = load_identity(base) + info.device_model = identity.device_model + info.app_version = identity.app_version + + enabled = getattr(ctx, "enable_commands", "") or "" + if enabled: + info.enabled_commands = [part.strip() for part in enabled.split(",") if part.strip()] + + status = _daemon_status() + if status is None: + return info + + daemon = status.get("daemon", {}) + info.daemon_running = True + info.daemon_version = daemon.get("version") + info.daemon_uptime = daemon.get("uptime_s") + rows = status.get("accounts", []) or [] + info.accounts_connected = sorted( + str(row.get("alias", "")) for row in rows if row.get("state") == "online" + ) + info.accounts_disconnected = sorted( + str(row.get("alias", "")) for row in rows if row.get("state") != "online" + ) + # `healthy` is about the accounts, not about the process: v1 reported every + # client the daemon held as connected, so a fully deaf daemon looked fine. + info.daemon_healthy = bool(daemon.get("ready")) and not info.accounts_disconnected + info.active_jobs = [ + str(job.get("name", "")) for job in (status.get("jobs") or []) if job.get("running") + ] + for row in rows: + if row.get("alias") == alias: + info.user_id = row.get("user_id") or info.user_id + info.username = row.get("username") or info.username + info.frozen = row.get("state") == "frozen" return info -def _daemon_alive(paths: Any) -> bool: - """Is there a live process behind the pid file? +def _telethon_layer() -> int: + return _layer() + + +def _layer() -> int: + with contextlib.suppress(Exception): + from telethon.tl.alltlobjects import LAYER + + return int(LAYER) + return 0 - A stale pid file is the normal aftermath of a crash, and reporting - `daemon_running: true` for one sends an agent into a retry loop against - a socket nobody is listening on. - """ - import os - try: - pid = int(paths.pid.read_text(encoding="utf-8").split()[0]) - except (OSError, ValueError, IndexError): - return False - try: - os.kill(pid, 0) - except OSError: - return False - return True +def _telethon_version() -> str: + with contextlib.suppress(Exception): + from tlgr.core.telethon_compat import telethon_version + + return telethon_version() + return "" + + +def _probe() -> dict[str, Any] | None: + """`/v1/status`, or None. Never starts a daemon to answer for it.""" + return _daemon_status() + + +def _daemon_status() -> dict[str, Any] | None: + """`/v1/status`, never starting a daemon to answer a question about it.""" + from tlgr.core.paths import default_base + from tlgr.transport.client import DaemonClient + + client = DaemonClient(default_base(), timeout=2.0, auto_start=False, no_restart=True) + with contextlib.suppress(Exception): + return client.probe_status() + return None SPEC_WHOAMI = OperationSpec( id="agent.whoami", - request=WhoamiReq, - response=Whoami, + request=WhoAmIReq, + response=WhoAmI, impl=whoami, - summary="Report the active account, daemon status and environment", + summary="Report the active account, daemon health and client identity", description=( - "The orientation call. `output_schema_version` is 2 for this build; " - "branch on it rather than probing for a renamed field." + "The first call an agent should make. `output_schema_version` says " + "which output contract it is talking to, and `layer` says how far " + "behind Telegram's current schema this build is." ), legacy_paths=("agent whoami",), needs_account=False, needs_auth=False, + needs_client=False, surface=Surface.LOCAL, idempotent=True, rate_class="local", - timeout_s=15, - columns=("account", "user_id", "username", "daemon_running"), + timeout_s=30, example={ "output_schema_version": 2, "account": "work", - "user_id": 4242, - "username": "me", + "user_id": 777, "daemon_running": True, - "config_dir": "/home/me/.tlgr", + "daemon_healthy": True, + "layer": 227, + "tlgr_version": "2.0.0", }, example_args="agent whoami", - tags=frozenset({"infrastructure", "agent-safe"}), + covers_partial=("updates.invoke-init-connection", "updates.invoke-with-layer"), + coverage_note=( + "reports the identity and layer this build declares; setting them is " + "`config set`, and `status` reports the connection." + ), + tags=frozenset({"agent-safe"}), ) @@ -408,3 +612,326 @@ async def completion(ctx: OpContext, req: CompletionReq) -> CompletionScript: example_args="completion bash", tags=frozenset({"infrastructure", "agent-safe", "text"}), ) + + +# --------------------------------------------------------------------------- +# capabilities +# --------------------------------------------------------------------------- + + +class Capabilities(Model): + """What this build can do, cannot do, and will not do. + + The third list is the one that matters. "Cannot" is a gap somebody may + close; "will not" is a decision, and an agent that cannot tell them apart + will keep asking for the second kind for ever. + """ + + layer: int = 0 + telethon_version: str = "" + tlgr_version: str = "" + event_types: int = 0 + unsupported_constructors: list[str] = [] + secret_chats: str = "" + pfs: str = "" + calls_media: str = "" + push: str = "" + presence_policy: str = "" + read_receipt_policy: str = "" + prohibited: list[dict[str, str]] = [] + premium_gated: list[str] = [] + bot_only: list[str] = [] + admin_only: list[str] = [] + limits: dict[str, Any] = {} + operations: int = 0 + + +class CapabilitiesReq(Request): + section: Annotated[ + str | None, + choice("protocol", "policy", "gates", "events", "limits", help="Restrict the report."), + ] = None + + +#: Things tlgr will not do, and why. Not a list of missing features: each of +#: these is a decision, and an agent that reads it stops asking. +_PROHIBITED: tuple[tuple[str, str], ...] = ( + ( + "fake a read receipt", + "Not calling messages.readHistory is fine — you simply did not read it. " + "Reading and then suppressing the receipt violates api terms 1.4.", + ), + ( + "suppress typing status", + "Same clause. tlgr sends setTyping when it is composing and never lies about it.", + ), + ( + "misrepresent online status", + "'Ghost mode' is explicitly forbidden by api terms 1.4. presence.mode defaults to " + "'off', which announces nothing rather than claiming to be offline while reading.", + ), + ( + "pass an integrity attestation", + "invokeWithGooglePlayIntegrity, invokeWithApnsSecret and invokeWithReCaptcha are " + "device-attestation flows. tlgr reports the demand and stops rather than " + "impersonating a phone.", + ), + ( + "execute a payment", + "Buying stars, paying an invoice, bidding in a gift auction and withdrawing " + "revenue are financial actions a person performs, not an agent.", + ), +) + +_PREMIUM_GATED = ( + "voice transcription", + "saved-message reaction tags", + "story stealth mode", + "uploading notification sounds", + "larger upload limits and folder counts", +) + +_BOT_ONLY = ( + "inline queries and callback answers", + "shipping and pre-checkout answers", + "business-connection messages", + "chat-boost updates", +) + +_ADMIN_ONLY = ( + "channel participant updates for other users", + "pending join requests", + "the admin log", +) + + +async def capabilities(ctx: OpContext, req: CapabilitiesReq) -> Capabilities: + """The honest-limits report, meant to be read before planning. + + Everything here is derived or fixed, never guessed: the layer and the + unparseable constructors come from the event taxonomy, the operation count + from the registry, and the policy entries are the decisions recorded in + ARCHITECTURE and the API terms. + """ + from tlgr import __version__ + from tlgr.core import eventtypes + from tlgr.registry import REGISTRY + + report = Capabilities( + layer=_layer(), + telethon_version=_telethon_version(), + tlgr_version=__version__, + event_types=len(eventtypes.TYPES), + unsupported_constructors=sorted(eventtypes.NEWER_THAN_LAYER_227), + operations=len(REGISTRY), + secret_chats=( + "envelope only: Telethon implements no MTProto 2.0 end-to-end layer, so tlgr " + "can report that encrypted traffic exists and acknowledge the qts, and cannot " + "read or send it" + ), + pfs=( + "not implemented: auth.bindTempAuthKey needs changes inside Telethon's " + "MTProtoSender. The practical mitigation is protecting the session file, which " + "is written 0600 and audited at start" + ), + calls_media=( + "signalling only: ring, accept, reject and hang up work; carrying the audio or " + "video stream needs tgcalls and is out of scope for a CLI" + ), + push=( + "not registered: the daemon holds a socket, so it has no need of push. " + "`tlgr events decode --push` reads a payload a phone relayed" + ), + presence_policy=( + "presence.mode defaults to 'off': tlgr announces nothing rather than claiming " + "to be offline while reading" + ), + read_receipt_policy=( + "never faked: reading without calling messages.readHistory is allowed, " + "suppressing a receipt after reading is not" + ), + prohibited=[{"action": action, "reason": reason} for action, reason in _PROHIBITED], + premium_gated=list(_PREMIUM_GATED), + bot_only=list(_BOT_ONLY), + admin_only=list(_ADMIN_ONLY), + ) + + if req.section: + return _section(report, req.section) + return report + + +def _section(report: Capabilities, section: str) -> Capabilities: + """Blank everything outside the requested section, keeping the shape.""" + keep = { + "protocol": {"layer", "telethon_version", "tlgr_version", "unsupported_constructors"}, + "policy": {"prohibited", "presence_policy", "read_receipt_policy"}, + "gates": {"premium_gated", "bot_only", "admin_only"}, + "events": {"event_types", "unsupported_constructors"}, + "limits": {"limits", "operations"}, + }[section] + trimmed = Capabilities() + for field in keep: + setattr(trimmed, field, getattr(report, field)) + return trimmed + + +SPEC_CAPABILITIES = OperationSpec( + id="agent.capabilities", + request=CapabilitiesReq, + response=Capabilities, + impl=capabilities, + summary="Report what this build can do, cannot do, and will not do", + description=( + "Three different things, deliberately separated. `unsupported_*` is " + "what this Telethon layer cannot parse; `premium_gated`/`bot_only`/" + "`admin_only` is what this *account* may not reach; `prohibited` is " + "what tlgr refuses on purpose, with the reason. Only the first is a " + "gap somebody might close." + ), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=30, + example={ + "layer": 227, + "tlgr_version": "2.0.0", + "event_types": 114, + "prohibited": [{"action": "fake a read receipt", "reason": "api terms 1.4"}], + }, + example_args="agent capabilities --section policy", + covers=("updates.session-pfs", "updates.sync-disable-updates"), + covers_partial=( + "updates.invoke-with-layer", + "updates.ops-single-updates-consumer", + "updates.presence-read-receipts-policy", + "updates.sync-old-layer-socket-reset", + ), + coverage_note=( + "states the policy and the layer bound; the switches themselves are " + "`config set`, and recovery is `daemon reconnect`." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# status — the one-screen summary +# --------------------------------------------------------------------------- + + +class HealthReq(Request): + check: Annotated[bool, opt("--check", help="Exit non-zero when anything is unhealthy.")] = False + + +async def account_status(ctx: OpContext, req: HealthReq) -> HealthSummary: + """One screen: account, connection, sync lag, daemon, floods. + + Deliberately the union of several groups rather than a link to them. The + states it surfaces — frozen, terms not accepted, an unconfirmed new login, + a flood the account still owes — are the ones in which *every other + command* starts failing, and a user whose sends are being refused should + not have to know which of five nouns to ask first. + """ + from tlgr.core.accounts import AccountManager + from tlgr.core.paths import default_base + + base = default_base() + manager = AccountManager(base) + alias = ctx.account or manager.get_active() or "" + account = manager.get_account(alias) if alias else None + + summary = HealthSummary( + account=alias, + user_id=account.user_id if account else None, + username=account.username if account else None, + layer=_telethon_layer(), + ) + + status = _probe() + if status is None: + summary.problems.append("the daemon is not running (tlgr daemon start)") + if req.check: + raise DaemonNotRunningError("the daemon is not answering on its socket") + return summary + + daemon = status.get("daemon", {}) + summary.daemon_running = True + summary.jobs_running = len([j for j in (status.get("jobs") or []) if j.get("running")]) + summary.webhook_enabled = bool((status.get("webhook") or {}).get("enabled")) + + rows = [row for row in (status.get("accounts") or []) if not alias or row.get("alias") == alias] + row = rows[0] if rows else {} + state = str(row.get("state", "unknown")) + summary.authorized = state not in ("needs_login", "unknown", "not_connected") + summary.connected = state == "online" + summary.dc_id = row.get("dc_id") + summary.proxy = row.get("proxy") + summary.behind_seconds = row.get("behind_seconds") + summary.flood_waits = int(row.get("flood_entries") or 0) + summary.daemon_healthy = bool(daemon.get("ready")) and state == "online" + + if state == "needs_login": + summary.problems.append(f"{alias} needs to log in again (tlgr account add)") + if state == "frozen": + summary.frozen = {"state": "frozen", "reason": row.get("reason") or ""} + summary.problems.append( + f"{alias} is frozen by Telegram; see `tlgr config app get --frozen` for the appeal link" + ) + if str(row.get("circuit", "closed")) != "closed": + summary.problems.append( + f"the send circuit breaker is open for {alias}: {row.get('circuit_reason') or 'spam flagged'}" + ) + if summary.flood_waits: + summary.problems.append( + f"{summary.flood_waits} rate-limit deadline(s) outstanding (tlgr daemon flood list)" + ) + if not daemon.get("ready"): + summary.problems.append("the daemon is running but not ready") + + if req.check and summary.problems: + raise DaemonError("; ".join(summary.problems)) + return summary + + +SPEC_STATUS = OperationSpec( + id="agent.status", + request=HealthReq, + response=HealthSummary, + impl=account_status, + summary="One-screen health summary: account, connection, sync lag, daemon, floods", + description=( + "The union of several groups on purpose. A frozen account, an open " + "circuit breaker, an outstanding flood deadline and a daemon that is " + "up but not ready are the states in which every *other* command " + "starts failing, and `--check` turns them into an exit code a monitor " + "can read." + ), + legacy_paths=("status",), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("account", "connected", "daemon_healthy", "behind_seconds", "problems"), + example={ + "account": "work", + "authorized": True, + "connected": True, + "daemon_running": True, + "daemon_healthy": True, + "layer": 227, + }, + example_args="status --check", + covers=("updates.invoke-with-layer",), + covers_partial=("updates.config-account-frozen", "updates.net-flood-wait"), + coverage_note=( + "surfaces the states; the detail is `config app get --frozen` and `daemon flood list`." + ), + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/ops/config.py b/tlgr/ops/config.py new file mode 100644 index 0000000..7dda924 --- /dev/null +++ b/tlgr/ops/config.py @@ -0,0 +1,1698 @@ +"""The `config` group: this installation's settings, and Telegram's own. + +Four different things wear the word "config", and keeping them apart is most +of this module's job: + +* `config get/set/list/keys/path/init/validate` — **local** settings, in + `config.toml`. Identity, transport, proxy, flood budget, presence policy, + event buffer. +* `config server get` — Telegram's MTProto configuration (`help.getConfig`): + `message_length_max`, `edit_time_limit`, the DC list. +* `config app get` — the client configuration (`help.getAppConfig`): the + limits and kill switches almost every feature is gated on, plus the + account-freeze fields that turn a bare `FROZEN_METHOD_INVALID` into + something actionable. +* `config info get`, `config country list`, `config promo get`, + `config suggestion list` — the flat read-only `help.*` endpoints. + +v1 had nine documented keys and parsed the file with `raw.get(x, default)` at +every call site, so a typo was silently the default. `config keys` is now +machine-readable and `config set` validates against it, which is the +difference between "tlgr ignored your setting" and an error naming the key. +""" + +from __future__ import annotations + +import contextlib +from typing import Annotated, Any + +from tlgr.core.errors import EXIT_EMPTY, NotFoundError, UsageError +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.config import ( + ConfigEntry, + ConfigKey, + ConfigPaths, + ConfigValue, + InitResult, + ValidationIssue, + ValidationReport, +) +from tlgr.models.net import ( + AppConfigDoc, + Country, + CountryCode, + DcOption, + InfoTopic, + PromoData, + ServerConfig, +) +from tlgr.models.page import Page +from tlgr.ops._params import arg, choice, opt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + + +# --------------------------------------------------------------------------- +# The key catalogue (§10.2) +# --------------------------------------------------------------------------- + + +def _key( + key: str, + section: str, + field: str, + type_: str, + default: Any, + help_: str, + *, + scope: str = "global", + restart: bool = False, + secret: bool = False, + choices: tuple[str, ...] = (), +) -> tuple[ConfigKey, tuple[str, str]]: + return ( + ConfigKey( + key=key, + type=type_, + default=default, + scope=scope, + section=section, + requires_restart=restart, + secret=secret, + help=help_, + choices=list(choices), + ), + (section, field), + ) + + +#: Every documented knob, machine-readable so an agent can discover them +#: without reading prose. `requires_restart` is not decoration: an identity or +#: transport key only takes effect on the next `initConnection`, and a `set` +#: that pretended otherwise would be a lie about what happened. +_CATALOGUE: tuple[tuple[ConfigKey, tuple[str, str]], ...] = ( + # -- identity: what Settings → Devices shows the user ------------------ + _key( + "client.device_model", + "identity", + "device_model", + "string", + "", + "Device name shown in Settings → Devices. Must be honest: it is how a " + "user recognises — and safely terminates — the tlgr session.", + restart=True, + ), + _key( + "client.system_version", + "identity", + "system_version", + "string", + "", + "System version sent in initConnection.", + restart=True, + ), + _key( + "client.lang_code", + "identity", + "lang_code", + "string", + "", + "Language tlgr asks the server to localise its own error and service messages into.", + restart=True, + ), + _key( + "client.system_lang_code", + "identity", + "system_lang_code", + "string", + "", + "System language sent in initConnection.", + restart=True, + ), + _key( + "client.tz_offset", + "identity", + "tz_offset", + "bool", + True, + "Send this host's UTC offset in initConnection params. Business hours " + "and scheduled-message display depend on it.", + restart=True, + ), + # -- network ----------------------------------------------------------- + _key( + "net.proxy", + "network", + "proxy", + "string", + "", + "Active proxy URL. Prefer `tlgr proxy set`, which reconnects for you.", + restart=True, + secret=True, + ), + _key( + "net.ipv6", + "network", + "ipv6", + "bool", + False, + "Connect over IPv6. One argument; matters on IPv6-only hosts.", + restart=True, + ), + _key( + "net.connect_timeout", + "network", + "connect_timeout", + "int", + 10, + "Seconds to wait for a connection before giving up.", + restart=True, + ), + _key( + "net.connection", + "network", + "connection", + "string", + "tcp_full", + "MTProto transport: tcp_full, tcp_abridged, tcp_intermediate, " + "tcp_obfuscated or http. Obfuscated helps on hostile networks.", + restart=True, + choices=("tcp_full", "tcp_abridged", "tcp_intermediate", "tcp_obfuscated", "http"), + ), + # -- daemon ------------------------------------------------------------ + _key("daemon.auto_start", "daemon", "auto_start", "bool", True, "Start the daemon on CLI use."), + _key( + "daemon.log_level", + "daemon", + "log_level", + "string", + "info", + "Daemon log level.", + choices=("debug", "info", "warning", "error"), + ), + _key( + "daemon.idle_timeout", + "daemon", + "idle_timeout", + "int", + 1800, + "Seconds of inactivity before the daemon stops; 0 disables. An idle " + "stop with catch-up off is a permanent sync hole.", + ), + _key( + "daemon.event_buffer", + "daemon", + "event_buffer", + "int", + 4096, + "Events kept per account for `--since` replay. Older ones produce a " + "`gap` frame rather than silence.", + restart=True, + ), + _key( + "daemon.event_workers", + "daemon", + "event_workers", + "int", + 8, + "Worker lanes the bus dispatches handlers on, keyed by chat.", + restart=True, + ), + _key( + "daemon.state_save_interval", + "daemon", + "state_save_interval", + "int", + 60, + "How often pts/qts and the entity cache are flushed. Telethon only " + "writes them on a clean disconnect.", + ), + _key( + "daemon.drain_seconds", + "daemon", + "drain_seconds", + "int", + 30, + "How long a shutdown waits for in-flight requests.", + ), + _key( + "daemon.resync_depth", + "daemon", + "resync_depth", + "int", + 50, + "Messages re-read per channel after a differenceTooLong.", + ), + # -- flood ------------------------------------------------------------- + _key( + "flood.sleep_threshold", + "flood", + "sleep_threshold", + "int", + 120, + "Seconds of FLOOD_WAIT tlgr will sleep off inside a request. Longer " + "waits come back as RATE_LIMITED with the deadline.", + ), + _key( + "flood.max_wait", + "flood", + "max_wait", + "int", + 600, + "Ceiling on the sleep threshold, whatever a caller asks for.", + ), + _key( + "flood.persist", + "flood", + "persist", + "bool", + True, + "Remember flood deadlines across restarts. Off means a fresh process " + "re-trips every wait it had already earned.", + ), + # -- presence ---------------------------------------------------------- + _key( + "presence.mode", + "presence", + "mode", + "string", + "off", + "off | online | mirror. Default off: tlgr announces nothing rather " + "than claiming to be offline while reading, which api terms 1.4 " + "forbids.", + choices=("off", "online", "mirror"), + ), + # -- limits ------------------------------------------------------------ + _key( + "limits.entity_cache", + "limits", + "entity_cache", + "int", + 20000, + "Peers Telethon keeps access hashes for in memory.", + restart=True, + ), + _key( + "limits.request_retries", + "limits", + "request_retries", + "int", + 5, + "Attempts per request before giving up.", + restart=True, + ), + _key( + "limits.dialog_scan_max", + "limits", + "dialog_scan_max", + "int", + 5000, + "How many dialogs a peer resolution will scan before answering " + "INDETERMINATE rather than 'not found'.", + ), + _key( + "limits.max_album", + "limits", + "max_album", + "int", + 10, + "Maximum files in one album.", + ), + # -- defaults ---------------------------------------------------------- + _key( + "defaults.output", + "defaults", + "output", + "string", + "human", + "Default output mode.", + choices=("human", "json", "plain"), + ), + _key( + "defaults.parse_mode", + "defaults", + "parse_mode", + "string", + "none", + "Default message parse mode. `none` because v1's markdown default " + "silently ate underscores and asterisks in ordinary text.", + choices=("none", "md", "html"), + ), + _key( + "defaults.require_account", + "defaults", + "require_account", + "bool", + False, + "Require -a on every command instead of falling back to a default.", + ), + _key( + "defaults.confirm_destructive", + "defaults", + "confirm_destructive", + "bool", + True, + "Prompt before a destructive command off a TTY.", + ), + _key( + "defaults.timezone", + "defaults", + "timezone", + "string", + "", + "Timezone for human date rendering. Empty means the host's.", + ), + _key( + "defaults.legacy_dates", + "defaults", + "legacy_dates", + "bool", + False, + "Print v1's `str(datetime)` spelling instead of RFC-3339.", + ), + _key( + "accounts.default", + "accounts", + "default", + "string", + "", + "Account used when -a is not given.", + ), + # -- security / policy ------------------------------------------------- + _key( + "security.require_token", + "security", + "require_token", + "bool", + False, + "Require X-Tlgr-Token on the IPC socket as well as the peer-uid check.", + restart=True, + ), + _key( + "security.peer_uid_check", + "security", + "peer_uid_check", + "bool", + True, + "Refuse socket connections from another uid.", + restart=True, + ), + _key( + "logging.redact", + "logging", + "redact", + "bool", + True, + "Redact access hashes, tokens and secrets from the log.", + ), +) + +KEYS: dict[str, ConfigKey] = {entry[0].key: entry[0] for entry in _CATALOGUE} +_FIELDS: dict[str, tuple[str, str]] = {entry[0].key: entry[1] for entry in _CATALOGUE} + +#: v1 spelled nine of these without a section prefix. §12.4: a documented name +#: does not stop working because the catalogue grew a namespace. +_LEGACY_KEYS: dict[str, str] = { + "output": "defaults.output", + "drop_author": "defaults.drop_author", + "delete_after": "defaults.delete_after", + "default_account": "accounts.default", + "require_account": "defaults.require_account", + "auto_start": "daemon.auto_start", + "log_level": "daemon.log_level", + "idle_timeout": "daemon.idle_timeout", + "flood_wait_max": "flood.sleep_threshold", +} + + +def _resolve_key(name: str) -> ConfigKey: + key = _LEGACY_KEYS.get(name, name) + found = KEYS.get(key) + if found is None: + raise NotFoundError(f"unknown config key {name!r}. Run: tlgr config keys") + return found + + +def _paths() -> Any: + from tlgr.core.paths import TlgrPaths + + return TlgrPaths() + + +def _raw() -> dict[str, Any]: + from tlgr.core.config import _load_toml + + return _load_toml(_paths().config) + + +def _write(document: dict[str, Any]) -> None: + from tlgr.core.config import _save_toml + + _save_toml(_paths().config, document) + + +def _stored(document: dict[str, Any], key: ConfigKey) -> tuple[Any, bool]: + section, field = _FIELDS[key.key] + block = document.get(section) + if isinstance(block, dict) and field in block: + return block[field], True + return key.default, False + + +def _coerce(key: ConfigKey, raw: str) -> Any: + """A CLI string into the key's declared type, or a usage error naming it.""" + if key.type == "bool": + lowered = raw.strip().lower() + if lowered in ("true", "yes", "on", "1"): + return True + if lowered in ("false", "no", "off", "0"): + return False + raise UsageError(f"{key.key} is a boolean; got {raw!r}", field="value") + if key.type == "int": + try: + return int(raw) + except ValueError as exc: + raise UsageError(f"{key.key} is an integer; got {raw!r}", field="value") from exc + if key.choices and raw not in key.choices: + raise UsageError( + f"{key.key} must be one of {', '.join(key.choices)}; got {raw!r}", field="value" + ) + return raw + + +def _redact(key: ConfigKey, value: Any) -> Any: + return "" if key.secret and value else value + + +# --------------------------------------------------------------------------- +# Local settings +# --------------------------------------------------------------------------- + + +class ConfigGetReq(Request): + key: Annotated[str, arg(0, metavar="KEY")] + source: Annotated[ + bool, opt("--source", help="Also report which file and section the value came from.") + ] = False + + +async def config_get(ctx: OpContext, req: ConfigGetReq) -> ConfigValue: + """Read one local key. Server-side configuration is a different noun.""" + key = _resolve_key(req.key) + value, present = _stored(_raw(), key) + return ConfigValue( + key=key.key, + value=_redact(key, value), + default=key.default, + source=(f"{_paths().config} [{key.section}]" if present else "default") + if req.source + else ("file" if present else "default"), + help=key.help, + requires_restart=key.requires_restart, + ) + + +SPEC_CONFIG_GET = OperationSpec( + id="config.get", + request=ConfigGetReq, + response=ConfigValue, + impl=config_get, + summary="Read one local configuration key", + legacy_paths=("config get",), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + columns=("key", "value", "source"), + example={"key": "daemon.idle_timeout", "value": 0, "default": 1800, "source": "file"}, + example_args="config get daemon.idle_timeout", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigSetReq(Request): + key: Annotated[str, arg(0, metavar="KEY")] + value: Annotated[str, arg(1, metavar="VALUE")] + apply: Annotated[ + bool, opt("--apply/--no-apply", help="Ask a running daemon to adopt the change now.") + ] = True + + +async def config_set(ctx: OpContext, req: ConfigSetReq) -> ConfigValue: + """Write one local key, validated against the catalogue. + + Validation is the point. v1 read the file with `raw.get(key, default)` at + every call site, so a typo or a wrong type was silently the default and + the user's setting simply never happened. + """ + key = _resolve_key(req.key) + section, field = _FIELDS[key.key] + value = _coerce(key, req.value) + document = _raw() + previous, present = _stored(document, key) + if present and previous == value: + ctx.mark_already() + return ConfigValue( + key=key.key, + value=_redact(key, value), + previous=_redact(key, previous), + already=True, + requires_restart=key.requires_restart, + ) + + document.setdefault(section, {})[field] = value + _write(document) + + applied = False + if req.apply: + applied = _reload_daemon() + if key.requires_restart and not applied: + ctx.warn(f"{key.key} takes effect on the next reconnect: tlgr daemon reconnect") + return ConfigValue( + key=key.key, + value=_redact(key, value), + previous=_redact(key, previous) if present else None, + default=key.default, + updated=True, + requires_restart=key.requires_restart, + applied=applied, + ) + + +def _reload_daemon() -> bool: + """Ask a running daemon to re-read the file. Absent daemon is not an error.""" + from tlgr.core.paths import default_base + from tlgr.transport.client import DaemonClient + + client = DaemonClient(default_base(), timeout=10.0, auto_start=False, no_restart=True) + with contextlib.suppress(Exception): + client.admin("reload", {"what": ["config", "policy"]}) + return True + return False + + +SPEC_CONFIG_SET = OperationSpec( + id="config.set", + request=ConfigSetReq, + response=ConfigValue, + impl=config_set, + summary="Write one local configuration key", + description=( + "Validated against `config keys`: a wrong type or an unknown key is " + "an error naming it, not a silent fallback to the default. Identity " + "and transport keys only take effect on the next `initConnection`, " + "and the response says so." + ), + legacy_paths=("config set",), + mutating=True, + idempotent=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=30, + example={"key": "daemon.idle_timeout", "value": 0, "previous": 1800, "updated": True}, + example_args="config set daemon.idle_timeout 0", + covers=( + "updates.event-report-message-delivery", + "updates.invoke-init-connection", + "updates.net-local-addr", + "updates.net-network-type", + "updates.net-parallel-connections", + "updates.net-proxy-for-calls", + "updates.net-transport-mode", + "updates.sync-dispatch-ordering", + ), + covers_partial=( + "updates.config-dns-fallback", + "updates.invoke-client-proxy-declare", + "updates.invoke-init-params-json", + "updates.net-flood-wait", + "updates.net-ipv6", + "updates.net-proxy-autoswitch", + "updates.net-proxy-system", + "updates.net-test-dc", + "updates.net-timeouts-retries", + "updates.ops-reconnect-health", + "updates.ops-single-updates-consumer", + "updates.presence-keepalive-period", + "updates.presence-set-online", + "updates.sync-catch-up-on-start", + "updates.sync-channel-short-poll", + "updates.sync-disable-updates", + "updates.sync-new-session-triggers-diff", + ), + coverage_note=( + "sets the switch; the behaviour it selects belongs to the group that " + "implements it (`proxy`, `sync`, `net`, `daemon`)." + ), + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigUnsetReq(Request): + key: Annotated[str, arg(0, metavar="KEY")] + apply: Annotated[ + bool, opt("--apply/--no-apply", help="Ask a running daemon to adopt the change now.") + ] = True + + +async def config_unset(ctx: OpContext, req: ConfigUnsetReq) -> ConfigValue: + """Remove a key, reverting it to its documented default.""" + key = _resolve_key(req.key) + section, field = _FIELDS[key.key] + document = _raw() + previous, present = _stored(document, key) + if not present: + ctx.mark_already() + return ConfigValue(key=key.key, default=key.default, already=True) + del document[section][field] + if not document[section]: + del document[section] + _write(document) + if req.apply: + _reload_daemon() + return ConfigValue( + key=key.key, previous=_redact(key, previous), default=key.default, removed=True + ) + + +SPEC_CONFIG_UNSET = OperationSpec( + id="config.unset", + request=ConfigUnsetReq, + response=ConfigValue, + impl=config_unset, + summary="Remove a local configuration key (revert to its default)", + legacy_paths=("config unset",), + mutating=True, + idempotent=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=30, + example={"key": "daemon.idle_timeout", "previous": 0, "default": 1800, "removed": True}, + example_args="config unset daemon.idle_timeout", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigListReq(Request): + section: Annotated[str | None, opt("--section", metavar="NAME", help="Only this section.")] = ( + None + ) + defaults: Annotated[ + bool, opt("--defaults", help="Include keys still at their default value.") + ] = False + + +async def config_list(ctx: OpContext, req: ConfigListReq) -> Page[ConfigEntry]: + """The effective configuration, with where each value came from. + + Secrets — proxy passwords, MTProxy secrets, the webhook token, `api_hash` + — are redacted. `config list` is the command people paste into bug + reports. + """ + document = _raw() + rows: list[ConfigEntry] = [] + for key in KEYS.values(): + if req.section and key.section != req.section and not key.key.startswith(f"{req.section}."): + continue + value, present = _stored(document, key) + if not present and not req.defaults: + continue + rows.append( + ConfigEntry( + key=key.key, + value=_redact(key, value), + default=key.default, + source="file" if present else "default", + scope=key.scope, + ) + ) + return build_page(rows, op="config.list", kind=PageKind.LOCAL, has_more=False, total=len(rows)) + + +SPEC_CONFIG_LIST = OperationSpec( + id="config.list", + request=ConfigListReq, + response=Page[ConfigEntry], + impl=config_list, + summary="Show the effective local configuration", + description="Secrets are redacted: this is the command people paste into bug reports.", + legacy_paths=("config list",), + paginated=PageKind.LOCAL, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + columns=("key", "value", "source"), + example={ + "items": [{"key": "daemon.idle_timeout", "value": 0, "source": "file"}], + "has_more": False, + }, + example_args="config list --defaults", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigKeysReq(Request): + section: Annotated[str | None, opt("--section", metavar="NAME", help="Only this section.")] = ( + None + ) + search: Annotated[ + str | None, opt("--search", metavar="TEXT", help="Substring match on key or help.") + ] = None + + +async def config_keys(ctx: OpContext, req: ConfigKeysReq) -> Page[ConfigKey]: + """Every documented knob, with its type, default and restart requirement. + + Machine-readable on purpose: an agent that has to discover the knobs from + prose will get one wrong, and a wrong key was silently the default in v1. + """ + rows = list(KEYS.values()) + if req.section: + rows = [ + row + for row in rows + if row.section == req.section or row.key.startswith(f"{req.section}.") + ] + if req.search: + needle = req.search.lower() + rows = [row for row in rows if needle in row.key.lower() or needle in row.help.lower()] + rows.sort(key=lambda row: row.key) + return build_page(rows, op="config.keys", kind=PageKind.LOCAL, has_more=False, total=len(rows)) + + +SPEC_CONFIG_KEYS = OperationSpec( + id="config.keys", + request=ConfigKeysReq, + response=Page[ConfigKey], + impl=config_keys, + summary="List every documented configuration key with its type and default", + legacy_paths=("config keys",), + paginated=PageKind.LOCAL, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + columns=("key", "type", "default", "requires_restart", "help"), + example={ + "items": [ + { + "key": "presence.mode", + "type": "string", + "default": "off", + "help": "off | online | mirror.", + } + ], + "has_more": False, + }, + example_args="config keys --section presence", + covers=( + "updates.invoke-init-params-json", + "updates.net-timeouts-retries", + "updates.presence-read-receipts-policy", + "updates.presence-set-online", + ), + covers_partial=("updates.invoke-init-connection",), + coverage_note="documents the knobs; writing one is `config set`.", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigPathReq(Request): + file: Annotated[ + str | None, + choice( + "config", + "jobs", + "webhook", + "sessions", + "logs", + "dead-letter", + "socket", + "pid", + "secrets", + help="Print just one path.", + ), + ] = None + + +async def config_path(ctx: OpContext, req: ConfigPathReq) -> ConfigPaths: + """Where everything lives. + + The session files and the secrets file are credential material at mode + 0600: exclude them from backups, and never paste their contents. + """ + paths = _paths() + report = ConfigPaths( + config_dir=str(paths.base), + config=str(paths.config), + jobs=str(paths.jobs), + webhook=str(paths.webhook), + secrets=str(paths.token), + sessions=str(paths.accounts), + logs=str(paths.logs), + socket=str(paths.socket), + pid=str(paths.pid), + dead_letter=str(paths.dead_letter), + ) + if req.file: + chosen = { + "config": report.config, + "jobs": report.jobs, + "webhook": report.webhook, + "sessions": report.sessions, + "logs": report.logs, + "dead-letter": report.dead_letter, + "socket": report.socket, + "pid": report.pid, + "secrets": report.secrets, + }[req.file] + return ConfigPaths(config_dir=report.config_dir, path=chosen) + return report + + +SPEC_CONFIG_PATH = OperationSpec( + id="config.path", + request=ConfigPathReq, + response=ConfigPaths, + impl=config_path, + summary="Print the paths of the configuration, jobs, webhook, session and log files", + legacy_paths=("config path",), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + columns=("config_dir", "config", "jobs", "webhook"), + example={"config_dir": "~/.tlgr", "config": "~/.tlgr/config.toml"}, + example_args="config path --file socket", + covers_partial=("updates.session-persistence",), + coverage_note="says where the session lives; persisting it is the session supervisor's.", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigInitReq(Request): + overwrite: Annotated[bool, opt("--overwrite", help="Replace files that already exist.")] = False + + +async def config_init(ctx: OpContext, req: ConfigInitReq) -> InitResult: + """Create the default configuration files, all at mode 0600. + + v1 wrote all three with `write_text`, world-readable — and `webhook.toml` + holds a token (SEC-07). + """ + from tlgr.core.paths import write_private + + paths = _paths() + paths.ensure_base() + created: list[str] = [] + skipped: list[str] = [] + for name, path, body in ( + ("config.toml", paths.config, _DEFAULT_CONFIG), + ("jobs.yaml", paths.jobs, _DEFAULT_JOBS), + ("webhook.toml", paths.webhook, _DEFAULT_WEBHOOK), + ): + if path.exists() and not req.overwrite: + skipped.append(name) + continue + write_private(path, body) + created.append(name) + if not created: + ctx.mark_already() + return InitResult(created=created, skipped=skipped, path=str(paths.base)) + + +_DEFAULT_CONFIG = """\ +[defaults] +output = "human" +# `none` because v1's markdown default silently ate `_`, `*` and backticks. +parse_mode = "none" + +[accounts] +default = "" + +[daemon] +auto_start = true +log_level = "info" +# 0 disables the idle stop. An idle stop with catch-up off is a sync hole. +idle_timeout = 0 + +[presence] +# tlgr announces nothing rather than claiming to be offline while reading. +mode = "off" +""" + +_DEFAULT_JOBS = """\ +# Gateway jobs. Add one non-interactively with: +# tlgr job add --name NAME --action 'reply:hello' +jobs: [] +""" + +_DEFAULT_WEBHOOK = """\ +[webhook] +enabled = false +url = "" +# Prefer the HMAC signature over a bearer token: it authenticates the body. +secret = "" +events = ["message_new"] + +[webhook.retry] +enabled = true +max_attempts = 5 +backoff_base = 2 + +[webhook.filters] +chats = [] +""" + + +SPEC_CONFIG_INIT = OperationSpec( + id="config.init", + request=ConfigInitReq, + response=InitResult, + impl=config_init, + summary="Create the default configuration files", + description="Written 0600 through the one writer that chmods before it renames (SEC-07).", + legacy_paths=("config init",), + mutating=True, + idempotent=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=30, + example={"created": ["config.toml", "jobs.yaml", "webhook.toml"], "path": "~/.tlgr"}, + example_args="config init", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +class ConfigValidateReq(Request): + strict: Annotated[bool, opt("--strict", help="Treat warnings (unknown keys) as errors.")] = ( + False + ) + file: Annotated[ + str | None, choice("config", "jobs", "webhook", help="Only validate one file.") + ] = None + + +async def config_validate(ctx: OpContext, req: ConfigValidateReq) -> ValidationReport: + """Check the three configuration files before the daemon has to. + + Also checks the *names*: a filter, processor, action or event name nobody + registered parses fine and then silently never matches, which is the + failure mode this command exists to convert into a message. + """ + from tlgr.core.config import load_app_config, load_webhook_config + + errors: list[ValidationIssue] = [] + warnings: list[ValidationIssue] = [] + checked: list[str] = [] + paths = _paths() + + if req.file in (None, "config"): + checked.append("config.toml") + try: + load_app_config(paths.base) + except Exception as exc: + errors.append(ValidationIssue(file="config.toml", message=str(exc))) + for section, block in _raw().items(): + if not isinstance(block, dict): + continue + for field in block: + if not any(pair == (section, field) for pair in _FIELDS.values()): + warnings.append( + ValidationIssue( + file="config.toml", + key=f"{section}.{field}", + message="not a documented key; run `tlgr config keys`", + ) + ) + + if req.file in (None, "jobs"): + checked.append("jobs.yaml") + errors.extend(_validate_jobs(paths.base)) + + if req.file in (None, "webhook"): + checked.append("webhook.toml") + try: + webhook = load_webhook_config(paths.base) + if webhook.enabled and not webhook.url: + errors.append( + ValidationIssue(file="webhook.toml", message="enabled but no url is set") + ) + for event in webhook.events: + _check_event(event, "webhook.toml", errors) + except Exception as exc: + errors.append(ValidationIssue(file="webhook.toml", message=str(exc))) + + if req.strict: + errors.extend(warnings) + warnings = [] + return ValidationReport( + ok=not errors, valid=not errors, files=checked, errors=errors, warnings=warnings + ) + + +def _validate_jobs(base: Any) -> list[ValidationIssue]: + from tlgr.actions import get_action + from tlgr.gateway.config import load_gateway_configs + + issues: list[ValidationIssue] = [] + try: + configs = load_gateway_configs(base) + except Exception as exc: + return [ValidationIssue(file="jobs.yaml", message=str(exc))] + for config in configs: + if not config.name: + issues.append(ValidationIssue(file="jobs.yaml", message="a job has no `name`")) + if not config.actions: + issues.append( + ValidationIssue( + file="jobs.yaml", key=config.name, message="has no actions and would do nothing" + ) + ) + for action in config.actions: + if get_action(action.name) is None: + issues.append( + ValidationIssue( + file="jobs.yaml", + key=config.name, + message=f"unknown action {action.name!r}", + ) + ) + for event in config.events: + _check_event(event, "jobs.yaml", issues, key=config.name) + return issues + + +def _check_event(name: str, file: str, into: list[ValidationIssue], key: str | None = None) -> None: + from tlgr.core import eventtypes + + try: + eventtypes.resolve_selectors(name) + except Exception: + into.append( + ValidationIssue( + file=file, + key=key, + message=f"unknown event type {name!r}; run `tlgr events list`", + ) + ) + + +SPEC_CONFIG_VALIDATE = OperationSpec( + id="config.validate", + request=ConfigValidateReq, + response=ValidationReport, + impl=config_validate, + summary="Validate the configuration, jobs and webhook files", + description=( + "Names as well as syntax: a filter, action or event name nobody " + "registered parses fine and then silently never matches." + ), + legacy_paths=("config validate",), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("ok", "files"), + example={"ok": True, "valid": True, "files": ["config.toml", "jobs.yaml", "webhook.toml"]}, + example_args="config validate --strict", + tags=frozenset({"infrastructure", "agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Server-side configuration +# --------------------------------------------------------------------------- + + +def _client(ctx: OpContext) -> Any: + client = getattr(ctx, "client", None) + if client is None: + raise UsageError("this operation needs a connected account") + return client + + +def _tl(value: Any) -> Any: + from tlgr.core.tl import tl_to_builtins + + return tl_to_builtins(value) + + +def _dc_options(config: Any) -> list[DcOption]: + out: list[DcOption] = [] + for option in getattr(config, "dc_options", None) or []: + out.append( + DcOption( + id=int(getattr(option, "id", 0) or 0), + ip_address=str(getattr(option, "ip_address", "") or ""), + port=int(getattr(option, "port", 0) or 0), + ipv6=bool(getattr(option, "ipv6", False)), + media_only=bool(getattr(option, "media_only", False)), + tcpo_only=bool(getattr(option, "tcpo_only", False)), + cdn=bool(getattr(option, "cdn", False)), + static=bool(getattr(option, "static", False)), + this_port_only=bool(getattr(option, "this_port_only", False)), + ) + ) + return out + + +class ConfigServerReq(Request): + key: Annotated[ + list[str], opt("--key", metavar="NAME", help="Print one field (repeatable).") + ] = [] + dc_options: Annotated[bool, opt("--dc-options", help="Include the dc_options array.")] = False + + +async def config_server_get(ctx: OpContext, req: ConfigServerReq) -> ServerConfig: + """`help.getConfig` — the server's own limits and endpoints. + + Feeding `message_length_max` into `message send` is what avoids a + MESSAGE_TOO_LONG round trip; `online_update_period_ms` is what + `presence.mode = online` refreshes on instead of a hard-coded minute. + """ + from telethon.tl import functions + + from tlgr.core.timefmt import fmt_dt, to_unix + + config = await _client(ctx)(functions.help.GetConfigRequest()) + date = getattr(config, "date", None) + report = ServerConfig( + expires=fmt_dt(getattr(config, "expires", None)), + test_mode=bool(getattr(config, "test_mode", False)), + this_dc=int(getattr(config, "this_dc", 0) or 0), + date=fmt_dt(date), + date_unix=to_unix(date), + chat_size_max=int(getattr(config, "chat_size_max", 0) or 0), + megagroup_size_max=int(getattr(config, "megagroup_size_max", 0) or 0), + message_length_max=int(getattr(config, "message_length_max", 0) or 0), + caption_length_max=int(getattr(config, "caption_length_max", 0) or 0), + online_update_period_ms=int(getattr(config, "online_update_period_ms", 0) or 0), + offline_blur_timeout_ms=int(getattr(config, "offline_blur_timeout_ms", 0) or 0), + offline_idle_timeout_ms=int(getattr(config, "offline_idle_timeout_ms", 0) or 0), + edit_time_limit=int(getattr(config, "edit_time_limit", 0) or 0), + revoke_time_limit=int(getattr(config, "revoke_time_limit", 0) or 0), + rating_e_decay=int(getattr(config, "rating_e_decay", 0) or 0), + forwarded_count_max=int(getattr(config, "forwarded_count_max", 0) or 0), + push_chat_period_ms=int(getattr(config, "push_chat_period_ms", 0) or 0), + dc_options=_dc_options(config) if req.dc_options else [], + ) + if req.key: + whole = _tl(config) + report.values = { + name: whole.get(name) for name in req.key if isinstance(whole, dict) and name in whole + } + missing = [name for name in req.key if name not in report.values] + if missing: + ctx.warn(f"help.getConfig has no field(s): {', '.join(missing)}") + return report + + +SPEC_CONFIG_SERVER = OperationSpec( + id="config.server.get", + request=ConfigServerReq, + response=ServerConfig, + impl=config_server_get, + summary="Read the MTProto server configuration (help.getConfig)", + aliases=("net.config",), + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=30, + columns=("this_dc", "message_length_max", "edit_time_limit", "revoke_time_limit"), + example={"this_dc": 4, "message_length_max": 4096, "edit_time_limit": 172800}, + example_args="config server get --dc-options", + covers=("updates.config-mtproto", "updates.presence-keepalive-period"), + covers_partial=("updates.config-dc-options",), + coverage_note="reads the config; enumerating the endpoints is `net dc list`.", + tags=frozenset({"agent-safe"}), +) + + +class ConfigAppReq(Request): + prefix: Annotated[ + str | None, + arg(0, metavar="KEY", required=False, help="Dotted key or prefix to filter."), + ] = None + frozen: Annotated[bool, opt("--frozen", help="Print only the account-freeze fields.")] = False + include_config: Annotated[bool, opt("--config", help="Also include help.getConfig.")] = False + + +async def config_app_get(ctx: OpContext, req: ConfigAppReq) -> AppConfigDoc: + """`help.getAppConfig` — the limits and kill switches everything is gated on. + + The freeze fields are why this is not merely diagnostic: without + `freeze_since_date`, `freeze_until_date` and `freeze_appeal_url`, a frozen + account produces a bare `FROZEN_METHOD_INVALID` on every send and nothing + that tells the user what to do about it. + """ + from telethon.tl import functions + + from tlgr.core.timefmt import fmt_unix + + result = await _client(ctx)(functions.help.GetAppConfigRequest(hash=0)) + if type(result).__name__ == "HelpAppConfigNotModified": + return AppConfigDoc(not_modified=True) + + values = _tl(getattr(result, "config", None)) + flat = _flatten_json_object(values) + report = AppConfigDoc(hash=int(getattr(result, "hash", 0) or 0), values=flat) + + for field, target in ( + ("freeze_since_date", "freeze_since_date"), + ("freeze_until_date", "freeze_until_date"), + ): + raw = flat.get(field) + if isinstance(raw, (int, float)): + setattr(report, target, fmt_unix(int(raw))) + appeal = flat.get("freeze_appeal_url") + if isinstance(appeal, str): + report.freeze_appeal_url = appeal + + if req.frozen: + report.values = {k: v for k, v in flat.items() if k.startswith("freeze_")} + elif req.prefix: + report.values = {k: v for k, v in flat.items() if k.startswith(req.prefix)} + if not report.values: + raise NotFoundError(f"no app-config key starts with {req.prefix!r}") + + if req.include_config: + report.config = await config_server_get(ctx, ConfigServerReq(dc_options=True)) + return report + + +def _flatten_json_object(value: Any) -> dict[str, Any]: + """A TL `JsonObject` tree → a flat `{key: value}` dict. + + `help.getAppConfig` returns a JSON document encoded as TL objects; leaving + it in that shape would make every consumer walk `{"_": "JsonObjectValue", + "key": …, "value": {"_": "JsonString", "value": …}}` by hand. + """ + out: dict[str, Any] = {} + for entry in (value or {}).get("value", []) if isinstance(value, dict) else []: + if not isinstance(entry, dict): + continue + key = entry.get("key") + if not isinstance(key, str): + continue + out[key] = _json_value(entry.get("value")) + return out + + +def _json_value(node: Any) -> Any: + if not isinstance(node, dict): + return node + kind = node.get("_") + if kind == "JsonNull": + return None + if kind == "JsonArray": + return [_json_value(item) for item in node.get("value", []) or []] + if kind == "JsonObject": + return _flatten_json_object(node) + return node.get("value") + + +SPEC_CONFIG_APP = OperationSpec( + id="config.app.get", + request=ConfigAppReq, + response=AppConfigDoc, + impl=config_app_get, + summary="Read the client (app) configuration (help.getAppConfig)", + description=( + "Almost every feature in Telegram has a limit or a kill switch here. " + "`--frozen` prints the account-freeze fields, which are what turn a " + "bare FROZEN_METHOD_INVALID into an actionable message." + ), + aliases=("settings.app-config",), + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=30, + example={"hash": 1834712, "values": {"reactions_user_max_default": 1}}, + example_args="config app get --frozen", + covers=("account.app-config", "updates.config-account-frozen", "updates.config-app"), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# The flat help.* endpoints +# --------------------------------------------------------------------------- + +_INFO_TOPICS = ( + "support", + "invite-text", + "premium-promo", + "peer-colors", + "timezones", + "languages", + "cdn", + "recent-links", + "emoji-keywords", + "deep-link", +) + + +class ConfigInfoReq(Request): + topic: Annotated[str, choice(*_INFO_TOPICS, help="Which endpoint to read.")] + value: Annotated[ + str | None, + arg( + 0, + metavar="VALUE", + required=False, + help="The tg:// link for deep-link, the query for emoji-keywords.", + ), + ] = None + lang: Annotated[ + str | None, opt("--lang", metavar="CODE", help="Language where the endpoint takes one.") + ] = None + search: Annotated[ + str | None, opt("--search", metavar="TEXT", help="Filter the returned list.") + ] = None + + +async def config_info_get(ctx: OpContext, req: ConfigInfoReq) -> InfoTopic: + """One of the flat read-only `help.*` endpoints. + + One command rather than ten thin siblings, because none of them carries an + option of its own. What they are *for* differs, though: `timezones` feeds + business hours, `peer-colors` ids are required by `account.updateColor`, + `languages` exists to choose `lang_code` (tlgr does not localise its own + output), and `premium-promo` prints prices — subscribing is a payment a + person performs. + """ + from telethon.tl import functions + + client = _client(ctx) + lang = req.lang or "en" + request, items_key = _info_request(req, lang, functions) + result = await client(request) + body = _tl(result) + items = body.get(items_key) if isinstance(body, dict) and items_key else None + rows = [row for row in (items or []) if isinstance(row, dict)] + if req.search: + needle = req.search.lower() + rows = [row for row in rows if needle in str(row).lower()] + topic = InfoTopic(topic=req.topic, items=rows, raw=body if isinstance(body, dict) else {}) + if not rows and not topic.raw: + raise NotFoundError(f"{req.topic} returned nothing") + return topic + + +def _info_request(req: ConfigInfoReq, lang: str, functions: Any) -> tuple[Any, str]: + """The request for a topic, and the field its list lives in.""" + if req.topic == "support": + return functions.help.GetSupportRequest(), "" + if req.topic == "invite-text": + return functions.help.GetInviteTextRequest(), "" + if req.topic == "premium-promo": + return functions.help.GetPremiumPromoRequest(), "period_options" + if req.topic == "peer-colors": + return functions.help.GetPeerColorsRequest(hash=0), "colors" + if req.topic == "timezones": + return functions.help.GetTimezonesListRequest(hash=0), "timezones" + if req.topic == "cdn": + return functions.help.GetCdnConfigRequest(), "public_keys" + if req.topic == "recent-links": + return functions.help.GetRecentMeUrlsRequest(referer=""), "urls" + if req.topic == "languages": + return functions.langpack.GetLanguagesRequest(lang_pack=""), "" + if req.topic == "emoji-keywords": + return functions.messages.GetEmojiKeywordsRequest(lang_code=lang), "keywords" + if req.topic == "deep-link": + if not req.value: + raise UsageError("deep-link needs the tg:// link as its argument", field="value") + return functions.help.GetDeepLinkInfoRequest(path=_deep_link_path(req.value)), "" + raise UsageError(f"unknown topic {req.topic!r}", field="topic") + + +def _deep_link_path(link: str) -> str: + """`tg://resolve?domain=x` → `resolve?domain=x`, which is what the API wants.""" + return link.removeprefix("tg://").removeprefix("https://t.me/").lstrip("/") + + +SPEC_CONFIG_INFO = OperationSpec( + id="config.info.get", + request=ConfigInfoReq, + response=InfoTopic, + impl=config_info_get, + summary="Read one of the server's flat informational endpoints", + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=60, + empty_exit=EXIT_EMPTY, + example={"topic": "timezones", "items": [{"id": "Europe/London", "utc_offset": 0}]}, + example_args="config info get timezones", + covers=( + "updates.config-cdn", + "updates.config-deep-link-info", + "updates.config-emoji-keywords", + "updates.config-invite-text", + "updates.config-peer-colors", + "updates.config-premium-promo", + "updates.config-recent-me-urls", + "updates.config-support", + "updates.config-timezones", + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Countries +# --------------------------------------------------------------------------- + +#: ISO-3166 alpha-2 → the regional-indicator pair that renders as its flag. +#: A pure client-side derivation: TDLib's getCountryFlagEmoji has no MTProto +#: counterpart, and neither does the preferred-language hint. +_FLAG_BASE = 0x1F1E6 + + +def _flag(iso2: str) -> str: + if len(iso2) != 2 or not iso2.isalpha(): + return "" + return "".join(chr(_FLAG_BASE + ord(char.upper()) - ord("A")) for char in iso2) + + +class CountryListReq(Request): + code: Annotated[str | None, opt("--code", metavar="ISO2", help="One country by ISO code.")] = ( + None + ) + search: Annotated[ + str | None, opt("--search", metavar="TEXT", help="Match on country name.") + ] = None + phone: Annotated[ + str | None, + opt("--phone", metavar="NUMBER", help="Classify a number: country, prefix, validity."), + ] = None + lang: Annotated[ + str | None, opt("--lang", metavar="CODE", help="Language for the localised names.") + ] = None + + +async def country_list(ctx: OpContext, req: CountryListReq) -> Page[Country]: + """Countries, phone prefixes and number patterns. + + `--phone` is the reason to have it: validating a number *before* `tlgr + login` turns a wasted `auth.sendCode` — and the flood budget it costs — + into a local error. + """ + from telethon.tl import functions + + result = await _client(ctx)( + functions.help.GetCountriesListRequest(lang_code=req.lang or "", hash=0) + ) + rows: list[Country] = [] + digits = "".join(char for char in (req.phone or "") if char.isdigit()) + + for entry in getattr(result, "countries", None) or []: + iso2 = str(getattr(entry, "iso2", "") or "") + codes = [ + CountryCode( + country_code=str(getattr(code, "country_code", "") or ""), + prefixes=[str(p) for p in (getattr(code, "prefixes", None) or [])], + patterns=[str(p) for p in (getattr(code, "patterns", None) or [])], + ) + for code in getattr(entry, "country_codes", None) or [] + ] + country = Country( + iso2=iso2, + name=str(getattr(entry, "name", "") or getattr(entry, "default_name", "") or ""), + default_name=str(getattr(entry, "default_name", "") or ""), + hidden=bool(getattr(entry, "hidden", False)), + flag_emoji=_flag(iso2), + preferred_language=iso2.lower(), + codes=codes, + ) + if digits: + matched = _match_phone(digits, codes) + if matched is None: + continue + country.matched_prefix, country.valid = matched + elif (req.code and iso2.upper() != req.code.upper()) or ( + req.search and req.search.lower() not in country.name.lower() + ): + continue + rows.append(country) + + if digits and not rows: + raise NotFoundError(f"no country claims the prefix of {req.phone!r}") + limit = int(getattr(ctx, "limit", None) or 300) + return build_page( + rows[:limit], + op="config.country.list", + kind=PageKind.LOCAL, + has_more=len(rows) > limit, + total=len(rows), + ) + + +def _match_phone(digits: str, codes: list[CountryCode]) -> tuple[str, bool] | None: + """The longest matching dial prefix, and whether the rest fits a pattern.""" + best: tuple[str, bool] | None = None + for code in codes: + if not digits.startswith(code.country_code): + continue + rest = digits[len(code.country_code) :] + prefixes = code.prefixes or [""] + for prefix in prefixes: + if not rest.startswith(prefix): + continue + valid = not code.patterns or any( + len(rest) == len(pattern.replace(" ", "")) for pattern in code.patterns + ) + candidate = (code.country_code + prefix, valid) + if best is None or len(candidate[0]) > len(best[0]): + best = candidate + return best + + +SPEC_COUNTRY_LIST = OperationSpec( + id="config.country.list", + request=CountryListReq, + response=Page[Country], + impl=country_list, + summary="List or look up countries, phone prefixes and number patterns", + description=( + "`--phone` classifies a number locally, which turns a wasted " + "`auth.sendCode` into an error before it costs the flood budget. The " + "flag emoji and the preferred language are derived client-side: " + "neither has an MTProto counterpart." + ), + aliases=("config.countries", "auth.countries"), + paginated=PageKind.LOCAL, + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=30, + columns=("iso2", "name", "flag_emoji", "codes"), + empty_exit=EXIT_EMPTY, + example={ + "items": [{"iso2": "GB", "name": "United Kingdom", "flag_emoji": "🇬🇧"}], + "has_more": False, + }, + example_args="config country list --phone +447700900000", + covers=( + "auth.countries-list", + "auth.prelogin-language", + "updates.config-countries", + "updates.config-country-lookup", + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Promo and suggestions +# --------------------------------------------------------------------------- + + +class PromoReq(Request): + hide: Annotated[bool, opt("--hide", help="Hide the current promo dialog.")] = False + + +async def config_promo_get(ctx: OpContext, req: PromoReq) -> PromoData: + """The promoted / PSA / sponsored chat the server pins to the dialog list. + + Official clients render it specially at the top of the list, so `chat + list` needs it for parity. A `proxy` flag means the promo arrived because + of an MTProxy sponsor — which is the reason tlgr does not declare its + proxy to the server by default. + """ + from telethon.tl import functions + + from tlgr.core.timefmt import fmt_unix + + client = _client(ctx) + result = await client(functions.help.GetPromoDataRequest()) + kind = type(result).__name__ + if kind == "HelpPromoDataEmpty": + return PromoData(kind="none", expires=fmt_unix(getattr(result, "expires", None))) + + peer = getattr(result, "peer", None) + from tlgr.core.tl import peer_marked_id + + report = PromoData( + kind="psa" if getattr(result, "psa_type", None) else "promo", + chat_id=peer_marked_id(peer), + psa_type=getattr(result, "psa_type", None), + psa_message=getattr(result, "psa_message", None), + proxy=bool(getattr(result, "proxy", False)), + expires=fmt_unix(getattr(result, "expires", None)), + pending_suggestions=[str(s) for s in (getattr(result, "pending_suggestions", None) or [])], + ) + if req.hide: + if peer is None: + ctx.mark_already() + return report + await client(functions.help.HidePromoDataRequest(peer=peer)) + report.hidden = True + return report + + +SPEC_PROMO = OperationSpec( + id="config.promo.get", + request=PromoReq, + response=PromoData, + impl=config_promo_get, + summary="Show (or hide) the promoted / PSA chat the server pins to the dialog list", + mutating=True, + idempotent=True, + surface=Surface.DAEMON, + rate_class="read", + timeout_s=30, + example={"kind": "psa", "psa_type": "covid", "expires": "2026-09-04T00:00:00Z"}, + example_args="config promo get", + covers=("updates.config-promo-psa", "updates.invoke-client-proxy-declare"), + coverage_note="", + tags=frozenset({"agent-safe", "mutating-checked"}), +) diff --git a/tlgr/ops/daemon.py b/tlgr/ops/daemon.py new file mode 100644 index 0000000..c2cb862 --- /dev/null +++ b/tlgr/ops/daemon.py @@ -0,0 +1,1386 @@ +"""The `daemon` group: lifecycle, health, floods and dead letters. + +Two halves that look alike and are not. `start`, `stop`, `restart`, `install`, +`uninstall`, `logs` and `status` run **outside** the daemon — they are how you +find out that it is not running, so they cannot need it to answer. Everything +else (`reconnect`, `save-state`, `flood *`, `dead-letter *`) runs inside it, +because it is asking about state only the running process has. + +`status` is the one worth reading the code of. v1 reported which clients the +daemon *held*: a client whose connection had died was still in the dict, still +listed under `accounts`, and the daemon still called itself healthy (COR-13, +COR-37). Here every account carries a state, a `pts` and a `behind_seconds`, +and `healthy` is false when any account needs a login, is frozen, or has +fallen behind — so "the process is alive" and "the daemon works" are separate +answers to separate questions. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import platform +import re +import subprocess +import sys +import time +from collections.abc import AsyncIterator +from datetime import datetime, timezone +from pathlib import Path +from typing import Annotated, Any + +from tlgr.core.errors import ( + EXIT_EMPTY, + DaemonError, + DaemonNotRunningError, + NotFoundError, + UsageError, +) +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.daemon import ( + AccountHealth, + DaemonStatus, + DeadLetter, + DeadLetterResult, + EventBusStatus, + FloodRecord, + FloodResult, + LifecycleResult, + LogLine, + ReconnectedAccount, + ReconnectResult, + SavedState, + SaveStateResult, + ServiceResult, +) +from tlgr.models.page import Page +from tlgr.models.peer import PeerRef +from tlgr.ops._params import choice, opt, parse_dt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + + +# --------------------------------------------------------------------------- +# Shared plumbing +# --------------------------------------------------------------------------- + + +def _base() -> Path: + from tlgr.core.paths import default_base + + return default_base() + + +def _writable_base(what: str) -> Path: + """The tlgr home, refused when it is somebody's live installation. + + A home carrying a `.production` marker belongs to a running daemon with + real accounts in it. Starting a second one there shares the session files, + and Telegram revokes an auth key it sees two clients on — so the marker is + a hard stop rather than a warning, with `TLGR_ALLOW_PRODUCTION_HOME=1` as + the escape hatch a person types on purpose. + """ + from tlgr.core.paths import refuse_production_home + + base = _base() + refuse_production_home(base) + return base + + +def _probe(timeout: float = 2.0) -> dict[str, Any] | None: + """`GET /v1/status`, without ever starting a daemon to answer it. + + `tlgr daemon status` exists to tell you the daemon is down; auto-starting + one to find out would make the question unanswerable. + """ + from tlgr.transport.client import DaemonClient + + client = DaemonClient(_base(), timeout=timeout, auto_start=False, no_restart=True) + with contextlib.suppress(Exception): + return client.probe_status() + return None + + +def _admin(action: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + from tlgr.transport.client import DaemonClient + + client = DaemonClient(_base(), timeout=30.0, auto_start=False, no_restart=True) + return client.admin(action, body or {}) + + +def _stamp(value: float | None) -> str | None: + if not value: + return None + return datetime.fromtimestamp(value, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _spanned(ctx: OpContext) -> list[str]: + """Which accounts an `--account all` daemon operation covers. + + Empty or `all` means every account the daemon holds. Naming one narrows + it. There is no "pick one for me": v1 did that and a two-account user + silently operated on the wrong identity (COR-02). + """ + alias = (ctx.account or "").strip() + daemon = getattr(ctx, "daemon", None) + sessions = getattr(daemon, "sessions", None) + known = list(getattr(sessions, "aliases", []) or []) + if alias and alias != "all": + if known and alias not in known: + raise NotFoundError(f"account {alias!r} is not connected. Run: tlgr daemon status") + return [alias] + return known + + +def _daemon(ctx: OpContext) -> Any: + daemon = getattr(ctx, "daemon", None) + if daemon is None: + raise DaemonError("this operation runs inside the daemon") + return daemon + + +def _telethon_layer() -> int: + with contextlib.suppress(Exception): + from telethon.tl.alltlobjects import LAYER + + return int(LAYER) + return 0 + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +def _spawn(base: Path, *, foreground: bool = False) -> Any: + """Start the daemon process. + + Spawned rather than imported, and not only because `ops/` may not import + `daemon/` (§2.2): a daemon that shares this process's file descriptors, + signal handlers and event loop is not the process a supervisor will start + later, so testing one would not test the other. + """ + command = [sys.executable, "-m", "tlgr.daemon.main", "--base", str(base)] + if foreground: + command.append("--foreground") + return subprocess.Popen(command) + return subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + +def _wait_ready(timeout: float) -> dict[str, Any] | None: + """Poll `/v1/status` until the daemon answers. + + Readiness is a *reply*, not a file. v1 waited for the socket to appear, + which happens at `bind()` — before any account has connected and before + the daemon can serve anything (ROB-07). + """ + from tlgr.transport.autostart import wait_ready + from tlgr.transport.client import DaemonClient + + client = DaemonClient(_base(), auto_start=False, no_restart=True) + return wait_ready(client.probe_status, timeout=timeout) + + +class DaemonStartReq(Request): + foreground: Annotated[ + bool, opt("--foreground", help="Run in the foreground instead of forking.") + ] = False + connect: Annotated[ + list[str], + opt("--connect", metavar="ALIAS", help="Only connect these accounts (repeatable)."), + ] = [] + catch_up: Annotated[ + bool, + opt( + "--catch-up/--no-catch-up", + help="Load the persisted pts/qts/seq and fetch the difference before dispatching.", + ), + ] = True + idle_timeout: Annotated[ + int | None, + opt( + "--idle-timeout", metavar="DURATION", kind="duration", help="0 disables the idle stop." + ), + ] = None + wait: Annotated[ + int, opt("--wait", metavar="DURATION", kind="duration", help="How long to wait for ready.") + ] = 30 + + +async def daemon_start(ctx: OpContext, req: DaemonStartReq) -> LifecycleResult: + """Start the update-receiving daemon. + + `catch_up` defaults to true and `idle_timeout` to 0 for good reason: v1 + combined an idle stop at 1,800 s with an effectively disabled catch-up, so + the daemon shut down, restarted on the next command, and never fetched + what it had missed. That combination is a guaranteed, permanent sync hole. + """ + from tlgr.core.process import read_pid + + base = _writable_base("tlgr daemon start") + existing = read_pid(base) + if existing: + running = _probe() or {} + ctx.mark_already() + return LifecycleResult( + started=False, + already=True, + pid=existing, + socket=str(running.get("daemon", {}).get("socket", "")), + ready=bool(running.get("daemon", {}).get("ready")), + catch_up=req.catch_up, + ) + + environment_note = _start_environment(req) + if req.foreground: + raise SystemExit(_spawn(base, foreground=True).wait()) + + process = _spawn(base) + status = _wait_ready(float(req.wait)) + if status is None: + raise DaemonError( + f"the daemon did not become ready within {req.wait}s; check the log: tlgr daemon logs" + ) + info = status.get("daemon", {}) + if environment_note: + ctx.warn(environment_note) + return LifecycleResult( + started=True, + pid=int(info.get("pid") or read_pid(base) or process.pid), + socket=str(info.get("socket", "")), + ready=bool(info.get("ready")), + accounts=[row.get("alias", "") for row in status.get("accounts", [])], + catch_up=req.catch_up, + ) + + +def _start_environment(req: DaemonStartReq) -> str: + """Apply the per-start overrides through the environment the child reads.""" + notes: list[str] = [] + if req.idle_timeout is not None: + os.environ["TLGR_IDLE_TIMEOUT"] = str(int(req.idle_timeout)) + notes.append(f"idle_timeout was set to {int(req.idle_timeout)}s for this run only") + if not req.catch_up: + os.environ["TLGR_CATCH_UP"] = "0" + notes.append( + "catch-up is disabled for this run: updates that arrive while the " + "daemon is down will not be recovered" + ) + if req.connect: + os.environ["TLGR_PRECONNECT"] = ",".join(req.connect) + return "; ".join(notes) + + +SPEC_DAEMON_START = OperationSpec( + id="daemon.start", + request=DaemonStartReq, + response=LifecycleResult, + impl=daemon_start, + summary="Start the update-receiving daemon", + description=( + "Waits for an HTTP 200 from `/v1/status`, not for the socket file: " + "the socket exists from `bind()`, before any account has connected " + "(ROB-07). Catch-up is on by default and the idle stop is off, " + "because the two together are what made v1 lose updates silently." + ), + legacy_paths=("daemon start",), + mutating=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=120, + example={"started": True, "pid": 41231, "ready": True, "catch_up": True}, + example_args="daemon start", + covers=( + "updates.stream-daemon-multi-account", + "updates.sync-catch-up-on-start", + "updates.sync-new-session-triggers-diff", + ), + covers_partial=("updates.ops-daemon-lifecycle",), + coverage_note="starting the process; stopping it cleanly is `daemon stop`.", + tags=frozenset({"agent-safe"}), +) + + +class DaemonStopReq(Request): + timeout: Annotated[ + int, + opt("--grace", metavar="DURATION", kind="duration", help="Drain period before SIGKILL."), + ] = 10 + + +async def daemon_stop(ctx: OpContext, req: DaemonStopReq) -> LifecycleResult: + """Stop the daemon, letting it flush pts and the entity cache first. + + Every shutdown path has to `await disconnect()`: a SIGKILL loses the + update state and the cached access hashes, and losing an access hash is + what makes the next catch-up silently skip a channel. + """ + from tlgr.core.process import read_pid + + base = _base() + pid = read_pid(base) + if pid is None: + return LifecycleResult(stopped=False, already=True) + + with contextlib.suppress(Exception): + _admin("stop", {"drain_s": float(req.timeout)}) + + deadline = time.monotonic() + max(1.0, float(req.timeout)) + while time.monotonic() < deadline: + if read_pid(base) is None: + return LifecycleResult(stopped=True, pid=pid) + time.sleep(0.1) + + from tlgr.core.process import stop_daemon + + stop_daemon(base) + for _ in range(20): + time.sleep(0.25) + if read_pid(base) is None: + return LifecycleResult(stopped=True, pid=pid) + raise DaemonError(f"the daemon (pid {pid}) did not stop within {req.timeout}s") + + +SPEC_DAEMON_STOP = OperationSpec( + id="daemon.stop", + request=DaemonStopReq, + response=LifecycleResult, + impl=daemon_stop, + summary="Stop the daemon", + description=( + "Asks it to drain in-flight requests and disconnect cleanly, then " + "falls back to SIGTERM. A killed daemon loses its `pts` and the " + "cached access hashes catch-up needs." + ), + legacy_paths=("daemon stop",), + mutating=True, + idempotent=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=60, + example={"stopped": True, "pid": 41231}, + example_args="daemon stop", + covers=("updates.ops-daemon-lifecycle",), + tags=frozenset({"agent-safe"}), +) + + +class DaemonRestartReq(Request): + timeout: Annotated[ + int, + opt("--grace", metavar="DURATION", kind="duration", help="Drain period before SIGKILL."), + ] = 10 + wait: Annotated[ + int, opt("--wait", metavar="DURATION", kind="duration", help="How long to wait for ready.") + ] = 30 + + +async def daemon_restart(ctx: OpContext, req: DaemonRestartReq) -> LifecycleResult: + """Stop and start, waiting for readiness at both ends.""" + from tlgr.core.process import read_pid + + base = _writable_base("tlgr daemon restart") + if read_pid(base) is not None: + await daemon_stop(ctx, DaemonStopReq(timeout=req.timeout)) + started = await daemon_start(ctx, DaemonStartReq(wait=req.wait)) + return LifecycleResult( + restarted=True, + pid=started.pid, + socket=started.socket, + ready=started.ready, + accounts=started.accounts, + ) + + +SPEC_DAEMON_RESTART = OperationSpec( + id="daemon.restart", + request=DaemonRestartReq, + response=LifecycleResult, + impl=daemon_restart, + summary="Restart the daemon", + legacy_paths=("daemon restart",), + mutating=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=180, + example={"restarted": True, "pid": 41999}, + example_args="daemon restart", + covers_partial=("updates.ops-daemon-lifecycle",), + coverage_note="a stop and a start; the lifecycle itself is `daemon stop`.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Service installation +# --------------------------------------------------------------------------- + + +def _supervisor(choice_: str) -> str: + if choice_ != "auto": + return choice_ + return "launchd" if platform.system() == "Darwin" else "systemd" + + +class DaemonInstallReq(Request): + supervisor: Annotated[ + str, choice("auto", "launchd", "systemd", help="Which service manager to install into.") + ] = "auto" + keep_alive: Annotated[ + bool, opt("--keep-alive/--no-keep-alive", help="Restart the daemon on crash.") + ] = True + + +async def daemon_install(ctx: OpContext, req: DaemonInstallReq) -> ServiceResult: + """Install as a *user* service: it holds session files under $HOME. + + Both backends force `idle_timeout` to 0. Under a supervisor a clean idle + exit is either a respawn loop or a daemon that never comes back (COR-39). + """ + base = _writable_base("tlgr daemon install") + kind = _supervisor(req.supervisor) + if kind == "launchd": + from tlgr.core import launchd + from tlgr.core.config import get_logs_dir + + if launchd.is_installed(): + ctx.mark_already() + return ServiceResult( + installed=True, already=True, supervisor=kind, path=str(launchd.PLIST_PATH) + ) + path = launchd.install(base, get_logs_dir(base)) + else: + from tlgr.core import systemd + + if systemd.is_installed(): + ctx.mark_already() + return ServiceResult( + installed=True, already=True, supervisor=kind, path=str(systemd.unit_path()) + ) + path = systemd.install(base) + if not req.keep_alive: + ctx.warn( + "--no-keep-alive is recorded but not honoured by the generated unit; " + "edit it directly to disable the restart" + ) + return ServiceResult(installed=True, supervisor=kind, unit=path.name, path=str(path)) + + +SPEC_DAEMON_INSTALL = OperationSpec( + id="daemon.install", + request=DaemonInstallReq, + response=ServiceResult, + impl=daemon_install, + summary="Install the daemon as a user service (auto-start, restart on crash)", + description=( + "macOS gets a LaunchAgent, Linux a systemd **user** unit — user, " + "because the daemon holds session files under $HOME and must run as " + "their owner." + ), + legacy_paths=("daemon install",), + mutating=True, + idempotent=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=60, + example={"installed": True, "supervisor": "launchd", "path": "~/Library/LaunchAgents/…"}, + example_args="daemon install", + covers_partial=("updates.ops-daemon-lifecycle",), + coverage_note="the supervisor half; running the daemon is `daemon start`/`stop`.", + tags=frozenset({"agent-safe"}), +) + + +class DaemonUninstallReq(Request): + stop: Annotated[bool, opt("--stop/--no-stop", help="Also stop a running daemon.")] = True + + +async def daemon_uninstall(ctx: OpContext, req: DaemonUninstallReq) -> ServiceResult: + """Remove the user service.""" + from tlgr.core import launchd, systemd + + removed = launchd.uninstall() if platform.system() == "Darwin" else False + removed = systemd.uninstall() or removed + stopped = False + if req.stop: + result = await daemon_stop(ctx, DaemonStopReq()) + stopped = result.stopped + if not removed: + ctx.mark_already() + return ServiceResult(uninstalled=removed, already=not removed, stopped=stopped) + + +SPEC_DAEMON_UNINSTALL = OperationSpec( + id="daemon.uninstall", + request=DaemonUninstallReq, + response=ServiceResult, + impl=daemon_uninstall, + summary="Remove the daemon service", + legacy_paths=("daemon uninstall",), + mutating=True, + destructive=True, + idempotent=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=60, + example={"uninstalled": True, "stopped": True}, + example_args="daemon uninstall", + covers_partial=("updates.ops-daemon-lifecycle",), + coverage_note="the supervisor half; running the daemon is `daemon start`/`stop`.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Logs +# --------------------------------------------------------------------------- + +_LOG_LEVELS = ("debug", "info", "warning", "error") +_LEVEL_RANK = {name: index for index, name in enumerate(_LOG_LEVELS)} +_PLAIN_LOG = re.compile(r"^(?P\S+)\s+(?P[A-Z]+)\s+(?P\S+)\s+(?P.*)$") + + +def _parse_log(line: str) -> LogLine: + """One log line, structured where it can be and verbatim where it cannot.""" + stripped = line.rstrip("\n") + if stripped.startswith("{"): + with contextlib.suppress(json.JSONDecodeError): + record = json.loads(stripped) + if isinstance(record, dict): + return LogLine( + ts=str(record.get("ts") or record.get("time") or ""), + level=str(record.get("level", "")).lower(), + account=record.get("account"), + logger=str(record.get("logger", "")), + message=str(record.get("message", "")), + raw=stripped, + ) + match = _PLAIN_LOG.match(stripped) + if match: + return LogLine( + ts=match.group("ts"), + level=match.group("level").lower(), + logger=match.group("logger"), + message=match.group("message"), + raw=stripped, + ) + return LogLine(message=stripped, raw=stripped) + + +def _wanted_log(entry: LogLine, level: str | None, account: str | None, grep: str | None) -> bool: + if level and _LEVEL_RANK.get(entry.level, 0) < _LEVEL_RANK.get(level, 0): + return False + if account and entry.account != account: + return False + return not (grep and grep.lower() not in entry.raw.lower()) + + +class DaemonLogsReq(Request): + follow: Annotated[bool, opt("--follow", "-f", help="Follow the log as it is written.")] = False + lines: Annotated[int, opt("--lines", metavar="N", ge=1, le=100000)] = 50 + level: Annotated[str | None, choice(*_LOG_LEVELS, help="Minimum level to show.")] = None + log_account: Annotated[ + str | None, + opt("--for-account", metavar="ALIAS", help="Only lines tagged with this account."), + ] = None + grep: Annotated[str | None, opt("--grep", metavar="TEXT", help="Substring filter.")] = None + + +async def daemon_logs(ctx: OpContext, req: DaemonLogsReq) -> AsyncIterator[dict[str, Any]]: + """The daemon log, tailed and filtered. + + Read here rather than exec'ing `tail`, so that `--level`, `--for-account` + and `--grep` mean the same thing whether or not you are following, and so + the output is structured rather than whatever the log formatter happened + to print. + """ + path = _base() / "logs" / "daemon.log" + if not path.exists(): + raise NotFoundError(f"no log file at {path}. Has the daemon ever started?") + + with path.open(encoding="utf-8", errors="replace") as handle: + tail = handle.readlines()[-req.lines :] + for line in tail: + entry = _parse_log(line) + if _wanted_log(entry, req.level, req.log_account, req.grep): + yield {"type": "log", **_log_frame(entry)} + if not req.follow: + return + handle.seek(0, os.SEEK_END) + while True: + line = handle.readline() + if not line: + await asyncio.sleep(0.25) + continue + entry = _parse_log(line) + if _wanted_log(entry, req.level, req.log_account, req.grep): + yield {"type": "log", **_log_frame(entry)} + + +def _log_frame(entry: LogLine) -> dict[str, Any]: + from tlgr.models.base import to_builtins + + frame = to_builtins(entry) + return frame if isinstance(frame, dict) else {"raw": entry.raw} + + +SPEC_DAEMON_LOGS = OperationSpec( + id="daemon.logs", + request=DaemonLogsReq, + response=None, + impl=daemon_logs, + summary="View or follow the daemon log", + description=( + "Structured lines with secrets redacted: an auth key, an access hash, " + "a proxy secret and a webhook token are never written to the log in " + "the first place." + ), + legacy_paths=("daemon logs",), + stream=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=900, + example={"type": "log", "level": "info", "message": "daemon ready with 1 account(s)"}, + example_args="daemon logs --lines 100 --level warning", + covers_partial=("updates.ops-daemon-lifecycle",), + coverage_note="the operator's view of the process; the lifecycle is `daemon stop`.", + tags=frozenset({"agent-safe", "frames", "live-stream"}), +) + + +# --------------------------------------------------------------------------- +# Status +# --------------------------------------------------------------------------- + + +def _account_health(row: dict[str, Any]) -> AccountHealth: + return AccountHealth( + alias=str(row.get("alias", "")), + state=str(row.get("state", "unknown")), + user_id=row.get("user_id"), + username=row.get("username"), + dc_id=row.get("dc_id"), + proxy=row.get("proxy"), + pts=row.get("pts"), + qts=row.get("qts"), + seq=row.get("seq"), + date=row.get("date"), + behind_seconds=row.get("behind_seconds"), + catching_up=bool(row.get("catch_up_pending")), + channels_tracked=int(row.get("channels_tracked") or 0), + last_update_at=row.get("last_update"), + connected_since=row.get("connected_since"), + reconnects=int(row.get("reconnects") or 0), + in_flight=int(row.get("in_flight") or 0), + resync_needed=list(row.get("resync_needed") or []), + flood_waits=int(row.get("flood_entries") or 0), + circuit=str(row.get("circuit", "closed")), + frozen=str(row.get("state", "")) == "frozen", + error=row.get("reason"), + ) + + +#: An account in one of these states is not doing its job, whatever the +#: process is doing. `healthy` has to be false for all of them, or the flag +#: means "a process exists" — which is the question nobody was asking. +_UNHEALTHY = frozenset({"needs_login", "frozen", "degraded", "stopped"}) + + +class DaemonStatusReq(Request): + check: Annotated[ + bool, opt("--check", help="Exit 11 when the daemon or any account is unhealthy.") + ] = False + + +async def daemon_status(ctx: OpContext, req: DaemonStatusReq) -> DaemonStatus: + """Daemon and per-account health, as two separate answers. + + `running` has always meant "a process is alive". `ready` and `healthy` + are the questions people were actually asking, and v1 could not tell them + apart: an account whose connection had died was still counted (COR-37). + """ + from tlgr.core.process import read_pid + + base = _base() + pid = read_pid(base) + status = _probe() + if status is None: + result = DaemonStatus(running=pid is not None, ready=False, healthy=False, pid=pid) + if req.check: + raise DaemonNotRunningError("the daemon is not answering on its socket") + return result + + info = status.get("daemon", {}) + rows = [_account_health(row) for row in status.get("accounts", [])] + if ctx.account and ctx.account != "all": + rows = [row for row in rows if row.alias == ctx.account] + unhealthy = [row for row in rows if row.state in _UNHEALTHY] + result = DaemonStatus( + running=True, + ready=bool(info.get("ready")), + healthy=bool(info.get("ready")) and not unhealthy, + pid=info.get("pid") or pid, + uptime_seconds=int(info.get("uptime_s") or 0), + version=str(info.get("version", "")), + protocol=int(info.get("protocol") or 0), + layer=_telethon_layer(), + socket=str(info.get("socket", "")), + socket_owner=os.getuid(), + managed_by=info.get("managed_by"), + accounts=rows, + events=EventBusStatus(**status["events"]) + if isinstance(status.get("events"), dict) + else None, + webhook=status.get("webhook") or {}, + jobs=status.get("jobs") or [], + connections={row.alias: row.state == "online" for row in rows}, + disconnected=sorted(row.alias for row in rows if row.state != "online"), + ) + if req.check and not result.healthy: + raise DaemonError( + "the daemon is not healthy: " + + (", ".join(f"{row.alias} is {row.state}" for row in unhealthy) or "not ready") + ) + return result + + +SPEC_DAEMON_STATUS = OperationSpec( + id="daemon.status", + request=DaemonStatusReq, + response=DaemonStatus, + impl=daemon_status, + summary="Show daemon and per-account connection health", + description=( + "`running` is about the process, `ready` about the socket, `healthy` " + "about the accounts. v1 had only the first and reported every client " + "it held as connected, so a fully deaf daemon looked fine (COR-37)." + ), + legacy_paths=("daemon status",), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("running", "ready", "healthy", "pid", "uptime_seconds", "disconnected"), + example={ + "running": True, + "ready": True, + "healthy": True, + "pid": 41231, + "uptime_seconds": 8123, + "accounts": [{"alias": "work", "state": "online", "pts": 91824}], + }, + example_args="daemon status --check", + covers=( + "bots.bot-updates-status", + "updates.ops-single-updates-consumer", + "updates.session-persistence", + ), + covers_partial=( + "updates.config-account-frozen", + "updates.net-connection-status", + "updates.ops-reconnect-health", + "updates.stream-daemon-multi-account", + "updates.sync-updating-indicator", + ), + coverage_note=( + "reports the state; the network detail is `net status`, the freeze " + "fields are `config app get`, and recovery is `daemon reconnect`." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Reconnect and save-state +# --------------------------------------------------------------------------- + + +class DaemonReconnectReq(Request): + reset_proxy: Annotated[ + bool, opt("--reset-proxy", help="Rebuild the client with the currently selected proxy.") + ] = False + catch_up: Annotated[ + bool, opt("--catch-up/--no-catch-up", help="Fetch the difference after reconnecting.") + ] = True + + +async def daemon_reconnect(ctx: OpContext, req: DaemonReconnectReq) -> ReconnectResult: + """Force a reconnect, and by default a catch-up with it. + + Also the documented recovery for a `TypeNotFoundError` from a constructor + of a newer layer: the guidance is to treat it like a 500 — reopen the + socket, re-`initConnection`, then `getDifference` — because a socket that + has met an unparseable constructor cannot be trusted to be in sync. + """ + daemon = _daemon(ctx) + out: list[ReconnectedAccount] = [] + for alias in _spanned(ctx): + session = daemon.sessions.get(alias) + if session is None: + out.append(ReconnectedAccount(alias=alias, error="not connected")) + continue + row = ReconnectedAccount(alias=alias) + try: + if req.reset_proxy: + await daemon.sessions.release(alias) + session = await daemon.sessions.ensure(alias) + else: + client = session.client + if client is not None: + await client.disconnect() + await client.connect() + row.reconnected = True + row.dc_id = getattr(getattr(session.client, "session", None), "dc_id", None) + if req.catch_up: + await session.catch_up() + session.resync_needed.clear() + row.caught_up = True + except Exception as exc: + row.error = f"{type(exc).__name__}: {exc}" + out.append(row) + if not out: + ctx.warn("no accounts are connected; nothing to reconnect") + return ReconnectResult(accounts=out) + + +SPEC_DAEMON_RECONNECT = OperationSpec( + id="daemon.reconnect", + request=DaemonReconnectReq, + response=ReconnectResult, + impl=daemon_reconnect, + summary="Force a reconnect (and catch-up) for one or every account", + mutating=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=180, + columns=("accounts.alias", "accounts.reconnected", "accounts.caught_up", "accounts.error"), + example={"accounts": [{"alias": "work", "reconnected": True, "caught_up": True}]}, + example_args="daemon reconnect", + covers=("updates.ops-reconnect-health", "updates.sync-old-layer-socket-reset"), + covers_partial=("updates.sync-new-session-triggers-diff",), + coverage_note="the manual recovery; the automatic one runs in the supervisor.", + tags=frozenset({"agent-safe"}), +) + + +class DaemonSaveStateReq(Request): + pass + + +async def daemon_save_state(ctx: OpContext, req: DaemonSaveStateReq) -> SaveStateResult: + """Flush pts/qts/seq and the entity cache to the session file now. + + Telethon persists only on `disconnect()`, so a SIGKILL'd daemon loses both + the update state and the access hashes that make channel catch-up + possible. The daemon does this on a timer; this is the manual trigger. + """ + from tlgr.core import telethon_compat as compat + + daemon = _daemon(ctx) + rows: list[SavedState] = [] + for alias in _spanned(ctx): + session = daemon.sessions.get(alias) + if session is None or session.client is None: + rows.append(SavedState(alias=alias, error="not connected")) + continue + row = SavedState(alias=alias) + try: + await compat.save_state(session.client) + state, channels = compat.session_state(session.client) + row.pts = state.get("pts") + row.qts = state.get("qts") + row.seq = state.get("seq") + row.date = state.get("date") + row.channels = len(channels) + row.entities = compat.entity_count(session.client) + except Exception as exc: + row.error = f"{type(exc).__name__}: {exc}" + rows.append(row) + daemon.bus.flush_state() + return SaveStateResult(accounts=rows) + + +SPEC_DAEMON_SAVE_STATE = OperationSpec( + id="daemon.save-state", + request=DaemonSaveStateReq, + response=SaveStateResult, + impl=daemon_save_state, + summary="Flush update state and the entity cache to the session file now", + description=( + "Telethon writes the session only on a clean `disconnect()`. A " + "SIGKILL therefore costs the `pts` progress *and* the cached access " + "hashes — and a channel whose access hash is gone is silently skipped " + "by the next catch-up." + ), + mutating=True, + idempotent=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=60, + columns=("accounts.alias", "accounts.pts", "accounts.channels", "accounts.entities"), + example={"accounts": [{"alias": "work", "pts": 91824, "channels": 12, "entities": 480}]}, + example_args="daemon save-state", + covers=("updates.sync-peer-cache-from-updates",), + covers_partial=("updates.session-persistence", "updates.sync-state-persistence"), + coverage_note="flushes it on demand; the periodic flush is the session supervisor's.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Floods +# --------------------------------------------------------------------------- + + +def _flood_kind(method: str) -> str: + lowered = method.lower() + for needle, kind in ( + ("slowmode", "slowmode"), + ("premium", "premium_wait"), + ("peer_flood", "peer_flood"), + ("takeout", "takeout_delay"), + ): + if needle in lowered: + return kind + return "flood_wait" + + +class FloodListReq(Request): + include_expired: Annotated[ + bool, opt("--include-expired", help="Also show deadlines that have already passed.") + ] = False + + +async def flood_list(ctx: OpContext, req: FloodListReq) -> Page[FloodRecord]: + """The rate-limit deadlines this installation still owes. + + tlgr keeps its own persistent store keyed `(account, method, peer)`. + Telethon remembers a `FloodWaitError` in memory and forgets it on exit, so + v1 re-hit every wait after a restart — and re-hitting a wait is how a + short one becomes a long one. + """ + daemon = _daemon(ctx) + rows: list[FloodRecord] = [] + aliases = _spanned(ctx) or [row.alias for row in daemon.accounts.list_accounts()] + for alias in aliases: + limiter = daemon.sessions.limiter(alias) + for deadline in limiter.flood.entries(include_expired=req.include_expired): + rows.append( + FloodRecord( + account=alias, + kind=_flood_kind(deadline.method), + method=deadline.method, + chat=deadline.peer or None, + wait_seconds=deadline.remaining, + until=_stamp(deadline.until), + circuit_open=limiter.breaker.open, + expired=deadline.remaining == 0, + ) + ) + rows.sort(key=lambda row: (-row.wait_seconds, row.account, row.method)) + limit = int(getattr(ctx, "limit", None) or 100) + return build_page(rows[:limit], op="daemon.flood.list", kind=PageKind.LOCAL, has_more=False) + + +SPEC_FLOOD_LIST = OperationSpec( + id="daemon.flood.list", + request=FloodListReq, + response=Page[FloodRecord], + impl=flood_list, + summary="List active rate-limit deadlines", + aliases=("daemon.floods",), + paginated=PageKind.LOCAL, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("account", "kind", "method", "wait_seconds", "until", "circuit_open"), + example={ + "items": [ + { + "account": "work", + "kind": "flood_wait", + "method": "SendMessageRequest", + "wait_seconds": 41, + "until": "2026-09-03T09:20:00Z", + } + ], + "has_more": False, + }, + example_args="daemon flood list", + covers=("updates.net-flood-wait",), + tags=frozenset({"agent-safe"}), +) + + +class FloodClearReq(Request): + method: Annotated[ + str | None, opt("--method", metavar="NAME", help="Only this request type.") + ] = None + chat: Annotated[ + PeerRef | None, opt("--chat", metavar="CHAT", kind="peer", help="Only this peer.") + ] = None + everything: Annotated[ + bool, opt("--every", help="Clear every remembered deadline for the account.") + ] = False + + +async def flood_clear(ctx: OpContext, req: FloodClearReq) -> FloodResult: + """Forget remembered deadlines and close the circuit breaker. + + Clearing a *live* server-side FLOOD_WAIT does not lift it — the next call + re-trips it, more expensively. This is for after the cause is fixed, or to + reopen an account an operator has actually looked at. + """ + daemon = _daemon(ctx) + if not (req.method or req.chat or req.everything): + raise UsageError("say what to clear: --method, --chat, or --every", field="method") + cleared = 0 + touched: list[str] = [] + for alias in _spanned(ctx) or [row.alias for row in daemon.accounts.list_accounts()]: + limiter = daemon.sessions.limiter(alias) + peer = req.chat.raw if req.chat is not None else None + cleared += ( + limiter.flood.forget(method=req.method or "", peer=peer) + if not req.everything + else _clear_all(limiter) + ) + limiter.reset_breaker() + touched.append(alias) + if not cleared: + ctx.mark_already() + return FloodResult(cleared=cleared, circuit_open=False, accounts=touched) + + +def _clear_all(limiter: Any) -> int: + count = len(limiter.flood.entries(include_expired=True)) + limiter.flood.clear() + return count + + +SPEC_FLOOD_CLEAR = OperationSpec( + id="daemon.flood.clear", + request=FloodClearReq, + response=FloodResult, + impl=flood_clear, + summary="Clear remembered rate-limit deadlines and reset the circuit breaker", + description=( + "Local memory only. Telegram's own wait is unaffected, so clearing a " + "deadline that has not actually passed simply spends the next request " + "learning that again." + ), + mutating=True, + destructive=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=30, + example={"cleared": 3, "circuit_open": False, "accounts": ["work"]}, + example_args="daemon flood clear --every", + covers_partial=("updates.net-flood-wait",), + coverage_note="the reset half; the accounting is `daemon flood list`.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# Dead letters +# --------------------------------------------------------------------------- + + +def _dead_letters(ctx: OpContext) -> tuple[Any, list[dict[str, Any]]]: + daemon = _daemon(ctx) + webhook = daemon.webhook + return webhook, webhook.read_dead_letters() + + +def _dead_letter_model(index: int, entry: dict[str, Any]) -> DeadLetter: + identifier = str(entry.get("delivery_id") or f"dl-{index}") + return DeadLetter( + id=identifier, + seq=int(entry.get("seq") or 0), + source=str(entry.get("source", "webhook")), + event=str(entry.get("event", "")), + account=str(entry.get("account", "")), + attempts=int(entry.get("attempts") or 1), + last_error=str(entry.get("reason", "")), + first_failed_at=str(entry.get("first_failed_at") or entry.get("ts", "")), + last_failed_at=str(entry.get("ts", "")), + ) + + +def _matches(entry: dict[str, Any], source: str, since: str | None, events: str | None) -> bool: + if source != "all" and str(entry.get("source", "webhook")) != source: + return False + if since and str(entry.get("ts", "")) < since: + return False + if events: + wanted = {part.strip() for part in events.split(",") if part.strip()} + if str(entry.get("event", "")) not in wanted: + return False + return True + + +class DeadLetterListReq(Request): + source: Annotated[str, choice("webhook", "job", "all", help="Which consumer failed.")] = "all" + since: Annotated[ + str | None, + opt("--since", metavar="WHEN", kind="datetime", help="Only entries after this time."), + ] = None + events: Annotated[ + str | None, opt("--events", metavar="TYPES", help="Filter by event type.") + ] = None + + +async def dead_letter_list(ctx: OpContext, req: DeadLetterListReq) -> Page[DeadLetter]: + """Events no consumer could be given. + + One store, shared by the webhook pusher and the gateway actions, at mode + 0600 and size-rotated. v1 appended full message text to a world-readable + file that grew without limit (SEC-06). + """ + _webhook, entries = _dead_letters(ctx) + since = _iso(req.since) + rows = [ + _dead_letter_model(index, entry) + for index, entry in enumerate(entries) + if _matches(entry, req.source, since, req.events) + ] + if ctx.account and ctx.account != "all": + rows = [row for row in rows if row.account == ctx.account] + limit = int(getattr(ctx, "limit", None) or 100) + return build_page( + rows[:limit], + op="daemon.dead-letter.list", + kind=PageKind.LOCAL, + has_more=len(rows) > limit, + total=len(rows), + ) + + +def _iso(value: str | None) -> str | None: + if not value: + return None + parsed = parse_dt(value) + return parsed.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") if parsed else value + + +SPEC_DEAD_LETTER_LIST = OperationSpec( + id="daemon.dead-letter.list", + request=DeadLetterListReq, + response=Page[DeadLetter], + impl=dead_letter_list, + summary="List events that could not be delivered", + aliases=("webhook.dead-letter.list", "job.dead-letter.list"), + paginated=PageKind.LOCAL, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("id", "source", "event", "account", "attempts", "last_error", "last_failed_at"), + example={ + "items": [ + { + "id": "0f3c…", + "source": "webhook", + "event": "message_new", + "account": "work", + "attempts": 3, + "last_error": "HTTP 502", + } + ], + "has_more": False, + }, + example_args="daemon dead-letter list", + covers_partial=("updates.stream-webhook-delivery",), + coverage_note="the failure store; delivery itself is the webhook pusher.", + empty_exit=EXIT_EMPTY, + tags=frozenset({"agent-safe"}), +) + + +class DeadLetterSendReq(Request): + source: Annotated[str, choice("webhook", "job", "all", help="Which consumer to re-drive.")] = ( + "all" + ) + id: Annotated[ + list[str], opt("--id", metavar="ID", help="Only these entries (repeatable).") + ] = [] + since: Annotated[ + str | None, opt("--since", metavar="WHEN", kind="datetime", help="Only entries after this.") + ] = None + keep_on_success: Annotated[ + bool, opt("--keep-on-success", help="Do not remove entries that deliver.") + ] = False + url: Annotated[str | None, opt("--url", metavar="URL", help="Deliver to this URL instead.")] = ( + None + ) + + +async def dead_letter_send(ctx: OpContext, req: DeadLetterSendReq) -> DeadLetterResult: + """Re-deliver what was dead-lettered, keeping the original delivery id. + + A receiver keyed on `Idempotency-Key` therefore sees a duplicate rather + than a new event, which is what makes a drain safe to run twice. + """ + webhook, entries = _dead_letters(ctx) + since = _iso(req.since) + wanted = set(req.id) + remaining: list[dict[str, Any]] = [] + attempted = delivered = failed = 0 + + for index, entry in enumerate(entries): + identifier = str(entry.get("delivery_id") or f"dl-{index}") + if wanted and identifier not in wanted: + remaining.append(entry) + continue + if not _matches(entry, req.source, since, None): + remaining.append(entry) + continue + attempted += 1 + ok, error = await webhook.deliver_once(entry, url=req.url or "") + if ok: + delivered += 1 + if req.keep_on_success: + remaining.append(entry) + else: + failed += 1 + entry["reason"] = error + entry["attempts"] = int(entry.get("attempts") or 1) + 1 + entry["ts"] = _now() + remaining.append(entry) + + webhook.write_dead_letters(remaining) + if attempted == 0: + ctx.mark_already() + return DeadLetterResult( + attempted=attempted, delivered=delivered, failed=failed, remaining=len(remaining) + ) + + +SPEC_DEAD_LETTER_SEND = OperationSpec( + id="daemon.dead-letter.send", + request=DeadLetterSendReq, + response=DeadLetterResult, + impl=dead_letter_send, + summary="Re-deliver dead-lettered events", + aliases=( + "webhook.dead-letter.drain", + "job.dead-letter.drain", + "daemon.dead-letter.drain", + ), + mutating=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=300, + example={"attempted": 4, "delivered": 3, "failed": 1, "remaining": 1}, + example_args="daemon dead-letter send", + covers_partial=("updates.stream-webhook-delivery",), + coverage_note="the replay half; the live delivery path is the webhook pusher.", + tags=frozenset({"agent-safe"}), +) + + +class DeadLetterDeleteReq(Request): + source: Annotated[str, choice("webhook", "job", "all", help="Restrict by consumer.")] = "all" + id: Annotated[ + list[str], opt("--id", metavar="ID", help="Only these entries (repeatable).") + ] = [] + until: Annotated[ + str | None, + opt("--until", metavar="WHEN", kind="datetime", help="Only entries older than this."), + ] = None + everything: Annotated[bool, opt("--every", help="Discard everything.")] = False + + +async def dead_letter_delete(ctx: OpContext, req: DeadLetterDeleteReq) -> DeadLetterResult: + """Discard dead-lettered events permanently.""" + webhook, entries = _dead_letters(ctx) + if not (req.id or req.until or req.everything): + raise UsageError("say what to delete: --id, --until, or --every", field="id") + until = _iso(req.until) + wanted = set(req.id) + remaining: list[dict[str, Any]] = [] + deleted = 0 + for index, entry in enumerate(entries): + identifier = str(entry.get("delivery_id") or f"dl-{index}") + drop = req.everything + if wanted: + drop = identifier in wanted + elif until: + drop = str(entry.get("ts", "")) < until + if drop and _matches(entry, req.source, None, None): + deleted += 1 + continue + remaining.append(entry) + webhook.write_dead_letters(remaining) + if deleted == 0: + ctx.mark_already() + return DeadLetterResult(deleted=deleted, remaining=len(remaining)) + + +SPEC_DEAD_LETTER_DELETE = OperationSpec( + id="daemon.dead-letter.delete", + request=DeadLetterDeleteReq, + response=DeadLetterResult, + impl=dead_letter_delete, + summary="Permanently discard dead-lettered events", + aliases=("webhook.dead-letter.clear", "job.dead-letter.clear"), + mutating=True, + destructive=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=60, + example={"deleted": 12, "remaining": 0}, + example_args="daemon dead-letter delete --every", + covers_partial=("updates.stream-webhook-delivery",), + coverage_note="the disposal half; delivery is the webhook pusher's.", + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/ops/events.py b/tlgr/ops/events.py new file mode 100644 index 0000000..db97c53 --- /dev/null +++ b/tlgr/ops/events.py @@ -0,0 +1,1327 @@ +"""The `events` group and `watch`: discovering, replaying and following events. + +Four of these five operations run without touching Telegram at all. That is +the point: an agent should be able to ask *what can arrive* (`events list`), +*what one looks like* (`events get`), and *what did arrive* (`events replay`) +before it commits to holding a stream open (`watch`). + +`watch` replaces v1's poller. v1 asked the daemon for `chat list` every two +seconds, then `message list` per chat, and emitted only new messages — so an +edit, a deletion, a read receipt, a reaction and every service message were +invisible, and twenty chats cost thirty HTTP round trips a minute whether or +not anything happened. Here the daemon holds one `events.Raw` handler per +account and a watcher is a bounded queue on the bus: nothing is polled, and +everything in the taxonomy is selectable. +""" + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import contextlib +import json +import sys +import time +from collections.abc import AsyncIterator +from typing import Annotated, Any + +from tlgr.core import eventtypes +from tlgr.core.errors import ( + EXIT_EMPTY, + IndeterminateError, + NotFoundError, + NotSupportedError, + UsageError, +) +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request, to_builtins +from tlgr.models.event import DecodedEvent, EventEnvelope, EventType, EventTypeDetail +from tlgr.models.page import Page +from tlgr.models.peer import PeerRef +from tlgr.ops import _send +from tlgr.ops._params import arg, choice, opt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + +_EXAMPLE_ENVELOPE: dict[str, Any] = { + "seq": 91824, + "ts": "2026-09-03T09:14:07Z", + "account": "work", + "type": "message_new", + "payload": {"id": 12345, "chat_id": 777123, "text": "on my way"}, + "chat_id": 777123, + "sender_id": 4242, +} + + +# --------------------------------------------------------------------------- +# Shared plumbing +# --------------------------------------------------------------------------- + + +def _row(name: str, spec: eventtypes.EventTypeSpec, sources: tuple[str, ...]) -> EventType: + return EventType( + type=name, + group=spec.group, + summary=spec.summary, + sources=list(sources), + telethon=spec.telethon, + box=spec.box, + bot_only=spec.bot_only, + since_layer=spec.since_layer, + available=spec.since_layer == 0, + derived=spec.derived, + ) + + +def _window(ctx: OpContext, op: str, default: int = 200) -> tuple[int, dict[str, Any]]: + from tlgr.core.pagination import decode_cursor + + limit = int(getattr(ctx, "limit", None) or default) + if limit < 1: + raise UsageError("--limit must be at least 1", field="limit") + token = getattr(ctx, "cursor", None) + state: dict[str, Any] = {} + if token: + state = decode_cursor(token, op=op, kind=PageKind.LOCAL, account=ctx.account) + return min(limit, 5000), state + + +async def _chat_ids(ctx: OpContext, refs: tuple[PeerRef, ...]) -> list[int]: + """`--chat` → marked ids, resolving through the account when it is connected. + + A watcher does not need a Telegram client, so the resolver may be absent; + a numeric id still works, and a username without a connected account is a + usage error naming the fix rather than a filter that silently matches + nothing. + """ + out: list[int] = [] + for ref in refs: + if ref.kind == "id": + out.append(int(ref.value)) + continue + if getattr(ctx, "resolver", None) is None: + raise UsageError( + f"{ref.raw!r} needs a connected account to resolve; " + "pass the numeric chat id, or start the daemon first", + field="chat", + ) + out.append(_send.peer_id_of(await _send.resolve(ctx, ref))) + return out + + +def _bus(ctx: OpContext) -> Any: + bus = getattr(ctx, "bus", None) + if bus is None: + raise UsageError("this operation needs the daemon's event bus") + return bus + + +def _accounts(ctx: OpContext) -> list[str]: + """The accounts an `--account all` operation spans, or the one given.""" + if ctx.account and ctx.account != "all": + return [ctx.account] + daemon = getattr(ctx, "daemon", None) + sessions = getattr(daemon, "sessions", None) + return list(getattr(sessions, "aliases", []) or []) + + +# --------------------------------------------------------------------------- +# events list +# --------------------------------------------------------------------------- + + +class EventListReq(Request): + group: Annotated[ + str | None, + choice(*eventtypes.GROUPS, help="Only this family."), + ] = None + raw: Annotated[ + bool, + opt("--raw", help="One row per raw TL update constructor instead of per type."), + ] = False + available: Annotated[ + bool, + opt( + "--available", + help="Only types this build can actually receive (hides bot-only and layer-229).", + ), + ] = False + search: Annotated[ + str | None, + opt("--search", metavar="TEXT", help="Substring match on type, constructor or summary."), + ] = None + + +async def event_list(ctx: OpContext, req: EventListReq) -> Page[EventType]: + """The subscribable surface, machine-readable. + + An agent that has to learn the vocabulary from prose will get it wrong; + this is the same table `docs/design/EVENTS.md` prints and `watch --events` + accepts, so there is exactly one source of truth for it. + """ + rows: list[EventType] = [] + for name, spec in sorted(eventtypes.TYPES.items()): + if req.group and spec.group != req.group: + continue + if req.available and (spec.bot_only or spec.since_layer): + continue + sources = eventtypes.constructors_for(name) + if req.raw: + for source in sources: + if req.available and source in eventtypes.NEWER_THAN_LAYER_227: + continue + row = _row(name, spec, (source,)) + row.available = source not in eventtypes.NEWER_THAN_LAYER_227 + rows.append(row) + else: + rows.append(_row(name, spec, sources)) + + if req.search: + needle = req.search.lower() + rows = [ + row + for row in rows + if needle in row.type + or needle in row.summary.lower() + or any(needle in source.lower() for source in row.sources) + ] + + limit, state = _window(ctx, "events.list") + offset = int(state.get("offset", 0)) + window = rows[offset : offset + limit] + return build_page( + window, + op="events.list", + kind=PageKind.LOCAL, + state={"offset": offset + len(window)}, + account=ctx.account, + has_more=offset + len(window) < len(rows), + total=len(rows), + ) + + +SPEC_EVENT_LIST = OperationSpec( + id="events.list", + request=EventListReq, + response=Page[EventType], + impl=event_list, + summary="List the event types tlgr can emit, with their source constructors", + description=( + "114 types covering every `Update*` constructor Telethon can parse, " + "plus the ones Telegram has added since. `--raw` lists the " + "constructors instead. These names are the only values `watch " + "--events`, `job add --events` and `webhook set --events` accept." + ), + # No `schema events` alias: placing one would turn the top-level `schema` + # command into a group and take v1's bare `tlgr schema` with it. The same + # taxonomy is reachable as `tlgr schema events`, which is `agent.schema`'s + # own positional (DECISIONS, 2026-09-03). + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + paginated=PageKind.LOCAL, + columns=("type", "group", "box", "summary"), + example={ + "items": [ + { + "type": "message_new", + "group": "message", + "summary": "A message arrived in any chat the account can see", + "sources": ["UpdateNewMessage"], + "box": "pts", + } + ], + "has_more": False, + "total": 114, + }, + example_args="events list --group message", + covers=( + "updates.event-ai-compose-tones", + "updates.event-autosave-settings", + "updates.event-bot-callback-query", + "updates.event-bot-ephemeral-callback", + "updates.event-bot-inline-query", + "updates.event-bot-message-reactions", + "updates.event-bot-stars-subscription", + "updates.event-bot-webhook-json", + "updates.event-channel-forwards", + "updates.event-channel-views", + "updates.event-chat-participants", + "updates.event-config-changed", + "updates.event-dc-options", + "updates.event-dialog-filters", + "updates.event-dialog-unread-mark", + "updates.event-emoji-game-info", + "updates.event-ephemeral-messages", + "updates.event-folder-peers", + "updates.event-group-call", + "updates.event-join-chat-webview-decision", + "updates.event-login-token", + "updates.event-message-deleted", + "updates.event-new-authorization", + "updates.event-new-message", + "updates.event-peer-blocked", + "updates.event-peer-wallpaper", + "updates.event-pinned-messages", + "updates.event-pts-changed", + "updates.event-read-contents", + "updates.event-read-monoforum", + "updates.event-saved-dialogs", + "updates.event-scheduled-deleted", + "updates.event-service-message", + "updates.event-stars-balance", + "updates.event-stories-stealth", + "updates.event-story-reaction", + "updates.event-typing", + "updates.event-user-phone", + "updates.event-view-forum-as-messages", + "updates.event-webview-result-sent", + ), + covers_partial=( + "updates.event-attach-menu-bots", + "updates.event-bot-business", + "updates.event-bot-commands", + "updates.event-bot-guest-chat-query", + "updates.event-bot-menu-button", + "updates.event-bot-payments", + "updates.event-bot-stopped", + "updates.event-channel-available-messages", + "updates.event-channel-participant", + "updates.event-chat-boost", + "updates.event-chat-refetch", + "updates.event-contacts-reset", + "updates.event-default-banned-rights", + "updates.event-dialog-pinned", + "updates.event-draft", + "updates.event-encrypted-chats", + "updates.event-extended-media", + "updates.event-geo-live-viewed", + "updates.event-history-ttl", + "updates.event-join-requests", + "updates.event-managed-bot", + "updates.event-message-edited", + "updates.event-message-id-map", + "updates.event-new-bot-connection", + "updates.event-new-channel-message", + "updates.event-notify-settings", + "updates.event-paid-reaction-privacy", + "updates.event-peer-located", + "updates.event-peer-settings", + "updates.event-phone-call", + "updates.event-pinned-forum-topics", + "updates.event-poll", + "updates.event-privacy", + "updates.event-quick-replies", + "updates.event-reactions", + "updates.event-read-discussion", + "updates.event-read-inbox", + "updates.event-read-outbox", + "updates.event-recent-reactions", + "updates.event-report-message-delivery", + "updates.event-saved-gifs", + "updates.event-saved-ringtones", + "updates.event-scheduled-new", + "updates.event-sent-phone-code", + "updates.event-service-notification", + "updates.event-star-gift-auction", + "updates.event-stars-revenue", + "updates.event-stickers-changed", + "updates.event-story-id", + "updates.event-story-new", + "updates.event-story-read", + "updates.event-transcription", + "updates.event-user-emoji-status", + "updates.event-user-name", + "updates.event-user-refetch", + "updates.event-user-status", + "updates.event-web-browser-settings", + "updates.event-webpage", + "updates.stream-event-types", + "updates.stream-raw-passthrough", + ), + coverage_note=( + "the catalogue half: the type exists and is selectable. Receiving one " + "is `watch`, which owns those ids fully." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# events get +# --------------------------------------------------------------------------- + + +class EventGetReq(Request): + type: Annotated[ + str, + arg(0, metavar="TYPE", help="An event type name, or a `raw:Constructor`."), + ] + example: Annotated[ + bool, opt("--example/--no-example", help="Include a synthetic example envelope.") + ] = True + json_schema: Annotated[ + bool, opt("--json-schema", help="Emit the payload as a JSON Schema object.") + ] = False + + +def _json_schema(payload: dict[str, str]) -> dict[str, Any]: + """The payload table as draft 2020-12. + + Deliberately loose: most payloads are the update's own fields made + JSON-safe, and a schema that claimed to be exhaustive about them would be + a promise the taxonomy does not make. + """ + properties: dict[str, Any] = {} + for name, described in payload.items(): + if name in ("…", "_"): + continue + base = described.split("—")[0].strip() + kind = { + "int": "integer", + "str": "string", + "bool": "boolean", + "object": "object", + "true": "boolean", + "false": "boolean", + }.get(base.replace(" | null", "").strip(), "string") + if base.startswith("list["): + kind = "array" + properties[name] = {"type": kind, "description": described} + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": properties, + "additionalProperties": True, + } + + +async def event_get(ctx: OpContext, req: EventGetReq) -> EventTypeDetail: + """One event type in full: sources, box, payload, and an example.""" + name = req.type.strip().lower() + if name.startswith("raw:"): + mapped = eventtypes.type_for_constructor(req.type[4:]) + if mapped is None: + raise NotFoundError(f"no event type is produced by {req.type[4:]!r}") + name = mapped + for legacy, expansion in eventtypes.ALIASES.items(): + if name == legacy and len(expansion) == 1: + name = expansion[0] + spec = eventtypes.TYPES.get(name) + if spec is None: + raise NotFoundError(f"unknown event type {req.type!r}; run `tlgr events list`") + + row = _row(name, spec, eventtypes.constructors_for(name)) + detail = EventTypeDetail( + type=row.type, + group=row.group, + summary=row.summary, + sources=row.sources, + telethon=row.telethon, + box=row.box, + bot_only=row.bot_only, + since_layer=row.since_layer, + available=row.available, + derived=row.derived, + payload=dict(spec.payload), + filters=["account", "chat", "sender", "type", "self_origin"], + ) + if req.json_schema: + detail.json_schema = _json_schema(spec.payload) + if req.example: + detail.example = EventEnvelope( + seq=1, + ts="2026-09-03T09:14:07Z", + account=ctx.account or "work", + type=name, + payload={key: None for key in spec.payload if key not in ("…",)}, + chat_id=-1001234567890, + ) + return detail + + +SPEC_EVENT_GET = OperationSpec( + id="events.get", + request=EventGetReq, + response=EventTypeDetail, + impl=event_get, + summary="Show one event type: payload, source constructors, sequence box, example", + description=( + "`box` is the field to read first: it says which sequence orders the " + "event, and therefore whether a gap in it is recoverable with `sync " + "difference` or simply lost." + ), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + example={ + "type": "message_new", + "group": "message", + "summary": "A message arrived in any chat the account can see", + "sources": ["UpdateNewMessage"], + "box": "pts", + "payload": {"message": "Message"}, + }, + example_args="events get message_new", + covers=( + "updates.event-attach-menu-bots", + "updates.event-bot-business", + "updates.event-bot-commands", + "updates.event-bot-guest-chat-query", + "updates.event-bot-menu-button", + "updates.event-bot-payments", + "updates.event-bot-stopped", + "updates.event-channel-available-messages", + "updates.event-channel-participant", + "updates.event-chat-boost", + "updates.event-chat-refetch", + "updates.event-contacts-reset", + "updates.event-default-banned-rights", + "updates.event-dialog-pinned", + "updates.event-draft", + "updates.event-encrypted-chats", + "updates.event-extended-media", + "updates.event-geo-live-viewed", + "updates.event-history-ttl", + "updates.event-join-requests", + "updates.event-managed-bot", + "updates.event-message-edited", + "updates.event-new-bot-connection", + "updates.event-notify-settings", + "updates.event-peer-located", + "updates.event-phone-call", + "updates.event-poll", + "updates.event-quick-replies", + "updates.event-read-discussion", + "updates.event-read-outbox", + "updates.event-saved-gifs", + "updates.event-scheduled-new", + "updates.event-service-notification", + "updates.event-stars-revenue", + "updates.event-story-id", + "updates.event-story-read", + "updates.event-user-emoji-status", + "updates.event-user-refetch", + "updates.event-web-browser-settings", + ), + covers_partial=("updates.stream-event-types", "updates.sync-min-constructors"), + coverage_note=( + "documents the type and its payload; receiving one is `watch`, and " + "min-constructor hydration happens on the bus." + ), + empty_exit=EXIT_EMPTY, + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# events decode +# --------------------------------------------------------------------------- + +#: `loc_key` prefixes Telegram uses in a push payload → tlgr event types. The +#: two that matter are not messages: `DC_UPDATE` and `SESSION_REVOKE` are +#: security events, and the second one means this session has been killed. +_PUSH_KEYS: dict[str, str] = { + "DC_UPDATE": "sync_dc_options", + "SESSION_REVOKE": "account_session_revoked", + "AUTH_REGION": "account_new_authorization", + "AUTH_UNKNOWN": "account_new_authorization", + "MESSAGE": "message_new", + "CHAT_MESSAGE": "message_new", + "CHANNEL_MESSAGE": "message_new", + "ENCRYPTED_MESSAGE": "secret_message", + "PHONE_CALL": "call_phone", + "READ_HISTORY": "read_inbox", + "MESSAGE_DELETED": "message_deleted", + "REACT": "message_reactions", + "GEO_LIVE_PENDING": "message_geo_live_viewed", + "STORY": "story_new", +} + + +def _push_event_type(loc_key: str) -> str: + key = (loc_key or "").upper() + for prefix, mapped in _PUSH_KEYS.items(): + if key == prefix or key.startswith(f"{prefix}_"): + return mapped + return "account_service_notification" + + +def _read_input(source: str | None) -> str: + if source in (None, "", "-"): + if sys.stdin is None or sys.stdin.isatty(): + raise UsageError("no input was given and stdin is a terminal", field="input") + return sys.stdin.read() + try: + with open(str(source), encoding="utf-8") as handle: + return handle.read() + except OSError as exc: + raise UsageError(f"{source}: {exc.strerror or exc}", field="input") from exc + + +def _decrypt_push(blob: bytes, auth_key: bytes) -> dict[str, Any]: + """MTProto 2.0 decryption of an encrypted push payload. + + Telegram encrypts a push notification with the *push* auth key, using the + same key derivation as a message: `msg_key` first, then AES-256-IGE. The + direction byte is not documented for push, so both are tried and the one + whose recomputed `msg_key` matches is the right one — which also means a + wrong key produces "could not be decrypted" rather than plausible rubbish. + """ + import hashlib + + from telethon.crypto import AES + + if len(auth_key) != 256: + raise UsageError(f"a push auth key is 256 bytes; this one is {len(auth_key)}", field="key") + if len(blob) < 16 or (len(blob) - 16) % 16: + raise UsageError("the push payload is not a multiple of the AES block size", field="input") + msg_key, body = blob[:16], blob[16:] + + for offset in (0, 8): + sha256_a = hashlib.sha256(msg_key + auth_key[offset : offset + 36]).digest() + sha256_b = hashlib.sha256(auth_key[offset + 40 : offset + 76] + msg_key).digest() + key = sha256_a[:8] + sha256_b[8:24] + sha256_a[24:32] + iv = sha256_b[:8] + sha256_a[8:24] + sha256_b[24:32] + plain = AES.decrypt_ige(body, key, iv) + computed = hashlib.sha256(auth_key[88 + offset : 88 + offset + 32] + plain).digest()[8:24] + if computed != msg_key: + continue + length = int.from_bytes(plain[:4], "little") + payload = plain[4 : 4 + length] + try: + decoded = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise IndeterminateError( + "the payload decrypted but is not the JSON push body; " + "the key may be for a different session" + ) from exc + if not isinstance(decoded, dict): + raise IndeterminateError("the decrypted push payload is not an object") + return decoded + + raise IndeterminateError( + "the push payload could not be decrypted with this key — it belongs to " + "another session, or the payload is truncated" + ) + + +class EventDecodeReq(Request): + input: Annotated[ + str | None, + arg( + 0, + metavar="INPUT", + required=False, + help="A file, or '-' for stdin: a JSON TL object, or a base64 push payload.", + ), + ] = None + push: Annotated[ + bool, opt("--push", help="The input is a Telegram push-notification payload.") + ] = False + key: Annotated[ + str | None, + opt( + "--key", + secret=True, + envvar="TLGR_PUSH_KEY", + help="The base64 push auth key used to decrypt the payload.", + ), + ] = None + raw: Annotated[ + bool, opt("--raw", help="Print the decoded TL object instead of the tlgr envelope.") + ] = False + + +async def event_decode(ctx: OpContext, req: EventDecodeReq) -> DecodedEvent: + """Turn a raw update or a push payload into the envelope tlgr would emit. + + Offline and account-free on purpose. tlgr does not register for push — + the daemon holds a socket, so it has no need of one — but a phone-relay + setup does, and `DC_UPDATE`/`SESSION_REVOKE` are security events somebody + has to be able to read. + """ + text = _read_input(req.input) + + if req.push: + try: + blob = base64.b64decode(text.strip(), validate=False) + except (binascii.Error, ValueError) as exc: + raise UsageError("the push payload is not base64", field="input") from exc + try: + body = json.loads(blob.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + # The CLI has already read the secret out of the environment, a + # file or stdin; it never travels as argv (STYLE §3). + if not req.key: + raise UsageError( + "this push payload is encrypted; supply the push auth key with " + "--key-env, --key-file or --key-stdin", + field="key", + ) from None + body = _decrypt_push(blob, base64.b64decode(req.key)) + if not isinstance(body, dict): + raise UsageError("the push payload is not a JSON object", field="input") + inner = body.get("data") + data: dict[str, Any] = inner if isinstance(inner, dict) else body + loc_key = str(data.get("loc_key", "")) + chat = data.get("chat_id") or data.get("channel_id") or data.get("from_id") + chat_id = ( + int(chat) if isinstance(chat, (int, str)) and str(chat).lstrip("-").isdigit() else None + ) + return DecodedEvent( + event=_push_event_type(loc_key), + account=ctx.account, + chat_id=chat_id, + sender_id=None, + data=dict(data), + raw=dict(body) if req.raw else None, + push=True, + ) + + try: + loaded = json.loads(text) + except json.JSONDecodeError as exc: + raise UsageError(f"the input is not JSON: {exc}", field="input") from exc + if not isinstance(loaded, dict): + raise UsageError("a TL update is a JSON object", field="input") + + constructor = str(loaded.get("_") or loaded.get("constructor") or "") + if not constructor: + raise UsageError( + "the object has no `_` naming its TL constructor; " + "`tlgr watch --with-raw` emits that form", + field="input", + ) + event_type = eventtypes.type_for_constructor(constructor) + if event_type is None: + reason = eventtypes.INTERNAL.get(constructor) + if reason: + raise NotSupportedError(f"{constructor} carries no event: {reason}") + raise NotFoundError( + f"{constructor} is not an update tlgr knows; run `tlgr events list --raw`" + ) + chat = loaded.get("chat_id") + return DecodedEvent( + event=event_type, + account=ctx.account, + chat_id=int(chat) if isinstance(chat, int) else None, + data={key: value for key, value in loaded.items() if key != "_"}, + raw=dict(loaded) if req.raw else None, + ) + + +SPEC_EVENT_DECODE = OperationSpec( + id="events.decode", + request=EventDecodeReq, + response=DecodedEvent, + impl=event_decode, + summary="Decode a raw TL update or an encrypted push payload into an event", + description=( + "Offline; no account needed. tlgr does not register for push " + "notifications — the daemon holds a socket — but a phone-relay setup " + "does, and `DC_UPDATE` and `SESSION_REVOKE` are security events: the " + "second one means this session has been terminated." + ), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + example={ + "event": "message_new", + "data": {"pts": 4213, "pts_count": 1}, + }, + example_args="events decode - --push", + covers=("updates.push-payload-decrypt",), + covers_partial=("updates.stream-raw-passthrough",), + coverage_note="decodes one update offline; the live passthrough is `watch --raw`.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# The shared selection used by watch and replay +# --------------------------------------------------------------------------- + + +class _Selection: + """The filter a watcher or a replay applies, resolved once.""" + + __slots__ = ("chats", "senders", "topic", "types") + + def __init__( + self, + types: frozenset[str], + chats: list[int], + senders: list[int], + topic: int | None, + ) -> None: + self.types = types + self.chats = set(chats) + self.senders = set(senders) + self.topic = topic + + def wants(self, event: EventEnvelope) -> bool: + if self.types and event.type not in self.types: + return False + if self.chats and (event.chat_id is None or event.chat_id not in self.chats): + return False + if self.senders and (event.sender_id is None or event.sender_id not in self.senders): + return False + return not (self.topic is not None and event.payload.get("top_msg_id") != self.topic) + + +async def _selection( + ctx: OpContext, + *, + events: str, + exclude: str | None, + chats: tuple[PeerRef, ...], + senders: tuple[PeerRef, ...], + topic: int | None, +) -> _Selection: + wanted = eventtypes.resolve_selectors(events) + if exclude: + wanted = wanted - eventtypes.resolve_selectors(exclude, allow_all=False) + if not wanted: + raise UsageError( + "--events and --exclude together select nothing; a watch that " + "matches nothing is indistinguishable from a broken daemon", + field="events", + ) + return _Selection( + wanted, + await _chat_ids(ctx, chats), + await _chat_ids(ctx, senders), + topic, + ) + + +# --------------------------------------------------------------------------- +# watch +# --------------------------------------------------------------------------- + + +class WatchReq(Request): + events: Annotated[ + str, + opt( + "--events", + metavar="TYPES", + help=( + "Types, groups, `raw:Constructor` names or `all`, " + "comma-separated. See `tlgr events list`." + ), + ), + ] = "new_message" + exclude: Annotated[ + str | None, + opt("--exclude", metavar="TYPES", help="Subtract these after --events is applied."), + ] = None + chat: Annotated[ + list[PeerRef], + opt("--chat", metavar="CHAT", kind="peer", help="Only events about this chat."), + ] = [] + sender: Annotated[ + list[PeerRef], + opt("--sender", metavar="USER", kind="user", help="Only events from this user."), + ] = [] + topic: Annotated[ + int | None, opt("--topic", metavar="ID", help="Only events inside this forum topic.") + ] = None + since: Annotated[ + int | None, + opt("--since", metavar="SEQ", help="Replay from this seq (exclusive) before following."), + ] = None + follow: Annotated[ + bool, opt("--follow/--no-follow", help="Keep streaming after the replay is drained.") + ] = True + max_events: Annotated[ + int | None, + opt("--max-events", metavar="N", help="Stop after this many events."), + ] = None + raw: Annotated[ + bool, opt("--raw", help="Emit only the raw TL update instead of the envelope.") + ] = False + with_raw: Annotated[ + bool, opt("--with-raw", help="Include the raw TL update beside the payload.") + ] = False + heartbeat: Annotated[ + int, + opt("--heartbeat", metavar="SECONDS", ge=0, help="Idle keepalive; 0 disables."), + ] = 15 + on_lag: Annotated[ + str, + choice( + "drop", + "block", + "fail", + help=( + "Falling behind: drop the oldest and report it, take a much " + "larger queue, or stop with exit 13." + ), + ), + ] = "drop" + follow_for: Annotated[ + int, + opt("--follow-for", metavar="SECONDS", ge=1, le=86400, help="Close the stream after this."), + ] = 3600 + print_cursor: Annotated[ + bool, opt("--print-cursor", help="Emit a final frame carrying the resume seq.") + ] = False + + +async def watch(ctx: OpContext, req: WatchReq) -> AsyncIterator[dict[str, Any]]: + """Follow the bus, as NDJSON frames. + + Push, never polling: the daemon already holds the update socket, so a + watcher is a bounded queue on the bus rather than v1's two-second + `chat list` + `message list` loop, which cost thirty round trips a minute + and could only ever report new messages. + + `--account all` multiplexes every connected account; each frame carries + its own `account`, and `seq` is per account because update state is. + """ + bus = _bus(ctx) + accounts = _accounts(ctx) + selection = await _selection( + ctx, + events=req.events, + exclude=req.exclude, + chats=tuple(req.chat), + senders=tuple(req.sender), + topic=req.topic, + ) + watching = accounts or [ctx.account] + + yield { + "type": "watching", + "accounts": watching, + "events": sorted(selection.types), + "chats": sorted(selection.chats), + "latest_seq": {alias: bus.latest_seq(alias) for alias in watching}, + } + + subscribers = [ + bus.subscribe( + alias, + types=selection.types, + maxsize=8192 if req.on_lag == "block" else 2048, + want_raw=req.raw or req.with_raw, + ) + for alias in watching + ] + delivered = 0 + last_seq: dict[str, int] = {} + reason = "closed" + try: + for alias in watching: + if req.since is None: + continue + replayed, gap = bus.replay(alias, req.since) + if gap is not None: + yield {**gap, "account": alias} + for event in replayed: + if not selection.wants(event): + continue + delivered += 1 + last_seq[alias] = event.seq + yield _frame(event, req) + if req.max_events and delivered >= req.max_events: + reason = "limit" + break + if reason == "limit": + break + + if req.follow and reason != "limit": + deadline = time.monotonic() + req.follow_for + heartbeat = float(req.heartbeat) if req.heartbeat else None + while reason == "closed": + remaining = deadline - time.monotonic() + if remaining <= 0: + reason = "timeout" + break + wait = min(heartbeat, remaining) if heartbeat else remaining + pending = [asyncio.ensure_future(sub.queue.get()) for sub in subscribers] + done, _ = await asyncio.wait( + pending, timeout=wait, return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + if task not in done: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + if not done: + if heartbeat: + yield {"type": "heartbeat", "ts": _now()} + continue + for task in done: + event = task.result() + lag = _take_lag(subscribers, event.account) + if lag: + if req.on_lag == "fail": + raise IndeterminateError( + f"this watcher fell behind and lost {lag} events; " + "resume from the last seq you saw with --since" + ) + yield {"type": "lag", "dropped": lag, "account": event.account} + if not selection.wants(event): + continue + delivered += 1 + last_seq[event.account] = event.seq + yield _frame(event, req) + if req.max_events and delivered >= req.max_events: + reason = "limit" + break + finally: + for subscriber in subscribers: + bus.unsubscribe(subscriber) + + # Outside the `finally`, deliberately: yielding while an async generator + # is being closed raises, and the resume cursor is worth having only when + # the stream ended on its own terms. + if req.print_cursor: + yield {"type": "cursor", "latest_seq": last_seq, "reason": reason} + + +def _take_lag(subscribers: list[Any], account: str) -> int: + for subscriber in subscribers: + if subscriber.account == account: + return int(subscriber.take_lag()) + return 0 + + +def _now() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _frame(event: EventEnvelope, req: WatchReq) -> dict[str, Any]: + if req.raw: + return {"type": event.type, "seq": event.seq, "account": event.account, "raw": event.raw} + frame = to_builtins(event) + if not isinstance(frame, dict): # pragma: no cover - EventEnvelope is a Struct + return {"type": event.type} + if not req.with_raw: + frame.pop("raw", None) + return frame + + +SPEC_WATCH = OperationSpec( + id="events.watch", + request=WatchReq, + response=None, + impl=watch, + summary="Stream live events from the daemon as newline-delimited JSON", + description=( + "Push-driven from the daemon's event bus, not polled: v1 asked for " + "`chat list` and then `message list` every two seconds and could only " + "report new messages. Every type in `tlgr events list` is selectable, " + "`--since ` replays the ring buffer first (with a `gap` frame " + "when it cannot reach that far back), and a watcher that falls behind " + "gets a `lag` frame rather than silence." + ), + aliases=("events.tail",), + legacy_paths=("watch",), + stream=True, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=900, + example={"type": "message_new", "seq": 91824}, + example_args="watch --events message_new,read_inbox --chat @alice", + covers=( + "bots.bot-side-update-stream", + "bots.bot-subscription-update", + "bots.ephemeral-message-view", + "contacts-users.user-status-watch", + "dialogs.typing-watch", + "dialogs.watch-dialog-events", + "giveaway.prize-stars", + "location.proximity-alert-event", + "location.viewed-receipt", + "messages-core.message-watch-events", + "updates.event-message-id-map", + "updates.event-new-channel-message", + "updates.event-paid-reaction-privacy", + "updates.event-peer-settings", + "updates.event-pinned-forum-topics", + "updates.event-privacy", + "updates.event-reactions", + "updates.event-read-inbox", + "updates.event-recent-reactions", + "updates.event-saved-ringtones", + "updates.event-sent-phone-code", + "updates.event-star-gift-auction", + "updates.event-stickers-changed", + "updates.event-story-new", + "updates.event-transcription", + "updates.event-user-name", + "updates.event-user-status", + "updates.event-webpage", + "updates.stream-event-types", + "updates.stream-raw-passthrough", + "updates.stream-watch-ndjson", + "updates.sync-min-constructors", + ), + covers_partial=( + "updates.event-ai-compose-tones", + "updates.event-attach-menu-bots", + "updates.event-autosave-settings", + "updates.event-bot-business", + "updates.event-bot-callback-query", + "updates.event-bot-commands", + "updates.event-bot-ephemeral-callback", + "updates.event-bot-guest-chat-query", + "updates.event-bot-inline-query", + "updates.event-bot-menu-button", + "updates.event-bot-message-reactions", + "updates.event-bot-payments", + "updates.event-bot-stars-subscription", + "updates.event-bot-stopped", + "updates.event-bot-webhook-json", + "updates.event-channel-available-messages", + "updates.event-channel-forwards", + "updates.event-channel-participant", + "updates.event-channel-views", + "updates.event-chat-boost", + "updates.event-chat-participants", + "updates.event-chat-refetch", + "updates.event-config-changed", + "updates.event-contacts-reset", + "updates.event-dc-options", + "updates.event-default-banned-rights", + "updates.event-dialog-filters", + "updates.event-dialog-pinned", + "updates.event-dialog-unread-mark", + "updates.event-draft", + "updates.event-emoji-game-info", + "updates.event-encrypted-chats", + "updates.event-ephemeral-messages", + "updates.event-extended-media", + "updates.event-folder-peers", + "updates.event-geo-live-viewed", + "updates.event-group-call", + "updates.event-history-ttl", + "updates.event-join-chat-webview-decision", + "updates.event-join-requests", + "updates.event-login-token", + "updates.event-managed-bot", + "updates.event-message-deleted", + "updates.event-message-edited", + "updates.event-new-authorization", + "updates.event-new-bot-connection", + "updates.event-new-message", + "updates.event-notify-settings", + "updates.event-peer-blocked", + "updates.event-peer-located", + "updates.event-peer-wallpaper", + "updates.event-phone-call", + "updates.event-pinned-messages", + "updates.event-poll", + "updates.event-pts-changed", + "updates.event-quick-replies", + "updates.event-read-contents", + "updates.event-read-discussion", + "updates.event-read-monoforum", + "updates.event-read-outbox", + "updates.event-report-message-delivery", + "updates.event-saved-dialogs", + "updates.event-saved-gifs", + "updates.event-scheduled-deleted", + "updates.event-scheduled-new", + "updates.event-service-message", + "updates.event-service-notification", + "updates.event-stars-balance", + "updates.event-stars-revenue", + "updates.event-stories-stealth", + "updates.event-story-id", + "updates.event-story-reaction", + "updates.event-story-read", + "updates.event-typing", + "updates.event-user-emoji-status", + "updates.event-user-phone", + "updates.event-user-refetch", + "updates.event-view-forum-as-messages", + "updates.event-web-browser-settings", + "updates.event-webview-result-sent", + "updates.stream-daemon-multi-account", + "updates.stream-event-filtering", + "updates.stream-resume-cursor", + "updates.sync-channel-short-poll", + "updates.sync-difference-too-long", + "updates.sync-dispatch-ordering", + "updates.sync-duplicate-suppression", + "updates.sync-peer-cache-from-updates", + "updates.sync-too-long", + "updates.sync-updating-indicator", + ), + coverage_note=( + "delivers every type; the catalogue half (what exists, what it means) " + "is `events list`/`events get`, and gap recovery is the `sync` group." + ), + tags=frozenset({"agent-safe", "frames", "live-stream"}), +) + + +# --------------------------------------------------------------------------- +# events replay +# --------------------------------------------------------------------------- + + +class EventReplayReq(Request): + since: Annotated[ + int | None, + opt("--since", metavar="SEQ", help="First seq (exclusive). Default: the whole buffer."), + ] = None + until: Annotated[ + int | None, opt("--until", metavar="SEQ", help="Stop at this seq (inclusive).") + ] = None + events: Annotated[str, opt("--events", metavar="TYPES", help="Filter the replay.")] = "all" + exclude: Annotated[ + str | None, opt("--exclude", metavar="TYPES", help="Subtract these types.") + ] = None + chat: Annotated[ + list[PeerRef], opt("--chat", metavar="CHAT", kind="peer", help="Only this chat.") + ] = [] + webhook: Annotated[ + bool, + opt("--webhook", help="Re-deliver the range to the configured webhook, not to stdout."), + ] = False + difference: Annotated[ + bool, + opt( + "--difference", + help="Rebuild a range older than the buffer with updates.getDifference.", + ), + ] = False + + +async def event_replay(ctx: OpContext, req: EventReplayReq) -> AsyncIterator[Page[EventEnvelope]]: + """Read the ring buffer without following it. + + The honest failure is the point. Asking for events after 91,820 when the + buffer starts at 95,000 does not return the newest page as though it were + the next one; it is INDETERMINATE with the oldest seq it does hold, so a + consumer knows it has a hole rather than believing it caught up. + """ + bus = _bus(ctx) + selection = await _selection( + ctx, + events=req.events, + exclude=req.exclude, + chats=tuple(req.chat), + senders=(), + topic=None, + ) + limit, state = _window(ctx, "events.replay", default=1000) + offset = int(state.get("offset", 0)) + + collected: list[EventEnvelope] = [] + for alias in _accounts(ctx) or [ctx.account]: + events, gap = bus.replay(alias, req.since if req.since is not None else 0) + if gap is not None and not req.difference: + raise IndeterminateError( + f"seq {req.since} is older than the buffer, which starts at " + f"{gap['from']}; {gap['lost']} events are not recoverable from " + "memory. Re-run with --difference to rebuild from Telegram, or " + "start from that seq." + ) + if gap is not None: + ctx.warn( + f"{gap['lost']} events before seq {gap['from']} were rebuilt from " + "updates.getDifference and may be incomplete" + ) + await _difference_backfill(ctx, alias) + collected.extend( + event + for event in events + if selection.wants(event) and (req.until is None or event.seq <= req.until) + ) + + collected.sort(key=lambda event: (event.account, event.seq)) + if req.webhook: + pushed = _push_to_webhook(ctx, collected) + ctx.warn(f"{pushed} events were re-queued for the webhook instead of printed") + collected = [] + + window = collected[offset : offset + limit] + yield build_page( + window, + op="events.replay", + kind=PageKind.LOCAL, + state={"offset": offset + len(window)}, + account=ctx.account, + has_more=offset + len(window) < len(collected), + total=len(collected), + ) + + +async def _difference_backfill(ctx: OpContext, alias: str) -> None: + """Ask the session to catch up so the gap is at least *narrowed*.""" + daemon = getattr(ctx, "daemon", None) + sessions = getattr(daemon, "sessions", None) + session = sessions.get(alias) if sessions is not None else None + if session is not None: + await session.catch_up() + + +def _push_to_webhook(ctx: OpContext, events: list[EventEnvelope]) -> int: + daemon = getattr(ctx, "daemon", None) + webhook = getattr(daemon, "webhook", None) + if webhook is None: + raise UsageError("no webhook is configured; run `tlgr webhook set --url …`") + for event in events: + webhook.enqueue(event) + return len(events) + + +SPEC_EVENT_REPLAY = OperationSpec( + id="events.replay", + request=EventReplayReq, + response=Page[EventEnvelope], + impl=event_replay, + summary="Replay buffered events from the daemon's ring buffer without following", + description=( + "Exit 3 when the range is inside the buffer and empty; exit 13, with " + "the oldest seq the daemon still holds, when `--since` predates it. " + "Returning the newest page instead would be a silent lie about having " + "caught up." + ), + stream=True, + paginated=PageKind.LOCAL, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=120, + columns=("seq", "ts", "type", "chat_id"), + empty_exit=EXIT_EMPTY, + example={ + "items": [_EXAMPLE_ENVELOPE], + "has_more": False, + "total": 1, + }, + example_args="events replay --since 91820 --events message_new", + covers=("updates.stream-resume-cursor",), + covers_partial=("updates.stream-watch-ndjson", "updates.sync-duplicate-suppression"), + coverage_note=( + "replays a range; following it live is `watch`, and de-duplication is " + "the consumer's job through the envelope's stable seq." + ), + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/ops/export.py b/tlgr/ops/export.py new file mode 100644 index 0000000..d027b28 --- /dev/null +++ b/tlgr/ops/export.py @@ -0,0 +1,596 @@ +"""The `export` group: Telegram's data export (takeout). + +A takeout is a *mode*, not a request. `account.initTakeoutSession` returns an +id, and every subsequent call — `upload.getFile` included — has to be wrapped +in `invokeWithTakeout` or it simply is not part of the export. `file_max_size` +is fixed at that moment and cannot be raised later. + +Two consequences shape this group. The session id is held by the daemon, in +memory, for the account it was opened on, so `export start` and `export +message download` are separate commands rather than one long-running call an +interrupted terminal would abandon. And `TAKEOUT_INIT_DELAY_X` — another +logged-in session has to approve the export, or 24 hours must pass if there is +none — is reported as a structured error with `retry_after` rather than slept +through in silence. +""" + +from __future__ import annotations + +import contextlib +import json +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Annotated, Any + +from tlgr.core.errors import EXIT_EMPTY, NotFoundError, RateLimitError, UsageError +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.export import ( + ExportedFile, + ExportResult, + MessageRange, + TakeoutSession, + TakeoutStatus, +) +from tlgr.models.message import Message +from tlgr.models.page import Page +from tlgr.models.peer import PeerRef +from tlgr.ops import _send +from tlgr.ops._params import opt, parse_dt +from tlgr.ops._serialize import message_to_model +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + +_SCOPES = ("contacts", "messages", "users", "chats", "megagroups", "channels", "bots", "files") + + +def _client(ctx: OpContext) -> Any: + client = getattr(ctx, "client", None) + if client is None: + raise UsageError("this operation needs a connected account") + return client + + +def _sessions(ctx: OpContext) -> Any: + daemon = getattr(ctx, "daemon", None) + return getattr(daemon, "sessions", None) + + +def _store(ctx: OpContext) -> dict[str, dict[str, Any]]: + """Open takeout sessions, per account, on the daemon. + + In memory rather than on disk, deliberately: a takeout id is only valid + for the connection that opened it, so persisting one across a restart + would hand back an id every subsequent request would be rejected with. + """ + daemon = getattr(ctx, "daemon", None) + if daemon is None: + raise UsageError("this operation runs inside the daemon") + existing = getattr(daemon, "takeouts", None) + if existing is None: + existing = {} + daemon.takeouts = existing + return existing + + +def _active(ctx: OpContext) -> dict[str, Any]: + entry = _store(ctx).get(ctx.account) + if not entry: + raise NotFoundError("no takeout session is open for this account. Run: tlgr export start") + return entry + + +async def _invoke(ctx: OpContext, request: Any) -> Any: + """Run *request* inside the account's takeout session when one is open. + + Outside a session the call still works — it simply runs against the normal + flood budget instead of the takeout one — which is why `export account + download` is useful before `export start` has been approved. + """ + from telethon.tl import functions + + client = _client(ctx) + entry = _store(ctx).get(ctx.account) + if not entry: + return await client(request) + return await client( + functions.InvokeWithTakeoutRequest(takeout_id=entry["takeout_id"], query=request) + ) + + +# --------------------------------------------------------------------------- +# export start / status / end +# --------------------------------------------------------------------------- + + +class ExportStartReq(Request): + contacts: Annotated[bool, opt("--contacts", help="Include contacts.")] = False + messages: Annotated[bool, opt("--messages", help="Include private-chat history.")] = False + users: Annotated[bool, opt("--users", help="Include private chats.")] = False + chats: Annotated[bool, opt("--chats", help="Include basic groups.")] = False + megagroups: Annotated[bool, opt("--megagroups", help="Include supergroups.")] = False + channels: Annotated[bool, opt("--channels", help="Include channels.")] = False + bots: Annotated[bool, opt("--bots", help="Include bot chats.")] = False + files: Annotated[bool, opt("--files", help="Include media files.")] = False + max_file_size: Annotated[ + int, + opt( + "--max-file-size", + metavar="BYTES", + help="file_max_size, declared up front and unchangeable afterwards.", + ), + ] = 100 * 1024 * 1024 + wait: Annotated[ + bool, opt("--wait", help="Report TAKEOUT_INIT_DELAY as a wait instead of failing.") + ] = False + + +async def export_start(ctx: OpContext, req: ExportStartReq) -> TakeoutSession: + """Open a takeout session. + + `file_max_size` cannot be changed later, so it is declared here. A + `TAKEOUT_INIT_DELAY_X` means another logged-in session has to approve the + export first — 24 hours if there is none — and it comes back as + RATE_LIMITED carrying `retry_after`, because "try again later" without + "how much later" is not actionable. + """ + from telethon.tl import functions + + store = _store(ctx) + if ctx.account in store: + entry = store[ctx.account] + ctx.mark_already() + return TakeoutSession( + takeout_id=entry["takeout_id"], + scope=entry["scope"], + started_at=entry["started_at"], + max_file_size=entry["max_file_size"], + already=True, + ) + + scope = [name for name in _SCOPES if getattr(req, name, False)] + if not scope: + raise UsageError( + "name at least one scope: --messages, --contacts, --channels, --files, …", + field="messages", + ) + + request = functions.account.InitTakeoutSessionRequest( + contacts=req.contacts, + message_users=req.users or req.messages, + message_chats=req.chats, + message_megagroups=req.megagroups, + message_channels=req.channels, + files=req.files, + file_max_size=req.max_file_size if req.files else None, + ) + try: + result = await _client(ctx)(request) + except Exception as exc: + raise _takeout_delay(exc, waiting=req.wait) from exc + + entry = { + "takeout_id": int(getattr(result, "id", 0) or 0), + "scope": scope, + "started_at": _now(), + "max_file_size": req.max_file_size, + } + _store(ctx)[ctx.account] = entry + return TakeoutSession( + takeout_id=entry["takeout_id"], + scope=scope, + started_at=entry["started_at"], + max_file_size=entry["max_file_size"], + ) + + +def _takeout_delay(exc: Exception, *, waiting: bool) -> Exception: + seconds = getattr(exc, "seconds", None) + if type(exc).__name__ != "TakeoutInitDelayError" and seconds is None: + return exc + hint = ( + "another logged-in session has to approve this export (or 24 hours must " + "pass if there is none). Approve it in Settings → Privacy → Data export." + ) + if waiting: + hint += f" Retry in {seconds}s." + return RateLimitError(f"the export cannot start yet: {hint}", wait_seconds=int(seconds or 0)) + + +def _now() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +SPEC_EXPORT_START = OperationSpec( + id="export.start", + request=ExportStartReq, + response=TakeoutSession, + impl=export_start, + summary="Open a Telegram data-export (takeout) session", + description=( + "The returned id wraps every subsequent request in `invokeWithTakeout`, " + "`upload.getFile` included. `file_max_size` cannot be changed later." + ), + aliases=("daemon.takeout.start",), + mutating=True, + idempotent=True, + surface=Surface.DAEMON, + rate_class="read", + timeout_s=120, + columns=("takeout_id", "scope", "started_at", "max_file_size"), + example={"takeout_id": 1234567890, "scope": ["messages", "files"], "max_file_size": 104857600}, + example_args="export start --messages --files", + covers_partial=( + "takeout.contacts", + "takeout.files", + "takeout.messages", + "takeout.personal-info", + "updates.takeout-session", + ), + coverage_note=( + "opens the session the other export commands run inside; each of them " + "owns the data it fetches." + ), + tags=frozenset({"agent-safe"}), +) + + +class ExportStatusReq(Request): + ranges: Annotated[ + bool, opt("--ranges/--no-ranges", help="Include messages.getSplitRanges output.") + ] = True + + +async def export_status(ctx: OpContext, req: ExportStatusReq) -> TakeoutStatus: + """The active takeout session and the message ranges it must be walked in. + + Split ranges are not advice. For private chats and basic groups the export + has to call `messages.getSplitRanges` and then wrap each range in + `invokeWithMessagesRange`, restarting pagination per range — Telethon does + none of that, so tlgr wraps it by hand. + """ + entry = _store(ctx).get(ctx.account) + if not entry: + return TakeoutStatus(active=False) + + status = TakeoutStatus( + active=True, + takeout_id=entry["takeout_id"], + started_at=entry["started_at"], + scope=entry["scope"], + max_file_size=entry["max_file_size"], + ) + if req.ranges: + from telethon.tl import functions + + with contextlib.suppress(Exception): + reply = await _invoke(ctx, functions.messages.GetSplitRangesRequest()) + status.ranges = [ + MessageRange( + min_id=int(getattr(row, "min_id", 0) or 0), + max_id=int(getattr(row, "max_id", 0) or 0), + ) + for row in (reply or []) + ] + return status + + +SPEC_EXPORT_STATUS = OperationSpec( + id="export.status", + request=ExportStatusReq, + response=TakeoutStatus, + impl=export_status, + summary="Show the active takeout session and its message ranges", + aliases=("daemon.takeout.status",), + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=60, + columns=("active", "takeout_id", "scope", "max_file_size"), + example={"active": True, "takeout_id": 1234567890, "scope": ["messages"]}, + example_args="export status", + covers=("updates.takeout-session", "updates.takeout-split-ranges"), + tags=frozenset({"agent-safe"}), +) + + +class ExportEndReq(Request): + failed: Annotated[ + bool, opt("--failed", help="Report the export as unsuccessful (success=false).") + ] = False + + +async def export_end(ctx: OpContext, req: ExportEndReq) -> ExportResult: + """Close the takeout session. + + Must be called: an open session blocks the next export, and the next + `export start` then fails with a delay that looks like Telegram refusing + rather than like a session nobody closed. + """ + from telethon.tl import functions + + store = _store(ctx) + entry = store.get(ctx.account) + if not entry: + ctx.mark_already() + return ExportResult(finished=False, success=not req.failed) + + with contextlib.suppress(Exception): + await _invoke(ctx, functions.account.FinishTakeoutSessionRequest(success=not req.failed)) + store.pop(ctx.account, None) + return ExportResult(finished=True, takeout_id=entry["takeout_id"], success=not req.failed) + + +SPEC_EXPORT_END = OperationSpec( + id="export.end", + request=ExportEndReq, + response=ExportResult, + impl=export_end, + summary="Close the takeout session", + description="An open session blocks the next export, so this is not optional.", + aliases=("export.finish", "daemon.takeout.finish"), + mutating=True, + idempotent=True, + surface=Surface.DAEMON, + rate_class="read", + timeout_s=60, + example={"finished": True, "takeout_id": 1234567890, "success": True}, + example_args="export end", + covers_partial=("takeout.messages", "updates.takeout-session"), + coverage_note="closes the session; the data comes from the download commands.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# export account download +# --------------------------------------------------------------------------- + + +class ExportAccountReq(Request): + out: Annotated[str, opt("--out", metavar="DIR", kind="path", help="Output directory.")] = ( + "./telegram-export" + ) + photos: Annotated[bool, opt("--photos", help="Profile photos.")] = False + sessions: Annotated[bool, opt("--sessions", help="Sessions and websites.")] = False + stories: Annotated[bool, opt("--stories", help="Story archive.")] = False + contacts: Annotated[bool, opt("--contacts", help="Contacts and top peers.")] = False + left_channels: Annotated[bool, opt("--left-channels", help="Channels I left.")] = False + everything: Annotated[bool, opt("--everything", help="All of the above.")] = True + + +async def export_account_download(ctx: OpContext, req: ExportAccountReq) -> ExportResult: + """Export the personal information a takeout covers. + + Every call is wrapped in `invokeWithTakeout` when a session is open, and + runs normally when one is not — which is deliberately useful: the + personal-info half needs no approval delay, so it works while the message + export is still waiting for one. + """ + from telethon.tl import functions + + out = Path(req.out).expanduser() + out.mkdir(parents=True, exist_ok=True) + want = _wanted(req) + written: list[ExportedFile] = [] + skipped: list[str] = [] + + jobs: list[tuple[str, str, Any]] = [ + ("profile", "profile.json", functions.users.GetFullUserRequest(id="me")), + ( + "photos", + "photos.json", + functions.photos.GetUserPhotosRequest(user_id="me", offset=0, max_id=0, limit=100), + ), + ("sessions", "sessions.json", functions.account.GetAuthorizationsRequest()), + ("sessions", "websites.json", functions.account.GetWebAuthorizationsRequest()), + ("contacts", "contacts.json", functions.contacts.GetSavedRequest()), + ( + "left_channels", + "left_channels.json", + functions.channels.GetLeftChannelsRequest(offset=0), + ), + ] + + for name, filename, request in jobs: + if name not in want: + continue + if name == "profile" and "photos" not in want and not req.everything: + continue + try: + if name == "profile": + request = functions.users.GetFullUserRequest(id=await _self(ctx)) + elif name == "photos": + request = functions.photos.GetUserPhotosRequest( + user_id=await _self(ctx), offset=0, max_id=0, limit=100 + ) + reply = await _invoke(ctx, request) + except Exception as exc: + skipped.append(f"{filename}: {type(exc).__name__}: {exc}") + continue + written.append(_dump(out / filename, reply)) + + if "stories" in want: + with contextlib.suppress(Exception): + reply = await _invoke( + ctx, + functions.stories.GetStoriesArchiveRequest( + peer=await _self(ctx), offset_id=0, limit=100 + ), + ) + written.append(_dump(out / "stories.json", reply)) + + for note in skipped: + ctx.warn(note) + return ExportResult(written=len(written), files=written, out=str(out), skipped=skipped) + + +def _wanted(req: ExportAccountReq) -> set[str]: + chosen = { + name + for name in ("photos", "sessions", "stories", "contacts", "left_channels") + if getattr(req, name, False) + } + if chosen and not any( + getattr(req, name) + for name in ("photos", "sessions", "stories", "contacts", "left_channels") + ): + chosen = set() + if not chosen: + chosen = {"photos", "sessions", "stories", "contacts", "left_channels"} + chosen.add("profile") + return chosen + + +async def _self(ctx: OpContext) -> Any: + from telethon.tl import types + + return types.InputUserSelf() + + +def _dump(path: Path, value: Any) -> ExportedFile: + from tlgr.core.tl import tl_to_builtins + + body = json.dumps(tl_to_builtins(value), ensure_ascii=False, indent=2) + path.write_text(body, encoding="utf-8") + return ExportedFile(path=str(path), kind=path.stem, bytes=len(body.encode("utf-8"))) + + +SPEC_EXPORT_ACCOUNT = OperationSpec( + id="export.account.download", + request=ExportAccountReq, + response=ExportResult, + impl=export_account_download, + summary="Export personal info: profile, photos, sessions, stories, contacts, left channels", + description=( + "Runs inside the takeout session when one is open, and normally when " + "it is not — so the personal-info half works while a message export " + "is still waiting for approval." + ), + surface=Surface.DAEMON, + rate_class="bulk", + timeout_s=900, + columns=("written", "out"), + example={"written": 5, "out": "./telegram-export"}, + example_args="export account download --out ./export", + covers=("takeout.contacts", "takeout.personal-info", "updates.takeout-export-run"), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# export message download +# --------------------------------------------------------------------------- + + +class ExportMessagesReq(Request): + out: Annotated[str, opt("--out", metavar="DIR", kind="path", help="Output directory.")] = ( + "./telegram-export" + ) + chat: Annotated[ + list[PeerRef], + opt("--chat", metavar="CHAT", kind="peer", help="Only these chats (repeatable)."), + ] = [] + since: Annotated[ + str | None, opt("--since", metavar="WHEN", kind="datetime", help="Only after this.") + ] = None + until: Annotated[ + str | None, opt("--until", metavar="WHEN", kind="datetime", help="Only before this.") + ] = None + files: Annotated[bool, opt("--files", help="Also download media.")] = False + per_chat: Annotated[ + int, opt("--per-chat", metavar="N", ge=1, le=100000, help="Messages per chat.") + ] = 1000 + + +async def export_message_download( + ctx: OpContext, req: ExportMessagesReq +) -> AsyncIterator[Page[Message]]: + """Export chat history inside the takeout session, as NDJSON on disk. + + Checkpointed per chat, because a takeout still meets FLOOD_WAIT — a + generous budget is not an absent one — and an export that has to restart + from message zero after four hours is an export nobody finishes. + """ + _active(ctx) # refuse to export outside a session: it would not be a takeout + out = Path(req.out).expanduser() + out.mkdir(parents=True, exist_ok=True) + client = _client(ctx) + since = parse_dt(req.since) if req.since else None + until = parse_dt(req.until) if req.until else None + + if not req.chat: + raise UsageError( + "name the chats to export with --chat; exporting every dialog is a " + "different, much longer operation and should be asked for explicitly", + field="chat", + ) + + for ref in req.chat: + peer = await _send.resolve(ctx, ref) + chat_id = _send.peer_id_of(peer) + path = out / f"chat_{chat_id}.jsonl" + rows: list[Message] = [] + with path.open("a", encoding="utf-8") as handle: + async for message in client.iter_messages(peer, limit=req.per_chat, offset_date=until): + stamp = getattr(message, "date", None) + if since is not None and stamp is not None and stamp < since: + break + model = message_to_model(message, chat_id=chat_id) + rows.append(model) + from tlgr.models.base import to_builtins + + handle.write(json.dumps(to_builtins(model), ensure_ascii=False) + "\n") + if req.files: + ctx.warn( + "--files is recorded but media are not downloaded here; use " + "`tlgr media download` per message, which shares the takeout session" + ) + yield build_page( + rows, + op="export.message.download", + kind=PageKind.HISTORY, + state={"chat_id": chat_id}, + account=ctx.account, + has_more=False, + total=len(rows), + ) + + +SPEC_EXPORT_MESSAGES = OperationSpec( + id="export.message.download", + request=ExportMessagesReq, + response=Page[Message], + impl=export_message_download, + summary="Export chat history inside the takeout session", + description=( + "One NDJSON file per chat, appended as it goes: a takeout still meets " + "FLOOD_WAIT, and an export that restarts from zero after four hours " + "is one nobody finishes." + ), + stream=True, + paginated=PageKind.HISTORY, + surface=Surface.DAEMON, + rate_class="bulk", + timeout_s=900, + columns=("id", "date", "text"), + empty_exit=EXIT_EMPTY, + example={ + "items": [ + { + "id": 12345, + "chat_id": 777123, + "date": "2026-09-03T09:14:07Z", + "date_unix": 1788340447, + } + ], + "has_more": False, + }, + example_args="export message download --chat @alice --out ./export", + covers=("takeout.files", "takeout.messages"), + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/ops/job.py b/tlgr/ops/job.py new file mode 100644 index 0000000..bdbde83 --- /dev/null +++ b/tlgr/ops/job.py @@ -0,0 +1,799 @@ +"""The `job` group: the gateway's rules, as data rather than as an editor. + +v1's `job add` opened `$EDITOR` on `jobs.yaml`. That is a perfectly good way +for a person to write a rule and a completely useless one for an agent, which +is the caller this whole product is for — so the flag form is the primary path +and `--edit` keeps the old behaviour. + +The other change is what a job can hear. v1 jobs only ever saw `NewMessage`, +because the engine registered Telethon's high-level handlers directly. A job +now declares `events:` from the same taxonomy `watch` and `webhook set` use, +and subscribes to the same bus — so "which events exist" has one answer across +the three places that ask. +""" + +from __future__ import annotations + +import contextlib +import json +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Annotated, Any + +from tlgr.core import eventtypes +from tlgr.core.errors import EXIT_EMPTY, NotFoundError, UsageError +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.daemon import Job, JobState, JobTestFrame +from tlgr.models.page import Page +from tlgr.models.peer import PeerRef +from tlgr.ops._params import arg, opt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + + +# --------------------------------------------------------------------------- +# jobs.yaml +# --------------------------------------------------------------------------- + + +def _jobs_path() -> Path: + from tlgr.core.paths import TlgrPaths + + return TlgrPaths().jobs + + +def _load_raw() -> dict[str, Any]: + """Read `jobs.yaml` as plain data, so a rewrite keeps what it did not touch. + + Round-tripping through `GatewayConfig` would silently drop every filter and + processor the parser does not model, which is how an "add a job" command + quietly deletes the four already there. + """ + import yaml + + path = _jobs_path() + if not path.exists(): + return {"jobs": []} + try: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + raise UsageError(f"{path} is not valid YAML: {exc}") from exc + if not isinstance(loaded, dict): + raise UsageError(f"{path} must be a mapping with a `jobs:` list") + loaded.setdefault("jobs", []) + if not isinstance(loaded["jobs"], list): + raise UsageError(f"{path}: `jobs` must be a list") + return loaded + + +def _save_raw(document: dict[str, Any]) -> None: + import yaml + + from tlgr.core.paths import write_private + + write_private( + _jobs_path(), + yaml.safe_dump(document, default_flow_style=False, sort_keys=False, allow_unicode=True), + ) + + +def _find(document: dict[str, Any], name: str) -> dict[str, Any]: + for entry in document["jobs"]: + if isinstance(entry, dict) and entry.get("name") == name: + return entry + raise NotFoundError(f"no job named {name!r}. Run: tlgr job list") + + +def _runner(ctx: OpContext) -> Any: + daemon = getattr(ctx, "daemon", None) + if daemon is None: + raise UsageError("this operation runs inside the daemon") + return daemon + + +# --------------------------------------------------------------------------- +# job list / get +# --------------------------------------------------------------------------- + + +class JobListReq(Request): + enabled_only: Annotated[bool, opt("--enabled-only", help="Hide disabled jobs.")] = False + + +async def job_list(ctx: OpContext, req: JobListReq) -> Page[JobState]: + """Every configured job, with what the running engine has done with it.""" + daemon = _runner(ctx) + live = {row.get("name"): row for row in daemon.list_jobs()} + document = _load_raw() + rows: list[JobState] = [] + for entry in document["jobs"]: + if not isinstance(entry, dict): + continue + name = str(entry.get("name", "")) + running = live.get(name, {}) + enabled = bool(entry.get("enabled", True)) + if req.enabled_only and not enabled: + continue + account = str(entry.get("account", "")) + if ctx.account and ctx.account != "all" and account and account != ctx.account: + continue + rows.append( + JobState( + name=name, + account=account, + enabled=enabled, + running=bool(running.get("running")), + events=[str(e) for e in (entry.get("events") or ["new_message"])], + matched=int(running.get("matched") or 0), + skipped=int(running.get("skipped") or 0), + errors=int(running.get("errors") or 0), + ) + ) + return build_page( + rows, + op="job.list", + kind=PageKind.LOCAL, + has_more=False, + total=len(rows), + ) + + +SPEC_JOB_LIST = OperationSpec( + id="job.list", + request=JobListReq, + response=Page[JobState], + impl=job_list, + summary="List gateway jobs and their state", + description=( + "Configured *and* running are different facts: a job can be enabled " + "in `jobs.yaml` and not running because its account will not connect." + ), + legacy_paths=("job list",), + paginated=PageKind.LOCAL, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("name", "account", "enabled", "running", "matched", "errors"), + example={ + "items": [ + { + "name": "archive", + "account": "work", + "enabled": True, + "running": True, + "events": ["new_message"], + } + ], + "has_more": False, + }, + example_args="job list", + covers_partial=("updates.stream-event-filtering",), + coverage_note="lists the rules; proving one fires is `job test`.", + tags=frozenset({"agent-safe"}), +) + + +class JobGetReq(Request): + name: Annotated[str, arg(0, metavar="NAME")] + explain: Annotated[ + bool, opt("--explain", help="Annotate each filter with the registry entry it resolves to.") + ] = False + + +async def job_get(ctx: OpContext, req: JobGetReq) -> JobState: + """One job's resolved pipeline: filters, processors, actions.""" + entry = _find(_load_raw(), req.name) + state = JobState( + name=req.name, + account=str(entry.get("account", "")), + enabled=bool(entry.get("enabled", True)), + events=[str(e) for e in (entry.get("events") or ["new_message"])], + filters=entry.get("filters") or {}, + processors=[str(p) for p in (entry.get("processors") or [])], + actions=[a for a in (entry.get("actions") or []) if isinstance(a, dict)], + ) + if req.explain: + state.filters = { + key: {"value": value, "resolves_to": _explain_filter(key)} + for key, value in (state.filters or {}).items() + } + return state + + +def _explain_filter(name: str) -> str: + """Which filter implementation a key resolves to, or that it resolves to none. + + "This job never fires" is almost always a filter name nobody registered, + and the pipeline's silence about it is the reason it takes an hour to find. + """ + with contextlib.suppress(Exception): + from tlgr.filters import get_filter + + found = get_filter(name) + if found is not None: + return getattr(found, "__name__", str(found)) + return "UNKNOWN — no filter is registered under this name; the job will never match" + + +SPEC_JOB_GET = OperationSpec( + id="job.get", + request=JobGetReq, + response=JobState, + impl=job_get, + summary="Show one job's resolved pipeline (filters, processors, actions)", + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + example={ + "name": "archive", + "account": "work", + "enabled": True, + "events": ["new_message"], + "actions": [{"forward": {"to": "@archive"}}], + }, + example_args="job get archive --explain", + covers_partial=("updates.stream-event-filtering",), + coverage_note="shows the rule; evaluating it against real events is `job test`.", + empty_exit=EXIT_EMPTY, + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# job add +# --------------------------------------------------------------------------- + + +def _pairs(values: list[str], what: str) -> dict[str, Any]: + out: dict[str, Any] = {} + for value in values: + key, sep, raw = value.partition("=") + if not sep: + raise UsageError(f"--{what} wants key=value, got {value!r}", field=what) + out[key.strip()] = _coerce(raw.strip()) + return out + + +def _coerce(raw: str) -> Any: + lowered = raw.lower() + if lowered in ("true", "yes"): + return True + if lowered in ("false", "no"): + return False + with contextlib.suppress(ValueError): + return int(raw) + if raw.startswith(("[", "{")): + with contextlib.suppress(json.JSONDecodeError): + return json.loads(raw) + return raw + + +def _action(spec: str) -> dict[str, Any]: + """`reply:hello` or `forward:to=@archive` → one action entry.""" + name, sep, rest = spec.partition(":") + name = name.strip() + if not name: + raise UsageError(f"--action wants NAME[:CONFIG], got {spec!r}", field="action") + if not sep or not rest: + return {name: {}} + if "=" in rest: + return {name: _pairs([part for part in rest.split(",") if part], "action")} + return {name: rest} + + +class JobAddReq(Request): + name: Annotated[str | None, opt("--name", metavar="NAME", help="Job name.")] = None + from_file: Annotated[ + str | None, + opt("--from-file", metavar="PATH", help="Read one job (or a jobs list) from YAML/JSON."), + ] = None + job_account: Annotated[ + str | None, opt("--for-account", metavar="ALIAS", help="Account the job runs on.") + ] = None + events: Annotated[ + str, opt("--events", metavar="TYPES", help="Event types the job subscribes to.") + ] = "new_message" + filter: Annotated[ + list[str], opt("--filter", metavar="KEY=VALUE", help="Filter entry (repeatable).") + ] = [] + action: Annotated[ + list[str], + opt("--action", metavar="SPEC", help="Action entry, e.g. 'reply:hello' (repeatable)."), + ] = [] + processor: Annotated[ + list[str], opt("--processor", metavar="NAME", help="Processor entry (repeatable).") + ] = [] + enabled: Annotated[bool, opt("--enabled/--disabled", help="Initial state.")] = True + edit: Annotated[ + bool, opt("--edit", help="Open jobs.yaml in $EDITOR instead (the v1 behaviour).") + ] = False + + +async def job_add(ctx: OpContext, req: JobAddReq) -> Job: + """Add a job from flags, a file, or an editor. + + The event names are validated here rather than at load time: `jobs.yaml` + parsing drops an event it does not recognise, so a typo used to produce a + job that simply never fired and never said why. + """ + if req.edit: + _open_editor() + return Job(name=req.name or "", enabled=True, already=True) + + document = _load_raw() + entries = _entries_from(req) + added: list[str] = [] + for entry in entries: + name = str(entry.get("name", "")) + if not name: + raise UsageError("a job needs a name (--name, or `name:` in the file)", field="name") + if any(isinstance(e, dict) and e.get("name") == name for e in document["jobs"]): + raise UsageError(f"a job named {name!r} already exists; remove it first", field="name") + eventtypes.resolve_selectors(entry.get("events") or ["new_message"]) + document["jobs"].append(entry) + added.append(name) + + _save_raw(document) + first = entries[0] + return Job( + name=str(first.get("name", "")), + account=str(first.get("account", "")), + enabled=bool(first.get("enabled", True)), + events=[str(e) for e in (first.get("events") or [])], + added=added, + ) + + +def _entries_from(req: JobAddReq) -> list[dict[str, Any]]: + if req.from_file: + return _entries_from_file(req.from_file) + entry: dict[str, Any] = { + "name": req.name, + "events": [e.strip() for e in req.events.split(",") if e.strip()], + "enabled": req.enabled, + } + if req.job_account: + entry["account"] = req.job_account + if req.filter: + entry["filters"] = _pairs(req.filter, "filter") + if req.processor: + entry["processors"] = list(req.processor) + if req.action: + entry["actions"] = [_action(spec) for spec in req.action] + if not entry.get("actions"): + raise UsageError("a job with no actions would do nothing; pass --action", field="action") + return [entry] + + +def _entries_from_file(source: str) -> list[dict[str, Any]]: + import sys + + import yaml + + if source == "-": + if sys.stdin is None or sys.stdin.isatty(): + raise UsageError("--from-file - was given but stdin is a terminal", field="from_file") + text = sys.stdin.read() + else: + try: + text = Path(source).read_text(encoding="utf-8") + except OSError as exc: + raise UsageError(f"{source}: {exc.strerror or exc}", field="from_file") from exc + try: + loaded = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise UsageError(f"{source} is not valid YAML or JSON: {exc}", field="from_file") from exc + if isinstance(loaded, dict) and isinstance(loaded.get("jobs"), list): + return [entry for entry in loaded["jobs"] if isinstance(entry, dict)] + if isinstance(loaded, dict): + return [loaded] + if isinstance(loaded, list): + return [entry for entry in loaded if isinstance(entry, dict)] + raise UsageError(f"{source} does not contain a job", field="from_file") + + +def _open_editor() -> None: + path = _jobs_path() + if not path.exists(): + from tlgr.core.paths import write_private + + write_private( + path, + "# Gateway jobs. See `tlgr job add --help` for the non-interactive form.\njobs: []\n", + ) + os.execlp(os.environ.get("EDITOR", "vi"), os.environ.get("EDITOR", "vi"), str(path)) + + +SPEC_JOB_ADD = OperationSpec( + id="job.add", + request=JobAddReq, + response=Job, + impl=job_add, + summary="Add a gateway job", + description=( + "v1 only opened `$EDITOR`, which no agent can drive. The flags are " + "the agent path, `--from-file -` takes YAML or JSON on stdin, and " + "`--edit` keeps the old behaviour." + ), + legacy_paths=("job add",), + mutating=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=30, + example={"name": "archive", "account": "work", "enabled": True, "added": ["archive"]}, + example_args="job add --name archive --action 'forward:to=@archive'", + covers_partial=("updates.stream-event-filtering",), + coverage_note="writes the rule; the filter vocabulary belongs to the gateway.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# enable / disable / remove / reload +# --------------------------------------------------------------------------- + + +async def _set_enabled(ctx: OpContext, name: str, enabled: bool) -> Job: + document = _load_raw() + entry = _find(document, name) + if bool(entry.get("enabled", True)) == enabled: + ctx.mark_already() + return Job(name=name, enabled=enabled, already=True) + entry["enabled"] = enabled + _save_raw(document) + daemon = getattr(ctx, "daemon", None) + if daemon is not None: + with contextlib.suppress(Exception): + await (daemon.enable_job(name) if enabled else daemon.disable_job(name)) + return Job(name=name, enabled=enabled, reloaded=daemon is not None) + + +class JobNameReq(Request): + name: Annotated[str, arg(0, metavar="NAME")] + + +async def job_enable(ctx: OpContext, req: JobNameReq) -> Job: + """Enable a disabled job, in `jobs.yaml` and in the running engine.""" + return await _set_enabled(ctx, req.name, True) + + +SPEC_JOB_ENABLE = OperationSpec( + id="job.enable", + request=JobNameReq, + response=Job, + impl=job_enable, + summary="Enable a disabled job", + legacy_paths=("job enable",), + mutating=True, + idempotent=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=30, + example={"name": "archive", "enabled": True}, + example_args="job enable archive", + covers_partial=("updates.stream-event-filtering",), + coverage_note="toggles a rule; the filtering itself is the gateway's.", + tags=frozenset({"agent-safe"}), +) + + +async def job_disable(ctx: OpContext, req: JobNameReq) -> Job: + """Disable a job without removing it.""" + return await _set_enabled(ctx, req.name, False) + + +SPEC_JOB_DISABLE = OperationSpec( + id="job.disable", + request=JobNameReq, + response=Job, + impl=job_disable, + summary="Disable a job without removing it", + legacy_paths=("job disable",), + mutating=True, + idempotent=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=30, + example={"name": "archive", "enabled": False}, + example_args="job disable archive", + covers_partial=("updates.stream-event-filtering",), + coverage_note="toggles a rule; the filtering itself is the gateway's.", + tags=frozenset({"agent-safe"}), +) + + +async def job_remove(ctx: OpContext, req: JobNameReq) -> Job: + """Remove a job from `jobs.yaml` and stop it.""" + document = _load_raw() + _find(document, req.name) + document["jobs"] = [ + entry + for entry in document["jobs"] + if not (isinstance(entry, dict) and entry.get("name") == req.name) + ] + _save_raw(document) + daemon = getattr(ctx, "daemon", None) + if daemon is not None: + with contextlib.suppress(Exception): + await daemon.remove_job(req.name) + return Job(name=req.name, enabled=False, removed=True) + + +SPEC_JOB_REMOVE = OperationSpec( + id="job.remove", + request=JobNameReq, + response=Job, + impl=job_remove, + summary="Remove a job", + legacy_paths=("job remove",), + mutating=True, + destructive=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=30, + example={"name": "archive", "enabled": False, "removed": True}, + example_args="job remove archive", + covers_partial=("updates.stream-event-filtering",), + coverage_note="deletes a rule; the filtering itself is the gateway's.", + tags=frozenset({"agent-safe"}), +) + + +class JobReloadReq(Request): + validate_only: Annotated[ + bool, opt("--validate-only", help="Parse and report without swapping the pipeline.") + ] = False + + +async def job_reload(ctx: OpContext, req: JobReloadReq) -> Job: + """Re-read `jobs.yaml` and swap the running pipeline. + + `--validate-only` parses and reports without swapping, which is the check + to run before a reload rather than after one: a config with a typo would + otherwise take effect as "that job is gone". + """ + from tlgr.gateway.config import load_gateway_configs + + daemon = _runner(ctx) + base = getattr(getattr(daemon, "paths", None), "base", None) + configs = load_gateway_configs(base) + problems: list[str] = [] + for config in configs: + if not config.name: + problems.append("a job has no `name`") + if not config.actions: + problems.append(f"job {config.name!r} has no actions and would do nothing") + for action in config.actions: + from tlgr.actions import get_action + + if get_action(action.name) is None: + problems.append(f"job {config.name!r} uses unknown action {action.name!r}") + + if req.validate_only or problems: + return Job(name="", enabled=True, loaded=len(configs), errors=problems) + + result = await daemon.reload_jobs() + return Job( + name="", + enabled=True, + reloaded=True, + loaded=len(configs), + added=sorted(result.get("added", [])), + removed_names=sorted(result.get("removed", [])), + changed=sorted(result.get("updated", [])), + ) + + +SPEC_JOB_RELOAD = OperationSpec( + id="job.reload", + request=JobReloadReq, + response=Job, + impl=job_reload, + summary="Hot-reload jobs.yaml without restarting the daemon", + legacy_paths=("job reload",), + mutating=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=60, + example={"name": "", "enabled": True, "reloaded": True, "loaded": 3, "added": ["archive"]}, + example_args="job reload --validate-only", + covers_partial=("updates.stream-webhook-delivery",), + coverage_note="reloads the consumers; delivery is the webhook pusher's.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# job test +# --------------------------------------------------------------------------- + + +class JobTestReq(Request): + name: Annotated[str, arg(0, metavar="NAME")] + event: Annotated[ + str | None, + opt("--event", metavar="TYPE", help="Synthesise an event of this type instead."), + ] = None + since: Annotated[ + int | None, opt("--since", metavar="SEQ", help="Replay buffered events from this seq.") + ] = None + chat: Annotated[ + PeerRef | None, opt("--chat", metavar="CHAT", kind="peer", help="Restrict the replay.") + ] = None + from_file: Annotated[ + str | None, opt("--from-file", metavar="PATH", help="Feed envelopes from NDJSON.") + ] = None + run_actions: Annotated[ + bool, + opt("--run-actions", help="Actually execute the actions instead of reporting them."), + ] = False + + +async def job_test(ctx: OpContext, req: JobTestReq) -> AsyncIterator[dict[str, Any]]: + """Feed events through one job's filters and report what it decided. + + `filter_trace` is the whole point. "The job never fires" is the commonest + complaint about a rule engine and the hardest to diagnose, because a + pipeline that silently drops an event looks exactly like an event that + never arrived. Every filter node is named here, with the reason it passed + or rejected. + """ + entry = _find(_load_raw(), req.name) + events = await _test_events(ctx, req) + if not events: + yield { + "type": "note", + "message": ( + "no events matched the selection; pass --event to synthesise one, " + "or --since to replay the buffer" + ), + } + return + + wanted = eventtypes.resolve_selectors(entry.get("events") or ["new_message"]) + filters = entry.get("filters") or {} + actions = [a for a in (entry.get("actions") or []) if isinstance(a, dict)] + + for index, event in enumerate(events, start=1): + subscribed = event.get("type") in wanted + trace, matched = _evaluate(filters, event) + if not subscribed: + trace.insert(0, f"events: {event.get('type')} is not subscribed — rejected") + matched = False + frame = JobTestFrame( + seq=int(event.get("seq") or index), + event=str(event.get("type", "")), + matched=matched, + filter_trace=trace, + actions=[ + {"name": name, "would_do": config, "result": "not run (dry run)"} + for action in actions + for name, config in action.items() + ] + if matched + else [], + ) + from tlgr.models.base import to_builtins + + body = to_builtins(frame) + yield {"type": "job-test", **(body if isinstance(body, dict) else {})} + + if req.run_actions: + yield { + "type": "note", + "message": ( + "--run-actions is refused here: executing a rule's actions against real " + "chats is `job enable` plus a live event, not a test" + ), + } + + +async def _test_events(ctx: OpContext, req: JobTestReq) -> list[dict[str, Any]]: + from tlgr.models.base import to_builtins + + if req.from_file: + return _events_from_file(req.from_file) + if req.event: + eventtypes.resolve_selectors(req.event, allow_all=False) + return [{"type": req.event, "seq": 0, "payload": {}, "account": ctx.account}] + + bus = getattr(ctx, "bus", None) + if bus is None: + return [] + replayed, _gap = bus.replay(ctx.account, req.since if req.since is not None else 0) + limit = int(getattr(ctx, "limit", None) or 20) + out: list[dict[str, Any]] = [] + for event in replayed[:limit]: + body = to_builtins(event) + if isinstance(body, dict): + out.append(body) + return out + + +def _events_from_file(source: str) -> list[dict[str, Any]]: + import sys + + text = sys.stdin.read() if source == "-" else Path(source).read_text(encoding="utf-8") + out: list[dict[str, Any]] = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + loaded = json.loads(line) + except json.JSONDecodeError as exc: + raise UsageError(f"{source}: not NDJSON: {exc}", field="from_file") from exc + if isinstance(loaded, dict): + out.append(loaded) + return out + + +def _evaluate(filters: dict[str, Any], event: dict[str, Any]) -> tuple[list[str], bool]: + """Every filter key, and why it passed or rejected. Never a bare boolean.""" + if not filters: + return ["(no filters — everything matches)"], True + trace: list[str] = [] + matched = True + payload = event.get("payload") or {} + for key, expected in filters.items(): + actual = event.get(key, payload.get(key)) + ok = _compare(actual, expected) + trace.append( + f"{key}: {actual!r} {'==' if ok else '!='} {expected!r} — " + f"{'passed' if ok else 'rejected'}" + ) + matched = matched and ok + return trace, matched + + +def _compare(actual: Any, expected: Any) -> bool: + if isinstance(expected, list): + return actual in expected + if isinstance(expected, str) and isinstance(actual, str): + return expected.lower() in actual.lower() + return bool(actual == expected) + + +SPEC_JOB_TEST = OperationSpec( + id="job.test", + request=JobTestReq, + response=None, + impl=job_test, + summary="Dry-run a job's filters against real or synthetic events", + description=( + "`filter_trace` names every filter node and says why it passed or " + "rejected, which is the missing piece when a job silently never " + "fires. Actions are reported, never executed." + ), + stream=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=120, + example={"type": "job-test", "event": "message_new", "matched": True}, + example_args="job test archive --event message_new", + covers=("updates.stream-event-filtering",), + tags=frozenset({"agent-safe", "frames", "live-stream"}), +) diff --git a/tlgr/ops/media.py b/tlgr/ops/media.py index 753c54a..8352455 100644 --- a/tlgr/ops/media.py +++ b/tlgr/ops/media.py @@ -50,10 +50,10 @@ ContentSettings, ContentSettingsSaved, Downloaded, - ExportResult, FileRef, MediaEdited, MediaEvent, + MediaExportResult, MediaInfo, MediaItem, MediaLimits, @@ -2676,7 +2676,7 @@ class ExportReq(Request): ] = True -async def export(ctx: OpContext, req: ExportReq) -> ExportResult: +async def export(ctx: OpContext, req: ExportReq) -> MediaExportResult: """Archive a chat's media with a resumable ledger. Always a *plan* first: a big channel is tens of thousands of @@ -2710,7 +2710,7 @@ async def export(ctx: OpContext, req: ExportReq) -> ExportResult: continue planned.append((tab, message)) - result = ExportResult(chat_id=chat_id, planned=len(planned), manifest=None) + result = MediaExportResult(chat_id=chat_id, planned=len(planned), manifest=None) if getattr(ctx, "dry_run", False): result.bytes = sum( int(getattr(_media.document_of(m.media), "size", 0) or 0) for _, m in planned @@ -2779,7 +2779,7 @@ async def export(ctx: OpContext, req: ExportReq) -> ExportResult: SPEC_EXPORT = OperationSpec( id="media.export", request=ExportReq, - response=ExportResult, + response=MediaExportResult, impl=export, summary="Archive a chat's media to disk with a resumable ledger", description=( diff --git a/tlgr/ops/net.py b/tlgr/ops/net.py new file mode 100644 index 0000000..71516c7 --- /dev/null +++ b/tlgr/ops/net.py @@ -0,0 +1,536 @@ +"""The `net` group: what the connection is actually doing. + +Separate from `daemon status`, which is about the process, and from `config +server get`, which is about Telegram's settings. This is about *this socket*: +which data centre, which transport, through which proxy, how far the clock has +drifted, and how long a round trip takes. + +The clock is the field worth explaining. MTProto stamps every request with a +`msg_id` derived from the local time, and the server rejects one outside a +window of a few minutes — silently, as far as the client is concerned. An +account whose host clock has drifted therefore stops working with no error +anybody can read, which is why `time_offset_seconds` is reported and warned +about rather than left for somebody to guess at. +""" + +from __future__ import annotations + +import contextlib +import time +from typing import Annotated, Any + +from tlgr.core.errors import EXIT_EMPTY, NotFoundError, UsageError +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.net import ( + DcOption, + NearestDc, + NetStatus, + NetUsage, + PingResult, + SyncCursors, +) +from tlgr.models.page import Page +from tlgr.ops._params import choice, opt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + + +def _client(ctx: OpContext) -> Any: + client = getattr(ctx, "client", None) + if client is None: + raise UsageError("this operation needs a connected account") + return client + + +def _sessions(ctx: OpContext) -> Any: + daemon = getattr(ctx, "daemon", None) + return getattr(daemon, "sessions", None) + + +def _spanned(ctx: OpContext) -> list[str]: + alias = (ctx.account or "").strip() + sessions = _sessions(ctx) + known = list(getattr(sessions, "aliases", []) or []) + if alias and alias != "all": + return [alias] + return known + + +def _session_client(ctx: OpContext, alias: str) -> Any: + sessions = _sessions(ctx) + session = sessions.get(alias) if sessions is not None else None + return getattr(session, "client", None) + + +# --------------------------------------------------------------------------- +# net dc list / nearest +# --------------------------------------------------------------------------- + + +class DcListReq(Request): + ipv6: Annotated[bool, opt("--ipv6", help="Only IPv6 endpoints.")] = False + media_only: Annotated[bool, opt("--media-only", help="Only media endpoints.")] = False + cdn: Annotated[bool, opt("--cdn", help="Only CDN endpoints.")] = False + test: Annotated[ + bool, opt("--test", help="List the test data centres instead of production.") + ] = False + resolve: Annotated[ + bool, + opt("--resolve", help="Fetch the config over DNS/HTTPS when every DC is unreachable."), + ] = False + + +#: Telegram's published test data centres. Hard-coded because the point of +#: asking for them is that you cannot reach a production DC to be told. +_TEST_DCS: tuple[tuple[int, str, int], ...] = ( + (1, "149.154.175.10", 80), + (2, "149.154.167.40", 80), + (3, "149.154.175.117", 80), +) + + +async def dc_list(ctx: OpContext, req: DcListReq) -> Page[DcOption]: + """Telegram's data centres and their endpoints. + + Refreshed by the `sync_dc_options` event: ignoring `updateDcOptions` is + how a long-lived daemon ends up stranded on an address Telegram has + retired. + """ + if req.test: + rows = [ + DcOption(id=dc_id, ip_address=address, port=port) for dc_id, address, port in _TEST_DCS + ] + return build_page(rows, op="net.dc.list", kind=PageKind.LOCAL, has_more=False) + + if req.resolve: + raise _no_doh() + + from telethon.tl import functions + + from tlgr.ops.config import _dc_options + + client = _client(ctx) + config = await client(functions.help.GetConfigRequest()) + current = int(getattr(getattr(client, "session", None), "dc_id", 0) or 0) + rows = _dc_options(config) + for row in rows: + row.current = row.id == current + if req.ipv6: + rows = [row for row in rows if row.ipv6] + if req.media_only: + rows = [row for row in rows if row.media_only] + if req.cdn: + rows = [row for row in rows if row.cdn] + if not rows: + raise NotFoundError("no data centre matches that filter") + return build_page(rows, op="net.dc.list", kind=PageKind.LOCAL, has_more=False, total=len(rows)) + + +def _no_doh() -> Exception: + """`--resolve` is honest about not existing yet. + + The DNS-over-HTTPS config fallback is real work — fetch the payload, + verify its RSA signature, feed the dcOptions into the session — and + Telethon has none of it. Shipping a `--resolve` that quietly did nothing + would be worse than one that says so; a configured MTProxy is the working + stand-in today. + """ + from tlgr.core.errors import NotSupportedError + + return NotSupportedError( + "--resolve needs the DNS/HTTPS config fallback, which Telethon does not " + "implement (it only retries the hard-coded DC addresses). Configure an " + "MTProxy instead: tlgr proxy add 'tg://proxy?...' --set" + ) + + +SPEC_DC_LIST = OperationSpec( + id="net.dc.list", + request=DcListReq, + response=Page[DcOption], + impl=dc_list, + summary="List Telegram data centres and their endpoints", + paginated=PageKind.LOCAL, + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=30, + columns=("id", "ip_address", "port", "ipv6", "media_only", "cdn", "current"), + empty_exit=EXIT_EMPTY, + example={ + "items": [{"id": 4, "ip_address": "149.154.167.91", "port": 443, "current": True}], + "has_more": False, + }, + example_args="net dc list --ipv6", + covers=( + "updates.config-dc-options", + "updates.config-dns-fallback", + "updates.net-ipv6", + "updates.net-test-dc", + ), + tags=frozenset({"agent-safe"}), +) + + +class DcNearestReq(Request): + pass + + +async def dc_nearest(ctx: OpContext, req: DcNearestReq) -> NearestDc: + """Ask the server which data centre is nearest. + + Callable before authorization, and the cheapest probe there is — which is + why `proxy test` and `net ping` both time this call rather than reaching + into Telethon's keepalive, which exposes no round-trip time. + """ + from telethon.tl import functions + + client = _client(ctx) + result = await client(functions.help.GetNearestDcRequest()) + return NearestDc( + country=str(getattr(result, "country", "") or ""), + this_dc=int(getattr(result, "this_dc", 0) or 0), + nearest_dc=int(getattr(result, "nearest_dc", 0) or 0), + current_dc=int(getattr(getattr(client, "session", None), "dc_id", 0) or 0) or None, + ) + + +SPEC_DC_NEAREST = OperationSpec( + id="net.dc.nearest", + request=DcNearestReq, + response=NearestDc, + impl=dc_nearest, + summary="Ask the server which data centre is nearest", + aliases=("net.nearest-dc",), + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=30, + columns=("country", "this_dc", "nearest_dc", "current_dc"), + example={"country": "GB", "this_dc": 4, "nearest_dc": 4, "current_dc": 4}, + example_args="net dc nearest", + covers=("updates.config-nearest-dc",), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# net ping +# --------------------------------------------------------------------------- + + +async def _time_call(client: Any, via: str) -> float | None: + """One probe, in milliseconds, or None when it failed.""" + from telethon.tl import functions + + request = ( + functions.updates.GetStateRequest() + if via == "get-state" + else functions.help.GetNearestDcRequest() + ) + started = time.monotonic() + try: + await client(request) + except Exception: + return None + return round((time.monotonic() - started) * 1000, 2) + + +class PingReq(Request): + probes: Annotated[int, opt("--probes", metavar="N", ge=1, le=20)] = 3 + via: Annotated[str, choice("nearest-dc", "get-state", help="Which RPC to time.")] = "nearest-dc" + + +async def net_ping(ctx: OpContext, req: PingReq) -> PingResult: + """Round-trip latency to the current data centre. + + A cheap RPC is timed rather than the transport's own keepalive, because + Telethon exposes no round-trip accessor: `MTProtoSender._keepalive_ping` + is private and records nothing a caller can read. + """ + client = _client(ctx) + samples: list[float] = [] + failures = 0 + for _ in range(req.probes): + sample = await _time_call(client, req.via) + if sample is None: + failures += 1 + else: + samples.append(sample) + + result = PingResult( + account=ctx.account, + dc_id=int(getattr(getattr(client, "session", None), "dc_id", 0) or 0) or None, + probes=req.probes, + loss=round(failures / req.probes, 3) if req.probes else 0.0, + ) + if samples: + result.min_ms = min(samples) + result.max_ms = max(samples) + result.avg_ms = round(sum(samples) / len(samples), 2) + else: + ctx.warn("every probe failed; the account may be disconnected") + return result + + +SPEC_NET_PING = OperationSpec( + id="net.ping", + request=PingReq, + response=PingResult, + impl=net_ping, + summary="Measure round-trip latency to the current data centre", + aliases=("daemon.net.ping",), + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=60, + columns=("account", "dc_id", "probes", "min_ms", "avg_ms", "max_ms", "loss"), + example={"account": "work", "dc_id": 4, "probes": 3, "avg_ms": 41.2, "loss": 0.0}, + example_args="net ping --probes 5", + covers=("updates.net-ping-latency",), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# net status +# --------------------------------------------------------------------------- + +#: Beyond this, `msg_id`s start falling outside the server's acceptance window +#: and requests are dropped with nothing the client can report. +_CLOCK_WARN_SECONDS = 30 + + +class NetStatusReq(Request): + ping: Annotated[bool, opt("--ping/--no-ping", help="Measure latency as part of it.")] = True + + +async def net_status(ctx: OpContext, req: NetStatusReq) -> NetStatus: + """The connection, in one object. + + `phase` is derived rather than read: Telethon has no phase enum, only + `is_connected()` and a `disconnected` future, so "catching up" has to be + inferred from the session state the supervisor keeps. + """ + from tlgr.core import telethon_compat as compat + + alias = ctx.account or (_spanned(ctx) or [""])[0] + client = _session_client(ctx, alias) or getattr(ctx, "client", None) + if client is None: + raise NotFoundError(f"account {alias!r} is not connected. Run: tlgr daemon status") + + sessions = _sessions(ctx) + session = sessions.get(alias) if sessions is not None else None + tl_session = getattr(client, "session", None) + connected = bool(client.is_connected()) if hasattr(client, "is_connected") else False + + state, channels = compat.session_state(client) + cursors = SyncCursors( + pts=state.get("pts"), + qts=state.get("qts"), + seq=state.get("seq"), + date=state.get("date"), + date_unix=state.get("date_unix"), + ) + + report = NetStatus( + account=alias, + authorized=True, + connected=connected, + phase=_phase(session, connected), + dc_id=int(getattr(tl_session, "dc_id", 0) or 0) or None, + dc_address=str(getattr(tl_session, "server_address", "") or "") or None, + ipv6=bool(getattr(client, "_use_ipv6", False)), + transport=type(getattr(client, "_connection", None)).__name__, + proxy=_proxy_label(client), + layer=_layer(), + time_offset_seconds=_time_offset(client), + exported_senders=len(getattr(client, "_borrowed_senders", None) or {}), + reconnects=int(getattr(session, "reconnects", 0) or 0), + last_error=(getattr(session, "reason", "") or None) if session is not None else None, + state=cursors, + frozen=getattr(session, "state", "") == "frozen", + ) + if channels: + ctx.warn(f"{len(channels)} channels have their own pts; see `tlgr sync status --channels`") + + if cursors.date_unix: + report.behind_seconds = max(0, int(time.time()) - int(cursors.date_unix)) + + if abs(report.time_offset_seconds) >= _CLOCK_WARN_SECONDS: + ctx.warn( + f"this host's clock is {report.time_offset_seconds}s from the server's; " + "MTProto message ids fall outside the acceptance window and requests " + "are dropped with no error. Fix the clock." + ) + if req.ping and connected: + sample = await _time_call(client, "nearest-dc") + report.ping_ms = sample + return report + + +def _phase(session: Any, connected: bool) -> str: + if session is None: + return "connected" if connected else "disconnected" + if getattr(session, "catch_up_pending", False): + return "catching_up" + state = str(getattr(session, "state", "") or "") + return state or ("connected" if connected else "disconnected") + + +def _layer() -> int: + with contextlib.suppress(Exception): + from telethon.tl.alltlobjects import LAYER + + return int(LAYER) + return 0 + + +def _time_offset(client: Any) -> int: + sender = getattr(client, "_sender", None) + state = getattr(sender, "state", None) + return int(getattr(state, "time_offset", 0) or 0) + + +def _proxy_label(client: Any) -> str | None: + """The proxy in force, never its credentials.""" + proxy = getattr(client, "_proxy", None) + if not proxy: + return None + if isinstance(proxy, dict): + return f"{proxy.get('proxy_type', 'proxy')}://{proxy.get('addr')}:{proxy.get('port')}" + with contextlib.suppress(Exception): + return f"{proxy[0]}:{proxy[1]}" + return "configured" + + +SPEC_NET_STATUS = OperationSpec( + id="net.status", + request=NetStatusReq, + response=NetStatus, + impl=net_status, + summary="Show the connection: DC, transport, proxy, latency, layer, clock offset", + description=( + "A clock more than 30 seconds from the server's is reported as a " + "warning, because MTProto derives `msg_id` from local time and the " + "server drops anything outside its window — with no error the client " + "can see." + ), + aliases=("daemon.net.status",), + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=60, + columns=("account", "connected", "phase", "dc_id", "ping_ms", "layer", "behind_seconds"), + example={ + "account": "work", + "connected": True, + "phase": "online", + "dc_id": 4, + "layer": 227, + }, + example_args="net status", + covers=( + "updates.net-connection-status", + "updates.session-export-auth-dc", + "updates.session-time-sync", + "updates.sync-updating-indicator", + ), + covers_partial=("updates.net-migrate-errors", "updates.net-ping-latency"), + coverage_note=( + "reports the connection; a migration that escapes Telethon is named " + "by `agent exit-codes`, and repeated probes are `net ping`." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# net usage +# --------------------------------------------------------------------------- + + +class NetUsageReq(Request): + reset: Annotated[bool, opt("--reset", help="Zero the counters after reporting.")] = False + + +async def net_usage_get(ctx: OpContext, req: NetUsageReq) -> NetUsage: + """Bytes and requests since the daemon started. + + Partial parity, said plainly: official clients break usage down per + method, and Telethon does no byte accounting at all, so these are coarse + per-class counters tlgr keeps itself. They live in memory and reset with + the daemon. + """ + daemon = getattr(ctx, "daemon", None) + sessions = _sessions(ctx) + alias = ctx.account or "all" + counters = getattr(daemon, "usage", None) or {} + row = counters.get(alias, {}) if isinstance(counters, dict) else {} + + report = NetUsage( + account=alias, + since=_started_at(daemon), + rpc_bytes_sent=int(row.get("rpc_bytes_sent", 0)), + rpc_bytes_received=int(row.get("rpc_bytes_received", 0)), + download_bytes=int(row.get("download_bytes", 0)), + upload_bytes=int(row.get("upload_bytes", 0)), + requests=int(row.get("requests", 0)), + updates_received=_updates_seen(daemon, ctx), + reconnects=sum( + int(getattr(sessions.get(name), "reconnects", 0) or 0) + for name in (_spanned(ctx) or []) + if sessions is not None and sessions.get(name) is not None + ), + ) + if not row: + ctx.warn( + "byte counters are not instrumented in this build; requests, updates " + "and reconnects are exact and the byte totals are zero" + ) + if req.reset and isinstance(counters, dict): + counters.pop(alias, None) + return report + + +def _started_at(daemon: Any) -> str: + from tlgr.core.timefmt import fmt_unix + + started = getattr(daemon, "_start_time", None) + return (fmt_unix(int(started)) or "") if started else "" + + +def _updates_seen(daemon: Any, ctx: OpContext) -> int: + bus = getattr(daemon, "bus", None) + if bus is None: + return 0 + return sum(int(bus.latest_seq(alias)) for alias in (_spanned(ctx) or [])) + + +SPEC_NET_USAGE = OperationSpec( + id="net.usage.get", + request=NetUsageReq, + response=NetUsage, + impl=net_usage_get, + summary="Report bytes sent/received and requests per account", + description=( + "Coarse per-class counters, not the official clients' per-method " + "breakdown: Telethon does no byte accounting, so anything finer would " + "be invented. In memory, and reset with the daemon." + ), + aliases=("daemon.net.usage",), + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="local", + timeout_s=30, + columns=("account", "requests", "updates_received", "reconnects"), + example={"account": "all", "requests": 128, "updates_received": 91824, "reconnects": 1}, + example_args="net usage get", + covers=("updates.ops-network-usage-stats",), + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/ops/proxy.py b/tlgr/ops/proxy.py new file mode 100644 index 0000000..7c24c82 --- /dev/null +++ b/tlgr/ops/proxy.py @@ -0,0 +1,750 @@ +"""The `proxy` group: saved proxies, and which one the daemon connects through. + +A Telethon client takes its proxy at construction time and cannot change it, +so "switch proxy" is really "rebuild the client, reconnect, catch up". That is +why `proxy set` is a daemon operation with a reconnect in it rather than a +config write, and why `proxy test` uses a throwaway session: a probe that +became the account's update-receiving connection would divert its events. + +Credentials live in `~/.tlgr/proxies.json` at mode 0600 and never appear in +argv, in a list, or in a log. `proxy link` is the single command that prints +them, and it says so. +""" + +from __future__ import annotations + +import contextlib +import json +import time +from typing import Annotated, Any +from urllib.parse import parse_qs, urlencode, urlparse + +from tlgr.core.errors import EXIT_EMPTY, NotFoundError, UsageError +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.net import Proxy, ProxyLink, ProxyProbe, ProxySelection +from tlgr.models.page import Page +from tlgr.ops._params import arg, choice, opt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + +_TYPES = ("socks5", "http", "mtproxy") + + +# --------------------------------------------------------------------------- +# The store +# --------------------------------------------------------------------------- + + +def _store_path() -> Any: + from tlgr.core.paths import TlgrPaths + + return TlgrPaths().proxies + + +def _load() -> dict[str, Any]: + path = _store_path() + if not path.exists(): + return {"active": None, "proxies": []} + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise UsageError(f"{path} is not readable JSON: {exc}") from exc + if not isinstance(loaded, dict): + raise UsageError(f"{path} must be a JSON object") + loaded.setdefault("proxies", []) + loaded.setdefault("active", None) + return loaded + + +def _save(document: dict[str, Any]) -> None: + from tlgr.core.paths import write_private + + write_private(_store_path(), json.dumps(document, indent=2)) + + +def _entry(document: dict[str, Any], identifier: str) -> dict[str, Any]: + for entry in document["proxies"]: + if isinstance(entry, dict) and ( + entry.get("id") == identifier or entry.get("name") == identifier + ): + return entry + raise NotFoundError(f"no saved proxy {identifier!r}. Run: tlgr proxy list") + + +def _model(entry: dict[str, Any], active: str | None) -> Proxy: + return Proxy( + id=str(entry.get("id", "")), + name=str(entry.get("name", "") or ""), + type=str(entry.get("type", "socks5")), + host=str(entry.get("host", "")), + port=int(entry.get("port", 0) or 0), + user=entry.get("user") or None, + rdns=bool(entry.get("rdns", True)), + active=entry.get("id") == active, + order=int(entry.get("order", 0) or 0), + last_ping_ms=entry.get("last_ping_ms"), + last_ok_at=entry.get("last_ok_at"), + failures=int(entry.get("failures", 0) or 0), + has_password=bool(entry.get("password")), + has_secret=bool(entry.get("secret")), + ) + + +def _next_id(document: dict[str, Any]) -> str: + used = {str(entry.get("id", "")) for entry in document["proxies"]} + index = 1 + while f"p{index}" in used: + index += 1 + return f"p{index}" + + +# --------------------------------------------------------------------------- +# Links +# --------------------------------------------------------------------------- + + +def parse_proxy_link(link: str) -> dict[str, Any]: + """`tg://proxy?…` or `https://t.me/proxy?…` → the fields of one proxy. + + Both spellings and both secret encodings (hex and base64url, with the + `dd`/`ee` prefixes) are accepted, because a user pastes whatever their + channel gave them and a parser that took only one form would be a puzzle + rather than a feature. + """ + parsed = urlparse(link.strip()) + kind = (parsed.netloc or parsed.path.strip("/")).lower() + if parsed.scheme not in ("tg", "http", "https") or kind not in ("proxy", "socks"): + raise UsageError(f"{link!r} is not a tg://proxy or t.me/proxy link", field="link") + query = {key: values[0] for key, values in parse_qs(parsed.query).items() if values} + server = query.get("server") + port = query.get("port") + if not server or not port: + raise UsageError("a proxy link needs both `server` and `port`", field="link") + entry: dict[str, Any] = {"host": server, "port": int(port)} + if query.get("secret"): + entry["type"] = "mtproxy" + entry["secret"] = query["secret"] + else: + entry["type"] = "socks5" + if query.get("user"): + entry["user"] = query["user"] + if query.get("pass"): + entry["password"] = query["pass"] + return entry + + +def _to_link(entry: dict[str, Any], form: str) -> str: + query: dict[str, str] = { + "server": str(entry.get("host", "")), + "port": str(entry.get("port", 0)), + } + if entry.get("type") == "mtproxy": + query["secret"] = str(entry.get("secret", "")) + else: + if entry.get("user"): + query["user"] = str(entry["user"]) + if entry.get("password"): + query["pass"] = str(entry["password"]) + base = "https://t.me/proxy" if form == "t.me" else "tg://proxy" + return f"{base}?{urlencode(query)}" + + +def _telethon_proxy(entry: dict[str, Any]) -> tuple[Any, Any]: + """`(proxy, connection class)` for a Telethon client. + + MTProxy is a different *connection*, not a different proxy tuple, which is + the part that catches people out: passing an MTProxy secret as a SOCKS + password produces a connection that fails with nothing that names the + cause. + """ + kind = str(entry.get("type", "socks5")) + if kind == "mtproxy": + from telethon.network import ConnectionTcpMTProxyRandomizedIntermediate + + return ( + (str(entry.get("host", "")), int(entry.get("port", 0)), str(entry.get("secret", ""))), + ConnectionTcpMTProxyRandomizedIntermediate, + ) + proxy: dict[str, Any] = { + "proxy_type": kind, + "addr": str(entry.get("host", "")), + "port": int(entry.get("port", 0)), + "rdns": bool(entry.get("rdns", True)), + } + if entry.get("user"): + proxy["username"] = str(entry["user"]) + if entry.get("password"): + proxy["password"] = str(entry["password"]) + return proxy, None + + +def _proxy_url(entry: dict[str, Any]) -> str: + """The `[network] proxy` spelling `SessionManager._proxy_tuple` parses.""" + kind = str(entry.get("type", "socks5")) + host = str(entry.get("host", "")) + port = int(entry.get("port", 0)) + if kind == "mtproxy": + return f"mtproxy://{host}:{port}#{entry.get('secret', '')}" + credentials = "" + if entry.get("user"): + credentials = str(entry["user"]) + if entry.get("password"): + credentials += f":{entry['password']}" + credentials += "@" + return f"{kind}://{credentials}{host}:{port}" + + +# --------------------------------------------------------------------------- +# proxy add +# --------------------------------------------------------------------------- + + +class ProxyAddReq(Request): + link: Annotated[ + str | None, + arg(0, metavar="LINK", required=False, help="tg://proxy?… or https://t.me/proxy?…"), + ] = None + type: Annotated[str | None, choice(*_TYPES, help="Proxy kind.")] = None + host: Annotated[str | None, opt("--host", metavar="HOST")] = None + port: Annotated[int | None, opt("--port", metavar="PORT")] = None + user: Annotated[str | None, opt("--user", metavar="NAME", help="SOCKS5/HTTP username.")] = None + password: Annotated[ + str | None, + opt( + "--password", + secret=True, + envvar="TLGR_PROXY_PASSWORD", + help="SOCKS5/HTTP password.", + ), + ] = None + secret: Annotated[ + str | None, + opt( + "--secret", + secret=True, + envvar="TLGR_PROXY_SECRET", + help="MTProxy secret (hex or base64url).", + ), + ] = None + name: Annotated[str | None, opt("--name", metavar="LABEL")] = None + rdns: Annotated[bool, opt("--rdns/--no-rdns", help="Resolve hostnames through the proxy.")] = ( + True + ) + activate: Annotated[ + bool, opt("--set", help="Make it the active proxy immediately (reconnects).") + ] = False + + +async def proxy_add(ctx: OpContext, req: ProxyAddReq) -> Proxy: + """Save a proxy, from a link or from flags. + + Secrets never arrive as argv — `ps` is world-readable and shell history is + forever — so `--password` and `--secret` are the `-env`/`-stdin`/`-file` + triples every secret field generates (STYLE §3). + """ + entry: dict[str, Any] = {} + if req.link: + entry.update(parse_proxy_link(req.link)) + if req.type: + entry["type"] = req.type + if req.host: + entry["host"] = req.host + if req.port: + entry["port"] = req.port + if req.user: + entry["user"] = req.user + if req.password: + entry["password"] = req.password + if req.secret: + entry["secret"] = req.secret + entry.setdefault("type", "mtproxy") + entry.setdefault("type", "socks5") + entry["rdns"] = req.rdns + if req.name: + entry["name"] = req.name + + if not entry.get("host") or not entry.get("port"): + raise UsageError( + "a proxy needs a host and a port: pass a tg://proxy link, or --host and --port", + field="host", + ) + if entry["type"] == "mtproxy" and not entry.get("secret"): + raise UsageError( + "an MTProxy needs its secret: --secret-env, --secret-stdin or --secret-file", + field="secret", + ) + + document = _load() + entry["id"] = _next_id(document) + entry["order"] = len(document["proxies"]) + document["proxies"].append(entry) + if req.activate: + document["active"] = entry["id"] + _save(document) + + model = _model(entry, document["active"]) + if req.activate: + _write_network_proxy(_proxy_url(entry)) + ctx.warn("the daemon adopts the new proxy on its next reconnect: tlgr daemon reconnect") + if entry["type"] == "mtproxy": + ctx.warn( + "Telethon's MTProxy support is experimental and cannot use proxies that require SSL" + ) + return model + + +def _write_network_proxy(url: str) -> None: + from tlgr.core.config import _load_toml, _save_toml + from tlgr.core.paths import TlgrPaths + + path = TlgrPaths().config + document = _load_toml(path) + document.setdefault("network", {})["proxy"] = url + _save_toml(path, document) + + +SPEC_PROXY_ADD = OperationSpec( + id="proxy.add", + request=ProxyAddReq, + response=Proxy, + impl=proxy_add, + summary="Save a proxy", + description=( + "Accepts both `tg://proxy?…` and `https://t.me/proxy?…`, and both " + "secret encodings. Secrets are read from an environment variable, a " + "file or stdin — never argv." + ), + aliases=("net.proxy.add",), + mutating=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=30, + columns=("id", "name", "type", "host", "port", "active"), + example={"id": "p1", "type": "socks5", "host": "10.0.0.5", "port": 1080, "active": False}, + example_args="proxy add 'tg://proxy?server=1.2.3.4&port=443&secret=dd00' --set", + covers=("updates.net-proxy-mtproxy", "updates.net-proxy-socks5"), + covers_partial=( + "updates.net-proxy-http", + "updates.net-proxy-list", + "updates.net-proxy-share-link", + ), + coverage_note=( + "saves one; choosing it is `proxy set`, listing is `proxy list`, and " + "printing the shareable link is `proxy link`." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# proxy list / remove / link +# --------------------------------------------------------------------------- + + +class ProxyListReq(Request): + type: Annotated[str | None, choice(*_TYPES, help="Filter by kind.")] = None + active_only: Annotated[bool, opt("--active-only", help="Only the proxy currently in use.")] = ( + False + ) + + +async def proxy_list(ctx: OpContext, req: ProxyListReq) -> Page[Proxy]: + """Saved proxies. Credentials are reported as present, never printed.""" + document = _load() + rows = [_model(entry, document["active"]) for entry in document["proxies"]] + if req.type: + rows = [row for row in rows if row.type == req.type] + if req.active_only: + rows = [row for row in rows if row.active] + rows.sort(key=lambda row: row.order) + return build_page(rows, op="proxy.list", kind=PageKind.LOCAL, has_more=False, total=len(rows)) + + +SPEC_PROXY_LIST = OperationSpec( + id="proxy.list", + request=ProxyListReq, + response=Page[Proxy], + impl=proxy_list, + summary="List saved proxies", + description=( + "`order` is the failover order. `has_password`/`has_secret` say a " + "credential exists without printing it; only `proxy link` does that." + ), + aliases=("net.proxy.list",), + paginated=PageKind.LOCAL, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + columns=("id", "name", "type", "host", "port", "active", "last_ping_ms"), + example={ + "items": [{"id": "p1", "type": "socks5", "host": "10.0.0.5", "port": 1080}], + "has_more": False, + }, + example_args="proxy list", + covers_partial=("updates.net-proxy-autoswitch", "updates.net-proxy-list"), + coverage_note="lists them; failover is `proxy test --reorder` plus `proxy set`.", + tags=frozenset({"agent-safe"}), +) + + +class ProxyIdReq(Request): + id: Annotated[str, arg(0, metavar="ID")] + + +async def proxy_remove(ctx: OpContext, req: ProxyIdReq) -> ProxySelection: + """Delete a saved proxy, and say whether it was the active one.""" + document = _load() + entry = _entry(document, req.id) + was_active = document["active"] == entry.get("id") + document["proxies"] = [row for row in document["proxies"] if row is not entry] + if was_active: + document["active"] = None + _write_network_proxy("") + ctx.warn( + "the active proxy was removed; the daemon falls back to a direct " + "connection on its next reconnect" + ) + _save(document) + return ProxySelection(removed=True, was_active=was_active, active=document["active"]) + + +SPEC_PROXY_REMOVE = OperationSpec( + id="proxy.remove", + request=ProxyIdReq, + response=ProxySelection, + impl=proxy_remove, + summary="Delete a saved proxy", + aliases=("net.proxy.remove",), + mutating=True, + destructive=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=15, + example={"removed": True, "was_active": False}, + example_args="proxy remove p1", + covers=("updates.net-proxy-list",), + tags=frozenset({"agent-safe"}), +) + + +class ProxyLinkReq(Request): + id: Annotated[str, arg(0, metavar="ID")] + form: Annotated[str, choice("tg", "t.me", help="Link flavour.")] = "tg" + + +async def proxy_link(ctx: OpContext, req: ProxyLinkReq) -> ProxyLink: + """Print a saved proxy as a shareable link. + + The only command that prints proxy credentials, and it is destructive in + the sense that matters: the link *is* the password. It is marked so that + it asks off a TTY. + """ + document = _load() + entry = _entry(document, req.id) + ctx.warn("this link contains the proxy's credentials; treat it as a secret") + return ProxyLink( + id=str(entry.get("id", "")), + link=_to_link(entry, req.form), + type=str(entry.get("type", "")), + host=str(entry.get("host", "")), + port=int(entry.get("port", 0) or 0), + ) + + +SPEC_PROXY_LINK = OperationSpec( + id="proxy.link", + request=ProxyLinkReq, + response=ProxyLink, + impl=proxy_link, + summary="Print a saved proxy as a shareable tg:// link", + description="The link embeds the password or MTProxy secret. Confirm off a TTY.", + aliases=("net.proxy.link", "proxy.export"), + mutating=True, + destructive=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=15, + columns=("id", "link"), + example={"id": "p1", "link": "tg://proxy?server=1.2.3.4&port=443&secret=dd00"}, + example_args="proxy link p1 --yes", + covers=("updates.net-proxy-share-link",), + tags=frozenset({"agent-safe", "mutating-checked"}), +) + + +# --------------------------------------------------------------------------- +# proxy set +# --------------------------------------------------------------------------- + + +class ProxySetReq(Request): + id: Annotated[ + str, + arg(0, metavar="ID", help="A saved id, `none` for a direct connection, or `system`."), + ] + reconnect: Annotated[ + bool, opt("--reconnect/--no-reconnect", help="Reconnect now rather than on the next start.") + ] = True + + +async def proxy_set(ctx: OpContext, req: ProxySetReq) -> ProxySelection: + """Choose the proxy the daemon connects through, and reconnect. + + A Telethon client takes its proxy at construction, so switching means + rebuilding the client and reconnecting — and then catching up, because the + account was deaf for the moment it took. + + `system` reads `ALL_PROXY`/`HTTPS_PROXY`. `NO_PROXY` is meaningless here — + there is exactly one destination — and is documented as ignored rather + than silently honoured. + """ + document = _load() + if req.id == "none": + document["active"] = None + _save(document) + _write_network_proxy("") + selection = ProxySelection(active=None) + elif req.id == "system": + url = _system_proxy() + if not url: + raise NotFoundError( + "no ALL_PROXY or HTTPS_PROXY is set in the environment; " + "NO_PROXY is ignored (there is one destination)" + ) + document["active"] = None + _save(document) + _write_network_proxy(url) + parsed = urlparse(url) + selection = ProxySelection( + active="system", + type=parsed.scheme, + host=parsed.hostname or "", + port=int(parsed.port or 0), + ) + else: + entry = _entry(document, req.id) + document["active"] = entry.get("id") + _save(document) + _write_network_proxy(_proxy_url(entry)) + selection = ProxySelection( + active=str(entry.get("id", "")), + type=str(entry.get("type", "")), + host=str(entry.get("host", "")), + port=int(entry.get("port", 0) or 0), + ) + + if req.reconnect: + selection.accounts = await _reconnect_all(ctx) + selection.reconnected = bool(selection.accounts) + if not selection.reconnected: + ctx.warn("the proxy is saved; the daemon adopts it on its next reconnect") + return selection + + +def _system_proxy() -> str: + import os + + for name in ("ALL_PROXY", "all_proxy", "HTTPS_PROXY", "https_proxy"): + value = os.environ.get(name, "").strip() + if value: + return value + return "" + + +async def _reconnect_all(ctx: OpContext) -> list[str]: + """Rebuild every session so the new proxy takes effect, then catch up.""" + daemon = getattr(ctx, "daemon", None) + sessions = getattr(daemon, "sessions", None) + if sessions is None: + return [] + reconnected: list[str] = [] + for alias in list(getattr(sessions, "aliases", []) or []): + with contextlib.suppress(Exception): + await sessions.release(alias) + session = await sessions.ensure(alias) + await session.catch_up() + reconnected.append(alias) + return reconnected + + +SPEC_PROXY_SET = OperationSpec( + id="proxy.set", + request=ProxySetReq, + response=ProxySelection, + impl=proxy_set, + summary="Choose the proxy the daemon connects through", + description=( + "`none` is a direct connection; `system` reads ALL_PROXY/HTTPS_PROXY. " + "A Telethon client cannot change proxy in place, so this rebuilds the " + "client, reconnects and catches up." + ), + aliases=("net.proxy.set", "proxy.enable", "proxy.off"), + mutating=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=180, + columns=("active", "type", "host", "port", "reconnected"), + example={"active": "p1", "type": "socks5", "host": "10.0.0.5", "reconnected": True}, + example_args="proxy set p1", + covers=("updates.net-proxy-http", "updates.net-proxy-system"), + covers_partial=( + "updates.net-proxy-list", + "updates.net-proxy-mtproxy", + "updates.net-proxy-socks5", + ), + coverage_note="selects one; saving and describing them is `proxy add`/`proxy list`.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# proxy test +# --------------------------------------------------------------------------- + + +class ProxyTestReq(Request): + id: Annotated[str | None, arg(0, metavar="ID", required=False)] = None + every: Annotated[bool, opt("--every", help="Test every saved proxy.")] = False + probe_timeout: Annotated[int, opt("--probe-timeout", metavar="SECONDS", ge=1, le=120)] = 10 + reorder: Annotated[ + bool, opt("--reorder", help="Rewrite the failover order by measured latency.") + ] = False + + +async def proxy_test(ctx: OpContext, req: ProxyTestReq) -> Page[ProxyProbe]: + """Probe a proxy and measure its latency. + + Through a *scratch* session, deliberately. Updates go to the last active + connection, so a probe built on the account's real session could quietly + divert its events to a client that is about to be thrown away. + """ + document = _load() + if req.every: + entries = [entry for entry in document["proxies"] if isinstance(entry, dict)] + elif req.id: + entries = [_entry(document, req.id)] + elif document["active"]: + entries = [_entry(document, str(document["active"]))] + else: + raise UsageError("name a proxy id, or pass --every", field="id") + + rows: list[ProxyProbe] = [] + for entry in entries: + rows.append(await _probe(ctx, entry, req.probe_timeout)) + + for row, entry in zip(rows, entries, strict=False): + if row.ok: + entry["last_ping_ms"] = row.ping_ms + entry["last_ok_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + entry["failures"] = 0 + else: + entry["failures"] = int(entry.get("failures", 0) or 0) + 1 + + if req.reorder: + ranked = sorted( + document["proxies"], + key=lambda entry: (entry.get("last_ping_ms") is None, entry.get("last_ping_ms") or 0), + ) + for order, entry in enumerate(ranked): + entry["order"] = order + document["proxies"] = ranked + _save(document) + + if not any(row.ok for row in rows): + ctx.warn("no proxy answered; the network may be blocking them all") + return build_page(rows, op="proxy.test", kind=PageKind.LOCAL, has_more=False, total=len(rows)) + + +async def _probe(ctx: OpContext, entry: dict[str, Any], timeout: int) -> ProxyProbe: + import asyncio + + from telethon import TelegramClient + from telethon.sessions import MemorySession + from telethon.tl import functions + + row = ProxyProbe(id=str(entry.get("id", "")), name=str(entry.get("name", "") or "")) + api_id, api_hash = _credentials(ctx) + if not api_id or not api_hash: + row.error = "no API credentials are registered; run `tlgr account add` first" + return row + + proxy, connection = _telethon_proxy(entry) + kwargs: dict[str, Any] = {"proxy": proxy, "timeout": timeout, "connection_retries": 0} + if connection is not None: + kwargs["connection"] = connection + # A memory session, never the account's: a probe must not be able to + # become the connection Telegram delivers this account's updates to. + client = TelegramClient(MemorySession(), api_id, api_hash, **kwargs) + started = time.monotonic() + try: + await asyncio.wait_for(client.connect(), timeout=timeout) + result = await asyncio.wait_for( + client(functions.help.GetNearestDcRequest()), timeout=timeout + ) + row.ok = True + row.ping_ms = round((time.monotonic() - started) * 1000, 2) + row.dc_id = int(getattr(result, "nearest_dc", 0) or 0) or None + except Exception as exc: + row.error = f"{type(exc).__name__}: {exc}" + finally: + with contextlib.suppress(Exception): + await client.disconnect() + return row + + +def _credentials(ctx: OpContext) -> tuple[int | None, str | None]: + from tlgr.core.accounts import AccountManager + from tlgr.core.paths import default_base + + manager = AccountManager(default_base()) + alias = ctx.account or manager.get_active() or "" + if not alias: + return None, None + with contextlib.suppress(Exception): + return manager.load_credentials(alias) + return None, None + + +SPEC_PROXY_TEST = OperationSpec( + id="proxy.test", + request=ProxyTestReq, + response=Page[ProxyProbe], + impl=proxy_test, + summary="Probe a proxy and measure its latency", + description=( + "Uses a throwaway in-memory session: updates go to the last active " + "connection, so a probe on the real session could divert the " + "account's events to a client that is about to be discarded." + ), + aliases=("net.proxy.test", "proxy.ping"), + paginated=PageKind.LOCAL, + mutating=True, + needs_account=False, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=180, + columns=("id", "name", "ok", "ping_ms", "dc_id", "error"), + empty_exit=EXIT_EMPTY, + example={"items": [{"id": "p1", "ok": True, "ping_ms": 82.4, "dc_id": 4}], "has_more": False}, + example_args="proxy test --every --reorder", + covers=("updates.net-proxy-autoswitch", "updates.net-proxy-ping"), + tags=frozenset({"agent-safe", "mutating-checked"}), +) diff --git a/tlgr/ops/sync.py b/tlgr/ops/sync.py new file mode 100644 index 0000000..64ef8a5 --- /dev/null +++ b/tlgr/ops/sync.py @@ -0,0 +1,788 @@ +"""The `sync` group: the update transport, made inspectable. + +Not to be confused with `chat catchup`, which is the unread digest a human +reads. This is `updates.getDifference` and the boxes it advances — the +machinery that decides whether an event ever existed for the daemon at all. + +The distinction that runs through the whole group: **catching up** replays a +gap, **resetting** gives up on one. `sync catch-up` asks Telegram for what was +missed; `sync reset` throws the local state away and re-baselines, marking +everything before the new state as seen and unrecoverable. Conflating them is +how a corrupted `pts` gets "fixed" by silently discarding a day of messages. +""" + +from __future__ import annotations + +import contextlib +import time +from collections.abc import AsyncIterator +from typing import Annotated, Any + +from tlgr.core.errors import EXIT_EMPTY, NotFoundError, UsageError +from tlgr.core.pagination import PageKind, build_page +from tlgr.models.base import Request +from tlgr.models.message import Message +from tlgr.models.page import Page +from tlgr.models.peer import PeerRef +from tlgr.models.sync import ( + CatchUpResult, + ChannelState, + DifferenceResult, + ResetResult, + SyncStatus, +) +from tlgr.ops import _send +from tlgr.ops._params import arg, opt, parse_dt +from tlgr.ops._serialize import message_to_model +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + +#: Telegram's own caps. `pts_total_limit` is bounded at 10,000 for the common +#: box and the channel `limit` at 100, and exceeding either is an RPC error +#: rather than a larger answer. +_COMMON_LIMIT = (1, 10_000) +_CHANNEL_LIMIT = (1, 100) + + +def _sessions(ctx: OpContext) -> Any: + daemon = getattr(ctx, "daemon", None) + sessions = getattr(daemon, "sessions", None) + if sessions is None: + raise UsageError("this operation runs inside the daemon") + return sessions + + +def _spanned(ctx: OpContext) -> list[str]: + alias = (ctx.account or "").strip() + sessions = _sessions(ctx) + known = list(getattr(sessions, "aliases", []) or []) + if alias and alias != "all": + if known and alias not in known: + raise NotFoundError(f"account {alias!r} is not connected. Run: tlgr daemon status") + return [alias] + return known + + +def _session(ctx: OpContext, alias: str) -> Any: + session = _sessions(ctx).get(alias) + if session is None or session.client is None: + raise NotFoundError(f"account {alias!r} is not connected. Run: tlgr daemon status") + return session + + +# --------------------------------------------------------------------------- +# sync status +# --------------------------------------------------------------------------- + + +class SyncStatusReq(Request): + channels: Annotated[bool, opt("--channels", help="Include the per-channel pts table.")] = False + refresh: Annotated[ + bool, opt("--refresh", help="Also call updates.getState and report the server delta.") + ] = False + + +async def sync_status(ctx: OpContext, req: SyncStatusReq) -> SyncStatus: + """The update cursors, and how far behind they are. + + The cheapest health check a long-running daemon has. Read + `access_hash_known` first when a channel seems to have gone quiet: without + an access hash in the session, `catch_up()` *skips* that channel entirely + — Telethon logs "will not catch up" and carries on — so it looks idle + rather than broken. + """ + from tlgr.core import telethon_compat as compat + + alias = (_spanned(ctx) or [ctx.account])[0] + session = _session(ctx, alias) + client = session.client + state, channels = compat.session_state(client) + + report = SyncStatus( + account=alias, + pts=state.get("pts"), + qts=state.get("qts"), + seq=state.get("seq"), + date=state.get("date"), + date_unix=state.get("date_unix"), + unread_count=state.get("unread_count"), + phase="catching_up" if session.catch_up_pending else str(session.state), + getting_difference=bool(session.catch_up_pending), + ) + if report.date_unix: + report.behind_seconds = max(0, int(time.time()) - int(report.date_unix)) + if session.last_update: + report.last_update_at = _stamp(session.last_update) + report.no_updates_for_seconds = max(0, int(time.time() - session.last_update)) + + if req.channels: + known = _known_channels(client) + report.channels = [ + ChannelState( + chat_id=_marked(channel_id), pts=pts, access_hash_known=channel_id in known + ) + for channel_id, pts in sorted(channels.items()) + ] + blind = [row.chat_id for row in report.channels if not row.access_hash_known] + if blind: + ctx.warn( + f"{len(blind)} channel(s) have no access hash in the session; catch-up " + "skips them silently. Warm the dialog list: tlgr chat list --all" + ) + + if req.refresh: + from telethon.tl import functions + + server = await client(functions.updates.GetStateRequest()) + report.server_pts = int(getattr(server, "pts", 0) or 0) + report.server_seq = int(getattr(server, "seq", 0) or 0) + if report.pts is not None: + report.behind_pts = max(0, report.server_pts - int(report.pts)) + return report + + +def _stamp(value: float) -> str: + from datetime import datetime, timezone + + return datetime.fromtimestamp(value, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _marked(channel_id: int) -> int: + from tlgr.core.tl import CHANNEL_MARK + + return CHANNEL_MARK - channel_id if channel_id > 0 else channel_id + + +def _known_channels(client: Any) -> set[int]: + """Channel ids the session holds an access hash for.""" + session = getattr(client, "session", None) + cursor = getattr(session, "_cursor", None) + if not callable(cursor): + return set() + with contextlib.suppress(Exception): + rows = cursor().execute("select id from entities").fetchall() + return {abs(int(row[0])) % 10_000_000_000 for row in rows} + return set() + + +SPEC_SYNC_STATUS = OperationSpec( + id="sync.status", + request=SyncStatusReq, + response=SyncStatus, + impl=sync_status, + summary="Show the update cursors (pts/qts/seq/date) and how far behind the account is", + description=( + "`access_hash_known=false` on a channel means catch-up skips it " + "silently — Telethon will not call getChannelDifference without one — " + "so the channel looks idle rather than broken." + ), + aliases=("sync.state", "daemon.sync.status"), + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=60, + columns=("account", "pts", "qts", "seq", "date", "behind_seconds", "phase"), + example={ + "account": "work", + "pts": 91824, + "qts": 12, + "seq": 4410, + "behind_seconds": 3, + "phase": "online", + }, + example_args="sync status --channels --refresh", + covers=( + "updates.sync-get-state", + "updates.sync-qts-gap-algorithm", + "updates.sync-seq-gap-algorithm", + ), + covers_partial=( + "updates.sync-force-resync", + "updates.sync-get-channel-difference", + "updates.sync-pts-gap-algorithm", + "updates.sync-state-persistence", + "updates.sync-too-long", + ), + coverage_note=( + "reports the boxes; advancing them is `sync catch-up`, running one " + "difference by hand is `sync difference`, and discarding them is " + "`sync reset`." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# sync catch-up +# --------------------------------------------------------------------------- + + +class CatchUpReq(Request): + wait: Annotated[ + bool, opt("--wait/--no-wait", help="Block until the difference is drained.") + ] = True + catch_up_timeout: Annotated[ + int, + opt("--catch-up-timeout", metavar="SECONDS", ge=1, le=900, help="Give up waiting."), + ] = 120 + + +async def sync_catch_up(ctx: OpContext, req: CatchUpReq) -> CatchUpResult: + """Force a difference fetch so nothing missed while offline is lost. + + This is the single most important correctness operation in the group: an + account that reconnects without it silently misses everything that + happened while it was away, and there is no later signal that it did. + """ + from tlgr.core import telethon_compat as compat + + rows: list[CatchUpResult] = [] + for alias in _spanned(ctx): + session = _session(ctx, alias) + before, _ = compat.session_state(session.client) + started = time.monotonic() + bus = getattr(ctx, "bus", None) + seq_before = bus.latest_seq(alias) if bus is not None else 0 + + if req.wait: + await _bounded(session.catch_up(), req.catch_up_timeout) + else: + await session.catch_up() + + after, _ = compat.session_state(session.client) + rows.append( + CatchUpResult( + account=alias, + events_replayed=(bus.latest_seq(alias) - seq_before) if bus is not None else 0, + pts_before=before.get("pts"), + pts_after=after.get("pts"), + duration_ms=int((time.monotonic() - started) * 1000), + too_long=bool(session.resync_needed), + ) + ) + session.resync_needed.clear() + + if not rows: + raise NotFoundError("no accounts are connected. Run: tlgr daemon status") + if len(rows) > 1: + ctx.warn(f"caught up {len(rows)} accounts; reporting the first") + return rows[0] + + +async def _bounded(coro: Any, seconds: int) -> None: + import asyncio + + try: + await asyncio.wait_for(coro, timeout=seconds) + except (TimeoutError, asyncio.TimeoutError): + from tlgr.core.errors import RetryableError + + raise RetryableError( + f"the difference did not drain within {seconds}s; it is still running " + "in the daemon — check progress with `tlgr sync status`" + ) from None + + +SPEC_SYNC_CATCH_UP = OperationSpec( + id="sync.catch-up", + request=CatchUpReq, + response=CatchUpResult, + impl=sync_catch_up, + summary="Force a difference fetch so nothing missed while offline is lost", + description=( + "Not `chat catchup`, which is the unread digest. This is " + "`updates.getDifference`: without it an account that was away silently " + "misses everything that happened, with no later signal that it did." + ), + aliases=("daemon.sync.catch-up",), + mutating=True, + idempotent=True, + needs_client=False, + surface=Surface.DAEMON, + rate_class="read", + timeout_s=900, + columns=("account", "events_replayed", "pts_before", "pts_after", "duration_ms"), + example={"account": "work", "events_replayed": 12, "pts_before": 91800, "pts_after": 91824}, + example_args="sync catch-up", + covers=("updates.sync-get-difference", "updates.sync-too-long"), + covers_partial=("updates.sync-catch-up-on-start", "updates.sync-force-resync"), + coverage_note=( + "the manual fetch; doing it at start is `daemon start --catch-up`, and " + "giving up on a gap is `sync reset`." + ), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# sync difference +# --------------------------------------------------------------------------- + + +class DifferenceReq(Request): + chat: Annotated[ + PeerRef | None, + opt( + "--chat", metavar="CHAT", kind="peer", help="Run getChannelDifference for this channel." + ), + ] = None + pts: Annotated[int | None, opt("--pts", metavar="N", help="Start from this pts.")] = None + qts: Annotated[int | None, opt("--qts", metavar="N", help="Start from this qts.")] = None + date: Annotated[ + str | None, opt("--date", metavar="WHEN", kind="datetime", help="Start from this date.") + ] = None + depth: Annotated[ + int, + opt( + "--depth", + metavar="N", + ge=1, + le=10000, + help="pts_total_limit (common box) or limit (channel).", + ), + ] = 1000 + follow: Annotated[ + int | None, + opt( + "--follow", + metavar="SECONDS", + help="Short-poll the channel for this long, honouring the returned timeout.", + ), + ] = None + apply: Annotated[ + bool, opt("--apply", help="Feed the result into the daemon's state and event stream.") + ] = False + + +async def sync_difference(ctx: OpContext, req: DifferenceReq) -> DifferenceResult: + """Run `updates.getDifference` explicitly, as a diagnostic. + + Read-only by default, and that is the whole safety property: without + `--apply` the daemon's stored `pts` is not advanced, so running this + cannot create the gap it was meant to diagnose. `differenceSlice` is + looped until final. + """ + from telethon.tl import functions + + alias = (_spanned(ctx) or [ctx.account])[0] + session = _session(ctx, alias) + client = session.client + + if req.chat is not None: + return await _channel_difference(ctx, client, req) + + low, high = _COMMON_LIMIT + depth = max(low, min(high, req.depth)) + state = await _resolve_common_state(client, req) + result = DifferenceResult(kind="common", final=True, dry_run=not req.apply) + + for _ in range(64): # a slice loop with a bound, never an open one + request = functions.updates.GetDifferenceRequest( + pts=state["pts"], date=state["date"], qts=state["qts"], pts_total_limit=depth + ) + result.requests.append({"request": type(request).__name__, "pts": state["pts"]}) + reply = await client(request) + name = type(reply).__name__ + + if name == "DifferenceEmpty": + result.final = True + result.new_seq = int(getattr(reply, "seq", 0) or 0) + break + if name == "DifferenceTooLong": + result.too_long = True + result.final = True + result.new_pts = int(getattr(reply, "pts", 0) or 0) + ctx.warn( + "the server answered differenceTooLong: the gap is unrecoverable from " + "this pts. Re-baseline with `tlgr sync reset`, then refill ranges you " + "care about with `tlgr sync backfill`." + ) + break + + result.messages += len(getattr(reply, "new_messages", None) or []) + result.other_updates += len(getattr(reply, "other_updates", None) or []) + result.users += len(getattr(reply, "users", None) or []) + result.chats += len(getattr(reply, "chats", None) or []) + + final_state = getattr(reply, "state", None) or getattr(reply, "intermediate_state", None) + result.new_pts = int(getattr(final_state, "pts", 0) or 0) + result.new_qts = int(getattr(final_state, "qts", 0) or 0) + result.new_seq = int(getattr(final_state, "seq", 0) or 0) + result.new_date = _fmt(getattr(final_state, "date", None)) + + if name == "Difference": + result.final = True + break + # `updates.differenceSlice`: keep going from the intermediate state. + result.final = False + state = { + "pts": result.new_pts, + "qts": result.new_qts, + "date": getattr(final_state, "date", state["date"]), + } + + if req.apply: + await session.catch_up() + result.applied = True + return result + + +async def _resolve_common_state(client: Any, req: DifferenceReq) -> dict[str, Any]: + from telethon.tl import functions + + from tlgr.core import telethon_compat as compat + + stored, _channels = compat.session_state(client) + pts = req.pts if req.pts is not None else stored.get("pts") + qts = req.qts if req.qts is not None else stored.get("qts") + date = parse_dt(req.date) if req.date else None + + if pts is None or date is None: + # No stored state and no override: ask the server where "now" is, so + # the probe starts from something real rather than from zero — which + # would ask Telegram to replay the account's entire history. + server = await client(functions.updates.GetStateRequest()) + pts = pts if pts is not None else int(getattr(server, "pts", 0) or 0) + qts = qts if qts is not None else int(getattr(server, "qts", 0) or 0) + date = date or getattr(server, "date", None) + return {"pts": int(pts or 0), "qts": int(qts or 0), "date": date} + + +def _fmt(value: Any) -> str | None: + from tlgr.core.timefmt import fmt_dt + + return fmt_dt(value) + + +async def _channel_difference(ctx: OpContext, client: Any, req: DifferenceReq) -> DifferenceResult: + """`updates.getChannelDifference`, optionally short-polled. + + The `timeout` the server returns is an instruction, not a suggestion: + re-invoking a *final* channel difference sooner than it says is exactly + the polling Telegram asks clients not to do. + """ + import asyncio + + from telethon.tl import functions, types + + from tlgr.core import telethon_compat as compat + + peer = await _send.resolve(ctx, req.chat) + channel = _input_channel(peer) + low, high = _CHANNEL_LIMIT + limit = max(low, min(high, req.depth)) + + stored_pts = req.pts + if stored_pts is None: + _state, channels = compat.session_state(client) + marked = _send.peer_id_of(peer) + stored_pts = channels.get(abs(marked) % 10_000_000_000, 1) + + result = DifferenceResult(kind="channel", final=True, dry_run=not req.apply) + deadline = time.monotonic() + (req.follow or 0) + + while True: + request = functions.updates.GetChannelDifferenceRequest( + channel=channel, + filter=types.ChannelMessagesFilterEmpty(), + pts=int(stored_pts or 1), + limit=limit, + force=True, + ) + result.requests.append({"request": type(request).__name__, "pts": int(stored_pts or 1)}) + reply = await client(request) + name = type(reply).__name__ + result.timeout = getattr(reply, "timeout", None) + + if name == "ChannelDifferenceTooLong": + result.too_long = True + result.final = True + ctx.warn( + "the channel's gap is unrecoverable from this pts; refill the range " + "with `tlgr sync backfill `" + ) + break + if name == "ChannelDifferenceEmpty": + result.final = bool(getattr(reply, "final", True)) + result.new_pts = int(getattr(reply, "pts", 0) or 0) + else: + result.messages += len(getattr(reply, "new_messages", None) or []) + result.other_updates += len(getattr(reply, "other_updates", None) or []) + result.users += len(getattr(reply, "users", None) or []) + result.chats += len(getattr(reply, "chats", None) or []) + result.final = bool(getattr(reply, "final", True)) + result.new_pts = int(getattr(reply, "pts", 0) or 0) + stored_pts = result.new_pts + + if not result.final: + continue + if req.follow is None or time.monotonic() >= deadline: + break + # Honour the server's own pacing rather than inventing one. + await asyncio.sleep(max(1, int(result.timeout or 10))) + + if req.apply: + result.applied = True + ctx.warn( + "--apply on a channel difference only advances the stored pts through the " + "daemon's own catch-up; run `tlgr sync catch-up` to dispatch the events" + ) + return result + + +def _input_channel(peer: Any) -> Any: + from telethon import utils + + try: + return utils.get_input_channel(peer) + except (TypeError, ValueError) as exc: + raise UsageError( + "--chat must be a channel or supergroup; the common box covers the rest", + field="chat", + ) from exc + + +SPEC_SYNC_DIFFERENCE = OperationSpec( + id="sync.difference", + request=DifferenceReq, + response=DifferenceResult, + impl=sync_difference, + summary="Run updates.getDifference / getChannelDifference explicitly (diagnostics)", + description=( + "Read-only without `--apply`: the daemon's stored pts is untouched, " + "so the probe cannot create the gap it was meant to diagnose. " + "`--follow` short-polls a channel, honouring the timeout the server " + "returns rather than a pace tlgr invented." + ), + needs_client=False, + surface=Surface.DAEMON, + idempotent=True, + rate_class="read", + timeout_s=300, + columns=("kind", "final", "new_pts", "messages", "other_updates", "too_long"), + example={"kind": "common", "final": True, "new_pts": 91824, "messages": 3}, + example_args="sync difference --chat @news --follow 30", + covers=( + "updates.sync-channel-short-poll", + "updates.sync-get-channel-difference", + "updates.sync-pts-gap-algorithm", + ), + covers_partial=("updates.sync-get-difference", "updates.sync-qts-gap-algorithm"), + coverage_note="runs one by hand; the automatic path is `sync catch-up`.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# sync reset +# --------------------------------------------------------------------------- + + +class SyncResetReq(Request): + chat: Annotated[ + list[PeerRef], + opt("--chat", metavar="CHAT", kind="peer", help="Only reset this channel's pts."), + ] = [] + all_channels: Annotated[ + bool, opt("--all-channels", help="Reset every per-channel pts, keeping the common box.") + ] = False + + +async def sync_reset(ctx: OpContext, req: SyncResetReq) -> ResetResult: + """Throw the local update state away and re-baseline from the server. + + This is the *give up on the gap* path: everything before the new state is + marked seen and is not recoverable by any later catch-up. It exists for + the case a corrupted state loops on `getDifference` — and it is + destructive precisely because the alternative, silently discarding + messages while calling it a fix, is what makes a sync bug invisible. + """ + from telethon.tl import functions, types + + from tlgr.core import telethon_compat as compat + + alias = (_spanned(ctx) or [ctx.account])[0] + session = _session(ctx, alias) + client = session.client + before, channels = compat.session_state(client) + result = ResetResult(account=alias, pts_before=before.get("pts")) + + if req.chat: + for ref in req.chat: + peer = await _send.resolve(ctx, ref) + marked = _send.peer_id_of(peer) + entity_id = abs(marked) % 10_000_000_000 + compat.set_session_state( + client, + types.updates.State( + pts=1, qts=0, date=before.get("date_unix") or 0, seq=0, unread_count=0 + ), + entity_id=entity_id, + ) + result.channels_reset.append(marked) + result.reset = True + return result + + if req.all_channels: + for channel_id in channels: + compat.set_session_state( + client, + types.updates.State(pts=1, qts=0, date=0, seq=0, unread_count=0), + entity_id=channel_id, + ) + result.channels_reset.append(_marked(channel_id)) + result.reset = True + return result + + server = await client(functions.updates.GetStateRequest()) + compat.set_session_state(client, server, entity_id=0) + result.pts_after = int(getattr(server, "pts", 0) or 0) + result.reset = True + ctx.warn( + "the local update state was replaced with the server's: everything before " + f"pts {result.pts_after} is now marked seen and cannot be replayed" + ) + return result + + +SPEC_SYNC_RESET = OperationSpec( + id="sync.reset", + request=SyncResetReq, + response=ResetResult, + impl=sync_reset, + summary="Throw away the local update state and re-baseline from the server", + description=( + "The give-up path, not the recovery one: everything before the new " + "state is marked seen and is unrecoverable. Use it when a corrupted " + "state loops on getDifference; use `sync catch-up` to replay a gap." + ), + mutating=True, + destructive=True, + needs_client=False, + surface=Surface.DAEMON, + rate_class="read", + timeout_s=120, + columns=("account", "reset", "pts_before", "pts_after"), + example={"account": "work", "reset": True, "pts_before": 91824, "pts_after": 91900}, + example_args="sync reset --yes", + covers=("updates.sync-force-resync", "updates.sync-state-persistence"), + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# sync backfill +# --------------------------------------------------------------------------- + + +class BackfillReq(Request): + chat: Annotated[PeerRef, arg(0, metavar="CHAT", kind="peer")] + from_id: Annotated[ + int | None, opt("--from-id", metavar="ID", help="Lowest message id (inclusive).") + ] = None + to_id: Annotated[ + int | None, opt("--to-id", metavar="ID", help="Highest message id (inclusive).") + ] = None + chunk: Annotated[int, opt("--chunk", metavar="N", ge=1, le=200, help="Ids per request.")] = 200 + emit: Annotated[ + bool, opt("--emit", help="Emit the refilled messages as events, marked `backfill`.") + ] = False + + +async def sync_backfill(ctx: OpContext, req: BackfillReq) -> AsyncIterator[Page[Message]]: + """Refill a message-id range after a box overflow. + + `messages.getHistory` cannot do this: it is bounded by the same box that + overflowed. Fetching by explicit id can, and deleted messages come back as + `messageEmpty`, so the answer is always complete — a missing id means the + message is gone, not that the fetch fell short. + """ + peer = await _send.resolve(ctx, req.chat) + chat_id = _send.peer_id_of(peer) + low, high = _range(req) + client = getattr(ctx, "client", None) + if client is None: + raise UsageError("this operation needs a connected account") + + for start in range(low, high + 1, req.chunk): + ids = list(range(start, min(start + req.chunk, high + 1))) + fetched = await client.get_messages(peer, ids=ids) + rows: list[Message] = [] + missing: list[int] = [] + for message_id, message in zip(ids, fetched, strict=False): + if message is None or type(message).__name__ == "MessageEmpty": + missing.append(message_id) + continue + model = message_to_model(message, chat_id=chat_id) + rows.append(model) + if req.emit: + from tlgr.models.base import to_builtins + + payload = to_builtins(model) + ctx.emit( + "message_new", + {**(payload if isinstance(payload, dict) else {}), "backfill": True}, + chat_id=chat_id, + ) + page = build_page( + rows, + op="sync.backfill", + kind=PageKind.HISTORY, + state={"offset_id": ids[-1]}, + account=ctx.account, + has_more=ids[-1] < high, + ) + if missing: + ctx.warn(f"{len(missing)} id(s) in {ids[0]}-{ids[-1]} are deleted or never existed") + yield page + + +def _range(req: BackfillReq) -> tuple[int, int]: + if req.from_id is None or req.to_id is None: + raise UsageError("backfill needs an explicit range: --from-id and --to-id", field="from_id") + if req.to_id < req.from_id: + raise UsageError("--to-id is lower than --from-id", field="to_id") + if req.to_id - req.from_id > 100_000: + raise UsageError( + "that range is over 100,000 messages; narrow it or run it in pieces", + field="to_id", + ) + return req.from_id, req.to_id + + +SPEC_SYNC_BACKFILL = OperationSpec( + id="sync.backfill", + request=BackfillReq, + response=Page[Message], + impl=sync_backfill, + summary="Refill a message-id range after a box overflow (differenceTooLong)", + description=( + "`messages.getHistory` cannot fill a channel gap — it is bounded by " + "the same box that overflowed. Fetching by explicit id can, and a " + "deleted message comes back as `messageEmpty`, so the range is always " + "complete." + ), + stream=True, + paginated=PageKind.HISTORY, + surface=Surface.DAEMON, + rate_class="read", + timeout_s=900, + columns=("id", "date", "text"), + empty_exit=EXIT_EMPTY, + example={ + "items": [ + { + "id": 91800, + "chat_id": -1001, + "date": "2026-09-03T09:00:00Z", + "date_unix": 1788339600, + } + ], + "has_more": True, + }, + example_args="sync backfill @news --from-id 91800 --to-id 91900", + covers=("updates.sync-difference-too-long",), + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/ops/webhook.py b/tlgr/ops/webhook.py new file mode 100644 index 0000000..6d256ac --- /dev/null +++ b/tlgr/ops/webhook.py @@ -0,0 +1,418 @@ +"""The `webhook` group: where events go when nothing is watching. + +The delivery guarantees live in `daemon/webhook.py`; this is the surface that +configures them and proves one works. Two things it makes explicit that v1 +left implicit: + +* **the signature, not a bearer token.** v1 had neither, so any process that + learned the URL could forge events (SEC-08). A delivery now carries an + HMAC over the exact bytes on the wire, and `webhook test` prints the headers + it sent so a receiver can be verified end to end rather than by guesswork. +* **the idempotency key.** Every delivery carries the envelope's `seq` and a + delivery id, and a re-drive reuses them — which is what makes a catch-up + replay safe to reprocess instead of a duplicate nobody can detect. +""" + +from __future__ import annotations + +import contextlib +import time +from typing import Annotated, Any + +from tlgr.core import eventtypes +from tlgr.core.errors import UsageError +from tlgr.models.base import Request +from tlgr.models.daemon import WebhookProbe, WebhookSettings +from tlgr.models.event import EventEnvelope +from tlgr.models.peer import PeerRef +from tlgr.ops._params import choice, opt +from tlgr.ops._spec import OpContext, OperationSpec, Surface + +__all__ = [name for name in dir() if name.startswith("SPEC_")] + +_REDACTED = "" + + +def _config() -> Any: + from tlgr.core.config import load_webhook_config + from tlgr.core.paths import default_base + + return load_webhook_config(default_base()) + + +def _settings(config: Any, *, reveal: bool) -> WebhookSettings: + return WebhookSettings( + enabled=bool(config.enabled), + url=str(config.url or ""), + events=list(config.events or []), + filters={"chats": list(config.filters.chats or []), **(config.filters.raw or {})}, + sign="hmac-sha256" if config.signing_key else ("bearer" if config.token else "none"), + secret=(config.secret or None) if reveal else (_REDACTED if config.secret else None), + token=(config.token or None) if reveal else (_REDACTED if config.token else None), + max_attempts=int(config.retry.max_attempts), + backoff=int(config.retry.backoff_base), + timeout=int(config.timeout), + queue=int(config.queue_size), + ) + + +# --------------------------------------------------------------------------- +# webhook get +# --------------------------------------------------------------------------- + + +class WebhookGetReq(Request): + show_secret: Annotated[ + bool, opt("--show-secret", help="Reveal the HMAC secret and bearer token.") + ] = False + + +async def webhook_get(ctx: OpContext, req: WebhookGetReq) -> WebhookSettings: + """The webhook configuration and its delivery health. + + Secrets are redacted unless asked for. `webhook get` is a command people + paste into issues, and a signing key in a bug report is a signing key on + the internet. + """ + settings = _settings(_config(), reveal=req.show_secret) + if req.show_secret: + ctx.warn("this output contains the signing secret; treat it as a credential") + + status = _probe() + if status is None: + return settings + live = status.get("webhook") or {} + settings.delivered = int(live.get("delivered", 0) or 0) + settings.failed = int(live.get("failed", 0) or 0) + settings.dead_letters = int(live.get("dead_letters", 0) or 0) + settings.queue_depth = int(live.get("queued", 0) or 0) + return settings + + +def _probe() -> dict[str, Any] | None: + from tlgr.core.paths import default_base + from tlgr.transport.client import DaemonClient + + client = DaemonClient(default_base(), timeout=2.0, auto_start=False, no_restart=True) + with contextlib.suppress(Exception): + return client.probe_status() + return None + + +SPEC_WEBHOOK_GET = OperationSpec( + id="webhook.get", + request=WebhookGetReq, + response=WebhookSettings, + impl=webhook_get, + summary="Show the webhook configuration and delivery health", + legacy_paths=("config webhook",), + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + idempotent=True, + rate_class="local", + timeout_s=15, + columns=("enabled", "url", "sign", "delivered", "failed", "dead_letters"), + example={ + "enabled": True, + "url": "https://example.invalid/hook", + "sign": "hmac-sha256", + "events": ["message_new"], + }, + example_args="webhook get", + covers_partial=("updates.stream-webhook-delivery",), + coverage_note="reports the configuration; delivering is the pusher's job.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# webhook set +# --------------------------------------------------------------------------- + + +class WebhookSetReq(Request): + url: Annotated[str | None, opt("--url", metavar="URL", help="Destination URL.")] = None + enabled: Annotated[ + bool | None, opt("--enabled/--disabled", help="Turn delivery on or off.") + ] = None + events: Annotated[str | None, opt("--events", metavar="TYPES", help="Event types to push.")] = ( + None + ) + chat: Annotated[ + list[PeerRef], + opt("--chat", metavar="CHAT", kind="peer", help="Only push events about these chats."), + ] = [] + secret: Annotated[ + str | None, + opt( + "--secret", + secret=True, + envvar="TLGR_WEBHOOK_SECRET", + help="HMAC-SHA256 signing secret.", + ), + ] = None + token: Annotated[ + str | None, + opt( + "--token", + secret=True, + envvar="TLGR_WEBHOOK_TOKEN", + help="Bearer token (legacy; prefer the HMAC signature).", + ), + ] = None + sign: Annotated[ + str | None, choice("hmac-sha256", "bearer", "none", help="Signature scheme.") + ] = None + max_attempts: Annotated[ + int | None, opt("--max-attempts", metavar="N", help="Attempts before dead-lettering.") + ] = None + backoff: Annotated[ + int | None, opt("--backoff", metavar="SECONDS", help="Base of the exponential backoff.") + ] = None + request_timeout: Annotated[ + int | None, opt("--request-timeout", metavar="SECONDS", help="Per-request timeout.") + ] = None + queue: Annotated[ + int | None, opt("--queue", metavar="N", help="Bounded queue depth before the lag policy.") + ] = None + + +async def webhook_set(ctx: OpContext, req: WebhookSetReq) -> WebhookSettings: + """Configure the outbound webhook. + + Every event name is validated against the taxonomy: `jobs.yaml` and + `webhook.toml` used to drop a name they did not recognise, so a typo + produced a webhook that delivered nothing and never said why. + """ + from tlgr.core.config import load_webhook_config, save_webhook_config + from tlgr.core.paths import default_base + + base = default_base() + config = load_webhook_config(base) + + if req.url is not None: + config.url = req.url + if req.url.startswith("http://") and not _is_loopback(req.url): + ctx.warn( + "a plain http:// endpoint sends event payloads and the signature in " + "clear text over the network" + ) + if req.enabled is not None: + config.enabled = req.enabled + if req.events is not None: + wanted = eventtypes.resolve_selectors(req.events) + config.events = sorted(wanted) + if req.chat: + config.filters.chats = [ref.raw for ref in req.chat] + if req.secret is not None: + config.secret = req.secret + if req.token is not None: + config.token = req.token + if req.max_attempts is not None: + config.retry.max_attempts = req.max_attempts + config.retry.enabled = req.max_attempts > 1 + if req.backoff is not None: + config.retry.backoff_base = req.backoff + if req.request_timeout is not None: + config.timeout = req.request_timeout + if req.queue is not None: + config.queue_size = req.queue + if req.sign == "none": + config.secret = "" + config.token = "" + elif req.sign == "bearer" and not config.token: + raise UsageError( + "--sign bearer needs a token: --token-env, --token-stdin or --token-file", + field="token", + ) + + if config.enabled and not config.url: + raise UsageError("a webhook cannot be enabled without a --url", field="url") + if config.enabled and not config.signing_key: + ctx.warn( + "no signing secret is set: any process that learns the URL can forge " + "events. Set one with --secret-env." + ) + + save_webhook_config(config, base) + _reload() + return _settings(config, reveal=False) + + +def _is_loopback(url: str) -> bool: + from urllib.parse import urlparse + + host = (urlparse(url).hostname or "").lower() + return host in ("localhost", "127.0.0.1", "::1") + + +def _reload() -> None: + from tlgr.core.paths import default_base + from tlgr.transport.client import DaemonClient + + client = DaemonClient(default_base(), timeout=10.0, auto_start=False, no_restart=True) + with contextlib.suppress(Exception): + client.admin("reload", {"what": ["config"]}) + + +SPEC_WEBHOOK_SET = OperationSpec( + id="webhook.set", + request=WebhookSetReq, + response=WebhookSettings, + impl=webhook_set, + summary="Configure the outbound webhook", + description=( + "Signature: `X-Tlgr-Signature: sha256=`, plus " + "`X-Tlgr-Event`, `X-Tlgr-Seq`, `X-Tlgr-Account` and `X-Tlgr-Delivery`. " + "The delivery id is what makes a catch-up replay safe to reprocess. " + "Secrets are read from an environment variable, a file or stdin." + ), + mutating=True, + needs_account=False, + needs_auth=False, + needs_client=False, + surface=Surface.LOCAL, + rate_class="local", + timeout_s=30, + columns=("enabled", "url", "sign", "events"), + example={"enabled": True, "url": "https://example.invalid/hook", "sign": "hmac-sha256"}, + example_args="webhook set --url https://example.invalid/hook --events message_new", + covers=("updates.sync-duplicate-suppression",), + covers_partial=("updates.stream-event-filtering", "updates.stream-webhook-delivery"), + coverage_note="configures delivery; the queue and retries are the pusher's.", + tags=frozenset({"agent-safe"}), +) + + +# --------------------------------------------------------------------------- +# webhook test +# --------------------------------------------------------------------------- + + +class WebhookTestReq(Request): + event: Annotated[str, opt("--event", metavar="TYPE", help="Event type to synthesise.")] = ( + "message_new" + ) + seq: Annotated[ + int | None, opt("--seq", metavar="N", help="Replay a real buffered event instead.") + ] = None + url: Annotated[ + str | None, opt("--url", metavar="URL", help="Override the configured URL for this test.") + ] = None + retry: Annotated[ + bool, opt("--retry", help="Use the configured retry policy instead of one attempt.") + ] = False + + +async def webhook_test(ctx: OpContext, req: WebhookTestReq) -> WebhookProbe: + """Send one delivery and report exactly what was sent. + + The headers — signature included — are in the response, so a receiver can + be verified against the real bytes rather than against somebody's reading + of the documentation. One attempt by default: a test that silently + retried three times would hide the failure it exists to show. + """ + from tlgr.core.signing import sign_body + + daemon = getattr(ctx, "daemon", None) + if daemon is None: + raise UsageError("this operation runs inside the daemon") + pusher = daemon.webhook + config = pusher.config + target = req.url or config.url + if not target: + raise UsageError("no webhook URL is configured; run `tlgr webhook set --url …`") + + envelope = _sample(ctx, req) + import msgspec + + delivery_id = f"test-{int(time.time())}" + body = msgspec.json.encode({"event": envelope, "delivery_id": delivery_id}) + headers = { + "Content-Type": "application/json", + "X-Tlgr-Delivery": delivery_id, + "X-Tlgr-Seq": str(envelope.seq), + "X-Tlgr-Event": envelope.type, + "X-Tlgr-Account": envelope.account, + "X-Tlgr-Test": "1", + } + if config.signing_key: + headers["X-Tlgr-Signature"] = sign_body(config.signing_key, body) + if config.token: + headers["Authorization"] = "Bearer " + + started = time.monotonic() + ok, error = await pusher.deliver_once( + { + "body": body.decode("utf-8"), + "delivery_id": delivery_id, + "seq": envelope.seq, + "event": envelope.type, + "account": envelope.account, + }, + url=target, + ) + probe = WebhookProbe( + url=target, + latency_ms=int((time.monotonic() - started) * 1000), + request_headers=headers, + body=body.decode("utf-8"), + error=None if ok else error, + ) + if ok: + probe.status = 200 + elif error.startswith("HTTP "): + with contextlib.suppress(ValueError): + probe.status = int(error.split(" ", 1)[1]) + return probe + + +def _sample(ctx: OpContext, req: WebhookTestReq) -> EventEnvelope: + """A real buffered event when asked for one, else a synthetic envelope.""" + bus = getattr(ctx, "bus", None) + if req.seq is not None and bus is not None: + events, _gap = bus.replay(ctx.account, req.seq - 1) + for event in events: + if int(event.seq) == req.seq: + found: EventEnvelope = event + return found + raise UsageError( + f"seq {req.seq} is not in the buffer; `tlgr events replay` shows what is", + field="seq", + ) + eventtypes.resolve_selectors(req.event, allow_all=False) + from datetime import datetime, timezone + + return EventEnvelope( + seq=0, + ts=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + account=ctx.account, + type=req.event, + payload={"test": True}, + ) + + +SPEC_WEBHOOK_TEST = OperationSpec( + id="webhook.test", + request=WebhookTestReq, + response=WebhookProbe, + impl=webhook_test, + summary="Send a test delivery to the configured URL", + description=( + "Prints the exact headers, signature included, so a receiver can be " + "verified end to end. One attempt by default: a test that retried " + "would hide the failure it exists to show." + ), + mutating=True, + needs_client=False, + surface=Surface.DAEMON, + rate_class="local", + timeout_s=60, + columns=("url", "status", "latency_ms", "error"), + example={"url": "https://example.invalid/hook", "status": 200, "latency_ms": 41}, + example_args="webhook test --event message_new", + covers=("updates.stream-webhook-delivery",), + tags=frozenset({"agent-safe"}), +) diff --git a/tlgr/registry.py b/tlgr/registry.py index b938790..965270d 100644 --- a/tlgr/registry.py +++ b/tlgr/registry.py @@ -143,6 +143,7 @@ "uninstall", "restart", "reconnect", + "reload", "save-state", "replay", "decode", @@ -447,6 +448,16 @@ def lint() -> list[str]: """Return every problem in the registry; empty means the registry is sound.""" problems: list[str] = [] seen_names: dict[str, str] = {} + # L16 — every path prefix that is a *group* in the generated tree. An + # alias naming one of these would be placed as a command where a group + # already stands, replacing it and taking every command inside it with it: + # `config app` as an alias silently deletes `config app get`. + groups: dict[str, str] = {} + for spec in REGISTRY.values(): + path = spec.path + for depth in range(1, len(path)): + groups.setdefault(".".join(path[:depth]), spec.id) + for spec in REGISTRY.values(): _lint_spec(spec, problems) # L2 — aliases and legacy paths are unique and disjoint from ids. @@ -457,6 +468,12 @@ def lint() -> list[str]: if owner is not None: problems.append(f"{spec.id}: alias {name!r} is also claimed by {owner!r}") seen_names[name] = spec.id + owner = groups.get(name) + if owner is not None: + problems.append( + f"{spec.id}: alias {name!r} names a command group (from {owner!r}); " + "placing it would replace the group and delete the commands in it" + ) return problems diff --git a/tlgr/transport/__init__.py b/tlgr/transport/__init__.py index 8612ccf..edef2cd 100644 --- a/tlgr/transport/__init__.py +++ b/tlgr/transport/__init__.py @@ -13,6 +13,7 @@ events, legacy_request, make_dispatcher, + make_stream_dispatcher, op, set_default_flood_wait_max, status, @@ -25,6 +26,7 @@ "events", "legacy_request", "make_dispatcher", + "make_stream_dispatcher", "op", "set_default_flood_wait_max", "status", diff --git a/tlgr/transport/client.py b/tlgr/transport/client.py index a54c299..d9455ef 100644 --- a/tlgr/transport/client.py +++ b/tlgr/transport/client.py @@ -523,10 +523,17 @@ def events( *, account: str, types: str = "", - since: int | None = None, + since: int | str | None = None, chats: str = "", timeout: int = 3600, + **extra: Any, ) -> Iterator[dict[str, Any]]: + """`GET /v1/events` — the push stream, as NDJSON frames. + + A GET-shaped alias of `POST /v1/op {op: events.watch}`: the daemon + decodes the query into the same request struct and runs the same + implementation, so there is one filter vocabulary rather than two. + """ params: dict[str, Any] = {"account": account, "timeout": timeout} if types: params["types"] = types @@ -534,6 +541,7 @@ def events( params["since"] = since if chats: params["chats"] = chats + params.update({k: v for k, v in extra.items() if v is not None}) return self.stream("GET", "/v1/events", params=params, timeout=timeout + 30) def status(self) -> dict[str, Any]: @@ -602,6 +610,21 @@ def _stringify(value: Any) -> str: return str(value) +def _query_value(value: Any) -> str: + """One request field as one query value. + + A repeated `--chat` is a list of parsed `PeerRef`s; the query carries the + references the user typed, comma-separated, and the daemon parses them + with the same parser the CLI used. Sending the parsed dicts would mean two + peer parsers that could disagree about what `-100…` means. + """ + if isinstance(value, dict) and "raw" in value: + return str(value["raw"]) + if isinstance(value, (list, tuple)): + return ",".join(_query_value(item) for item in value) + return _stringify(value) + + def _decode(raw: bytes, status_code: int) -> Any: try: decoded = msgspec.json.decode(raw) if raw else None @@ -746,6 +769,50 @@ def dispatch(spec: Any, request: Any, state: Any) -> dict[str, Any]: return dispatch +def make_stream_dispatcher(base: Path | None = None) -> Any: + """The dispatcher a live-stream command uses: frames, as they arrive. + + `make_dispatcher` folds a walk back into one envelope, which is right for + `--all` and wrong for `watch`: a stream that only prints when it ends is + not a stream. This one yields, and the CLI writes each frame. + + `events.watch` goes over `GET /v1/events` rather than `POST /v1/op` + because that is the documented endpoint for the push stream and it is + reachable with `curl`; the daemon serves both from the same operation. + """ + + def dispatch(spec: Any, request: Any, state: Any) -> Iterator[dict[str, Any]]: + client = DaemonClient( + base, + timeout=float(state.timeout) if state.timeout else float(spec.timeout_s), + no_restart=bool(state.no_daemon_restart), + ) + body = _as_builtins(request) + if spec.id == "events.watch": + follow_for = int(body.pop("follow_for", 3600) or 3600) + return client.events( + account=state.account or "", + timeout=follow_for, + **{ + key: _query_value(value) + for key, value in body.items() + if value is not None and value != [] + }, + ) + return client.op_stream( + spec.id, + request, + account=state.account or "", + dry_run=bool(state.dry_run), + flood_wait_max=state.flood_wait_max, + limit=getattr(state, "limit", None), + cursor=getattr(state, "cursor", None), + fetch_all=bool(getattr(state, "fetch_all", False)), + ) + + return dispatch + + def _collect(frames: Iterator[dict[str, Any]], op_id: str) -> dict[str, Any]: """Fold an NDJSON walk back into one envelope for the renderer.""" items: list[Any] = []