diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6734de80..7bbc5d9c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,12 +19,13 @@ jobs: - uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python }} - - run: uv sync --frozen --group dev + - run: uv sync --frozen --group dev --extra files - run: uv run python tools/generate_client.py --check - - run: uv run pytest -q + - run: uv run python tools/generate_public.py --check + - run: uv run --extra files pytest -q # Keep the byte-frozen runtime and unrelated examples out of formatting. - - run: uv run ruff check tools/generate_client.py tinyercot/catalog.py tests - - run: uv run ruff format --check tools/generate_client.py tinyercot/catalog.py tests + - run: uv run ruff check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests + - run: uv run ruff format --check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests - run: uv build - run: uv export --frozen --no-dev --no-emit-project --output-file /tmp/tinyercot-requirements.txt - run: uv venv /tmp/tinyercot-wheel diff --git a/.gitignore b/.gitignore index 57537455..b96b718b 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ tests/cassettes/*.yaml # claude .remember/ + +# Local bounded retrieval evidence +.local-evidence/ diff --git a/CLAUDE.md b/CLAUDE.md index 761edfe5..c89f5319 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,16 +2,18 @@ TinyERCOT preserves its legacy public Python API. The opt-in `tinyercot.catalog` module supplies offline metadata only. Read `docs/public-foundation.md` before -adding current data capabilities. +adding current data capabilities. Read `docs/public-retrieval.md` for the opt-in +two-endpoint API client, annual file adapter, and ESR website feed. ## Local checks ```bash -uv sync --frozen --group dev +uv sync --frozen --group dev --extra files uv run python tools/generate_client.py --check -uv run pytest -q -uv run ruff check tools/generate_client.py tinyercot/catalog.py tests -uv run ruff format --check tools/generate_client.py tinyercot/catalog.py tests +uv run python tools/generate_public.py --check +uv run --extra files pytest -q +uv run ruff check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests +uv run ruff format --check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests uv build ``` @@ -29,6 +31,13 @@ It has no `--refresh`, `--cache-products`, or `--pandas` option. - `api_response_fields.json` is the frozen legacy field cache. It does not verify current schemas. `products.json` is historical evidence, not a generation input. - `tinyercot/catalog.py` reads bundled public metadata without data requests. +- `tinyercot/public/` contains opt-in clients and separate errors and row models. +- `tools/generate_public.py` generates current models from pinned response fields. + New endpoints need actual field evidence. Cached legacy rows do not qualify. + +Real integration tests require explicit opt-in paths and an installed wheel. +The normal suite blocks sockets and skips them. Do not print credential files, +auth responses, tokens, or HTTPX request objects from real requests. Do not edit the generated file manually. Keep the three legacy runtime files unchanged for this milestone. New clients, errors, schema policies, file diff --git a/MANIFEST.in b/MANIFEST.in index 57afe9fc..0038c8c4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include api_response_fields.json uv.lock recursive-include tools *.py *.json recursive-include tests *.py *.json -recursive-include docs *.md +recursive-include docs *.md *.json diff --git a/README.md b/README.md index ecd11b0c..3154c9e9 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ Python client with 102 legacy typed ERCOT Public API endpoint families. -Legacy imports, signatures, and behavior stay fixed. Current server compatibility -is not verified. The opt-in offline catalog describes observed public sources -and access boundaries; it adds no data retrieval. +Legacy imports, signatures, and behavior stay fixed. Legacy server compatibility +is not verified. The opt-in `tinyercot.public` surface supports two current DAM +price endpoints, bounded annual DAM file samples, and the rolling website ESR +feed. The separate offline catalog records observed sources and access boundaries. ## Install @@ -12,7 +13,7 @@ and access boundaries; it adds no data retrieval. uv add tinyercot ``` -## Setup +## Legacy setup ```bash export ERCOT_USERNAME="your-username" @@ -110,16 +111,50 @@ Secure, Certified, EWS, private participant records, and customer data are restricted and excluded. Public API data requests still need an ERCOT account. See [scope, provenance, and adapter boundaries](docs/public-foundation.md). +## Opt-in public retrieval + +The new client has **2 generated typed operations in 2 products out of 243 +observed public data paths across 98 product namespaces**. This is partial +coverage. Unknown row schemas fail closed. Restricted services remain excluded. + +```python +from datetime import date +from tinyercot.public import Credentials, PublicClient, WebClient, coverage + +scopes = coverage() # Offline; no credentials or network requests. +with PublicClient(Credentials.from_env()) as client: + page = client.dam_prices( + start=date(2026, 9, 4), end=date(2026, 9, 4), + settlement_point="HB_HOUSTON", size=2, + ) + capacity = client.dam_capacity_prices( + start=date(2026, 9, 4), end=date(2026, 9, 4), + ancillary_type="REGUP", size=2, + ) + receipt = page.receipt # Public URL, UTC retrieval time, byte count, SHA-256. + +with WebClient() as client: + snapshot = client.esr() # Anonymous website feed, separate from Public Data API. + stale = snapshot.is_stale() +``` + +Use an existing secure environment injection method for credentials. The new +client does not load `.env` or authenticate on import. It has per-instance +request and byte limits. Its errors, row models, and pagination are separate +from the legacy DataFrame and exception contracts. +See [exact coverage, annual files, evidence, and limits](docs/public-retrieval.md). + ## Development and legacy generation Generation uses hash-pinned local inputs. It never fetches an upstream URL. ```bash -uv sync --frozen --group dev +uv sync --frozen --group dev --extra files uv run python tools/generate_client.py --check -uv run pytest -q -uv run ruff check tools/generate_client.py tinyercot/catalog.py tests -uv run ruff format --check tools/generate_client.py tinyercot/catalog.py tests +uv run python tools/generate_public.py --check +uv run --extra files pytest -q +uv run ruff check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests +uv run ruff format --check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests uv build ``` @@ -128,4 +163,6 @@ Use `--output /tmp/legacy.py` to write a review copy. Paths do not depend on the working directory. `--check` never writes. Missing inputs or changed hashes fail before output is written. The former authenticated `--refresh` and `--cache-products` developer commands are removed. Metadata refresh and current -API generation need a separate reviewed tool; neither is part of this milestone. +API generation use separate inputs. `tools/generate_public.py` generates only +the two source-backed current models from `tools/inputs/current/`. It never +downloads upstream specifications or overwrites the legacy generated file. diff --git a/docs/evidence/fixture-provenance.json b/docs/evidence/fixture-provenance.json new file mode 100644 index 00000000..1c43110a --- /dev/null +++ b/docs/evidence/fixture-provenance.json @@ -0,0 +1,171 @@ +{ + "fixtures": [ + { + "fixture": "capacity-current.json", + "sha256": "3fa8d075dadb0baf6cb0b4656ec87e887a7c85c6d96ab2923dc3e42bd8373251", + "transformation": "Exact public response bytes.", + "source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap?deliveryDateFrom=2026-09-04&deliveryDateTo=2026-09-04&ancillaryType=REGUP&size=2&page=1&sort=deliveryDate&dir=desc", + "retrieved_at": "2026-09-05 20:54:00.360875+00:00", + "sha256": "3fa8d075dadb0baf6cb0b4656ec87e887a7c85c6d96ab2923dc3e42bd8373251", + "byte_count": 1336, + "status": 200 + } + }, + { + "fixture": "capacity-oldest.json", + "sha256": "ed8731e3b8bb8e9c682403f11b0722f6c61f935d5a1ce23dac219b164623f855", + "transformation": "Exact public response bytes.", + "source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap?ancillaryType=REGUP&size=1&page=1&sort=deliveryDate&dir=asc", + "retrieved_at": "2026-09-05 20:54:03.804946+00:00", + "sha256": "ed8731e3b8bb8e9c682403f11b0722f6c61f935d5a1ce23dac219b164623f855", + "byte_count": 1237, + "status": 200 + } + }, + { + "fixture": "annual-cells.json", + "sha256": "3be74d1ea482d14e14318431a68e9a52433296be07ecc79fe5171732156bee70", + "transformation": "Header and first four rows from 2010 Dec_1 and 2026 Aug; tests construct small XLSX archives from these observed cells.", + "sources": [ + { + "source_url": "https://www.ercot.com/misdownload/servlets/mirDownload?doclookupId=283205860&reportTypeId=13060", + "retrieved_at": "2026-09-05T20:29:36.736580+00:00", + "bytes": 241081, + "sha256": "aca9db5d9dbb124ec7a5afcf13d245b7f984c93958d7e81a9b86a9008f695312", + "status": 200, + "document_id": "283205860", + "report_type_id": "13060", + "source_published_at": "2013-03-01T13:17:18-05:00", + "friendly_name": "DAMLZHBSPP_2010", + "members": [ + { + "name": "rpt.00013060.0000000000000000.DAMLZHBSPP_2010.xlsx", + "bytes": 291305 + } + ] + }, + { + "source_url": "https://www.ercot.com/misdownload/servlets/mirDownload?doclookupId=1268441709&reportTypeId=13060", + "retrieved_at": "2026-09-05T20:29:39.667899+00:00", + "bytes": 1356199, + "sha256": "6ca3b9751f3868b00bc8b48e9fef66350cf090170b23fff9a747a26923486a34", + "status": 200, + "document_id": "1268441709", + "report_type_id": "13060", + "source_published_at": "2026-08-30T08:02:17-05:00", + "friendly_name": "DAMLZHBSPP_2026", + "members": [ + { + "name": "rpt.00013060.0000000000000000.DAMLZHBSPP_2026.xlsx", + "bytes": 1950568 + } + ] + } + ] + }, + { + "fixture": "dam-annual-list.json", + "sha256": "43495cb7a429a314c43a97121aae818c30d688651a19af6f077e0a6ad47d7091", + "source": { + "source_url": "https://www.ercot.com/misapp/servlets/IceDocListJsonWS?reportTypeId=13060", + "retrieved_at": "2026-09-05T20:27:20.742164+00:00", + "status": 200, + "bytes": 7797, + "sha256": "43495cb7a429a314c43a97121aae818c30d688651a19af6f077e0a6ad47d7091", + "content_type": null, + "http_last_modified": null + }, + "transformation": "Exact public response bytes." + }, + { + "fixture": "dam-current-page1.json", + "sha256": "dc70cef89b0d7ce0fb53a6441f8d5bdd0f79b5d60fff6125253046017cf37d18", + "source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices", + "query": { + "deliveryDateFrom": "2026-09-04", + "deliveryDateTo": "2026-09-04", + "settlementPoint": "HB_HOUSTON", + "size": 2, + "page": 1, + "sort": "deliveryDate", + "dir": "desc" + }, + "retrieved_at": "2026-09-05T20:28:34.745619+00:00", + "bytes": 1384, + "sha256": "dc70cef89b0d7ce0fb53a6441f8d5bdd0f79b5d60fff6125253046017cf37d18", + "status": 200 + }, + "transformation": "Exact public response bytes." + }, + { + "fixture": "dam-current-page2.json", + "sha256": "62e38531d8bf6a2e8ab858f0dd005982c4a145e229a6197188386731f905efa9", + "source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices", + "query": { + "deliveryDateFrom": "2026-09-04", + "deliveryDateTo": "2026-09-04", + "settlementPoint": "HB_HOUSTON", + "size": 2, + "page": 2, + "sort": "deliveryDate", + "dir": "desc" + }, + "retrieved_at": "2026-09-05T20:28:36.897114+00:00", + "bytes": 1384, + "sha256": "62e38531d8bf6a2e8ab858f0dd005982c4a145e229a6197188386731f905efa9", + "status": 200 + }, + "transformation": "Exact public response bytes." + }, + { + "fixture": "dam-oldest.json", + "sha256": "c887c6583e05dede55a538e49f1d19136d6b733df1542eaf87f7b24672357ef6", + "source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices", + "query": { + "settlementPoint": "HB_HOUSTON", + "size": 1, + "page": 1, + "sort": "deliveryDate", + "dir": "asc" + }, + "retrieved_at": "2026-09-05T20:28:42.357641+00:00", + "bytes": 1279, + "sha256": "c887c6583e05dede55a538e49f1d19136d6b733df1542eaf87f7b24672357ef6", + "status": 200 + }, + "transformation": "Exact public response bytes." + }, + { + "fixture": "dam-product.json", + "sha256": "39af5e785f5d8da682d2552a1c5aab34b6d8a129f0396536d2257be9527eef74", + "source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd", + "query": {}, + "retrieved_at": "2026-09-05T20:28:32.451913+00:00", + "bytes": 1372, + "sha256": "39af5e785f5d8da682d2552a1c5aab34b6d8a129f0396536d2257be9527eef74", + "status": 200 + }, + "transformation": "Exact public response bytes." + }, + { + "fixture": "esr-live.json", + "sha256": "04d60f3488471982e2190123f0be5998c6bd0acbb00e74a5304d7b28a577c007", + "source": { + "source_url": "https://www.ercot.com/api/1/services/read/dashboards/energy-storage-resources.json", + "retrieved_at": "2026-09-05T20:27:20.661195+00:00", + "status": 200, + "bytes": 88668, + "sha256": "51095e4a3a63915ea4d73ee77831461d8e5da20f866fadece1f5c23a80b23633", + "content_type": "application/json", + "http_last_modified": null + }, + "transformation": "First two and last two rows from each source day; source timestamps preserved." + } + ] +} diff --git a/docs/evidence/installed-receipts.json b/docs/evidence/installed-receipts.json new file mode 100644 index 00000000..fd37e5a2 --- /dev/null +++ b/docs/evidence/installed-receipts.json @@ -0,0 +1,119 @@ +{ + "installed_module": "isolated site-packages/tinyercot/__init__.py (Python 3.14)", + "receipts": [ + { + "kind": "typed-api-current", + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices?settlementPoint=HB_HOUSTON&page=1&size=2&sort=deliveryDate&dir=asc&deliveryDateFrom=2026-09-04&deliveryDateTo=2026-09-04", + "retrieved_at": "2026-09-05 21:07:45.518924+00:00", + "sha256": "a15b73447a360b4e75809f3211d80f448b79717cfa0d0ed52a6e2c152856895f", + "byte_count": 1383, + "status": 200, + "rows": 2, + "page": 1, + "total_pages": 12 + }, + { + "kind": "typed-api-current", + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices?settlementPoint=HB_HOUSTON&page=2&size=2&sort=deliveryDate&dir=asc&deliveryDateFrom=2026-09-04&deliveryDateTo=2026-09-04", + "retrieved_at": "2026-09-05 21:07:47.526077+00:00", + "sha256": "1ff1da41ba3ecad8ef29b6d5f5bf274bd18a44b6301fe38ea71f6af41a8b7cdc", + "byte_count": 1383, + "status": 200, + "rows": 2, + "page": 2, + "total_pages": 12 + }, + { + "kind": "typed-api-historical", + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices?settlementPoint=HB_HOUSTON&page=1&size=1&sort=deliveryDate&dir=asc", + "retrieved_at": "2026-09-05 21:07:54.843199+00:00", + "sha256": "c887c6583e05dede55a538e49f1d19136d6b733df1542eaf87f7b24672357ef6", + "byte_count": 1279, + "status": 200, + "rows": 1, + "earliest_observed_date": "2023-12-13" + }, + { + "kind": "typed-api-after-token-reacquisition", + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices?settlementPoint=HB_HOUSTON&page=1&size=1&sort=deliveryDate&dir=desc&deliveryDateFrom=2026-09-04&deliveryDateTo=2026-09-04", + "retrieved_at": "2026-09-05 21:07:57.004796+00:00", + "sha256": "59913cba9f9ec61b4163c47999d5b5490e6edbeb5d7bf363defa1c5e3c458e45", + "byte_count": 1336, + "status": 200, + "rows": 1 + }, + { + "kind": "typed-capacity-current", + "source_url": "https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap?ancillaryType=REGUP&page=1&size=2&sort=deliveryDate&dir=desc&deliveryDateFrom=2026-09-04&deliveryDateTo=2026-09-04", + "retrieved_at": "2026-09-05 21:07:59.139161+00:00", + "sha256": "2bb390340e7bd53c5813eb9feec867fb9344e60b9d8f7a406b36c4c9a4c40db4", + "byte_count": 1336, + "status": 200, + "rows": 2 + }, + { + "kind": "typed-capacity-historical", + "source_url": "https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap?ancillaryType=REGUP&page=1&size=1&sort=deliveryDate&dir=asc", + "retrieved_at": "2026-09-05 21:08:02.329327+00:00", + "sha256": "ed8731e3b8bb8e9c682403f11b0722f6c61f935d5a1ce23dac219b164623f855", + "byte_count": 1237, + "status": 200, + "rows": 1, + "earliest_observed_date": "2023-12-13" + }, + { + "kind": "typed-esr", + "source_url": "https://www.ercot.com/api/1/services/read/dashboards/energy-storage-resources.json", + "retrieved_at": "2026-09-05 21:08:02.436736+00:00", + "sha256": "ce893e76c9ae91f8d0d5ef6224cb0ebe6660a17cf688280a34a34b736f67a558", + "byte_count": 90155, + "status": 200, + "rows": 482, + "source_updated_at": "2026-09-05 16:06:01-05:00", + "stale": false + }, + { + "kind": "typed-public-mis-listing", + "source_url": "https://www.ercot.com/misapp/servlets/IceDocListJsonWS?reportTypeId=13060", + "retrieved_at": "2026-09-05 21:08:04.547771+00:00", + "sha256": "43495cb7a429a314c43a97121aae818c30d688651a19af6f077e0a6ad47d7091", + "byte_count": 7797, + "status": 200, + "documents": 17 + }, + { + "kind": "typed-annual-sample", + "source_url": "https://www.ercot.com/misdownload/servlets/mirDownload?doclookupId=283205860&reportTypeId=13060", + "retrieved_at": "2026-09-05 20:52:21.196539+00:00", + "sha256": "aca9db5d9dbb124ec7a5afcf13d245b7f984c93958d7e81a9b86a9008f695312", + "byte_count": 241081, + "status": 200, + "year": 2010, + "source_published_at": "2013-03-01 13:17:18-05:00", + "document_id": "283205860", + "rows": 4, + "member": "rpt.00013060.0000000000000000.DAMLZHBSPP_2010.xlsx", + "sheet": "Dec_1", + "truncated": true, + "initial_cache_hit": true, + "verified_cache_reuse": true + }, + { + "kind": "typed-annual-sample", + "source_url": "https://www.ercot.com/misdownload/servlets/mirDownload?doclookupId=1268441709&reportTypeId=13060", + "retrieved_at": "2026-09-05 20:52:23.837681+00:00", + "sha256": "6ca3b9751f3868b00bc8b48e9fef66350cf090170b23fff9a747a26923486a34", + "byte_count": 1356199, + "status": 200, + "year": 2026, + "source_published_at": "2026-08-30 08:02:17-05:00", + "document_id": "1268441709", + "rows": 4, + "member": "rpt.00013060.0000000000000000.DAMLZHBSPP_2026.xlsx", + "sheet": "Aug", + "truncated": true, + "initial_cache_hit": true, + "verified_cache_reuse": true + } + ] +} diff --git a/docs/public-retrieval-task-note.md b/docs/public-retrieval-task-note.md new file mode 100644 index 00000000..43d6f030 --- /dev/null +++ b/docs/public-retrieval-task-note.md @@ -0,0 +1,113 @@ +# Public retrieval task note + +Status: Bounded retrieval milestone complete; direct stacked PR handoff. +Do not merge this PR or foundation PR #1. + +Branch: `feat/public-retrieval`. +PR base: `feat/public-foundation`, commit +`4b26c8a91efbba668dbbc40f564bbe812d4ae792`. +Remote main remains audit base `d1daad25df42d3fff41f88b907d99ef325b970e0`. + +## Scope + +The captain's later inbox instruction authorized real Public API access through +a local vault export. This supersedes the original Stage-0 public-metadata-only +stopping point. Stage-0 evidence remains in `docs/task-note.md` and PR #1. + +- Add opt-in `PublicClient` with two generated models: NP4-190-CD DAM settlement + prices and NP4-188-CD DAM capacity clearing prices. Generate offline from + hash-pinned, observed response fields. Reject unknown fields and null rows. +- Add bounded pagination for DAM settlement prices, explicit credentials, + synchronized ID-token acquisition and expiry margin, one 401 reacquisition, + safe errors, pacing, retry and response/request ceilings. +- Add anonymous `WebClient` for public report-13060 annual ZIP/XLSX samples and + the rolling website ESR feed. Require a receipt cache for annual downloads. + Keep source publication, retrieval time, local market fields, and hashes. +- Add all 39 audited family boundaries and all 257 observed API operation + states to opt-in `coverage()`. Two API paths and two website subsets are + covered within their stated limits. No entire broad family is marked covered. +- Keep all three legacy runtime files unchanged. Existing imports, defaults, + signatures, exceptions, pagination, and DataFrame behavior stay fixed. + +Generated typed coverage is **2 operations in 2 products out of 243 observed +data paths across 98 product namespaces**. The other 241 data paths remain +deferred. Stage 0 recorded 40 missing and 203 unverified cached row schemas. +Current evidence verifies only these two endpoints. The website ESR adapter +does not implement the separate four-second Public Data API. + +## Real evidence + +An isolated installed wheel passed real current and historical API fetches for +both generated models. Current selections used 2026-09-04, HB_HOUSTON or REGUP, +and at most two rows per page. Both oldest-first selections returned one row +dated 2023-12-13. These are observations, not complete-retention guarantees. +Two DAM pages and a fetch after explicit token reacquisition passed. + +The installed client decoded 482 ESR observations with time/freshness checks. +It listed 17 public annual files, selected only 2010 and 2026, and decoded four +rows from Dec_1 and Aug respectively. Cache reuse passed. Each annual file was +downloaded once during source discovery and once by the installed adapter. +Later installed probes used the cache for both files. No all-history or +all-sheet extraction ran. Public receipts and fixture provenance are in +`docs/evidence/`. The evidence-class table and all source families are in +`docs/public-retrieval.md`. + +Real authorized ERCOT authentication occurred. Credential and token values +were not printed or retained in evidence, source, tests, commits, or PR text. +The vault-created local `.env` is ignored and mode 0600. No restricted request, +paid access, bulk history, forecaster work, no-mistakes run, or merge occurred. + +## Exact local validation + +The following final checks passed: + +```bash +uv run python tools/generate_client.py --check +uv run python tools/generate_public.py --check +uv run --extra files pytest -q +UV_PROJECT_ENVIRONMENT=/tmp/tinyercot-py311 uv run --frozen --python 3.11 --extra files pytest -q +uv run ruff check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests +uv run ruff format --check tools/generate_client.py tools/generate_public.py tools/probe_public.py tinyercot/catalog.py tinyercot/public tests +uv build +uv export --frozen --no-dev --no-emit-project --output-file /tmp/tinyercot-requirements.txt +uv pip sync --python /tmp/tinyercot-wheel/bin/python /tmp/tinyercot-requirements.txt +uv pip install --python /tmp/tinyercot-wheel/bin/python --no-deps dist/tinyercot-0.2.2-py3-none-any.whl +/tmp/tinyercot-wheel/bin/python -I tests/wheel_smoke.py +git diff --check +``` + +Results: **941 passed, 1 skipped** on Python 3.14.0 and 3.11.14. The skipped +integration entry point requires explicit environment opt-in. All normal tests +block sockets. Ruff checked 26 authored Python files. Both generators reproduced +their pinned output. The built wheel passed isolated imports and metadata use +with sockets blocked and without the optional XLSX dependency. The source and +wheel archives exclude the local credential file and bulk source files. + +Real installed-client commands, run separately from the offline suite: + +```bash +uv venv /tmp/tinyercot-retrieval-wheel +uv pip install --python /tmp/tinyercot-retrieval-wheel/bin/python 'dist/tinyercot-0.2.2-py3-none-any.whl[files]' +uv pip install --python /tmp/tinyercot-retrieval-wheel/bin/python --reinstall --no-deps dist/tinyercot-0.2.2-py3-none-any.whl +/tmp/tinyercot-retrieval-wheel/bin/python -I tests/wheel_smoke.py +/tmp/tinyercot-retrieval-wheel/bin/python -I tools/probe_public.py --live --credentials-file /Users/kevin/.treehouse/tinyercot-1bad13/1/tinyercot/.env --output /Users/kevin/.treehouse/tinyercot-1bad13/1/tinyercot/.local-evidence/installed +``` + +Google-style documentation review covered authored Python summaries, Args, +Returns, Raises, Yields, and Attributes sections. The repository has no static +type-check command or PEP 561 contract. Existing unrelated example/frozen-file +Ruff findings remain excluded, as recorded in Stage 0. No unrelated work was +discarded or reformatted. CI runs offline on Python 3.11 and 3.14. +Diff review found that an interrupted HTTP-200 token body did not enter the +transport retry path. The fix and a mocked partial-token regression test pass. + +## Deferred and excluded + +No implementation blocker remains for this bounded milestone. The 241 other +data paths, generic API archive/bundle operations, other MIS products, rolling +feeds, schema epochs, all-filter contracts, as-of revisions, full history, +automatic polling, async retrieval, and package-wide static typing are deferred. +The 39-family table marks 33 deferred public/conditional boundaries, three +restricted families, and three unavailable gaps. Secure, Certified, participant +EWS, private telemetry/bids/COP/awards, settlements, and customer data remain +excluded. Retired AS-offer/SASM sources and hourly load 2001 remain unavailable. diff --git a/docs/public-retrieval.md b/docs/public-retrieval.md new file mode 100644 index 00000000..9c00a169 --- /dev/null +++ b/docs/public-retrieval.md @@ -0,0 +1,174 @@ +# Opt-in public retrieval + +This additive milestone preserves the legacy client. Import retrieval tools +from `tinyercot.public`. The Stage-0 catalog remains an audit snapshot. +Use `coverage()` for retrieval scope states. + +## Coverage and evidence + +On 2026-09-05 the isolated installed wheel retrieved and decoded public data. +Generated typed API coverage is **2 operations in 2 products**: NP4-190-CD and +NP4-188-CD. The first installed-client checkpoint covered NP4-190-CD alone; +NP4-188-CD was added from two later source-backed field observations. +The denominator is **243 observed public data paths in 98 product namespaces** +(242 public-reports paths and one public-data ESR path). This is a bounded +vertical slice, not broad typed coverage. The legacy 102 endpoints do not add +to current verified coverage. Website ESR is separate from the four-second +Public Data API. A namespace count does not establish an EMIL product census. + +| Evidence class | Exact supported or observed scope | +| --- | --- | +| Typed installed-client current fetch | NP4-190-CD DAM settlement prices: two pages of two HB_HOUSTON rows for 2026-09-04; another row after explicit ID-token reacquisition. NP4-188-CD: two REGUP capacity-price rows for 2026-09-04. | +| Typed installed-client historical fetch | NP4-190-CD HB_HOUSTON and NP4-188-CD REGUP: one oldest-first row each returned 2023-12-13. NP4-180-ER report 13060: four rows each from 2010/Dec_1 and 2026/Aug workbooks, with truncation and cache reuse verified. | +| Typed installed-client live fetch | Website ESR: 482 rolling rows in the final installed probe; source offset/epoch and freshness checks passed. Not four-second API coverage. | +| Real source retrieval only | NP4-190-CD product metadata, retained as a fixture. This is not an implemented product-metadata method. | +| Fixture-only behavior | Rate limits, transport retries, expired tokens, 401/403, malformed JSON/ZIP/XLSX, cache mutation, repeated pages, DST transitions. These faults were simulated, not induced on ERCOT. | +| Deferred schema | The other 241 observed data paths. Stage 0 recorded 40 missing and 203 unverified cached schemas. Current field evidence verifies two endpoints. No guessed raw or typed models. | +| Restricted | Secure, Certified, EWS, private telemetry, bids/COP/awards, participant settlements and customer data. No requests. | +| Unavailable | Retired total-AS-offer and SASM sources and the unavailable hourly-load 2001 year, as recorded in the audit. | + +Only two selected annual files were downloaded. Each was retrieved once during +source discovery and once through the installed adapter. Later uses reuse a +hash-verified cache. No all-year iteration or bulk historical extraction ran. +Authorized Public API authentication did occur in this milestone. No credential +or token values were retained in evidence, fixtures, logs, source, or PR text. + +The oldest row is the earliest returned observation for the selected point at +the retrieval time. It is not proof of complete history or stable retention. +Archive publication time, data date, and retrieval time remain separate. +No adapter provides an as-of vintage, revision merge, or complete-year claim. +Typed installed-client evidence means generated or explicit Pydantic row +decoding. This milestone does not add a package-wide PEP 561 typing claim. + +## Use annual files + +Install the optional XLSX decoder with `uv add 'tinyercot[files]'`. +Listing and live metadata do not require this extra. + +```python +from pathlib import Path +from tinyercot.public import WebClient, sample_dam_archive + +with WebClient() as client: + documents, listing_receipt = client.dam_archives() + document = next(d for d in documents if d.friendly_name == "DAMLZHBSPP_2010") + download = client.download_dam_archive(document, cache=Path(".ercot-cache")) + sample = sample_dam_archive(download, sheet="Dec_1", max_rows=4) + assert sample.truncated # A sample does not establish complete annual coverage. +``` + +The adapter uses the source page's `mirDownload` route. A discovery request to +`ViewReport` returned HTTP 200 with an HTML "No Document" page. HTTP status alone +does not verify a file. The adapter checks the public listing, received byte +count, ZIP structure, XLSX headers, cell types, and cached hash. It rejects +partial or changed cache entries. The caller must inspect and repair them +explicitly. It does not overwrite a prior document when publication changes. +Cache writes use POSIX exclusive-file and hard-link operations. + +Each call selects one listed public file. Each decode selects one named sheet +and at most 1,000 rows, with a bounded scan and archive expansion limits. +The cache preserves document ID, publication time, original retrieval time, +and hash. Workbook prices use Decimal conversion of numeric source cells. +Date, hour-ending, and repeated-hour fields stay separate. No guessed UTC +mapping joins the source's ambiguous local market intervals. + +## HTTP, token, and time contracts + +Defaults allow 20 HTTP attempts per client, including authentication and +retries, at least 2.1 seconds between starts, and at most 4 MB per response. +These are per-client ceilings, not an account-wide rate limiter. Callers that +run several clients must coordinate their total request rate. +The new surface retries 429, 502, 503, 504 and transport failures within its +attempt budget. It respects bounded Retry-After delays. It does not follow +redirects or retry 403. One 401 can trigger one ID-token reacquisition. +The client reacquires before the documented expiry with a 60-second margin. +`refresh_token()` means a new ID-token POST, not use of a returned refresh token. +The verified form-encoded request keeps credentials out of URLs. + +The ESR decoder checks source offsets against America/Chicago and checks epoch +milliseconds against each timestamp. It retains the source DST flag without +guessing its meaning. Source update time and freshness remain visible; a stale +snapshot is not silently described as current. The rolling feed supplies no +history or latency guarantee. Future and old source timestamps are stale. +Mocked DST fixtures cover the repeated fall hour and the missing spring hour. + +## Reproduction and evidence + +The generators read only hash-pinned local inputs. Current row pins include +actual response field descriptors and query provenance. Nullability is unknown; +the two models reject nulls and cover observed non-null rows only. Field order +comes from each response, while unknown/missing names or source types fail. +The generated models do not establish coverage of every filter or schema epoch. + +`docs/evidence/fixture-provenance.json` records public source hashes, retrieval +times, fixture hashes, and excerpt transformations. `installed-receipts.json` +records installed-client fetches without headers, credential values, or row +values. API fixtures are small public responses. ESR is an explicit excerpt. +Workbook tests build tiny XLSX files from recorded source cells. Full annual +archives and raw live snapshots are not checked in. + +The normal suite blocks network access. The real integration entry point is +skipped unless `TINYERCOT_LIVE_TESTS=1`. It also requires explicit absolute paths +in `TINYERCOT_INSTALLED_PYTHON`, `TINYERCOT_CREDENTIALS_FILE`, and +`TINYERCOT_PROBE_OUTPUT`. Use an isolated installed wheel with the files extra. +The credential path must be a regular mode-0600 file with the three ERCOT keys +and JSON-quoted values, as produced by the authorized local vault export. +Do not commit that file. `tools/probe_public.py --live` performs a fixed small +2026-09-04/oldest-price check, one rolling ESR request, one public listing, and +the two selected annual samples. Reuse its output cache to avoid redownloads. + +Primary sources: ERCOT's [current Public Reports specification](https://apiexplorer.ercot.com/developer/apis/pubapi-apim-api?export=true&api-version=2022-04-01-preview), +[authentication guide](https://developer.ercot.com/applications/pubapi/user-guide/registration-and-authentication/), +[known limits](https://developer.ercot.com/applications/pubapi/known-limits/), +[public annual DAM product](https://www.ercot.com/mp/data-products/data-product-details?id=np4-180-er), +and [ESR website feed](https://www.ercot.com/api/1/services/read/dashboards/energy-storage-resources.json). + +## Audited source families + +This table enumerates all 39 audited families, including 33 deferred public or +conditional boundaries, three restricted families, and three unavailable gaps. +Supported narrow subsets above do not mark an entire family covered. Product +IDs and primary ERCOT URLs are retained in `tinyercot/public/_coverage.json`. +This inventory includes website collections as well as EMIL products. + +| Family | State | Product or collection IDs | +| --- | --- | --- | +| DAM prices, lambda, constraints | deferred | NP4-183-CD, NP4-188-CD, NP4-190-CD, NP4-191-CD, NP4-523-CD | +| RT SCED prices and constraints | deferred | NP6-787-CD, NP6-788-CD, NP6-322-CD, NP6-86-CD, NP6-905-CD | +| Price corrections and investigations | deferred | NP4-196-M, NP4-197-M, NP4-46-AN, NP4-47-AN, NP4-48-AN, NP4-49-AN | +| Historical hub/zone and AS price files | deferred | NP4-180-ER, NP6-785-ER, NP4-181-ER | +| RTD indicative prices and capacity prices | deferred | NP6-970-CD, NP6-325-CD, NP6-329-CD | +| RTC+B ancillary prices, capability, and SOG | deferred | NP6-323-CD, NP6-324-CD, NP6-326-CD, NP6-327-CD, NP6-328-CD, NP6-331-CD, NP6-332-CD | +| Historical RT adders and RTC+B archives | deferred | NP6-792-ER, NP6-793-ER, NP6-794-ER, NP6-795-ER, NP6-796-ER | +| DAM plans, energy totals, AS offers/demand curves | deferred | NP4-33-CD, NP4-19-CD, NP4-192-CD, NP4-193-CD, NP4-212-CD, NP4-532-CD, NP1-302 | +| Legacy total AS offers | unavailable | NP4-179-CD | +| RUC demand curves, deployment factors, constraints | deferred | NP4-213-CD, NP4-214-CD, NP4-215-CD, NP5-525-CD, NP5-526-CD, NP5-527-CD, NP5-528-CD, NP5-520-ER, NP5-753-CD, NP5-754-CD, NP5-755-CD, NP5-108-CD, NP3-764-CD | +| System and zonal actual load | deferred | NP6-345-CD, NP6-346-CD, NP6-344-CD, NP6-235-CD, GEN-55-CD | +| Load forecasts | deferred | NP3-565-CD, NP3-566-CD, NP3-560-CD, NP3-561-CD, NP3-562-CD | +| Long hourly-load archive | deferred | WEBSITE-LOAD-HISTORY | +| Wind and solar actual/forecast series | deferred | NP4-732-CD, NP4-733-CD, NP4-737-CD, NP4-738-CD, NP4-742-CD, NP4-743-CD, NP4-745-CD, NP4-746-CD | +| Renewable forecast models and intra-hour forecasts | deferred | NP4-442-CD, NP4-443-CD, NP4-751-CD, NP4-752-CD | +| Storage four-second Public Data API | deferred | RPTESR-M | +| Live storage, fuel mix, and generation outages | deferred | GEN-545-UI, GEN-544-UI, GEN-546-UI | +| Live grid conditions, reserves, supply and demand | deferred | GEN-530-UI, GEN-518-UI, GEN-506-UI, GEN-536-UI, NP6-904-CD, NP6-906-UI, GEN-547-UI | +| Live price, weather, combined renewable displays | deferred | GEN-502-UI, GEN-523-UI, GEN-522-UI, GEN-526-UI, GEN-540-UI, GEN-542-UI, GEN-539-UI, GEN-537-UI, GEN-507-UI | +| Outage capacity, unplanned outages, adequacy | deferred | NP3-233-CD, NP1-346-ER, NP3-763-CD, NP3-161-CD, NP3-162-CD, OPG-103-ER | +| DC tie schedules, flows, and state-estimator aggregates | deferred | NP3-765-CD, NP6-626-CD, NP6-625-CD, GEN-538-UI | +| 2-day dispatch, energy curves, bids and offers | deferred | NP3-906-EX, NP3-907-EX, NP3-908-ER, NP3-909-ER, NP3-910-ER, NP3-911-ER | +| 3-day and event disclosures | deferred | NP3-257-EX, NP3-914-EX, NP3-915-EX, NP3-916-EX, NP3-987-EX | +| 60-day SCED and DAM disclosures, including ESR | deferred | NP3-965-ER, NP3-966-ER | +| COP snapshot and all updates | deferred | NP1-301, NP3-991-EX | +| Legacy SASM disclosures | unavailable | NP3-990-EX | +| Settlement-point and electrical-bus mapping | deferred | NP4-160-SG, NP4-158-SG, NP4-200-CD, NP4-231-CD, NP4-159-CD | +| Public CRR auctions, ownership, and PTP results | deferred | NP7-802-M, NP7-803-M, NP7-535-SG, NP7-536-SG, NP7-157-SG, NP7-464-CD, NP4-194-CD | +| Scarcity, fuel cost, demand response, and integration reports | deferred | NP4-790-CD, NP4-791-CD, NP4-412-CD, NP4-494-ER, NP3-107, NP3-108, NP3-109, NP3-110, NP4-760-ER, NP4-765-ER, EIA-930-ER | +| Load profiles, loss factors, and settlement aggregates | deferred | ZP18-68-M, ZP18-67-M, ZP18-265-SG, NP13-9-SG, NP13-14-SG, NP13-262-SG, NP13-268-SG, NP9-598, NP1-300, COMS-770-SG | +| Adequacy, planning, CDR, MORA and SARA history | deferred | NP3-774-M, NP3-784-M, NP3-773-M, NP3-240-M, PG7-048-M | +| Interconnection, resource lists and DG reports | deferred | PG7-201-ER, NP3-988-ER, NP16-533-M, NP3-823-M, NP12-215-ER, NP16-474-M | +| Frequency events and public notices | deferred | NP12-261-M, NP12-265-M, NP6-87-AN, OPG-453-AN, OPG-158, ZP4-405-M, NP4-50-AN | +| Weather archive contractual boundary | deferred | WEBSITE-WEATHER-1996-2000, NP4-722-CD | +| Unavailable hourly-load year | unavailable | WEBSITE-LOAD-2001 | +| Certified participant settlements and retail usage | restricted | NP9-170-SG, NP9-566-SG, ZP12-245, COMS-448 | +| Secure models, network ratings and ECEII | restricted | NP4-500-SG, NP6-216-ER, NP3-217-CD, NP3-459-SG | +| Participant EWS, telemetry, private bids/COP/awards | restricted | NP4-302-UI, NP4-303-UI | +| Zonal-era and other old public collections | deferred | WEBSITE-ZONAL-ARCHIVES | diff --git a/pyproject.toml b/pyproject.toml index 0abe12e9..d6aa0ea3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,11 @@ Homepage = "https://github.com/kvkenyon/tinyercot" Repository = "https://github.com/kvkenyon/tinyercot" Issues = "https://github.com/kvkenyon/tinyercot/issues" +[project.optional-dependencies] +files = [ + "openpyxl>=3.1.5", +] + [dependency-groups] dev = [ "ipython>=9.8.0", @@ -42,3 +47,4 @@ pythonpath = [".", "tests"] [tool.setuptools.package-data] tinyercot = ["_catalog.json"] +"tinyercot.public" = ["_coverage.json"] diff --git a/tests/fixtures/public/annual-cells.json b/tests/fixtures/public/annual-cells.json new file mode 100644 index 00000000..a82e3e02 --- /dev/null +++ b/tests/fixtures/public/annual-cells.json @@ -0,0 +1,82 @@ +{ + "2010": { + "sheet": "Dec_1", + "rows": [ + [ + "Delivery Date", + "Hour Ending", + "Repeated Hour Flag", + "Settlement Point", + "Settlement Point Price" + ], + [ + "12/01/2010", + "01:00", + "N", + "HB_BUSAVG", + 34.7 + ], + [ + "12/01/2010", + "01:00", + "N", + "HB_HOUSTON", + 35.33 + ], + [ + "12/01/2010", + "01:00", + "N", + "HB_HUBAVG", + 34.64 + ], + [ + "12/01/2010", + "01:00", + "N", + "HB_NORTH", + 34.41 + ] + ] + }, + "2026": { + "sheet": "Aug", + "rows": [ + [ + "Delivery Date", + "Hour Ending", + "Repeated Hour Flag", + "Settlement Point", + "Settlement Point Price" + ], + [ + "08/01/2026", + "01:00", + "N", + "HB_BUSAVG", + 35.15 + ], + [ + "08/01/2026", + "01:00", + "N", + "HB_HOUSTON", + 35.14 + ], + [ + "08/01/2026", + "01:00", + "N", + "HB_HUBAVG", + 35.25 + ], + [ + "08/01/2026", + "01:00", + "N", + "HB_NORTH", + 35.14 + ] + ] + } +} diff --git a/tests/fixtures/public/capacity-current.json b/tests/fixtures/public/capacity-current.json new file mode 100644 index 00000000..3a6fb144 --- /dev/null +++ b/tests/fixtures/public/capacity-current.json @@ -0,0 +1 @@ +{"_meta":{"totalRecords":24,"pageSize":2,"totalPages":12,"currentPage":1,"query":{"parameterCount":3,"parameters":{"deliveryDateFrom":"2026-09-04","deliveryDateTo":"2026-09-04","ancillaryType":"REGUP"},"sortedBy":"deliveryDate: DESC"}},"report":{"reportName":"dam_clear_price_for_cap","reportDisplayName":"DAM Clearing Prices for Capacity","reportId":"12329","reportEMIL":"NP4-188-CD","downloadLimit":2000000},"fields":[{"name":"deliveryDate","label":"Delivery Date","cardinality":1,"dataType":"DATE","searchable":true,"sortable":true,"hasRange":true},{"name":"hourEnding","label":"Hour Ending","cardinality":2,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"ancillaryType","label":"Ancillary Type","cardinality":3,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"MCPC","label":"MCPC","cardinality":4,"dataType":"DOUBLE","searchable":true,"sortable":true,"hasRange":true},{"name":"DSTFlag","label":"DST Flag","cardinality":5,"dataType":"BOOLEAN","searchable":true,"sortable":true,"hasRange":false}],"data":[["2026-09-04","01:00","REGUP",0.51,false],["2026-09-04","02:00","REGUP",0.49,false]],"_links":{"self":{"href":"https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap"},"parent":{"href":"https://api.ercot.com/api/public-reports/np4-188-cd"}}} \ No newline at end of file diff --git a/tests/fixtures/public/capacity-oldest.json b/tests/fixtures/public/capacity-oldest.json new file mode 100644 index 00000000..c54c80f7 --- /dev/null +++ b/tests/fixtures/public/capacity-oldest.json @@ -0,0 +1 @@ +{"_meta":{"totalRecords":23975,"pageSize":1,"totalPages":23975,"currentPage":1,"query":{"parameterCount":1,"parameters":{"ancillaryType":"REGUP"},"sortedBy":"deliveryDate: ASC"}},"report":{"reportName":"dam_clear_price_for_cap","reportDisplayName":"DAM Clearing Prices for Capacity","reportId":"12329","reportEMIL":"NP4-188-CD","downloadLimit":2000000},"fields":[{"name":"deliveryDate","label":"Delivery Date","cardinality":1,"dataType":"DATE","searchable":true,"sortable":true,"hasRange":true},{"name":"hourEnding","label":"Hour Ending","cardinality":2,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"ancillaryType","label":"Ancillary Type","cardinality":3,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"MCPC","label":"MCPC","cardinality":4,"dataType":"DOUBLE","searchable":true,"sortable":true,"hasRange":true},{"name":"DSTFlag","label":"DST Flag","cardinality":5,"dataType":"BOOLEAN","searchable":true,"sortable":true,"hasRange":false}],"data":[["2023-12-13","24:00","REGUP",1.83,false]],"_links":{"self":{"href":"https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap"},"parent":{"href":"https://api.ercot.com/api/public-reports/np4-188-cd"}}} \ No newline at end of file diff --git a/tests/fixtures/public/dam-annual-list.json b/tests/fixtures/public/dam-annual-list.json new file mode 100644 index 00000000..afa80a28 --- /dev/null +++ b/tests/fixtures/public/dam-annual-list.json @@ -0,0 +1 @@ +{"ListDocsByRptTypeRes":{"DocumentList":[{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1356199","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2026","ConstructedName":"rpt.00013060.0000000000000000.20260830.080217640.DAMLZHBSPP_2026.zip","DocID":"1268441709","PublishDate":"2026-08-30T08:02:17-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"2050483","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2025","ConstructedName":"rpt.00013060.0000000000000000.20260101.084409719.DAMLZHBSPP_2025.zip","DocID":"1177667469","PublishDate":"2026-01-01T08:44:09-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"2110124","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2024","ConstructedName":"rpt.00013060.0000000000000000.20250101.080733817.DAMLZHBSPP_2024.zip","DocID":"1065468714","PublishDate":"2025-01-01T08:07:33-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"2081453","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2023","ConstructedName":"rpt.00013060.0000000000000000.20240101.081011828.DAMLZHBSPP_2023.zip","DocID":"969803138","PublishDate":"2024-01-01T08:10:11-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"2106454","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2022","ConstructedName":"rpt.00013060.0000000000000000.20230101.081114733.DAMLZHBSPP_2022.zip","DocID":"886627599","PublishDate":"2023-01-01T08:11:13-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1997884","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2021","ConstructedName":"rpt.00013060.0000000000000000.20220101.081333707.DAMLZHBSPP_2021.zip","DocID":"814918746","PublishDate":"2022-01-01T08:13:33-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"2021142","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2020","ConstructedName":"rpt.00013060.0000000000000000.20210101.080809625.DAMLZHBSPP_2020.zip","DocID":"751351545","PublishDate":"2021-01-01T08:08:09-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"2013424","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2019","ConstructedName":"rpt.00013060.0000000000000000.20200101.080319632.DAMLZHBSPP_2019.zip","DocID":"694281912","PublishDate":"2020-01-01T08:03:19-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1765054","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2018","ConstructedName":"rpt.00013060.0000000000000000.20190101.080239873.DAMLZHBSPP_2018.zip","DocID":"642564845","PublishDate":"2019-01-01T08:02:39-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1773462","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2017","ConstructedName":"rpt.00013060.0000000000000000.20180101.080254200.DAMLZHBSPP_2017.zip","DocID":"592990685","PublishDate":"2018-01-01T08:02:53-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1779055","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2016","ConstructedName":"rpt.00013060.0000000000000000.20170101.080337663.DAMLZHBSPP_2016.zip","DocID":"547013266","PublishDate":"2017-01-01T08:03:36-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1765691","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2015","ConstructedName":"rpt.00013060.0000000000000000.20160101.080309215.DAMLZHBSPP_2015.zip","DocID":"505281022","PublishDate":"2016-01-01T08:03:08-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1752978","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2014","ConstructedName":"rpt.00013060.0000000000000000.20150101.080339110.DAMLZHBSPP_2014.zip","DocID":"468450666","PublishDate":"2015-01-01T08:03:37-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1759810","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2013","ConstructedName":"rpt.00013060.0000000000000000.20140101.080309000.DAMLZHBSPP_2013.zip","DocID":"387408239","PublishDate":"2014-01-01T08:03:09-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1810760","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2012","ConstructedName":"rpt.00013060.0000000000000000.20130301.132227000.DAMLZHBSPP_2012.zip","DocID":"283206776","PublishDate":"2013-03-01T13:22:27-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"1869131","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2011","ConstructedName":"rpt.00013060.0000000000000000.20130301.132053000.DAMLZHBSPP_2011.zip","DocID":"283206764","PublishDate":"2013-03-01T13:20:53-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}},{"Document":{"ExpiredDate":"4712-12-31T00:00:00-06:00","ILMStatus":"EXT","SecurityStatus":"P","ContentSize":"241081","Extension":"zip","ReportTypeID":"13060","Prefix":"rpt","FriendlyName":"DAMLZHBSPP_2010","ConstructedName":"rpt.00013060.0000000000000000.20130301.131718000.DAMLZHBSPP_2010.zip","DocID":"283205860","PublishDate":"2013-03-01T13:17:18-05:00","ReportName":"Historical DAM Load Zone and Hub Prices","DUNS":"0000000000000000","DocCount":"0"}}]}} \ No newline at end of file diff --git a/tests/fixtures/public/dam-current-page1.json b/tests/fixtures/public/dam-current-page1.json new file mode 100644 index 00000000..5853b2dd --- /dev/null +++ b/tests/fixtures/public/dam-current-page1.json @@ -0,0 +1 @@ +{"_meta":{"totalRecords":24,"pageSize":2,"totalPages":12,"currentPage":1,"query":{"parameterCount":3,"parameters":{"deliveryDateFrom":"2026-09-04","deliveryDateTo":"2026-09-04","settlementPoint":"HB_HOUSTON"},"sortedBy":"deliveryDate: DESC"}},"report":{"reportName":"dam_stlmnt_pnt_prices","reportDisplayName":"DAM Settlement Point Prices","reportId":"12331","reportEMIL":"NP4-190-CD","downloadLimit":2000000},"fields":[{"name":"deliveryDate","label":"Delivery Date","cardinality":1,"dataType":"DATE","searchable":true,"sortable":true,"hasRange":true},{"name":"hourEnding","label":"Hour Ending","cardinality":2,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"settlementPoint","label":"Settlement Point","cardinality":3,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"settlementPointPrice","label":"Settlement Point Price","cardinality":4,"dataType":"DOUBLE","searchable":true,"sortable":true,"hasRange":true},{"name":"DSTFlag","label":"DST Flag","cardinality":5,"dataType":"BOOLEAN","searchable":true,"sortable":true,"hasRange":false}],"data":[["2026-09-04","01:00","HB_HOUSTON",28.76,false],["2026-09-04","02:00","HB_HOUSTON",26.25,false]],"_links":{"self":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices"},"parent":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd"}}} \ No newline at end of file diff --git a/tests/fixtures/public/dam-current-page2.json b/tests/fixtures/public/dam-current-page2.json new file mode 100644 index 00000000..9e65f930 --- /dev/null +++ b/tests/fixtures/public/dam-current-page2.json @@ -0,0 +1 @@ +{"_meta":{"totalRecords":24,"pageSize":2,"totalPages":12,"currentPage":2,"query":{"parameterCount":3,"parameters":{"deliveryDateFrom":"2026-09-04","deliveryDateTo":"2026-09-04","settlementPoint":"HB_HOUSTON"},"sortedBy":"deliveryDate: DESC"}},"report":{"reportName":"dam_stlmnt_pnt_prices","reportDisplayName":"DAM Settlement Point Prices","reportId":"12331","reportEMIL":"NP4-190-CD","downloadLimit":2000000},"fields":[{"name":"deliveryDate","label":"Delivery Date","cardinality":1,"dataType":"DATE","searchable":true,"sortable":true,"hasRange":true},{"name":"hourEnding","label":"Hour Ending","cardinality":2,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"settlementPoint","label":"Settlement Point","cardinality":3,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"settlementPointPrice","label":"Settlement Point Price","cardinality":4,"dataType":"DOUBLE","searchable":true,"sortable":true,"hasRange":true},{"name":"DSTFlag","label":"DST Flag","cardinality":5,"dataType":"BOOLEAN","searchable":true,"sortable":true,"hasRange":false}],"data":[["2026-09-04","03:00","HB_HOUSTON",25.87,false],["2026-09-04","04:00","HB_HOUSTON",25.98,false]],"_links":{"self":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices"},"parent":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd"}}} \ No newline at end of file diff --git a/tests/fixtures/public/dam-oldest.json b/tests/fixtures/public/dam-oldest.json new file mode 100644 index 00000000..22cc4941 --- /dev/null +++ b/tests/fixtures/public/dam-oldest.json @@ -0,0 +1 @@ +{"_meta":{"totalRecords":23951,"pageSize":1,"totalPages":23951,"currentPage":1,"query":{"parameterCount":1,"parameters":{"settlementPoint":"HB_HOUSTON"},"sortedBy":"deliveryDate: ASC"}},"report":{"reportName":"dam_stlmnt_pnt_prices","reportDisplayName":"DAM Settlement Point Prices","reportId":"12331","reportEMIL":"NP4-190-CD","downloadLimit":2000000},"fields":[{"name":"deliveryDate","label":"Delivery Date","cardinality":1,"dataType":"DATE","searchable":true,"sortable":true,"hasRange":true},{"name":"hourEnding","label":"Hour Ending","cardinality":2,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"settlementPoint","label":"Settlement Point","cardinality":3,"dataType":"VARCHAR","searchable":true,"sortable":true,"hasRange":false},{"name":"settlementPointPrice","label":"Settlement Point Price","cardinality":4,"dataType":"DOUBLE","searchable":true,"sortable":true,"hasRange":true},{"name":"DSTFlag","label":"DST Flag","cardinality":5,"dataType":"BOOLEAN","searchable":true,"sortable":true,"hasRange":false}],"data":[["2023-12-13","24:00","HB_HOUSTON",16.48,false]],"_links":{"self":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices"},"parent":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd"}}} \ No newline at end of file diff --git a/tests/fixtures/public/dam-product.json b/tests/fixtures/public/dam-product.json new file mode 100644 index 00000000..95665bf8 --- /dev/null +++ b/tests/fixtures/public/dam-product.json @@ -0,0 +1 @@ +{"emilId":"NP4-190-CD","name":"DAM Settlement Point Prices","description":"The Settlement Point Prices for all Resource Nodes, Load Zones, and Trading Hubs from the Day-Ahead Market.","status":"Active","reportTypeId":12331,"audience":"Public","generationFrequency":"Event - Per DAM Run","securityClassification":"Public","lastUpdated":"2024-07-10","firstRun":"2010-11-29","eceii":"","channel":"Public,EWS,Data Portal","userGuide":null,"postingType":"Report","market":"Nodal","extractSubscriber":"","xsdName":"Current Day Reports XSD","misPostingLocation":"","certificateRole":"","fileType":"zip,csv,xml","ddlName":"","misDisplayDuration":31,"archiveDuration":2555,"notificationType":"","contentType":"DATA","downloadLimit":500,"lastPostDatetime":"2026-09-05T12:32:38","bundle":1,"protocolRules":{"NP4.5.3(2)(b)":"https://www.ercot.com/mp/data-products?protocolRules=NP4.5.3(2)(b)"},"artifacts":[{"reportTypeId":12331,"displayName":"DAM Settlement Point Prices","_links":{"endpoint":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices"}}}],"_links":{"self":{"href":"https://api.ercot.com/api/public-reports/np4-190-cd"},"parent":{"href":"https://api.ercot.com/api/public-reports"},"archive":{"href":"https://api.ercot.com/api/public-reports/archive/np4-190-cd"},"bundle":{"href":"https://api.ercot.com/api/public-reports/bundle/np4-190-cd"}}} \ No newline at end of file diff --git a/tests/fixtures/public/esr-live.json b/tests/fixtures/public/esr-live.json new file mode 100644 index 00000000..3835c858 --- /dev/null +++ b/tests/fixtures/public/esr-live.json @@ -0,0 +1,85 @@ +{ + "lastUpdated": "2026-09-05 15:26:01-0500", + "previousDay": { + "dayDate": "2026-09-04 03:00:00-0500", + "data": [ + { + "tagCLastTime": "2026-09-04 00:00:00", + "dstFlag": "N", + "totalCharging": -1258.227, + "totalDischarging": 32.117, + "netOutput": -1226.11, + "timestamp": "2026-09-04 00:00:00-0500", + "epoch": 1788498000000 + }, + { + "tagCLastTime": "2026-09-04 00:05:00", + "dstFlag": "N", + "totalCharging": -1304.835, + "totalDischarging": 35.149, + "netOutput": -1269.687, + "timestamp": "2026-09-04 00:05:00-0500", + "epoch": 1788498300000 + }, + { + "tagCLastTime": "2026-09-04 23:50:00", + "dstFlag": "N", + "totalCharging": -342.519, + "totalDischarging": 1066.38, + "netOutput": 723.901, + "timestamp": "2026-09-04 23:50:00-0500", + "epoch": 1788583800000 + }, + { + "tagCLastTime": "2026-09-04 23:55:00", + "dstFlag": "N", + "totalCharging": -322.159, + "totalDischarging": 902.39, + "netOutput": 580.231, + "timestamp": "2026-09-04 23:55:00-0500", + "epoch": 1788584100000 + } + ] + }, + "currentDay": { + "dayDate": "2026-09-05 03:00:00-0500", + "data": [ + { + "tagCLastTime": "2026-09-05 00:00:00", + "dstFlag": "N", + "totalCharging": -354.183, + "totalDischarging": 813.958, + "netOutput": 459.775, + "timestamp": "2026-09-05 00:00:00-0500", + "epoch": 1788584400000 + }, + { + "tagCLastTime": "2026-09-05 00:05:00", + "dstFlag": "N", + "totalCharging": -276.922, + "totalDischarging": 721.049, + "netOutput": 444.127, + "timestamp": "2026-09-05 00:05:00-0500", + "epoch": 1788584700000 + }, + { + "tagCLastTime": "2026-09-05 15:20:00", + "dstFlag": "N", + "totalCharging": -405.779, + "totalDischarging": 143.732, + "netOutput": -262.047, + "timestamp": "2026-09-05 15:20:00-0500", + "epoch": 1788639600000 + }, + { + "tagCLastTime": "2026-09-05 15:25:00", + "dstFlag": "N", + "totalCharging": -388.095, + "totalDischarging": 132.303, + "netOutput": -255.791, + "timestamp": "2026-09-05 15:25:00-0500", + "epoch": 1788639900000 + } + ] + } +} diff --git a/tests/test_public_api.py b/tests/test_public_api.py new file mode 100644 index 00000000..2d51d738 --- /dev/null +++ b/tests/test_public_api.py @@ -0,0 +1,284 @@ +import datetime +import hashlib +import json +from decimal import Decimal +from pathlib import Path +from urllib.parse import parse_qs + +import httpx +import pytest + +from tinyercot.public import ( + AccessDeniedError, + AuthenticationError, + Credentials, + Limits, + PublicClient, + SchemaMismatchError, +) +from tinyercot.public._http import Payload, Receipt +from tinyercot.public.api import PRICE_PATH, TOKEN_URL, decode_prices + +FIXTURES = Path(__file__).parent / "fixtures/public" +CREDS = Credentials( + "synthetic+user@example.test", "synthetic&password=+", "synthetic-key" +) + + +def payload(body): + raw = json.dumps(body).encode() + return Payload( + raw, + Receipt( + "https://api.ercot.com/public-test", + datetime.datetime.now(datetime.UTC), + hashlib.sha256(raw).hexdigest(), + len(raw), + ), + ) + + +@pytest.mark.parametrize( + "name", ["dam-current-page1", "dam-current-page2", "dam-oldest"] +) +def test_observed_current_and_oldest_rows_decode(name): + body = json.loads((FIXTURES / f"{name}.json").read_text()) + response = decode_prices(payload(body), expected_page=body["_meta"]["currentPage"]) + assert response.rows and isinstance(response.rows[0].settlementPointPrice, Decimal) + assert response.rows[0].settlementPoint == "HB_HOUSTON" + if name == "dam-oldest": + assert response.rows[0].deliveryDate == datetime.date(2023, 12, 13) + assert response.rows[0].hourEnding == "24:00" + + +def test_reordered_fields_decode_by_declared_names(): + body = json.loads((FIXTURES / "dam-current-page1.json").read_text()) + expected = decode_prices(payload(body), expected_page=1).rows + body["fields"].reverse() + for row in body["data"]: + row.reverse() + assert decode_prices(payload(body), expected_page=1).rows == expected + + +@pytest.mark.parametrize( + "change", + [ + "missing_field", + "extra_field", + "duplicate_field", + "field_type", + "short_row", + "long_row", + "null", + "extra_key", + "bool_string", + "numeric_string", + "wrong_page", + "missing_meta", + "error", + "not_json", + ], +) +def test_unknown_current_shapes_fail_closed(change): + body = json.loads((FIXTURES / "dam-current-page1.json").read_text()) + if change == "missing_field": + body["fields"].pop() + elif change == "extra_field": + body["fields"].append({"name": "new", "dataType": "VARCHAR"}) + elif change == "duplicate_field": + body["fields"].append(body["fields"][0]) + elif change == "field_type": + body["fields"][0]["dataType"] = "VARCHAR" + elif change == "short_row": + body["data"][0].pop() + elif change == "long_row": + body["data"][0].append("extra") + elif change == "null": + body["data"][0][3] = None + elif change == "bool_string": + body["data"][0][4] = "false" + elif change == "numeric_string": + body["data"][0][3] = "28.76" + elif change == "extra_key": + body["data"][0] = { + **dict( + zip([f["name"] for f in body["fields"]], body["data"][0], strict=True) + ), + "new": 1, + } + elif change == "wrong_page": + body["_meta"]["currentPage"] = 2 + elif change == "missing_meta": + body.pop("_meta") + elif change == "error": + body = {"error": "unauthorized"} + result = payload(body) + if change == "not_json": + result = Payload(b"error", result.receipt) + with pytest.raises(SchemaMismatchError): + decode_prices(result, expected_page=1) + + +def test_form_auth_pagination_refresh_and_receipts(): + calls = [] + token_count = 0 + + def handler(request): + nonlocal token_count + assert CREDS.password not in str(request.url) + calls.append(request) + if str(request.url) == TOKEN_URL: + token_count += 1 + fields = parse_qs(request.content.decode()) + assert fields["username"] == [CREDS.username] and fields["password"] == [ + CREDS.password + ] + return httpx.Response( + 200, + json={"id_token": f"synthetic-token-{token_count}", "expires_in": 3600}, + ) + assert request.url.path.endswith(PRICE_PATH) + page = request.url.params["page"] + return httpx.Response( + 200, content=(FIXTURES / f"dam-current-page{page}.json").read_bytes() + ) + + with PublicClient( + CREDS, limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + assert calls == [] + pages = list( + client.price_pages( + start=datetime.date(2026, 9, 4), + end=datetime.date(2026, 9, 4), + settlement_point="HB_HOUSTON", + size=2, + max_pages=2, + ) + ) + assert len(pages) == 2 and sum(len(p.rows) for p in pages) == 4 + assert token_count == 1 and len(calls) == 3 + assert pages[0].receipt.sha256 != pages[1].receipt.sha256 + client.refresh_token() + assert token_count == 2 + client.dam_prices(settlement_point="HB_HOUSTON", size=2) + assert calls[-1].headers["Authorization"] == "Bearer synthetic-token-2" + assert CREDS.password not in repr(CREDS) + assert all(CREDS.password not in repr(p.receipt) for p in pages) + assert client._token is None + + +@pytest.mark.parametrize( + "status, expected, auth_count", + [(401, AuthenticationError, 2), (403, AccessDeniedError, 1)], +) +def test_terminal_access_errors_do_not_loop_or_expose_secrets( + status, expected, auth_count +): + tokens = [] + + def handler(request): + if str(request.url) == TOKEN_URL: + tokens.append(1) + return httpx.Response( + 200, json={"id_token": "secret-token", "expires_in": 3600} + ) + return httpx.Response(status, json={"secret": CREDS.password}) + + with PublicClient( + CREDS, limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + with pytest.raises(expected) as caught: + client.dam_prices(settlement_point="HB_HOUSTON", size=1) + assert len(tokens) == auth_count + assert CREDS.password not in str(caught.value) and "secret-token" not in str( + caught.value + ) + + +def test_one_401_reacquisition_and_expiry_skew(): + tokens, gets = [], [] + + def handler(request): + if str(request.url) == TOKEN_URL: + tokens.append(1) + return httpx.Response( + 200, json={"id_token": "synthetic", "expires_in": 3600} + ) + gets.append(1) + if len(gets) == 1: + return httpx.Response(401) + return httpx.Response( + 200, content=(FIXTURES / "dam-current-page1.json").read_bytes() + ) + + with PublicClient( + CREDS, limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + now = [100.0] + client._http.clock = lambda: now[0] + client.dam_prices(settlement_point="HB_HOUSTON", size=2) + assert len(tokens) == 2 and len(gets) == 2 + now[0] = 3641 + client.dam_prices(settlement_point="HB_HOUSTON", size=2) + assert len(tokens) == 3 + + +def test_current_generator_is_independent_and_hash_pinned(monkeypatch, tmp_path): + from tools import generate_public + + assert generate_public.render() == generate_public.OUTPUT.read_text() + pin = tmp_path / "dam-prices.json" + pin.write_text("{}") + monkeypatch.setattr(generate_public, "INPUT", pin) + with pytest.raises(ValueError, match="Unverified"): + generate_public.render() + + +@pytest.mark.parametrize("fixture", ["capacity-current", "capacity-oldest"]) +def test_capacity_endpoint_uses_its_own_pinned_schema(fixture): + from tinyercot.public import DamCapacityPrice + + calls = [] + + def handler(request): + calls.append(request) + if str(request.url) == TOKEN_URL: + return httpx.Response( + 200, json={"id_token": "synthetic", "expires_in": 3600} + ) + assert request.url.path.endswith("/np4-188-cd/dam_clear_price_for_cap") + assert request.url.params["ancillaryType"] == "REGUP" + return httpx.Response(200, content=(FIXTURES / f"{fixture}.json").read_bytes()) + + with PublicClient( + CREDS, limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + page = client.dam_capacity_prices( + ancillary_type="REGUP", size=2, oldest_first=fixture.endswith("oldest") + ) + assert page.rows and isinstance(page.rows[0], DamCapacityPrice) + assert isinstance(page.rows[0].MCPC, Decimal) + assert page.rows[0].ancillaryType == "REGUP" + if fixture.endswith("oldest"): + assert page.rows[0].deliveryDate == datetime.date(2023, 12, 13) + assert len(calls) == 2 + + +def test_capacity_endpoint_rejects_settlement_price_schema(): + def handler(request): + if str(request.url) == TOKEN_URL: + return httpx.Response( + 200, json={"id_token": "synthetic", "expires_in": 3600} + ) + return httpx.Response( + 200, content=(FIXTURES / "dam-current-page1.json").read_bytes() + ) + + with ( + PublicClient( + CREDS, limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client, + pytest.raises(SchemaMismatchError), + ): + client.dam_capacity_prices(ancillary_type="REGUP", size=2) diff --git a/tests/test_public_archives.py b/tests/test_public_archives.py new file mode 100644 index 00000000..f3a0cd70 --- /dev/null +++ b/tests/test_public_archives.py @@ -0,0 +1,183 @@ +import datetime +import io +import json +import zipfile +from dataclasses import replace +from pathlib import Path + +import httpx +import pytest + +from tinyercot.public import ( + AccessDeniedError, + Limits, + SchemaMismatchError, + WebClient, + sample_dam_archive, +) +from tinyercot.public._http import LimitError +from tinyercot.public.archives import checked_zip +from tinyercot.public.web import DOWNLOAD_URL, LIST_URL + +FIXTURES = Path(__file__).parent / "fixtures/public" + + +def workbook_zip(*, extra=False, bad_header=False): + openpyxl = pytest.importorskip("openpyxl") + book = openpyxl.Workbook() + sheet = book.active + sheet.title = "Dec_1" + # Cells transcribed from the first four rows of the real 2010 source file. + rows = json.loads((FIXTURES / "annual-cells.json").read_text())["2010"]["rows"] + if bad_header: + rows[0][0] = "Unknown column" + for row in rows: + sheet.append([*row, "unexpected"] if extra else row) + inner = io.BytesIO() + book.save(inner) + book.close() + outer = io.BytesIO() + with zipfile.ZipFile(outer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("DAMLZHBSPP_2010.xlsx", inner.getvalue()) + return outer.getvalue() + + +def source_fixture(raw): + body = json.loads((FIXTURES / "dam-annual-list.json").read_text()) + entry = next( + e + for e in body["ListDocsByRptTypeRes"]["DocumentList"] + if e["Document"]["FriendlyName"] == "DAMLZHBSPP_2010" + ) + entry["Document"]["ContentSize"] = str(len(raw)) + body["ListDocsByRptTypeRes"]["DocumentList"] = [entry] + return body + + +def test_download_decode_receipt_cache_and_revision_identity(tmp_path): + raw = workbook_zip() + listing = source_fixture(raw) + calls = [] + + def handler(request): + calls.append(request) + assert "Authorization" not in request.headers + if str(request.url).startswith(LIST_URL): + return httpx.Response(200, json=listing) + assert str(request.url).startswith(DOWNLOAD_URL) + return httpx.Response(200, content=raw) + + with WebClient( + limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + documents, receipt = client.dam_archives() + assert receipt.sha256 and len(documents) == 1 + download = client.download_dam_archive(documents[0], cache=tmp_path) + assert download.path.read_bytes() == raw + sample = sample_dam_archive(download, sheet="Dec_1", max_rows=2) + assert sample.rows[0].delivery_date == datetime.date(2010, 12, 1) + assert str(sample.rows[1].settlement_point_price) == "35.33" + assert sample.truncated + cached = client.download_dam_archive(documents[0], cache=tmp_path) + assert ( + cached.cache_hit and cached.receipt == download.receipt and len(calls) == 2 + ) + revised = replace( + documents[0], + published_at=documents[0].published_at + datetime.timedelta(days=1), + ) + with pytest.raises(AccessDeniedError): + client.download_dam_archive(revised, cache=tmp_path) + download.path.write_bytes(b"tampered") + with pytest.raises(SchemaMismatchError): + client.download_dam_archive(documents[0], cache=tmp_path) + assert len(calls) == 2 + + +@pytest.mark.parametrize("security", ["C", "S", "", "unknown"]) +def test_nonpublic_listing_never_allows_download(security): + body = json.loads((FIXTURES / "dam-annual-list.json").read_text()) + body["ListDocsByRptTypeRes"]["DocumentList"][0]["Document"]["SecurityStatus"] = ( + security + ) + with ( + WebClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json=body) + ) + ) as client, + pytest.raises(AccessDeniedError), + ): + client.dam_archives() + + +@pytest.mark.parametrize( + "bad", + ["html", "traversal", "absolute", "duplicate", "symlink", "members", "expanded"], +) +def test_archive_structure_fails_closed(bad): + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + if bad == "traversal": + archive.writestr("../outside", b"x") + elif bad == "absolute": + archive.writestr("/outside", b"x") + elif bad == "duplicate": + archive.writestr("same", b"x") + with pytest.warns(UserWarning): + archive.writestr("same", b"y") + elif bad == "symlink": + item = zipfile.ZipInfo("link") + item.external_attr = 0o120777 << 16 + archive.writestr(item, b"outside") + elif bad == "members": + archive.writestr("one", b"x") + archive.writestr("two", b"y") + else: + archive.writestr("one", b"too large") + raw = b"No Document" if bad == "html" else stream.getvalue() + with pytest.raises((SchemaMismatchError, LimitError)): + checked_zip( + raw, max_members=1, max_expanded_bytes=1 if bad == "expanded" else 100 + ) + + +@pytest.mark.parametrize("bad", ["extra", "header"]) +def test_unknown_workbook_columns_are_not_silently_dropped(tmp_path, bad): + raw = workbook_zip(extra=bad == "extra", bad_header=bad == "header") + listing = source_fixture(raw) + + def handler(request): + return ( + httpx.Response(200, json=listing) + if str(request.url).startswith(LIST_URL) + else httpx.Response(200, content=raw) + ) + + with WebClient( + limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + documents, _ = client.dam_archives() + downloaded = client.download_dam_archive(documents[0], cache=tmp_path) + with pytest.raises(SchemaMismatchError): + sample_dam_archive(downloaded, sheet="Dec_1") + + +def test_failed_transfer_and_html_do_not_create_complete_receipt(tmp_path): + raw = b"No Document" + listing = source_fixture(raw) + + def handler(request): + return ( + httpx.Response(200, json=listing) + if str(request.url).startswith(LIST_URL) + else httpx.Response(200, content=raw) + ) + + with WebClient( + limits=Limits(min_interval=0), transport=httpx.MockTransport(handler) + ) as client: + documents, _ = client.dam_archives() + with pytest.raises(SchemaMismatchError): + client.download_dam_archive(documents[0], cache=tmp_path) + assert list(tmp_path.iterdir()) == [] diff --git a/tests/test_public_coverage.py b/tests/test_public_coverage.py new file mode 100644 index 00000000..9678d42f --- /dev/null +++ b/tests/test_public_coverage.py @@ -0,0 +1,43 @@ +from collections import Counter + +from tinyercot.catalog import operations +from tinyercot.public import coverage + + +def test_every_audited_family_and_operation_has_an_explicit_scope(): + entries = coverage() + assert len(entries) == 39 + 257 + 2 + assert len({entry.key for entry in entries}) == len(entries) + assert Counter(entry.status for entry in entries)["covered"] == 4 + assert {entry.status for entry in entries} == { + "covered", + "deferred", + "restricted", + "unavailable", + } + assert all(entry.source_urls and entry.scope for entry in entries) + family = [entry for entry in entries if entry.key.startswith("family:")] + assert len(family) == 39 and all(entry.status != "covered" for entry in family) + assert all( + entry.status == "restricted" + for entry in family + if "Certified participant" in entry.title + or "Secure models" in entry.title + or "Participant EWS" in entry.title + ) + + +def test_typed_counts_do_not_claim_broad_api_coverage(): + observed = [operation for operation in operations() if operation.kind == "data"] + typed = [ + entry + for entry in coverage() + if entry.key.startswith("api:") and entry.status == "covered" + ] + assert len(observed) == 243 + assert len({(op.service, op.path.split("/")[1]) for op in observed}) == 98 + assert len(typed) == 2 + assert {entry.title.split("/")[1] for entry in typed} == { + "np4-190-cd", + "np4-188-cd", + } diff --git a/tests/test_public_http.py b/tests/test_public_http.py new file mode 100644 index 00000000..24a7bd28 --- /dev/null +++ b/tests/test_public_http.py @@ -0,0 +1,129 @@ +import datetime +from email.utils import format_datetime + +import httpx +import pytest + +from tinyercot.public import LimitError, Limits, RateLimitError, SourceUnavailableError +from tinyercot.public._http import _HTTP + + +@pytest.mark.parametrize("status", [429, 502, 503, 504]) +def test_bounded_retry_honors_wait_and_no_terminal_sleep(status): + calls, sleeps = [], [] + + def handler(request): + calls.append(request) + return httpx.Response(status, headers={"Retry-After": "3"}) + + client = _HTTP( + Limits(min_interval=0), httpx.MockTransport(handler), sleep=sleeps.append + ) + try: + with pytest.raises(RateLimitError if status == 429 else SourceUnavailableError): + client.request("GET", "https://www.ercot.com/test") + assert len(calls) == 3 and sleeps == [3, 3] + finally: + client.close() + + +def test_retry_after_http_date_and_excessive_wait(): + client = _HTTP(Limits()) + try: + future = datetime.datetime.now(datetime.UTC) + datetime.timedelta(seconds=20) + assert 18 <= client._retry_delay(format_datetime(future, usegmt=True), 0) <= 20 + with pytest.raises(RateLimitError): + client._retry_delay("120", 0) + with pytest.raises(RateLimitError): + client._retry_delay("NaN", 0) + finally: + client.close() + + +def test_request_budget_byte_budget_and_pacing(): + sleeps, calls = [], [] + + def handler(request): + calls.append(request) + return httpx.Response(200, content=b"1234") + + client = _HTTP( + Limits(max_requests=2, max_bytes=3), + httpx.MockTransport(handler), + clock=lambda: 0, + sleep=sleeps.append, + ) + try: + for _ in range(2): + with pytest.raises(LimitError, match="byte"): + client.request("GET", "https://www.ercot.com/test") + with pytest.raises(LimitError, match="request"): + client.request("GET", "https://www.ercot.com/test") + assert sleeps == [2.1] and len(calls) == 2 + finally: + client.close() + + +def test_transport_error_is_redacted(): + def error(request): + raise httpx.ReadTimeout("synthetic-secret", request=request) + + client = _HTTP(Limits(attempts=1), httpx.MockTransport(error)) + try: + with pytest.raises(SourceUnavailableError) as caught: + client.request( + "POST", + "https://example.test/token", + data={"password": "synthetic-secret"}, + authentication=True, + ) + assert "synthetic-secret" not in str(caught.value) + assert caught.value.__context__ is None + finally: + client.close() + + +def test_interrupted_auth_body_retries_without_retaining_partial_token(): + class BrokenBody(httpx.SyncByteStream): + def __iter__(self): + yield b'{"id_token":"synthetic-partial' + raise httpx.ReadTimeout("synthetic-secret") + + calls, sleeps = [], [] + + def handler(request): + calls.append(request) + if len(calls) == 1: + return httpx.Response(200, stream=BrokenBody()) + return httpx.Response(200, json={"id_token": "synthetic-complete"}) + + client = _HTTP( + Limits(min_interval=0), httpx.MockTransport(handler), sleep=sleeps.append + ) + try: + result = client.request( + "POST", "https://example.test/token", authentication=True + ) + assert result.json() == {"id_token": "synthetic-complete"} + assert result.receipt.sha256 == "" and result.receipt.byte_count == 0 + assert len(calls) == 2 and sleeps == [1] + finally: + client.close() + + +def test_redirect_is_not_followed(): + calls = [] + + def redirect(request): + calls.append(request) + return httpx.Response( + 302, headers={"Location": "https://restricted.example.test"} + ) + + client = _HTTP(Limits(), httpx.MockTransport(redirect)) + try: + with pytest.raises(SourceUnavailableError): + client.request("GET", "https://www.ercot.com/test") + assert len(calls) == 1 + finally: + client.close() diff --git a/tests/test_public_live.py b/tests/test_public_live.py new file mode 100644 index 00000000..1c1607f1 --- /dev/null +++ b/tests/test_public_live.py @@ -0,0 +1,95 @@ +import datetime +import hashlib +import json +from pathlib import Path + +import httpx +import pytest + +from tinyercot.public import SchemaMismatchError, WebClient +from tinyercot.public._http import Payload, Receipt +from tinyercot.public.live import EsrRow, decode_esr + +RAW = (Path(__file__).parent / "fixtures/public/esr-live.json").read_bytes() +RECEIPT = Receipt( + "https://www.ercot.com/api/1/services/read/dashboards/energy-storage-resources.json", + datetime.datetime(2026, 9, 5, 20, 27, 20, tzinfo=datetime.UTC), + hashlib.sha256(RAW).hexdigest(), + len(RAW), +) + + +def test_real_feed_fixture_and_freshness(): + result = decode_esr(Payload(RAW, RECEIPT)) + assert len(result.current_day.data) == 4 and len(result.previous_day.data) == 4 + assert not result.is_stale() + assert result.is_stale(max_age=datetime.timedelta(seconds=1)) + assert result.current_day.data[0].utc == datetime.datetime( + 2026, 9, 5, 5, tzinfo=datetime.UTC + ) + assert result.current_day.data[0].dstFlag == "N" + assert result.current_day.dayDate == "2026-09-05 03:00:00-0500" + + +@pytest.mark.parametrize( + "timestamp, epoch", + [ + ("2026-11-01 01:30:00-0500", 1793514600000), + ("2026-11-01 01:30:00-0600", 1793518200000), + ("2026-03-08 01:55:00-0600", 1772956500000), + ("2026-03-08 03:00:00-0500", 1772956800000), + ], +) +def test_offset_epoch_disambiguate_dst_without_guessing_flag(timestamp, epoch): + row = EsrRow( + tagCLastTime=timestamp[:19], + timestamp=timestamp, + epoch=epoch, + dstFlag="unknown-meaning", + totalCharging=-1, + totalDischarging=2, + netOutput=1, + ) + assert int(row.utc.timestamp() * 1000) == epoch + assert row.dstFlag == "unknown-meaning" + + +@pytest.mark.parametrize( + "change", + ["epoch", "offset", "flag_type", "missing", "extra", "duplicate", "null", "local"], +) +def test_feed_drift_and_conflicting_times_fail_closed(change): + body = json.loads(RAW) + row = body["currentDay"]["data"][0] + if change == "epoch": + row["epoch"] += 1 + elif change == "offset": + row["timestamp"] = row["timestamp"].replace("-0500", "-0600") + elif change == "flag_type": + row["dstFlag"] = False + elif change == "missing": + row.pop("totalCharging") + elif change == "extra": + row["new"] = 1 + elif change == "duplicate": + body["currentDay"]["data"].append(row) + elif change == "null": + row["netOutput"] = None + elif change == "local": + row["tagCLastTime"] = "2026-09-05 02:00:00" + with pytest.raises(SchemaMismatchError): + decode_esr(Payload(json.dumps(body).encode(), RECEIPT)) + + +def test_web_fetch_is_anonymous_and_only_one_capture(): + calls = [] + + def handler(request): + calls.append(request) + assert "Authorization" not in request.headers + assert "Ocp-Apim-Subscription-Key" not in request.headers + return httpx.Response(200, content=RAW) + + with WebClient(transport=httpx.MockTransport(handler)) as client: + assert client.esr().current_day.data + assert len(calls) == 1 diff --git a/tests/test_public_live_opt_in.py b/tests/test_public_live_opt_in.py new file mode 100644 index 00000000..b8cc0b19 --- /dev/null +++ b/tests/test_public_live_opt_in.py @@ -0,0 +1,41 @@ +"""Explicit integration entry point; ordinary test runs never load credentials.""" + +import os +import subprocess +from pathlib import Path + +import pytest + + +@pytest.mark.skipif( + os.environ.get("TINYERCOT_LIVE_TESTS") != "1", + reason="Real public requests require TINYERCOT_LIVE_TESTS=1 and an installed wheel", +) +def test_installed_public_sources(): + """Run the bounded installed-wheel probe only on explicit operator request.""" + required = ( + "TINYERCOT_INSTALLED_PYTHON", + "TINYERCOT_CREDENTIALS_FILE", + "TINYERCOT_PROBE_OUTPUT", + ) + if not all(os.environ.get(name) for name in required): + pytest.fail("The three documented integration paths are required") + root = Path(__file__).resolve().parents[1] + result = subprocess.run( + [ + os.environ[required[0]], + "-I", + str(root / "tools/probe_public.py"), + "--live", + "--credentials-file", + os.environ[required[1]], + "--output", + os.environ[required[2]], + ], + capture_output=True, + timeout=300, + check=False, + ) + assert result.returncode == 0, ( + "Installed probe failed; inspect public receipt status" + ) diff --git a/tests/test_public_provenance.py b/tests/test_public_provenance.py new file mode 100644 index 00000000..4723ddae --- /dev/null +++ b/tests/test_public_provenance.py @@ -0,0 +1,34 @@ +"""Check that compact public fixtures and current generation share evidence.""" + +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_every_public_fixture_has_a_hash_and_source_transformation(): + records = json.loads((ROOT / "docs/evidence/fixture-provenance.json").read_text())[ + "fixtures" + ] + fixtures = ROOT / "tests/fixtures/public" + assert {r["fixture"] for r in records} == {p.name for p in fixtures.glob("*.json")} + for record in records: + digest = hashlib.sha256((fixtures / record["fixture"]).read_bytes()).hexdigest() + assert digest == record["sha256"] + assert record["transformation"] + if record["transformation"] == "Exact public response bytes.": + assert record["source"]["sha256"] == digest + + +def test_current_model_fields_come_from_retained_response_bytes(): + for contract_name, fixture_name in ( + ("dam-prices", "dam-current-page1"), + ("dam-capacity", "capacity-current"), + ): + contract = json.loads( + (ROOT / f"tools/inputs/current/{contract_name}.json").read_text() + ) + raw = (ROOT / f"tests/fixtures/public/{fixture_name}.json").read_bytes() + assert hashlib.sha256(raw).hexdigest() == contract["response_source"]["sha256"] + assert json.loads(raw)["fields"] == contract["fields"] diff --git a/tests/wheel_smoke.py b/tests/wheel_smoke.py index f4146389..28aea408 100644 --- a/tests/wheel_smoke.py +++ b/tests/wheel_smoke.py @@ -34,4 +34,9 @@ def deny_network(*args, **kwargs): assert len(tinyercot._generated.__all__) == 35 assert tinyercot.np3_910_er._2d_agg_dsr_loads assert tinyercot.np4_190_cd.DamStlmntPntPricesResponse().to_df().columns.empty -print("Installed-wheel legacy imports and offline catalog passed") +public = importlib.import_module("tinyercot.public") +assert len(public.coverage()) == 298 +assert public.Credentials("test", "test", "test") +with public.WebClient(): + pass +print("Installed-wheel legacy imports and opt-in public metadata passed") diff --git a/tinyercot/catalog.py b/tinyercot/catalog.py index 18bca098..87d87a1d 100644 --- a/tinyercot/catalog.py +++ b/tinyercot/catalog.py @@ -50,7 +50,7 @@ def classify_access(security_classification: str | None) -> Access: @dataclass(frozen=True) class Operation: - """An observed API operation with no current retrieval implementation. + """An observed API operation in the immutable Stage-0 metadata snapshot. Attributes: service: ERCOT route namespace: public-reports or public-data (ESR). @@ -66,7 +66,8 @@ class Operation: source_url: Public OpenAPI export used for this observation. observed_at: Audit date; not a data publication or availability date. access: Public source classification; API authentication is still needed. - support: Metadata only. Typed and raw current retrieval are unsupported. + support: Metadata only. See tinyercot.public.coverage for opt-in + retrieval support. """ service: str @@ -92,8 +93,8 @@ def operations(*, service: str | None = None) -> tuple[Operation, ...]: An unknown namespace returns an empty tuple. Returns: - Immutable observations from 2026-09-05. All current retrieval remains - unsupported, including paths that have a legacy method or cached fields. + Immutable Stage-0 observations from 2026-09-05. These records do not + establish retrieval support. See tinyercot.public.coverage for that. """ snapshot = json.loads(files("tinyercot").joinpath("_catalog.json").read_text()) return tuple( diff --git a/tinyercot/public/__init__.py b/tinyercot/public/__init__.py new file mode 100644 index 00000000..60e1ad4e --- /dev/null +++ b/tinyercot/public/__init__.py @@ -0,0 +1,56 @@ +"""Opt-in bounded retrieval of selected public ERCOT data sources. + +Use PublicClient for current DAM API prices and WebClient for public annual +files and the ESR live feed. Legacy tinyercot imports and behavior are unchanged. +""" + +from ._generated import DamCapacityPrice, DamPrice +from ._http import ( + AccessDeniedError, + AuthenticationError, + LimitError, + Limits, + PublicDataError, + RateLimitError, + Receipt, + SchemaMismatchError, + SourceUnavailableError, +) +from .api import Credentials, PricePage, PublicClient +from .archives import ( + ArchiveDocument, + ArchivePrice, + ArchiveSample, + Download, + sample_dam_archive, +) +from .coverage import Coverage, coverage +from .live import EsrRow, EsrSnapshot +from .web import WebClient + +__all__ = [ + "AccessDeniedError", + "ArchiveDocument", + "ArchivePrice", + "ArchiveSample", + "AuthenticationError", + "Coverage", + "Credentials", + "DamCapacityPrice", + "DamPrice", + "Download", + "EsrRow", + "EsrSnapshot", + "LimitError", + "Limits", + "PricePage", + "PublicClient", + "PublicDataError", + "RateLimitError", + "Receipt", + "SchemaMismatchError", + "SourceUnavailableError", + "WebClient", + "coverage", + "sample_dam_archive", +] diff --git a/tinyercot/public/_coverage.json b/tinyercot/public/_coverage.json new file mode 100644 index 00000000..55f21107 --- /dev/null +++ b/tinyercot/public/_coverage.json @@ -0,0 +1,778 @@ +{ + "observed_at": "2026-09-05", + "audit_family_sha256": "9e227a6c87560d89975e417d542dd8f974dd179f8dd4490356c5b8e261318bef", + "families": [ + { + "key": "family:DAM prices, lambda, constraints", + "title": "DAM prices, lambda, constraints", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-183-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-188-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-190-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-191-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-523-cd" + ], + "product_ids": [ + "NP4-183-CD", + "NP4-188-CD", + "NP4-190-CD", + "NP4-191-CD", + "NP4-523-CD" + ] + }, + { + "key": "family:RT SCED prices and constraints", + "title": "RT SCED prices and constraints", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-787-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-788-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-322-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-86-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-905-cd" + ], + "product_ids": [ + "NP6-787-CD", + "NP6-788-CD", + "NP6-322-CD", + "NP6-86-CD", + "NP6-905-CD" + ] + }, + { + "key": "family:Price corrections and investigations", + "title": "Price corrections and investigations", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-196-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-197-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-46-an", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-47-an", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-48-an", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-49-an" + ], + "product_ids": [ + "NP4-196-M", + "NP4-197-M", + "NP4-46-AN", + "NP4-47-AN", + "NP4-48-AN", + "NP4-49-AN" + ] + }, + { + "key": "family:Historical hub/zone and AS price files", + "title": "Historical hub/zone and AS price files", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-180-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-785-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-181-er" + ], + "product_ids": [ + "NP4-180-ER", + "NP6-785-ER", + "NP4-181-ER" + ] + }, + { + "key": "family:RTD indicative prices and capacity prices", + "title": "RTD indicative prices and capacity prices", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-970-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-325-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-329-cd" + ], + "product_ids": [ + "NP6-970-CD", + "NP6-325-CD", + "NP6-329-CD" + ] + }, + { + "key": "family:RTC+B ancillary prices, capability, and SOG", + "title": "RTC+B ancillary prices, capability, and SOG", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-323-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-324-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-326-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-327-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-328-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-331-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-332-cd" + ], + "product_ids": [ + "NP6-323-CD", + "NP6-324-CD", + "NP6-326-CD", + "NP6-327-CD", + "NP6-328-CD", + "NP6-331-CD", + "NP6-332-CD" + ] + }, + { + "key": "family:Historical RT adders and RTC+B archives", + "title": "Historical RT adders and RTC+B archives", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-792-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-793-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-794-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-795-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-796-er" + ], + "product_ids": [ + "NP6-792-ER", + "NP6-793-ER", + "NP6-794-ER", + "NP6-795-ER", + "NP6-796-ER" + ] + }, + { + "key": "family:DAM plans, energy totals, AS offers/demand curves", + "title": "DAM plans, energy totals, AS offers/demand curves", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-33-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-19-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-192-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-193-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-212-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-532-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np1-302" + ], + "product_ids": [ + "NP4-33-CD", + "NP4-19-CD", + "NP4-192-CD", + "NP4-193-CD", + "NP4-212-CD", + "NP4-532-CD", + "NP1-302" + ] + }, + { + "key": "family:Legacy total AS offers", + "title": "Legacy total AS offers", + "status": "unavailable", + "scope": "Retired for new publication per ERCOT notice. Last data 2025-12-05 posted 2025-12-04. Current OpenAPI retains path; current EMIL omits product. Preserve import for history.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-179-cd" + ], + "product_ids": [ + "NP4-179-CD" + ] + }, + { + "key": "family:RUC demand curves, deployment factors, constraints", + "title": "RUC demand curves, deployment factors, constraints", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-213-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-214-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-215-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-525-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-526-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-527-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-528-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-520-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-753-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-754-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-755-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np5-108-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-764-cd" + ], + "product_ids": [ + "NP4-213-CD", + "NP4-214-CD", + "NP4-215-CD", + "NP5-525-CD", + "NP5-526-CD", + "NP5-527-CD", + "NP5-528-CD", + "NP5-520-ER", + "NP5-753-CD", + "NP5-754-CD", + "NP5-755-CD", + "NP5-108-CD", + "NP3-764-CD" + ] + }, + { + "key": "family:System and zonal actual load", + "title": "System and zonal actual load", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-345-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-346-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-344-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-235-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-55-cd" + ], + "product_ids": [ + "NP6-345-CD", + "NP6-346-CD", + "NP6-344-CD", + "NP6-235-CD", + "GEN-55-CD" + ] + }, + { + "key": "family:Load forecasts", + "title": "Load forecasts", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-565-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-566-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-560-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-561-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-562-cd" + ], + "product_ids": [ + "NP3-565-CD", + "NP3-566-CD", + "NP3-560-CD", + "NP3-561-CD", + "NP3-562-CD" + ] + }, + { + "key": "family:Long hourly-load archive", + "title": "Long hourly-load archive", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/gridinfo/load/load_hist" + ], + "product_ids": [ + "WEBSITE-LOAD-HISTORY" + ] + }, + { + "key": "family:Wind and solar actual/forecast series", + "title": "Wind and solar actual/forecast series", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-732-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-733-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-737-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-738-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-742-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-743-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-745-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-746-cd" + ], + "product_ids": [ + "NP4-732-CD", + "NP4-733-CD", + "NP4-737-CD", + "NP4-738-CD", + "NP4-742-CD", + "NP4-743-CD", + "NP4-745-CD", + "NP4-746-CD" + ] + }, + { + "key": "family:Renewable forecast models and intra-hour forecasts", + "title": "Renewable forecast models and intra-hour forecasts", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-442-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-443-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-751-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-752-cd" + ], + "product_ids": [ + "NP4-442-CD", + "NP4-443-CD", + "NP4-751-CD", + "NP4-752-CD" + ] + }, + { + "key": "family:Storage four-second Public Data API", + "title": "Storage four-second Public Data API", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://apiexplorer.ercot.com/api-details#api=esrapi-apim-api" + ], + "product_ids": [ + "RPTESR-M" + ] + }, + { + "key": "family:Live storage, fuel mix, and generation outages", + "title": "Live storage, fuel mix, and generation outages", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-545-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-544-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-546-ui" + ], + "product_ids": [ + "GEN-545-UI", + "GEN-544-UI", + "GEN-546-UI" + ] + }, + { + "key": "family:Live grid conditions, reserves, supply and demand", + "title": "Live grid conditions, reserves, supply and demand", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-530-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-518-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-506-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-536-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-904-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-906-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-547-ui" + ], + "product_ids": [ + "GEN-530-UI", + "GEN-518-UI", + "GEN-506-UI", + "GEN-536-UI", + "NP6-904-CD", + "NP6-906-UI", + "GEN-547-UI" + ] + }, + { + "key": "family:Live price, weather, combined renewable displays", + "title": "Live price, weather, combined renewable displays", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-502-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-523-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-522-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-526-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-540-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-542-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-539-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-537-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-507-ui" + ], + "product_ids": [ + "GEN-502-UI", + "GEN-523-UI", + "GEN-522-UI", + "GEN-526-UI", + "GEN-540-UI", + "GEN-542-UI", + "GEN-539-UI", + "GEN-537-UI", + "GEN-507-UI" + ] + }, + { + "key": "family:Outage capacity, unplanned outages, adequacy", + "title": "Outage capacity, unplanned outages, adequacy", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-233-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np1-346-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-763-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-161-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-162-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=opg-103-er" + ], + "product_ids": [ + "NP3-233-CD", + "NP1-346-ER", + "NP3-763-CD", + "NP3-161-CD", + "NP3-162-CD", + "OPG-103-ER" + ] + }, + { + "key": "family:DC tie schedules, flows, and state-estimator aggregates", + "title": "DC tie schedules, flows, and state-estimator aggregates", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-765-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-626-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-625-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=gen-538-ui" + ], + "product_ids": [ + "NP3-765-CD", + "NP6-626-CD", + "NP6-625-CD", + "GEN-538-UI" + ] + }, + { + "key": "family:2-day dispatch, energy curves, bids and offers", + "title": "2-day dispatch, energy curves, bids and offers", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-906-ex", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-907-ex", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-908-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-909-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-910-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-911-er" + ], + "product_ids": [ + "NP3-906-EX", + "NP3-907-EX", + "NP3-908-ER", + "NP3-909-ER", + "NP3-910-ER", + "NP3-911-ER" + ] + }, + { + "key": "family:3-day and event disclosures", + "title": "3-day and event disclosures", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-257-ex", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-914-ex", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-915-ex", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-916-ex", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-987-ex" + ], + "product_ids": [ + "NP3-257-EX", + "NP3-914-EX", + "NP3-915-EX", + "NP3-916-EX", + "NP3-987-EX" + ] + }, + { + "key": "family:60-day SCED and DAM disclosures, including ESR", + "title": "60-day SCED and DAM disclosures, including ESR", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-965-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-966-er" + ], + "product_ids": [ + "NP3-965-ER", + "NP3-966-ER" + ] + }, + { + "key": "family:COP snapshot and all updates", + "title": "COP snapshot and all updates", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np1-301", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-991-ex" + ], + "product_ids": [ + "NP1-301", + "NP3-991-EX" + ] + }, + { + "key": "family:Legacy SASM disclosures", + "title": "Legacy SASM disclosures", + "status": "unavailable", + "scope": "Four generated endpoints persist in current OpenAPI. New publication ended: final 2025-12-04 operating data posted 2026-02-03. EMIL still Active. Preserve history and imports.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-990-ex" + ], + "product_ids": [ + "NP3-990-EX" + ] + }, + { + "key": "family:Settlement-point and electrical-bus mapping", + "title": "Settlement-point and electrical-bus mapping", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-160-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-158-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-200-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-231-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-159-cd" + ], + "product_ids": [ + "NP4-160-SG", + "NP4-158-SG", + "NP4-200-CD", + "NP4-231-CD", + "NP4-159-CD" + ] + }, + { + "key": "family:Public CRR auctions, ownership, and PTP results", + "title": "Public CRR auctions, ownership, and PTP results", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np7-802-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np7-803-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np7-535-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np7-536-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np7-157-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np7-464-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-194-cd" + ], + "product_ids": [ + "NP7-802-M", + "NP7-803-M", + "NP7-535-SG", + "NP7-536-SG", + "NP7-157-SG", + "NP7-464-CD", + "NP4-194-CD" + ] + }, + { + "key": "family:Scarcity, fuel cost, demand response, and integration reports", + "title": "Scarcity, fuel cost, demand response, and integration reports", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-790-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-791-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-412-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-494-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-107", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-108", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-109", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-110", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-760-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-765-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=eia-930-er" + ], + "product_ids": [ + "NP4-790-CD", + "NP4-791-CD", + "NP4-412-CD", + "NP4-494-ER", + "NP3-107", + "NP3-108", + "NP3-109", + "NP3-110", + "NP4-760-ER", + "NP4-765-ER", + "EIA-930-ER" + ] + }, + { + "key": "family:Load profiles, loss factors, and settlement aggregates", + "title": "Load profiles, loss factors, and settlement aggregates", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=zp18-68-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=zp18-67-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=zp18-265-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np13-9-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np13-14-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np13-262-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np13-268-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np9-598", + "https://www.ercot.com/mp/data-products/data-product-details?id=np1-300", + "https://www.ercot.com/mp/data-products/data-product-details?id=coms-770-sg" + ], + "product_ids": [ + "ZP18-68-M", + "ZP18-67-M", + "ZP18-265-SG", + "NP13-9-SG", + "NP13-14-SG", + "NP13-262-SG", + "NP13-268-SG", + "NP9-598", + "NP1-300", + "COMS-770-SG" + ] + }, + { + "key": "family:Adequacy, planning, CDR, MORA and SARA history", + "title": "Adequacy, planning, CDR, MORA and SARA history", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-774-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-784-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-773-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-240-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=pg7-048-m", + "https://www.ercot.com/gridinfo/resource" + ], + "product_ids": [ + "NP3-774-M", + "NP3-784-M", + "NP3-773-M", + "NP3-240-M", + "PG7-048-M" + ] + }, + { + "key": "family:Interconnection, resource lists and DG reports", + "title": "Interconnection, resource lists and DG reports", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=pg7-201-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-988-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np16-533-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-823-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np12-215-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np16-474-m", + "https://www.ercot.com/gridinfo/resource" + ], + "product_ids": [ + "PG7-201-ER", + "NP3-988-ER", + "NP16-533-M", + "NP3-823-M", + "NP12-215-ER", + "NP16-474-M" + ] + }, + { + "key": "family:Frequency events and public notices", + "title": "Frequency events and public notices", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np12-261-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np12-265-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-87-an", + "https://www.ercot.com/mp/data-products/data-product-details?id=opg-453-an", + "https://www.ercot.com/mp/data-products/data-product-details?id=opg-158", + "https://www.ercot.com/mp/data-products/data-product-details?id=zp4-405-m", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-50-an", + "https://www.ercot.com/services/comm/mkt_notices" + ], + "product_ids": [ + "NP12-261-M", + "NP12-265-M", + "NP6-87-AN", + "OPG-453-AN", + "OPG-158", + "ZP4-405-M", + "NP4-50-AN" + ] + }, + { + "key": "family:Weather archive contractual boundary", + "title": "Weather archive contractual boundary", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-722-cd", + "https://www.ercot.com/mktinfo/loadprofile" + ], + "product_ids": [ + "WEBSITE-WEATHER-1996-2000", + "NP4-722-CD" + ] + }, + { + "key": "family:Unavailable hourly-load year", + "title": "Unavailable hourly-load year", + "status": "unavailable", + "scope": "ERCOT explicitly states no 2001 hourly load data available. Do not interpolate and label it authoritative recovered history.", + "source_urls": [ + "https://www.ercot.com/gridinfo/load/load_hist" + ], + "product_ids": [ + "WEBSITE-LOAD-2001" + ] + }, + { + "key": "family:Certified participant settlements and retail usage", + "title": "Certified participant settlements and retail usage", + "status": "restricted", + "scope": "Excluded from public retrieval.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np9-170-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np9-566-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=zp12-245", + "https://www.ercot.com/mp/data-products/data-product-details?id=coms-448" + ], + "product_ids": [ + "NP9-170-SG", + "NP9-566-SG", + "ZP12-245", + "COMS-448" + ] + }, + { + "key": "family:Secure models, network ratings and ECEII", + "title": "Secure models, network ratings and ECEII", + "status": "restricted", + "scope": "Excluded from public retrieval.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-500-sg", + "https://www.ercot.com/mp/data-products/data-product-details?id=np6-216-er", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-217-cd", + "https://www.ercot.com/mp/data-products/data-product-details?id=np3-459-sg" + ], + "product_ids": [ + "NP4-500-SG", + "NP6-216-ER", + "NP3-217-CD", + "NP3-459-SG" + ] + }, + { + "key": "family:Participant EWS, telemetry, private bids/COP/awards", + "title": "Participant EWS, telemetry, private bids/COP/awards", + "status": "restricted", + "scope": "Excluded from public retrieval.", + "source_urls": [ + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-302-ui", + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-303-ui", + "https://developer.ercot.com/applications/ews/ews/" + ], + "product_ids": [ + "NP4-302-UI", + "NP4-303-UI" + ] + }, + { + "key": "family:Zonal-era and other old public collections", + "title": "Zonal-era and other old public collections", + "status": "deferred", + "scope": "No current public retrieval for the whole family; selected supported subsets have separate entries.", + "source_urls": [ + "https://www.ercot.com/mktrules" + ], + "product_ids": [ + "WEBSITE-ZONAL-ARCHIVES" + ] + } + ] +} diff --git a/tinyercot/public/_generated.py b/tinyercot/public/_generated.py new file mode 100644 index 00000000..d8be44e2 --- /dev/null +++ b/tinyercot/public/_generated.py @@ -0,0 +1,65 @@ +# Generated by tools/generate_public.py from pinned public field metadata. +"""Observed non-null DAM rows; unknown schemas fail decoding.""" + +from datetime import date +from decimal import Decimal + +from pydantic import BaseModel, ConfigDict, StrictBool, StrictStr + + +class DamPrice(BaseModel): + """A non-null row from the observed public DAM field contract. + + Attributes: + deliveryDate: Source operating date, without an inferred timezone. + hourEnding: Source hour-ending label, including 24:00. + settlementPoint: Source hub, zone, or resource-node identifier. + settlementPointPrice: Decimal source price in dollars per MWh. + DSTFlag: Raw source flag; no repeated-hour meaning is inferred. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + deliveryDate: date + hourEnding: StrictStr + settlementPoint: StrictStr + settlementPointPrice: Decimal + DSTFlag: StrictBool + + +DAM_FIELDS = { + "deliveryDate": "DATE", + "hourEnding": "VARCHAR", + "settlementPoint": "VARCHAR", + "settlementPointPrice": "DOUBLE", + "DSTFlag": "BOOLEAN", +} + + +class DamCapacityPrice(BaseModel): + """A non-null row from the observed public DAM field contract. + + Attributes: + deliveryDate: Source operating date, without an inferred timezone. + hourEnding: Source hour-ending label, including 24:00. + ancillaryType: Source ancillary-service identifier. + MCPC: Decimal source capacity clearing price. + DSTFlag: Raw source flag; no repeated-hour meaning is inferred. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + deliveryDate: date + hourEnding: StrictStr + ancillaryType: StrictStr + MCPC: Decimal + DSTFlag: StrictBool + + +CAPACITY_FIELDS = { + "deliveryDate": "DATE", + "hourEnding": "VARCHAR", + "ancillaryType": "VARCHAR", + "MCPC": "DOUBLE", + "DSTFlag": "BOOLEAN", +} diff --git a/tinyercot/public/_http.py b/tinyercot/public/_http.py new file mode 100644 index 00000000..ab72889c --- /dev/null +++ b/tinyercot/public/_http.py @@ -0,0 +1,285 @@ +"""Bounded HTTP and safe errors for the opt-in public adapters.""" + +import datetime +import hashlib +import json +import math +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from email.utils import parsedate_to_datetime +from typing import Any + +import httpx + + +class PublicDataError(Exception): + """An opt-in retrieval cannot return a verified public result.""" + + +class AuthenticationError(PublicDataError): + """Public API authentication failed without disclosing response details.""" + + +class AccessDeniedError(PublicDataError): + """The source denied access; no entitlement workaround is attempted.""" + + +class SchemaMismatchError(PublicDataError): + """A response differs from the supported observed source contract.""" + + +class LimitError(PublicDataError): + """A request, byte, page, or archive limit was reached.""" + + +class RateLimitError(PublicDataError): + """The rate limit exceeds the configured retry budget.""" + + +class SourceUnavailableError(PublicDataError): + """A source did not return the requested public data.""" + + +@dataclass(frozen=True) +class Limits: + """Caller-selected ceilings for one adapter instance. + + Attributes: + max_requests: Total HTTP attempts, including authentication and retries. + max_bytes: Maximum decoded bytes in each HTTP response. + attempts: Maximum attempts per HTTP operation. + min_interval: Minimum seconds between request starts. + max_retry_wait: Maximum accepted server-directed retry delay. + """ + + max_requests: int = 20 + max_bytes: int = 4_000_000 + attempts: int = 3 + min_interval: float = 2.1 + max_retry_wait: float = 60.0 + + def __post_init__(self) -> None: + if ( + type(self.max_requests) is not int + or type(self.attempts) is not int + or not 1 <= self.max_requests <= 100 + or not 1 <= self.attempts <= 5 + ): + raise ValueError("Request limits must be positive and bounded") + if type(self.max_bytes) is not int or not 1 <= self.max_bytes <= 16_000_000: + raise ValueError("max_bytes must be between 1 and 16000000") + if not math.isfinite(self.min_interval) or self.min_interval < 0: + raise ValueError("min_interval must be finite and nonnegative") + if not math.isfinite(self.max_retry_wait) or self.max_retry_wait < 0: + raise ValueError("max_retry_wait must be finite and nonnegative") + + +@dataclass(frozen=True) +class Receipt: + """Public payload identity without credentials or response headers. + + Attributes: + source_url: Public source URL with public query parameters only. + retrieved_at: UTC retrieval time, separate from source publication time. + sha256: Hash of the decoded response bytes. + byte_count: Number of decoded bytes hashed. + status: HTTP success status. + """ + + source_url: str + retrieved_at: datetime.datetime + sha256: str + byte_count: int + status: int = 200 + + +@dataclass(frozen=True) +class Payload: + """Bounded bytes and their receipt for internal source decoding. + + Attributes: + body: Complete decoded response bytes within the byte ceiling. + receipt: Public source identity and retrieval time. + """ + + body: bytes = field(repr=False) + receipt: Receipt + + def json(self) -> Any: + """Decode JSON without including source bytes in an error. + + Returns: + The decoded JSON value, with decimals preserved. + + Raises: + SchemaMismatchError: The response is not valid JSON. + """ + from decimal import Decimal + + try: + return json.loads(self.body, parse_float=Decimal) + except (ValueError, UnicodeError): + pass + raise SchemaMismatchError("Source returned invalid JSON") + + +class _HTTP: + """Own a synchronous transport with a finite shared attempt budget.""" + + def __init__( + self, + limits: Limits, + transport: httpx.BaseTransport | None = None, + *, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + ) -> None: + """Create the bounded transport without a network request. + + Args: + limits: Attempt, byte, and wait ceilings for this instance. + transport: Optional mocked HTTPX transport. + sleep: Wait function, replaceable by an offline test. + clock: Monotonic clock used for pacing and token lifetime. + """ + self.limits = limits + self.client = httpx.Client(transport=transport, timeout=30, trust_env=False) + self.sleep = sleep + self.clock = clock + self.requests = 0 + self.last_start: float | None = None + self.lock = threading.RLock() + self.closed = False + + def close(self) -> None: + """Close connections and prevent further requests.""" + self.closed = True + self.client.close() + + def _pace(self) -> None: + """Wait for the next request slot and charge one attempt. + + Raises: + PublicDataError: This instance is closed or has used its budget. + """ + if self.closed: + raise PublicDataError("Adapter is closed") + if self.requests >= self.limits.max_requests: + raise LimitError("HTTP request budget exhausted") + if self.last_start is not None: + delay = self.last_start + self.limits.min_interval - self.clock() + if delay > 0: + self.sleep(delay) + self.last_start = self.clock() + self.requests += 1 + + def _retry_delay(self, value: str | None, attempt: int) -> float: + """Resolve the source wait header within the configured limit. + + Args: + value: Retry-After seconds or an HTTP date, if present. + attempt: Zero-based attempt index for the fallback wait. + + Returns: + A nonnegative wait in seconds. + + Raises: + RateLimitError: The header is invalid or exceeds the wait budget. + """ + delay = min(2**attempt, self.limits.max_retry_wait) + if value: + try: + delay = float(value) + except ValueError: + try: + stamp = parsedate_to_datetime(value) + delay = ( + stamp - datetime.datetime.now(datetime.UTC) + ).total_seconds() + except (TypeError, ValueError, OverflowError): + raise RateLimitError("Unsupported Retry-After value") from None + if not math.isfinite(delay) or delay > self.limits.max_retry_wait: + raise RateLimitError("Retry-After exceeds the wait budget") + return max(0.0, delay) + + def request( + self, + method: str, + url: str, + *, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + authentication: bool = False, + ) -> Payload: + """Fetch within limits without retaining sensitive HTTP error objects. + + Args: + method: GET for public data or POST for ID-token acquisition. + url: Fixed adapter source URL; never a caller-provided redirect. + params: Public query parameters only. + headers: Authentication headers, when needed. + data: Form-encoded auth fields; never placed in a URL or receipt. + authentication: Whether to suppress auth payload provenance. + + Returns: + Bounded response bytes and a credential-free receipt. + + Raises: + PublicDataError: Status, transport, schema, or budget failure. + """ + with self.lock: + for attempt in range(self.limits.attempts): + self._pace() + failure = None + retry_after = None + status = 0 + try: + with self.client.stream( + method, + url, + params=params, + headers=headers, + data=data, + follow_redirects=False, + ) as response: + status = response.status_code + if status == 200: + raw = bytearray() + for chunk in response.iter_bytes(chunk_size=65_536): + raw.extend(chunk) + if len(raw) > self.limits.max_bytes: + raise LimitError("Response byte budget exceeded") + body = bytes(raw) + receipt = Receipt( + url if authentication else str(response.url), + datetime.datetime.now(datetime.UTC), + "" + if authentication + else hashlib.sha256(body).hexdigest(), + 0 if authentication else len(body), + ) + return Payload(body, receipt) + retry_after = response.headers.get("retry-after") + except httpx.HTTPError: + # Raise outside the except block to discard the HTTPX context. + failure = "transport" + if status == 401 or ( + authentication + and failure is None + and status not in (0, 429, 502, 503, 504) + ): + raise AuthenticationError("Public API authentication failed") + if status == 403: + raise AccessDeniedError("Source denied public access") + retryable = failure is not None or status in (429, 502, 503, 504) + if not retryable: + raise SourceUnavailableError(f"Source returned HTTP {status}") + if attempt + 1 == self.limits.attempts: + if status == 429: + raise RateLimitError("Rate-limit attempt budget exhausted") + raise SourceUnavailableError("HTTP attempt budget exhausted") + self.sleep(self._retry_delay(retry_after, attempt)) + raise AssertionError("Unreachable request state") diff --git a/tinyercot/public/api.py b/tinyercot/public/api.py new file mode 100644 index 00000000..e76233d3 --- /dev/null +++ b/tinyercot/public/api.py @@ -0,0 +1,463 @@ +"""Opt-in DAM price retrieval with explicit credentials and bounded pagination.""" + +import datetime +import math +import os +import re +from collections.abc import Iterator +from dataclasses import dataclass, field +from decimal import Decimal +from typing import Generic, Self, TypeVar + +import httpx +from pydantic import ValidationError + +from ._generated import CAPACITY_FIELDS, DAM_FIELDS, DamCapacityPrice, DamPrice +from ._http import ( + _HTTP, + AuthenticationError, + Limits, + Payload, + Receipt, + SchemaMismatchError, +) + +BASE = "https://api.ercot.com/api/public-reports" +PRICE_PATH = "/np4-190-cd/dam_stlmnt_pnt_prices" +CAPACITY_PATH = "/np4-188-cd/dam_clear_price_for_cap" +Row = TypeVar("Row", DamPrice, DamCapacityPrice) +TOKEN_URL = "https://ercotb2c.b2clogin.com/ercotb2c.onmicrosoft.com/B2C_1_PUBAPI-ROPC-FLOW/oauth2/v2.0/token" +CLIENT_ID = "fec253ea-0d06-4272-a5e6-b478baeecd70" + + +@dataclass(frozen=True) +class Credentials: + """Public API account credentials, excluded from repr output. + + Attributes: + username: ERCOT Public API account name. + password: ERCOT Public API account password. + subscription_key: Subscription key for the public data product. + """ + + username: str = field(repr=False) + password: str = field(repr=False) + subscription_key: str = field(repr=False) + + def __post_init__(self) -> None: + if not all((self.username, self.password, self.subscription_key)): + raise AuthenticationError("All three Public API credentials are required") + + @classmethod + def from_env(cls) -> Self: + """Read the three legacy ERCOT environment names on explicit request. + + Returns: + Credentials without loading dotenv files or authenticating. + + Raises: + AuthenticationError: A required environment value is absent. + """ + return cls( + *( + os.environ.get(name, "") + for name in ( + "ERCOT_USERNAME", + "ERCOT_PASSWORD", + "ERCOT_SUBSCRIPTION_KEY", + ) + ) + ) + + +@dataclass(frozen=True) +class PricePage(Generic[Row]): + """Decoded price rows with the original public envelope and receipt. + + Attributes: + rows: Source rows decoded using declared field names and order. + meta: Original pagination and query metadata. + raw: Complete public response bytes; no request credentials or headers. + receipt: Retrieval time and exact response hash. + """ + + rows: tuple[Row, ...] + meta: dict + raw: bytes = field(repr=False) + receipt: Receipt + + +def _decode_page( + payload: Payload, + *, + expected_page: int, + fields: dict, + row_model: type[Row], + price_field: str, +) -> PricePage[Row]: + """Decode the observed five-column schema using response-declared order. + + Args: + payload: Bounded public JSON bytes and receipt. + expected_page: Requested one-based page number. + fields: Pinned response field names and source types. + row_model: Generated model for this fixed endpoint. + price_field: Numeric price field in the pinned model. + + Returns: + Typed non-null rows plus the complete original envelope. + + Raises: + SchemaMismatchError: Fields, row widths/types, or pagination disagree. + """ + body = payload.json() + try: + declared = body["fields"] + if not isinstance(declared, list): + raise TypeError + names = [entry["name"] for entry in declared] + if ( + len(names) != len(set(names)) + or {e["name"]: e["dataType"] for e in declared} != fields + ): + raise ValueError + meta = body["_meta"] + if type(meta["currentPage"]) is not int or meta["currentPage"] != expected_page: + raise ValueError + if type(meta["totalPages"]) is not int or meta["totalPages"] < 0: + raise ValueError + if meta["totalPages"] and expected_page > meta["totalPages"]: + raise ValueError + data = body["data"] + if not isinstance(data, list): + raise TypeError + rows = [] + for row in data: + if isinstance(row, list): + if len(row) != len(names): + raise ValueError + row = dict(zip(names, row, strict=True)) + if not isinstance(row, dict) or set(row) != set(fields): + raise ValueError + if type(row["deliveryDate"]) is not str or not re.fullmatch( + r"\d{4}-\d{2}-\d{2}", row["deliveryDate"] + ): + raise ValueError + if type(row[price_field]) not in (Decimal, int): + raise ValueError + model = row_model.model_validate(row) + if not getattr(model, price_field).is_finite(): + raise ValueError + rows.append(model) + if not meta["totalPages"] and rows: + raise ValueError + return PricePage(tuple(rows), meta, payload.body, payload.receipt) + except (KeyError, TypeError, ValueError, ValidationError): + pass + raise SchemaMismatchError( + "DAM price response differs from the observed field/page contract" + ) + + +def decode_prices(payload: Payload, *, expected_page: int) -> PricePage[DamPrice]: + """Decode observed DAM settlement prices. + + Args: + payload: Bounded public JSON bytes and receipt. + expected_page: Requested one-based page number. + + Returns: + Typed non-null rows and the original public envelope. + + Raises: + SchemaMismatchError: Fields, rows, or pagination differ from the contract. + """ + return _decode_page( + payload, + expected_page=expected_page, + fields=DAM_FIELDS, + row_model=DamPrice, + price_field="settlementPointPrice", + ) + + +class PublicClient: + """A synchronous client restricted to two observed public DAM price sources. + + Use as a context manager. Credentials and tokens stay on this instance. + This client has no participant routes, generic raw URL method, or async API. + """ + + def __init__( + self, + credentials: Credentials, + *, + limits: Limits | None = None, + transport: httpx.BaseTransport | None = None, + ) -> None: + """Create a client without making an HTTP request. + + Args: + credentials: Explicit public-account credentials. + limits: Per-instance request, byte, pacing, and retry ceilings. + transport: Optional HTTPX transport for offline tests. + """ + self._credentials = credentials + self._http = _HTTP(limits or Limits(), transport) + self._token: str | None = None + self._expires_at = 0.0 + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def close(self) -> None: + """Close the transport and discard this instance's cached token.""" + self._token = None + self._http.close() + + def refresh_token(self) -> None: + """Acquire a new ID token; ERCOT does not support ID-token refresh. + + Form-encoded acquisition was verified against the public auth service. + Secret values stay out of request URLs, receipts, and exceptions. + + Raises: + AuthenticationError: The auth response has no usable token lifetime. + PublicDataError: Transport, access, or request limits fail. + """ + with self._http.lock: + self._token = None + started = self._http.clock() + response = self._http.request( + "POST", + TOKEN_URL, + authentication=True, + data={ + "username": self._credentials.username, + "password": self._credentials.password, + "grant_type": "password", + "scope": f"openid {CLIENT_ID} offline_access", + "client_id": CLIENT_ID, + "response_type": "id_token", + }, + ).json() + try: + token = response["id_token"] + lifetime = float(response["expires_in"]) + if ( + not isinstance(token, str) + or not token + or not math.isfinite(lifetime) + or lifetime <= 60 + ): + raise ValueError + except (KeyError, TypeError, ValueError): + raise AuthenticationError("Invalid public ID-token response") from None + self._token = token + self._expires_at = started + min(lifetime, 3600) - 60 + + def dam_prices( + self, + *, + start: datetime.date | None = None, + end: datetime.date | None = None, + settlement_point: str, + page: int = 1, + size: int = 100, + oldest_first: bool = False, + ) -> PricePage[DamPrice]: + """Fetch one bounded page from the observed DAM price endpoint. + + Args: + start: Inclusive source delivery-date lower filter. + end: Inclusive source delivery-date upper filter. + settlement_point: Exact public settlement point identifier. + page: One-based page number, not an automatic history traversal. + size: Client-selected page ceiling, from 1 through 1000. + oldest_first: Sort by delivery date ascending instead of descending. + With omitted dates, size=1 gives an oldest-available observation. + + Returns: + Typed price rows with raw public envelope and retrieval receipt. + + Raises: + ValueError: Local query bounds are invalid. + PublicDataError: Authentication, transport, or schema checks fail. + """ + if not settlement_point or len(settlement_point) > 128: + raise ValueError("A settlement point is required") + if ( + type(page) is not int + or page < 1 + or type(size) is not int + or not 1 <= size <= 1000 + ): + raise ValueError("Invalid page or client page-size ceiling") + if start and end and start > end: + raise ValueError("start must not follow end") + params = { + "settlementPoint": settlement_point, + "page": page, + "size": size, + "sort": "deliveryDate", + "dir": "asc" if oldest_first else "desc", + } + if start: + params["deliveryDateFrom"] = start.isoformat() + if end: + params["deliveryDateTo"] = end.isoformat() + payload = self._price_payload(PRICE_PATH, params) + result = decode_prices(payload, expected_page=page) + if len(result.rows) > size: + raise SchemaMismatchError("Source exceeded requested page size") + return result + + def dam_capacity_prices( + self, + *, + start: datetime.date | None = None, + end: datetime.date | None = None, + ancillary_type: str, + page: int = 1, + size: int = 100, + oldest_first: bool = False, + ) -> PricePage[DamCapacityPrice]: + """Fetch one bounded NP4-188-CD capacity-price page. + + Args: + start: Inclusive source delivery-date lower filter. + end: Inclusive source delivery-date upper filter. + ancillary_type: Exact public ancillary-service identifier. + page: One-based page number. + size: Client-selected row ceiling, from 1 through 1000. + oldest_first: Sort delivery date ascending for oldest observations. + + Returns: + Typed non-null rows and the original public envelope and receipt. + + Raises: + ValueError: Local query bounds are invalid. + PublicDataError: Authentication, transport, or schema checks fail. + """ + if not ancillary_type or len(ancillary_type) > 128: + raise ValueError("An ancillary-service identifier is required") + if ( + type(page) is not int + or page < 1 + or type(size) is not int + or not 1 <= size <= 1000 + ): + raise ValueError("Invalid page or client page-size ceiling") + if start and end and start > end: + raise ValueError("start must not follow end") + params = { + "ancillaryType": ancillary_type, + "page": page, + "size": size, + "sort": "deliveryDate", + "dir": "asc" if oldest_first else "desc", + } + if start: + params["deliveryDateFrom"] = start.isoformat() + if end: + params["deliveryDateTo"] = end.isoformat() + result = _decode_page( + self._price_payload(CAPACITY_PATH, params), + expected_page=page, + fields=CAPACITY_FIELDS, + row_model=DamCapacityPrice, + price_field="MCPC", + ) + if len(result.rows) > size: + raise SchemaMismatchError("Source exceeded requested page size") + return result + + def _price_payload(self, path: str, params: dict) -> Payload: + """Use the instance token for one fixed public price route. + + Args: + path: One of the two implemented public price paths. + params: Validated public query fields. + + Returns: + Bounded public JSON bytes and receipt. + + Raises: + PublicDataError: Route, authentication, or transport checks fail. + """ + if path not in (PRICE_PATH, CAPACITY_PATH): + raise ValueError("Unsupported price path") + with self._http.lock: + for reacquisition in range(2): + if self._token is None or self._http.clock() >= self._expires_at: + self.refresh_token() + try: + return self._http.request( + "GET", + BASE + path, + params=params, + headers={ + "Authorization": f"Bearer {self._token}", + "Ocp-Apim-Subscription-Key": self._credentials.subscription_key, + }, + ) + except AuthenticationError: + self._token = None + if reacquisition: + raise AuthenticationError( + "Public API rejected the new ID token" + ) from None + raise AuthenticationError("Public API token reacquisition failed") + + def price_pages( + self, + *, + start: datetime.date, + end: datetime.date, + settlement_point: str, + size: int = 100, + max_pages: int = 2, + ) -> Iterator[PricePage[DamPrice]]: + """Yield a bounded selection of pages, preserving each page receipt. + + Args: + start: Required delivery-date lower filter. + end: Required delivery-date upper filter, at most 31 days after start. + settlement_point: Exact public settlement point identifier. + size: Rows requested per page, subject to the client ceiling. + max_pages: Explicit sampling limit, from 1 through 10. + + Yields: + Pages starting at one. Reaching max_pages is a partial selection, + not a complete-history result; totalPages remains in page metadata. + + Raises: + ValueError: The caller's window or page bound is invalid. + PublicDataError: A request or page consistency check fails. + """ + if ( + type(max_pages) is not int + or not 1 <= max_pages <= 10 + or not 0 <= (end - start).days <= 31 + ): + raise ValueError("Pagination requires a window <=31 days and <=10 pages") + seen = set() + for page in range(1, max_pages + 1): + result = self.dam_prices( + start=start, + end=end, + settlement_point=settlement_point, + page=page, + size=size, + oldest_first=True, + ) + identity = tuple(row.model_dump_json() for row in result.rows) + if identity and identity in seen: + raise SchemaMismatchError("Source repeated a page of rows") + seen.add(identity) + yield result + if page >= result.meta["totalPages"]: + break diff --git a/tinyercot/public/archives.py b/tinyercot/public/archives.py new file mode 100644 index 00000000..aa27caf0 --- /dev/null +++ b/tinyercot/public/archives.py @@ -0,0 +1,244 @@ +"""Bounded sampling of the observed public DAM annual ZIP/XLSX format.""" + +import datetime +import hashlib +import io +import re +import stat +import zipfile +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path, PurePosixPath + +from pydantic import BaseModel, ConfigDict, StrictStr + +from ._http import AccessDeniedError, LimitError, Receipt, SchemaMismatchError + +HEADER = ( + "Delivery Date", + "Hour Ending", + "Repeated Hour Flag", + "Settlement Point", + "Settlement Point Price", +) + + +@dataclass(frozen=True) +class ArchiveDocument: + """One public MIS listing entry, not a guarantee of interval completeness. + + Attributes: + document_id: Source document identity used for receipts and downloads. + report_type_id: Fixed public DAM annual archive report type, 13060. + friendly_name: Source year-file label. + published_at: Source publication timestamp with its original offset. + byte_count: Advertised download size, separate from the received size. + security: Raw MIS security code; only P is supported. + """ + + document_id: str + report_type_id: int + friendly_name: str + published_at: datetime.datetime + byte_count: int + security: str + + +@dataclass(frozen=True) +class Download: + """One verified local archive file and its source receipt. + + Attributes: + document: Original listing metadata, including publication time. + path: Complete file, committed atomically after validation. + receipt: Original download receipt; cache reuse keeps its retrieval time. + cache_hit: Whether a prior complete hash-verified download was reused. + """ + + document: ArchiveDocument + path: Path + receipt: Receipt + cache_hit: bool = False + + +class ArchivePrice(BaseModel): + """A row from the observed DAM annual workbook format. + + Attributes: + delivery_date: Source delivery date from the workbook. + hour_ending: Original hour-ending text, including 24:00. + repeated_hour_flag: Raw workbook flag, without a guessed timezone mapping. + settlement_point: Original settlement point identifier. + settlement_point_price: Decimal conversion of the source numeric cell. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + delivery_date: datetime.date + hour_ending: StrictStr + repeated_hour_flag: StrictStr + settlement_point: StrictStr + settlement_point_price: Decimal + + +@dataclass(frozen=True) +class ArchiveSample: + """A bounded worksheet selection, never a complete annual-history claim. + + Attributes: + rows: Decoded source rows up to the caller's limit. + member: Original XLSX member name in the outer ZIP. + sheet: Original worksheet name. + truncated: Whether another data row exists after the sampling limit. + source_sha256: Exact outer archive hash. + """ + + rows: tuple[ArchivePrice, ...] + member: str + sheet: str + truncated: bool + source_sha256: str + + +def checked_zip( + raw: bytes, *, max_members: int, max_expanded_bytes: int +) -> zipfile.ZipFile: + """Inspect archive structure before reading any member data. + + Args: + raw: Complete compressed archive within the download byte budget. + max_members: Maximum member count, including directory entries. + max_expanded_bytes: Maximum sum of advertised decompressed member sizes. + + Returns: + An open ZipFile. The caller must close it. + + Raises: + SchemaMismatchError: ZIP syntax or member paths are unsafe. + LimitError: A member or expanded-byte ceiling is exceeded. + """ + try: + archive = zipfile.ZipFile(io.BytesIO(raw)) + except zipfile.BadZipFile: + raise SchemaMismatchError("Source did not return a ZIP archive") from None + members = archive.infolist() + if ( + len(members) > max_members + or sum(m.file_size for m in members) > max_expanded_bytes + ): + archive.close() + raise LimitError("Archive expansion budget exceeded") + seen = set() + for member in members: + path = PurePosixPath(member.filename) + mode = member.external_attr >> 16 + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.filename + or ":" in member.filename + or member.filename in seen + or stat.S_ISLNK(mode) + or member.flag_bits & 1 + ): + archive.close() + raise SchemaMismatchError("Unsafe or duplicate archive member") + seen.add(member.filename) + return archive + + +def sample_dam_archive( + download: Download, + *, + sheet: str, + max_rows: int = 100, +) -> ArchiveSample: + """Decode a bounded worksheet sample from a verified local annual file. + + Requires the files extra. The decoder checks both ZIP layers before using + openpyxl read-only mode. It does not extract files or follow external links. + + Args: + download: A public report-13060 file with matching receipt hash. + sheet: Exact worksheet name; no automatic all-sheet traversal occurs. + max_rows: Maximum decoded data rows, from 1 through 1000. + + Returns: + Source rows, workbook identities, and an explicit truncation flag. + + Raises: + ImportError: Install tinyercot[files] for the XLSX decoder. + PublicDataError: Access, hash, shape, or archive limits fail. + ValueError: The row sampling bound is invalid. + """ + if type(max_rows) is not int or not 1 <= max_rows <= 1000: + raise ValueError("max_rows must be between 1 and 1000") + if download.document.security != "P" or download.document.report_type_id != 13060: + raise AccessDeniedError("Only public DAM annual files are supported") + if download.path.is_symlink() or download.path.stat().st_size > 4_000_000: + raise LimitError("Local archive size or path is invalid") + raw = download.path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != download.receipt.sha256: + raise SchemaMismatchError("Archive does not match its download receipt") + from openpyxl import load_workbook + + try: + with checked_zip(raw, max_members=24, max_expanded_bytes=32_000_000) as outer: + members = [m for m in outer.infolist() if not m.is_dir()] + if len(members) != 1 or not members[0].filename.lower().endswith(".xlsx"): + raise SchemaMismatchError("Unsupported annual archive container") + member = members[0].filename + workbook_bytes = outer.read(members[0]) + with checked_zip( + workbook_bytes, max_members=512, max_expanded_bytes=64_000_000 + ): + pass + workbook = load_workbook( + io.BytesIO(workbook_bytes), read_only=True, data_only=True, keep_links=False + ) + try: + if sheet not in workbook.sheetnames: + raise SchemaMismatchError("Requested source worksheet is absent") + iterator = workbook[sheet].iter_rows(values_only=True) + header = next(iterator) + if tuple(header) != HEADER: + raise SchemaMismatchError("Annual workbook headers changed") + rows = [] + truncated = False + for scanned, cells in enumerate(iterator, start=1): + if scanned > max_rows * 2 + 1: + raise LimitError("Worksheet scan budget exceeded") + if all(x is None for x in cells): + continue + if len(rows) == max_rows: + truncated = True + break + if len(cells) != 5: + raise SchemaMismatchError("Annual workbook has extra columns") + day, hour, repeated, point, price = cells[:5] + if ( + not isinstance(day, str) + or not re.fullmatch(r"\d{2}/\d{2}/\d{4}", day) + or type(price) not in (int, float) + ): + raise SchemaMismatchError("Annual workbook cell types changed") + amount = Decimal(str(price)) + if not amount.is_finite(): + raise SchemaMismatchError("Annual workbook price is nonfinite") + rows.append( + ArchivePrice( + delivery_date=datetime.date( + int(day[6:10]), int(day[:2]), int(day[3:5]) + ), + hour_ending=hour, + repeated_hour_flag=repeated, + settlement_point=point, + settlement_point_price=amount, + ) + ) + return ArchiveSample(tuple(rows), member, sheet, truncated, digest) + finally: + workbook.close() + except (zipfile.BadZipFile, KeyError, TypeError, ValueError, StopIteration): + pass + raise SchemaMismatchError("Annual workbook differs from the observed XLSX contract") diff --git a/tinyercot/public/coverage.py b/tinyercot/public/coverage.py new file mode 100644 index 00000000..d295dfdb --- /dev/null +++ b/tinyercot/public/coverage.py @@ -0,0 +1,91 @@ +"""Explicit scope states for every audited family and observed API operation.""" + +import json +from dataclasses import dataclass +from importlib.resources import files +from typing import Literal + +from tinyercot.catalog import operations + + +@dataclass(frozen=True) +class Coverage: + """An exact retrieval scope, separate from broad family completeness. + + Attributes: + key: Stable source or operation identity, including service and verb. + title: Human-readable family or implemented capability. + status: Covered bounded scope, deferred work, restricted, or unavailable. + scope: What the status covers and what it does not establish. + source_urls: Primary ERCOT references for this entry. + product_ids: Relevant source product identifiers, where established. + """ + + key: str + title: str + status: Literal["covered", "deferred", "restricted", "unavailable"] + scope: str + source_urls: tuple[str, ...] + product_ids: tuple[str, ...] = () + + +def coverage() -> tuple[Coverage, ...]: + """List all audited families, API operations, and supported web subsets. + + Returns: + Immutable scope records. A covered endpoint or web subset does not mark + its entire family covered. Deferred or unknown schemas grant no access. + The Stage-0 metadata catalog remains an unchanged observation snapshot. + """ + snapshot = json.loads( + files("tinyercot.public").joinpath("_coverage.json").read_text() + ) + entries = [ + Coverage( + **{k: v for k, v in row.items() if k not in {"source_urls", "product_ids"}}, + source_urls=tuple(row["source_urls"]), + product_ids=tuple(row["product_ids"]), + ) + for row in snapshot["families"] + ] + for operation in operations(): + supported = (operation.service, operation.method, operation.path) in { + ("public-reports", "GET", "/np4-190-cd/dam_stlmnt_pnt_prices"), + ("public-reports", "GET", "/np4-188-cd/dam_clear_price_for_cap"), + } + entries.append( + Coverage( + f"api:{operation.service}:{operation.method}:{operation.path}", + operation.path, + "covered" if supported else "deferred", + "Bounded non-null DAM price pages; observed field schema only. No full history, all-filter, or schema-epoch claim." + if supported + else "No opt-in retrieval implementation or verified current row contract.", + (operation.source_url,), + ) + ) + entries.extend( + ( + Coverage( + "web:dam-annual", + "Public DAM annual ZIP/XLSX samples", + "covered", + "Report 13060 public listing, one selected file per call, receipt cache, and <=1000 rows from one named worksheet. Verified 2010 and 2026 samples, not complete years.", + ( + "https://www.ercot.com/mp/data-products/data-product-details?id=np4-180-er", + ), + ("NP4-180-ER",), + ), + Coverage( + "web:esr", + "Public rolling ESR aggregates", + "covered", + "One snapshot with source offset/epoch validation and explicit freshness. No historical API or automatic polling.", + ( + "https://www.ercot.com/api/1/services/read/dashboards/energy-storage-resources.json", + ), + ("GEN-545-UI",), + ), + ) + ) + return tuple(entries) diff --git a/tinyercot/public/live.py b/tinyercot/public/live.py new file mode 100644 index 00000000..a56bdef7 --- /dev/null +++ b/tinyercot/public/live.py @@ -0,0 +1,174 @@ +"""Typed observations of the public rolling ESR dashboard feed.""" + +import datetime +from dataclasses import dataclass, field +from decimal import Decimal +from zoneinfo import ZoneInfo + +from pydantic import ( + BaseModel, + ConfigDict, + StrictInt, + StrictStr, + field_validator, + model_validator, +) + +from ._http import Payload, Receipt, SchemaMismatchError + + +class EsrRow(BaseModel): + """One observed storage aggregate, with source time evidence retained. + + Attributes: + tagCLastTime: Source local timestamp text, not a publication timestamp. + dstFlag: Raw source flag with no inferred repeated-hour meaning. + totalCharging: Source charging MW, preserving the source sign convention. + totalDischarging: Source discharging MW. + netOutput: Source net MW; no new calculation or rounding is applied. + timestamp: Source timestamp with its explicit UTC offset. + epoch: Source epoch milliseconds, checked against timestamp. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + tagCLastTime: StrictStr + dstFlag: StrictStr + totalCharging: Decimal + totalDischarging: Decimal + netOutput: Decimal + timestamp: datetime.datetime + epoch: StrictInt + + @field_validator("timestamp", mode="before") + @classmethod + def source_time(cls, value: str) -> datetime.datetime: + """Parse the observed offset-bearing source format. + + Args: + value: Timestamp text in the public feed's documented observation. + + Returns: + Offset-aware datetime without replacing the source offset. + + Raises: + ValueError: The source text does not match the observed format. + """ + return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S%z") + + @model_validator(mode="after") + def check_time_evidence(self): + """Require epoch, local text, and Texas offset to describe one instant. + + Returns: + This row after time and finite-number checks. + + Raises: + ValueError: Source timestamps conflict or a power value is nonfinite. + """ + local = datetime.datetime.fromisoformat(self.tagCLastTime) + texas = self.timestamp.astimezone(ZoneInfo("America/Chicago")) + if ( + int(self.timestamp.timestamp() * 1000) != self.epoch + or self.timestamp.replace(tzinfo=None) != local + or texas.replace(tzinfo=None) != local + or texas.utcoffset() != self.timestamp.utcoffset() + ): + raise ValueError("Conflicting timestamp evidence") + if not all( + x.is_finite() + for x in (self.totalCharging, self.totalDischarging, self.netOutput) + ): + raise ValueError("Nonfinite power value") + return self + + @property + def utc(self) -> datetime.datetime: + """The UTC instant established by the source offset and epoch. + + Returns: + The same instant in UTC; DST flags do not change its meaning. + """ + return self.timestamp.astimezone(datetime.UTC) + + +class EsrDay(BaseModel): + """One source day section, without inventing a history window. + + Attributes: + dayDate: Original source day marker, including its offset and time. + data: Typed, ordered storage aggregate observations. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + dayDate: StrictStr + data: tuple[EsrRow, ...] + + +@dataclass(frozen=True) +class EsrSnapshot: + """A single rolling feed capture with explicit freshness evidence. + + Attributes: + last_updated: Source update timestamp with its UTC offset. + previous_day: Source previous-day section. + current_day: Source current-day section. + receipt: Retrieval time and full response hash. + raw: Original JSON bytes, including all source labels. + """ + + last_updated: datetime.datetime + previous_day: EsrDay + current_day: EsrDay + receipt: Receipt + raw: bytes = field(repr=False) + + def is_stale( + self, *, max_age: datetime.timedelta = datetime.timedelta(minutes=10) + ) -> bool: + """Compare source update time with this capture's retrieval time. + + Args: + max_age: Caller-selected acceptable age, not a source SLA. + + Returns: + True for old data or a source update time in the future. + + Raises: + ValueError: max_age is negative. + """ + if max_age.total_seconds() < 0: + raise ValueError("max_age must be nonnegative") + age = self.receipt.retrieved_at - self.last_updated + return age < datetime.timedelta(0) or age > max_age + + +def decode_esr(payload: Payload) -> EsrSnapshot: + """Decode the observed rolling feed and validate its time evidence. + + Args: + payload: Bounded public JSON response and receipt. + + Returns: + An ESR snapshot with source and retrieval times kept separate. + + Raises: + SchemaMismatchError: Shape, field values, or timestamp evidence changes. + """ + body = payload.json() + try: + if set(body) != {"lastUpdated", "previousDay", "currentDay"}: + raise ValueError + updated = datetime.datetime.strptime(body["lastUpdated"], "%Y-%m-%d %H:%M:%S%z") + days = [] + for name in ("previousDay", "currentDay"): + day = EsrDay.model_validate(body[name]) + stamps = [row.epoch for row in day.data] + if stamps != sorted(set(stamps)): + raise ValueError + days.append(day) + return EsrSnapshot(updated, *days, payload.receipt, payload.body) + except (KeyError, TypeError, ValueError): + pass + raise SchemaMismatchError( + "ESR feed differs from the observed shape or time contract" + ) diff --git a/tinyercot/public/web.py b/tinyercot/public/web.py new file mode 100644 index 00000000..74a2cbcf --- /dev/null +++ b/tinyercot/public/web.py @@ -0,0 +1,248 @@ +"""Public-only MIS annual file and rolling ESR feed adapters.""" + +import datetime +import hashlib +import json +import os +from pathlib import Path +from typing import Self + +import httpx + +from ._http import _HTTP, AccessDeniedError, Limits, Receipt, SchemaMismatchError +from .archives import ArchiveDocument, Download, checked_zip +from .live import EsrSnapshot, decode_esr + +LIST_URL = "https://www.ercot.com/misapp/servlets/IceDocListJsonWS" +DOWNLOAD_URL = "https://www.ercot.com/misdownload/servlets/mirDownload" +ESR_URL = ( + "https://www.ercot.com/api/1/services/read/dashboards/energy-storage-resources.json" +) + + +class WebClient: + """An anonymous client for two verified public website source routes. + + No ERCOT credential headers are used. Only public DAM annual archives and + the aggregate ESR feed are implemented. Downloads require a receipt cache. + """ + + def __init__( + self, + *, + limits: Limits | None = None, + transport: httpx.BaseTransport | None = None, + ) -> None: + """Create the web adapter without any network request. + + Args: + limits: Per-instance request, response, and retry ceilings. + transport: Optional HTTPX transport for offline tests. + """ + self._http = _HTTP(limits or Limits(), transport) + self._listed: dict[str, ArchiveDocument] = {} + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def close(self) -> None: + """Release this instance's HTTP connections.""" + self._http.close() + + def esr(self) -> EsrSnapshot: + """Fetch one rolling ESR aggregate snapshot without polling. + + Returns: + Typed source observations, freshness evidence, and raw receipt. + + Raises: + PublicDataError: Transport, byte limits, or source shape checks fail. + """ + return decode_esr(self._http.request("GET", ESR_URL)) + + def dam_archives(self) -> tuple[tuple[ArchiveDocument, ...], Receipt]: + """List only public report-13060 annual files. + + Returns: + Source-order documents and the listing receipt. Nonpublic records + fail closed. Listing a year does not prove complete annual data. + + Raises: + PublicDataError: Access, transport, or listing shape checks fail. + """ + payload = self._http.request("GET", LIST_URL, params={"reportTypeId": 13060}) + body = payload.json() + self._listed = {} + try: + entries = body["ListDocsByRptTypeRes"]["DocumentList"] + if not isinstance(entries, list) or len(entries) > 100: + raise ValueError + documents = [] + for entry in entries: + item = entry["Document"] + if item["SecurityStatus"] != "P": + raise AccessDeniedError("MIS listing contains a nonpublic document") + if ( + item["Extension"].lower() != "zip" + or int(item["ReportTypeID"]) != 13060 + ): + raise ValueError + identity = item["DocID"] + if ( + not isinstance(identity, str) + or not identity.isdecimal() + or len(identity) > 24 + ): + raise ValueError + name = item["FriendlyName"] + if ( + not isinstance(name, str) + or not name.startswith("DAMLZHBSPP_") + or not name.removeprefix("DAMLZHBSPP_").isdecimal() + ): + raise ValueError + stamp = datetime.datetime.fromisoformat(item["PublishDate"]) + size = int(item["ContentSize"]) + if stamp.tzinfo is None or size <= 0 or identity in self._listed: + raise ValueError + document = ArchiveDocument(identity, 13060, name, stamp, size, "P") + self._listed[identity] = document + documents.append(document) + return tuple(documents), payload.receipt + except (KeyError, TypeError, ValueError): + self._listed = {} + raise SchemaMismatchError( + "MIS listing differs from the observed public annual contract" + ) + + def download_dam_archive( + self, document: ArchiveDocument, *, cache: Path + ) -> Download: + """Download one listed public file or reuse its verified local receipt. + + The source page uses mirDownload for files. ViewReport can return an + HTTP-200 HTML 'No Document' page and is deliberately not used here. + Files and receipts use document identity; revised documents do not + overwrite a previous version. No automatic year traversal occurs. + + Args: + document: An unchanged public entry listed by this client instance. + cache: Local directory for complete files and credential-free receipts. + + Returns: + A hash-verified complete ZIP and its original publication metadata. + + Raises: + PublicDataError: Access, transfer, archive, cache, or limit checks fail. + OSError: Local cache storage cannot be created or written. + """ + if ( + document.security != "P" + or document.report_type_id != 13060 + or self._listed.get(document.document_id) != document + ): + raise AccessDeniedError( + "Download requires this client's public listing evidence" + ) + if document.byte_count > min(self._http.limits.max_bytes, 4_000_000): + from ._http import LimitError + + raise LimitError("Advertised archive exceeds the download ceiling") + cache = Path(cache) + if cache.is_symlink(): + raise SchemaMismatchError("Receipt cache cannot be a symlink") + cache.mkdir(parents=True, exist_ok=True) + path = cache / f"13060-{document.document_id}.zip" + receipt_path = cache / f"13060-{document.document_id}.json" + if path.is_symlink() or receipt_path.is_symlink(): + raise SchemaMismatchError("Cached archive or receipt is a symlink") + temporary = cache / f".{document.document_id}.part" + if temporary.exists() or temporary.is_symlink(): + raise SchemaMismatchError("Incomplete download needs explicit cache repair") + if path.exists() or receipt_path.exists(): + try: + if path.stat().st_size > min(self._http.limits.max_bytes, 4_000_000): + raise ValueError + raw = path.read_bytes() + if receipt_path.stat().st_size > 16_000: + raise ValueError + saved = json.loads(receipt_path.read_text()) + expected_url = str( + httpx.URL( + DOWNLOAD_URL, + params={ + "doclookupId": document.document_id, + "reportTypeId": 13060, + }, + ) + ) + if ( + saved["source_url"] != expected_url + or saved["document_id"] != document.document_id + or saved["source_published_at"] != document.published_at.isoformat() + or saved["sha256"] != hashlib.sha256(raw).hexdigest() + or saved["byte_count"] != len(raw) + ): + raise ValueError + receipt = Receipt( + saved["source_url"], + datetime.datetime.fromisoformat(saved["retrieved_at"]), + saved["sha256"], + saved["byte_count"], + ) + if ( + receipt.retrieved_at.tzinfo is None + or len(raw) != document.byte_count + ): + raise ValueError + return Download(document, path, receipt, True) + except (OSError, KeyError, TypeError, ValueError): + pass + raise SchemaMismatchError( + "Incomplete or changed cache; explicit repair is required" + ) + payload = self._http.request( + "GET", + DOWNLOAD_URL, + params={"doclookupId": document.document_id, "reportTypeId": 13060}, + ) + if payload.receipt.byte_count != document.byte_count: + raise SchemaMismatchError("Downloaded size differs from the public listing") + with checked_zip( + payload.body, max_members=24, max_expanded_bytes=32_000_000 + ) as archive: + files = [item for item in archive.infolist() if not item.is_dir()] + if len(files) != 1 or not files[0].filename.lower().endswith(".xlsx"): + raise SchemaMismatchError("Expected one XLSX member in the annual ZIP") + receipt = payload.receipt + record = { + "source_url": receipt.source_url, + "retrieved_at": receipt.retrieved_at.isoformat(), + "sha256": receipt.sha256, + "byte_count": receipt.byte_count, + "document_id": document.document_id, + "source_published_at": document.published_at.isoformat(), + "friendly_name": document.friendly_name, + } + # Keep failed writes separate. A partial cache never triggers a redownload. + descriptor = os.open( + temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 + ) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload.body) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary, path) + finally: + temporary.unlink(missing_ok=True) + descriptor = os.open( + receipt_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 + ) + with os.fdopen(descriptor, "w") as stream: + json.dump(record, stream, indent=2) + stream.write("\n") + return Download(document, path, receipt) diff --git a/tools/generate_public.py b/tools/generate_public.py new file mode 100644 index 00000000..bb81f62d --- /dev/null +++ b/tools/generate_public.py @@ -0,0 +1,114 @@ +"""Generate current row models only from pinned observed field metadata.""" + +import argparse +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +INPUT = ROOT / "tools/inputs/current/dam-prices.json" +CAPACITY_INPUT = ROOT / "tools/inputs/current/dam-capacity.json" +MANIFEST = ROOT / "tools/inputs/current/provenance.json" +OUTPUT = ROOT / "tinyercot/public/_generated.py" +TYPES = { + "DATE": "date", + "VARCHAR": "StrictStr", + "DOUBLE": "Decimal", + "BOOLEAN": "StrictBool", +} +DESCRIPTIONS = { + "deliveryDate": "Source operating date, without an inferred timezone.", + "hourEnding": "Source hour-ending label, including 24:00.", + "settlementPoint": "Source hub, zone, or resource-node identifier.", + "settlementPointPrice": "Decimal source price in dollars per MWh.", + "ancillaryType": "Source ancillary-service identifier.", + "MCPC": "Decimal source capacity clearing price.", + "DSTFlag": "Raw source flag; no repeated-hour meaning is inferred.", +} + + +def render() -> str: + """Render the two observed non-null DAM row contracts. + + Returns: + Model source for two verified endpoints and their ordered fields. + + Raises: + ValueError: A pin, field type, or supported endpoint differs. + """ + lines = [ + "# Generated by tools/generate_public.py from pinned public field metadata.", + '"""Observed non-null DAM rows; unknown schemas fail decoding."""', + "", + "from datetime import date", + "from decimal import Decimal", + "", + "from pydantic import BaseModel, ConfigDict, StrictBool, StrictStr", + "", + ] + for path, expected_path, model, constant in ( + (INPUT, "/np4-190-cd/dam_stlmnt_pnt_prices", "DamPrice", "DAM_FIELDS"), + ( + CAPACITY_INPUT, + "/np4-188-cd/dam_clear_price_for_cap", + "DamCapacityPrice", + "CAPACITY_FIELDS", + ), + ): + raw = path.read_bytes() + expected = json.loads(MANIFEST.read_text())["inputs"][path.name] + if hashlib.sha256(raw).hexdigest() != expected: + raise ValueError("Unverified current response metadata") + contract = json.loads(raw) + if contract["path"] != expected_path: + raise ValueError("No row generator policy for this endpoint") + fields = contract["fields"] + if not fields or len({f["name"] for f in fields}) != len(fields): + raise ValueError("Missing or duplicate response fields") + if any( + f["name"] not in DESCRIPTIONS or f["dataType"] not in TYPES for f in fields + ): + raise ValueError("Unsupported response field") + lines.extend( + [ + "", + f"class {model}(BaseModel):", + ' """A non-null row from the observed public DAM field contract.', + "", + " Attributes:", + ] + ) + lines.extend(f" {f['name']}: {DESCRIPTIONS[f['name']]}" for f in fields) + lines.extend( + [ + ' """', + "", + ' model_config = ConfigDict(extra="forbid", frozen=True)', + "", + ] + ) + lines.extend(f" {f['name']}: {TYPES[f['dataType']]}" for f in fields) + lines.extend(["", "", f"{constant} = {{"]) + lines.extend( + f" {json.dumps(f['name'])}: {json.dumps(f['dataType'])}," for f in fields + ) + lines.extend(["}", ""]) + return "\n".join(lines) + + +def main() -> None: + """Write or check additive models without changing legacy generation.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + output = render() + if args.check: + if OUTPUT.read_text() != output: + raise SystemExit("Current generated models differ from the pinned inputs") + print("Current models match pinned response metadata") + else: + OUTPUT.write_text(output) + + +if __name__ == "__main__": + main() diff --git a/tools/inputs/current/dam-capacity.json b/tools/inputs/current/dam-capacity.json new file mode 100644 index 00000000..d11c9f73 --- /dev/null +++ b/tools/inputs/current/dam-capacity.json @@ -0,0 +1,157 @@ +{ + "format_version": 1, + "service": "public-reports", + "path": "/np4-188-cd/dam_clear_price_for_cap", + "observed_at": "2026-09-05", + "query_parameters": [ + { + "name": "deliveryDateFrom", + "in": "query", + "schema": { + "type": "string", + "format": "yyyy-MM-dd" + } + }, + { + "name": "deliveryDateTo", + "in": "query", + "schema": { + "type": "string", + "format": "yyyy-MM-dd" + } + }, + { + "name": "hourEnding", + "in": "query", + "schema": { + "type": "string", + "format": "abc123" + } + }, + { + "name": "ancillaryType", + "in": "query", + "schema": { + "type": "string", + "format": "abc123" + } + }, + { + "name": "MCPCFrom", + "in": "query", + "schema": { + "type": "number", + "format": "####.###" + } + }, + { + "name": "MCPCTo", + "in": "query", + "schema": { + "type": "number", + "format": "####.###" + } + }, + { + "name": "DSTFlag", + "in": "query", + "schema": { + "type": "boolean", + "format": "true | false" + } + }, + { + "name": "page", + "in": "query", + "description": "Page number of returned values in the collection.", + "schema": { + "type": "integer", + "format": "###" + } + }, + { + "name": "size", + "in": "query", + "description": "Number of returned items per page.", + "schema": { + "type": "integer", + "format": "###" + } + }, + { + "name": "sort", + "in": "query", + "description": "Defines field by which to sort the returned resource values.", + "schema": { + "type": "string", + "format": "abc123" + } + }, + { + "name": "dir", + "in": "query", + "description": "Defines sort order of returned values based on the primary business key of the resource.", + "schema": { + "type": "string", + "format": "abc123" + } + } + ], + "fields": [ + { + "name": "deliveryDate", + "label": "Delivery Date", + "cardinality": 1, + "dataType": "DATE", + "searchable": true, + "sortable": true, + "hasRange": true + }, + { + "name": "hourEnding", + "label": "Hour Ending", + "cardinality": 2, + "dataType": "VARCHAR", + "searchable": true, + "sortable": true, + "hasRange": false + }, + { + "name": "ancillaryType", + "label": "Ancillary Type", + "cardinality": 3, + "dataType": "VARCHAR", + "searchable": true, + "sortable": true, + "hasRange": false + }, + { + "name": "MCPC", + "label": "MCPC", + "cardinality": 4, + "dataType": "DOUBLE", + "searchable": true, + "sortable": true, + "hasRange": true + }, + { + "name": "DSTFlag", + "label": "DST Flag", + "cardinality": 5, + "dataType": "BOOLEAN", + "searchable": true, + "sortable": true, + "hasRange": false + } + ], + "nullability_policy": "Reject nulls; source field metadata does not establish nullability. This model supports the observed non-null price rows only.", + "source_url": "https://apiexplorer.ercot.com/developer/apis/pubapi-apim-api?export=true&api-version=2022-04-01-preview", + "query_source_sha256": "b978bf35fbf9dcb8edca14025d4b5d3108582885a6c1b92a9bf445e69b08a7f8", + "response_source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-188-cd/dam_clear_price_for_cap?deliveryDateFrom=2026-09-04&deliveryDateTo=2026-09-04&ancillaryType=REGUP&size=2&page=1&sort=deliveryDate&dir=desc", + "retrieved_at": "2026-09-05 20:54:00.360875+00:00", + "sha256": "3fa8d075dadb0baf6cb0b4656ec87e887a7c85c6d96ab2923dc3e42bd8373251", + "byte_count": 1336, + "status": 200 + } +} diff --git a/tools/inputs/current/dam-prices.json b/tools/inputs/current/dam-prices.json new file mode 100644 index 00000000..3d1414e5 --- /dev/null +++ b/tools/inputs/current/dam-prices.json @@ -0,0 +1,162 @@ +{ + "format_version": 1, + "service": "public-reports", + "path": "/np4-190-cd/dam_stlmnt_pnt_prices", + "observed_at": "2026-09-05", + "query_parameters": [ + { + "name": "deliveryDateFrom", + "in": "query", + "schema": { + "type": "string", + "format": "yyyy-MM-dd" + } + }, + { + "name": "deliveryDateTo", + "in": "query", + "schema": { + "type": "string", + "format": "yyyy-MM-dd" + } + }, + { + "name": "hourEnding", + "in": "query", + "schema": { + "type": "string", + "format": "abc123" + } + }, + { + "name": "settlementPoint", + "in": "query", + "schema": { + "type": "string", + "format": "abc123" + } + }, + { + "name": "settlementPointPriceFrom", + "in": "query", + "schema": { + "type": "number", + "format": "####.###" + } + }, + { + "name": "settlementPointPriceTo", + "in": "query", + "schema": { + "type": "number", + "format": "####.###" + } + }, + { + "name": "DSTFlag", + "in": "query", + "schema": { + "type": "boolean", + "format": "true | false" + } + }, + { + "name": "page", + "in": "query", + "schema": { + "type": "integer", + "format": "###" + } + }, + { + "name": "size", + "in": "query", + "schema": { + "type": "integer", + "format": "###" + } + }, + { + "name": "sort", + "in": "query", + "schema": { + "type": "string", + "format": "abc123" + } + }, + { + "name": "dir", + "in": "query", + "schema": { + "type": "string", + "format": "abc123" + } + } + ], + "fields": [ + { + "name": "deliveryDate", + "label": "Delivery Date", + "cardinality": 1, + "dataType": "DATE", + "searchable": true, + "sortable": true, + "hasRange": true + }, + { + "name": "hourEnding", + "label": "Hour Ending", + "cardinality": 2, + "dataType": "VARCHAR", + "searchable": true, + "sortable": true, + "hasRange": false + }, + { + "name": "settlementPoint", + "label": "Settlement Point", + "cardinality": 3, + "dataType": "VARCHAR", + "searchable": true, + "sortable": true, + "hasRange": false + }, + { + "name": "settlementPointPrice", + "label": "Settlement Point Price", + "cardinality": 4, + "dataType": "DOUBLE", + "searchable": true, + "sortable": true, + "hasRange": true + }, + { + "name": "DSTFlag", + "label": "DST Flag", + "cardinality": 5, + "dataType": "BOOLEAN", + "searchable": true, + "sortable": true, + "hasRange": false + } + ], + "nullability_policy": "Reject nulls; source field metadata does not establish nullability. This model supports the observed non-null price rows only.", + "source_url": "https://apiexplorer.ercot.com/developer/apis/pubapi-apim-api?export=true&api-version=2022-04-01-preview", + "query_source_sha256": "b978bf35fbf9dcb8edca14025d4b5d3108582885a6c1b92a9bf445e69b08a7f8", + "response_source": { + "source_url": "https://api.ercot.com/api/public-reports/np4-190-cd/dam_stlmnt_pnt_prices", + "query": { + "deliveryDateFrom": "2026-09-04", + "deliveryDateTo": "2026-09-04", + "settlementPoint": "HB_HOUSTON", + "size": 2, + "page": 1, + "sort": "deliveryDate", + "dir": "desc" + }, + "retrieved_at": "2026-09-05T20:28:34.745619+00:00", + "bytes": 1384, + "sha256": "dc70cef89b0d7ce0fb53a6441f8d5bdd0f79b5d60fff6125253046017cf37d18", + "status": 200 + } +} diff --git a/tools/inputs/current/provenance.json b/tools/inputs/current/provenance.json new file mode 100644 index 00000000..87516366 --- /dev/null +++ b/tools/inputs/current/provenance.json @@ -0,0 +1,7 @@ +{ + "format_version": 1, + "inputs": { + "dam-prices.json": "7b0a79a756ecfafb3f1d72d5c244b9ce06fdcba40d99f10ab05cd08a341e6bc4", + "dam-capacity.json": "46b3352eb03262a48d47ca499b5a2a7a96bdbb9ba1c769108428b0018f884969" + } +} diff --git a/tools/probe_public.py b/tools/probe_public.py new file mode 100644 index 00000000..b7413633 --- /dev/null +++ b/tools/probe_public.py @@ -0,0 +1,174 @@ +"""Run a small real-source check with an installed wheel, only with --live. + +The credential file is the JSON-quoted, mode-0600 local vault export used for +this task. Receipts contain public source metadata and hashes, never headers. +""" + +import argparse +import datetime +import json +import logging +import stat +from dataclasses import asdict +from pathlib import Path + +import tinyercot +from tinyercot.public import Credentials, PublicClient, WebClient, sample_dam_archive + + +def main() -> None: + """Check installed current/history retrieval and write public-only receipts. + + Raises: + SystemExit: Explicit consent, installed-package, or credential checks fail. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--live", action="store_true") + parser.add_argument("--credentials-file", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if not args.live: + raise SystemExit("Real requests require --live") + root = Path(__file__).resolve().parents[1] + installed = Path(tinyercot.__file__).resolve() + if installed.parent == root / "tinyercot": + raise SystemExit("Run with an isolated interpreter and the installed wheel") + credential_file = args.credentials_file + if ( + credential_file.is_symlink() + or stat.S_IMODE(credential_file.stat().st_mode) != 0o600 + ): + raise SystemExit("Credentials require a regular mode-0600 file") + if not credential_file.is_file(): + raise SystemExit("Credentials require a regular file") + logging.disable(logging.CRITICAL) + records = {"installed_module": str(installed), "receipts": []} + args.output.mkdir(parents=True, exist_ok=True) + + def record(kind, receipt, **metadata): + """Add public source evidence without request headers or row values. + + Args: + kind: Evidence class for this installed-client operation. + receipt: Public URL, retrieval time, byte count, and response hash. + **metadata: Public counts and source publication or selection fields. + """ + records["receipts"].append({"kind": kind, **asdict(receipt), **metadata}) + (args.output / "installed-receipts.json").write_text( + json.dumps(records, indent=2, default=str) + "\n" + ) + + try: + values = { + key: json.loads(value) + for line in credential_file.read_text().splitlines() + for key, value in [line.split("=", 1)] + } + credentials = Credentials( + *( + values[name] + for name in ( + "ERCOT_USERNAME", + "ERCOT_PASSWORD", + "ERCOT_SUBSCRIPTION_KEY", + ) + ) + ) + del values + day = datetime.date(2026, 9, 4) + with PublicClient(credentials) as client: + for page in client.price_pages( + start=day, end=day, settlement_point="HB_HOUSTON", size=2, max_pages=2 + ): + assert page.rows and all(row.deliveryDate == day for row in page.rows) + record( + "typed-api-current", + page.receipt, + rows=len(page.rows), + page=page.meta["currentPage"], + total_pages=page.meta["totalPages"], + ) + page = client.dam_prices( + settlement_point="HB_HOUSTON", size=1, oldest_first=True + ) + assert len(page.rows) == 1 + record( + "typed-api-historical", + page.receipt, + rows=1, + earliest_observed_date=str(page.rows[0].deliveryDate), + ) + client.refresh_token() + page = client.dam_prices( + start=day, end=day, settlement_point="HB_HOUSTON", size=1 + ) + assert page.rows + record( + "typed-api-after-token-reacquisition", page.receipt, rows=len(page.rows) + ) + page = client.dam_capacity_prices( + start=day, end=day, ancillary_type="REGUP", size=2 + ) + assert len(page.rows) == 2 and all( + row.deliveryDate == day for row in page.rows + ) + record("typed-capacity-current", page.receipt, rows=len(page.rows)) + page = client.dam_capacity_prices( + ancillary_type="REGUP", size=1, oldest_first=True + ) + assert len(page.rows) == 1 + record( + "typed-capacity-historical", + page.receipt, + rows=1, + earliest_observed_date=str(page.rows[0].deliveryDate), + ) + with WebClient() as client: + snapshot = client.esr() + record( + "typed-esr", + snapshot.receipt, + rows=len(snapshot.previous_day.data) + len(snapshot.current_day.data), + source_updated_at=str(snapshot.last_updated), + stale=snapshot.is_stale(), + ) + documents, receipt = client.dam_archives() + record("typed-public-mis-listing", receipt, documents=len(documents)) + for year, sheet in ((2010, "Dec_1"), (2026, "Aug")): + document = next( + item + for item in documents + if item.friendly_name == f"DAMLZHBSPP_{year}" + ) + download = client.download_dam_archive( + document, cache=args.output / "cache" + ) + sample = sample_dam_archive(download, sheet=sheet, max_rows=4) + assert len(sample.rows) == 4 and sample.truncated + reused = client.download_dam_archive( + document, cache=args.output / "cache" + ) + assert reused.cache_hit and reused.receipt == download.receipt + record( + "typed-annual-sample", + download.receipt, + year=year, + source_published_at=str(document.published_at), + document_id=document.document_id, + rows=len(sample.rows), + member=sample.member, + sheet=sample.sheet, + truncated=sample.truncated, + initial_cache_hit=download.cache_hit, + verified_cache_reuse=True, + ) + except Exception as error: # noqa: BLE001 - Do not expose third-party request objects. + # Some third-party exceptions retain requests. Never print their values. + raise SystemExit( + f"Installed probe failed: {type(error).__name__}; public receipts retained" + ) from None + print("Installed public-source probe passed; public-only receipts written") + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index 5922efd0..ba61458a 100644 --- a/uv.lock +++ b/uv.lock @@ -82,6 +82,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -295,6 +304,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/73/edeacba3167b1ca66d51b1a5a14697c2c40098b5ffa01811c67b1785a5ab/numpy-2.4.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b", size = 12489376, upload-time = "2025-12-20T16:18:16.524Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -638,6 +659,11 @@ dependencies = [ { name = "tqdm" }, ] +[package.optional-dependencies] +files = [ + { name = "openpyxl" }, +] + [package.dev-dependencies] dev = [ { name = "ipython" }, @@ -652,10 +678,12 @@ requires-dist = [ { name = "cachetools", specifier = ">=6.2.4" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "httpx-retries", specifier = ">=0.4.5" }, + { name = "openpyxl", marker = "extra == 'files'", specifier = ">=3.1.5" }, { name = "pandas", specifier = ">=2.0" }, { name = "pydantic", specifier = ">=2.10.0" }, { name = "tqdm", specifier = ">=4.67.1" }, ] +provides-extras = ["files"] [package.metadata.requires-dev] dev = [