diff --git a/.github/workflows/documentation-freshness.yml b/.github/workflows/documentation-freshness.yml new file mode 100644 index 0000000..f3309f4 --- /dev/null +++ b/.github/workflows/documentation-freshness.yml @@ -0,0 +1,52 @@ +name: Monitor Sent contract and documentation freshness + +on: + workflow_dispatch: + release: + types: [published] + schedule: + - cron: "17 9 * * 1" + +permissions: + contents: read + +jobs: + freshness-and-routing: + runs-on: ubuntu-latest + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + SENT_ROUTING_EVAL_MODEL: ${{ vars.SENT_ROUTING_EVAL_MODEL || 'gpt-5-mini' }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Install validators + run: python -m pip install -r requirements-dev.txt + + - name: Compare live Sent contract and documentation facts + run: >- + python scripts/check_live_contract.py + --timeout 15 + --retries 2 + --output artifacts/documentation-freshness.json + + - name: Upload freshness diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: documentation-freshness + path: artifacts/documentation-freshness.json + if-no-files-found: warn + + - name: Run model-backed routing evaluation + if: always() && env.OPENAI_API_KEY != '' + run: python scripts/run_model_routing_eval.py + + - name: Explain skipped model evaluation + if: always() && env.OPENAI_API_KEY == '' + run: echo "OPENAI_API_KEY is not configured; freshness monitoring still ran, but model routing evaluation was skipped." diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8e0e2b5..9a64324 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -7,8 +7,6 @@ on: branches: [main] release: types: [published] - schedule: - - cron: "17 9 * * 1" permissions: contents: read @@ -60,31 +58,5 @@ jobs: - name: Test Sent API contract regressions run: python scripts/test_contracts.py - live-contract-and-routing: - if: github.event_name == 'schedule' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - SENT_ROUTING_EVAL_MODEL: ${{ vars.SENT_ROUTING_EVAL_MODEL || 'gpt-5-mini' }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: requirements-dev.txt - - - name: Install validators - run: python -m pip install -r requirements-dev.txt - - - name: Compare live Sent OpenAPI with manifest - run: python scripts/check_live_contract.py - - - name: Run model-backed routing evaluation - if: env.OPENAI_API_KEY != '' - run: python scripts/run_model_routing_eval.py - - - name: Explain skipped model evaluation - if: env.OPENAI_API_KEY == '' - run: echo "OPENAI_API_KEY is not configured; live contract drift still ran, but model routing evaluation was skipped." + - name: Test freshness monitor simulations + run: python scripts/test_live_contract.py diff --git a/docs/PUBLIC_RELEASE.md b/docs/PUBLIC_RELEASE.md index 1caf959..ea6b73d 100644 --- a/docs/PUBLIC_RELEASE.md +++ b/docs/PUBLIC_RELEASE.md @@ -5,7 +5,7 @@ Use this checklist for every public release. ## Repository and package - Confirm the GitHub repository is public and the release commit is on the default branch. -- Confirm the repository root contains `plugin.json`, `mcp.json`, `skills/`, and `assets/` for GitHub auto-discovery. +- Confirm the repository root contains `plugin.json`, `mcp.json`, `public-surface.json`, `skills/`, and `assets/` for GitHub auto-discovery. - Confirm `.claude-plugin/marketplace.json` points to `./claude-plugins/sent`. - Confirm the Claude plugin contains its manifest, README, license, skills, commands, assets, and HTTPS MCP configuration. - Update the version consistently before release; plugin names are stable public identifiers and must not be renamed casually. @@ -18,11 +18,13 @@ python3 scripts/generate_adapters.py --check python3 scripts/validate.py python3 scripts/test_validation_gates.py python3 scripts/test_fixtures.py +python3 scripts/test_contracts.py +python3 scripts/test_live_contract.py claude plugin validate . --strict claude plugin validate ./claude-plugins/sent --strict ``` -CI must pass on the exact commit being submitted. Resolve warnings as well as errors before submission. +CI must pass on the exact commit being submitted. Resolve warnings as well as errors before submission. The dedicated contract/documentation freshness workflow runs on schedule, manual dispatch, and release; review its uploaded JSON artifact separately from pull-request validation. ## Public-facing review diff --git a/packages/sent/README.md b/packages/sent/README.md index c369ffc..c352adc 100644 --- a/packages/sent/README.md +++ b/packages/sent/README.md @@ -85,6 +85,10 @@ python3 scripts/generate_adapters.py python3 scripts/validate.py python3 scripts/test_validation_gates.py python3 scripts/test_fixtures.py +python3 scripts/test_contracts.py +python3 scripts/test_live_contract.py ``` For current product and API behavior, use the [Sent MCP documentation](https://docs.sent.dm/start/llm-docs/mcp-server), the [machine-readable documentation index](https://docs.sent.dm/llms.txt), and the [Sent API reference](https://docs.sent.dm/reference/api). + +Live network checks are isolated from pull-request validation. The scheduled/manual/release freshness workflow compares normalized facts from the source catalog, classifies drift separately from source failures, and uploads a JSON diagnostic artifact. diff --git a/schemas/sent/documentation-sources.json b/schemas/sent/documentation-sources.json index 351a6ff..02a3d29 100644 --- a/schemas/sent/documentation-sources.json +++ b/schemas/sent/documentation-sources.json @@ -6,7 +6,21 @@ "url": "https://docs.sent.dm/llms.txt", "kind": "documentation-index", "affected_skills": ["sent"], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "docs.index.template_definition_listed", + "extractor": "contains", + "pattern": "https://docs.sent.dm/llms/reference/api/template-definition.txt", + "expected": true + }, + { + "fact": "docs.index.webhook_events_listed", + "extractor": "contains", + "pattern": "https://docs.sent.dm/llms/start/webhooks/event-types.txt", + "expected": true + } + ] }, { "id": "openapi", @@ -22,14 +36,29 @@ "waba-embedded-signup", "waba-template-author" ], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [] }, { "id": "templates", "url": "https://docs.sent.dm/llms/reference/api/template-definition.txt", "kind": "documentation", "affected_skills": ["sent-templates", "template-builder-ui", "waba-template-author"], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "template.body_max_length", + "extractor": "regex_int", + "pattern": "Body text[^\\n]*Maximum (\\d+) characters", + "expected": 1024 + }, + { + "fact": "template.button_total_limit", + "extractor": "regex_int", + "pattern": "maximum of (\\d+) buttons", + "expected": 10 + } + ] }, { "id": "webhooks", @@ -40,35 +69,105 @@ "sender-profile-architect", "waba-embedded-signup" ], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "webhook.message_delivered_event", + "extractor": "contains", + "pattern": "\"event\": \"message.delivered\"", + "expected": true + }, + { + "fact": "webhook.template_event_omitted", + "extractor": "contains", + "pattern": "omitted for template events", + "expected": true + } + ] }, { "id": "sender-profiles", "url": "https://docs.sent.dm/llms/start/concepts/sender-profiles.txt", "kind": "documentation", "affected_skills": ["sender-profile-architect", "sms-10dlc-registration", "waba-embedded-signup"], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "profile.organization_scope_header", + "extractor": "contains", + "pattern": "x-profile-id", + "expected": true + }, + { + "fact": "profile.rcs_agent_dedicated", + "extractor": "contains", + "pattern": "RCS Agent", + "expected": true + } + ] }, { "id": "waba", "url": "https://docs.sent.dm/llms/start/guides/connect-whatsapp.txt", "kind": "documentation", "affected_skills": ["waba-embedded-signup", "waba-template-author"], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "waba.id_field", + "extractor": "contains", + "pattern": "waba_id", + "expected": true + }, + { + "fact": "waba.access_token_field", + "extractor": "contains", + "pattern": "access_token", + "expected": true + } + ] }, { "id": "routing-rcs", "url": "https://docs.sent.dm/llms/reference/channel-routing.txt", "kind": "documentation", "affected_skills": ["messaging-performance-analyzer", "rcs-agent-onboarding", "sent-messaging"], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "routing.multiple_explicit_channels", + "extractor": "contains", + "pattern": "broadcast list, not a fallback priority list", + "expected": true + }, + { + "fact": "routing.rcs_supported", + "extractor": "contains", + "pattern": "Pins the message to RCS", + "expected": true + } + ] }, { "id": "10dlc", "url": "https://docs.sent.dm/llms/start/advanced/10dlc-registration.txt", "kind": "documentation", "affected_skills": ["sms-10dlc-registration"], - "last_verified": "2026-08-09" + "last_verified": "2026-08-09", + "checks": [ + { + "fact": "10dlc.tcr_submission", + "extractor": "contains", + "pattern": "submit to TCR", + "expected": true + }, + { + "fact": "10dlc.opt_in_consent", + "extractor": "contains", + "pattern": "consent", + "expected": true + } + ] } ] } diff --git a/scripts/check_live_contract.py b/scripts/check_live_contract.py index 51df776..c4650db 100644 --- a/scripts/check_live_contract.py +++ b/scripts/check_live_contract.py @@ -1,18 +1,29 @@ #!/usr/bin/env python3 -"""Compare the checked-in critical contract manifest with Sent's live v3 OpenAPI.""" +"""Monitor Sent OpenAPI and documentation sources at normalized fact level. + +Exit codes: + 0 - every observed fact matches + 1 - semantic drift or a conflict between authoritative sources + 2 - source unavailability, malformed content, or monitor configuration error +""" from __future__ import annotations import argparse +import datetime import json +import re import sys import urllib.request +from collections import Counter, defaultdict from pathlib import Path -from typing import Any +from typing import Any, Callable ROOT = Path(__file__).resolve().parents[1] MANIFEST_PATH = ROOT / "schemas" / "sent" / "v3-contract-manifest.json" +CATALOG_PATH = ROOT / "schemas" / "sent" / "documentation-sources.json" +Fetcher = Callable[[str, float], str | bytes] def resolve(document: dict[str, Any], value: dict[str, Any]) -> dict[str, Any]: @@ -29,78 +40,461 @@ def resolve(document: dict[str, Any], value: dict[str, Any]) -> dict[str, Any]: return merged -def schema(document: dict[str, Any], name: str) -> dict[str, Any]: - return resolve(document, document["components"]["schemas"][name]) +def schema(document: dict[str, Any], name: str) -> dict[str, Any] | None: + try: + value = document["components"]["schemas"][name] + return resolve(document, value) + except (KeyError, TypeError, ValueError): + return None -def compare(document: dict[str, Any], manifest: dict[str, Any]) -> list[str]: - errors: list[str] = [] +def _fact( + source: dict[str, Any], + name: str, + expected: Any, + observed: Any, + status: str | None = None, + detail: str | None = None, +) -> dict[str, Any]: + if status is None: + status = "match" if observed == expected else "drift" + result = { + "fact": name, + "status": status, + "expected": expected, + "observed": observed, + "source_url": source["url"], + "affected_skills": source.get("affected_skills", []), + } + if detail: + result["detail"] = detail + return result + + +def _source_status(facts: list[dict[str, Any]]) -> str: + statuses = {fact["status"] for fact in facts} + for candidate in ("unavailable", "malformed", "conflict", "drift"): + if candidate in statuses: + return candidate + return "ok" + + +def _source_result(source: dict[str, Any], facts: list[dict[str, Any]], attempts: int = 1) -> dict[str, Any]: + return { + "id": source["id"], + "url": source["url"], + "kind": source["kind"], + "affected_skills": source.get("affected_skills", []), + "last_verified": source.get("last_verified"), + "status": _source_status(facts), + "attempts": attempts, + "facts": facts, + } + + +def evaluate_openapi( + document: dict[str, Any], manifest: dict[str, Any], source: dict[str, Any] +) -> dict[str, Any]: + if not isinstance(document, dict) or not isinstance(document.get("paths"), dict): + fact = _fact(source, "openapi.document", "valid OpenAPI object", None, "malformed") + return _source_result(source, [fact]) + + facts: list[dict[str, Any]] = [] + live_paths = document.get("paths", {}) for path, methods in manifest["critical_paths"].items(): - if path not in document.get("paths", {}): - errors.append(f"missing live path: {path}") - continue - live_methods = {method.upper() for method in document["paths"][path] if method.lower() in {"get", "post", "put", "patch", "delete"}} - if live_methods != set(methods): - errors.append(f"method drift at {path}: manifest={methods}, live={sorted(live_methods)}") + value = live_paths.get(path) + observed = ( + sorted( + method.upper() + for method in value + if method.lower() in {"get", "post", "put", "patch", "delete"} + ) + if isinstance(value, dict) + else None + ) + facts.append(_fact(source, f"openapi.path.{path}.methods", sorted(methods), observed)) for retired in manifest["retired_guidance_paths"]: - if retired in document.get("paths", {}): - errors.append(f"retired guidance path is live again and needs review: {retired}") + facts.append(_fact(source, f"openapi.retired_path.{retired}.present", False, retired in live_paths)) template = schema(document, "SentDmServicesEndpointsCustomerAPIv3RequestsCreateTemplateRequest") - if set(template.get("properties", {})) != set(manifest["template_create"]["allowed_fields"]): - errors.append("template create field drift") + facts.append( + _fact( + source, + "template.allowed_fields", + sorted(manifest["template_create"]["allowed_fields"]), + sorted(template.get("properties", {})) if template else None, + ) + ) + facts.append( + _fact( + source, + "template.required_fields", + sorted(manifest["template_create"]["required_fields"]), + sorted(template.get("required", [])) if template else None, + ) + ) body_content = schema(document, "SentDmServicesCommonEntitiesTemplateBodyContent") - if body_content.get("properties", {}).get("template", {}).get("maxLength") != manifest["template_create"]["body_max_length"]: - errors.append("template body maximum drift") - button_description = schema(document, "SentDmServicesCommonEntitiesTemplateButton")["properties"]["type"].get("description", "") - for button_type in manifest["template_create"]["button_types"]: - if button_type not in button_description: - errors.append(f"template button type absent from live description: {button_type}") - - create_profile = schema(document, "SentDmServicesEndpointsCustomerAPIv3RequestsCreateProfileRequest") - if set(manifest["profile"]["create_required_fields"]) - set(create_profile.get("required", [])): - errors.append("profile create required-field drift") - waba = schema(document, "SentDmServicesEndpointsCustomerAPIv3RequestsWhatsappBusinessAccountCredentials") - if set(waba.get("required", [])) != set(manifest["profile"]["waba_required_fields"]): - errors.append("WABA credential required-field drift") - complete = schema(document, "SentDmServicesEndpointsCustomerAPIv3RequestsProfilesCompleteProfileRequest") - if set(complete.get("required", [])) != set(manifest["profile"]["complete_required_fields"]): - errors.append("profile completion request drift") - - campaign = schema(document, "SentDmServicesEndpointsCustomerAPIv3RequestsCampaignsCampaignData") - if set(campaign.get("required", [])) != set(manifest["campaign"]["required_fields"]): - errors.append("campaign required-field drift") - use_cases = schema(document, "SentDmServicesCommonEnumsMessagingUseCaseUS").get("enum", []) - if use_cases != manifest["campaign"]["use_case_values"]: - errors.append("campaign use-case enum drift") - samples = schema(document, "SentDmServicesEndpointsCustomerAPIv3RequestsCampaignsCampaignUseCaseData")["properties"]["sampleMessages"] - if (samples.get("minItems"), samples.get("maxItems")) != ( - manifest["campaign"]["sample_min"], - manifest["campaign"]["sample_max"], - ): - errors.append("campaign sample-count drift") - return errors - - -def main() -> int: + body_maximum = ( + body_content.get("properties", {}).get("template", {}).get("maxLength") + if body_content + else None + ) + facts.append( + _fact( + source, + "template.body_max_length", + manifest["template_create"]["body_max_length"], + body_maximum, + ) + ) + button = schema(document, "SentDmServicesCommonEntitiesTemplateButton") + description = button.get("properties", {}).get("type", {}).get("description", "") if button else "" + observed_buttons = sorted( + button_type + for button_type in manifest["template_create"]["button_types"] + if button_type in description + ) + facts.append( + _fact( + source, + "template.button_types", + sorted(manifest["template_create"]["button_types"]), + observed_buttons if button else None, + ) + ) + + schema_facts = ( + ( + "profile.create_required_fields", + "SentDmServicesEndpointsCustomerAPIv3RequestsCreateProfileRequest", + manifest["profile"]["create_required_fields"], + ), + ( + "profile.waba_required_fields", + "SentDmServicesEndpointsCustomerAPIv3RequestsWhatsappBusinessAccountCredentials", + manifest["profile"]["waba_required_fields"], + ), + ( + "profile.complete_required_fields", + "SentDmServicesEndpointsCustomerAPIv3RequestsProfilesCompleteProfileRequest", + manifest["profile"]["complete_required_fields"], + ), + ( + "campaign.required_fields", + "SentDmServicesEndpointsCustomerAPIv3RequestsCampaignsCampaignData", + manifest["campaign"]["required_fields"], + ), + ) + for fact_name, schema_name, expected in schema_facts: + value = schema(document, schema_name) + facts.append( + _fact(source, fact_name, sorted(expected), sorted(value.get("required", [])) if value else None) + ) + + use_cases = schema(document, "SentDmServicesCommonEnumsMessagingUseCaseUS") + facts.append( + _fact( + source, + "campaign.use_case_values", + manifest["campaign"]["use_case_values"], + use_cases.get("enum") if use_cases else None, + ) + ) + campaign_use_case = schema( + document, "SentDmServicesEndpointsCustomerAPIv3RequestsCampaignsCampaignUseCaseData" + ) + samples = campaign_use_case.get("properties", {}).get("sampleMessages", {}) if campaign_use_case else {} + facts.append( + _fact( + source, + "campaign.sample_count_range", + [manifest["campaign"]["sample_min"], manifest["campaign"]["sample_max"]], + [samples.get("minItems"), samples.get("maxItems")] if campaign_use_case else None, + ) + ) + return _source_result(source, facts) + + +def evaluate_document(text: str, source: dict[str, Any]) -> dict[str, Any]: + if not isinstance(text, str) or not text.strip(): + return _source_result( + source, + [_fact(source, "source.content", "non-empty text", None, "malformed")], + ) + facts: list[dict[str, Any]] = [] + for check in source.get("checks", []): + fact_name = check.get("fact", "unnamed") + expected = check.get("expected") + extractor = check.get("extractor") + pattern = check.get("pattern") + if extractor == "contains" and isinstance(pattern, str): + facts.append(_fact(source, fact_name, expected, pattern in text)) + continue + if extractor in {"regex_int", "regex_string"} and isinstance(pattern, str): + match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE) + if match is None: + facts.append( + _fact( + source, + fact_name, + expected, + None, + "malformed", + f"required pattern did not match: {pattern}", + ) + ) + continue + observed: Any = match.group(1) + if extractor == "regex_int": + try: + observed = int(observed) + except ValueError: + facts.append(_fact(source, fact_name, expected, observed, "malformed", "capture is not an integer")) + continue + facts.append(_fact(source, fact_name, expected, observed)) + continue + facts.append(_fact(source, fact_name, expected, None, "malformed", f"unsupported extractor {extractor!r}")) + return _source_result(source, facts) + + +def default_fetcher(url: str, timeout: float) -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": "sent-plugin-freshness-check/1"}) + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + + +def _unavailable(source: dict[str, Any], error: Exception, attempts: int) -> dict[str, Any]: + fact = _fact( + source, + "source.availability", + "available", + f"{type(error).__name__}: {error}", + "unavailable", + ) + return _source_result(source, [fact], attempts) + + +def _malformed(source: dict[str, Any], detail: str, attempts: int) -> dict[str, Any]: + fact = _fact(source, "source.parse", "parseable content", None, "malformed", detail) + return _source_result(source, [fact], attempts) + + +def _apply_conflicts(results: list[dict[str, Any]]) -> None: + observations: dict[str, list[dict[str, Any]]] = defaultdict(list) + for result in results: + if result["status"] in {"unavailable", "malformed"}: + continue + for fact in result["facts"]: + if fact["status"] in {"match", "drift"} and fact["observed"] is not None: + observations[fact["fact"]].append(fact) + for fact_name, facts in observations.items(): + values = {json.dumps(fact["observed"], sort_keys=True) for fact in facts} + if len(values) <= 1: + continue + rendered = sorted(json.loads(value) for value in values) + for fact in facts: + fact["status"] = "conflict" + fact["detail"] = f"authoritative sources disagree: {rendered}" + for result in results: + result["status"] = _source_status(result["facts"]) + + +def run_monitor( + catalog: dict[str, Any], + manifest: dict[str, Any], + fetcher: Fetcher = default_fetcher, + *, + timeout: float = 10, + retries: int = 2, +) -> dict[str, Any]: + results: list[dict[str, Any]] = [] + for source in catalog.get("sources", []): + payload: str | bytes | None = None + last_error: Exception | None = None + attempts = 0 + for attempts in range(1, retries + 2): + try: + payload = fetcher(source["url"], timeout) + last_error = None + break + except Exception as exc: # network adapters expose several timeout/error types + last_error = exc + if last_error is not None: + results.append(_unavailable(source, last_error, attempts)) + continue + if isinstance(payload, bytes): + try: + payload = payload.decode("utf-8") + except UnicodeDecodeError as exc: + results.append(_malformed(source, str(exc), attempts)) + continue + if not isinstance(payload, str): + results.append(_malformed(source, "fetcher did not return text or bytes", attempts)) + continue + if source.get("kind") == "openapi": + try: + document = json.loads(payload) + except json.JSONDecodeError as exc: + results.append(_malformed(source, f"invalid JSON: {exc}", attempts)) + continue + result = evaluate_openapi(document, manifest, source) + result["attempts"] = attempts + results.append(result) + else: + result = evaluate_document(payload, source) + result["attempts"] = attempts + results.append(result) + + _apply_conflicts(results) + counts = Counter(result["status"] for result in results) + diagnostics = [ + fact + for result in results + for fact in result["facts"] + if fact["status"] != "match" + ] + return { + "schema_version": 1, + "generated_at": datetime.datetime.now(datetime.UTC).isoformat(), + "summary": { + "sources": len(results), + **{status: counts[status] for status in ("ok", "drift", "conflict", "malformed", "unavailable")}, + }, + "sources": results, + "diagnostics": diagnostics, + } + + +def exit_code(report: dict[str, Any]) -> int: + statuses = {source["status"] for source in report.get("sources", [])} + if statuses & {"malformed", "unavailable"}: + return 2 + if statuses & {"drift", "conflict"}: + return 1 + return 0 + + +def compare(document: dict[str, Any], manifest: dict[str, Any]) -> list[str]: + """Compatibility wrapper returning human-readable OpenAPI drift messages.""" + source = { + "id": "openapi", + "url": manifest["source"], + "kind": "openapi", + "affected_skills": [], + "last_verified": manifest.get("verified_at"), + } + result = evaluate_openapi(document, manifest, source) + return [ + f"{fact['fact']}: expected={fact['expected']!r}, observed={fact['observed']!r}" + for fact in result["facts"] + if fact["status"] != "match" + ] + + +def _configuration_report(detail: str) -> dict[str, Any]: + source = { + "id": "monitor-configuration", + "url": "local", + "kind": "configuration", + "affected_skills": [], + "last_verified": None, + } + result = _malformed(source, detail, 0) + return { + "schema_version": 1, + "generated_at": datetime.datetime.now(datetime.UTC).isoformat(), + "summary": {"sources": 1, "ok": 0, "drift": 0, "conflict": 0, "malformed": 1, "unavailable": 0}, + "sources": [result], + "diagnostics": result["facts"], + } + + +def _write_report(path: Path | None, report: dict[str, Any]) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--openapi", type=Path, help="read a local OpenAPI document instead of the live URL") - args = parser.parse_args() - manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + parser.add_argument("--catalog", type=Path, default=CATALOG_PATH) + parser.add_argument("--manifest", type=Path, default=MANIFEST_PATH) + parser.add_argument("--openapi", type=Path, help="compatibility mode: check one local OpenAPI document") + parser.add_argument("--source-dir", type=Path, help="read .json/.txt files instead of the network") + parser.add_argument("--only", action="append", default=[], help="check only a named catalog source") + parser.add_argument("--timeout", type=float, default=10.0) + parser.add_argument("--retries", type=int, default=2) + parser.add_argument("--output", type=Path, help="write the JSON diagnostic artifact") + args = parser.parse_args(argv) + + try: + catalog = json.loads(args.catalog.read_text(encoding="utf-8")) + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + report = _configuration_report(str(exc)) + _write_report(args.output, report) + print(f"Freshness monitor configuration error: {exc}", file=sys.stderr) + return 2 + if args.timeout <= 0 or args.retries < 0: + report = _configuration_report("timeout must be positive and retries must be non-negative") + _write_report(args.output, report) + print("Freshness monitor configuration error: invalid timeout/retries", file=sys.stderr) + return 2 + + sources = catalog.get("sources", []) if args.openapi: - document = json.loads(args.openapi.read_text(encoding="utf-8")) + sources = [source for source in sources if source.get("id") == "openapi"] + elif args.only: + requested = set(args.only) + sources = [source for source in sources if source.get("id") in requested] + filtered_catalog = {**catalog, "sources": sources} + if not sources: + report = _configuration_report("no catalog sources selected") + _write_report(args.output, report) + print("Freshness monitor configuration error: no catalog sources selected", file=sys.stderr) + return 2 + + if args.openapi: + local_openapi = args.openapi + + def fetcher(url: str, timeout: float) -> bytes: + return local_openapi.read_bytes() + + elif args.source_dir: + source_dir = args.source_dir + kinds = {source["url"]: source.get("kind") for source in sources} + identifiers = {source["url"]: source["id"] for source in sources} + + def fetcher(url: str, timeout: float) -> bytes: + suffix = ".json" if kinds[url] == "openapi" else ".txt" + return (source_dir / f"{identifiers[url]}{suffix}").read_bytes() + else: - request = urllib.request.Request(manifest["source"], headers={"User-Agent": "sent-plugin-contract-check/1"}) - with urllib.request.urlopen(request, timeout=30) as response: - document = json.load(response) - errors = compare(document, manifest) - if errors: - print("Live Sent contract drift detected:", file=sys.stderr) - for error in errors: - print(f"- {error}", file=sys.stderr) - return 1 - print("Live Sent v3 critical contract matches the checked-in manifest.") - return 0 + fetcher = default_fetcher + + report = run_monitor( + filtered_catalog, + manifest, + fetcher, + timeout=args.timeout, + retries=args.retries, + ) + _write_report(args.output, report) + code = exit_code(report) + summary = report["summary"] + print( + "Freshness results: " + + ", ".join(f"{status}={summary[status]}" for status in ("ok", "drift", "conflict", "malformed", "unavailable")) + ) + for diagnostic in report["diagnostics"]: + print( + f"- {diagnostic['status']} {diagnostic['fact']}: " + f"expected={diagnostic['expected']!r}, observed={diagnostic['observed']!r} " + f"({diagnostic['source_url']})", + file=sys.stderr, + ) + return code if __name__ == "__main__": diff --git a/scripts/test_contracts.py b/scripts/test_contracts.py index 9bc822e..926cf3e 100644 --- a/scripts/test_contracts.py +++ b/scripts/test_contracts.py @@ -147,6 +147,23 @@ def test_source_catalog_covers_freshness_domains(self) -> None: self.assertTrue(set(source["affected_skills"]) <= set(metadata.skills)) datetime.date.fromisoformat(source["last_verified"]) + def test_network_monitoring_is_isolated_from_pr_validation(self) -> None: + validation = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + freshness = (ROOT / ".github" / "workflows" / "documentation-freshness.yml").read_text(encoding="utf-8") + self.assertNotIn("schedule:", validation) + self.assertNotIn("python scripts/check_live_contract.py\n", validation) + for required in ( + "workflow_dispatch:", + "release:", + "schedule:", + "python scripts/check_live_contract.py", + "--output artifacts/documentation-freshness.json", + "if: always()", + "actions/upload-artifact@", + "python scripts/run_model_routing_eval.py", + ): + self.assertIn(required, freshness) + class BundledExampleTests(unittest.TestCase): def test_every_markdown_json_block_parses(self) -> None: diff --git a/scripts/test_live_contract.py b/scripts/test_live_contract.py new file mode 100644 index 0000000..91a7601 --- /dev/null +++ b/scripts/test_live_contract.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Offline simulations for Sent contract and documentation freshness monitoring.""" + +from __future__ import annotations + +import copy +import json +import unittest +from pathlib import Path + +import check_live_contract as CHECKER + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = json.loads((ROOT / "schemas" / "sent" / "v3-contract-manifest.json").read_text(encoding="utf-8")) + + +def minimal_openapi() -> dict: + schemas = { + "SentDmServicesEndpointsCustomerAPIv3RequestsCreateTemplateRequest": { + "type": "object", + "required": MANIFEST["template_create"]["required_fields"], + "properties": {name: {} for name in MANIFEST["template_create"]["allowed_fields"]}, + }, + "SentDmServicesCommonEntitiesTemplateBodyContent": { + "type": "object", + "properties": {"template": {"type": "string", "maxLength": MANIFEST["template_create"]["body_max_length"]}}, + }, + "SentDmServicesCommonEntitiesTemplateButton": { + "type": "object", + "properties": {"type": {"description": " ".join(MANIFEST["template_create"]["button_types"])}}, + }, + "SentDmServicesEndpointsCustomerAPIv3RequestsCreateProfileRequest": { + "type": "object", + "required": MANIFEST["profile"]["create_required_fields"], + }, + "SentDmServicesEndpointsCustomerAPIv3RequestsWhatsappBusinessAccountCredentials": { + "type": "object", + "required": MANIFEST["profile"]["waba_required_fields"], + }, + "SentDmServicesEndpointsCustomerAPIv3RequestsProfilesCompleteProfileRequest": { + "type": "object", + "required": MANIFEST["profile"]["complete_required_fields"], + }, + "SentDmServicesEndpointsCustomerAPIv3RequestsCampaignsCampaignData": { + "type": "object", + "required": MANIFEST["campaign"]["required_fields"], + }, + "SentDmServicesCommonEnumsMessagingUseCaseUS": { + "type": "string", + "enum": MANIFEST["campaign"]["use_case_values"], + }, + "SentDmServicesEndpointsCustomerAPIv3RequestsCampaignsCampaignUseCaseData": { + "type": "object", + "properties": { + "sampleMessages": { + "type": "array", + "minItems": MANIFEST["campaign"]["sample_min"], + "maxItems": MANIFEST["campaign"]["sample_max"], + } + }, + }, + } + paths = { + path: {method.lower(): {"responses": {"200": {"description": "ok"}}} for method in methods} + for path, methods in MANIFEST["critical_paths"].items() + } + return copy.deepcopy({"openapi": "3.1.0", "paths": paths, "components": {"schemas": schemas}}) + + +def source(identifier: str, url: str, kind: str, checks: list[dict] | None = None) -> dict: + value = { + "id": identifier, + "url": url, + "kind": kind, + "affected_skills": ["sent"], + "last_verified": "2026-08-09", + } + if checks is not None: + value["checks"] = checks + return value + + +class OpenApiDriftTests(unittest.TestCase): + def assert_drift(self, document: dict, fact_fragment: str) -> None: + result = CHECKER.evaluate_openapi(document, MANIFEST, source("openapi", "https://example.com/openapi.json", "openapi")) + self.assertEqual(result["status"], "drift") + self.assertTrue(any(fact_fragment in fact["fact"] for fact in result["facts"] if fact["status"] == "drift")) + + def test_endpoint_removal(self) -> None: + document = minimal_openapi() + document["paths"].pop("/v3/messages") + self.assert_drift(document, "path./v3/messages.methods") + + def test_enum_addition(self) -> None: + document = minimal_openapi() + document["components"]["schemas"]["SentDmServicesCommonEnumsMessagingUseCaseUS"]["enum"].append("NEW_CASE") + self.assert_drift(document, "campaign.use_case_values") + + def test_property_change(self) -> None: + document = minimal_openapi() + document["components"]["schemas"]["SentDmServicesEndpointsCustomerAPIv3RequestsCreateTemplateRequest"]["properties"]["newField"] = {} + self.assert_drift(document, "template.allowed_fields") + + +class SourceFailureTests(unittest.TestCase): + def test_source_conflict_is_semantic_exit_one(self) -> None: + check = {"fact": "template.body_max_length", "extractor": "regex_int", "pattern": r"Limit: (\d+)", "expected": 1024} + catalog = {"sources": [ + source("first", "https://example.com/first.txt", "documentation", [check]), + source("second", "https://example.com/second.txt", "documentation", [check]), + ]} + bodies = {"https://example.com/first.txt": "Limit: 1024", "https://example.com/second.txt": "Limit: 2048"} + report = CHECKER.run_monitor(catalog, MANIFEST, lambda url, timeout: bodies[url], timeout=1, retries=0) + self.assertEqual(CHECKER.exit_code(report), 1) + self.assertEqual({item["status"] for item in report["sources"]}, {"conflict"}) + + def test_timeout_retries_are_bounded_and_exit_two(self) -> None: + calls = 0 + + def timeout_fetcher(url: str, timeout: float) -> str: + nonlocal calls + calls += 1 + raise TimeoutError("synthetic timeout") + + catalog = {"sources": [source("docs", "https://example.com/docs.txt", "documentation", [])]} + report = CHECKER.run_monitor(catalog, MANIFEST, timeout_fetcher, timeout=0.01, retries=2) + self.assertEqual(calls, 3) + self.assertEqual(report["sources"][0]["status"], "unavailable") + self.assertEqual(CHECKER.exit_code(report), 2) + + def test_malformed_openapi_response_exits_two(self) -> None: + catalog = {"sources": [source("openapi", "https://example.com/openapi.json", "openapi")]} + report = CHECKER.run_monitor(catalog, MANIFEST, lambda url, timeout: "{", timeout=1, retries=0) + self.assertEqual(report["sources"][0]["status"], "malformed") + self.assertEqual(CHECKER.exit_code(report), 2) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/validate.py b/scripts/validate.py index 7acafce..1fdcb41 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -343,7 +343,7 @@ def validate_documentation_sources(validation: Validation) -> None: if not isinstance(source, dict): continue validation.check( - set(source) == {"id", "url", "kind", "affected_skills", "last_verified"}, + set(source) == {"id", "url", "kind", "affected_skills", "last_verified", "checks"}, f"documentation source {index} fields drifted", ) identifier = source.get("id") @@ -370,6 +370,28 @@ def validate_documentation_sources(validation: Validation) -> None: except ValueError: valid_date = False validation.check(valid_date, f"documentation source {identifier} requires a valid last_verified date") + checks = source.get("checks") + validation.check(isinstance(checks, list), f"documentation source {identifier} checks must be an array") + if isinstance(checks, list): + for check_index, check in enumerate(checks, 1): + validation.check( + isinstance(check, dict) + and set(check) == {"fact", "extractor", "pattern", "expected"}, + f"documentation source {identifier} check {check_index} fields drifted", + ) + if not isinstance(check, dict): + continue + validation.check( + check.get("extractor") in {"contains", "regex_int", "regex_string"}, + f"documentation source {identifier} check {check_index} has invalid extractor", + ) + validation.check( + isinstance(check.get("fact"), str) + and bool(check.get("fact")) + and isinstance(check.get("pattern"), str) + and bool(check.get("pattern")), + f"documentation source {identifier} check {check_index} requires fact and pattern", + ) def validate_skills(validation: Validation) -> None: