From 4b827974cd4f29d803395cacb401b8e18582b744 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 9 Sep 2026 19:53:44 -0400 Subject: [PATCH] Add private KFH outreach action aggregates with legacy compatibility --- CHANGELOG.md | 6 + KFH_ANALYTICS_CONTRACT.md | 14 ++ KFH_OUTREACH_RELEASE.md | 116 ++++++++++++ OPERATIONS.md | 4 + SOT.md | 14 ++ contracts/kfh-v1/outreach-empty.json | 112 ++++++++++++ contracts/kfh-v1/outreach-sample.json | 172 ++++++++++++++++++ contracts/kfh-v1/outreach-unavailable.json | 55 ++++++ .../0017_add_kfh_outreach_attribution.sql | 16 ++ package-lock.json | 4 +- package.json | 2 +- src/kfhAnalytics.ts | 98 ++++++++-- src/kfhContract.ts | 26 ++- src/kfhOutreachContract.ts | 48 +++++ tests/kfh-analytics.test.mjs | 69 ++++++- tests/release-control.test.mjs | 4 +- 16 files changed, 725 insertions(+), 35 deletions(-) create mode 100644 KFH_OUTREACH_RELEASE.md create mode 100644 contracts/kfh-v1/outreach-empty.json create mode 100644 contracts/kfh-v1/outreach-sample.json create mode 100644 contracts/kfh-v1/outreach-unavailable.json create mode 100644 migrations/0017_add_kfh_outreach_attribution.sql create mode 100644 src/kfhOutreachContract.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e27dee4..3fe16f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [1.34.0] - 2026-09-09 (review candidate) + +- Add strict KFH v3 public outreach labels on broad actions with atomic independent daily margins and additive migration 0017. +- Preserve v1/v2 ingestion, old-table rollback compatibility and 400-day aggregate retention; emit strict report 1.2 with honest unclassified history. +- Add matching producer/consumer fixtures, privacy/reconciliation/rollback tests and coordinated release instructions. No migration or production deployment performed. + ## [1.33.0] - 2026-09-05 - Added owner-authorized KFH ingestion v2 for default-on/opt-out collection without falsely asserting consent. Retained strict v1 compatibility and all suppression/privacy limits. diff --git a/KFH_ANALYTICS_CONTRACT.md b/KFH_ANALYTICS_CONTRACT.md index e002a24..6ee9bed 100644 --- a/KFH_ANALYTICS_CONTRACT.md +++ b/KFH_ANALYTICS_CONTRACT.md @@ -1,5 +1,19 @@ # Kingston Food Help analytics contract +## Current reviewed contract — 1.34.0, public outreach action attribution + +Jamie authorized the coordinated code changes, review branches, commits/pushes and pull requests on 2026-09-09. These are review candidates; this work does not authorize or record a main merge, migration application, production deployment, secret or settings change. The current production report pasted by Jamie contains observed activity; it is not a website-health or outcome receipt. + +The coordinated target is Kingston 0.2.0 (ingestion v3), Lighthouse 1.34.0 (report 1.2) and Smith 0.29.0 (strict 1.0/1.1/1.2 consumer). Deploy Smith first; apply Lighthouse migration 0017 after its review and before manually promoting Lighthouse; deploy the website last. Preserve cached v1/v2 ingestion and old reports for compatibility. See the release review for exact scope and rollback. + +Ingestion v3 requires all three fixed public labels on page_view and the existing four broad clicks. It adds only source reddit, campaign outreach_2026_09 and content post_02. Installation remains an unattributed browser signal. v1/v2 parsing stays exact and does not accept action attribution or new enums. + +One atomic D1 batch maintains the original kfh_daily totals and page margins and the new kfh_outreach_daily independent day/event/dimension/value margins. New labels map to old fallbacks in the original table so old Workers can read it. There are no raw events, combined source/campaign/content tuples, visitor/session IDs, provider IDs, destinations, search/filter/location data or new abuse identifiers. Existing GPC/DNT, origins, body/rate limits and credentials remain. Both tables retain today plus 399 previous UTC days under the existing pruning schedule. + +Report 1.2 reconciles exact totals, three independent outreach margins and classified/unclassified counts. It replaces the fallback portion of page-view discovery with actual v3 labels without double counting. Historical actions stay unclassified; old page-view labels are never assigned to them. A missing new table, failed query or inconsistent report is unavailable, with null metrics. Public 204 is fail-soft receipt, not proof of persistence. No other site/report/storage semantics change. + +`KFH_ANALYTICS_CONTRACT.md` and `KFH_OUTREACH_RELEASE.md` govern this reviewed expansion. Two strict TypeScript contract files and three additional producer fixtures are identical in Smith. Migration 0016 remains unchanged; 0017 is additive and has not been applied. + ## Current owner-authorized revision — ingestion 2 / report 1.1 Jamie explicitly changed Kingston to default-on aggregate analytics with an opt-out in Your privacy and no popup, and authorized the coordinated changes and production deployment. Existing saved no choices, GPC/DNT, dev_mode and noAnalytics suppression remain effective. No additional context, identifiers, raw history, queue, retries or third-party analytics are introduced. diff --git a/KFH_OUTREACH_RELEASE.md b/KFH_OUTREACH_RELEASE.md new file mode 100644 index 0000000..cbe3afb --- /dev/null +++ b/KFH_OUTREACH_RELEASE.md @@ -0,0 +1,116 @@ +# Coordinated outreach analytics release review + +Prepared 2026-09-09 for Jamie's review. Branch `codex/kfh-outreach-analytics` in all three repositories. Implementation, local testing and branch/PR publication are authorized. Nothing here is a migration or production deployment receipt. + +## Owner release sequence + +1. Review and merge **Agent Smith 0.29.0**. Its existing main-push workflow tests, deploys and registers commands. Verify that deployment succeeds before changing the producer. The command definition itself is unchanged; this is consumer compatibility for old KFH reports and new 1.2. +2. Review and merge **Lighthouse 1.34.0**. Main publication uploads a Worker version; it does **not** promote production. Separately approve and apply exactly `migrations/0017_add_kfh_outreach_attribution.sql` to the existing Lighthouse `DB`, after confirming 0016 and the pending-migration inventory. Verify the new constrained table and successful migration receipt. Do not apply unrelated pending migrations. Then explicitly run the existing manual, main-only `Deploy Lighthouse to Cloudflare` GitHub workflow and verify its deployment receipt. Validate the protected KFH report 1.2 and the existing private command through owner-approved production verification. +3. Review and merge **Kingston Food Help 0.2.0** last. Its native Cloudflare Pages main integration builds and deploys with the existing analytics-enabled setting. Verify the deployed build, opt-out, navigation and saved/offline/update behavior; physical-phone handoff remains a release check. Then replace the Facebook/Reddit links in the outreach plan and record the edit date. + +Do not merge all three simultaneously: a v3 website published before Lighthouse will have its new events dropped by the old parser. An old Smith reader cannot consume report 1.2. A new Lighthouse Worker without migration 0017 returns unavailable KFH reporting and cannot persist v3 batches. + +## 1. Exact files and settings affected + +The inventories below list the coordinated review files. No dependency version, binding, route, secret, schedule, DNS, Pages setting or GitHub workflow changes. Only package root version metadata changes in the lockfiles. No resource data or service-worker source changes. + +### Kingston Food Help 0.1.0 → 0.2.0 + +- `CHANGELOG.md` +- `README.md` +- `SOT.md` +- `docs/ANALYTICS_POLICY.md` +- `docs/ANALYTICS_REVIEW.md` +- `docs/APPROVAL_REGISTER.md` +- `docs/OPERATIONS.md` +- `docs/OUTREACH_LOG.csv` +- `docs/OUTREACH_PLAN.md` +- `docs/OUTREACH_RELEASE.md` +- `package-lock.json` +- `package.json` +- `src/analytics.ts` +- `src/app.ts` +- `tests/e2e/analytics.spec.ts` +- `tests/e2e/server.mjs` +- `tests/unit/analytics.test.ts` + +### Lighthouse 1.33.0 → 1.34.0 + +- `CHANGELOG.md` +- `KFH_ANALYTICS_CONTRACT.md` +- `KFH_OUTREACH_RELEASE.md` +- `OPERATIONS.md` +- `SOT.md` +- `contracts/kfh-v1/outreach-empty.json` +- `contracts/kfh-v1/outreach-sample.json` +- `contracts/kfh-v1/outreach-unavailable.json` +- `migrations/0017_add_kfh_outreach_attribution.sql` +- `package-lock.json` +- `package.json` +- `src/kfhAnalytics.ts` +- `src/kfhContract.ts` +- `src/kfhOutreachContract.ts` +- `tests/kfh-analytics.test.mjs` +- `tests/release-control.test.mjs` + +### Agent Smith 0.28.2 → 0.29.0 + +- `CHANGELOG.md` +- `KFH_OUTREACH_RELEASE.md` +- `KFH_REPORTING_CONTRACT.md` +- `OPERATIONS.md` +- `SOT.md` +- `VERSION` +- `package-lock.json` +- `package.json` +- `src/contracts/kfh-v1/outreach-empty.json` +- `src/contracts/kfh-v1/outreach-sample.json` +- `src/contracts/kfh-v1/outreach-unavailable.json` +- `src/contracts/kfhContract.ts` +- `src/contracts/kfhOutreachContract.ts` +- `src/logic/kfhReport.ts` +- `tests/kfh_command.test.ts` +- `tests/kfh_runtime.test.ts` + +## 2. Visible user impact + +The website's existing privacy details explain public outreach action totals and completion of a click already sent when leaving the page. Calls and links still navigate immediately. Default-on collection, remembered off, GPC/DNT/operator suppression and the simple directory remain. Smith's existing ephemeral /kfh shows Recorded page loads, independent outreach loads/actions, unclassified history and campaign-tag coverage. Full UTC windows and outcome/privacy limitations stay visible. There are no new posts, public dashboards or scheduled reports. + +## 3. Information and verification impact + +No food-support record, location, hours, eligibility, warning, source URL or verification date changes. The supplied Lunch by George email and Reddit offer remain unverified submissions for a separate content review. Page-view counts mean app startup under the existing one-attempt semantics, not proof that resource data loaded. Historical actions cannot be reconstructed by joining old page labels. Zero history is distinct from query failure; sparse observations do not establish completeness or health. + +## 4. Analytics and privacy impact + +Strict ingestion v3 adds only three fixed public labels: reddit, outreach_2026_09 and post_02. Page starts and the four existing broad click types carry source/campaign/content labels; installation stays unattributed. The server stores independent daily day/event/dimension/value counts, never a combined source/campaign/content record or visitor journey. The existing table keeps overall counts and legacy-compatible page margins. A single atomic write prevents partial totals across the two tables. Report 1.2 reconciles every dimension and classified/unclassified count; old 1.0/1.1 contracts remain supported by Smith. + +No provider/destination identity, search/filter/location context, URL/referrer text, timestamped raw events, user/session ID, new analytics cookie, fingerprinting or vendor. Existing rotating minute abuse counters and protected-report credentials remain. Aggregate retention is 400 days in both tables under the existing prune schedule. New event delivery remains best effort: a public 204 does not promise persistence, client labels/origins are not authentication, and privacy/blocking/offline use limits coverage. + +## 5. Offline and cache impact + +The service worker still ignores analytics and never caches, retries, queues or replays events. Initial hidden/offline startup attempts remain dropped without replay. Only already-started visible action requests get keepalive and survive pagehide/visibility loss; pending page/install requests abort. Opt-out/privacy suppression and offline events still abort pending requests. The 1,500ms timer depends on a runnable page realm and cannot guarantee a deadline after the browser freezes or destroys it. Website navigation and offline access do not await analytics. The generated precache revision changes with the new application bundle; saved old clients remain accepted through v1/v2 compatibility. + +## 6. Verification and unresolved limits + +Local checks cover strict payload rejection and privacy controls, new/legacy mixed history, constrained SQL and atomic rollback across both tables, 400-day pruning, missing migration, corrupt margins, exact producer/consumer fixtures, report limits and redirects, native Worker command execution, and delayed click-response completion through real same-tab/new-tab Chromium navigation. The navigation fixture uses a loopback collector with an explicit local-network permission; no traffic reaches the production collector. The full PWA suite covers first/repeat visits, offline launch, application/data update, rollback and stale-cache removal. + +Final local verification: Lighthouse typecheck and 214 tests passed; Smith typecheck, governance, 126 tests (including the native Worker command) and local Worker bundle inspection passed; Kingston 91 unit tests, production build and 17 Chromium checks passed. PR descriptions will record remote CI results. Shared contract files and all seven fixtures match byte-for-byte. Local Windows execution uses Node 24.12.0 and the existing lockfile-resolved dependencies. Worktree dependency paths avoid a Vite issue with `#` in the original directory; no package versions changed. The Worker runtime needs execution outside the restricted sandbox. Local success is not a physical-phone/iOS test, remote migration receipt or production rollout proof. No production reports, D1 rows, secrets or Discord messages were read or written during implementation. + +## 7. External effects and verified release controls + +Read-only control-plane checks on 2026-09-09 confirmed the Pages production branch is main, preview branches include only review/*, and the analytics tag/token settings are null. This codex/ review branch does not match that Pages preview rule. Lighthouse's main and non-main Workers Builds triggers both use `npx wrangler versions upload`; branch publication can create a preview/version, not an active-production promotion. Smith has no Workers Builds trigger and uses the checked-in GitHub main-push deployment workflow. Recheck if settings change before release. + +Canonical Lighthouse account: `eb1a8dd5723031d94e57642e3eaaebda`; Worker `buscore-lighthouse`; D1 `lighthouse`, binding `DB`, database ID `e46f2daa-7e97-45a3-9bf0-49003a42850c`. The new table is in the existing database; no new Cloudflare resource is created by these review branches. + +Owner review alone does not run a migration or dispatch Lighthouse production. Merging Smith/Kingston starts their existing deployment automation; merging Lighthouse uploads only. Branch pushes and PR creation are the only publication actions in this work. Existing automation may record checks and a Lighthouse preview upload. No merge, tag, release, settings/secret change, migration, production promotion or outreach message is performed by the preparer. + +## Rollback + +Review exact immutable versions/commits before executing rollback. Roll the website back to v2 first, then Lighthouse to its compatible old Worker if needed; Smith 0.29.0 accepts old reports. Roll Smith back only after the producer again serves an older report. Migration 0017 leaves the old table/constraints untouched, and new labels have legacy fallbacks there. Never relabel historic actions using old page-view attribution. + +Worker rollback does not roll back D1. Old Lighthouse versions cannot prune the new table: include a separately approved purge of the unused outreach table or continued retention maintenance if that rollback persists. Do not drop or modify the original daily table. A later v3 rollout after a purge must show those earlier actions as unclassified. Keep the normal website application-update/rollback/cache verification and saved privacy preferences. + +## References + +- [Fetch keepalive](https://developer.mozilla.org/en-US/docs/Web/API/Request/keepalive) describes the browser transport feature; it is not a guarantee of delivery. +- [D1 prepared-statement batch](https://developers.cloudflare.com/d1/worker-api/d1-database/#batch) describes transactional batch behavior. Local SQLite fixtures test the checked-in SQL; they do not apply it to Cloudflare. diff --git a/OPERATIONS.md b/OPERATIONS.md index 6843f6c..ef6ddae 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -1,5 +1,9 @@ # Lighthouse Operations and Diagnostics +## 2026-09-09 outreach review candidate + +The coordinated review changes and exact owner-controlled release order are in [KFH_OUTREACH_RELEASE.md](KFH_OUTREACH_RELEASE.md). Branch publication is authorized; production activation is pending owner review. Older dated operations below remain historical where the new release review supersedes them. + ## Kingston default-on rollout — 1.33.0 Jamie explicitly changed Kingston to default-on aggregate analytics with an opt-out in Your privacy and no popup, and authorized the coordinated changes and production deployment. Existing saved no choices, GPC/DNT, dev_mode and noAnalytics suppression remain effective. No additional context, identifiers, raw history, queue, retries or third-party analytics are introduced. diff --git a/SOT.md b/SOT.md index e58641f..e49ad20 100644 --- a/SOT.md +++ b/SOT.md @@ -1,5 +1,19 @@ # Lighthouse — Source of Truth +## Review candidate — 1.34.0, public outreach action attribution + +Jamie authorized the coordinated code changes, review branches, commits/pushes and pull requests on 2026-09-09. These are review candidates; this work does not authorize or record a main merge, migration application, production deployment, secret or settings change. The current production report pasted by Jamie contains observed activity; it is not a website-health or outcome receipt. + +The coordinated target is Kingston 0.2.0 (ingestion v3), Lighthouse 1.34.0 (report 1.2) and Smith 0.29.0 (strict 1.0/1.1/1.2 consumer). Deploy Smith first; apply Lighthouse migration 0017 after its review and before manually promoting Lighthouse; deploy the website last. Preserve cached v1/v2 ingestion and old reports for compatibility. See the release review for exact scope and rollback. + +Ingestion v3 requires all three fixed public labels on page_view and the existing four broad clicks. It adds only source reddit, campaign outreach_2026_09 and content post_02. Installation remains an unattributed browser signal. v1/v2 parsing stays exact and does not accept action attribution or new enums. + +One atomic D1 batch maintains the original kfh_daily totals and page margins and the new kfh_outreach_daily independent day/event/dimension/value margins. New labels map to old fallbacks in the original table so old Workers can read it. There are no raw events, combined source/campaign/content tuples, visitor/session IDs, provider IDs, destinations, search/filter/location data or new abuse identifiers. Existing GPC/DNT, origins, body/rate limits and credentials remain. Both tables retain today plus 399 previous UTC days under the existing pruning schedule. + +Report 1.2 reconciles exact totals, three independent outreach margins and classified/unclassified counts. It replaces the fallback portion of page-view discovery with actual v3 labels without double counting. Historical actions stay unclassified; old page-view labels are never assigned to them. A missing new table, failed query or inconsistent report is unavailable, with null metrics. Public 204 is fail-soft receipt, not proof of persistence. No other site/report/storage semantics change. + +`KFH_ANALYTICS_CONTRACT.md` and `KFH_OUTREACH_RELEASE.md` govern this reviewed expansion. Two strict TypeScript contract files and three additional producer fixtures are identical in Smith. Migration 0016 remains unchanged; 0017 is additive and has not been applied. + ## Current source — 1.33.0, Kingston default-on policy Jamie explicitly changed Kingston to default-on aggregate analytics with an opt-out in Your privacy and no popup, and authorized the coordinated changes and production deployment. Existing saved no choices, GPC/DNT, dev_mode and noAnalytics suppression remain effective. No additional context, identifiers, raw history, queue, retries or third-party analytics are introduced. diff --git a/contracts/kfh-v1/outreach-empty.json b/contracts/kfh-v1/outreach-empty.json new file mode 100644 index 0000000..434c978 --- /dev/null +++ b/contracts/kfh-v1/outreach-empty.json @@ -0,0 +1,112 @@ +{ + "view": "kfh", + "report_contract_version": "1.2", + "site_key": "kingston_food_help", + "generated_at": "2026-09-04T12:00:00.000Z", + "source": { + "availability": "available", + "reason": "no_observed_history", + "first_observed_day": null, + "last_observed_day": null + }, + "windows": { + "today": { + "start_day": "2026-09-04", + "end_day": "2026-09-04", + "partial": true, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "latest_complete_day": { + "start_day": "2026-09-03", + "end_day": "2026-09-03", + "partial": false, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "last_7_complete_days": { + "start_day": "2026-08-28", + "end_day": "2026-09-03", + "partial": false, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "previous_7_complete_days": { + "start_day": "2026-08-21", + "end_day": "2026-08-27", + "partial": false, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "last_30_complete_days": { + "start_day": "2026-08-05", + "end_day": "2026-09-03", + "partial": false, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + } + }, + "discovery_last_7_complete_days": { + "sources": [], + "campaigns": [], + "contents": [] + }, + "outreach_last_7_complete_days": { + "classified": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0 + }, + "unclassified": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0 + }, + "sources": [], + "campaigns": [], + "contents": [] + }, + "limitations": { + "coverage": "observed_only", + "counts_are": "observed_activity_not_people_or_service_outcomes", + "attribution": "separate_daily_public_outreach_action_totals", + "activity_is_health": false, + "raw_events_stored": false, + "identifiers_reported": false, + "aggregate_retention_days": 400 + } +} diff --git a/contracts/kfh-v1/outreach-sample.json b/contracts/kfh-v1/outreach-sample.json new file mode 100644 index 0000000..97fd518 --- /dev/null +++ b/contracts/kfh-v1/outreach-sample.json @@ -0,0 +1,172 @@ +{ + "view": "kfh", + "report_contract_version": "1.2", + "site_key": "kingston_food_help", + "generated_at": "2026-09-04T12:00:00.000Z", + "source": { + "availability": "available", + "reason": "observed_activity", + "first_observed_day": "2026-09-03", + "last_observed_day": "2026-09-03" + }, + "windows": { + "today": { + "start_day": "2026-09-04", + "end_day": "2026-09-04", + "partial": true, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "latest_complete_day": { + "start_day": "2026-09-03", + "end_day": "2026-09-03", + "partial": false, + "counts": { + "page_views": 3, + "resource_calls": 1, + "help_211": 0, + "directions": 1, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "last_7_complete_days": { + "start_day": "2026-08-28", + "end_day": "2026-09-03", + "partial": false, + "counts": { + "page_views": 3, + "resource_calls": 1, + "help_211": 0, + "directions": 1, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "previous_7_complete_days": { + "start_day": "2026-08-21", + "end_day": "2026-08-27", + "partial": false, + "counts": { + "page_views": 0, + "resource_calls": 0, + "help_211": 0, + "directions": 0, + "official_sources": 0, + "pwa_installs": 0 + } + }, + "last_30_complete_days": { + "start_day": "2026-08-05", + "end_day": "2026-09-03", + "partial": false, + "counts": { + "page_views": 3, + "resource_calls": 1, + "help_211": 0, + "directions": 1, + "official_sources": 0, + "pwa_installs": 0 + } + } + }, + "discovery_last_7_complete_days": { + "sources": [ + { + "value": "reddit", + "count": 2 + }, + { + "value": "facebook", + "count": 1 + } + ], + "campaigns": [ + { + "value": "outreach_2026_09", + "count": 2 + }, + { + "value": "launch_2026_09", + "count": 1 + } + ], + "contents": [ + { + "value": "post_02", + "count": 2 + }, + { + "value": "post_01", + "count": 1 + } + ] + }, + "outreach_last_7_complete_days": { + "classified": { + "page_views": 2, + "resource_calls": 0, + "help_211": 0, + "directions": 1, + "official_sources": 0 + }, + "unclassified": { + "page_views": 1, + "resource_calls": 1, + "help_211": 0, + "directions": 0, + "official_sources": 0 + }, + "sources": [ + { + "event": "directions", + "value": "reddit", + "count": 1 + }, + { + "event": "page_views", + "value": "reddit", + "count": 2 + } + ], + "campaigns": [ + { + "event": "directions", + "value": "outreach_2026_09", + "count": 1 + }, + { + "event": "page_views", + "value": "outreach_2026_09", + "count": 2 + } + ], + "contents": [ + { + "event": "directions", + "value": "post_02", + "count": 1 + }, + { + "event": "page_views", + "value": "post_02", + "count": 2 + } + ] + }, + "limitations": { + "coverage": "observed_only", + "counts_are": "observed_activity_not_people_or_service_outcomes", + "attribution": "separate_daily_public_outreach_action_totals", + "activity_is_health": false, + "raw_events_stored": false, + "identifiers_reported": false, + "aggregate_retention_days": 400 + } +} diff --git a/contracts/kfh-v1/outreach-unavailable.json b/contracts/kfh-v1/outreach-unavailable.json new file mode 100644 index 0000000..9c82173 --- /dev/null +++ b/contracts/kfh-v1/outreach-unavailable.json @@ -0,0 +1,55 @@ +{ + "view": "kfh", + "report_contract_version": "1.2", + "site_key": "kingston_food_help", + "generated_at": "2026-09-04T12:00:00.000Z", + "source": { + "availability": "unavailable", + "reason": "query_failed", + "first_observed_day": null, + "last_observed_day": null + }, + "windows": { + "today": { + "start_day": "2026-09-04", + "end_day": "2026-09-04", + "partial": true, + "counts": null + }, + "latest_complete_day": { + "start_day": "2026-09-03", + "end_day": "2026-09-03", + "partial": false, + "counts": null + }, + "last_7_complete_days": { + "start_day": "2026-08-28", + "end_day": "2026-09-03", + "partial": false, + "counts": null + }, + "previous_7_complete_days": { + "start_day": "2026-08-21", + "end_day": "2026-08-27", + "partial": false, + "counts": null + }, + "last_30_complete_days": { + "start_day": "2026-08-05", + "end_day": "2026-09-03", + "partial": false, + "counts": null + } + }, + "discovery_last_7_complete_days": null, + "outreach_last_7_complete_days": null, + "limitations": { + "coverage": "observed_only", + "counts_are": "observed_activity_not_people_or_service_outcomes", + "attribution": "separate_daily_public_outreach_action_totals", + "activity_is_health": false, + "raw_events_stored": false, + "identifiers_reported": false, + "aggregate_retention_days": 400 + } +} diff --git a/migrations/0017_add_kfh_outreach_attribution.sql b/migrations/0017_add_kfh_outreach_attribution.sql new file mode 100644 index 0000000..712502e --- /dev/null +++ b/migrations/0017_add_kfh_outreach_attribution.sql @@ -0,0 +1,16 @@ +-- Apply after owner approval and before the 1.34 Worker is promoted. +-- Additive: existing totals/constraints stay readable by rollback Workers. +-- No raw events, identities or joint source/campaign/content rows. +CREATE TABLE IF NOT EXISTS kfh_outreach_daily ( + day TEXT NOT NULL CHECK (day GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'), + event TEXT NOT NULL CHECK (event IN ('page_views', 'resource_calls', 'help_211', 'directions', 'official_sources')), + dimension TEXT NOT NULL, + value TEXT NOT NULL, + count INTEGER NOT NULL CHECK (typeof(count) = 'integer' AND count > 0), + PRIMARY KEY (day, event, dimension, value), + CHECK ( + (dimension = 'source' AND value IN ('direct_unknown', 'facebook', 'community', 'search', 'other', 'reddit')) OR + (dimension = 'campaign' AND value IN ('none', 'launch_2026_09', 'outreach_2026_09')) OR + (dimension = 'content' AND value IN ('none', 'post_01', 'poster_01', 'post_02')) + ) +) WITHOUT ROWID; diff --git a/package-lock.json b/package-lock.json index 371cc8d..a418303 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "buscore-lighthouse", - "version": "1.33.0", + "version": "1.34.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "buscore-lighthouse", - "version": "1.33.0", + "version": "1.34.0", "license": "ISC", "devDependencies": { "@cloudflare/workers-types": "^4.20260305.0", diff --git a/package.json b/package.json index 61e3f30..af1491e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "buscore-lighthouse", - "version": "1.33.0", + "version": "1.34.0", "description": "Standalone deterministic metrics worker: manifest proxy + fixed daily counters + protected on-demand reporting.", "scripts": { "dev": "wrangler dev", diff --git a/src/kfhAnalytics.ts b/src/kfhAnalytics.ts index 50e3693..02510cd 100644 --- a/src/kfhAnalytics.ts +++ b/src/kfhAnalytics.ts @@ -1,5 +1,7 @@ -import { KFH_SITE_KEY, KFH_ORIGINS, KFH_SOURCES, KFH_CAMPAIGNS, KFH_CONTENTS, KFH_COUNT_KEYS, KFH_WINDOW_KEYS, KFH_LIMITATIONS, type CountKey, type Counts, type WindowKey, type KfhReport, isKfhReport } from "./kfhContract.js"; +import { KFH_SITE_KEY, KFH_ORIGINS, KFH_SOURCES, KFH_CAMPAIGNS, KFH_CONTENTS, KFH_COUNT_KEYS, KFH_WINDOW_KEYS, KFH_OUTREACH_LIMITATIONS, type CountKey, type Counts, type WindowKey, type KfhReport, isKfhReport } from "./kfhContract.js"; +import { KFH_OUTREACH_SOURCES, KFH_OUTREACH_CAMPAIGNS, KFH_OUTREACH_CONTENTS, KFH_ATTRIBUTABLE_KEYS, type AttributableKey, type Outreach, type OutreachCounts, type OutreachRow } from "./kfhOutreachContract.js"; type Row = { day: string; metric: string; value: string; count: number }; +type StoredOutreach = OutreachRow & { day: string; dimension: string }; type Dimension = { value: string; count: number }; function object(value: unknown): value is Record { @@ -9,32 +11,50 @@ function member(value: unknown, allowed: readonly string[]): value is string { return typeof value === "string" && allowed.includes(value); } -export function parseKfhEvent(value: unknown): { counter: CountKey; source?: string; campaign?: string; content?: string } | null { +export function parseKfhEvent(value: unknown): { counter: CountKey; outreach?: true; source?: string; campaign?: string; content?: string } | null { if (!object(value)) return null; if (value.site_key !== KFH_SITE_KEY || value.page !== "directory") return null; const legacy = value.contract_version === 1 && value.consent === true; - const optOut = value.contract_version === 2 && value.collection_mode === "opt_out"; + const outreach = value.contract_version === 3 && value.collection_mode === "opt_out"; + const optOut = (value.contract_version === 2 || outreach) && value.collection_mode === "opt_out"; if (!legacy && !optOut) return null; const keys = ["site_key", "contract_version", legacy ? "consent" : "collection_mode", "page", "event_name"]; - if (value.event_name === "page_view") { + if (value.event_name === "page_view" || (outreach && (value.event_name === "contact_click" || value.event_name === "outbound_click"))) { keys.push("source", "campaign", "content"); + if (value.event_name !== "page_view") keys.push("event_value"); if (Object.keys(value).some(key => !keys.includes(key))) return null; const source = value.source === undefined ? "direct_unknown" : value.source; const campaign = value.campaign === undefined ? "none" : value.campaign; const content = value.content === undefined ? "none" : value.content; - if (!member(source, KFH_SOURCES) || !member(campaign, KFH_CAMPAIGNS) || !member(content, KFH_CONTENTS)) return null; - return { counter: "page_views", source, campaign, content }; + if (!member(source, outreach ? KFH_OUTREACH_SOURCES : KFH_SOURCES) + || !member(campaign, outreach ? KFH_OUTREACH_CAMPAIGNS : KFH_CAMPAIGNS) + || !member(content, outreach ? KFH_OUTREACH_CONTENTS : KFH_CONTENTS)) return null; + // v3 must carry all labels: missing attribution is not silently classified. + if (outreach && [value.source, value.campaign, value.content].some(label => label === undefined)) return null; + const counter = value.event_name === "page_view" ? "page_views" : actionCounter(value.event_name, value.event_value); + return counter ? { counter, ...(outreach ? { outreach: true as const } : {}), source, campaign, content } : null; } if (value.event_name !== "pwa_install") keys.push("event_value"); if (Object.keys(value).some(key => !keys.includes(key))) return null; if (value.event_name === "pwa_install") return { counter: "pwa_installs" }; - if (value.event_name === "contact_click" && value.event_value === "resource_call") return { counter: "resource_calls" }; - if (value.event_name === "contact_click" && value.event_value === "help_211") return { counter: "help_211" }; - if (value.event_name === "outbound_click" && value.event_value === "directions") return { counter: "directions" }; - if (value.event_name === "outbound_click" && value.event_value === "official_source") return { counter: "official_sources" }; + const counter = actionCounter(value.event_name, value.event_value); + return counter ? { counter } : null; +} + +function actionCounter(name: unknown, value: unknown): AttributableKey | null { + if (name === "contact_click" && value === "resource_call") return "resource_calls"; + if (name === "contact_click" && value === "help_211") return "help_211"; + if (name === "outbound_click" && value === "directions") return "directions"; + if (name === "outbound_click" && value === "official_source") return "official_sources"; return null; } +// Keep the old table valid for a rolled-back Worker, including its page margins. +const legacyLabel = (dimension: string, value: string) => dimension === "source" + ? member(value, KFH_SOURCES) ? value : "other" + : dimension === "campaign" ? member(value, KFH_CAMPAIGNS) ? value : "none" + : member(value, KFH_CONTENTS) ? value : "none"; + export async function readKfhBody(request: Request): Promise { if (!request.body) return null; const reader = request.body.getReader(); @@ -68,26 +88,36 @@ export async function ingestKfhEvent( if (!event || !(await allowRate())) return; const dimensions = [["event", event.counter]]; if (event.counter === "page_views") { - dimensions.push(["source", event.source!], ["campaign", event.campaign!], ["content", event.content!]); + dimensions.push(["source", legacyLabel("source", event.source!)], ["campaign", legacyLabel("campaign", event.campaign!)], ["content", legacyLabel("content", event.content!)]); } - // D1 batch is atomic: a view cannot have a partially written attribution set. - await db.batch(dimensions.map(([metric, value]) => db.prepare( + const statements = dimensions.map(([metric, value]) => db.prepare( "INSERT INTO kfh_daily(day, metric, value, count) VALUES (?, ?, ?, 1) ON CONFLICT(day, metric, value) DO UPDATE SET count = count + 1", - ).bind(day(now), metric, value))); + ).bind(day(now), metric, value)); + if (event.outreach) for (const [dimension, value] of [["source", event.source], ["campaign", event.campaign], ["content", event.content]]) { + statements.push(db.prepare("INSERT INTO kfh_outreach_daily(day, event, dimension, value, count) VALUES (?, ?, ?, ?, 1) ON CONFLICT(day, event, dimension, value) DO UPDATE SET count = count + 1") + .bind(day(now), event.counter, dimension, value)); + } + // All totals and independent margins succeed together; never store a raw event. + await db.batch(statements); } export async function pruneKfhData(db: D1Database, now: Date = new Date()): Promise { - await db.prepare("DELETE FROM kfh_daily WHERE day < ?").bind(shiftDay(now, -399)).run(); + await db.batch(["kfh_daily", "kfh_outreach_daily"].map(table => db.prepare(`DELETE FROM ${table} WHERE day < ?`).bind(shiftDay(now, -399)))); } export async function buildKfhReport(db: D1Database, now: Date = new Date()): Promise { let rows: Row[] = []; + let outreachRows: StoredOutreach[] = []; let available = true; try { const result = await db.prepare("SELECT day, metric, value, count FROM kfh_daily WHERE day >= ? AND day <= ? ORDER BY day, metric, value") .bind(shiftDay(now, -399), day(now)).all(); if (!result.success || !Array.isArray(result.results)) throw new Error("unavailable"); rows = result.results; + const outreach = await db.prepare("SELECT day, event, dimension, value, count FROM kfh_outreach_daily WHERE day >= ? AND day <= ? ORDER BY day, event, dimension, value") + .bind(shiftDay(now, -399), day(now)).all(); + if (!outreach.success || !Array.isArray(outreach.results)) throw new Error("unavailable"); + outreachRows = outreach.results; // Fail closed on corrupt/incompatible aggregate rows; no raw values leave here. for (const row of rows) { const allowed = row.metric === "event" ? KFH_COUNT_KEYS : row.metric === "source" ? KFH_SOURCES @@ -95,11 +125,16 @@ export async function buildKfhReport(db: D1Database, now: Date = new Date()): Pr if (!/^\d{4}-\d{2}-\d{2}$/.test(row.day) || !member(row.value, allowed) || !Number.isSafeInteger(row.count) || row.count < 1) throw new Error("unavailable"); } + for (const row of outreachRows) { + const allowed = row.dimension === "source" ? KFH_OUTREACH_SOURCES : row.dimension === "campaign" ? KFH_OUTREACH_CAMPAIGNS : row.dimension === "content" ? KFH_OUTREACH_CONTENTS : []; + if (!/^\d{4}-\d{2}-\d{2}$/.test(row.day) || !member(row.event, KFH_ATTRIBUTABLE_KEYS) + || !member(row.value, allowed) || !Number.isSafeInteger(row.count) || row.count < 1) throw new Error("unavailable"); + } } catch { available = false; rows = []; } - return kfhReportFromRows(available ? rows : null, now); + return kfhReportFromRows(available ? rows : null, now, available ? outreachRows : []); } -function kfhReportFromRows(input: Row[] | null, now: Date): KfhReport { +function kfhReportFromRows(input: Row[] | null, now: Date, outreachRows: StoredOutreach[] = []): KfhReport { const available = input !== null; const rows = input ?? []; const eventDays = rows.filter(row => row.metric === "event").map(row => row.day).sort(); @@ -120,10 +155,32 @@ function kfhReportFromRows(input: Row[] | null, now: Date): KfhReport { for (const row of rows) if (row.metric === metric && row.day >= shiftDay(now, -7) && row.day <= shiftDay(now, -1)) { totals.set(row.value, (totals.get(row.value) ?? 0) + row.count); } - return [...totals].map(([value, count]) => ({ value, count })).sort((a, b) => b.count - a.count || a.value.localeCompare(b.value)); + for (const row of outreachRows) if (row.event === "page_views" && row.dimension === metric && inWeek(row.day)) { + const fallback = legacyLabel(metric, row.value); + totals.set(fallback, (totals.get(fallback) ?? 0) - row.count); + totals.set(row.value, (totals.get(row.value) ?? 0) + row.count); + } + return [...totals].filter(([, count]) => count !== 0).map(([value, count]) => ({ value, count })).sort((a, b) => b.count - a.count || a.value.localeCompare(b.value)); + }; + const inWeek = (date: string) => date >= shiftDay(now, -7) && date <= shiftDay(now, -1); + const outreach: Outreach = { + classified: Object.fromEntries(KFH_ATTRIBUTABLE_KEYS.map(key => [key, 0])) as OutreachCounts, + unclassified: Object.fromEntries(KFH_ATTRIBUTABLE_KEYS.map(key => [key, 0])) as OutreachCounts, + sources: [], campaigns: [], contents: [], }; + for (const [dimension, key] of [["source", "sources"], ["campaign", "campaigns"], ["content", "contents"]] as const) { + const totals = new Map(); + for (const row of outreachRows) if (row.dimension === dimension && inWeek(row.day)) { + const id = `${row.event}:${row.value}`; + const total = totals.get(id) ?? { event: row.event, value: row.value, count: 0 }; + total.count += row.count; totals.set(id, total); + if (dimension === "source") outreach.classified[row.event] += row.count; + } + outreach[key] = [...totals.values()].sort((a, b) => a.event.localeCompare(b.event) || b.count - a.count || a.value.localeCompare(b.value)); + } + if (available) for (const key of KFH_ATTRIBUTABLE_KEYS) outreach.unclassified[key] = windows.last_7_complete_days.counts![key] - outreach.classified[key]; const report: KfhReport = { - view: "kfh", report_contract_version: "1.1", site_key: KFH_SITE_KEY, generated_at: now.toISOString(), + view: "kfh", report_contract_version: "1.2", site_key: KFH_SITE_KEY, generated_at: now.toISOString(), source: { availability: available ? "available" : "unavailable", reason: !available ? "query_failed" : eventDays.length ? "observed_activity" : "no_observed_history", @@ -131,7 +188,8 @@ function kfhReportFromRows(input: Row[] | null, now: Date): KfhReport { }, windows, discovery_last_7_complete_days: available ? { sources: rank("source"), campaigns: rank("campaign"), contents: rank("content") } : null, - limitations: KFH_LIMITATIONS, + outreach_last_7_complete_days: available ? outreach : null, + limitations: KFH_OUTREACH_LIMITATIONS, }; return isKfhReport(report) ? report : kfhReportFromRows(null, now); } diff --git a/src/kfhContract.ts b/src/kfhContract.ts index 8e9ba85..f8961e2 100644 --- a/src/kfhContract.ts +++ b/src/kfhContract.ts @@ -1,3 +1,4 @@ +import { KFH_OUTREACH_SOURCES, KFH_OUTREACH_CAMPAIGNS, KFH_OUTREACH_CONTENTS, isKfhOutreach, type Outreach } from "./kfhOutreachContract.js"; // Kingston's sensitive-use directory keeps only separate daily totals. // No raw event, provider context, identity or cross-dimension journey is stored. export const KFH_SITE_KEY = "kingston_food_help"; @@ -19,13 +20,15 @@ export const KFH_LIMITATIONS = { export const KFH_LEGACY_LIMITATIONS = { ...KFH_LIMITATIONS, counts_are: "consented_activity_not_people_or_service_outcomes" } as const; +export const KFH_OUTREACH_LIMITATIONS = { ...KFH_LIMITATIONS, attribution: "separate_daily_public_outreach_action_totals" } as const; + export type CountKey = typeof KFH_COUNT_KEYS[number]; export type WindowKey = typeof KFH_WINDOW_KEYS[number]; export type Counts = Record; type Dimension = { value: string; count: number }; export type KfhReport = { view: "kfh"; - report_contract_version: "1.0" | "1.1"; + report_contract_version: "1.0" | "1.1" | "1.2"; site_key: typeof KFH_SITE_KEY; generated_at: string; source: { @@ -36,7 +39,8 @@ export type KfhReport = { }; windows: Record; discovery_last_7_complete_days: { sources: Dimension[]; campaigns: Dimension[]; contents: Dimension[] } | null; - limitations: typeof KFH_LIMITATIONS | typeof KFH_LEGACY_LIMITATIONS; + outreach_last_7_complete_days?: Outreach | null; + limitations: typeof KFH_LIMITATIONS | typeof KFH_LEGACY_LIMITATIONS | typeof KFH_OUTREACH_LIMITATIONS; }; const isObject = (value: unknown): value is Record => !!value && typeof value === "object" && !Array.isArray(value); @@ -58,12 +62,13 @@ export function kfhWindowDays(now: Date): Record { // Strict shared producer/consumer contract. No runtime schema compiler is needed. export function isKfhReport(value: unknown): value is KfhReport { - if (!exact(value, ["view", "report_contract_version", "site_key", "generated_at", "source", "windows", "discovery_last_7_complete_days", "limitations"])) return false; - if (value.view !== "kfh" || (value.report_contract_version !== "1.0" && value.report_contract_version !== "1.1") || value.site_key !== KFH_SITE_KEY) return false; + const outreach = isObject(value) && value.report_contract_version === "1.2"; + if (!exact(value, [...(outreach ? ["outreach_last_7_complete_days"] : []), "view", "report_contract_version", "site_key", "generated_at", "source", "windows", "discovery_last_7_complete_days", "limitations"])) return false; + if (value.view !== "kfh" || (value.report_contract_version !== "1.0" && value.report_contract_version !== "1.1" && !outreach) || value.site_key !== KFH_SITE_KEY) return false; if (typeof value.generated_at !== "string" || !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(value.generated_at) || !Number.isFinite(Date.parse(value.generated_at)) || new Date(value.generated_at).toISOString() !== value.generated_at) return false; if (!exact(value.limitations, Object.keys(KFH_LIMITATIONS))) return false; - const limitations = value.report_contract_version === "1.0" ? KFH_LEGACY_LIMITATIONS : KFH_LIMITATIONS; + const limitations = value.report_contract_version === "1.0" ? KFH_LEGACY_LIMITATIONS : outreach ? KFH_OUTREACH_LIMITATIONS : KFH_LIMITATIONS; for (const [key, expected] of Object.entries(limitations)) if (value.limitations[key] !== expected) return false; const source = value.source; if (!exact(source, ["availability", "reason", "first_observed_day", "last_observed_day"])) return false; @@ -90,11 +95,11 @@ export function isKfhReport(value: unknown): value is KfhReport { } } const discovery = value.discovery_last_7_complete_days; - if (unavailable) return discovery === null; + if (unavailable) return discovery === null && (!outreach || value.outreach_last_7_complete_days === null); if (!exact(discovery, ["sources", "campaigns", "contents"])) return false; const windows = value.windows as KfhReport["windows"]; const views = windows.last_7_complete_days.counts!.page_views; - for (const [key, allowed] of [["sources", KFH_SOURCES], ["campaigns", KFH_CAMPAIGNS], ["contents", KFH_CONTENTS]] as const) { + for (const [key, allowed] of [["sources", outreach ? KFH_OUTREACH_SOURCES : KFH_SOURCES], ["campaigns", outreach ? KFH_OUTREACH_CAMPAIGNS : KFH_CAMPAIGNS], ["contents", outreach ? KFH_OUTREACH_CONTENTS : KFH_CONTENTS]] as const) { const rows = discovery[key]; if (!Array.isArray(rows) || rows.length > allowed.length) return false; const seen = new Set(); @@ -106,6 +111,13 @@ export function isKfhReport(value: unknown): value is KfhReport { } if (!Number.isSafeInteger(total) || total !== views) return false; } + if (outreach && !isKfhOutreach(value.outreach_last_7_complete_days, windows.last_7_complete_days.counts!)) return false; + if (outreach) for (const key of ["sources", "campaigns", "contents"] as const) { + const allViews = (discovery[key] as { value: string; count: number }[]); + for (const row of (value.outreach_last_7_complete_days as Outreach)[key]) { + if (row.event === "page_views" && row.count > (allViews.find(all => all.value === row.value)?.count ?? 0)) return false; + } + } // These windows overlap by definition. A contradictory report is unavailable. for (const key of KFH_COUNT_KEYS) { if (windows.latest_complete_day.counts![key] > windows.last_7_complete_days.counts![key] diff --git a/src/kfhOutreachContract.ts b/src/kfhOutreachContract.ts new file mode 100644 index 0000000..94f8ed3 --- /dev/null +++ b/src/kfhOutreachContract.ts @@ -0,0 +1,48 @@ +// Public outreach labels only. Rows are separate daily margins, never visitor journeys. +export const KFH_LEGACY_SOURCES = ["direct_unknown", "facebook", "community", "search", "other"] as const; +export const KFH_LEGACY_CAMPAIGNS = ["none", "launch_2026_09"] as const; +export const KFH_LEGACY_CONTENTS = ["none", "post_01", "poster_01"] as const; +export const KFH_OUTREACH_SOURCES = [...KFH_LEGACY_SOURCES, "reddit"] as const; +export const KFH_OUTREACH_CAMPAIGNS = [...KFH_LEGACY_CAMPAIGNS, "outreach_2026_09"] as const; +export const KFH_OUTREACH_CONTENTS = [...KFH_LEGACY_CONTENTS, "post_02"] as const; +export const KFH_ATTRIBUTABLE_KEYS = ["page_views", "resource_calls", "help_211", "directions", "official_sources"] as const; +export type AttributableKey = typeof KFH_ATTRIBUTABLE_KEYS[number]; +export type OutreachCounts = Record; +export type OutreachRow = { event: AttributableKey; value: string; count: number }; +export type Outreach = { + classified: OutreachCounts; + unclassified: OutreachCounts; + sources: OutreachRow[]; + campaigns: OutreachRow[]; + contents: OutreachRow[]; +}; + +const object = (v: unknown): v is Record => !!v && typeof v === "object" && !Array.isArray(v); +const exact = (v: unknown, keys: readonly string[]): v is Record => object(v) + && Object.keys(v).length === keys.length && keys.every(key => Object.prototype.hasOwnProperty.call(v, key)); +const count = (v: unknown): v is number => typeof v === "number" && Number.isSafeInteger(v) && v >= 0; + +export function isKfhOutreach(value: unknown, totals: OutreachCounts): value is Outreach { + if (!exact(value, ["classified", "unclassified", "sources", "campaigns", "contents"])) return false; + if (!exact(value.classified, KFH_ATTRIBUTABLE_KEYS) || !exact(value.unclassified, KFH_ATTRIBUTABLE_KEYS)) return false; + for (const event of KFH_ATTRIBUTABLE_KEYS) { + const known = value.classified[event], unknown = value.unclassified[event]; + if (!count(known) || !count(unknown) || !Number.isSafeInteger(known + unknown) || known + unknown !== totals[event]) return false; + } + for (const [key, allowed] of [["sources", KFH_OUTREACH_SOURCES], ["campaigns", KFH_OUTREACH_CAMPAIGNS], ["contents", KFH_OUTREACH_CONTENTS]] as const) { + const rows = value[key]; + if (!Array.isArray(rows) || rows.length > KFH_ATTRIBUTABLE_KEYS.length * allowed.length) return false; + const seen = new Set(); + const sums = Object.fromEntries(KFH_ATTRIBUTABLE_KEYS.map(event => [event, 0])) as OutreachCounts; + for (const row of rows) { + if (!exact(row, ["event", "value", "count"]) || typeof row.event !== "string" + || !(KFH_ATTRIBUTABLE_KEYS as readonly string[]).includes(row.event) || typeof row.value !== "string" + || !(allowed as readonly string[]).includes(row.value) || !count(row.count) || row.count === 0) return false; + const id = row.event + ":" + row.value; + if (seen.has(id)) return false; + seen.add(id); sums[row.event as AttributableKey] += row.count; + } + for (const event of KFH_ATTRIBUTABLE_KEYS) if (!Number.isSafeInteger(sums[event]) || sums[event] !== value.classified[event]) return false; + } + return true; +} diff --git a/tests/kfh-analytics.test.mjs b/tests/kfh-analytics.test.mjs index 87a7992..83e0300 100644 --- a/tests/kfh-analytics.test.mjs +++ b/tests/kfh-analytics.test.mjs @@ -30,12 +30,12 @@ before(async () => { try { const results = []; for (const statement of statements) results.push(await statement.run()); sqlite.run("COMMIT"); return results; } catch (error) { sqlite.run("ROLLBACK"); throw error; } } }; - for (const name of ["0008_add_site_event_rate_limit.sql", "0016_add_kfh_daily.sql"]) { + for (const name of ["0008_add_site_event_rate_limit.sql", "0016_add_kfh_daily.sql", "0017_add_kfh_outreach_attribution.sql"]) { await db.exec(fs.readFileSync(new URL(`../migrations/${name}`, import.meta.url), "utf8")); } }); after(() => { sqlite?.close(); }); -beforeEach(async () => { await db.exec("DELETE FROM kfh_daily; DELETE FROM site_event_rate_limit;"); }); +beforeEach(async () => { await db.exec("DELETE FROM kfh_daily; DELETE FROM kfh_outreach_daily; DELETE FROM site_event_rate_limit;"); }); test("strict Kingston payload rejects sensitive fields and action attribution", () => { assert.equal(parseKfhEvent(payload()).counter, "page_views"); @@ -176,13 +176,13 @@ test("strict response contract rejects identity, unsafe numbers and invented cov test("v2 default-on contract is explicit, isolated and persists the same bounded aggregates", async () => { const current = {site_key: KFH_SITE_KEY, contract_version: 2, collection_mode: "opt_out", page: "directory", event_name: "page_view"}; assert.deepEqual(parseKfhEvent(current), parseKfhEvent(payload())); - for (const extra of [{consent:true}, {consent:false}, {collection_mode:"opt_in"}, {collection_mode:undefined}, {contract_version:3}, {resource_id:"private"}, {search:"private"}]) assert.equal(parseKfhEvent({...current,...extra}), null); + for (const extra of [{consent:true}, {consent:false}, {collection_mode:"opt_in"}, {collection_mode:undefined}, {contract_version:4}, {resource_id:"private"}, {search:"private"}]) assert.equal(parseKfhEvent({...current,...extra}), null); assert.equal(parseKfhEvent({...payload(),collection_mode:"opt_out"}), null); await ingestKfhEvent(current, db, KFH_ORIGINS[0], async () => true, now); const rows = await db.prepare("SELECT metric,value,count FROM kfh_daily ORDER BY metric").all(); assert.equal(rows.results.length,4); assert.ok(rows.results.every(row => row.count === 1)); const report = await buildKfhReport(db,now); - assert.equal(report.report_contract_version,"1.1"); + assert.equal(report.report_contract_version,"1.2"); assert.equal(report.limitations.counts_are,"observed_activity_not_people_or_service_outcomes"); }); @@ -192,3 +192,64 @@ test("KFH report legacy compatibility cannot mislabel current default-on counts" assert.equal(isKfhReport({...legacy,report_contract_version:"1.1"}),false); assert.equal(isKfhReport({...legacy,report_contract_version:1.0}),false); }); + +const v3 = (overrides = {}) => ({ site_key: KFH_SITE_KEY, contract_version: 3, collection_mode: "opt_out", page: "directory", event_name: "page_view", source: "reddit", campaign: "outreach_2026_09", content: "post_02", ...overrides }); +test("v3 accepts only fixed outreach labels and broad events, preserving v1/v2 rejection", () => { + for (const event of [{}, { event_name: "contact_click", event_value: "resource_call" }, { event_name: "contact_click", event_value: "help_211" }, { event_name: "outbound_click", event_value: "directions" }, { event_name: "outbound_click", event_value: "official_source" }]) assert.equal(parseKfhEvent(v3(event)).outreach, true); + for (const change of [{ source: undefined }, { campaign: undefined }, { content: undefined }, { source: "private-person" }, { consent: true }, { session_id: "private" }, { resource_id: "private" }, { search: "private" }, { referrer: "private" }, { event_value: "extra" }, { event_name: "pwa_install" }, { event_name: "outbound_click", event_value: "resource_call" }, { contract_version: 2 }]) assert.equal(parseKfhEvent(v3(change)), null, JSON.stringify(change)); + const { source, campaign, content, ...install } = v3({ event_name: "pwa_install" }); + assert.deepEqual(parseKfhEvent(install), { counter: "pwa_installs" }); + assert.equal(parseKfhEvent({ ...install, event_value: "device" }), null); +}); + +test("v3 persists atomic independent margins; legacy totals remain readable on rollback", async () => { + const accept = body => ingestKfhEvent(body, db, KFH_ORIGINS[0], async () => true, new Date("2026-09-03T12:00:00Z")); + await accept(payload({ source: "facebook", campaign: "launch_2026_09", content: "post_01" })); + await accept(v3()); await accept(v3()); + await accept(v3({ event_name: "outbound_click", event_value: "directions" })); + await accept(payload({ event_name: "contact_click", event_value: "resource_call" })); + const report = await buildKfhReport(db, now); + assert.equal(isKfhReport(report), true); + assert.deepEqual(report.discovery_last_7_complete_days.sources, [{ value: "reddit", count: 2 }, { value: "facebook", count: 1 }]); + const outreach = report.outreach_last_7_complete_days; + assert.equal(outreach.classified.page_views, 2); assert.equal(outreach.unclassified.page_views, 1); + assert.equal(outreach.classified.directions, 1); assert.equal(outreach.unclassified.resource_calls, 1); + assert.ok(outreach.sources.every(row => row.value === "reddit")); + assert.equal((await db.prepare("SELECT count FROM kfh_daily WHERE metric='source' AND value='other'").first()).count, 2); + assert.equal((await db.prepare("SELECT count FROM kfh_daily WHERE metric='campaign' AND value='none'").first()).count, 2); + const rows = (await db.prepare("SELECT * FROM kfh_outreach_daily").all()).results; + assert.equal(rows.length, 6); + assert.ok(rows.every(row => Object.keys(row).sort().join(',') === 'count,day,dimension,event,value')); + assert.deepEqual(report, JSON.parse(fs.readFileSync(new URL('../contracts/kfh-v1/outreach-sample.json', import.meta.url), 'utf8'))); + await db.exec(fs.readFileSync(new URL('../migrations/0017_add_kfh_outreach_attribution.sql', import.meta.url), 'utf8')); + assert.deepEqual(await buildKfhReport(db, now), report); +}); + +test("failure in new attribution rolls back both tables, while missing migration reports unavailable", async () => { + await db.exec("CREATE TRIGGER fixture_reject_outreach BEFORE INSERT ON kfh_outreach_daily WHEN NEW.dimension='content' BEGIN SELECT RAISE(ABORT, 'fixture'); END;"); + try { + await assert.rejects(ingestKfhEvent(v3(), db, KFH_ORIGINS[0], async () => true, now)); + for (const table of ['kfh_daily', 'kfh_outreach_daily']) assert.equal((await db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).first()).n, 0); + } finally { await db.exec('DROP TRIGGER fixture_reject_outreach'); } + const noMigration = { prepare(sql) { if (sql.includes('kfh_outreach_daily')) throw new Error('missing table'); return db.prepare(sql); } }; + const report = await buildKfhReport(noMigration, now); + assert.equal(report.source.reason, 'query_failed'); assert.equal(report.outreach_last_7_complete_days, null); + assert.deepEqual(report, JSON.parse(fs.readFileSync(new URL('../contracts/kfh-v1/outreach-unavailable.json', import.meta.url), 'utf8'))); +}); + +test("v3 UTC windows, retention and inconsistent margins fail honestly", async () => { + const accept = (date) => ingestKfhEvent(v3(), db, KFH_ORIGINS[0], async () => true, date); + const empty = await buildKfhReport(db, now); + assert.deepEqual(empty, JSON.parse(fs.readFileSync(new URL('../contracts/kfh-v1/outreach-empty.json', import.meta.url), 'utf8'))); + for (const offset of [-400, -399, -8, -7, -1, 0]) await accept(new Date(now.getTime() + offset * 86400000)); + const report = await buildKfhReport(db, now); + assert.equal(report.outreach_last_7_complete_days.classified.page_views, 2); + assert.deepEqual(Object.values(report.windows).map(w => w.counts.page_views), [1, 1, 2, 1, 3]); + await pruneKfhData(db, now); + for (const table of ['kfh_daily', 'kfh_outreach_daily']) assert.equal((await db.prepare(`SELECT COUNT(DISTINCT day) AS n FROM ${table}`).first()).n, 5); + for (const mutate of [r => r.outreach_last_7_complete_days.classified.page_views++, r => r.outreach_last_7_complete_days.sources[0].count++, r => r.outreach_last_7_complete_days.sources[0].resource_id='private', r => r.outreach_last_7_complete_days.sources[0].event='pwa_installs', r => r.outreach_last_7_complete_days.sources.push(r.outreach_last_7_complete_days.sources[0]), r => r.outreach_last_7_complete_days.sources[0].value='facebook', r => r.report_contract_version='1.1']) { + const changed = structuredClone(report); mutate(changed); assert.equal(isKfhReport(changed), false); + } + await db.exec("DELETE FROM kfh_outreach_daily WHERE dimension='content' AND day='2026-09-03'"); + assert.equal((await buildKfhReport(db, now)).source.reason, 'query_failed'); +}); diff --git a/tests/release-control.test.mjs b/tests/release-control.test.mjs index 79480cb..4c114f9 100644 --- a/tests/release-control.test.mjs +++ b/tests/release-control.test.mjs @@ -54,7 +54,9 @@ test("operator scripts expose reads and upload without a direct production bypas }); test("the current governed bundle and historical release-control receipt stay synchronized", () => { - assert.equal(packageJson.version, "1.33.0"); + assert.equal(packageJson.version, "1.34.0"); + assert.match(sot, /^## Review candidate — 1\.34\.0, public outreach action attribution$/m); + assert.match(changelog, /^## \[1\.34\.0\] - 2026-09-09 \(review candidate\)$/m); assert.match(sot, /^## Kingston Food Help — v1\.32\.0 review candidate \(2026-09-04\)$/m); assert.match(changelog, /^## \[1\.32\.0\] - 2026-09-04$/m); assert.equal(packageLock.version, packageJson.version);