diff --git a/.github/workflows/durable-imports.yml b/.github/workflows/durable-imports.yml new file mode 100644 index 00000000..262e48cc --- /dev/null +++ b/.github/workflows/durable-imports.yml @@ -0,0 +1,36 @@ +name: Durable Imports checks + +on: + pull_request: + paths: + - 'applications/durable-imports/**' + - '.github/workflows/durable-imports.yml' + push: + branches: [main] + paths: + - 'applications/durable-imports/**' + - '.github/workflows/durable-imports.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + defaults: + run: + working-directory: applications/durable-imports + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24.21.0 + cache: npm + cache-dependency-path: applications/durable-imports/package-lock.json + - run: npm ci --ignore-scripts + - run: npm run format:check + - run: npm run build + - run: npm test + +# Cloud recovery tests require an explicitly provisioned dedicated fixture. diff --git a/applications/durable-imports/.env.example b/applications/durable-imports/.env.example new file mode 100644 index 00000000..23ed2994 --- /dev/null +++ b/applications/durable-imports/.env.example @@ -0,0 +1,12 @@ +PGHOST=your-service-hostname +PGPORT=5432 +PGDATABASE=postgres +PGUSER=imports_api +PGPASSWORD=replace-with-runtime-password +PGSSLROOTCERT=/absolute/path/cloud-ca.pem +PGSSLMODE=verify-full +CASEY_TOKEN=replace-with-a-random-token-of-at-least-32-characters +MORGAN_TOKEN=replace-with-a-different-random-token +PORT=4000 +# Run worker with imports_worker and its separate password. +# Run migrations with imports_migration, never either runtime role. diff --git a/applications/durable-imports/.gitignore b/applications/durable-imports/.gitignore new file mode 100644 index 00000000..c2f8f34e --- /dev/null +++ b/applications/durable-imports/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +.env +*.pem diff --git a/applications/durable-imports/.node-version b/applications/durable-imports/.node-version new file mode 100644 index 00000000..df6ae337 --- /dev/null +++ b/applications/durable-imports/.node-version @@ -0,0 +1 @@ +24.21.0 diff --git a/applications/durable-imports/README.md b/applications/durable-imports/README.md new file mode 100644 index 00000000..4de84e8a --- /dev/null +++ b/applications/durable-imports/README.md @@ -0,0 +1,189 @@ +# Durable contact imports + +An Express API accepts a small contact import, immediately returns its identifier, and lets its account retrieve progress and results. A separate pg-boss worker processes the durable payload in ClickHouse Managed Postgres. No email is sent and no external service is called. + +The important boundaries are visible in the code: + +- [Submission](src/imports.ts) inserts the import and enqueues its job on the **same node-postgres client transaction**, using pg-boss's `Db.executeSql` adapter. Account locking coordinates quotas and concurrent request retries. +- [The worker](src/handler.ts) receives pg-boss's transaction through `transactional: true`. Contact rows, application completion and queue completion commit together. A thrown error or killed process rolls back the handler writes; the already claimed job can be retried. +- Database uniqueness on `(account_id, request_id)`, `(import_id, email)` and `(import_id, row_index)` backs up the application coordination. Contacts are **import-scoped results**: the same email in distinct imports is allowed. + +This example pins Node 24.21.0, Express 5.2.1, node-postgres 8.23.1 and pg-boss 12.35.1. The queue schema is migrated separately with its owner. Runtime startup never performs DDL. + +## Create a Cloud fixture + +Use a dedicated test service. The shape below was verified for AWS `us-east-1`; inspect current availability and pricing before creating your own. This example uses PostgreSQL 18 without HA. + +```bash +export ORG_ID=your-organization-id +clickhousectl cloud postgres create --org-id "$ORG_ID" \ + --name durable-imports --provider aws --region us-east-1 \ + --size c6gd.large --pg-version 18 --ha-type none --json \ + > /private/path/service.json +chmod 600 /private/path/service.json +``` + +Save the returned service ID, hostname, port, username and password privately. Creation is asynchronous. Repeat the following until its status is `running`: + +```bash +export SERVICE_ID=your-created-service-id +clickhousectl cloud postgres get "$SERVICE_ID" --org-id "$ORG_ID" --json +clickhousectl cloud postgres certs get "$SERVICE_ID" --org-id "$ORG_ID" \ + --output /private/path/cloud-ca.pem +``` + +See the [ClickHouse Managed Postgres documentation](https://clickhouse.com/docs/products/managed-postgres/). A running service incurs charges; deleting it after testing avoids continuing charges. Keep the CLI receipt out of Git and logs. + +## Install and explicitly migrate + +Install the pinned Node release and PostgreSQL client in your development environment. From this directory: + +```bash +npm ci --ignore-scripts +npm run build +npm test +``` + +Create a private setup environment file. Use the Cloud receipt for the administrator, and generate three distinct random database passwords of at least 24 characters. Use separate random URL-safe tokens of 32–128 characters for the seeded accounts. + +```dotenv +PGHOST=your-service-hostname +PGPORT=5432 +PGDATABASE=postgres +PGSSLROOTCERT=/absolute/path/cloud-ca.pem +PGSSLMODE=verify-full +ADMIN_USER=receipt-username +ADMIN_PASSWORD=receipt-password +MIGRATION_PASSWORD=distinct-random-owner-password +APP_PASSWORD=distinct-random-api-password +WORKER_PASSWORD=distinct-random-worker-password +CASEY_TOKEN=distinct-random-account-token +MORGAN_TOKEN=another-random-account-token +``` + +Export these fields to child processes, then bootstrap as the administrator: + +```bash +set -a +source /private/path/setup.env +set +a +export PGUSER="$ADMIN_USER" PGPASSWORD="$ADMIN_PASSWORD" +psql -X -f sql/bootstrap.sql +export PGUSER=imports_migration PGPASSWORD="$MIGRATION_PASSWORD" +psql -X -f sql/migrate.sql +psql -X -f sql/seed.sql +npm run db:queue +psql -X -f sql/grants.sql +``` + +`bootstrap.sql` creates the roles and owned schemas once. It revokes public schema creation on this dedicated fixture. `migrate.sql`, `seed.sql` and `db:queue` are repeatable; rerun them to check the setup. The owner command invokes the pinned pg-boss migrations and creates a nonpartitioned queue. It omits the immutable `partition` option from later queue updates. No database `CREATE` grant is needed because the administrator already created both schemas. + +For migration upgrades, stop workers, inspect the pinned library's migration plans, run the owner command and review the grants against that release before restarting. Changing the dependency alone does not migrate runtime databases. Application migration version 1 has a destructive dedicated-fixture reset rather than a production downgrade path. + +### Runtime permissions + +`imports_api` may read application records, update the account quota columns, insert imports, reconcile terminal failures, and insert jobs into the owned queue's shared partition. It cannot claim jobs or insert contact results. + +`imports_worker` may read the payload, insert results, update application outcomes, claim/settle/delete queue jobs, update queue maintenance metadata, and clean queue dependencies. Two column grants on `import_jobs.version` allow the flow and monitoring cadence gates; its actual schema version remains immutable. Both roles can execute `job_now()`. They cannot create schemas, tables, queues or partitions, truncate tables, rewrite payloads, change existing contact results, or rebuild indexes. + +Ordinary worker supervision stays enabled for expiry, retries and retention. Scheduling, persistent statistics/warning partitions and automatic index rebuilding are disabled to avoid runtime DDL. Index maintenance and library upgrades belong to the owner. These grants are specific to pg-boss 12.35.1, not a universal pg-boss permission recipe. + +The runtime credentials are trusted shared application roles. Account isolation is enforced by server queries and token mappings, not PostgreSQL row-level security; a holder of those database credentials can access other accounts' rows within the granted permissions. + +## Run the API and worker + +Create separate private runtime files containing only connection fields, the respective role/password, and `PORT=4000`. The API file also contains `CASEY_TOKEN` and `MORGAN_TOKEN`; the worker does not need those tokens. Use [.env.example](.env.example) as a template. Do not copy administrator or migration credentials into runtime files. + +In one shell: + +```bash +set -a; source /private/path/api.env; set +a +unset ADMIN_USER ADMIN_PASSWORD MIGRATION_PASSWORD APP_PASSWORD WORKER_PASSWORD +npm start +``` + +In another: + +```bash +set -a; source /private/path/worker.env; set +a +unset ADMIN_USER ADMIN_PASSWORD MIGRATION_PASSWORD APP_PASSWORD WORKER_PASSWORD +npm run worker +``` + +The API binds to `127.0.0.1`. Both clients verify the Cloud CA and hostname explicitly; `PGSSLMODE` also configures `psql`. There is no option to disable verification. + +Tokens map to Casey and Morgan's seeded account IDs on the server. Requests cannot set their owner. Tokens are bearer credentials, not passwords, browser sessions or a user-management system. Rotating them requires restarting the API with new configuration. There is no cookie authentication or CORS configuration. Public deployment would need HTTPS termination, credential management and traffic controls beyond this loopback example. + +## Submit, poll and read results + +Use synthetic contacts only. Keep the same request UUID when retrying the same submission: + +```bash +export REQUEST_ID=12345678-1234-4234-8234-123456789abc +curl -sS http://127.0.0.1:4000/imports \ + -H "Authorization: Bearer $CASEY_TOKEN" -H 'Content-Type: application/json' \ + -d '{"requestId":"12345678-1234-4234-8234-123456789abc","rows":[{"email":" Casey@Example.test ","name":" Casey Example "}]}' +# Copy the returned import ID: +export IMPORT_ID=returned-import-id +curl -sS "http://127.0.0.1:4000/imports/$IMPORT_ID" \ + -H "Authorization: Bearer $CASEY_TOKEN" +curl -sS "http://127.0.0.1:4000/imports/$IMPORT_ID/results" \ + -H "Authorization: Bearer $CASEY_TOKEN" +``` + +| Route | Result | +| --- | --- | +| `POST /imports` | `202` on creation; `200` for a matching retained request; `409` for different normalized content with the same account/request ID | +| `GET /imports/:id` | Account-scoped state, queue progress, row count and terminal error/result count; `404` for missing or foreign records | +| `GET /imports/:id/results` | Up to 100 ordered normalized rows after success; `409` before success | + +Bodies are capped at 64 KiB and 1–100 rows. Only the documented fields are accepted. Email normalization trims and lowercases a deliberately basic ASCII syntax; it does **not** validate deliverability. Names are NFKC normalized, trimmed, whitespace collapsed, and limited to 80 Unicode codepoints. NUL, invalid Unicode and duplicate normalized emails in one import are rejected. Fingerprints retain row order, so a reordered retry conflicts. + +An account may have five pending imports and submit 50 new imports per account quota window. The window resets on the first new submission at least 24 hours after its stored start. Matching retries neither reset the window nor consume another slot. This is a resettable fixed window, not a sliding 24-hour count. Quota rejection is `429`; `Retry-After: 60` is a suggested polling delay, not a guarantee that quota will be available in a minute. Input errors are `400` or `413`; authentication failures are `401`. + +## Retries and durable outcomes + +The queue permits an initial attempt plus two retries, with one-second exponential backoff capped at four seconds. Claims expire after 20 seconds and heartbeat every 10 seconds. Handlers have a 25-second database transaction limit; imports are deliberately short and bounded. Each process has one local handler and a five-connection queue pool, separate from its five-connection application pool. + +A worker exception rolls back contacts, application success and queue completion together. The claim itself was committed earlier, so supervision can move an abandoned attempt back to retry. Multiple executions can occur; the handler checks the durable import state and database constraints prevent duplicate import-scoped effects. This does not provide exactly-once email, payment or other external effects. + +Queued jobs have one-hour retention and terminal job metadata is deleted after one day. The application reconciles queue failures/cancellations into durable terminal records on worker startup, periodic worker ticks, submission and status reads. If metadata disappears before reconciliation, it reports `Queue metadata no longer available`, not a fabricated cause or perpetual processing. Durable payloads, request fingerprints and results have no application purge route in this small example; operators need a retention policy for a production system. + +## Real Cloud acceptance + +Cloud tests require the setup environment plus `PGUSER=imports_api` and its password. They create records, use owner credentials for isolated fault/retention fixtures, and start runtime-only child processes. Use a dedicated service and run from a freshly bootstrapped schema: + +```bash +set -a; source /private/path/setup.env; set +a +export PGUSER=imports_api PGPASSWORD="$APP_PASSWORD" +npm run test:cloud +``` + +The tests exercise verified TLS controls, forbidden runtime DDL and mutations, atomic enqueue rollback, eight competing request retries, two worker processes, quotas, transactional handler failure, retry exhaustion, runtime retention, an actual `SIGKILL` after uncommitted effects, completion replay, account-scoped HTTP behavior and confirmed API/worker restarts. The faults are bounded, require `NODE_ENV=test`, and cannot be requested through HTTP. Private acceptance logs are kept outside the repository. + +## Cleanup + +Stop API and worker processes first. To reset only this dedicated fixture's schemas/roles, explicitly restore administrator credentials in a setup shell: + +```bash +set -a; source /private/path/setup.env; set +a +export PGUSER="$ADMIN_USER" PGPASSWORD="$ADMIN_PASSWORD" +psql -X -f sql/cleanup.sql +``` + +To remove your Cloud fixture, use its exact saved ID and verify that it disappears from the organization listing: + +```bash +clickhousectl cloud postgres delete "$SERVICE_ID" --org-id "$ORG_ID" +clickhousectl cloud postgres list --org-id "$ORG_ID" --json +``` + +Deletion is asynchronous and destroys the service's data. Retain needed source/evidence before deleting it. + +## Primary references + +- [pg-boss transaction adapters](https://pgboss.io/api/adapters) +- [pg-boss transactional workers](https://pgboss.io/api/workers) +- [Pinned pg-boss 12.35.1 source](https://github.com/timgit/pg-boss/tree/12.35.1) +- [node-postgres transactions](https://node-postgres.com/features/transactions) +- [Express API](https://expressjs.com/en/5x/api.html) diff --git a/applications/durable-imports/package-lock.json b/applications/durable-imports/package-lock.json new file mode 100644 index 00000000..5fb99512 --- /dev/null +++ b/applications/durable-imports/package-lock.json @@ -0,0 +1,1664 @@ +{ + "name": "durable-imports", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "durable-imports", + "version": "1.0.0", + "dependencies": { + "express": "5.2.1", + "pg": "8.23.1", + "pg-boss": "12.35.1" + }, + "devDependencies": { + "@types/express": "5.0.6", + "@types/node": "24.19.1", + "@types/pg": "8.23.1", + "prettier": "3.9.9", + "typescript": "7.0.2" + }, + "engines": { + "node": "24.21.0" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.3.tgz", + "integrity": "sha512-dPfW8NFiOF4wOHc7+N/QSxlY9cfSsenewGbAz8C8U/MULPd/YZ27LvJUIlzaXie7e6Ove9YunJGgC9tbHD2cKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.19.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.1.tgz", + "integrity": "sha512-aS3/DG0oM05K0RIXXP+hKjinGG5IgSSVGzswZxW3O0sS3pH4/fycXundUC9XsszgKCk4gHXylTEK6hyFxVxnoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@types/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/@types/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cron-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.10.1.tgz", + "integrity": "sha512-pKRrRagItwk9rGIStcUyMxFX34x56zoX3KDf9HJ4ttwkXIK1qxK63EvXvEDmlxI9AdPJ+Y7TEKRftRlW5eSS7A==", + "license": "MIT", + "dependencies": { + "luxon": "^3.7.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/non-error": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/non-error/-/non-error-0.1.0.tgz", + "integrity": "sha512-TMB1uHiGsHRGv1uYclfhivcnf0/PdFp2pNqRxXjncaAsjYMoisaQJI+SSZCqRq+VliwRTC8tsMQfmrWjDMhkPQ==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-boss": { + "version": "12.35.1", + "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-12.35.1.tgz", + "integrity": "sha512-2eGu2HhelLE0jaifdEMMtuy1pgoF9fycs5deHEwQiaXUcxuPQ9udNxb27VquNO8XgYTM3b1cWnsfSuc8RzXuWw==", + "license": "MIT", + "dependencies": { + "cron-parser": "^5.10.1", + "pg": "^8.23.1", + "rrule-temporal": "^2.2.7", + "serialize-error": "^13.0.2" + }, + "bin": { + "pg-boss": "dist/cli.js" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/prettier": { + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.9.tgz", + "integrity": "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/rrule-temporal": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/rrule-temporal/-/rrule-temporal-2.2.8.tgz", + "integrity": "sha512-+wWUe0J02oMh1UWKIt1RTu2H0Cz+gP11O/iweK0MN2UP6JLGeepTUkbSBa90Hxq0W+ywSxM9IlJNouIUdMOwhg==", + "license": "MIT", + "dependencies": { + "temporal-spec": "^1.0.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serialize-error": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-13.0.2.tgz", + "integrity": "sha512-G+YKwp9I6IvWMe5RlwqxdmTPIcq/G8HK/sU51gdV1XAJznYkRSTB0ph+zB9AfSxG0fx6xrqYA6i9HeIcHFTXow==", + "license": "MIT", + "dependencies": { + "non-error": "^0.1.0", + "type-fest": "^5.4.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/tagged-tag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", + "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/temporal-spec": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/temporal-spec/-/temporal-spec-1.0.1.tgz", + "integrity": "sha512-wxVoanmDeavXie1vu2JaQ3WIc3JZnWAOYFBsJyATaVsXsycKYUflGsyBmrRSnoCpZJpwPyr38VpgSUlQ8CbFxg==", + "license": "Apache-2.0" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-fest": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.10.0.tgz", + "integrity": "sha512-NoSdpq/WEiAg5sjmBkmV/hfxv6HJH4NqPNrqjtSO5CwRmpsDfaf4begxW34KdJykH/l1yHtwBWQkCRdoXO8mPA==", + "license": "(MIT OR CC0-1.0)", + "dependencies": { + "tagged-tag": "^1.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/applications/durable-imports/package.json b/applications/durable-imports/package.json new file mode 100644 index 00000000..90d31cc2 --- /dev/null +++ b/applications/durable-imports/package.json @@ -0,0 +1,31 @@ +{ + "name": "durable-imports", + "version": "1.0.0", + "private": true, + "type": "module", + "engines": { + "node": "24.21.0" + }, + "scripts": { + "build": "tsc", + "start": "node dist/src/server.js", + "worker": "node dist/src/worker.js", + "db:queue": "node dist/scripts/migrate-queue.js", + "test": "node --test dist/test/domain.test.js", + "test:cloud": "node --test --test-concurrency=1 dist/test/cloud.test.js", + "format": "prettier --write src test scripts package.json tsconfig.json", + "format:check": "prettier --check src test scripts package.json tsconfig.json" + }, + "dependencies": { + "express": "5.2.1", + "pg": "8.23.1", + "pg-boss": "12.35.1" + }, + "devDependencies": { + "@types/express": "5.0.6", + "@types/node": "24.19.1", + "@types/pg": "8.23.1", + "typescript": "7.0.2", + "prettier": "3.9.9" + } +} diff --git a/applications/durable-imports/scripts/migrate-queue.ts b/applications/durable-imports/scripts/migrate-queue.ts new file mode 100644 index 00000000..c3a41701 --- /dev/null +++ b/applications/durable-imports/scripts/migrate-queue.ts @@ -0,0 +1,15 @@ +import { queue, QUEUE, QUEUE_OPTIONS } from "../src/queue.js"; +if (process.env.PGUSER !== "imports_migration") + throw new Error("Queue setup requires imports_migration"); +// The explicit owner-only command invokes pg-boss's pinned standard migrations. +// Runtime instances use migrate:false and never create queues or partitions. +const boss = queue(false, true); +try { + await boss.start(); + await boss.createQueue(QUEUE, QUEUE_OPTIONS); + const { partition: _partition, ...mutableOptions } = QUEUE_OPTIONS; + await boss.updateQueue(QUEUE, mutableOptions); + console.log("Queue schema version", await boss.schemaVersion()); +} finally { + await boss.stop(); +} diff --git a/applications/durable-imports/sql/bootstrap.sql b/applications/durable-imports/sql/bootstrap.sql new file mode 100644 index 00000000..3230db96 --- /dev/null +++ b/applications/durable-imports/sql/bootstrap.sql @@ -0,0 +1,28 @@ +\set ON_ERROR_STOP on +\getenv migration_password MIGRATION_PASSWORD +\getenv api_password APP_PASSWORD +\getenv worker_password WORKER_PASSWORD +SELECT length(:'migration_password') >= 24 AND length(:'api_password') >= 24 + AND length(:'worker_password') >= 24 AND :'migration_password' <> :'api_password' + AND :'migration_password' <> :'worker_password' AND :'api_password' <> :'worker_password' + AS passwords_valid \gset +\if :passwords_valid +\else + DO $$ BEGIN RAISE EXCEPTION 'Use distinct random passwords of at least 24 characters'; END $$; +\endif +BEGIN; +CREATE ROLE imports_migration LOGIN PASSWORD :'migration_password'; +CREATE ROLE imports_api LOGIN PASSWORD :'api_password'; +CREATE ROLE imports_worker LOGIN PASSWORD :'worker_password'; +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +CREATE SCHEMA contact_imports AUTHORIZATION imports_migration; +CREATE SCHEMA import_jobs AUTHORIZATION imports_migration; +REVOKE ALL ON SCHEMA contact_imports, import_jobs FROM PUBLIC; +GRANT USAGE ON SCHEMA contact_imports, import_jobs TO imports_api, imports_worker; +ALTER ROLE imports_api SET statement_timeout = '15s'; +ALTER ROLE imports_api SET lock_timeout = '5s'; +ALTER ROLE imports_api SET idle_in_transaction_session_timeout = '15s'; +ALTER ROLE imports_worker SET statement_timeout = '15s'; +ALTER ROLE imports_worker SET lock_timeout = '5s'; +ALTER ROLE imports_worker SET idle_in_transaction_session_timeout = '25s'; +COMMIT; diff --git a/applications/durable-imports/sql/cleanup.sql b/applications/durable-imports/sql/cleanup.sql new file mode 100644 index 00000000..a9dc7c18 --- /dev/null +++ b/applications/durable-imports/sql/cleanup.sql @@ -0,0 +1,7 @@ +\set ON_ERROR_STOP on +-- Destructive, dedicated-fixture cleanup by the administrator after processes stop. +DROP SCHEMA IF EXISTS contact_imports CASCADE; +DROP SCHEMA IF EXISTS import_jobs CASCADE; +DROP ROLE IF EXISTS imports_api; +DROP ROLE IF EXISTS imports_worker; +DROP ROLE IF EXISTS imports_migration; diff --git a/applications/durable-imports/sql/grants.sql b/applications/durable-imports/sql/grants.sql new file mode 100644 index 00000000..c7d5f9c0 --- /dev/null +++ b/applications/durable-imports/sql/grants.sql @@ -0,0 +1,24 @@ +\set ON_ERROR_STOP on +-- Run after application/queue migrations and owner-only queue creation. +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA import_jobs FROM PUBLIC; +GRANT SELECT ON contact_imports.accounts, contact_imports.imports, contact_imports.contacts + TO imports_api, imports_worker; +-- FOR UPDATE for quota coordination also needs UPDATE on an account column. +GRANT UPDATE (window_start, submission_count) ON contact_imports.accounts TO imports_api; +GRANT INSERT ON contact_imports.imports TO imports_api; +GRANT UPDATE (state, error, finished_at) ON contact_imports.imports TO imports_api; +GRANT INSERT ON contact_imports.contacts TO imports_worker; +GRANT UPDATE (state, error, finished_at, result_count) ON contact_imports.imports TO imports_worker; + +GRANT SELECT ON import_jobs.version, import_jobs.queue, import_jobs.job, import_jobs.job_common + TO imports_api, imports_worker; +GRANT INSERT ON import_jobs.job_common TO imports_api; +-- The worker must claim/settle jobs and run ordinary expiry/retention supervision. +GRANT INSERT, UPDATE, DELETE ON import_jobs.job, import_jobs.job_common TO imports_worker; +GRANT UPDATE ON import_jobs.queue TO imports_worker; +-- Supervisor cadence gates, not the schema version or index-rebuild gate. +GRANT UPDATE (flow_on, monitor_backoff_on) ON import_jobs.version TO imports_worker; +GRANT SELECT, UPDATE, DELETE ON import_jobs.job_dependency TO imports_worker; +GRANT EXECUTE ON FUNCTION import_jobs.job_now() TO imports_api, imports_worker; +-- No schema CREATE, queue creation/helper EXECUTE, TRUNCATE or index ownership. +-- Scheduling, persistent stats partitions and runtime index rebuilding are disabled. diff --git a/applications/durable-imports/sql/migrate.sql b/applications/durable-imports/sql/migrate.sql new file mode 100644 index 00000000..10934230 --- /dev/null +++ b/applications/durable-imports/sql/migrate.sql @@ -0,0 +1,56 @@ +\set ON_ERROR_STOP on +BEGIN; +CREATE TABLE IF NOT EXISTS contact_imports.schema_migrations ( + version integer PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +SELECT NOT EXISTS (SELECT 1 FROM contact_imports.schema_migrations WHERE version = 1) + AS apply_migration \gset +\if :apply_migration +CREATE TABLE contact_imports.accounts ( + id uuid PRIMARY KEY, + name text NOT NULL, + window_start timestamptz NOT NULL DEFAULT clock_timestamp(), + submission_count integer NOT NULL DEFAULT 0 CHECK (submission_count BETWEEN 0 AND 50) +); +CREATE TABLE contact_imports.imports ( + id uuid PRIMARY KEY, + account_id uuid NOT NULL REFERENCES contact_imports.accounts(id), + request_id uuid NOT NULL, + fingerprint char(64) NOT NULL CHECK (fingerprint ~ '^[0-9a-f]{64}$'), + payload jsonb NOT NULL CHECK (jsonb_typeof(payload) = 'array'), + row_count integer NOT NULL CHECK (row_count BETWEEN 1 AND 100), + state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'succeeded', 'failed')), + result_count integer, + error text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + finished_at timestamptz, + CONSTRAINT imports_request_key UNIQUE (account_id, request_id), + CONSTRAINT imports_account_key UNIQUE (id, account_id), + CONSTRAINT payload_count CHECK (jsonb_array_length(payload) = row_count), + CONSTRAINT terminal_result CHECK ( + (state = 'queued' AND result_count IS NULL AND finished_at IS NULL AND error IS NULL) + OR (state = 'succeeded' AND result_count IS NOT NULL AND result_count = row_count + AND finished_at IS NOT NULL AND error IS NULL) + OR (state = 'failed' AND result_count IS NULL AND finished_at IS NOT NULL AND error IS NOT NULL) + ) +); +CREATE INDEX imports_account_created ON contact_imports.imports (account_id, created_at, id); +CREATE INDEX imports_pending ON contact_imports.imports (account_id) WHERE state = 'queued'; +CREATE TABLE contact_imports.contacts ( + import_id uuid NOT NULL, + account_id uuid NOT NULL, + row_index integer NOT NULL CHECK (row_index BETWEEN 1 AND 100), + email varchar(254) NOT NULL, + name varchar(80) NOT NULL, + CONSTRAINT contacts_import_key PRIMARY KEY (import_id, email), + CONSTRAINT contacts_row_key UNIQUE (import_id, row_index), + CONSTRAINT contacts_account_fk FOREIGN KEY (import_id, account_id) + REFERENCES contact_imports.imports (id, account_id), + CHECK (length(email) BETWEEN 3 AND 254 AND length(btrim(name)) BETWEEN 1 AND 80) +); +INSERT INTO contact_imports.schema_migrations (version) VALUES (1); +\else +\echo Application migration already applied +\endif +COMMIT; diff --git a/applications/durable-imports/sql/seed.sql b/applications/durable-imports/sql/seed.sql new file mode 100644 index 00000000..0ae73b23 --- /dev/null +++ b/applications/durable-imports/sql/seed.sql @@ -0,0 +1,5 @@ +\set ON_ERROR_STOP on +INSERT INTO contact_imports.accounts (id, name) VALUES + ('00000000-0000-0000-0000-000000000001', 'Casey'), + ('00000000-0000-0000-0000-000000000002', 'Morgan') +ON CONFLICT (id) DO NOTHING; diff --git a/applications/durable-imports/src/app.ts b/applications/durable-imports/src/app.ts new file mode 100644 index 00000000..541cf24b --- /dev/null +++ b/applications/durable-imports/src/app.ts @@ -0,0 +1,52 @@ +import express, { type ErrorRequestHandler } from "express"; +import { authorize } from "./auth.js"; +import { ApiError, MAX_BYTES, uuid } from "./input.js"; +import type { Imports } from "./imports.js"; +export function createApp(imports: Imports) { + const app = express(); + app.disable("x-powered-by"); + app.get("/health", (_, res) => res.json({ ok: true })); + app.use( + "/imports", + authorize(), + express.json({ limit: MAX_BYTES, strict: true }), + ); + app.post("/imports", async (req, res) => { + const result = await imports.submit(res.locals.accountId, req.body); + res + .status(result.replay ? 200 : 202) + .location(`/imports/${result.id}`) + .json(result); + }); + app.get("/imports/:id", async (req, res) => { + if (!uuid(req.params.id)) throw new ApiError(400, "invalid_import_id"); + res.json( + await imports.status(res.locals.accountId, req.params.id.toLowerCase()), + ); + }); + app.get("/imports/:id/results", async (req, res) => { + if (!uuid(req.params.id)) throw new ApiError(400, "invalid_import_id"); + res.json({ + rows: await imports.results( + res.locals.accountId, + req.params.id.toLowerCase(), + ), + }); + }); + app.use((_, res) => res.status(404).json({ error: "not_found" })); + const errors: ErrorRequestHandler = (error, _req, res, _next) => { + if (error instanceof ApiError) { + if (error.status === 429) res.set("Retry-After", "60"); + res.status(error.status).json({ error: error.code }); + } else if (error.type === "entity.too.large") + res.status(413).json({ error: "body_too_large" }); + else if (error.type === "entity.parse.failed") + res.status(400).json({ error: "invalid_json" }); + else { + console.error("Import request failed"); + res.status(500).json({ error: "internal_error" }); + } + }; + app.use(errors); + return app; +} diff --git a/applications/durable-imports/src/auth.ts b/applications/durable-imports/src/auth.ts new file mode 100644 index 00000000..d7986c23 --- /dev/null +++ b/applications/durable-imports/src/auth.ts @@ -0,0 +1,39 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import type { RequestHandler } from "express"; +import { CASEY, MORGAN } from "./input.js"; +export function accountTokens(): { id: string; digest: Buffer }[] { + const casey = process.env.CASEY_TOKEN; + const morgan = process.env.MORGAN_TOKEN; + if ( + !casey || + !morgan || + casey === morgan || + ![casey, morgan].every((value) => /^[A-Za-z0-9_-]{32,128}$/.test(value)) + ) + throw new Error( + "Set distinct, random CASEY_TOKEN and MORGAN_TOKEN (32–128 URL-safe characters)", + ); + return [ + { id: CASEY, digest: hash(casey) }, + { id: MORGAN, digest: hash(morgan) }, + ]; +} +function hash(value: string): Buffer { + return createHash("sha256").update(value).digest(); +} +export function authorize(tokens = accountTokens()): RequestHandler { + return (req, res, next) => { + const header = req.header("authorization"); + const token = header?.startsWith("Bearer ") ? header.slice(7) : ""; + const digest = hash(token.length <= 128 ? token : ""); + const account = tokens.find((candidate) => + timingSafeEqual(candidate.digest, digest), + ); + if (!account) { + res.status(401).json({ error: "unauthorized" }); + return; + } + res.locals.accountId = account.id; + next(); + }; +} diff --git a/applications/durable-imports/src/database.ts b/applications/durable-imports/src/database.ts new file mode 100644 index 00000000..5670d05d --- /dev/null +++ b/applications/durable-imports/src/database.ts @@ -0,0 +1,62 @@ +import { readFileSync } from "node:fs"; +import pg, { type PoolClient, type PoolConfig } from "pg"; +import type { Db } from "pg-boss"; + +export function databaseConfig(): PoolConfig { + const required = (name: string): string => { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; + }; + const port = Number(process.env.PGPORT ?? "5432"); + if (!Number.isInteger(port) || port < 1 || port > 65535) + throw new Error("Invalid PGPORT"); + const host = required("PGHOST"); + return { + host, + port, + database: required("PGDATABASE"), + user: required("PGUSER"), + password: required("PGPASSWORD"), + ssl: { + ca: readFileSync(required("PGSSLROOTCERT"), "utf8"), + rejectUnauthorized: true, + servername: host, + }, + max: 5, + connectionTimeoutMillis: 15_000, + statement_timeout: 15_000, + idleTimeoutMillis: 10_000, + }; +} +export function pool(): pg.Pool { + const result = new pg.Pool(databaseConfig()); + result.on("error", () => console.error("Database idle connection error")); + return result; +} +// The adapter executes every pg-boss statement on this already-open transaction. +export function transactionAdapter(client: PoolClient): Db { + return { executeSql: (text, values) => client.query(text, values) }; +} +export async function transaction( + db: pg.Pool, + callback: (client: PoolClient) => Promise, +): Promise { + const client = await db.connect(); + let discarded: Error | undefined; + try { + await client.query("BEGIN"); + const value = await callback(client); + await client.query("COMMIT"); + return value; + } catch (error) { + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + discarded = rollbackError as Error; + } + throw error; + } finally { + client.release(discarded); + } +} diff --git a/applications/durable-imports/src/handler.ts b/applications/durable-imports/src/handler.ts new file mode 100644 index 00000000..234d2e60 --- /dev/null +++ b/applications/durable-imports/src/handler.ts @@ -0,0 +1,71 @@ +import { setTimeout as delay } from "node:timers/promises"; +import type { Db, Job } from "pg-boss"; +import { uuid } from "./input.js"; +export type ImportJob = { importId: string }; +export type Fault = { + importId: string; + failAttempts: number; + pauseAfterEffects: boolean; +}; +export function testFault(): Fault | undefined { + if (!process.env.TEST_IMPORT_ID) return undefined; + const failAttempts = Number(process.env.TEST_FAIL_ATTEMPTS ?? "0"); + if ( + process.env.NODE_ENV !== "test" || + !uuid(process.env.TEST_IMPORT_ID) || + !Number.isInteger(failAttempts) || + failAttempts < 0 || + failAttempts > 3 + ) + throw new Error("Invalid test-only worker fault"); + return { + importId: process.env.TEST_IMPORT_ID, + failAttempts, + pauseAfterEffects: process.env.TEST_PAUSE_AFTER_EFFECTS === "true", + }; +} +export async function processImport( + jobs: Job[], + tx: Db, + fault?: Fault, +) { + const job = jobs[0]; + if (!job || !uuid(job.data?.importId)) + throw new Error("Invalid import job reference"); + const { + rows: [record], + } = await tx.executeSql( + "SELECT * FROM contact_imports.imports WHERE id = $1 FOR UPDATE", + [job.data.importId], + ); + if (!record) throw new Error("Import no longer exists"); + if (record.state !== "queued") + return { state: record.state, rowsImported: record.result_count }; + await tx.executeSql( + `INSERT INTO contact_imports.contacts (import_id, account_id, row_index, email, name) + SELECT $1::uuid, $2::uuid, r.position::integer, r.record->>'email', r.record->>'name' + FROM jsonb_array_elements($3::jsonb) WITH ORDINALITY AS r(record, position) + ON CONFLICT (import_id, email) DO NOTHING`, + [record.id, record.account_id, JSON.stringify(record.payload)], + ); + const { + rows: [counts], + } = await tx.executeSql( + "SELECT count(*)::integer AS total FROM contact_imports.contacts WHERE import_id = $1", + [record.id], + ); + if (counts.total !== record.row_count) + throw new Error("Import effects disagree with the durable payload"); + await tx.executeSql( + "UPDATE contact_imports.imports SET state = 'succeeded', result_count = $2, finished_at = clock_timestamp() WHERE id = $1", + [record.id, counts.total], + ); + if (fault && fault.importId === record.id) { + process.send?.({ type: "effects_written", importId: record.id }); + if (job.retryCount < fault.failAttempts) + throw new Error("Bounded test-only import failure"); + if (fault.pauseAfterEffects) + await delay(15_000, undefined, { signal: job.signal }); + } + return { state: "succeeded", rowsImported: counts.total }; +} diff --git a/applications/durable-imports/src/imports.ts b/applications/durable-imports/src/imports.ts new file mode 100644 index 00000000..b02ffd2e --- /dev/null +++ b/applications/durable-imports/src/imports.ts @@ -0,0 +1,130 @@ +import { randomUUID } from "node:crypto"; +import type { Pool, PoolClient } from "pg"; +import type { PgBoss } from "pg-boss"; +import { transaction, transactionAdapter } from "./database.js"; +import { ApiError, MAX_DAILY, MAX_PENDING, normalize } from "./input.js"; +import { QUEUE } from "./queue.js"; + +// Terminal queue outcomes are copied to the durable application record. Missing +// metadata is reported honestly, including after queue retention or operator deletion. +export async function reconcile( + db: Pick, + accountId?: string, +): Promise { + await db.query( + ` + UPDATE contact_imports.imports AS i SET state = 'failed', finished_at = clock_timestamp(), + error = CASE j.state::text + WHEN 'failed' THEN 'Queue retry limit exhausted' + WHEN 'cancelled' THEN 'Queue job cancelled' + WHEN 'completed' THEN 'Queue completion has no application result' + ELSE 'Queue metadata no longer available' END + FROM (SELECT x.id, q.state FROM contact_imports.imports x + LEFT JOIN import_jobs.job q ON q.id = x.id AND q.name = $1 + WHERE x.state = 'queued' AND ($2::uuid IS NULL OR x.account_id = $2) + AND (q.id IS NULL OR q.state::text IN ('failed', 'cancelled', 'completed'))) j + WHERE i.id = j.id AND i.state = 'queued'`, + [QUEUE, accountId ?? null], + ); +} +export class Imports { + constructor( + private db: Pool, + private boss: PgBoss, + ) {} + async submit( + accountId: string, + input: unknown, + afterEnqueue?: () => Promise, + ) { + if (afterEnqueue && process.env.NODE_ENV !== "test") + throw new Error("Submission fault requires test mode"); + const normalized = normalize(input); + return transaction(this.db, async (client) => { + const { + rows: [account], + } = await client.query( + "SELECT id, window_start, submission_count FROM contact_imports.accounts WHERE id = $1 FOR UPDATE", + [accountId], + ); + if (!account) throw new ApiError(401, "unknown_account"); + const { + rows: [existing], + } = await client.query( + "SELECT id, fingerprint FROM contact_imports.imports WHERE account_id = $1 AND request_id = $2", + [accountId, normalized.requestId], + ); + if (existing) { + if (existing.fingerprint !== normalized.fingerprint) + throw new ApiError(409, "request_id_conflict"); + return { id: existing.id as string, replay: true }; + } + await reconcile(client, accountId); + const { + rows: [limits], + } = await client.query( + `SELECT + count(*)::integer AS pending, + clock_timestamp() >= $2::timestamptz + interval '24 hours' AS reset + FROM contact_imports.imports WHERE account_id = $1 AND state = 'queued'`, + [accountId, account.window_start], + ); + const count = limits.reset ? 0 : account.submission_count; + if (limits.pending >= MAX_PENDING || count >= MAX_DAILY) + throw new ApiError(429, "account_import_limit"); + await client.query( + "UPDATE contact_imports.accounts SET submission_count = $2, window_start = CASE WHEN $3 THEN clock_timestamp() ELSE window_start END WHERE id = $1", + [accountId, count + 1, limits.reset], + ); + const id = randomUUID(); + await client.query( + "INSERT INTO contact_imports.imports (id, account_id, request_id, fingerprint, payload, row_count) VALUES ($1, $2, $3, $4, $5, $6)", + [ + id, + accountId, + normalized.requestId, + normalized.fingerprint, + JSON.stringify(normalized.rows), + normalized.rows.length, + ], + ); + const jobId = await this.boss.send( + QUEUE, + { importId: id }, + { id, db: transactionAdapter(client) }, + ); + if (jobId !== id) throw new Error("Queue did not accept the import job"); + // Internal acceptance seam, never accepted from an HTTP request. + if (afterEnqueue) await afterEnqueue(); + return { id, replay: false }; + }); + } + async status(accountId: string, id: string) { + return transaction(this.db, async (client) => { + await reconcile(client, accountId); + const { + rows: [row], + } = await client.query( + `SELECT i.id, i.request_id AS "requestId", i.state, + i.row_count AS "rowCount", i.result_count AS "rowsImported", i.error, + i.created_at AS "createdAt", i.finished_at AS "finishedAt", j.state::text AS "queueState", + j.retry_count AS "retryCount" + FROM contact_imports.imports i LEFT JOIN import_jobs.job j ON j.id = i.id AND j.name = $3 + WHERE i.id = $1 AND i.account_id = $2`, + [id, accountId, QUEUE], + ); + if (!row) throw new ApiError(404, "import_not_found"); + return row; + }); + } + async results(accountId: string, id: string) { + const status = await this.status(accountId, id); + if (status.state !== "succeeded") + throw new ApiError(409, "import_not_succeeded"); + const { rows } = await this.db.query( + "SELECT email, name FROM contact_imports.contacts WHERE import_id = $1 AND account_id = $2 ORDER BY row_index", + [id, accountId], + ); + return rows; + } +} diff --git a/applications/durable-imports/src/input.ts b/applications/durable-imports/src/input.ts new file mode 100644 index 00000000..114717f1 --- /dev/null +++ b/applications/durable-imports/src/input.ts @@ -0,0 +1,81 @@ +import { createHash } from "node:crypto"; + +export const MAX_ROWS = 100; +export const MAX_BYTES = 64 * 1024; +export const MAX_PENDING = 5; +export const MAX_DAILY = 50; +export const CASEY = "00000000-0000-0000-0000-000000000001"; +export const MORGAN = "00000000-0000-0000-0000-000000000002"; +export type Contact = { email: string; name: string }; +export class ApiError extends Error { + constructor( + public status: number, + public code: string, + ) { + super(code); + } +} +export function uuid(value: unknown): value is string { + return ( + typeof value === "string" && + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + value, + ) + ); +} +function object(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} +function exact(value: Record, keys: string[]): boolean { + return ( + JSON.stringify(Object.keys(value).sort()) === JSON.stringify(keys.sort()) + ); +} +export function normalize(input: unknown): { + requestId: string; + rows: Contact[]; + fingerprint: string; +} { + if ( + !object(input) || + !exact(input, ["requestId", "rows"]) || + !uuid(input.requestId) || + !Array.isArray(input.rows) || + input.rows.length < 1 || + input.rows.length > MAX_ROWS + ) + throw new ApiError(400, "invalid_import"); + if (Buffer.byteLength(JSON.stringify(input)) > MAX_BYTES) + throw new ApiError(413, "body_too_large"); + const emails = new Set(); + const rows = input.rows.map((row): Contact => { + if ( + !object(row) || + !exact(row, ["email", "name"]) || + typeof row.email !== "string" || + typeof row.name !== "string" || + !row.name.isWellFormed() + ) + throw new ApiError(400, "invalid_contact"); + const email = row.email.trim().toLowerCase(); + const name = row.name.normalize("NFKC").trim().replace(/\s+/gu, " "); + if ( + email.length > 254 || + !/^[a-z0-9._+-]+@[a-z0-9.-]+\.[a-z]{2,63}$/.test(email) || + name.length === 0 || + [...name].length > 80 || + name.includes("\0") || + emails.has(email) + ) + throw new ApiError(400, "invalid_contact"); + emails.add(email); + return { email, name }; + }); + return { + requestId: input.requestId.toLowerCase(), + rows, + fingerprint: createHash("sha256") + .update(JSON.stringify(rows)) + .digest("hex"), + }; +} diff --git a/applications/durable-imports/src/queue.ts b/applications/durable-imports/src/queue.ts new file mode 100644 index 00000000..f81ddad0 --- /dev/null +++ b/applications/durable-imports/src/queue.ts @@ -0,0 +1,46 @@ +import { PgBoss } from "pg-boss"; +import { databaseConfig } from "./database.js"; +export const QUEUE = "contact-imports"; +export const SCHEMA = "import_jobs"; +export const RETRY_LIMIT = 2; +export const QUEUE_OPTIONS = { + retryLimit: RETRY_LIMIT, + retryDelay: 1, + retryBackoff: true, + retryDelayMax: 4, + expireInSeconds: 20, + heartbeatSeconds: 10, + retentionSeconds: 3600, + deleteAfterSeconds: 86400, + partition: false, + notify: false, +}; +export function queue(worker = false, migrate = false): PgBoss { + const boss = new PgBoss({ + ...databaseConfig(), + schema: SCHEMA, + migrate, + createSchema: false, + supervise: worker, + schedule: false, + reindex: false, + persistQueueStats: false, + persistWarnings: false, + superviseIntervalSeconds: 2, + monitorIntervalSeconds: 2, + queueCacheIntervalSeconds: 10, + maintenanceIntervalSeconds: 60, + max: 5, + }); + // Never print SQL parameters, credentials or contact payloads to application logs. + boss.on("error", () => console.error("Queue operation error")); + boss.on("warning", (warning) => + console.error( + "Queue warning", + warning.data && "type" in warning.data + ? String(warning.data.type) + : "unspecified", + ), + ); + return boss; +} diff --git a/applications/durable-imports/src/server.ts b/applications/durable-imports/src/server.ts new file mode 100644 index 00000000..5531ad4b --- /dev/null +++ b/applications/durable-imports/src/server.ts @@ -0,0 +1,25 @@ +import { createApp } from "./app.js"; +import { pool } from "./database.js"; +import { Imports } from "./imports.js"; +import { queue } from "./queue.js"; +if (process.env.PGUSER !== "imports_api") + throw new Error("Server requires the imports_api role"); +const db = pool(); +const boss = queue(); +await boss.start(); +const port = Number(process.env.PORT ?? "4000"); +if (!Number.isInteger(port) || port < 1 || port > 65535) + throw new Error("Invalid PORT"); +const server = createApp(new Imports(db, boss)).listen(port, "127.0.0.1", () => + console.log("Import API listening on loopback"), +); +let stopping = false; +async function stop() { + if (stopping) return; + stopping = true; + server.close(); + await boss.stop(); + await db.end(); +} +process.on("SIGTERM", () => void stop()); +process.on("SIGINT", () => void stop()); diff --git a/applications/durable-imports/src/worker.ts b/applications/durable-imports/src/worker.ts new file mode 100644 index 00000000..a3f88d55 --- /dev/null +++ b/applications/durable-imports/src/worker.ts @@ -0,0 +1,46 @@ +import type { Db, Job } from "pg-boss"; +import { pool } from "./database.js"; +import { processImport, testFault, type ImportJob } from "./handler.js"; +import { reconcile } from "./imports.js"; +import { QUEUE, queue } from "./queue.js"; +if (process.env.PGUSER !== "imports_worker") + throw new Error("Worker requires the imports_worker role"); +const db = pool(); +const boss = queue(true); +const fault = testFault(); +// Preserve terminal outcomes before the queue supervisor can prune old metadata. +await reconcile(db); +await boss.start(); +await boss.work( + QUEUE, + { + transactional: true, + transactionTimeoutSeconds: 25, + batchSize: 1, + localConcurrency: 1, + pollingIntervalSeconds: 0.5, + }, + (jobs: Job[], tx: Db) => processImport(jobs, tx, fault), +); +console.log("Import worker ready"); +process.send?.({ type: "ready" }); +let reconciling = false; +const timer = setInterval(() => { + if (reconciling) return; + reconciling = true; + void reconcile(db) + .catch(() => console.error("Import outcome reconciliation failed")) + .finally(() => { + reconciling = false; + }); +}, 1000); +let stopping = false; +async function stop() { + if (stopping) return; + stopping = true; + clearInterval(timer); + await boss.stop({ graceful: true, timeout: 25_000 }); + await db.end(); +} +process.on("SIGTERM", () => void stop()); +process.on("SIGINT", () => void stop()); diff --git a/applications/durable-imports/test/cloud.test.ts b/applications/durable-imports/test/cloud.test.ts new file mode 100644 index 00000000..a08d8082 --- /dev/null +++ b/applications/durable-imports/test/cloud.test.ts @@ -0,0 +1,653 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { spawn, execFileSync, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { setTimeout as delay } from "node:timers/promises"; +import { after, afterEach, before, test } from "node:test"; +import pg from "pg"; +import { checkServerIdentity, type ConnectionOptions } from "node:tls"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { databaseConfig } from "../src/database.js"; +import { Imports } from "../src/imports.js"; +import { CASEY, MORGAN, ApiError } from "../src/input.js"; +import { QUEUE, queue } from "../src/queue.js"; + +process.env.NODE_ENV = "test"; +const config = databaseConfig(); +const owner = new pg.Pool({ + ...config, + user: "imports_migration", + password: process.env.MIGRATION_PASSWORD, +}); +const api = new pg.Pool({ + ...config, + user: "imports_api", + password: process.env.APP_PASSWORD, +}); +const workerDb = new pg.Pool({ + ...config, + user: "imports_worker", + password: process.env.WORKER_PASSWORD, +}); +const producer = queue(); +const imports = new Imports(api, producer); +const children = new Set(); +const casey = CASEY; +const morgan = MORGAN; +const input = (requestId = randomUUID()) => ({ + requestId, + rows: [ + { email: " Casey@Example.test ", name: " Casey Example " }, + { email: "morgan@example.test", name: "Morgan Example" }, + ], +}); + +async function eventually( + read: () => Promise, + acceptable: (value: T) => boolean, + timeout = 60_000, +): Promise { + const deadline = Date.now() + timeout; + let last: T; + do { + last = await read(); + if (acceptable(last)) return last; + await delay(200); + } while (Date.now() < deadline); + throw new Error( + `Timed out waiting for expected state: ${JSON.stringify(last)}`, + ); +} +function launch( + script: "server" | "worker", + fault: Record = {}, +) { + const env: NodeJS.ProcessEnv = {}; + // Child processes receive only their runtime password, never owner/admin credentials. + for (const key of [ + "PATH", + "HOME", + "PGHOST", + "PGPORT", + "PGDATABASE", + "PGSSLROOTCERT", + ]) + env[key] = process.env[key]; + Object.assign(env, { + NODE_ENV: "test", + PGUSER: script === "server" ? "imports_api" : "imports_worker", + PGPASSWORD: + script === "server" + ? process.env.APP_PASSWORD + : process.env.WORKER_PASSWORD, + PORT: "4081", + ...fault, + }); + if (script === "server") + Object.assign(env, { + CASEY_TOKEN: process.env.CASEY_TOKEN, + MORGAN_TOKEN: process.env.MORGAN_TOKEN, + }); + const child = spawn(process.execPath, [`dist/src/${script}.js`], { + env, + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + let output = ""; + child.stdout!.on("data", (data) => { + output += data.toString(); + }); + child.stderr!.on("data", (data) => { + output += data.toString(); + }); + children.add(child); + child.once("exit", () => children.delete(child)); + return { child, output: () => output }; +} +async function readyWorker(fault: Record = {}) { + const proc = launch("worker", fault); + await eventually( + async () => ({ + alive: proc.child.exitCode === null, + output: proc.output(), + }), + (s) => s.output.includes("Import worker ready"), + 20_000, + ); + return proc; +} +async function stop(child: ChildProcess, signal: NodeJS.Signals = "SIGTERM") { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill(signal); + await Promise.race([ + exited, + delay(30_000).then(() => { + throw new Error("Process did not stop"); + }), + ]); + assert(child.exitCode !== null || child.signalCode !== null); +} +async function counts(id: string) { + const { + rows: [row], + } = await owner.query( + `SELECT i.state, i.result_count, j.state::text AS job_state, j.retry_count, + (SELECT count(*)::integer FROM contact_imports.contacts c WHERE c.import_id=i.id) AS effects + FROM contact_imports.imports i LEFT JOIN import_jobs.job j ON j.id=i.id WHERE i.id=$1`, + [id], + ); + return row; +} +async function settled(id: string, state = "succeeded") { + return eventually( + () => counts(id), + (r) => + r.state === state && + (state !== "succeeded" || r.job_state === "completed"), + ); +} +async function request( + path: string, + token = process.env.CASEY_TOKEN!, + body?: unknown, +) { + return fetch(`http://127.0.0.1:4081${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); +} +before(async () => { + await producer.start(); +}); +afterEach(async () => { + for (const child of [...children]) await stop(child); +}); +after(async () => { + for (const child of [...children]) await stop(child); + await producer.stop(); + await Promise.all([owner.end(), api.end(), workerDb.end()]); +}); + +test( + "verified TLS, same-endpoint controls and restricted runtime roles", + { timeout: 40_000 }, + async () => { + assert.equal( + (await api.query("SELECT current_user AS role")).rows[0].role, + "imports_api", + ); + const goodSsl = config.ssl as ConnectionOptions; + const certDir = mkdtempSync(join(tmpdir(), "imports-wrong-ca-")); + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + join(certDir, "key.pem"), + "-out", + join(certDir, "ca.pem"), + "-days", + "1", + "-subj", + "/CN=unrelated-test-ca", + ], + { stdio: "ignore" }, + ); + const wrongCa = readFileSync(join(certDir, "ca.pem"), "utf8"); + rmSync(certDir, { recursive: true }); + for (const [ssl, expected] of [ + [ + { ...goodSsl, ca: wrongCa }, + /SELF_SIGNED_CERT_IN_CHAIN|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT_LOCALLY/, + ], + [ + { + ...goodSsl, + checkServerIdentity: ( + _host: string, + certificate: Parameters[1], + ) => checkServerIdentity("wrong-hostname.example.test", certificate), + }, + /ERR_TLS_CERT_ALTNAME_INVALID/, + ], + ] as const) { + const bad = new pg.Client({ ...config, ssl }); + try { + await assert.rejects( + bad.connect(), + (error: NodeJS.ErrnoException) => ( + console.log("Negative TLS control", error.code), + expected.test(error.code ?? "") + ), + ); + } finally { + await bad.end(); + } + } + assert.equal( + (await api.query("SELECT 1 AS positive_control")).rows[0] + .positive_control, + 1, + ); + for (const db of [api, workerDb]) { + await assert.rejects( + db.query( + "SELECT import_jobs.create_queue('runtime-forbidden', '{}'::jsonb)", + ), + (e: pg.DatabaseError) => e.code === "42501", + ); + for (const sql of [ + "CREATE TABLE contact_imports.runtime_ddl(id int)", + "UPDATE import_jobs.version SET version=version", + "UPDATE contact_imports.imports SET payload='[]'::jsonb WHERE false", + ]) { + await assert.rejects( + db.query(sql), + (error: pg.DatabaseError) => error.code === "42501", + ); + } + } + await assert.rejects( + api.query("UPDATE import_jobs.job SET state='active' WHERE false"), + (e: pg.DatabaseError) => e.code === "42501", + ); + await assert.rejects( + workerDb.query( + "UPDATE contact_imports.contacts SET name='changed' WHERE false", + ), + (e: pg.DatabaseError) => e.code === "42501", + ); + console.log( + "TLS trust and hostname failures verified; positive same-endpoint connection and role denials passed", + ); + }, +); + +test( + "enqueue rollback, competing retries and account-scoped request retention", + { timeout: 40_000 }, + async () => { + const submission = input(); + const beforeCount = ( + await owner.query( + "SELECT submission_count FROM contact_imports.accounts WHERE id=$1", + [casey], + ) + ).rows[0].submission_count; + await assert.rejects( + imports.submit(casey, submission, async () => { + throw new Error("After both writes"); + }), + /After both writes/, + ); + assert.equal( + ( + await owner.query( + "SELECT count(*)::integer AS n FROM contact_imports.imports WHERE request_id=$1", + [submission.requestId], + ) + ).rows[0].n, + 0, + ); + assert.equal( + (await owner.query("SELECT count(*)::integer AS n FROM import_jobs.job")) + .rows[0].n, + 0, + ); + assert.equal( + ( + await owner.query( + "SELECT submission_count FROM contact_imports.accounts WHERE id=$1", + [casey], + ) + ).rows[0].submission_count, + beforeCount, + ); + const repeated = await Promise.all( + Array.from({ length: 8 }, () => imports.submit(casey, submission)), + ); + assert.equal(new Set(repeated.map((r) => r.id)).size, 1); + assert.equal(repeated.filter((r) => !r.replay).length, 1); + assert.equal( + ( + await owner.query( + "SELECT count(*)::integer AS n FROM import_jobs.job WHERE id=$1", + [repeated[0]!.id], + ) + ).rows[0].n, + 1, + ); + await assert.rejects( + imports.submit(casey, { + ...submission, + rows: [...submission.rows].reverse(), + }), + (e: ApiError) => e.status === 409, + ); + await assert.rejects( + imports.status(morgan, repeated[0]!.id), + (e: ApiError) => e.status === 404, + ); + const first = await readyWorker(); + const second = await readyWorker(); + const outcome = await settled(repeated[0]!.id); + assert.equal(outcome.effects, 2); + assert.equal(outcome.result_count, 2); + // A separate import may contain the same email; contacts are import-scoped results. + const other = await imports.submit(casey, input()); + await settled(other.id); + assert.equal((await imports.results(casey, other.id)).length, 2); + const replayJob = randomUUID(); + await producer.send( + QUEUE, + { importId: repeated[0]!.id }, + { id: replayJob }, + ); + await eventually( + async () => + ( + await owner.query( + "SELECT state::text AS state FROM import_jobs.job WHERE id=$1", + [replayJob], + ) + ).rows[0]?.state, + (state) => state === "completed", + ); + assert.equal((await counts(repeated[0]!.id)).effects, 2); + await stop(first.child); + await stop(second.child); + console.log( + "Eight concurrent repeats produced one import/job; two worker processes and completion replay produced no duplicate effects", + ); + }, +); + +test("bounded account submission limits are coordinated by database locks", async () => { + const pending = []; + for (let i = 0; i < 5; i++) + pending.push(await imports.submit(morgan, input())); + await assert.rejects( + imports.submit(morgan, input()), + (e: ApiError) => e.status === 429, + ); + assert.equal( + ( + await imports.submit( + morgan, + input( + ( + await owner.query( + "SELECT request_id FROM contact_imports.imports WHERE id=$1", + [pending[0]!.id], + ) + ).rows[0].request_id, + ), + ) + ).replay, + true, + ); + const worker = await readyWorker(); + for (const p of pending) await settled(p.id); + await stop(worker.child); + await owner.query( + "UPDATE contact_imports.accounts SET submission_count=50 WHERE id=$1", + [morgan], + ); + await assert.rejects( + imports.submit(morgan, input()), + (e: ApiError) => e.status === 429, + ); + await owner.query( + "UPDATE contact_imports.accounts SET window_start=clock_timestamp()-interval '25 hours' WHERE id=$1", + [morgan], + ); + await imports.submit(morgan, input()); + assert.equal( + ( + await owner.query( + "SELECT submission_count FROM contact_imports.accounts WHERE id=$1", + [morgan], + ) + ).rows[0].submission_count, + 1, + ); +}); + +test( + "handler failure rolls effects and completion back, then retries successfully", + { timeout: 60_000 }, + async () => { + const submitted = await imports.submit(casey, input()); + const worker = await readyWorker({ + TEST_IMPORT_ID: submitted.id, + TEST_FAIL_ATTEMPTS: "1", + }); + await eventually( + () => counts(submitted.id), + (r) => r.job_state === "retry", + ); + const rolledBack = await counts(submitted.id); + assert.equal(rolledBack.state, "queued"); + assert.equal(rolledBack.effects, 0); + assert.equal(rolledBack.result_count, null); + const final = await settled(submitted.id); + assert.equal(final.retry_count, 1); + assert.equal(final.effects, 2); + assert(!worker.output().includes("Queue operation error"), worker.output()); + await stop(worker.child); + console.log( + "Fault after contacts and result update left zero effects/no completion; retry committed both", + ); + }, +); + +test( + "three failed attempts become durable failure; runtime retention preserves reported outcome", + { timeout: 100_000 }, + async () => { + const submitted = await imports.submit(casey, input()); + const worker = await readyWorker({ + TEST_IMPORT_ID: submitted.id, + TEST_FAIL_ATTEMPTS: "3", + }); + await eventually( + () => imports.status(casey, submitted.id), + (r) => r.state === "failed", + ); + const final = await counts(submitted.id); + assert.equal(final.retry_count, 2); + assert.equal(final.job_state, "failed"); + assert.equal(final.effects, 0); + await owner.query( + "UPDATE import_jobs.job SET completed_on=clock_timestamp()-interval '2 days', keep_until=clock_timestamp()-interval '2 days' WHERE id=$1", + [submitted.id], + ); + // Rewind only the test queue's maintenance cadence. A runtime-only supervisor performs DELETE. + await owner.query( + "UPDATE import_jobs.queue SET maintain_on=clock_timestamp()-interval '2 minutes' WHERE name=$1", + [QUEUE], + ); + await eventually( + async () => + ( + await owner.query( + "SELECT count(*)::integer AS n FROM import_jobs.job WHERE id=$1", + [submitted.id], + ) + ).rows[0].n, + (n) => n === 0, + 75_000, + ); + assert.equal( + (await imports.status(casey, submitted.id)).error, + "Queue retry limit exhausted", + ); + assert(!worker.output().includes("Queue operation error"), worker.output()); + await stop(worker.child); + console.log( + "Retry limit exhausted after three attempts; runtime retention deleted queue metadata, durable failure remained", + ); + }, +); + +test( + "real SIGKILL after uncommitted effects recovers through runtime expiry without duplicates", + { timeout: 100_000 }, + async () => { + const submitted = await imports.submit(casey, input()); + const doomed = launch("worker", { + TEST_IMPORT_ID: submitted.id, + TEST_PAUSE_AFTER_EFFECTS: "true", + }); + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error("No effects-written kill point")), + 25_000, + ); + doomed.child.on("message", (message: any) => { + if ( + message.type === "effects_written" && + message.importId === submitted.id + ) { + clearTimeout(timeout); + resolve(); + } + }); + }); + assert.equal((await counts(submitted.id)).effects, 0); + assert.equal((await counts(submitted.id)).state, "queued"); + assert.equal((await counts(submitted.id)).job_state, "active"); + await stop(doomed.child, "SIGKILL"); + const survivor = await readyWorker(); + const recovered = await settled(submitted.id); + assert(recovered.retry_count >= 1); + assert.equal(recovered.effects, 2); + assert.equal(recovered.result_count, 2); + assert( + !survivor.output().includes("Queue operation error"), + survivor.output(), + ); + await stop(survivor.child); + console.log( + "SIGKILL confirmed exited; active claim recovered via supervisor, effects and completion committed once", + ); + }, +); + +test( + "HTTP account isolation, malformed requests and real API/worker restart", + { timeout: 60_000 }, + async () => { + const server = launch("server"); + await eventually( + async () => { + try { + return (await fetch("http://127.0.0.1:4081/health")).status; + } catch { + return 0; + } + }, + (status) => status === 200, + 20_000, + ); + assert.equal( + (await request("/imports", "wrong-token", input())).status, + 401, + ); + assert.equal( + ( + await request("/imports", process.env.CASEY_TOKEN!, { + ...input(), + accountId: morgan, + }) + ).status, + 400, + ); + assert.equal( + ( + await request("/imports", process.env.CASEY_TOKEN!, { + ...input(), + rows: [{ email: "x@example.test", name: "bad\u0000name" }], + }) + ).status, + 400, + ); + assert.equal( + ( + await request("/imports", process.env.CASEY_TOKEN!, { + requestId: randomUUID(), + rows: "x".repeat(70_000), + }) + ).status, + 413, + ); + const submittedResponse = await request( + "/imports", + process.env.CASEY_TOKEN!, + input(), + ); + assert.equal(submittedResponse.status, 202); + const submitted = (await submittedResponse.json()) as { id: string }; + assert.equal( + (await request(`/imports/${submitted.id}`, process.env.MORGAN_TOKEN!)) + .status, + 404, + ); + assert.equal( + ( + await request( + `/imports/${submitted.id}/results`, + process.env.MORGAN_TOKEN!, + ) + ).status, + 404, + ); + assert.equal( + (await request(`/imports/${submitted.id}/results`)).status, + 409, + ); + await stop(server.child); + const worker = await readyWorker(); + await settled(submitted.id); + await stop(worker.child); + const newWorker = await readyWorker(); + const restarted = launch("server"); + await eventually( + async () => { + try { + return (await fetch("http://127.0.0.1:4081/health")).status; + } catch { + return 0; + } + }, + (status) => status === 200, + 20_000, + ); + const result = await request(`/imports/${submitted.id}/results`); + assert.equal(result.status, 200); + assert.equal(((await result.json()) as { rows: unknown[] }).rows.length, 2); + await stop(restarted.child); + await stop(newWorker.child); + console.log( + "Native HTTP acceptance and confirmed API/worker process restarts retained the durable result", + ); + }, +); + +test("lost queue metadata is explicit rather than eternal processing", async () => { + const submitted = await imports.submit(casey, input()); + await owner.query("DELETE FROM import_jobs.job WHERE id=$1", [submitted.id]); + const result = await imports.status(casey, submitted.id); + assert.equal(result.state, "failed"); + assert.equal(result.error, "Queue metadata no longer available"); +}); diff --git a/applications/durable-imports/test/domain.test.ts b/applications/durable-imports/test/domain.test.ts new file mode 100644 index 00000000..98f21f1f --- /dev/null +++ b/applications/durable-imports/test/domain.test.ts @@ -0,0 +1,75 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { normalize, ApiError } from "../src/input.js"; +const input = () => ({ + requestId: randomUUID(), + rows: [ + { email: " SYNTHETIC@EXAMPLE.TEST ", name: " Ｃａｓｅｙ\t Example " }, + ], +}); +test("normalization and request retry fingerprint are deterministic", () => { + const value = normalize(input()); + assert.deepEqual(value.rows, [ + { email: "synthetic@example.test", name: "Casey Example" }, + ]); + const replay = normalize({ + requestId: value.requestId.toUpperCase(), + rows: value.rows, + }); + assert.equal(value.fingerprint, replay.fingerprint); + assert.equal(value.requestId, replay.requestId); +}); +test("malformed/extra fields, duplicate emails and invalid Unicode are rejected", () => { + for (const value of [ + null, + [], + { ...input(), accountId: randomUUID() }, + { ...input(), rows: [{ email: "x@example.test", name: "\ud800" }] }, + { ...input(), rows: [{ email: "x@example.test", name: "hidden\0" }] }, + { + ...input(), + rows: [ + { email: "x@example.test", name: "One" }, + { email: "X@example.test", name: "Two" }, + ], + }, + ]) + assert.throws(() => normalize(value), ApiError); +}); +test("row, byte and PostgreSQL codepoint bounds are independent", () => { + assert.throws( + () => + normalize({ + ...input(), + rows: Array.from({ length: 101 }, (_, n) => ({ + email: `x${n}@example.test`, + name: "A", + })), + }), + ApiError, + ); + assert.throws( + () => + normalize({ + ...input(), + rows: [{ email: "x@example.test", name: "x".repeat(70_000) }], + }), + (e: unknown) => e instanceof ApiError && e.status === 413, + ); + assert.equal( + normalize({ + ...input(), + rows: [{ email: "x@example.test", name: "😀".repeat(80) }], + }).rows[0]!.name.length, + 160, + ); + assert.throws( + () => + normalize({ + ...input(), + rows: [{ email: "x@example.test", name: "😀".repeat(81) }], + }), + ApiError, + ); +}); diff --git a/applications/durable-imports/tsconfig.json b/applications/durable-imports/tsconfig.json new file mode 100644 index 00000000..716d033a --- /dev/null +++ b/applications/durable-imports/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2024", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": ".", + "outDir": "dist", + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "scripts/**/*.ts", "test/**/*.ts"] +}