diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cb90b9..aa1fdfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: - name: Test run: | npm test + npm run verify:docs npm run examples:check npm run benchmark:compare:local diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 59cb9dd..3eaf3a4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -41,7 +41,9 @@ jobs: run: npm run check - name: Test - run: npm test + run: | + npm test + npm run verify:docs - name: Build package run: npm run build:package diff --git a/README.md b/README.md index 01e5966..29953d9 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,11 @@ ThimbleDB is a Cloudflare-first database for small, read-heavy web -applications. Browsers read encrypted immutable objects through an -authenticated storage broker and retain them in memory and IndexedDB caches. -Writes and key grants use the same small authority. +applications. Browsers read through an authenticated authority and retain +scope-separated data in memory and encrypted IndexedDB caches. The default +read path returns encrypted immutable objects; deployments can explicitly +enable bounded decoded read bundles for eligible cold point reads. Writes and +key grants use the same small authority. Cloudflare Workers and R2 are the reference deployment. Azure Blob Storage, Amazon S3, and a local filesystem adapter implement the same provider-neutral @@ -21,55 +23,152 @@ Licensed under the [Apache License 2.0](LICENSE). ## Architecture +### Identity, session, and client bootstrap + ```mermaid -flowchart LR - subgraph Browser["Browser trust boundary"] - App["Web application"] - Client["ThimbleDB client"] - Memory["Decoded memory LRU"] - IDB["Device-key-encrypted IndexedDB"] - ScopeKey["Non-extractable scope key"] - - App --> Client - Client --> Memory - Client --> IDB - Client --> ScopeKey - end +sequenceDiagram + autonumber + participant User as User + participant IdP as External OIDC provider + participant App as Browser application + participant Client as ThimbleDB client + participant Cache as Memory + encrypted IndexedDB + participant Authority as In-app or separate authority + participant Auth as Private auth store + + User->>IdP: Authenticate with authorization code + PKCE + IdP-->>App: Short-lived access token + App->>Authority: Exchange token for opaque session + Authority->>Authority: Verify signature, issuer, audience, scope, and role + Authority->>Auth: Map identity and create revocable session + Authority-->>App: HttpOnly cookie and CSRF token + + App->>Client: createThimbleClient() + Client->>Authority: GET /api/config + Authority->>Auth: Reload user and calculate current grants + Authority-->>Client: Scope, layouts, indexes, generation, and capabilities + Client->>Authority: GET authorised scope-key grant + Authority-->>Client: Current and readable historical scope keys + Client->>Client: Import non-extractable decrypt-only keys + Client->>Cache: Open authority-and-scope cache namespace + Client-->>App: Ready typed client +``` - subgraph ReadPath["Brokered private-read boundary"] - Domain["Authenticated object endpoint"] - Objects["TDB1 gzip + AES-GCM envelopes"] - Domain --> Objects - end +1. External OIDC owns credentials, MFA, recovery, and token issuance. Service + principals use short-lived application tokens through the same exchange. +2. The authority maps the external identity to a stable internal principal, + stores only an opaque session digest, and recalculates grants on requests. +3. The browser imports authorised scope keys as non-extractable, memory-only + CryptoKeys. Persistent cache values use a separate device key. - subgraph Authority["Authenticated authority boundary"] - Auth["Authentication and scope authorisation"] - Grant["Short-lived key grant"] - Write["Validation and conditional write"] - Auth --> Grant - Auth --> Write - end +### Read flow - Client -- "Session-authorised ciphertext reads" --> Domain - Client -- "Mutations" --> Auth - Grant -- "Memory-only CryptoKey" --> ScopeKey - Write -- "Encrypted objects" --> Objects +```mermaid +flowchart TD + Read["Point read or bounded query"] --> HeadCached{"Collection HEAD cached?"} + HeadCached -- No --> BundleCheck{"Eligible cold point read
and bundle advertised?"} + HeadCached -- Yes --> HeadFresh{"HEAD inside its TTL?"} + HeadFresh -- Yes --> Resolve["Resolve referenced index,
snapshot, or trie objects"] + HeadFresh -- No --> Revalidate["Authenticated HEAD revalidation
with If-None-Match"] + Revalidate --> HeadResult{"HEAD result"} + HeadResult -- "304" --> Resolve + HeadResult -- "Changed" --> Objects + HeadResult -- "Network unavailable
and cached HEAD usable" --> Resolve + HeadResult -- "Network unavailable
and no usable cache" --> ReadError["Return explicit read error"] + Resolve --> ValuesCached{"Required immutable values cached?"} + ValuesCached -- Yes --> Plan["Validate document or query plan,
predicate, ordering, projection, and limit"] + ValuesCached -- No --> Objects + + BundleCheck -- Yes --> Bundle["Authority revalidates read grant,
reads encrypted HEAD and required objects"] + Bundle --> BundleLimit{"At most 4 objects and 4 MiB
with authenticated size metadata?"} + BundleLimit -- Yes --> Decoded["Return decoded cache values
over HTTPS with no-store"] + BundleLimit -- No --> Objects + BundleCheck -- No --> Objects["Revalidate read grant and return
authenticated TDB1 objects"] + Objects --> BrowserDecrypt["Browser decrypts and validates
HEAD, index, snapshot, or trie objects"] + Decoded --> DeviceCache["Encrypt decoded values with device key
and update the scoped cache"] + BrowserDecrypt --> DeviceCache --> Plan + Plan --> ReadResult["Document, or bounded query result
with point, index, or scan plan"] ``` +1. Warm reads stay in the authority-and-scope cache while the mutable HEAD is + fresh. Expired HEADs use conditional revalidation, and a usable cache can + remain available during a network failure. +2. Cold point reads can use one + bounded decoded bundle when explicitly enabled; every ineligible or failed + bundle falls back to authenticated TDB1 object reads. +3. Queries remain bounded and report whether they used a point, declared + index, covering projection, or collection scan plan. + +### Mutation and cache-synchronisation flow + +```mermaid +flowchart TD + Mutation["Create, replace, delete, restore, purge,
scope erase, or index rebuild"] + Mutation --> Request["Session + CSRF + exact Origin
scope + layout generation"] + Request --> Guards{"Operation allowed by maintenance state
and generation current?"} + Guards -- No --> Reject["Return explicit maintenance
or layout-changed error"] + Guards -- Yes --> Grant["Reload user and current write or admin grant"] + Grant --> Authorised{"Authorised?"} + Authorised -- No --> Deny["Return explicit forbidden response"] + Authorised -- Yes --> Load["Read current HEAD and affected immutable objects"] + Load --> Validate["Validate route, ID, document, limits,
layout, and complete index configuration"] + Validate --> Immutable["Create immutable document, root,
and index objects"] + Immutable --> Publish["Publish one HEAD with If-Match"] + Publish --> Conflict{"ETag conflict?"} + Conflict -- Yes --> Retry{"Bounded retry remains?"} + Retry -- Yes --> Load + Retry -- No --> ConflictError["Return explicit conflict"] + Conflict -- No --> Commit["Return committed values and new ETag"] + Commit --> Cache["Update the current scoped cache"] + Cache --> Tabs["Notify other tabs through BroadcastChannel"] + Tabs --> Result["Committed mutation result"] +``` + +1. The authority validates every mutation, creates immutable document and + index objects, and publishes their references through one conditional HEAD. +2. Successful writes update the current cache and notify other tabs. Logout + revokes the session and clears the affected browser cache namespace. + +### Deployment, scaling, and storage flow + ```mermaid flowchart TB - Engine["ThimbleDB protocol
cache + scopes + TDB1 + conditional HEAD"] - Contract["ObjectStore abstraction
get + put + delete + list + ETag conditions"] - - Engine --> Contract - Contract --> R2["Cloudflare R2
preferred"] - Contract --> Local["Local filesystem
development"] - Contract --> Azure["Azure Blob Storage
supported"] - Contract --> S3["Amazon S3
supported"] + Browser["Browser application"] --> Origin["One public browser origin"] + + subgraph InApp["In-app authority"] + Combined["Application + ThimbleDB authority
secrets colocated
one release and scaling policy"] + end + + subgraph Separate["Separate Worker or service"] + Router["Path router or application gateway"] + Application["Application assets or server"] + Authority["ThimbleDB authority + secrets
independent release, limits,
logs, region, and scaling"] + Router -- "/" --> Application + Router -- "/api/* and /studio/*" --> Authority + end + + Origin -- "In-app" --> Combined + Origin -- "Separate" --> Router + + Combined --> Contract["ObjectStore contract
get + put + delete + list + ETag conditions"] + Authority --> Contract + + Contract --> R2["Cloudflare R2
private data + auth bindings"] + Contract --> S3["Amazon S3
private data + auth buckets"] + Contract --> Azure["Azure Blob Storage
private data + auth containers"] + Contract --> Local["Local filesystem
two roots, one process"] ``` +In-app deployment has the lowest operational floor and scales the application +and authority together. A separate Worker or service isolates credentials, +deployments, failures, observability, regional placement, and runtime scaling. +It does not remove per-collection conditional-write contention or change the +browser API. + The stored object and encryption protocol stays the same across providers. -Only bindings, credentials, and browser read authorisation differ. +Only bindings, credentials, and deployment primitives differ. The local +filesystem adapter remains single-process; use shared cloud object storage +before horizontally scaling a Node authority. ## Features @@ -163,9 +262,11 @@ Use the browser/core API from `thimbledb`, external identity primitives from `thimbledb/authority/node` or `thimbledb/authority/cloudflare`. Consumers supply their own domain, storage, OIDC application, and secrets. -The authority can share the application deployment or run as a separate -service behind the same public browser origin. See -[Authority deployment modes](docs/AUTHORITY-DEPLOYMENT.md). +The authority can run in-app with the application or as a separate Worker, +container, function, or Node service behind the same public browser origin. +In-app deployment minimises operations. A separate authority isolates secrets, +releases, failures, and scaling. See +[In-app and separate authority deployment](docs/AUTHORITY-DEPLOYMENT.md). After the authority session exists: @@ -251,19 +352,20 @@ layout decision thresholds. | Document | Purpose | | --- | --- | | [Quickstart](docs/QUICKSTART.md) | Package, authority, browser client, and verification setup | +| [Configuration reference](docs/CONFIGURATION.md) | Authority options, environment variables, provider settings, defaults, and template coverage | | [Implementation prompts](docs/IMPLEMENTATION-PROMPTS.md) | Copy-paste integration, deployment, migration, and review prompts | | [npm publishing](docs/NPM-PUBLISHING.md) | OIDC trusted publisher setup and release process | | [Use cases](docs/USE-CASES.md) | Fit criteria and application-specific guides | | [Architecture](docs/ARCHITECTURE.md) | Components, data flow, and scope model | -| [Authority deployment modes](docs/AUTHORITY-DEPLOYMENT.md) | Embedded and separate authority topologies and decision criteria | -| [System diagrams](docs/DIAGRAMS.md) | Trust boundaries, sequences, keys, and providers | +| [In-app and separate authority deployment](docs/AUTHORITY-DEPLOYMENT.md) | Topologies, scaling opportunities, trust boundaries, and decision criteria | +| [System diagrams](docs/DIAGRAMS.md) | Trust, deployment, read, write, deletion, key, migration, Studio, and provider flows | | [Storage providers](docs/STORAGE-PROVIDERS.md) | Provider abstraction and conformance requirements | | [Security](docs/SECURITY.md) | Threat model, encryption, keys, and revocation | | [Authentication](docs/AUTHENTICATION.md) | External identity mapping, sessions, and scope grants | | [Deletion and retention](docs/DELETION-RETENTION.md) | Tombstones, restoration, scope erasure, and physical collection | | [Adaptive layouts](docs/ADAPTIVE-LAYOUTS.md) | Snapshot/trie recommendations and explicit migration | | [Protocol](docs/PROTOCOL.md) | Binary envelope and object layout | -| [Versioning](docs/VERSIONING.md) | Package, protocol, key, and v1 compatibility rules | +| [Versioning](docs/VERSIONING.md) | Package, protocol, key, and release compatibility rules | | [Public API](docs/PUBLIC-API.md) | Stable package exports and authority integration | | [Evaluation harness](docs/EVALUATION.md) | Browser harness, sample application, and benchmark usage | | [Benchmarks](docs/BENCHMARKS.md) | R2 browser methodology, results, and limitations | @@ -272,6 +374,7 @@ layout decision thresholds. | [Azure deployment](docs/DEPLOYMENT-AZURE.md) | Container Apps and Blob Storage | | [AWS deployment](docs/DEPLOYMENT-AWS.md) | Lambda container and private S3 buckets | | [Operations](docs/OPERATIONS.md) | Keys, backup, metrics, incidents, and cleanup | +| [Website privacy](docs/WEBSITE-PRIVACY.md) | Static-site data handling and Cloudflare Web Analytics disclosure | ## When to use ThimbleDB diff --git a/deploy/cloudflare/wrangler.example.jsonc b/deploy/cloudflare/wrangler.example.jsonc index 1552ca3..92443b5 100644 --- a/deploy/cloudflare/wrangler.example.jsonc +++ b/deploy/cloudflare/wrangler.example.jsonc @@ -17,8 +17,8 @@ "THIMBLE_HEAD_TTL_MS": "1000", "THIMBLE_READ_BUNDLES": "false", "THIMBLE_COLLECTION_LAYOUTS": "", + "THIMBLE_COLLECTION_INDEXES": "{}", "THIMBLE_COLLECTIONS": "", - "THIMBLE_RETIRED_COLLECTION_LAYOUTS": "", "THIMBLE_DELETE_RETENTION_DAYS": "30", "THIMBLE_DELETE_GRACE_DAYS": "7", "THIMBLE_MAINTENANCE_MODE": "false", diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fe305dd..0a5513f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -7,8 +7,8 @@ storage abstraction. See [System diagrams](DIAGRAMS.md) for trust boundaries, request sequences, scope separation, and provider layouts. -See [Authority deployment modes](AUTHORITY-DEPLOYMENT.md) for embedded and -separately deployed authority choices. +See [In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md) for +placement, scaling, secret-boundary, and same-origin choices. See [Storage providers](STORAGE-PROVIDERS.md) for the provider contract and conformance requirements. diff --git a/docs/AUTHENTICATION.md b/docs/AUTHENTICATION.md index 2f006f8..a02acc0 100644 --- a/docs/AUTHENTICATION.md +++ b/docs/AUTHENTICATION.md @@ -106,9 +106,9 @@ account security version, provider, expiry, CSRF token, and issued scope grants. Cookies use HttpOnly, SameSite=Strict, and Secure outside local development. -Embedded and separately deployed authorities should normally remain behind one +In-app and separately deployed authorities should normally remain behind one public browser origin. See -[Authority deployment modes](AUTHORITY-DEPLOYMENT.md). +[In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md). On every authenticated request the authority reloads the current internal user record and recalculates grants. Role or tenant removal observed during a later diff --git a/docs/AUTHORITY-DEPLOYMENT.md b/docs/AUTHORITY-DEPLOYMENT.md index 36ebf63..221f3ec 100644 --- a/docs/AUTHORITY-DEPLOYMENT.md +++ b/docs/AUTHORITY-DEPLOYMENT.md @@ -1,23 +1,31 @@ -# Authority deployment modes +# In-app and separate authority deployment ThimbleDB separates the browser client from the authenticated authority. The -authority can run in the application's deployment or as a separately operated -Worker, container, or function. +authority can run in-app inside the application's deployment or as a separately +operated Worker, container, function, or Node service. This is a process and deployment choice. In both modes, expose application and authority routes through one public browser origin unless an independently reviewed cross-origin session design replaces the default Strict cookie contract. +The browser API, object format, collection definitions, and storage-provider +contract stay the same. The choice changes the runtime trust boundary, +deployment lifecycle, failure isolation, and which workloads can scale +independently. See the +[in-app topology](DIAGRAMS.md#in-app-embedded-authority-deployment) and +[separate-service topology](DIAGRAMS.md#separate-worker-or-service-deployment) +for the detailed request and secret boundaries. + ## Decision summary -| Consideration | Embedded authority | Separate authority service | +| Consideration | In-app (embedded) authority | Separate Worker or service | | --- | --- | --- | | Deployment units | One application deployment | Application plus authority deployment | | Browser origin | Naturally the same | Use a gateway or path route to preserve one public origin | | Secrets | Application runtime also holds authority and storage secrets | Storage credentials and master key remain outside the application runtime | | Release cadence | Application and authority change together | Authority can be upgraded independently | -| Scaling | Application and authority scale together | Reads, writes, and application rendering can scale separately | +| Scaling | Application and authority capacity scale as one unit | Authority traffic and application rendering can scale independently | | Failure boundary | One runtime can affect both application and data API | Application and authority failures are isolated | | Operations | Simplest setup and observability | More routing, monitoring, version coordination, and incident paths | | Latency | No service-to-service hop inside the deployment | Gateway and service routing may add latency | @@ -28,7 +36,7 @@ local files, Azure Blob Storage, S3, or R2 in either deployment model. A Cloudflare authority can share one Worker deployment with application assets or run as a dedicated routed Worker. -## Embedded authority +## In-app (embedded) authority The authority starts as part of the application deployment. The application and data API normally share one release, hostname, logs, and scaling policy. @@ -62,6 +70,10 @@ Node example: import { startNodeAuthority, } from "thimbledb/authority/node"; +import { + collectionIndexes, + collectionLayouts, +} from "./collections.js"; await startNodeAuthority({ studio: true, @@ -84,9 +96,10 @@ Choose this mode when: - a same-origin browser path should require no additional routing layer Avoid it when a compromise of the application runtime must not expose the -storage credential or deployment master key. +storage credential or deployment master key, or when application rendering and +authority traffic need different scaling or release controls. -## Separate authority service +## Separate Worker or service The authority runs in its own Worker, container, Lambda function, Container App, or Node service. The browser application remains a normal ThimbleDB @@ -122,6 +135,29 @@ Choose this mode when: The additional cost is real: another deployment, route, health check, log stream, alert set, version boundary, and incident path must be operated. +## Scaling opportunities + +Both modes can use a platform that scales horizontally. Separating the +authority does not partition collection data or remove conditional-write +contention. It creates an independent runtime and operations boundary. + +| Scaling pressure | In-app authority | Separate Worker or service | +| --- | --- | --- | +| Static assets and application rendering | Scale with authentication and data API traffic | Scale without adding authority instances | +| Sessions, key grants, object reads, and read bundles | Compete with application work for the same runtime limits | Receive a dedicated route, limits, logs, and scaling policy | +| Write bursts and index maintenance | Share application CPU, memory, concurrency, and rollout risk | Can use dedicated capacity and maintenance windows | +| Regional placement | Follows the application deployment | Can run near object storage while the application uses another region or edge | +| Failure isolation | Saturation or failure can affect the whole application | Authority saturation can be isolated from application rendering | +| Cost and cold starts | One deployment has the lowest operating floor | A second deployment can add idle cost, cold starts, and gateway latency | + +Cloudflare Workers scale per request in either topology. A separate Worker is +useful when independent routes, limits, releases, observability, or secret +isolation matter. It is not an automatic throughput improvement. + +For Node deployments, use a shared cloud object store before running multiple +authority instances. The local filesystem provider is intentionally limited to +one process and is not a scale-out storage backend. + ## Same-origin browser boundary The default ThimbleDB session cookie is `HttpOnly` and `SameSite=Strict`. @@ -157,7 +193,7 @@ objects and sends them over HTTPS with `no-store`. Leave the capability disabled if the deployment requires every read response above TLS to remain a TDB1 envelope. -An embedded authority removes one internal routing boundary. A separate +An in-app authority removes one internal routing boundary. A separate authority can instead be placed near object storage and scaled independently. Neither choice changes the number of browser requests once the same public route reaches the authority. @@ -179,7 +215,7 @@ runtime, and storage region. Both modes enforce the same sessions, scope grants, encryption, deletion, and conditional-write rules. -Embedded mode has a larger runtime blast radius because application server +In-app mode has a larger runtime blast radius because application server code and authority secrets coexist. Separate mode narrows that secret boundary but adds routing and service-to-service configuration that can itself be misconfigured. @@ -195,7 +231,7 @@ In either mode: ## Recommendation -Start embedded for one small application unless a concrete security, -operations, or scaling requirement justifies a separate authority. Move the -authority into a separate service without changing application collection -code, storage layout, or browser query semantics. +Start in-app for one small application unless a concrete security, operations, +or scaling requirement justifies a separate Worker or service. Move the +authority later without changing application collection code, storage layout, +or browser query semantics. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 0000000..ef6921b --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,186 @@ +# Configuration reference + +ThimbleDB authorities can be configured through code options and environment +settings. Prefer code for collection layouts and indexes because it is easier +to type-check and review. Keep secrets in the deployment platform's secret +store. + +Code options take precedence over matching environment settings. + +## Authority code options + +Both `startNodeAuthority()` and `createCloudflareAuthority()` accept: + +| Option | Purpose | Default | +| --- | --- | --- | +| `collectionLayouts` | Explicit `trie` or `snapshot` layout by collection | Trie for collections not listed | +| `collectionIndexes` | Complete declared secondary-index definitions | No indexes | +| `collections` | Bounded Studio collection catalogue | Names inferred from configured layouts and indexes | +| `studio` | Enable Studio APIs and Node asset hosting | `false` | +| `studioOrigin` | Additional exact origin accepted for Studio mutations | Local Node authority origin when Studio is enabled; otherwise none | +| `readBundles` | Advertise bounded decoded point-read bundles | `false` | + +## Common authority settings + +These settings are supported by both the Node and Cloudflare authorities +unless noted otherwise. + +| Setting | Purpose | Default or requirement | +| --- | --- | --- | +| `THIMBLE_MASTER_KEY` | Base64 deployment master key with at least 32 decoded bytes | Required outside the local Node provider; local Node creates a protected key file when absent | +| `THIMBLE_ALLOWED_ORIGIN` | Exact browser origin accepted for state-changing requests | Required except local Node, which defaults to `http://127.0.0.1:5173` | +| `THIMBLE_PREFIX` | Application prefix inside the data store | `demo` | +| `THIMBLE_KEY_VERSION` | Active scope-key version for writes | `1` | +| `THIMBLE_READ_KEY_VERSIONS` | Comma-separated historical key versions that remain readable | Empty | +| `THIMBLE_HEAD_TTL_MS` | Browser mutable-HEAD revalidation interval | `1000` | +| `THIMBLE_COLLECTION_LAYOUTS` | Comma-separated `collection=trie|snapshot` mappings | Empty; trie is the fallback | +| `THIMBLE_COLLECTION_INDEXES` | JSON object containing the complete active index definitions | `{}` | +| `THIMBLE_DELETE_RETENTION_DAYS` | Restore window for retained deletions | `30` | +| `THIMBLE_DELETE_GRACE_DAYS` | Additional delay before expired tombstones leave the live layout | `7` | +| `THIMBLE_MAINTENANCE_MODE` | Reject normal writes while maintenance is running | `false` | +| `THIMBLE_STUDIO` | Enable Studio | `false` | +| `THIMBLE_STUDIO_ORIGIN` | Additional exact Studio origin | None, except local Node defaults to its authority origin | +| `THIMBLE_COLLECTIONS` | Comma-separated Studio collection catalogue | Empty | +| `THIMBLE_READ_BUNDLES` | Enable the trusted-authority decoded bundle path | `false` | + +`THIMBLE_READ_BUNDLES=true` changes the read transport trust boundary. Review +[Security](SECURITY.md) before enabling it. + +## Identity settings + +Configure at least one production OIDC provider. + +Microsoft Entra: + +| Setting | Purpose | +| --- | --- | +| `ENTRA_TENANT_ID` | Exact Entra tenant | +| `ENTRA_AUDIENCE` | API audience | +| `ENTRA_REQUIRED_SCOPE` | Optional required delegated scope | +| `ENTRA_REQUIRED_ROLE` | Optional required application role | + +Generic OIDC: + +| Setting | Purpose | +| --- | --- | +| `OIDC_PROVIDER_ID` | Stable route-safe provider identifier | +| `OIDC_ISSUER` | Exact token issuer | +| `OIDC_AUDIENCE` | Required audience | +| `OIDC_JWKS_URI` | HTTPS JWKS endpoint | +| `OIDC_ALLOWED_TENANTS` | Optional comma-separated tenant allowlist | +| `OIDC_REQUIRED_SCOPE` | Optional required delegated scope | +| `OIDC_REQUIRED_ROLE` | Optional required application role | + +At least one required scope or role must be configured for each provider. +When both are set, both must be present. + +## Node-only runtime settings + +| Setting | Purpose | Default | +| --- | --- | --- | +| `NODE_ENV` | Process mode used to prohibit the local development identity in production | Unset | +| `THIMBLE_PROVIDER` | Object-store adapter: `local`, `azure`, `s3`, or `r2` | `azure` when `AZURE_STORAGE_CONNECTION_STRING` is set; otherwise `local` | +| `THIMBLE_HOST` | Authority listen address | `127.0.0.1` | +| `THIMBLE_PORT` | Authority listen port | `8787` | +| `THIMBLE_SECURE_COOKIES` | Force Secure session cookies | `true` for cloud providers; otherwise `false` unless explicitly enabled | +| `THIMBLE_SESSION_TTL_SECONDS` | Opaque session lifetime | `3600` | +| `THIMBLE_AUTH_RATE_LIMIT` | Authentication attempts per durable rate window | `5` | +| `THIMBLE_AUTH_RATE_WINDOW_MS` | Authentication rate window | `60000` | +| `THIMBLE_SCOPE_CACHE_MAX` | Maximum cached scope runtimes | `100` | +| `THIMBLE_SCOPE_CACHE_TTL_MS` | Scope-runtime cache lifetime | `900000` | +| `THIMBLE_TRUSTED_PROXY_IPS` | Comma-separated immediate proxy addresses trusted for `X-Forwarded-For` | Empty | +| `THIMBLE_DISABLE_IP_RATE_LIMIT` | Disable source-IP limiting when the proxy boundary cannot be verified | `false` | + +## Local Node settings + +| Setting | Purpose | Default | +| --- | --- | --- | +| `THIMBLE_LOCAL_DATA_ROOT` | Local application-object directory | `.thimble-data` | +| `THIMBLE_LOCAL_AUTH_ROOT` | Local authentication-object directory | `.thimble-auth` | +| `THIMBLE_LOCAL_SECRET_ROOT` | Local master-key directory | `.thimble-data` | +| `THIMBLE_DEV_IDENTITY` | Enable the loopback-only development identity | `false` | +| `THIMBLE_DEV_SUBJECT` | Development identity subject | `local-developer` | +| `THIMBLE_DEV_DISPLAY_NAME` | Development identity display name | `Local developer` | + +The development identity requires a local provider, a non-production process, +a loopback listen host, and loopback application and Studio origins. + +## Node provider settings + +Azure Blob Storage: + +| Setting | Purpose | Default | +| --- | --- | --- | +| `AZURE_STORAGE_CONNECTION_STRING` | Server-only Blob Storage connection string | Required | +| `AZURE_STORAGE_CONTAINER` | Data container | `thimbledb` | +| `AZURE_AUTH_STORAGE_CONTAINER` | Authentication container | `-auth` | + +Amazon S3: + +| Setting | Purpose | Default | +| --- | --- | --- | +| `S3_BUCKET` | Data bucket | Required | +| `S3_AUTH_BUCKET` | Authentication bucket | Required | +| `AWS_REGION` | AWS region used by the SDK | `us-east-1` | +| `S3_ENDPOINT` | Optional S3-compatible endpoint | AWS endpoint | +| `S3_FORCE_PATH_STYLE` | Use path-style bucket addressing | `false` | + +The AWS SDK uses its normal credential chain. + +R2 through the S3 adapter: + +| Setting | Purpose | +| --- | --- | +| `R2_ACCOUNT_ID` | Cloudflare account ID | +| `R2_BUCKET` | Data bucket | +| `R2_AUTH_BUCKET` | Authentication bucket | +| `R2_ACCESS_KEY_ID` | R2 API access key ID | +| `R2_SECRET_ACCESS_KEY` | R2 API secret | + +Prefer the native Worker bindings for the Cloudflare reference deployment. + +## Cloudflare bindings + +| Binding | Purpose | Required | +| --- | --- | --- | +| `DB` | Private R2 data bucket | Yes | +| `AUTH_DB` | Separate private R2 authentication bucket | Yes | +| `ASSETS` | Static application and Studio assets | Optional | +| `AUTH_RATE_LIMITER` | Native low-latency source-IP rate limit | Optional | + +Cloudflare uses a one-hour session lifetime and a bounded in-memory scope +runtime cache. Those values are not environment-configurable in the current +Worker authority. + +## Maintenance command settings + +These settings apply to command-line maintenance. They are not all authority +runtime settings. + +| Setting | Purpose | +| --- | --- | +| `THIMBLE_SCOPE_ID` | Target scope | +| `THIMBLE_COLLECTIONS` | Target collection list | +| `THIMBLE_COLLECTION` | Single collection for a layout migration | +| `THIMBLE_SOURCE_LAYOUT` | Existing `trie` or `snapshot` layout for a layout migration | +| `THIMBLE_TARGET_LAYOUT` | Replacement `trie` or `snapshot` layout for a layout migration | +| `THIMBLE_COLLECTION_LAYOUTS` | Active layout mapping | +| `THIMBLE_RETIRED_COLLECTION_LAYOUTS` | Retired layouts eligible for quiescent cleanup | +| `THIMBLE_COLLECTION_INDEXES` | Complete active index definitions | +| `THIMBLE_MIGRATION_QUIESCENT` | Confirms writes are blocked for migration commands | +| `THIMBLE_MAINTENANCE_QUIESCENT` | Confirms writes are blocked for destructive retention maintenance | + +Do not add `THIMBLE_RETIRED_COLLECTION_LAYOUTS` to a Worker and expect it to +perform cleanup. Run the documented maintenance command in a controlled +environment with the matching provider credentials and master key. + +## Supplied template coverage + +The Cloudflare Wrangler example exposes current collection, Studio, and +read-bundle settings. + +The checked-in Azure and AWS infrastructure templates intentionally expose a +smaller core setting set. Their deployment guides list the optional settings +that require a reviewed derived template. Do not assume a shell variable is +passed into Container Apps or Lambda unless the infrastructure template maps +it into the container environment. diff --git a/docs/DELETION-RETENTION.md b/docs/DELETION-RETENTION.md index 82cf9ac..d33d001 100644 --- a/docs/DELETION-RETENTION.md +++ b/docs/DELETION-RETENTION.md @@ -1,6 +1,9 @@ # Deletion and retention ThimbleDB uses retained logical deletion before physical collection. +See the +[retained deletion flow](DIAGRAMS.md#retained-deletion-restore-and-physical-collection) +for the document, restore, purge, and quiescent-maintenance boundaries. ## Document lifecycle diff --git a/docs/DEPLOYMENT-AWS.md b/docs/DEPLOYMENT-AWS.md index e82e2e8..2a9b37f 100644 --- a/docs/DEPLOYMENT-AWS.md +++ b/docs/DEPLOYMENT-AWS.md @@ -10,7 +10,8 @@ The CloudFormation template is `deploy/aws/template.yaml`. The Lambda authority can be deployed with the application or as a separate service behind the same public application gateway. See -[Authority deployment modes](AUTHORITY-DEPLOYMENT.md). +[In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md) for the +scaling and operations tradeoffs. ## Build the Lambda image @@ -59,12 +60,27 @@ Generate the recommended Entra delegated scope and application roles with machine access, use a service principal and required application role rather than a static global key. -To serve the package-owned Studio from the same authority, set -`THIMBLE_STUDIO=true` and configure the exact public origin in -`THIMBLE_STUDIO_ORIGIN`. +## Template capability -Set `THIMBLE_READ_BUNDLES=true` to advertise the bounded cold point-read -optimization. +The checked-in CloudFormation template exposes the core provider, OIDC, key +version, collection-layout, and retention settings. It does not currently +expose: + +- `THIMBLE_COLLECTION_INDEXES` +- `THIMBLE_COLLECTIONS` +- `THIMBLE_HEAD_TTL_MS` +- `THIMBLE_STUDIO` or `THIMBLE_STUDIO_ORIGIN` +- `THIMBLE_READ_BUNDLES` + +The supplied deployment therefore leaves Studio, covering indexes, and read +bundles disabled. Use a reviewed derived template or another Node deployment +configuration when those optional features are required. Setting variables +only in the deployment shell does not pass them into the Lambda function. + +The template still passes its legacy `RetiredCollectionLayouts` value into +the Lambda environment. The authority runtime does not perform retired-layout +cleanup. Pass `THIMBLE_RETIRED_COLLECTION_LAYOUTS` to the separate quiescent +retention command when cleanup is required. For key rotation, deploy `KeyVersion` as the current write version and `ReadKeyVersions` as the comma-separated historical versions that remain diff --git a/docs/DEPLOYMENT-AZURE.md b/docs/DEPLOYMENT-AZURE.md index d4bd6b0..d274492 100644 --- a/docs/DEPLOYMENT-AZURE.md +++ b/docs/DEPLOYMENT-AZURE.md @@ -11,7 +11,8 @@ The Bicep template is `deploy/azure/main.bicep`. The Container App authority can share an application deployment boundary or run independently behind Front Door or another same-origin gateway. See -[Authority deployment modes](AUTHORITY-DEPLOYMENT.md). +[In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md) for the +scaling and operations tradeoffs. ## Prerequisites @@ -83,12 +84,21 @@ Generate the recommended Entra delegated scope and application roles with fragment with the existing application registration rather than replacing unrelated entries. -To serve the package-owned Studio from the same Container App, set -`THIMBLE_STUDIO=true` and set `THIMBLE_STUDIO_ORIGIN` to the exact public -authority origin. +## Template capability -Set `THIMBLE_READ_BUNDLES=true` to advertise the bounded cold point-read -optimization. +The checked-in Bicep template exposes the core provider, OIDC, key-version, +collection-layout, and retention settings. It does not currently expose: + +- `THIMBLE_COLLECTION_INDEXES` +- `THIMBLE_COLLECTIONS` +- `THIMBLE_HEAD_TTL_MS` +- `THIMBLE_STUDIO` or `THIMBLE_STUDIO_ORIGIN` +- `THIMBLE_READ_BUNDLES` + +The supplied deployment therefore leaves Studio, covering indexes, and read +bundles disabled. Use a reviewed derived template or another Container Apps +configuration when those optional features are required. Setting variables +only in the deployment shell does not add them to the Container App. For key rotation, set `keyVersion` to the current write version and `readKeyVersions` to the comma-separated historical versions that remain diff --git a/docs/DEPLOYMENT-CLOUDFLARE.md b/docs/DEPLOYMENT-CLOUDFLARE.md index cebff33..9fedbfd 100644 --- a/docs/DEPLOYMENT-CLOUDFLARE.md +++ b/docs/DEPLOYMENT-CLOUDFLARE.md @@ -2,10 +2,10 @@ Cloudflare Workers and R2 are the reference ThimbleDB deployment. -The authority can share the application Worker deployment or run as a -separately routed Worker. See -[Authority deployment modes](AUTHORITY-DEPLOYMENT.md). In both modes, prefer -one public browser origin. +The authority can run in-app inside the application Worker or as a separately +routed Worker with its own release, limits, logs, and scaling boundary. See +[In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md). In both +modes, prefer one public browser origin. The design uses: @@ -49,7 +49,7 @@ applies to Standard storage. Never attach a custom domain to the auth bucket. ## 4. Configure the application domain -For the embedded reference deployment, use one Worker hostname: +For the in-app reference deployment, use one Worker hostname: ```text db.example.com diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index a6e4b64..217f884 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -28,8 +28,11 @@ The generated app includes: - browser memory and IndexedDB caches - typed notes collection - equality and range indexes +- an explicit covering projection for indexed title lookup +- opt-in bounded point-read bundles - fluent queries - deletion and restore +- packaged same-origin Studio - Vite frontend Run: @@ -96,3 +99,19 @@ Before deployment: 6. Run authentication, logout, scope, and deletion tests. The authority refuses to start with the development identity in production. + +## Documentation diagrams + +Mermaid remains the source format in repository Markdown. The static website +uses content-addressed SVGs so diagram pages require no client-side rendering +runtime. + +After changing a Mermaid block under `docs`, regenerate the website assets: + +```powershell +npm run site:diagrams +npm run site:build +``` + +The generator pins Mermaid CLI 12.0.0, omits embedded font data, removes stale +diagram assets, and writes only files referenced by the current Markdown. diff --git a/docs/DIAGRAMS.md b/docs/DIAGRAMS.md index bdadbcb..832d3fc 100644 --- a/docs/DIAGRAMS.md +++ b/docs/DIAGRAMS.md @@ -1,7 +1,7 @@ # System diagrams These Mermaid diagrams describe the ThimbleDB 3.1 data, identity, query, -index, migration, and deployment paths. +index, deletion, migration, and deployment paths. ## Trust boundaries @@ -24,7 +24,7 @@ flowchart LR Exchange["OIDC token exchange"] Session["Opaque revocable session"] Authorizer["Current scope-grant calculation"] - Broker["Authenticated encrypted-object broker"] + Broker["Authenticated ciphertext-object and decoded-bundle broker"] Mutation["Mutation validation and execution"] Exchange --> Session @@ -50,7 +50,7 @@ flowchart LR end UI -- "Session cookie + CSRF + requested scope" --> Authority - Client -- "Brokered ciphertext reads" --> Broker + Client -- "Ciphertext objects or opt-in decoded bundles" --> Broker Broker --> Heads Broker --> Pages Mutation -- "Create immutable objects" --> Pages @@ -68,7 +68,7 @@ flowchart LR The browser receives decrypt-only scope keys. It never receives a storage credential, an authority write key, or a database-wide administrator key. -## Embedded authority deployment +## In-app (embedded) authority deployment ```mermaid flowchart LR @@ -94,11 +94,11 @@ flowchart LR Secrets --> Authority ``` -Embedded mode keeps one deployment, release cadence, public origin, and +In-app mode keeps one deployment, release cadence, public origin, and scaling policy. Application server code and authority secrets share one runtime trust boundary. -## Separate authority deployment +## Separate Worker or service deployment ```mermaid flowchart LR @@ -111,7 +111,7 @@ flowchart LR App["Application assets or application service"] - subgraph AuthorityService["Separate authority deployment"] + subgraph AuthorityService["Separate Worker or service deployment"] Authority["ThimbleDB authority"] Broker["Object broker and read bundles"] Mutation["Validation and conditional writes"] @@ -174,8 +174,10 @@ flowchart TD Point{"ID equality?"} Bundle{"Cold cache and bundle endpoint available?"} BundleRead["One bounded authority read bundle"] + BundleValidate["Validate bundled HEAD and immutable values"] Index{"Matching declared index with indexable values?"} - Head["Read collection HEAD"] + PointHead["Read collection HEAD"] + IndexHead["Read collection HEAD"] IndexPage["Read immutable encrypted index page"] Candidates["Resolve bounded candidate IDs"] Covered{"Explicit selected fields and all query fields covered?"} @@ -188,10 +190,10 @@ flowchart TD Query --> Validate --> Point Point -- Yes --> Bundle - Bundle -- Yes --> BundleRead --> Result - Bundle -- No or fallback --> ReadDocs + Bundle -- Yes --> BundleRead --> BundleValidate --> Result + Bundle -- No or fallback --> PointHead --> ReadDocs Point -- No --> Index - Index -- Yes --> Head --> IndexPage --> Candidates --> Covered + Index -- Yes --> IndexHead --> IndexPage --> Candidates --> Covered Covered -- Yes --> Projection --> Predicate Covered -- No --> ReadDocs Index -- No --> Scan @@ -202,7 +204,7 @@ flowchart TD Arrays and objects are not secondary-index values. Queries using those values fall back to a bounded scan instead of returning an empty indexed result. -## Indexed query read sequence +## Point-read and indexed-query sequence ```mermaid sequenceDiagram @@ -220,7 +222,7 @@ sequenceDiagram Client->>Broker: GET bounded point-read bundle Broker->>Broker: Require current read grant and enforce object/byte limits Broker-->>Client: Decoded HEAD and immutable cache values over HTTPS - Client->>Cache: Store returned objects + Client->>Cache: Store returned values with device-key encryption else HEAD missing or stale Client->>Broker: GET encrypted HEAD with session Broker->>Store: Read object with ETag condition @@ -244,7 +246,7 @@ sequenceDiagram Client->>Client: Enforce bounded collection scan end - Client->>Client: Decrypt, validate candidates, order, and limit + Client->>Client: Decrypt when required, validate candidates, order, and limit Client-->>App: Documents plus point/index/scan plan ``` @@ -283,6 +285,32 @@ Document roots and secondary indexes become visible through the same conditional HEAD write. A process that sees existing index references but has no matching index configuration refuses to rewrite the collection. +## Retained deletion, restore, and physical collection + +```mermaid +flowchart LR + Live["Live document"] + Delete["Authorised DELETE"] + Tombstone["Encrypted tombstone
original document retained"] + Hidden["Reads and scans hide document"] + Restore{"Restore requested
inside restore window?"} + Restored["Authorised restore
new live generation"] + Grace["Restore window expires
then purge grace elapses"] + Purge["Authorised purge removes tombstone
from current live layout"] + Unreachable["Old immutable generations
become unreachable"] + Quiescent["All authorities in maintenance mode
quiescent flag confirmed"] + Collection["Maintenance deletes unreachable objects
and retired layout prefixes"] + + Live --> Delete --> Tombstone --> Hidden --> Restore + Restore -- Yes --> Restored --> Live + Restore -- No --> Grace --> Purge --> Unreachable + Unreachable --> Quiescent --> Collection +``` + +Logical deletion is immediate, restoration is bounded by the configured +retention window, and physical object deletion is a separate operator-approved +maintenance step. Object-store age rules cannot determine reachability. + ## Key hierarchy, rotation, and browser lifetime ```mermaid diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 715c87b..de6af39 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -66,7 +66,7 @@ On first local startup: 2. It derives one scope encryption key and one node-address HMAC key. 3. A validated external identity is mapped to a stable internal user UUID. 4. The browser receives an HttpOnly opaque session cookie. -5. The authorised key endpoint returns the scope key once. +5. The authorised key endpoint returns the current readable scope-key grant. 6. The browser imports it as a non-extractable memory-only CryptoKey. 7. Persistent cache values are encrypted with a separate non-extractable browser device key. diff --git a/docs/FAQ.md b/docs/FAQ.md index 7ca3d71..7b765ae 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -36,18 +36,20 @@ includes a Node authority with local filesystem, Azure Blob Storage, Amazon S3, and S3-compatible adapters. The stored protocol remains the same across providers. -## Does the authority run inside the application? +## Can the authority run in-app or as a separate Worker? -It can. The authority can share the application deployment or run as a -separate Worker, container, function, or Node service. A separate process -should normally remain behind the same public browser origin through -path-based routing so Strict cookies, CSRF, Studio, browser caches, and logout -coordination retain the documented behaviour. +Both are supported. The authority can run in-app inside the application +deployment or as a separate Worker, container, function, or Node service. A +separate process should normally remain behind the same public browser origin +through path-based routing so Strict cookies, CSRF, Studio, browser caches, +and logout coordination retain the documented behaviour. -Use an embedded authority for the smallest operational surface. Use a separate +Use an in-app authority for the smallest operational surface. Use a separate authority when storage-secret isolation, independent release control, failure -isolation, or independent scaling justifies another service. See -[Authority deployment modes](AUTHORITY-DEPLOYMENT.md). +isolation, regional placement, or independent scaling justifies another +service. Separation creates a scaling boundary; it does not remove collection +write contention. See +[In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md). ## Does ThimbleDB store passwords? @@ -159,6 +161,20 @@ and exposes guarded maintenance operations. Studio starts read-only and never receives object-storage credentials. See [ThimbleDB Studio](STUDIO.md). +## How do I report a bug or documentation problem? + +Use the [GitHub issue tracker](https://github.com/Jason-Doyle/thimble/issues) +for reproducible defects, documentation corrections, and feature proposals. +Do not include credentials, tokens, private application data, or other +sensitive information in a public issue. + +## Does thimbledb.com use analytics cookies? + +The static documentation site uses Cloudflare Web Analytics. Cloudflare +describes that service as privacy-first analytics that does not use cookies or +collect visitors' personal data. The site has no user accounts, comments, or +contact forms. See [Website privacy](WEBSITE-PRIVACY.md). + ## What licence does ThimbleDB use? ThimbleDB is available under the Apache License 2.0. diff --git a/docs/IMPLEMENTATION-PROMPTS.md b/docs/IMPLEMENTATION-PROMPTS.md index 8720e27..0ffba21 100644 --- a/docs/IMPLEMENTATION-PROMPTS.md +++ b/docs/IMPLEMENTATION-PROMPTS.md @@ -14,7 +14,7 @@ Application context: - Package manager: [package manager] - OIDC provider: [Entra/Auth0/other] - Authority platform: [Cloudflare/Node] -- Authority topology: [embedded deployment/separate service behind same-origin route] +- Authority topology: [in-app deployment/separate Worker or service behind same-origin route] - Collections: [collection list] - Scope model: [per-user/per-tenant/both] @@ -24,8 +24,8 @@ Requirements: 3. Use external OIDC authentication. Do not add local passwords, password hashes, recovery tokens, passkey storage, or MFA secrets to ThimbleDB. 4. Exchange the provider access token at `/api/auth/oidc//session`. 5. Build the browser client from `/api/config` and the scope key grant. -6. Preserve one public browser origin whether the authority is embedded or - separately deployed. +6. Preserve one public browser origin whether the authority runs in-app or as + a separately deployed Worker or service. 7. Keep provider credentials and object-storage credentials server-side. 8. Keep scope keys memory-only as non-extractable CryptoKeys. 9. Pass `collectionLayouts`, `layoutGeneration`, and `configurationUrl` to `ThimbleClient`. diff --git a/docs/NPM-PUBLISHING.md b/docs/NPM-PUBLISHING.md index b28eb1c..be6c132 100644 --- a/docs/NPM-PUBLISHING.md +++ b/docs/NPM-PUBLISHING.md @@ -59,8 +59,10 @@ unused write-capable npm automation tokens. 1. Update `package.json`, `package-lock.json`, and `CHANGELOG.md` through a pull request. -2. Merge only after the protected `verify` check and required review pass. -3. Create a matching tag such as `v2.0.1`. +2. Merge only after the protected `verify` check passes and repository branch + protection requirements are satisfied. A repository administrator may use + the documented bypass when the sole maintainer cannot self-approve. +3. Create a matching tag such as `vX.Y.Z`. 4. Publish a GitHub release for that tag. 5. The release event runs `publish.yml`. 6. The workflow verifies that the tag matches the package version, runs @@ -79,8 +81,8 @@ published manually before the trusted publisher exists. - Package dependencies and build tools run in a job without OIDC permission. - The workflow has read-only repository content access and `id-token: write`. - Publishing is tied to this repository and the exact `publish.yml` workflow. -- Protected `main` rules require CI and code-owner approval before version - changes can merge. +- Protected `main` rules require CI and review before version changes merge, + with an administrator bypass reserved for the sole-maintainer case. The repository is public, so npm automatically generates a signed provenance attestation for trusted publishes. If the repository becomes private, diff --git a/docs/PUBLIC-API.md b/docs/PUBLIC-API.md index 6cac8c8..9b454b2 100644 --- a/docs/PUBLIC-API.md +++ b/docs/PUBLIC-API.md @@ -110,8 +110,8 @@ authentication, object broker, key grant, write, deletion, linking, administration, retention, and layout-migration endpoints used by the reference deployment. -See [Authority deployment modes](AUTHORITY-DEPLOYMENT.md) before choosing one -application deployment or a separately operated authority service. +See [In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md) before +choosing an in-app authority or a separately operated Worker or service. ### `thimbledb/authority/cloudflare` @@ -212,8 +212,8 @@ roles. See [Machine and service access](SERVICE-ACCESS.md). ## Compatibility commitments - Semantic versioning applies from package version `1.0.0`. -- TDB1 envelopes and committed protocol fixtures remain readable throughout - the 1.x line. +- TDB1 envelopes and committed protocol fixtures remain readable in every + release that claims TDB1 compatibility. - New optional object layouts may be added in a minor release. - Removing or changing an exported symbol, authority endpoint, stored field, or required configuration value is a major-version change. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index cf2a8fd..5865ba3 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -12,12 +12,14 @@ The browser API is the same for every authority. Choose one deployment shape: -- embed the authority in the application deployment -- run the authority in a separate Worker, container, function, or Node service +- run the authority in-app inside the application deployment for the smallest + operational surface +- run the authority as a separate Worker, container, function, or Node service + for independent secret, release, failure, and scaling boundaries In either case, preserve one public browser origin through the application server or a path-based gateway. See -[Authority deployment modes](AUTHORITY-DEPLOYMENT.md). +[In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md). ## Fast local evaluation @@ -65,8 +67,8 @@ For a Node authority backed by Amazon S3 or R2 through the S3 API: npm install @aws-sdk/client-s3 ``` -Cloudflare Worker, browser-only, and local Node deployments do not need either -provider SDK. +Browser clients, Cloudflare Worker deployments, and local-file Node +authorities do not need either cloud provider SDK. ## Cloudflare Worker and R2 @@ -109,7 +111,6 @@ Use a Wrangler configuration with caller-owned resources: "THIMBLE_READ_BUNDLES": "true", "THIMBLE_COLLECTION_LAYOUTS": "", "THIMBLE_COLLECTION_INDEXES": "{}", - "THIMBLE_RETIRED_COLLECTION_LAYOUTS": "", "THIMBLE_DELETE_RETENTION_DAYS": "30", "THIMBLE_DELETE_GRACE_DAYS": "7", "THIMBLE_MAINTENANCE_MODE": "false", @@ -193,6 +194,13 @@ records under `.thimble-auth`. It is intended for one Node process. Before selecting `azure`, install `@azure/storage-blob`. Before selecting `s3` or `r2`, install `@aws-sdk/client-s3`. For provider environment variables, trusted proxy, and secure-cookie configuration, use the deployment guides. +For the complete option and environment-variable matrix, see +[Configuration reference](CONFIGURATION.md). + +Both authority examples explicitly enable read bundles. This lets the trusted +authority return bounded decoded cache values over HTTPS. Disable the option +if every application-data response above TLS must remain a TDB1 envelope. See +[Security](SECURITY.md). ## Browser client diff --git a/docs/README.md b/docs/README.md index 24faafe..281a2ee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,16 +4,19 @@ | --- | --- | | [Quickstart](QUICKSTART.md) | Package, authority, browser client, and verification setup | | [Local development](DEVELOPMENT.md) | Scaffolding, development identity, and production transition | +| [Configuration reference](CONFIGURATION.md) | Authority options, environment variables, provider settings, defaults, and template coverage | | [Queries and indexes](QUERIES-INDEXES.md) | Typed collections, bounded predicates, query plans, and declared indexes | | [Logical migration](MIGRATION.md) | Portable archives and JSON, CSV, lowdb, SQLite, PostgreSQL, and Firestore adapters | | [Vibe-coded app fit guide](VIBE-CODED-APPS.md) | Exact workload decision criteria for AI-assisted small apps | | [Implementation prompts](IMPLEMENTATION-PROMPTS.md) | Copy-paste integration, deployment, migration, and review prompts | +| [Starter examples](EXAMPLES.md) | Checked-in examples, generated application, and maintained starter repositories | +| [Frequently asked questions](FAQ.md) | Direct answers about fit, providers, identity, performance, and limits | | [npm publishing](NPM-PUBLISHING.md) | OIDC trusted publisher setup and release process | | [Use cases](USE-CASES.md) | Fit criteria and application-specific guides | | [Database comparisons](COMPARISONS.md) | Workload comparisons with D1, SQLite, Firestore, lowdb, and direct object storage | | [Architecture](ARCHITECTURE.md) | Components, data flow, scopes, and provider model | -| [Authority deployment modes](AUTHORITY-DEPLOYMENT.md) | Embedded and separate authority topologies, decision criteria, and origin boundaries | -| [System diagrams](DIAGRAMS.md) | Trust boundaries, sequences, keys, scopes, and providers | +| [In-app and separate authority deployment](AUTHORITY-DEPLOYMENT.md) | Topologies, scaling opportunities, trust boundaries, and origin requirements | +| [System diagrams](DIAGRAMS.md) | Trust, deployment, read, write, deletion, key, migration, Studio, and provider flows | | [Storage providers](STORAGE-PROVIDERS.md) | Provider abstraction, support levels, and conformance | | [Security](SECURITY.md) | Threat model, keys, encryption, revocation, and XSS boundary | | [Authentication](AUTHENTICATION.md) | External identity mappings, sessions, and scope grants | @@ -22,7 +25,7 @@ | [Deletion and retention](DELETION-RETENTION.md) | Tombstones, restore windows, scope erasure, and quiescent collection | | [Adaptive layouts](ADAPTIVE-LAYOUTS.md) | Snapshot/trie evidence, recommendations, and migration | | [Protocol](PROTOCOL.md) | Binary envelope and object layout | -| [Versioning](VERSIONING.md) | Package, protocol, key, and v1 compatibility rules | +| [Versioning](VERSIONING.md) | Package, protocol, key, and release compatibility rules | | [Public API](PUBLIC-API.md) | Stable package exports and authority integration | | [Evaluation harness](EVALUATION.md) | Browser harness, sample store, and benchmark usage | | [Benchmarks](BENCHMARKS.md) | R2 browser methodology, results, and limitations | @@ -31,3 +34,4 @@ | [Azure deployment](DEPLOYMENT-AZURE.md) | Container Apps and Blob Storage | | [AWS deployment](DEPLOYMENT-AWS.md) | Lambda container and private S3 buckets | | [Operations](OPERATIONS.md) | Keys, backup, metrics, incidents, and cleanup | +| [Website privacy](WEBSITE-PRIVACY.md) | Static-site data handling and Cloudflare Web Analytics disclosure | diff --git a/docs/VERSIONING.md b/docs/VERSIONING.md index 85595c8..49b816c 100644 --- a/docs/VERSIONING.md +++ b/docs/VERSIONING.md @@ -14,6 +14,22 @@ Package version `1.0.0` freezes the documented exports in [Public package API](PUBLIC-API.md). Semantic versioning applies to the root, auth, authority, and provider subpaths. +## Version 1.0 + +Version 1.0 provides: + +- Apache-2.0 licensing +- stable documented package and authority exports +- Node and Cloudflare authority implementations +- Entra and generic OIDC identity mapping and administration +- retained deletion and quiescent physical collection +- explicit snapshot/trie recommendations and migration +- TDB1 read compatibility fixtures + +Performance is not part of the compatibility guarantee. Published R2 evidence +shows fast warm cache reads, while cold reads and external session creation can +take seconds from distant regions. + ## Version 2.0 Version 2 separates cloud storage SDKs from the base package: @@ -109,19 +125,3 @@ compatible extension. The `npm run migrate:keys` command rewrites selected live collections under the current write key. Old objects are retained until an operator-approved offline cleanup. - -## Version 1.0 - -Version 1.0 provides: - -- Apache-2.0 licensing -- stable documented package and authority exports -- Node and Cloudflare authority implementations -- Entra and generic OIDC identity mapping and administration -- retained deletion and quiescent physical collection -- explicit snapshot/trie recommendations and migration -- TDB1 read compatibility fixtures - -Performance is not part of the compatibility guarantee. Published R2 evidence -shows fast warm cache reads, while cold reads and external session creation can -take seconds from distant regions. diff --git a/docs/WEBSITE-PRIVACY.md b/docs/WEBSITE-PRIVACY.md new file mode 100644 index 0000000..c6c9581 --- /dev/null +++ b/docs/WEBSITE-PRIVACY.md @@ -0,0 +1,39 @@ +# Website privacy + +This notice covers the public documentation website at `thimbledb.com`. It +does not govern applications that use the ThimbleDB package. Each application +operator remains responsible for its own privacy notice, identity provider, +data model, retention policy, and legal obligations. + +## Information collected by the site + +The website is a static documentation site. It does not provide user accounts, +comments, contact forms, or application data storage. It does not receive +documents, scope keys, provider credentials, session cookies, or exports from +applications that use ThimbleDB. + +Search and copy controls run in the browser. Mermaid source is pre-rendered +into site-hosted SVG assets during development and is not sent to an external +rendering service when a visitor opens the site. + +Cloudflare hosts the site and Cloudflare Web Analytics is enabled. Cloudflare +describes Web Analytics as privacy-first analytics that does not use cookies +or collect visitors' personal data: + +- [Cloudflare Web Analytics](https://developers.cloudflare.com/web-analytics/about/) + +Cloudflare may process ordinary network and request information as the hosting +and security provider under its +[privacy policy](https://www.cloudflare.com/privacypolicy/). + +## External services + +Links to GitHub, npm, Microsoft, Cloudflare, AWS, Firebase, SQLite, OpenID, and +other external sites are governed by those services' privacy policies. + +## Support and corrections + +Report documentation defects or privacy concerns through the +[public issue tracker](https://github.com/Jason-Doyle/thimble/issues). Do not +include credentials, access tokens, private application data, or other +sensitive information in a public issue. diff --git a/docs/use-cases/AI-CONTEXT.md b/docs/use-cases/AI-CONTEXT.md index ac1d60a..bb32a76 100644 --- a/docs/use-cases/AI-CONTEXT.md +++ b/docs/use-cases/AI-CONTEXT.md @@ -70,7 +70,7 @@ controls. A model should not silently create permanent records. ```text Add user-controlled structured context to this AI-assisted application with -ThimbleDB 1.x. +ThimbleDB 3.x. Use the signed-in user's scope. Store preferences and tool settings as snapshots; sources, checkpoints, and approved facts as tries. Require explicit diff --git a/docs/use-cases/CATALOGUE.md b/docs/use-cases/CATALOGUE.md index 9ecfaad..0444b09 100644 --- a/docs/use-cases/CATALOGUE.md +++ b/docs/use-cases/CATALOGUE.md @@ -62,7 +62,7 @@ query. ## Scaffold prompt ```text -Build a small catalogue or reference library with ThimbleDB 1.x. +Build a small catalogue or reference library with ThimbleDB 3.x. Model JSON documents with stable IDs, category, summary, tags, active status, and update timestamp. Start with snapshot layout and use a 10-second HEAD TTL. diff --git a/docs/use-cases/FIELD-GUIDE.md b/docs/use-cases/FIELD-GUIDE.md index fee0cc1..18b3e03 100644 --- a/docs/use-cases/FIELD-GUIDE.md +++ b/docs/use-cases/FIELD-GUIDE.md @@ -58,7 +58,7 @@ the application also defines conflict and retry semantics. ```text Build an offline-readable field guide and inspection workflow with ThimbleDB -1.x. +3.x. Use snapshot layout for sites and inspection templates, trie for assets and inspection records, and content caching in the browser. Show last-checked time diff --git a/docs/use-cases/PER-USER-WORKSPACE.md b/docs/use-cases/PER-USER-WORKSPACE.md index 2bc51b8..e3e9518 100644 --- a/docs/use-cases/PER-USER-WORKSPACE.md +++ b/docs/use-cases/PER-USER-WORKSPACE.md @@ -73,7 +73,7 @@ tombstone every user collection through the administrator scope-erasure API. ## Scaffold prompt ```text -Add a per-user workspace to this application using ThimbleDB 1.x. +Add a per-user workspace to this application using ThimbleDB 3.x. Store profile, preferences, drafts, and saved items in the signed-in user's `user:` scope. Use snapshot layout for profile and preferences. Start diff --git a/docs/use-cases/PROGRESS-JOURNAL.md b/docs/use-cases/PROGRESS-JOURNAL.md index a650e14..4c73f08 100644 --- a/docs/use-cases/PROGRESS-JOURNAL.md +++ b/docs/use-cases/PROGRESS-JOURNAL.md @@ -48,7 +48,7 @@ await db.write("sessions", "session-2026-09-24", { ## Scaffold prompt ```text -Build a private progress journal with ThimbleDB 1.x. +Build a private progress journal with ThimbleDB 3.x. Store profile and templates as snapshots. Store sessions and milestones as tries in the signed-in user's `user:` scope. Use stable IDs, structured diff --git a/docs/use-cases/TENANT-OPERATIONS.md b/docs/use-cases/TENANT-OPERATIONS.md index af201e7..fde17f9 100644 --- a/docs/use-cases/TENANT-OPERATIONS.md +++ b/docs/use-cases/TENANT-OPERATIONS.md @@ -69,7 +69,7 @@ Do not reuse a user-scope client for tenant data. ## Scaffold prompt ```text -Add a small multi-tenant operations portal using ThimbleDB 1.x. +Add a small multi-tenant operations portal using ThimbleDB 3.x. Map OIDC tenant and role claims to `tenant:` scopes. Store procedures and checklist templates as snapshots, equipment and inspections as tries, and diff --git a/package.json b/package.json index 730d998..3f6dbc0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "thimbledb", "version": "3.1.0", - "description": "Browser-oriented encrypted object-storage database with Node and Cloudflare authorities.", + "description": "Encrypted browser-first JSON database for small web apps, backed by object storage with in-app or separate Cloudflare and Node authorities.", "private": false, "license": "Apache-2.0", "author": "Jason Doyle", @@ -9,21 +9,28 @@ "type": "git", "url": "git+https://github.com/Jason-Doyle/thimble.git" }, - "homepage": "https://github.com/Jason-Doyle/thimble#readme", + "homepage": "https://thimbledb.com", "bugs": { "url": "https://github.com/Jason-Doyle/thimble/issues" }, "keywords": [ "cloudflare", + "cloudflare-r2", + "cloudflare-workers", "r2", "database", "database-admin", "browser-database", + "json-database", "indexeddb", "object-storage", "encryption", + "oidc", + "aws-s3", + "azure-blob-storage", "serverless", "typescript", + "webcrypto", "vibe-coding" ], "publishConfig": { @@ -45,24 +52,7 @@ "LICENSE", "NOTICE", "public/thimbledb-logo.png", - "docs/QUICKSTART.md", - "docs/DEVELOPMENT.md", - "docs/EXAMPLES.md", - "docs/IMPLEMENTATION-PROMPTS.md", - "docs/MIGRATION.md", - "docs/NPM-PUBLISHING.md", - "docs/QUERIES-INDEXES.md", - "docs/SERVICE-ACCESS.md", - "docs/STUDIO.md", - "docs/USE-CASES.md", - "docs/use-cases", - "docs/ADAPTIVE-LAYOUTS.md", - "docs/AUTHORITY-DEPLOYMENT.md", - "docs/AUTHENTICATION.md", - "docs/DELETION-RETENTION.md", - "docs/PUBLIC-API.md", - "docs/PROTOCOL.md", - "docs/VERSIONING.md" + "docs" ], "exports": { ".": { @@ -132,8 +122,10 @@ "test:e2e": "playwright test", "verify:package": "node scripts/verify-package.mjs", "verify:scaffold": "node scripts/verify-scaffold.mjs", + "verify:docs": "node scripts/verify-docs.mjs", "site:dev": "npm --prefix site run dev", "site:check": "npm --prefix site run check", + "site:diagrams": "npm --prefix site run generate:diagrams", "site:build": "npm --prefix site run build", "site:preview": "npm --prefix site run preview", "site:test": "npm --prefix site run test:worker && playwright test --config site/playwright.config.ts", diff --git a/scripts/verify-docs.mjs b/scripts/verify-docs.mjs new file mode 100644 index 0000000..614b6c9 --- /dev/null +++ b/scripts/verify-docs.mjs @@ -0,0 +1,384 @@ +import { + existsSync, + readFileSync, + readdirSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import path from "node:path"; + +const root = process.cwd(); +const docsRoot = path.join(root, "docs"); +const markdownFiles = [ + path.join(root, "README.md"), + path.join(root, "CHANGELOG.md"), + ...walk(docsRoot).filter((file) => file.endsWith(".md")), +]; +const errors = []; +const anchorCache = new Map(); + +for (const file of markdownFiles) { + const content = readFileSync(file, "utf8"); + const relative = path.relative(root, file).replaceAll("\\", "/"); + const fences = content + .split(/\r?\n/) + .filter((line) => line.startsWith("```")).length; + if (fences % 2 !== 0) { + errors.push(`${relative}: unbalanced fenced code blocks`); + } + if ( + relative.startsWith("docs/use-cases/") && + content.includes("ThimbleDB 1.x") + ) { + errors.push(`${relative}: stale ThimbleDB 1.x prompt`); + } + if (relative !== "README.md" && relative !== "docs/README.md") { + const headings = withoutFencedCode(content).match(/^# .+$/gm) ?? []; + if (headings.length !== 1) { + errors.push( + `${relative}: expected one level-one heading, found ${headings.length}`, + ); + } + } + for (const match of content.matchAll( + /\[[^\]]*]\(([^)]+)\)/g, + )) { + const raw = match[1].trim(); + if (/^[a-z][a-z\d+.-]*:/i.test(raw)) { + continue; + } + const hashIndex = raw.indexOf("#"); + const target = + hashIndex === -1 ? raw : raw.slice(0, hashIndex); + const fragment = + hashIndex === -1 ? "" : raw.slice(hashIndex + 1); + const decoded = decodeURIComponent( + target.replace(/^<|>$/g, ""), + ); + const resolved = decoded + ? path.resolve(path.dirname(file), decoded) + : file; + if (!existsSync(resolved)) { + errors.push(`${relative}: missing link target ${raw}`); + continue; + } + if ( + fragment && + resolved.toLowerCase().endsWith(".md") && + !markdownAnchors(resolved).has( + decodeURIComponent(fragment).toLowerCase(), + ) + ) { + errors.push(`${relative}: missing link anchor ${raw}`); + } + } +} + +const docsIndex = readFileSync( + path.join(docsRoot, "README.md"), + "utf8", +); +for (const file of readdirSync(docsRoot, { + withFileTypes: true, +})) { + if ( + file.isFile() && + file.name.endsWith(".md") && + !["README.md"].includes(file.name) && + !docsIndex.includes(`](${file.name})`) + ) { + errors.push(`docs/README.md: missing ${file.name}`); + } +} + +const metadataSource = readFileSync( + path.join(root, "site", "src", "data", "docs.ts"), + "utf8", +); +const metadataIds = [ + ...metadataSource.matchAll(/\bid:\s*"([^"]+)"/g), +].map((match) => match[1]); +const metadataTitles = new Map( + [ + ...metadataSource.matchAll( + /\{\s*id:\s*"([^"]+)",\s*title:\s*"([^"]+)"/g, + ), + ].map((match) => [match[1], match[2]]), +); +const duplicateIds = metadataIds.filter( + (id, index) => metadataIds.indexOf(id) !== index, +); +duplicateIds.forEach((id) => + errors.push(`site docs metadata: duplicate id ${id}`), +); + +const expectedIds = walk(docsRoot) + .filter((file) => file.endsWith(".md")) + .map((file) => + path.relative(docsRoot, file).replaceAll("\\", "/"), + ) + .filter( + (file) => + file !== "README.md" && file !== "NPM-PUBLISHING.md", + ) + .map(documentId); +expectedIds.push("changelog"); +for (const id of expectedIds) { + if (!metadataIds.includes(id)) { + errors.push(`site docs metadata: missing id ${id}`); + } +} +for (const id of metadataIds) { + if (!expectedIds.includes(id)) { + errors.push(`site docs metadata: no source for ${id}`); + } +} +for (const file of walk(docsRoot).filter( + (candidate) => + candidate.endsWith(".md") && + !candidate.endsWith(`${path.sep}README.md`) && + !candidate.endsWith(`${path.sep}NPM-PUBLISHING.md`), +)) { + const id = documentId( + path.relative(docsRoot, file).replaceAll("\\", "/"), + ); + const heading = /^# (.+)$/m.exec( + withoutFencedCode(readFileSync(file, "utf8")), + )?.[1]; + if (metadataTitles.get(id) !== heading) { + errors.push( + `site docs metadata: title for ${id} does not match "${heading ?? "missing"}"`, + ); + } +} +if (metadataTitles.get("changelog") !== "Changelog") { + errors.push( + 'site docs metadata: title for changelog does not match "Changelog"', + ); +} + +const packageJson = JSON.parse( + readFileSync(path.join(root, "package.json"), "utf8"), +); +if (!packageJson.files?.includes("docs")) { + errors.push("package.json: complete docs directory is not packaged"); +} +if (packageJson.homepage !== "https://thimbledb.com") { + errors.push("package.json: homepage is not https://thimbledb.com"); +} +if ( + !packageJson.description?.includes( + "Encrypted browser-first JSON database", + ) || + !packageJson.description.includes("in-app or separate") +) { + errors.push( + "package.json: description does not match the current product and deployment model", + ); +} +for (const keyword of [ + "browser-database", + "json-database", + "object-storage", + "cloudflare-workers", + "aws-s3", + "azure-blob-storage", + "oidc", +]) { + if (!packageJson.keywords?.includes(keyword)) { + errors.push(`package.json: missing keyword ${keyword}`); + } +} + +const changelog = readFileSync( + path.join(root, "CHANGELOG.md"), + "utf8", +); +const firstRelease = + /^## (\d+\.\d+\.\d+)(?:\s+-\s+.*)?$/m.exec( + changelog, + )?.[1]; +if (firstRelease !== packageJson.version) { + errors.push( + `CHANGELOG.md: first release ${firstRelease ?? "missing"} does not match ${packageJson.version}`, + ); +} + +const configuration = readFileSync( + path.join(docsRoot, "CONFIGURATION.md"), + "utf8", +); +const readme = readFileSync(path.join(root, "README.md"), "utf8"); +const readmeDiagramCount = [ + ...readme.matchAll(/```mermaid\s*\r?\n[\s\S]*?\r?\n```/g), +].length; +if (readmeDiagramCount < 4) { + errors.push( + `README.md: expected at least 4 detailed Mermaid diagrams, found ${readmeDiagramCount}`, + ); +} +for (const heading of [ + "### Identity, session, and client bootstrap", + "### Read flow", + "### Mutation and cache-synchronisation flow", + "### Deployment, scaling, and storage flow", +]) { + if (!readme.includes(heading)) { + errors.push(`README.md: missing ${heading}`); + } +} +const sourceFiles = walk(path.join(root, "src")).filter((file) => + file.endsWith(".ts"), +); +const configuredNames = new Set(); +for (const file of sourceFiles) { + const source = readFileSync(file, "utf8"); + for (const pattern of [ + /process\.env\.([A-Z][A-Z0-9_]+)/g, + /required(?:Environment)?\("([A-Z][A-Z0-9_]+)"\)/g, + ]) { + for (const match of source.matchAll(pattern)) { + configuredNames.add(match[1]); + } + } +} +const workerSource = readFileSync( + path.join(root, "src", "cloudflare-worker.ts"), + "utf8", +); +const workerEnvironment = + /export type CloudflareAuthorityEnv = \{([\s\S]*?)^};/m.exec( + workerSource, + )?.[1] ?? ""; +for (const match of workerEnvironment.matchAll( + /^\s{2}([A-Z][A-Z0-9_]+)\??:/gm, +)) { + configuredNames.add(match[1]); +} +configuredNames.delete("AWS_LAMBDA_FUNCTION_NAME"); +for (const name of configuredNames) { + if (!configuration.includes(`\`${name}\``)) { + errors.push(`docs/CONFIGURATION.md: missing ${name}`); + } +} + +const diagramDirectory = path.join( + root, + "site", + "public", + "diagrams", +); +const expectedDiagramAssets = new Set( + walk(docsRoot) + .filter((file) => file.endsWith(".md")) + .flatMap((file) => [ + ...readFileSync(file, "utf8").matchAll( + /```mermaid\s*\r?\n([\s\S]*?)\r?\n```/g, + ), + ]) + .map((match) => + `${createHash("sha256") + .update(match[1].replace(/\r\n/g, "\n").trim()) + .digest("hex") + .slice(0, 20)}.svg`, + ), +); +const actualDiagramAssets = new Set( + readdirSync(diagramDirectory).filter((file) => + file.endsWith(".svg"), + ), +); +for (const asset of expectedDiagramAssets) { + if (!actualDiagramAssets.has(asset)) { + errors.push( + `site/public/diagrams: missing ${asset}; run npm run site:diagrams`, + ); + } +} +for (const asset of actualDiagramAssets) { + if (!expectedDiagramAssets.has(asset)) { + errors.push( + `site/public/diagrams: stale ${asset}; run npm run site:diagrams`, + ); + } +} + +if (errors.length > 0) { + console.error(errors.join("\n")); + process.exit(1); +} + +console.log( + `Verified ${markdownFiles.length} Markdown files and ${metadataIds.length} website document routes.`, +); + +function walk(directory) { + return readdirSync(directory, { + withFileTypes: true, + }).flatMap((entry) => { + const entryPath = path.join(directory, entry.name); + return entry.isDirectory() ? walk(entryPath) : [entryPath]; + }); +} + +function documentId(file) { + const withoutExtension = file.replace(/\.md$/i, ""); + if (withoutExtension.startsWith("use-cases/")) { + return `use-cases/${path.posix + .basename(withoutExtension) + .toLowerCase()}`; + } + + if (withoutExtension.startsWith("compare/")) { + return `compare/${path.posix + .basename(withoutExtension) + .toLowerCase()}`; + } + const special = { + "USE-CASES": "use-cases", + COMPARISONS: "comparisons", + "VIBE-CODED-APPS": "vibe-coded-apps", + FAQ: "faq", + BENCHMARKS: "benchmarks", + SECURITY: "security", + }; + return special[withoutExtension] ?? withoutExtension.toLowerCase(); +} + +function withoutFencedCode(content) { + let fenced = false; + return content + .split(/\r?\n/) + .filter((line) => { + if (line.startsWith("```")) { + fenced = !fenced; + return false; + } + return !fenced; + }) + .join("\n"); +} + +function markdownAnchors(file) { + const cached = anchorCache.get(file); + if (cached) { + return cached; + } + const anchors = new Set(); + const duplicateCounts = new Map(); + const content = withoutFencedCode(readFileSync(file, "utf8")); + for (const match of content.matchAll(/^#{1,6}\s+(.+?)\s*#*\s*$/gm)) { + const base = match[1] + .replace(/\[([^\]]+)]\([^)]+\)/g, "$1") + .replace(/<[^>]+>/g, "") + .replace(/[`*_~]/g, "") + .toLowerCase() + .trim() + .replace(/[^\p{L}\p{N}\s_-]/gu, "") + .replace(/\s+/g, "-"); + const count = duplicateCounts.get(base) ?? 0; + duplicateCounts.set(base, count + 1); + anchors.add(count === 0 ? base : `${base}-${count}`); + } + anchorCache.set(file, anchors); + return anchors; +} diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 4b1e096..e56cf80 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -146,6 +146,24 @@ try { "index.html", ), ); + for (const documentationPath of [ + ["docs", "ARCHITECTURE.md"], + ["docs", "AUTHORITY-DEPLOYMENT.md"], + ["docs", "CONFIGURATION.md"], + ["docs", "SECURITY.md"], + ["docs", "WEBSITE-PRIVACY.md"], + ["docs", "compare", "CLOUDFLARE-D1.md"], + ["docs", "use-cases", "CATALOGUE.md"], + ]) { + await assertPresent( + path.join( + consumer, + "node_modules", + "thimbledb", + ...documentationPath, + ), + ); + } runCliVersion(consumer, rootPackage.version); runNode( diff --git a/site/astro.config.mjs b/site/astro.config.mjs index f1a0483..7152904 100644 --- a/site/astro.config.mjs +++ b/site/astro.config.mjs @@ -2,9 +2,13 @@ import { fileURLToPath } from "node:url"; import sitemap from "@astrojs/sitemap"; import { unified } from "@astrojs/markdown-remark"; import { defineConfig } from "astro/config"; +import remarkMermaidDiagrams from "./src/lib/remark-mermaid-diagrams.js"; import remarkRepositoryLinks from "./src/lib/remark-repository-links.js"; const repositoryRoot = fileURLToPath(new URL("../", import.meta.url)); +const diagramsDirectory = fileURLToPath( + new URL("./public/diagrams", import.meta.url), +); export default defineConfig({ site: "https://thimbledb.com", @@ -18,6 +22,12 @@ export default defineConfig({ markdown: { processor: unified({ remarkPlugins: [ + [ + remarkMermaidDiagrams, + { + publicDirectory: diagramsDirectory, + }, + ], [ remarkRepositoryLinks, { diff --git a/site/package.json b/site/package.json index 0ed1ec4..73c06f8 100644 --- a/site/package.json +++ b/site/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "prepare:assets": "node scripts/prepare-assets.mjs", + "generate:diagrams": "node scripts/generate-diagrams.mjs", "dev": "npm run prepare:assets && astro dev", "check": "npm run prepare:assets && astro check", "build": "npm run prepare:assets && astro check && astro build", diff --git a/site/public/diagrams/00741aca01dd3f86612e.svg b/site/public/diagrams/00741aca01dd3f86612e.svg new file mode 100644 index 0000000..082cccd --- /dev/null +++ b/site/public/diagrams/00741aca01dd3f86612e.svg @@ -0,0 +1 @@ +Browser cachesPrivate auth storeAuthoritycreateThimbleClientApplicationBrowser cachesPrivate auth storeAuthoritycreateThimbleClientApplicationalt[Encrypted scope]Create connectionGET /api/config with session cookieReload current user and sessionRecalculate current grantsScope, CSRF, layouts, indexes, generation, active key IDGET scope key grantRequire current read grantCurrent and readable historical scope keysImport non-extractable decrypt-only CryptoKeysCreate authority-and-scope namespaceReady typed client \ No newline at end of file diff --git a/site/public/diagrams/147485b5115ec3f4ffed.svg b/site/public/diagrams/147485b5115ec3f4ffed.svg new file mode 100644 index 0000000..86a14b1 --- /dev/null +++ b/site/public/diagrams/147485b5115ec3f4ffed.svg @@ -0,0 +1 @@ +

Validated user or service identity

Internal principal
current roles + tenant memberships

Default scope authorizer

user:<uuid>
read + write

tenant:<id>
read

tenant:<id>
read + write
tenant writer or admin role

tenant:<id>
admin
tenant admin role

role:<role>
read

 \ No newline at end of file diff --git a/site/public/diagrams/1dfc8cb88c7aac4dabf7.svg b/site/public/diagrams/1dfc8cb88c7aac4dabf7.svg new file mode 100644 index 0000000..ae5ed95 --- /dev/null +++ b/site/public/diagrams/1dfc8cb88c7aac4dabf7.svg @@ -0,0 +1 @@ +

One application deployment

Browser application

External OIDC provider

Application and Studio assets

ThimbleDB authority
sessions + broker + writes

Application server or Worker logic

Private application object storage

Separate private auth storage

Master key and provider credentials

 \ No newline at end of file diff --git a/site/public/diagrams/2e76976c4fd220d46ab7.svg b/site/public/diagrams/2e76976c4fd220d46ab7.svg new file mode 100644 index 0000000..e1388d6 --- /dev/null +++ b/site/public/diagrams/2e76976c4fd220d46ab7.svg @@ -0,0 +1 @@ +

npx thimbledb create

Loopback Node authority
local provider

Explicit loopback-only
development identity

Vite app
typed notes + declared indexes

doctor + build + browser CRUD verification

Production authority

External OIDC

R2, S3, Azure Blob,
or controlled local storage

 \ No newline at end of file diff --git a/site/public/diagrams/4e36152ae5a803b417ab.svg b/site/public/diagrams/4e36152ae5a803b417ab.svg new file mode 100644 index 0000000..be0bfef --- /dev/null +++ b/site/public/diagrams/4e36152ae5a803b417ab.svg @@ -0,0 +1 @@ +

thimbledb npm package

Version-matched Studio assets

Node authority
serves /studio/

thimbledb studio-assets

Cloudflare asset build
/studio/

Studio browser

OIDC session

Explicit scope grants

Authority Studio and document APIs

 \ No newline at end of file diff --git a/site/public/diagrams/5628899b5012f929f172.svg b/site/public/diagrams/5628899b5012f929f172.svg new file mode 100644 index 0000000..ce0ad3a --- /dev/null +++ b/site/public/diagrams/5628899b5012f929f172.svg @@ -0,0 +1 @@ +Private object storageAuthenticated object brokerScope-namespaced cachesThimbleDB clientApplicationPrivate object storageAuthenticated object brokerScope-namespaced cachesThimbleDB clientApplicationalt[Cold point read and bundle endpoint advertised][HEAD missing or stale]alt[Explicit projection is covered][Full documents required]alt[Index plan][Scan plan]where(...).orderBy(...).take(...).get()Validate query and choose point, index, or scan planRead collection HEADGET bounded point-read bundleRequire current read grant and enforce object/byte limitsDecoded HEAD and immutable cache values over HTTPSStore returned values with device-key encryptionGET encrypted HEAD with sessionRead object with ETag conditionEncrypted HEADEncrypted HEADStore decrypted value under device-key encryptionRead referenced immutable index pageFetch index page on cache missResolve candidate IDs within maxScanValidate predicate and order from declared projectionsBuild projected documents without full-document readsResolve snapshot once or shared trie nodesFetch only missing immutable objectsEnforce bounded collection scanDecrypt when required, validate candidates, order, and limitDocuments plus point/index/scan plan \ No newline at end of file diff --git a/site/public/diagrams/589f254c1ee8289973b2.svg b/site/public/diagrams/589f254c1ee8289973b2.svg new file mode 100644 index 0000000..8ed1cc7 --- /dev/null +++ b/site/public/diagrams/589f254c1ee8289973b2.svg @@ -0,0 +1 @@ +

Human administrator

Service principal or workload identity

External OIDC provider

Interactive access token

Short-lived application token

ThimbleDB OIDC session exchange

Opaque revocable session

Current explicit scope grants

Live viewer or automation endpoint

 \ No newline at end of file diff --git a/site/public/diagrams/737720e06ef2016a1a1a.svg b/site/public/diagrams/737720e06ef2016a1a1a.svg new file mode 100644 index 0000000..3580a96 --- /dev/null +++ b/site/public/diagrams/737720e06ef2016a1a1a.svg @@ -0,0 +1 @@ +

Separate Worker or service deployment

One public browser origin

/

/api/* and /studio/*

Browser application

External OIDC provider

Path router or application gateway

Application assets or application service

ThimbleDB authority

Object broker and read bundles

Validation and conditional writes

Private application object storage

Separate private auth storage

Authority-only secrets

 \ No newline at end of file diff --git a/site/public/diagrams/897ea0c26deb6b3cb03a.svg b/site/public/diagrams/897ea0c26deb6b3cb03a.svg new file mode 100644 index 0000000..c0fa702 --- /dev/null +++ b/site/public/diagrams/897ea0c26deb6b3cb03a.svg @@ -0,0 +1 @@ +Other browser tabsCurrent browser cachesR2, S3, Blob, or local storeAuth serviceWorker or Node authorityApplicationOther browser tabsCurrent browser cachesR2, S3, Blob, or local storeAuth serviceWorker or Node authorityApplicationalt[ETag conflict][Commit]Mutation + session + CSRF + scope + generationReload user and recalculate grantsCurrent write grant or denialRead HEAD and affected immutable objectsValidate document and update configured indexesCreate immutable document/root objectsCreate immutable index pagesPublish one HEAD with all document and index references using If-MatchPrecondition failedReload current HEAD and retryNew HEAD ETagCommitted object bundleApply new HEAD and immutable objectsBroadcast committed bundle \ No newline at end of file diff --git a/site/public/diagrams/9b4aa8045c4fb8fa072b.svg b/site/public/diagrams/9b4aa8045c4fb8fa072b.svg new file mode 100644 index 0000000..0bbcb3b --- /dev/null +++ b/site/public/diagrams/9b4aa8045c4fb8fa072b.svg @@ -0,0 +1 @@ +

Browser application

External OIDC provider

Cloudflare Worker
static assets + authority + object broker

Private R2 data binding

Separate private R2 auth binding

Encrypted data bucket

Encrypted identity and session bucket

Worker secrets
master key and OIDC configuration

 \ No newline at end of file diff --git a/site/public/diagrams/9d3eb2d431c870257603.svg b/site/public/diagrams/9d3eb2d431c870257603.svg new file mode 100644 index 0000000..1725430 --- /dev/null +++ b/site/public/diagrams/9d3eb2d431c870257603.svg @@ -0,0 +1 @@ +

Rollback protection

JSON, CSV, lowdb, SQLite,
PostgreSQL, Firestore, or ThimbleDB

Explicit migration adapter

Plaintext logical archive
manifest + deterministic NDJSON + SHA-256

Checksum, shape, count,
path, and tombstone validation

Archive-wide target preflight

Quiescent write window

Create, replace, or merge import

Read-back full-document verification

Encrypted ThimbleDB collections
or explicit external target

Separate encrypted provider backup

 \ No newline at end of file diff --git a/site/public/diagrams/b39ab811f8e20b9226e4.svg b/site/public/diagrams/b39ab811f8e20b9226e4.svg new file mode 100644 index 0000000..1afaf49 --- /dev/null +++ b/site/public/diagrams/b39ab811f8e20b9226e4.svg @@ -0,0 +1 @@ +

Key or layout changed

Deployment master key
platform secret

HKDF-SHA-256

Scope data key vN
AES-256-GCM

Scope address key vN
HMAC-SHA-256

System-auth key
AES-256-GCM

TDB1 encrypted objects

Private deterministic object addresses

Encrypted users, identity mappings, sessions, and limits

Short-lived authorised key grant

Non-extractable decrypt-only keys
memory only

Non-extractable device cache key

Encrypted IndexedDB values

Configuration check
layout generation + active scope key ID

Destroy scoped cache and reload client

 \ No newline at end of file diff --git a/site/public/diagrams/bc36843257ac2c3323b6.svg b/site/public/diagrams/bc36843257ac2c3323b6.svg new file mode 100644 index 0000000..0d7979e --- /dev/null +++ b/site/public/diagrams/bc36843257ac2c3323b6.svg @@ -0,0 +1 @@ +

ThimbleDB protocol
scopes + TDB1 + layouts + indexes + conditional HEAD

ObjectStore contract
get + put + delete + list + ETag conditions

Cloudflare reference

Local development

Azure deployment

AWS deployment

Worker authority

Private R2 data and auth bindings

Node authority

Single-process filesystem store

Container Apps authority

Separate data and auth Blob containers

Lambda container authority

Separate private data and auth S3 buckets

 \ No newline at end of file diff --git a/site/public/diagrams/c5bc1a796ce8e93f2840.svg b/site/public/diagrams/c5bc1a796ce8e93f2840.svg new file mode 100644 index 0000000..9b06136 --- /dev/null +++ b/site/public/diagrams/c5bc1a796ce8e93f2840.svg @@ -0,0 +1 @@ +

Yes

Yes

No or fallback

No

Yes

Yes

No

No

Typed query expression

Validate version, values, limits, depth, and scan bound

ID equality?

Cold cache and bundle endpoint available?

One bounded authority read bundle

Validate bundled HEAD and immutable values

Matching declared index with indexable values?

Read collection HEAD

Read collection HEAD

Read immutable encrypted index page

Resolve bounded candidate IDs

Explicit selected fields and all query fields covered?

Read declared projections from index page

Read candidate documents
coalesce shared immutable reads

Re-evaluate complete predicate

Read bounded collection

Apply deterministic ordering and limit

Return documents and plan metadata

 \ No newline at end of file diff --git a/site/public/diagrams/f4570032f31e22ff6018.svg b/site/public/diagrams/f4570032f31e22ff6018.svg new file mode 100644 index 0000000..598ad21 --- /dev/null +++ b/site/public/diagrams/f4570032f31e22ff6018.svg @@ -0,0 +1 @@ +

Yes

No

Live document

Authorised DELETE

Encrypted tombstone
original document retained

Reads and scans hide document

Restore requested
inside restore window?

Authorised restore
new live generation

Restore window expires
then purge grace elapses

Authorised purge removes tombstone
from current live layout

Old immutable generations
become unreachable

All authorities in maintenance mode
quiescent flag confirmed

Maintenance deletes unreachable objects
and retired layout prefixes

 \ No newline at end of file diff --git a/site/public/diagrams/f9b79a62f37a16efb8ec.svg b/site/public/diagrams/f9b79a62f37a16efb8ec.svg new file mode 100644 index 0000000..5fc7c63 --- /dev/null +++ b/site/public/diagrams/f9b79a62f37a16efb8ec.svg @@ -0,0 +1 @@ +

Platform secret boundary

Separate private authentication storage

Private data object storage

Application authority boundary

Browser trust boundary

Session cookie + CSRF + requested scope

Ciphertext objects or opt-in decoded bundles

Create immutable objects

Conditional HEAD publication

Application UI

ThimbleDB client

Scope-namespaced memory cache

Scope-namespaced, device-key-encrypted IndexedDB

Non-extractable decrypt-only scope keys

OIDC token exchange

Opaque revocable session

Current scope-grant calculation

Authenticated ciphertext-object and decoded-bundle broker

Mutation validation and execution

Mutable collection HEAD records

Immutable encrypted document and index pages

Identity mappings and users

Session digests and CSRF state

Authentication rate-limit state

Deployment master key

Storage credential or platform binding

 \ No newline at end of file diff --git a/site/scripts/generate-diagrams.mjs b/site/scripts/generate-diagrams.mjs new file mode 100644 index 0000000..d442da0 --- /dev/null +++ b/site/scripts/generate-diagrams.mjs @@ -0,0 +1,148 @@ +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + unlink, + writeFile, +} from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const siteRoot = fileURLToPath(new URL("../", import.meta.url)); +const repositoryRoot = path.resolve(siteRoot, ".."); +const docsRoot = path.join(repositoryRoot, "docs"); +const outputDirectory = path.join(siteRoot, "public", "diagrams"); +const temporaryDirectory = await mkdtemp( + path.join(tmpdir(), "thimbledb-diagrams-"), +); + +try { + const sourceFiles = (await walk(docsRoot)).filter((file) => + file.endsWith(".md"), + ); + const diagrams = []; + for (const file of sourceFiles) { + diagrams.push( + ...extractDiagrams(await readFile(file, "utf8")), + ); + } + await mkdir(outputDirectory, { recursive: true }); + const expectedFiles = new Set(); + + for (const diagram of diagrams) { + const hash = sourceHash(diagram); + const fileName = `${hash}.svg`; + expectedFiles.add(fileName); + const inputPath = path.join(temporaryDirectory, `${hash}.mmd`); + const outputPath = path.join(outputDirectory, fileName); + await writeFile(inputPath, `${diagram}\n`, "utf8"); + runMermaid(inputPath, outputPath); + } + + for (const entry of await readdir(outputDirectory, { + withFileTypes: true, + })) { + if ( + entry.isFile() && + entry.name.endsWith(".svg") && + !expectedFiles.has(entry.name) + ) { + await unlink(path.join(outputDirectory, entry.name)); + } + } + + /** + * @param {string} directory + * @returns {Promise} + */ + async function walk(directory) { + const entries = await readdir(directory, { + withFileTypes: true, + }); + /** @type {string[]} */ + const files = []; + for (const entry of entries) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await walk(entryPath))); + } else { + files.push(entryPath); + } + } + return files; + } + + console.log( + `Generated ${diagrams.length} static Mermaid diagrams.`, + ); +} finally { + await rm(temporaryDirectory, { + recursive: true, + force: true, + }); +} + +/** + * @param {string} markdown + * @returns {string[]} + */ +function extractDiagrams(markdown) { + return [ + ...markdown.matchAll( + /```mermaid\s*\r?\n([\s\S]*?)\r?\n```/g, + ), + ].map((match) => normalizeSource(match[1])); +} + +/** @param {string} source */ +function sourceHash(source) { + return createHash("sha256") + .update(normalizeSource(source)) + .digest("hex") + .slice(0, 20); +} + +/** @param {string} source */ +function normalizeSource(source) { + return source.replace(/\r\n/g, "\n").trim(); +} + +/** + * @param {string} inputPath + * @param {string} outputPath + */ +function runMermaid(inputPath, outputPath) { + const executable = process.platform === "win32" ? "npx.cmd" : "npx"; + const result = spawnSync( + executable, + [ + "--yes", + "@mermaid-js/mermaid-cli@12.0.0", + "--input", + inputPath, + "--output", + outputPath, + "--theme", + "neutral", + "--backgroundColor", + "transparent", + "--no-font-embed", + "--quiet", + ], + { + cwd: repositoryRoot, + shell: process.platform === "win32", + stdio: "inherit", + }, + ); + if (result.status !== 0) { + throw new Error( + `Mermaid rendering failed for ${path.basename(inputPath)}${result.error ? `: ${result.error.message}` : ""}`, + ); + } +} diff --git a/site/src/components/Footer.astro b/site/src/components/Footer.astro index 4a5ccf2..b2000bc 100644 --- a/site/src/components/Footer.astro +++ b/site/src/components/Footer.astro @@ -28,13 +28,16 @@ import { site } from "../data/site";
- Apache-2.0 licensed. + Apache-2.0 licensed. Documentation and evidence are sourced from the public repository.
diff --git a/site/src/content.config.ts b/site/src/content.config.ts index c78b137..fe8be65 100644 --- a/site/src/content.config.ts +++ b/site/src/content.config.ts @@ -6,6 +6,8 @@ const docs = defineCollection({ pattern: [ "**/*.md", "!README.md", + // Contributor release operations are public repository documentation, + // but not application-user documentation for the product website. "!NPM-PUBLISHING.md", ], base: new URL("../../docs", import.meta.url), diff --git a/site/src/data/docs.ts b/site/src/data/docs.ts index 8e40168..449c9c7 100644 --- a/site/src/data/docs.ts +++ b/site/src/data/docs.ts @@ -37,6 +37,15 @@ export const docs: DocMeta[] = [ order: 12, featured: true, }, + { + id: "configuration", + title: "Configuration reference", + description: + "Review authority options, environment variables, provider settings, defaults, and deployment-template coverage.", + group: "Start here", + order: 14, + featured: true, + }, { id: "use-cases", title: "Use cases", @@ -48,7 +57,7 @@ export const docs: DocMeta[] = [ }, { id: "vibe-coded-apps", - title: "Should you use ThimbleDB?", + title: "Should you use ThimbleDB for a vibe-coded app?", description: "Use an exact fit check to decide whether a vibe-coded or AI-assisted small app matches ThimbleDB.", group: "Start here", @@ -91,7 +100,7 @@ export const docs: DocMeta[] = [ }, { id: "use-cases/tenant-operations", - title: "Multi-tenant operations portal", + title: "Multi-tenant internal operations portal", description: "Separate tenant and user records for a read-heavy internal operations application.", group: "Use cases", @@ -99,7 +108,7 @@ export const docs: DocMeta[] = [ }, { id: "use-cases/field-guide", - title: "Field guide and inspections", + title: "Offline-readable field guide and inspections", description: "Keep reference material available from browser cache while submitting connected inspection records.", group: "Use cases", @@ -107,7 +116,7 @@ export const docs: DocMeta[] = [ }, { id: "use-cases/catalogue", - title: "Catalogue or reference library", + title: "Small catalogue or reference library", description: "Serve a small bounded catalogue with snapshot reads and measured growth thresholds.", group: "Use cases", @@ -115,7 +124,7 @@ export const docs: DocMeta[] = [ }, { id: "use-cases/progress-journal", - title: "Progress or activity journal", + title: "Progress, training, or activity journal", description: "Store user-owned entries, summaries, and profile data without a separate database engine.", group: "Use cases", @@ -123,7 +132,7 @@ export const docs: DocMeta[] = [ }, { id: "use-cases/ai-context", - title: "Structured AI application context", + title: "Structured context for an AI-assisted application", description: "Keep bounded preferences and task context separate from model prompts and provider tokens.", group: "Use cases", @@ -172,7 +181,7 @@ export const docs: DocMeta[] = [ }, { id: "compare/object-storage", - title: "ThimbleDB and direct object storage", + title: "ThimbleDB and direct JSON in object storage", description: "Decide whether direct JSON objects are enough or a versioned storage protocol is justified.", group: "Compare", @@ -189,9 +198,9 @@ export const docs: DocMeta[] = [ }, { id: "authority-deployment", - title: "Authority deployment modes", + title: "In-app and separate authority deployment", description: - "Choose an embedded or separately deployed authority and preserve the browser origin, secret, and operations boundaries.", + "Compare in-app and separate Worker or service topologies, scaling opportunities, trust boundaries, and same-origin requirements.", group: "Understand", order: 15, featured: true, @@ -200,7 +209,7 @@ export const docs: DocMeta[] = [ id: "diagrams", title: "System diagrams", description: - "Review trust boundaries, query and index flows, key rotation, service access, migration, and provider topology.", + "Review trust boundaries, deployment styles, read and write flows, deletion, key rotation, migration, and provider topology.", group: "Understand", order: 20, }, @@ -363,6 +372,14 @@ export const docs: DocMeta[] = [ group: "Reference", order: 50, }, + { + id: "website-privacy", + title: "Website privacy", + description: + "Understand what the static documentation site collects and how Cloudflare Web Analytics is used.", + group: "Reference", + order: 55, + }, { id: "faq", title: "Frequently asked questions", diff --git a/site/src/data/site.ts b/site/src/data/site.ts index ff552b8..981277b 100644 --- a/site/src/data/site.ts +++ b/site/src/data/site.ts @@ -4,8 +4,11 @@ export const site = { name: "ThimbleDB", url: "https://thimbledb.com", description: - "Encrypted browser-first database for small web applications, backed by object storage.", + "Encrypted browser-first JSON database for small web apps, backed by object storage with in-app or separate Cloudflare and Node authorities.", repository: "https://github.com/Jason-Doyle/thimble", + issues: "https://github.com/Jason-Doyle/thimble/issues", + license: + "https://github.com/Jason-Doyle/thimble/blob/main/LICENSE", npm: "https://www.npmjs.com/package/thimbledb", version: packageJson.version, author: "Jason Doyle", diff --git a/site/src/lib/remark-mermaid-diagrams.js b/site/src/lib/remark-mermaid-diagrams.js new file mode 100644 index 0000000..5aa1601 --- /dev/null +++ b/site/src/lib/remark-mermaid-diagrams.js @@ -0,0 +1,93 @@ +import { + existsSync, + readFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import path from "node:path"; + +/** + * @param {{ publicDirectory: string }} options + */ +export default function remarkMermaidDiagrams(options) { + const publicDirectory = path.resolve(options.publicDirectory); + + /** @param {import("mdast").Root} tree */ + return (tree) => { + let currentHeading = "System"; + for (const [index, child] of tree.children.entries()) { + if (child.type === "heading") { + currentHeading = plainText(child) || currentHeading; + } + if (child.type === "code" && child.lang === "mermaid") { + const source = normalizeSource(child.value); + const hash = createHash("sha256") + .update(source) + .digest("hex") + .slice(0, 20); + const fileName = `${hash}.svg`; + const assetPath = path.join(publicDirectory, fileName); + if (!existsSync(assetPath)) { + throw new Error( + `Missing generated Mermaid asset ${fileName}. Run npm --prefix site run generate:diagrams.`, + ); + } + const dimensions = svgDimensions( + readFileSync(assetPath, "utf8"), + ); + const caption = `${currentHeading} diagram`; + tree.children[index] = { + type: "html", + value: [ + '
', + `${escapeHtml(caption)}`, + `
${escapeHtml(caption)}
`, + '
', + "View Mermaid source", + `
${escapeHtml(source)}
`, + "
", + "
", + ].join(""), + }; + } + } + }; +} + +/** + * @param {import("mdast").Nodes} node + * @returns {string} + */ +function plainText(node) { + if ("value" in node && typeof node.value === "string") { + return node.value; + } + if (!("children" in node) || !Array.isArray(node.children)) { + return ""; + } + return node.children.map(plainText).join(""); +} + +/** @param {string} svg */ +function svgDimensions(svg) { + const match = + /]*\bviewBox="[-\d.]+ [-\d.]+ ([\d.]+) ([\d.]+)"/.exec( + svg, + ); + return match + ? ` width="${Math.ceil(Number(match[1]))}" height="${Math.ceil(Number(match[2]))}"` + : ""; +} + +/** @param {string} source */ +function normalizeSource(source) { + return source.replace(/\r\n/g, "\n").trim(); +} + +/** @param {string} value */ +function escapeHtml(value) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} diff --git a/site/src/pages/index.astro b/site/src/pages/index.astro index b528f51..5e9c8c7 100644 --- a/site/src/pages/index.astro +++ b/site/src/pages/index.astro @@ -43,7 +43,7 @@ const websiteSchema = { @@ -351,6 +351,74 @@ const websiteSchema = { Azure Blob Storage Local filesystem + +
+
+
+

Two deployment styles

+

Keep the authority in-app, or give it its own boundary.

+
+

+ The browser API and stored data stay the same. The choice controls + secret isolation, release ownership, failure containment, and + which traffic can scale independently. +

+
+ +
+
+
+

In-app authority

+ One deployment +
+ +

+ Lowest operating overhead. Application and authority share + secrets, releases, failures, logs, limits, and scaling. +

+
+ +
+
+

Separate authority

+ Independent runtime +
+ +

+ Separate secrets, releases, limits, logs, failures, regions, and + scaling. Adds routing and another service to operate. +

+
+
+ + + Compare deployment and scaling tradeoffs + +
diff --git a/site/src/pages/llms-full.txt.ts b/site/src/pages/llms-full.txt.ts index afb677e..4bd7e31 100644 --- a/site/src/pages/llms-full.txt.ts +++ b/site/src/pages/llms-full.txt.ts @@ -8,7 +8,11 @@ export const prerender = true; const includedIds = docs.map((doc) => doc.id); export async function GET() { - const entries = await getCollection("docs"); + const [documentation, project] = await Promise.all([ + getCollection("docs"), + getCollection("project"), + ]); + const entries = [...documentation, ...project]; const byId = new Map(entries.map((entry) => [entry.id, entry])); const sections = includedIds.flatMap((id) => { const entry = byId.get(id); diff --git a/site/src/pages/llms.txt.ts b/site/src/pages/llms.txt.ts index 1f60a61..5abbd4a 100644 --- a/site/src/pages/llms.txt.ts +++ b/site/src/pages/llms.txt.ts @@ -23,6 +23,7 @@ scope and one collection. - [Should you use ThimbleDB for a vibe-coded app?](${site.url}/vibe-coded-apps/): Exact fit and rejection criteria for AI-assisted small apps. - [Quickstart](${site.url}/docs/quickstart/): Install the package and configure Cloudflare, Node, and browser clients. - [Local development](${site.url}/docs/development/): Scaffold a complete local app with the safe development identity. +- [Configuration reference](${site.url}/docs/configuration/): Authority options, environment variables, defaults, provider settings, and template coverage. - [ThimbleDB Studio](${site.url}/docs/studio/): Package-owned management UI for explicit scopes, bounded queries, index health, exports, and guarded operations. - [Starter examples](${site.url}/examples/): Local notes and Cloudflare authority examples. - [Node starter repository](https://github.com/Jason-Doyle/thimbledb-node-starter): Local-first authenticated notes application. @@ -33,7 +34,8 @@ scope and one collection. ## Architecture and security - [Architecture](${site.url}/docs/architecture/): Browser cache, authority, object storage, and scopes. -- [Authority deployment modes](${site.url}/docs/authority-deployment/): Embedded and separate authority services, decision criteria, and same-origin requirements. +- [In-app and separate authority deployment](${site.url}/docs/authority-deployment/): Topologies, scaling opportunities, trust boundaries, and same-origin requirements. +- [System diagrams](${site.url}/docs/diagrams/): Rendered trust, deployment, read, write, deletion, key, migration, Studio, and provider flows. - [Security](${site.url}/security/): Threat model, encryption, key handling, and browser boundaries. - [Authentication](${site.url}/docs/authentication/): External OIDC identities and revocable sessions. - [Machine and service access](${site.url}/docs/service-access/): Entra roles, service principals, live viewers, and why there is no global admin key. @@ -47,6 +49,7 @@ scope and one collection. - [Tradeoffs](${site.url}/docs/tradeoffs/): Verified behaviour, unknowns, and poor-fit workloads. - [R2 browser benchmarks](${site.url}/benchmarks/): Multi-region evidence and limitations. - [Public package API](${site.url}/docs/public-api/): Stable package exports. +- [Website privacy](${site.url}/docs/website-privacy/): Static-site data handling and Cloudflare Web Analytics disclosure. - [GitHub repository](${site.repository}): Source, tests, examples, fixtures, and raw evidence. - [npm package](${site.npm}): Published package. diff --git a/site/src/scripts/site.ts b/site/src/scripts/site.ts index 6fe7dae..558b175 100644 --- a/site/src/scripts/site.ts +++ b/site/src/scripts/site.ts @@ -2,7 +2,11 @@ const copyLabel = "Copy"; for (const block of document.querySelectorAll(".prose pre")) { const code = block.querySelector("code"); - if (!code || block.parentElement?.classList.contains("code-shell")) { + if ( + !code || + block.dataset.language === "mermaid" || + block.parentElement?.classList.contains("code-shell") + ) { continue; } const shell = document.createElement("div"); diff --git a/site/src/styles/global.css b/site/src/styles/global.css index acfa305..2cbcbaa 100644 --- a/site/src/styles/global.css +++ b/site/src/styles/global.css @@ -1405,6 +1405,133 @@ summary { font-size: 0.72rem; } +.deployment-choice { + margin-top: 4rem; + border-top: 1px solid rgb(120 180 255 / 0.16); + padding-top: 3rem; +} + +.deployment-choice-heading { + display: grid; + grid-template-columns: minmax(0, 1.15fr) minmax(18rem, 0.85fr); + gap: 3rem; + align-items: end; +} + +.deployment-choice-heading h3 { + max-width: 18ch; + margin: 0.55rem 0 0; + font-size: clamp(2rem, 4vw, 3.15rem); + letter-spacing: -0.055em; + line-height: 1; +} + +.deployment-choice-heading > p { + margin: 0; + color: #afc2dd; +} + +.deployment-choice-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 1rem; + margin: 2rem 0 1.4rem; +} + +.deployment-choice-grid article { + border: 1px solid rgb(116 179 255 / 0.2); + border-radius: var(--radius-lg); + padding: clamp(1.25rem, 3vw, 2rem); + background: linear-gradient( + 145deg, + rgb(255 255 255 / 0.075), + rgb(255 255 255 / 0.025) + ); +} + +.deployment-choice-grid header { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + gap: 1rem; + align-items: baseline; +} + +.deployment-choice-grid header h4 { + margin: 0; + font-size: 1.15rem; + font-weight: 800; +} + +.deployment-choice-grid header strong { + color: #7fdcff; + font-size: 0.7rem; + letter-spacing: 0.09em; + text-transform: uppercase; +} + +.deployment-choice-grid article > p { + margin: 1.25rem 0 0; + color: #afc2dd; +} + +.topology-line { + display: flex; + flex-wrap: wrap; + gap: 0.55rem; + align-items: center; + margin-top: 1.35rem; +} + +.topology-line span, +.topology-branches span { + border: 1px solid rgb(120 180 255 / 0.24); + border-radius: 0.55rem; + padding: 0.62rem 0.75rem; + color: #d9e8ff; + background: rgb(2 12 31 / 0.64); + font-size: 0.75rem; + font-weight: 750; + text-align: center; +} + +.topology-line .accent-node { + border-color: rgb(66 215 255 / 0.56); + color: var(--white); + background: rgb(12 114 255 / 0.18); +} + +.topology-line i { + color: #42d7ff; + font-style: normal; +} + +.topology-split { + margin-top: 1.35rem; +} + +.topology-split .topology-line { + justify-content: center; + margin-top: 0; +} + +.topology-branches { + display: grid; + grid-template-columns: 0.75fr 1.25fr; + gap: 0.55rem; + margin-top: 0.55rem; +} + +.topology-branches span:last-child { + border-color: rgb(66 215 255 / 0.56); + color: var(--white); + background: rgb(12 114 255 / 0.18); +} + +.light-link { + color: #9ee8ff; +} + .docs-landing-hero { padding-block: 5rem 5.5rem; color: var(--white); @@ -1792,6 +1919,57 @@ summary { scrollbar-color: #2e5a96 transparent; } +.mermaid-figure { + overflow: hidden; + border: 1px solid var(--slate-200); + border-radius: var(--radius-lg); + margin: 1.75rem 0; + background: #fff; + box-shadow: var(--shadow-sm); +} + +.prose .mermaid-static { + display: block; + width: 100%; + max-width: 100%; + height: auto; + margin: 0; + border: 0; + border-radius: 0; + padding: clamp(1rem, 3vw, 2rem); + background: #fff; +} + +.mermaid-figure figcaption { + border-top: 1px solid var(--slate-200); + padding: 0.7rem 1rem; + color: var(--slate-600); + background: var(--slate-50); + font-size: 0.78rem; + font-weight: 750; +} + +.mermaid-source { + border-top: 1px solid var(--slate-200); + background: var(--ink-950); +} + +.mermaid-source summary { + padding: 0.75rem 1rem; + color: #b9cdea; + cursor: pointer; + font-size: 0.76rem; + font-weight: 750; +} + +.mermaid-source pre, +.mermaid-source .astro-code { + border: 0; + border-radius: 0; + margin: 0; + box-shadow: none; +} + .code-shell .prose pre, .code-shell pre { margin: 0; @@ -2129,6 +2307,11 @@ summary { grid-template-columns: 1fr; } + .deployment-choice-heading, + .deployment-choice-grid { + grid-template-columns: 1fr; + } + .architecture-flow article { min-height: auto; } diff --git a/site/tests/site.spec.ts b/site/tests/site.spec.ts index e5fd4a8..0bcccaf 100644 --- a/site/tests/site.spec.ts +++ b/site/tests/site.spec.ts @@ -19,7 +19,7 @@ test("homepage presents the product and complete SEO metadata", async ({ ).toHaveAttribute("href", "https://thimbledb.com/"); await expect(page.locator('meta[name="description"]')).toHaveAttribute( "content", - /encrypted, browser-first database/i, + /encrypted browser-first JSON database/i, ); await expect( page.locator('script[type="application/ld+json"]'), @@ -38,6 +38,29 @@ test("homepage presents the product and complete SEO metadata", async ({ }); await studioImage.scrollIntoViewIfNeeded(); await expect(studioImage).toHaveJSProperty("complete", true); + await expect( + page.getByRole("link", { name: "Privacy", exact: true }), + ).toHaveAttribute("href", "/docs/website-privacy/"); + await expect( + page.getByRole("link", { name: "Issues", exact: true }), + ).toHaveAttribute( + "href", + "https://github.com/Jason-Doyle/thimble/issues", + ); + await expect( + page.getByRole("heading", { + level: 3, + name: "Keep the authority in-app, or give it its own boundary.", + }), + ).toBeVisible(); + await expect( + page.getByRole("link", { + name: /Compare deployment and scaling tradeoffs/, + }), + ).toHaveAttribute("href", "/docs/authority-deployment/"); + await expect( + page.locator('script[src*="MermaidDiagrams"]'), + ).toHaveCount(0); await assertNoHorizontalOverflow(page); }); @@ -77,22 +100,41 @@ test("repository documentation renders with rewritten internal links", async ({ await expect( page.getByRole("heading", { level: 1, - name: "Authority deployment modes", + name: "In-app and separate authority deployment", }), ).toBeVisible(); await expect( page.getByRole("heading", { level: 2, - name: "Embedded authority", + name: "In-app (embedded) authority", }), ).toBeVisible(); await expect( page.getByRole("heading", { level: 2, - name: "Separate authority service", + name: "Separate Worker or service", + }), + ).toBeVisible(); + await expect( + page.getByRole("heading", { + level: 2, + name: "Scaling opportunities", }), ).toBeVisible(); await assertNoHorizontalOverflow(page); + + await page.goto("/docs/website-privacy/"); + await expect( + page.getByRole("heading", { + level: 1, + name: "Website privacy", + }), + ).toBeVisible(); + await expect( + page.getByRole("link", { + name: "Cloudflare Web Analytics", + }), + ).toBeVisible(); }); test("documentation search returns relevant repository pages", async ({ @@ -108,6 +150,36 @@ test("documentation search returns relevant repository pages", async ({ ).toBeVisible(); }); +test("Mermaid source renders as diagrams with accessible fallback", async ({ + page, +}) => { + await page.goto("/docs/diagrams/"); + await expect( + page.locator('script[src*="MermaidDiagrams"]'), + ).toHaveCount(0); + await expect(page.locator(".mermaid-figure")).toHaveCount(16); + await expect(page.locator(".mermaid-static")).toHaveCount(16); + await expect(page.locator(".mermaid-source")).toHaveCount(16); + await expect(page.locator(".mermaid-figure .code-shell")).toHaveCount(0); + const firstDiagram = page.locator(".mermaid-static").first(); + await firstDiagram.scrollIntoViewIfNeeded(); + await expect(firstDiagram).toHaveJSProperty("complete", true); + expect( + await firstDiagram.evaluate( + (image: HTMLImageElement) => image.naturalWidth, + ), + ).toBeGreaterThan(100); + expect( + Number(await firstDiagram.getAttribute("width")), + ).toBeGreaterThan(100); + + await page.goto("/docs/authority-deployment/"); + await page.goto("/docs/authority-deployment/"); + await expect( + page.locator('script[src*="MermaidDiagrams"]'), + ).toHaveCount(0); +}); + test("FAQ publishes structured answers and user-facing content", async ({ page, }) => { @@ -152,6 +224,9 @@ test("AI discovery routes publish explicit access and decision content", async ( expect(llmsText).toContain("Logical migration"); expect(llmsText).toContain("Machine and service access"); expect(llmsText).toContain("ThimbleDB Studio"); + expect(llmsText).toContain("In-app and separate authority deployment"); + expect(llmsText).toContain("System diagrams"); + expect(llmsText).toContain("Configuration reference"); const full = await request.get("/llms-full.txt"); expect(full.ok()).toBe(true); @@ -161,6 +236,13 @@ test("AI discovery routes publish explicit access and decision content", async ( expect(fullText).toContain("# Logical migration"); expect(fullText).toContain("# Machine and service access"); expect(fullText).toContain("# ThimbleDB Studio"); + expect(fullText).toContain( + "# In-app and separate authority deployment", + ); + expect(fullText).toContain("# System diagrams"); + expect(fullText).toContain("# Configuration reference"); + expect(fullText).toContain("# Website privacy"); + expect(fullText).toContain("## 3.1.0"); await page.goto("/vibe-coded-apps/"); await expect(