feat: pre-generated monthly notary schedule for ghost keys

.github/workflows/rust-api-tests.yml

@@ -43,7 +43,9 @@ jobs:
       - name: Cache cargo
         uses: Swatinem/rust-cache@v2
         with:
-          workspaces: rust
+          workspaces: |
+            rust
+            rust/cli
 
       - name: Formatting
         working-directory: rust
@@ -63,3 +65,10 @@ jobs:
       - name: Test
         working-directory: rust
         run: cargo test -p ghostkey-api
+
+      # generate_notary_keys.sh mints years of notary keys in one master-key
+      # session, and a mistake in it (say, a second "quarterly" key for a quarter)
+      # is only found after the keys are in use. Runs against a throwaway
+      # master key.
+      - name: Notary schedule generator
+        run: bash rust/cli/test_notary_schedule.sh

hugo-site/static/js/donation-success.js

@@ -165,15 +165,52 @@ function generateTestCertificate() {
   displayCertificate(publicKey, privateKey, unblindedSignature);
 }
 
+// The notary certificate this payment was quoted. It must be the one we blind
+// against: the server signs with its pair, and notary keys rotate monthly.
+// localStorage is only a fallback (an API without this endpoint), because every
+// tab shares it and another checkout may have overwritten it with a different
+// month's certificate for the same amount.
+async function fetchQuotedNotaryCertificate(paymentIntentId) {
+  try {
+    const apiUrl = window.ghostkeyApiUrl;
+    // Bounded, so a hung request falls back instead of leaving the page
+    // spinning. AbortController rather than AbortSignal.timeout, which older
+    // browsers lack (it would throw and silently skip this fetch every time).
+    // The timer covers the body as well as the headers.
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), 20000);
+    try {
+      const response = await fetch(
+        `${apiUrl}/notary-certificate/${encodeURIComponent(paymentIntentId)}`,
+        { signal: controller.signal }
+      );
+      if (!response.ok) {
+        console.warn("Could not fetch quoted notary certificate:", response.status);
+        return null;
+      }
+      const data = await response.json();
+      return data.notary_certificate_base64 || null;
+    } finally {
+      clearTimeout(timer);
+    }
+  } catch (error) {
+    console.warn("Could not fetch quoted notary certificate:", error);
+    return null;
+  }
+}
+
 async function generateAndSignCertificate(paymentIntentId) {
   console.log("Starting generateAndSignCertificate");
   try {
-      // Read the canonical notary key first, fall back to the legacy
+      let notaryCertificateBase64 = await fetchQuotedNotaryCertificate(paymentIntentId);
+      // Fallback: read the canonical notary key first, then the legacy
       // delegate key for sessions opened before the rename. We write the
       // canonical key on migration so future reads are clean, but we do
       // NOT delete the legacy key for one release in case an old cached
       // tab reads it. Removal tracked for a future release in freenet/web#24.
-      let notaryCertificateBase64 = localStorage.getItem('notary_certificate_base64');
+      if (!notaryCertificateBase64) {
+        notaryCertificateBase64 = localStorage.getItem('notary_certificate_base64');
+      }
       if (!notaryCertificateBase64) {
         notaryCertificateBase64 = localStorage.getItem('delegate_certificate_base64');
         if (notaryCertificateBase64) {
@@ -209,9 +246,11 @@ async function generateAndSignCertificate(paymentIntentId) {
         headers: { 
           'Content-Type': 'application/json',
         },
-        body: JSON.stringify({ 
-          payment_intent_id: paymentIntentId, 
-          blinded_ghost_key_base64: blindedPublicKey
+        body: JSON.stringify({
+          payment_intent_id: paymentIntentId,
+          blinded_ghost_key_base64: blindedPublicKey,
+          // Lets the server refuse a mismatch before spending the donation.
+          notary_certificate_base64: notaryCertificateBase64
         }),
         credentials: 'same-origin'
       });

hugo-site/themes/freenet/layouts/shortcodes/stripe-donation-form.html

@@ -2,9 +2,10 @@
     <form id="payment-form">
     {{/* These tiers are NOT free to change. Each one needs a matching per-amount
          notary keypair on the API host (notary_certificate_{amount}.pem and
-         notary_signing_key_{amount}.pem, see rust/api/src/delegates.rs). An
-         amount with no keypair fails certificate signing AFTER the card has been
-         charged, leaving the donor paid up with no key.
+         notary_signing_key_{amount}.pem, see rust/api/src/delegates.rs), in the
+         flat files AND in every month of the notary schedule, so adding a tier
+         means a master-key session (rust/api/README.md). An amount with no
+         keypair fails when the donation is quoted, before any charge.
 
          Fixed tiers are also load-bearing for anonymity: the amount is written
          into the certificate and is visible to anyone who verifies it, so a

rust/api/README.md

@@ -2,94 +2,47 @@
 
 WARNING: This file is publicly readable, do NOT put anything secret in here.
 
+## Where it runs
+
+gkapi runs on **nova** as `gkapi.service` (unit: `/etc/systemd/system/gkapi.service`,
+binary: `/home/gkapi/bin/ghostkey-api`, user `gkapi`). It was migrated from vega on
+2026-08-28. Caddy terminates TLS for `gkapi.freenet.org` and owns ACME (see
+`/etc/caddy/gkapi.caddy`), and proxies to the API on `:8081`. So the binary binds an
+unprivileged port and needs no file capability. The unit file's comments explain why the
+TLS flags and `setcap` are gone and why the ufw `deny 8081` rule must stay.
+
 ## Deploying gkapi
 
 There is **no CI deployment for this crate**. `deploy.yml` builds the Hugo site and
 publishes it to GitHub Pages; it never touches the API. `rust-api-tests.yml` only runs
 fmt, build and test. Merging a change to `rust/api` therefore ships nothing: the binary
-on vega has to be replaced by hand, and has in the past sat months behind main.
-
-There is also no Rust toolchain on vega (`~gkapi/.cargo` exists but `bin/` is empty), so
-the binary is built elsewhere and copied. vega and nova are both Ubuntu 24.04 on the same
-glibc, so a release build from nova runs there as-is. Check with `ldd --version` on both
-before relying on that.
-
-### The step that is easy to miss
-
-The service runs as the unprivileged `gkapi` user and binds **port 80** for the HTTP-01
-ACME challenge server, which needs `CAP_NET_BIND_SERVICE`. That capability lives on the
-binary as a file capability, and **`cp` does not preserve it**. Replace the binary without
-re-running `setcap` and the service binds 443, logs `Listening on 0.0.0.0:443`, then panics
-at `main.rs` with `PermissionDenied` on the port 80 bind and systemd restart-loops until it
-gives up.
-
-The same trap catches the rollback: copying a backup *back* into place also produces a file
-with no capability, so a panicking service stays panicking and it looks like the new build
-is at fault. `mv` preserves the capability because it keeps the inode; `cp` does not.
-
-### Procedure
+has to be replaced by hand, and has in the past sat months behind main.
 
 ```bash
 # 1. build on nova, from main, and prove the build contains what you expect
 cd ~/code/freenet/web/main/rust && cargo build --release -p ghostkey-api
 strings target/release/ghostkey-api | grep -c payment_claim   # sanity: expect non-zero
 
-# 2. upload and re-check after transfer
-scp target/release/ghostkey-api vega:/tmp/ghostkey-api.new
-ssh vega 'strings /tmp/ghostkey-api.new | grep -q payment_claim && echo ok'
-
-# 3. stage it at its final location, fully prepared, WITHOUT displacing the live binary
-ssh vega '
-  sudo cp /tmp/ghostkey-api.new /home/gkapi/bin/ghostkey-api.staged
-  sudo chown gkapi:gkapi        /home/gkapi/bin/ghostkey-api.staged
-  sudo chmod 775                /home/gkapi/bin/ghostkey-api.staged
-  sudo setcap cap_net_bind_service+ep /home/gkapi/bin/ghostkey-api.staged
-  sudo getcap /home/gkapi/bin/ghostkey-api.staged
-'
-# Must print: /home/gkapi/bin/ghostkey-api.staged cap_net_bind_service=ep
-# If it does not, STOP. Nothing has changed yet and the live binary is untouched.
-
-# 4. swap with two adjacent renames, then restart
-ssh vega '
-  set -e
-  STAMP=$(date +%Y%m%d-%H%M%S)
-  sudo mv /home/gkapi/bin/ghostkey-api         /home/gkapi/bin/ghostkey-api.rollback-$STAMP
-  sudo mv /home/gkapi/bin/ghostkey-api.staged  /home/gkapi/bin/ghostkey-api
-  echo "rollback binary: /home/gkapi/bin/ghostkey-api.rollback-$STAMP"
-  sudo systemctl reset-failed gkapi
-  sudo systemctl restart gkapi
-'
+# 2. stage it next to the live binary, then swap with two adjacent renames
+sudo install -o gkapi -g gkapi -m 755 target/release/ghostkey-api /home/gkapi/bin/ghostkey-api.staged
+STAMP=$(date +%Y%m%d-%H%M%S)
+sudo mv /home/gkapi/bin/ghostkey-api        /home/gkapi/bin/ghostkey-api.rollback-$STAMP
+sudo mv /home/gkapi/bin/ghostkey-api.staged /home/gkapi/bin/ghostkey-api
+echo "rollback binary: /home/gkapi/bin/ghostkey-api.rollback-$STAMP"
+sudo systemctl restart gkapi
 ```
 
-Two things about that shape are deliberate, and both exist to avoid a worse failure than
-the one being fixed:
-
-- **Prepare the capability on the staged file, and verify it, before anything is
-  displaced.** The `getcap` gate in step 3 is the whole point of splitting the steps: if it
-  fails you stop with the running service completely untouched. Do not restart to see what
-  happens.
-- **Do not `mv` the old binary away in one command and `cp` the new one in with another.**
-  Between those two the path does not exist, and anything that restarts the unit in that
-  window (a crash, an OOM, a reboot, someone running `systemctl restart` out of order) fails
-  with `status=203/EXEC` and stays down. Step 4 closes that to two adjacent renames.
-  `mv` is also what carries the capability across, since it keeps the inode.
-
-`sudo` on `getcap` is not decoration: it lives in `/usr/sbin`, which is not on a normal
-user's `PATH`, so a bare `getcap` can fail as "command not found" and skip the check
-entirely.
+Do not `mv` the old binary away in one command and copy the new one in with a later one.
+Between those two the path does not exist, and anything that restarts the unit in that
+window fails with `status=203/EXEC` and stays down.
 
 ### Verifying a deploy
 
 ```bash
-ssh vega 'systemctl is-active gkapi; strings /home/gkapi/bin/ghostkey-api | grep -c payment_claim'
+systemctl is-active gkapi
 curl -s https://gkapi.freenet.org/                       # {"message":"Hello, world!"}
-curl -s -o /dev/null -w '%{http_code}\n' http://gkapi.freenet.org/.well-known/acme-challenge/probe
 ```
 
-That last one matters: a 404 means the port 80 challenge listener is up. Connection refused
-means the capability is missing and certificate renewal will fail at the next attempt even
-if HTTPS looks healthy.
-
 Then confirm every donation tier still resolves its notary keypair, which is a separate
 failure mode from the binary (see the tier comment in
 `hugo-site/themes/freenet/layouts/shortcodes/stripe-donation-form.html`).
@@ -109,39 +62,135 @@ done
 
 ### Rollback
 
-Substitute `<stamp>` with the timestamp step 4 printed.
+Substitute `<stamp>` with the timestamp the deploy printed.
+
+```bash
+sudo mv /home/gkapi/bin/ghostkey-api.rollback-<stamp> /home/gkapi/bin/ghostkey-api
+sudo systemctl restart gkapi
+```
+
+**Once a notary schedule month is live, do not roll back to a binary that predates the
+schedule** (anything before freenet/web#192). That binary ignores `notary_period` and signs
+with the flat files, so every donation quoted from a month directory and not yet signed is
+charged and gets a signature that does not unblind. The signing call succeeds, so the
+PaymentIntent stays marked `certificate_signed` and the donor cannot retry. If it happens
+anyway, roll forward, then clear `certificate_signed` only on the PaymentIntents of donors
+who report that key generation failed (the success page shows the error; the PaymentIntent
+id is in its URL), then tell each of them to reload that success page. Do not clear it in bulk on every PaymentIntent
+with `notary_period`: the server cannot tell which signatures failed to unblind, and a
+donor whose key worked would be able to mint a second one.
+
+## Notary keys and the monthly schedule
+
+The notary directory (`--notary-dir`, `/home/gkapi/delegate-keys` on nova) holds one
+keypair per donation tier: `notary_certificate_<amount>.pem` and
+`notary_signing_key_<amount>.pem` (older deployments use `delegate_*` names, which are
+still read). The certificate's info string carries the amount and a date,
+`delegate-key-created`, which every ghost key issued from it shows publicly.
+
+To keep that date current without taking the master key out of storage every month, the
+directory can also hold a schedule: one subdirectory per month, `YYYY-MM/`, each a complete
+set of tiers.
+
+### How the API uses it
+
+- **Quoting** (`/create-donation`, `/update-donation`) uses the newest `YYYY-MM/` that is not
+  in the future, or the flat files if there is none, and records which on the PaymentIntent
+  as `notary_period` metadata. If the current month is missing it logs an error and keeps
+  issuing from the newest earlier month, so a schedule that runs out degrades to a stale
+  date, not an outage. If the chosen month lacks a tier, quoting that tier fails, before
+  any charge.
+- **Blinding** happens on the success page after the charge, possibly in a later month.
+  The page fetches the certificate its PaymentIntent was quoted
+  (`GET /notary-certificate/{payment_intent_id}`) rather than trusting localStorage, which
+  every tab shares.
+- **Signing** (`/sign-certificate`) uses exactly the pair the PaymentIntent was quoted
+  from, never whatever is current by then, and refuses (409, before marking the payment
+  spent) if the client says it blinded against a different certificate. A PaymentIntent
+  with no `notary_period` (quoted from the flat files, including everything before the
+  schedule existed) signs with the flat files.
+
+So a month directory, and the flat files, must never be **deleted or replaced** while a
+donation quoted from it could still be in checkout: that breaks the donation after the
+charge. Keep past months; they are small. Never regenerate into a live directory.
+
+### Generating (master key mounted, one session)
+
+```bash
+rust/cli/generate_notary_keys.sh --master-key <master_signing_key.pem> \
+  --notary-dir <drive>/notary-schedule --start-month 2026-10 --months 120
+```
+
+By default $1 and $5 are dated monthly, and $20 and up are dated quarterly (1 January,
+April, July or October).
+- **Why not monthly for everything:** the date partitions each tier's anonymity set, and the
+  higher tiers see too few donors a month for a monthly date to be safe (see the script
+  header).
+- **Why not yearly:** an application that wants a *recent* ghost key can still use the date.
+  While the schedule covers the current month, a key is never shown more than three months
+  older than it is.
+
+The script verifies every pair against the compiled-in Freenet master key. It builds each
+month in a hidden directory before renaming it into place, and refuses to touch an existing
+month. When extending a schedule mid-quarter, it reuses that quarter's pairs, after
+checking them. `rust/cli/test_notary_schedule.sh` tests it (CI runs it).
+To extend later, run it again with `--start-month` just after the last month.
+
+Keep the output on the encrypted master-key drive. It is the backup.
+
+### Installing on nova: a rolling window
+
+Do not copy the whole schedule into the API's directory. If the API process were
+compromised, every future month's notary key would leak with it, and forged ghost keys
+dated years ahead would be indistinguishable from real ones. Instead keep the schedule
+root-only and let a daily timer publish the current and next month:
 
 ```bash
-ssh vega '
-  set -e
-  sudo cp /home/gkapi/bin/ghostkey-api.rollback-<stamp> /home/gkapi/bin/ghostkey-api.staged
-  sudo chown gkapi:gkapi /home/gkapi/bin/ghostkey-api.staged
-  sudo chmod 775 /home/gkapi/bin/ghostkey-api.staged
-  sudo setcap cap_net_bind_service+ep /home/gkapi/bin/ghostkey-api.staged
-  sudo getcap /home/gkapi/bin/ghostkey-api.staged
-  sudo mv /home/gkapi/bin/ghostkey-api.staged /home/gkapi/bin/ghostkey-api
-  sudo systemctl reset-failed gkapi && sudo systemctl restart gkapi
-'
+# once, from the mounted drive
+sudo install -d -m 700 -o root -g root /var/lib/gkapi-notary-schedule
+sudo cp -a <drive>/notary-schedule/20??-?? /var/lib/gkapi-notary-schedule/
+sudo chown -R root:root /var/lib/gkapi-notary-schedule
+sudo install -m 755 rust/api/publish_notary_window.sh /usr/local/sbin/publish-notary-window
 ```
 
-The `setcap` line is required on the way back too, and this is the part that is genuinely
-easy to get wrong under pressure: copying a backup *into place* produces a file with no
-capability, so the service carries on panicking and it reads as "the new build is broken"
-rather than "the copy dropped a capability". If a rollback does not fix the panic, run
-`sudo getcap` on the live binary before concluding anything about the build.
+`/etc/systemd/system/gkapi-notary-window.service`:
 
-## letsencrypt
+```ini
+[Unit]
+Description=Publish the next months of the ghost key notary schedule to gkapi
 
-Verify that certificate was automatically renewed by root cron job on vega by looking at write times
-of cert files:
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/publish-notary-window /var/lib/gkapi-notary-schedule /home/gkapi/delegate-keys gkapi:gkapi 1
+```
 
+`/etc/systemd/system/gkapi-notary-window.timer`:
+
+```ini
+[Unit]
+Description=Daily ghost key notary schedule publish
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+
+[Install]
+WantedBy=timers.target
 ```
-root@vega:/home/gkapi# cd /etc/letsencrypt/live/gkapi.freenet.org/
-root@vega:/etc/letsencrypt/live/gkapi.freenet.org# ls -l
-total 4
--rw-r--r-- 1 root ssl-cert 692 Aug  5 00:40 README
-lrwxrwxrwx 1 root root      41 Oct  4 05:31 cert.pem -> ../../archive/gkapi.freenet.org/cert2.pem
-lrwxrwxrwx 1 root root      42 Oct  4 05:31 chain.pem -> ../../archive/gkapi.freenet.org/chain2.pem
-lrwxrwxrwx 1 root root      46 Oct  4 05:31 fullchain.pem -> ../../archive/gkapi.freenet.org/fullchain2.pem
-lrwxrwxrwx 1 root root      44 Oct  4 05:31 privkey.pem -> ../../archive/gkapi.freenet.org/privkey2.pem
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl start gkapi-notary-window.service   # publish now; check its output
+sudo systemctl enable --now gkapi-notary-window.timer
 ```
+
+The publisher never replaces a published month, and it stages each month in a hidden
+directory, so the API never sees a partial one. It exits non-zero if the schedule has no
+directory for the current month, and warns when 12 or fewer months remain; both show in
+`systemctl status gkapi-notary-window` and the journal. No gkapi restart is needed; the
+files are read per request.
+
+**Order matters on first rollout:** deploy the binary from #192 first and let it run on the
+flat files, and only then enable the timer. A binary that predates the schedule never reads
+the month directories, and rolling back to one after a month is live breaks in-flight
+donations (see Rollback).