diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 040b42a..0f8fed6 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -45,6 +45,29 @@ Example shape: The numbers above illustrate the JSON schema only; they are not Rivet performance claims or release thresholds. +## Measuring a packaged app + +`raco rivet measure` reports what a host app actually weighs, so release CI +can gate budgets instead of eyeballing dist listings: + +- `raco rivet measure size` lists every artifact in `dist/` with its byte + count. With `--budget `, each artifact is matched against the + budget file's rules — `{"rules": [{"glob": "*.dmg", "max-bytes": + 100000000}, ...]}`, first matching glob wins — and the command exits + nonzero when any artifact exceeds its budget, so a bundle-size regression + fails the build instead of shipping. `--artifact ` narrows the + report to one file. +- `raco rivet measure startup [--artifact ] [--seconds N]` runs the + launch-survival smoke (the same gate `raco rivet package` and `verify` + use) and reports the elapsed window, so the artifact's ability to boot on + the packaging machine is measured rather than assumed. + +Per-RPC latency stays with the integration benchmarks above: the +`--benchmark` mode measures the real embedded runtime end to end, which a +detached CLI cannot reach into a packaged GUI host. A practical release +budget file lives in the product repository next to `rivet-schema.json` and +is updated in the same review that changes packaging. + ## CI behavior Every `Embedded Roundtrip` job first runs the existing correctness integration path, then starts a fresh integration process with `--benchmark`. The second step records one JSON report for that runner and revision. Starting a fresh process is important because the embedded Racket runtime is intentionally one-shot within a process and startup is itself one of the measured costs. diff --git a/rivet-cli/main.rkt b/rivet-cli/main.rkt index 2ea0d93..8e50e79 100644 --- a/rivet-cli/main.rkt +++ b/rivet-cli/main.rkt @@ -64,6 +64,8 @@ " [--published-at ] [--rollout <0-100>] [--rollback-allowed]\n" " compose a verified family update manifest\n" " raco rivet i18n verify shared/i18n parity and regenerate embedded tables\n" + " raco rivet measure report packaged artifact sizes and launch survival\n" + " raco rivet measure size --budget fail when artifacts exceed budgets\n" " raco rivet compliance generate SBOM/notices and run the license audit\n" " raco rivet verify re-verify the current packaged artifact\n" " raco rivet verify --production verify production trust/notarization requirements\n" @@ -295,6 +297,11 @@ (define-values (keys files) (apply values (generate-i18n! (current-project!)))) (say "i18n: ~a keys across zh/en; wrote ~a embedded table~a" keys files (if (= files 1) "" "s"))] + [(cons "measure" measure-args) + ;; Report packaged-app measurements (issue #262): dist artifact sizes + ;; against optional budgets, and the launch-survival window. + (define run-measure (command "measure" 'run-measure)) + (run-measure measure-args)] [(list "compliance") (define generate-compliance-artifacts! (command "compliance" 'generate-compliance-artifacts!)) diff --git a/rivet-cli/measure.rkt b/rivet-cli/measure.rkt new file mode 100644 index 0000000..bcb3eb3 --- /dev/null +++ b/rivet-cli/measure.rkt @@ -0,0 +1,196 @@ +#lang racket/base + +;; Measurement for host apps (issue #262). The migration docs defined stop +;; rules referencing startup/package-size/latency thresholds, so release CI +;; needs one command that reports what a packaged app actually weighs and +;; whether it still fits its budgets. Size is fully deterministic here; +;; startup wraps the launch-smoke survival window; per-RPC latency stays +;; with the integration benchmarks (docs/benchmarks.md --benchmark mode), +;; which already measure the real embedded runtime end to end. + +(require file/glob + json + racket/file + racket/list + racket/format + racket/match + racket/path + racket/string + "launch-smoke.rkt" + "project.rkt") + +(provide measure-size + measure-startup + parse-budget-file) + +(define dist-directory "dist") + +(define (dist-artifacts root) + (define dist (build-path root dist-directory)) + (if (directory-exists? dist) + (sort + (for/list ([path (in-list (directory-list dist #:build? #t))] + #:when (file-exists? path)) + path) + #:key path->string + string= mb 1.0) + (format "~a (~a MiB)" count (~r mb #:precision 1)) + (format "~a bytes" count))) + +;; Budget files are JSON: {"rules": [{"glob": "*.dmg", "max-bytes": +;; 100000000}, ...]}. The first matching rule wins, so list the most +;; specific glob first. +(define (parse-budget-file path) + (unless (file-exists? path) + (error 'measure "budget file ~a does not exist" path)) + (define document + (with-handlers ([exn:fail? + (lambda (e) + (error 'measure "budget file ~a is not valid JSON: ~a" + path (exn-message e)))]) + (call-with-input-file path read-json))) + (define rules (hash-ref document 'rules (lambda () #f))) + (unless (and (list? rules) + (for/and ([rule (in-list rules)]) + (and (hash? rule) + (string? (hash-ref rule 'glob (lambda () #f))) + (exact-positive-integer? + (hash-ref rule 'max-bytes (lambda () #f)))))) + (error 'measure + "budget file ~a must contain {\"rules\": [{\"glob\": ..., \"max-bytes\": ...}, ...]}" + path)) + rules) + +(define (budget-for rules artifact) + (define name (path->string (file-name-from-path artifact))) + (for/first ([rule (in-list rules)] + #:when (glob-match? (hash-ref rule 'glob) name)) + (hash-ref rule 'max-bytes))) + +(define (measure-size project + #:budget-path [budget-path #f] + #:artifact [artifact-filter #f]) + (define root (rivet-project-root project)) + (define artifacts + (if artifact-filter + (filter (lambda (path) + (equal? (path->string (file-name-from-path path)) + artifact-filter)) + (dist-artifacts root)) + (dist-artifacts root))) + (when (null? artifacts) + (error 'measure + "no packaged artifacts under ~a; run `raco rivet package` or `raco rivet release` first" + (build-path root dist-directory))) + (define rules (and budget-path (parse-budget-file budget-path))) + (define violations '()) + (define rows + (for/list ([artifact (in-list artifacts)]) + (define bytes (file-size artifact)) + (define budget (and rules (budget-for rules artifact))) + (when (and budget (> bytes budget)) + (set! violations + (cons (cons artifact (cons bytes budget)) violations))) + (hash 'name (path->string (file-name-from-path artifact)) + 'bytes bytes + 'budget-bytes budget))) + (hash 'artifacts (reverse rows) + 'violations (reverse violations))) + +(define (display-size-report report) + (for ([row (in-list (hash-ref report 'artifacts))]) + (define budget (hash-ref row 'budget-bytes)) + (printf " ~a: ~a~a\n" + (hash-ref row 'name) + (format-bytes (hash-ref row 'bytes)) + (if budget + (format " [budget ~a]" (format-bytes budget)) + ""))) + (for ([violation (in-list (hash-ref report 'violations))]) + (define artifact (car violation)) + (define bytes (car (cdr violation))) + (define budget (cdr (cdr violation))) + (printf " OVER BUDGET: ~a is ~a against a ~a budget\n" + (path->string (file-name-from-path artifact)) + (format-bytes bytes) + (format-bytes budget)))) + +(define (measure-startup project + #:artifact [artifact-filter #f] + #:seconds [seconds 5.0]) + (define root (rivet-project-root project)) + (define artifacts + (if artifact-filter + (filter (lambda (path) + (equal? (path->string (file-name-from-path path)) + artifact-filter)) + (dist-artifacts root)) + (dist-artifacts root))) + (when (null? artifacts) + (error 'measure + "no packaged artifacts under ~a; run `raco rivet package` first" + (build-path root dist-directory))) + ;; The most recent packaged GUI artifact is the smoke subject; dist keeps + ;; manifests and SBOMs too, so pick the largest executable-like file. + (define artifact + (argmax file-size + (filter (lambda (path) + (not (regexp-match? #rx"[.](json|txt|sha256|sig|zip)$" + (path->string path)))) + artifacts))) + (define started (current-inexact-milliseconds)) + (launch-smoke! artifact + #:seconds seconds + #:who 'measure-startup) + (define elapsed (/ (- (current-inexact-milliseconds) started) 1000.0)) + (hash 'artifact (path->string (file-name-from-path artifact)) + 'survival-seconds elapsed + 'required-seconds seconds)) + +(define (display-startup-report report) + (printf " ~a survived its ~a second launch window (~a elapsed)\n" + (hash-ref report 'artifact) + (hash-ref report 'required-seconds) + (~r (hash-ref report 'survival-seconds) #:precision 2))) + +(define (run-measure args) + (define find-project + (dynamic-require "project.rkt" 'find-project)) + (define project + (or (find-project) + (error 'rivet "no rivet.rktd found in this directory or its parents"))) + (match args + [(list "size") + (define report (measure-size project)) + (display-size-report report) + (unless (null? (hash-ref report 'violations)) + (error 'measure "size budgets exceeded; see the OVER BUDGET lines above"))] + [(list "size" "--budget" budget-path) + (define report (measure-size project #:budget-path budget-path)) + (display-size-report report) + (unless (null? (hash-ref report 'violations)) + (error 'measure "size budgets exceeded; see the OVER BUDGET lines above"))] + [(list "size" "--artifact" artifact) + (display-size-report (measure-size project #:artifact artifact))] + [(list "startup") + (display-startup-report (measure-startup project))] + [(list "startup" "--artifact" artifact) + (display-startup-report (measure-startup project #:artifact artifact))] + [(list "startup" "--seconds" seconds-text) + (define seconds + (with-handlers ([exn:fail? (lambda (_) #f)]) + (string->number seconds-text))) + (unless (and (real? seconds) (positive? seconds)) + (error 'measure "--seconds requires a positive number")) + (display-startup-report (measure-startup project #:seconds seconds))] + [_ + (error 'measure + "unknown measurement ~a; use `size [--budget ] [--artifact ]` or `startup [--artifact ] [--seconds N]`" + (string-join (if (null? args) '("(none)") args) " "))])) + +(provide run-measure) diff --git a/rivet/tests/measure.rkt b/rivet/tests/measure.rkt new file mode 100644 index 0000000..052e046 --- /dev/null +++ b/rivet/tests/measure.rkt @@ -0,0 +1,100 @@ +#lang racket/base + +(require json + racket/file + racket/port + racket/string + rackunit + "../../rivet-cli/measure.rkt" + "../../rivet-cli/project.rkt" + "../../rivet-cli/scaffold.rkt") + +;; The measurement surface is the dist/ tree, so the fixtures are plain +;; files standing in for packaged artifacts — no native packaging needed. + +(define temp-root (make-temporary-file "rivet-measure-~a" 'directory)) +(dynamic-wind + void + (lambda () + (define project-root (create-project! "measured" temp-root)) + (define project (load-project project-root)) + (define dist (build-path project-root "dist")) + (make-directory* dist) + (define (stage-artifact name bytes) + (call-with-output-file (build-path dist name) + (lambda (out) (write-bytes (make-bytes bytes 65) out)))) + (stage-artifact "measured-0.1.0-macos.dmg" 3000) + (stage-artifact "measured-0.1.0-macos-arm64.zip" 1500) + (stage-artifact "update-stable.json" 40) + + ;; No budgets: every artifact is listed with its byte count and no + ;; violations exist. + (define plain (measure-size project)) + (check-equal? (length (hash-ref plain 'artifacts)) 3) + (check-equal? (hash-ref plain 'violations) '()) + (check-true + (for/and ([row (in-list (hash-ref plain 'artifacts))]) + (and (exact-positive-integer? (hash-ref row 'bytes)) + (not (hash-ref row 'budget-bytes))))) + + ;; A budget file gates by glob, first match wins. + (define budget-path (build-path temp-root "budgets.json")) + (call-with-output-file budget-path + (lambda (out) + (write-json + (hasheq 'rules + (list (hasheq 'glob "*.dmg" 'max-bytes 2000) + (hasheq 'glob "*" 'max-bytes 100000))) + out))) + (define gated (measure-size project #:budget-path budget-path)) + (check-equal? (length (hash-ref gated 'violations)) 1) + (define violation (car (hash-ref gated 'violations))) + (check-true + (string-suffix? + (path->string (car violation)) "measured-0.1.0-macos.dmg")) + (check-equal? (cdr violation) (cons 3000 2000)) + + ;; A tight global budget flags everything; relaxing it clears the gate. + (call-with-output-file budget-path + #:exists 'truncate + (lambda (out) + (write-json + (hasheq 'rules (list (hasheq 'glob "*" 'max-bytes 200000))) + out))) + (check-equal? (hash-ref (measure-size project #:budget-path budget-path) + 'violations) + '()) + + ;; A clean budget file without dist artifacts is an error, not an + ;; empty pass: measuring nothing would read as a green gate. + (define empty-root (build-path temp-root "empty")) + (make-directory* empty-root) + (call-with-output-file (build-path empty-root "rivet.rktd") + (lambda (out) + (displayln + "#hasheq((name . \"Empty\") (backend . \"app/backend.rkt\") (module . \"backend\") (entry . \"start\") (protocol . 1))" + out))) + (check-exn #rx"no packaged artifacts" + (lambda () (measure-size (load-project empty-root)))) + + ;; Malformed budget files fail loudly instead of gating nothing. + (define malformed (build-path temp-root "malformed.json")) + (call-with-output-file malformed + (lambda (out) (display "{\"rules\": [{\"glob\": \"*.dmg\"}]}" out))) + (check-exn #rx"must contain" + (lambda () (parse-budget-file malformed))) + (check-exn #rx"budget file .* does not exist" + (lambda () (parse-budget-file + (build-path temp-root "absent.json")))) + + ;; The artifact filter narrows a single row. + (define filtered + (measure-size project #:artifact "update-stable.json")) + (check-equal? (length (hash-ref filtered 'artifacts)) 1) + (check-equal? + (hash-ref (car (hash-ref filtered 'artifacts)) 'bytes) + 40)) + + (lambda () + (when (directory-exists? temp-root) + (delete-directory/files temp-root))))