diff --git a/README.md b/README.md index d16ec40..c439e05 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,8 @@ API unwraps it. A header whose key ID, label, wrapped DEK, or wrapped length was changed but still parses is usually refused by the API (`not_found`, `key_not_allowed`, or `invalid_ciphertext`), which gives exit code 1. A change that makes the header invalid, such as a label or wrapped DEK that is no -longer UTF-8 or a length out of range, gives exit code 4, as does any change -to the body. +longer UTF-8, a wrapped DEK that is no longer base64, or a length out of +range, gives exit code 4, as does any change to the body. ## Operational notes @@ -166,7 +166,7 @@ All integers are big-endian. | label length | 2 bytes | 0 to 1024 | | label | variable | UTF-8 | | wrapped length | 4 bytes | Length of the next field, 1 to 65536 | -| wrapped DEK | variable | The API's `wrapped` string, byte for byte | +| wrapped DEK | variable | The API's `wrapped` value, in the standard padded base64 the API sends | The body is a sequence of chunks, each the AEAD ciphertext of one plaintext chunk followed by its 16-byte tag. Every chunk but the last holds exactly @@ -187,10 +187,12 @@ go run scripts/probe.go # record live wrap/unwrap shapes, see docs/api-not ``` The unit tests cover the file format, including tamper and header-validation -cases. The client and command tests run against an in-memory mock of the API +cases. The command tests run against an in-memory mock of the API built with `net/http/httptest`, selected with `RYPT_API_URL`. The integration tests build the binary, run full round trips against the live API, and skip when `RYPT_TOKEN` or `RYPT_KEY_ID` is unset. They never print the token. +The API client itself is the [github.com/x6c-co/rypt](https://github.com/x6c-co/rypt) +module, which has its own tests. [docs/api-notes.md](docs/api-notes.md) records the observed behaviour of the `wrap` and `unwrap` endpoints. diff --git a/docs/api-notes.md b/docs/api-notes.md index bec4681..6f4c712 100644 --- a/docs/api-notes.md +++ b/docs/api-notes.md @@ -76,9 +76,17 @@ Response `200`, when no `dek` was sent: not document its internal structure. The length can differ on other tiers (`free` and `wrapped` keys use a rypt-held data key under a shared Cloud KMS root), so the program treats `wrapped` as an opaque string. (openapi) -- Program decision: store `wrapped` byte for byte as UTF-8, never decode it, - and refuse a `wrap` response whose `wrapped` is empty or longer than 65,536 - bytes, the header's limit. The observed 188 characters are far below that. +- Program decision: store `wrapped` in the header as the standard padded + base64 the API sends, and refuse a `wrap` response whose `wrapped` is empty + or longer than 65,536 bytes, the header's limit. The observed 188 + characters are far below that. The client library, + [github.com/x6c-co/rypt](https://github.com/x6c-co/rypt), hands `wrapped` + over as decoded bytes, so the program encodes it again for the header and + decodes the header's value before `unwrap`. An encoder writes one + standard padded base64 string for each byte string, so the header holds + the API's string exactly. A header whose `wrapped` value is not standard padded base64 + cannot have come from the API, and is refused with exit code 4 before any + API call. ## unwrap @@ -164,7 +172,7 @@ Status codes for `wrap` and `unwrap`: - No `retry-after` header was observed on any response. (live) - Responses from Cloudflare in front of the service (for example a 502 or - 504) may not carry the JSON envelope. The client falls back to the + 504) may not carry the JSON envelope. The client library falls back to the `x-request-id` header and the HTTP status. (inference; not observed) ## Consequences for the program @@ -172,17 +180,17 @@ Status codes for `wrap` and `unwrap`: - A tampered label or `wrapped` DEK in a file header is refused by `unwrap` with `400 invalid_ciphertext`, and a tampered key ID with `404 not_found` (or `403 key_not_allowed` for an API token restricted to particular keys). - (The OpenAPI description also lists `400 invalid_base64`, which a `wrapped` - value that is no longer base64 would presumably get.) A changed + A `wrapped` value that is no longer base64 never reaches the API: the + program refuses it locally, exit code 4. A changed `wrapped` length that still parses is usually refused the same way. These reach the program as API refusals, exit code 1, before any chunk is decrypted. A change that makes the header fail local validation (a label or `wrapped` - value that is no longer UTF-8, or a length out of range) is exit code 4 + value that is no longer UTF-8 or base64, or a length out of range) is exit code 4 with no API call. With the correct DEK, every one of these changes also fails chunk authentication, because the whole header is the chunk associated data; the format tests check every header bit that way. -- The client retries a connection error and HTTP 502, 503, and 504, at most - twice. It does not retry a TLS certificate or protocol error, which would +- The client library retries a connection error and HTTP 502, 503, and 504 + on `wrap` and `unwrap`, at most twice. It does not retry a TLS certificate or protocol error, which would fail the same way again, or any other status. - Rotating a key so that the version a `wrapped` value names is no longer enabled makes `unwrap` return `400 invalid_ciphertext`. (openapi) diff --git a/go.mod b/go.mod index e54ab26..a5172f8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/x6c-co/rypt-cli go 1.27.0 require ( + github.com/x6c-co/rypt v0.1.0 golang.org/x/crypto v0.58.0 golang.org/x/sys v0.49.0 golang.org/x/term v0.47.0 diff --git a/go.sum b/go.sum index dca889c..3478441 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/x6c-co/rypt v0.1.0 h1:Te7nRwj4bHQHjLQKjqm3LBP0MlEgHzBe0V3dkBkc6Yk= +github.com/x6c-co/rypt v0.1.0/go.mod h1:CdxA8wPiJuR0J/k1h+H5UTdoaoLYJOWmm7HhVtyWFk0= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 63b131f..854e512 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "io" + "strings" "time" - "github.com/x6c-co/rypt-cli/internal/client" + "github.com/x6c-co/rypt" + "github.com/x6c-co/rypt-cli/internal/format" ) @@ -116,7 +118,7 @@ func ioError(err error) error { // exitCode maps an error to its exit code. func exitCode(err error) int { var ee *exitError - var ce *client.Error + var ce *rypt.Error switch { case err == nil: return ExitOK @@ -186,8 +188,10 @@ func Run(ctx context.Context, env Env) int { // Tests set it to avoid sleeping. var clientBackoff func(retry int) time.Duration -// newClient returns an API client configured from the environment. -func newClient(env Env) (*client.Client, error) { +// newClient returns an API client configured from the environment. It makes +// at most three attempts at each call, retrying only what is safe to repeat; +// wrap and unwrap always are. +func newClient(env Env) (*rypt.Client, error) { token := env.Getenv("RYPT_TOKEN") if token == "" { return nil, usageErrorf("RYPT_TOKEN is not set") @@ -197,14 +201,18 @@ func newClient(env Env) (*client.Client, error) { return nil, usageErrorf("RYPT_TOKEN contains a character that is not printable ASCII") } } - c, err := client.New(client.Options{ - BaseURL: env.Getenv("RYPT_API_URL"), - Token: token, - UserAgent: "rypt-cli/" + env.Version, - Backoff: clientBackoff, - }) + opts := []rypt.Option{ + rypt.WithBaseURL(env.Getenv("RYPT_API_URL")), + rypt.WithUserAgent("rypt-cli/" + env.Version), + rypt.WithMaxAttempts(3), + } + if clientBackoff != nil { + opts = append(opts, rypt.WithBackoff(clientBackoff)) + } + c, err := rypt.NewClient(token, opts...) if err != nil { - return nil, usageErrorf("RYPT_API_URL: %v", err) + // The token was checked above, so the base URL is what is wrong. + return nil, usageErrorf("RYPT_API_URL: %s", strings.TrimPrefix(err.Error(), "rypt: ")) } return c, nil } diff --git a/internal/cli/decrypt.go b/internal/cli/decrypt.go index 78fe474..33671ef 100644 --- a/internal/cli/decrypt.go +++ b/internal/cli/decrypt.go @@ -2,6 +2,7 @@ package cli import ( "context" + "encoding/base64" "errors" "fmt" "io" @@ -71,7 +72,15 @@ func runDecrypt(ctx context.Context, env Env, opts *options) error { if len(h.Label) > 0 { aad = h.Label } - dek, err := api.Unwrap(ctx, format.FormatKeyID(h.KeyID), string(h.Wrapped), aad) + // A wrapped DEK that is not base64 cannot have come from the API: the + // header is damaged, and no operation is spent finding that out. + // The decoder skips line breaks, so a field of only line breaks decodes + // to nothing. + wrapped, err := base64.StdEncoding.AppendDecode(nil, h.Wrapped) + if err != nil || len(wrapped) == 0 { + return fmt.Errorf("%s: the wrapped data key in the header is not base64: %w", describe(opts.input), format.ErrIntegrity) + } + dek, err := api.Unwrap(ctx, format.FormatKeyID(h.KeyID), wrapped, aad) if err != nil { return fmt.Errorf("unwrap: %w", err) } diff --git a/internal/cli/encrypt.go b/internal/cli/encrypt.go index 958fd97..2cb2db4 100644 --- a/internal/cli/encrypt.go +++ b/internal/cli/encrypt.go @@ -2,6 +2,7 @@ package cli import ( "context" + "encoding/base64" "fmt" "io" "unicode/utf8" @@ -87,22 +88,24 @@ func runEncrypt(ctx context.Context, env Env, opts *options) (err error) { if opts.label != "" { label = []byte(opts.label) } - wrapped, err := api.Wrap(ctx, format.FormatKeyID(keyID), label) + dk, err := api.GenerateDataKey(ctx, format.FormatKeyID(keyID), label) if err != nil { return fmt.Errorf("wrap: %w", err) } - dek := wrapped.DEK + dek := dk.Plaintext defer clear(dek) if len(dek) != format.KeySize { return &exitError{code: ExitAPI, err: fmt.Errorf("wrap: the API returned a %d-byte data key, not %d", len(dek), format.KeySize)} } + // The header holds the wrapped DEK as the API sends it: standard padded + // base64. h := &format.Header{ Cipher: cipher, ChunkSize: format.DefaultChunkSize, KeyID: keyID, Label: label, - Wrapped: []byte(wrapped.Wrapped), + Wrapped: base64.StdEncoding.AppendEncode(nil, dk.Wrapped), } if _, err := h.Marshal(); err != nil { // The client bounds the wrapped DEK, so this means the API returned a diff --git a/internal/cli/integrity_test.go b/internal/cli/integrity_test.go index e85f420..af5eb7e 100644 --- a/internal/cli/integrity_test.go +++ b/internal/cli/integrity_test.go @@ -179,6 +179,10 @@ func TestHeaderRejected(t *testing.T) { return ct }}, {"wrapped invalid UTF-8", rebuild(func(h *specHeader) { h.Wrapped = append([]byte("CiQA"), 0xc3, 0x28) })}, + // The API sends wrapped as base64, so anything else is damage. + {"wrapped not base64", rebuild(func(h *specHeader) { h.Wrapped = []byte("CiQA!!!!") })}, + {"wrapped unpadded base64", rebuild(func(h *specHeader) { h.Wrapped = []byte("CiQAWQ") })}, + {"wrapped newline only", rebuild(func(h *specHeader) { h.Wrapped = []byte("\n") })}, {"wrapped length past the end", func(ct []byte, hdr specHeader) []byte { n := hdr.offWrappedLen() binary.BigEndian.PutUint32(ct[n:], 60000) diff --git a/internal/client/client.go b/internal/client/client.go deleted file mode 100644 index 250d09c..0000000 --- a/internal/client/client.go +++ /dev/null @@ -1,332 +0,0 @@ -// Package client is a minimal client for the rypt API's wrap and unwrap -// operations. See docs/api-notes.md for the observed request and response -// shapes. -package client - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -// DefaultBaseURL is the production API. -const DefaultBaseURL = "https://api.rypt.dev" - -// RequestTimeout bounds each HTTP request, including reading the response. -const RequestTimeout = 30 * time.Second - -// MaxAttempts is the number of times a request is sent: one try and up to -// two retries. -const MaxAttempts = 3 - -const ( - // dekLen is the only DEK length the program accepts. - dekLen = 32 - // maxWrappedLen is the largest "wrapped" value a file header can hold - // (the header's wrapped length limit). - maxWrappedLen = 65536 - // maxResponseBytes caps how much of a response body is read. - maxResponseBytes = 1 << 20 - // defaultUserAgent is sent when Options.UserAgent is empty. - defaultUserAgent = "rypt" -) - -// Options configures a Client. -type Options struct { - // BaseURL is the API base URL. Empty means DefaultBaseURL. It is - // validated with ParseBaseURL. - BaseURL string - // Token is the API token. It is sent only in the Authorization header. - Token string - // UserAgent is sent in the User-Agent header. - UserAgent string - // HTTPClient, if not nil, replaces the default client, which has a - // RequestTimeout timeout and does not follow redirects. Tests use it. - HTTPClient *http.Client - // Backoff, if not nil, returns the delay before retry n (1 or 2). Tests - // use it to avoid sleeping. - Backoff func(retry int) time.Duration -} - -// Client calls the rypt API. -type Client struct { - base *url.URL - token string - userAgent string - http *http.Client - backoff func(retry int) time.Duration -} - -// WrapResult is the result of a successful wrap. -type WrapResult struct { - // DEK is the 32-byte data key the server generated. The caller zeroes - // it when done. - DEK []byte - // Wrapped is the API's "wrapped" string, exactly as received. - Wrapped string -} - -// Kind classifies an Error. -type Kind int - -const ( - // KindAPI means the API answered with an error status. - KindAPI Kind = iota + 1 - // KindTransport means the API could not be reached, or the connection - // failed before a complete response arrived. - KindTransport - // KindResponse means the API answered 200 with a body that does not - // have the expected shape. - KindResponse -) - -// Error is the only error type Wrap and Unwrap return, apart from ctx.Err() -// (context.Canceled or context.DeadlineExceeded) when ctx is done. All text -// that came from the server has had control characters removed. -type Error struct { - Kind Kind - StatusCode int // HTTP status; 0 for KindTransport - Code string // API "error" code; may be empty - Message string // API "message", or a description of the failure - RequestID string // API "request_id", else the x-request-id header - Cap int64 // "cap" of ops_cap_reached - ResetsAt string // "resets_at" of ops_cap_reached - Attempts int // requests sent - Err error // the underlying transport error, for KindTransport -} - -func (e *Error) Error() string { - var b strings.Builder - switch e.Kind { - case KindTransport: - msg := e.Message - if msg == "" { - msg = "cannot reach the rypt API" - } - b.WriteString(msg) - switch { - case e.Attempts == 1: - b.WriteString(" after 1 attempt") - case e.Attempts > 1: - fmt.Fprintf(&b, " after %d attempts", e.Attempts) - } - if e.Err != nil { - // Transport errors can quote bytes the server sent, such as a - // malformed status line. - b.WriteString(": ") - b.WriteString(sanitize(e.Err.Error())) - } - return b.String() - case KindResponse: - fmt.Fprintf(&b, "malformed API response (HTTP %d)", e.StatusCode) - if e.Message != "" { - b.WriteString(": ") - b.WriteString(e.Message) - } - default: - b.WriteString("API error") - if e.Code != "" { - b.WriteString(" ") - b.WriteString(e.Code) - } - fmt.Fprintf(&b, " (HTTP %d)", e.StatusCode) - if e.Message != "" { - b.WriteString(": ") - b.WriteString(e.Message) - } - var limits []string - if e.Cap > 0 { - limits = append(limits, "cap "+strconv.FormatInt(e.Cap, 10)) - } - if e.ResetsAt != "" { - limits = append(limits, "resets at "+e.ResetsAt) - } - if len(limits) > 0 { - b.WriteString("; ") - b.WriteString(strings.Join(limits, ", ")) - } - } - if e.RequestID != "" { - fmt.Fprintf(&b, " (request_id %s)", e.RequestID) - } - return b.String() -} - -func (e *Error) Unwrap() error { return e.Err } - -// ParseBaseURL validates an API base URL. The scheme must be https, or http -// when the host is localhost, 127.0.0.1 or ::1. The URL must have a host and -// no user info, query or fragment. A trailing slash is removed from the -// path. -func ParseBaseURL(raw string) (*url.URL, error) { - if raw == "" { - raw = DefaultBaseURL - } - // The URL is not echoed in errors: it could carry a password. - u, err := url.Parse(raw) - if err != nil { - return nil, errors.New("invalid API base URL") - } - switch { - case u.Opaque != "": - return nil, errors.New("invalid API base URL") - case u.User != nil: - return nil, errors.New("API base URL must not contain user info") - case strings.Contains(raw, "?"): - // Checked on the raw string so that a bare "?" is refused too. - return nil, errors.New("API base URL must not contain a query") - case strings.Contains(raw, "#"): - return nil, errors.New("API base URL must not contain a fragment") - case u.Hostname() == "": - return nil, errors.New("API base URL must have a host") - } - host := u.Host - switch u.Scheme { - case "https": - case "http": - if !isLoopbackName(u.Hostname()) { - return nil, errors.New("API base URL must use https unless the host is localhost, 127.0.0.1 or ::1") - } - // net/http bypasses a proxy only for the exact name "localhost", so - // "LOCALHOST" would send the token through HTTP_PROXY in clear text. - host = strings.ToLower(host) - default: - return nil, errors.New("API base URL must use https") - } - return &url.URL{ - Scheme: u.Scheme, - Host: host, - Path: strings.TrimRight(u.Path, "/"), - RawPath: strings.TrimRight(u.RawPath, "/"), - }, nil -} - -// isLoopbackName reports whether host is one of the names for which plain -// http is allowed. It deliberately matches names, not address ranges. The -// comparison ignores ASCII case only: strings.EqualFold would also accept -// non-ASCII look-alikes such as "localhoſt". -func isLoopbackName(host string) bool { - return strings.ToLower(host) == "localhost" || host == "127.0.0.1" || host == "::1" -} - -// New returns a Client. It returns an error if the base URL is invalid, or -// if the token is empty or contains a control character, which cannot be -// sent in an HTTP header. -func New(opts Options) (*Client, error) { - base, err := ParseBaseURL(opts.BaseURL) - if err != nil { - return nil, err - } - if opts.Token == "" { - return nil, errors.New("API token is empty") - } - // Not a format check: a control character cannot go in an HTTP header, - // and net/http would refuse every attempt. - if strings.ContainsFunc(opts.Token, func(r rune) bool { return r < 0x20 || r == 0x7f }) { - return nil, errors.New("API token contains control characters") - } - c := &Client{ - base: base, - token: opts.Token, - userAgent: opts.UserAgent, - http: opts.HTTPClient, - backoff: opts.Backoff, - } - if c.userAgent == "" { - c.userAgent = defaultUserAgent - } - if c.http == nil { - c.http = defaultHTTPClient() - } - if c.backoff == nil { - c.backoff = defaultBackoff - } - return c, nil -} - -// defaultHTTPClient returns a client with a RequestTimeout timeout that never -// follows redirects, so the token and body are never re-sent to another URL. -func defaultHTTPClient() *http.Client { - var transport *http.Transport - if t, ok := http.DefaultTransport.(*http.Transport); ok { - transport = t.Clone() - } else { - transport = &http.Transport{Proxy: http.ProxyFromEnvironment} - } - transport.Proxy = httpsOnlyProxy(transport.Proxy) - return &http.Client{ - Transport: transport, - Timeout: RequestTimeout, - CheckRedirect: func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse - }, - } -} - -// httpsOnlyProxy returns a proxy function that uses next for https requests -// only. Plain http is only ever allowed to a loopback host, and through a -// proxy the token would travel in clear text. https requests still use -// HTTPS_PROXY, through a CONNECT tunnel that carries the TLS session. -func httpsOnlyProxy(next func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { - return func(r *http.Request) (*url.URL, error) { - if r.URL.Scheme != "https" || next == nil { - return nil, nil - } - return next(r) - } -} - -// defaultBackoff returns the delay before retry n. -func defaultBackoff(retry int) time.Duration { - if retry <= 1 { - return 250 * time.Millisecond - } - return 750 * time.Millisecond -} - -// wrapRequest is the wrap body. It never has a "dek" field: the server -// generates the DEK. An empty aad is omitted, so the body is {}. -type wrapRequest struct { - AAD []byte `json:"aad,omitempty"` -} - -// unwrapRequest is the unwrap body. An empty aad is omitted. -type unwrapRequest struct { - Wrapped string `json:"wrapped"` - AAD []byte `json:"aad,omitempty"` -} - -// Wrap asks the server to generate a DEK and wrap it under keyID. aad is -// sent only when it is not empty. -func (c *Client) Wrap(ctx context.Context, keyID string, aad []byte) (*WrapResult, error) { - rep, err := c.call(ctx, keyID, "wrap", wrapRequest{AAD: aad}) - if err != nil { - return nil, err - } - var res dekFields - wrapped, err := rep.decode(true, &res) - if err != nil { - return nil, err - } - return &WrapResult{DEK: res.DEK, Wrapped: wrapped}, nil -} - -// Unwrap asks the server to unwrap wrapped under keyID and returns the -// 32-byte DEK. aad is sent only when it is not empty. The caller zeroes the -// returned slice when done. -func (c *Client) Unwrap(ctx context.Context, keyID, wrapped string, aad []byte) ([]byte, error) { - rep, err := c.call(ctx, keyID, "unwrap", unwrapRequest{Wrapped: wrapped, AAD: aad}) - if err != nil { - return nil, err - } - var res dekFields - if _, err := rep.decode(false, &res); err != nil { - return nil, err - } - return res.DEK, nil -} diff --git a/internal/client/client_test.go b/internal/client/client_test.go deleted file mode 100644 index c6e4342..0000000 --- a/internal/client/client_test.go +++ /dev/null @@ -1,429 +0,0 @@ -package client - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync/atomic" - "testing" - "time" -) - -func TestRoundTrip(t *testing.T) { - for _, aad := range [][]byte{nil, {}, []byte("label"), bytes.Repeat([]byte{0xff, 0x00}, 512)} { - m := newMockServer(t, "") - c := newTestClient(t, m.URL) - ctx := ctxTimeout(t) - - res, err := c.Wrap(ctx, testKeyID, aad) - if err != nil { - t.Fatalf("Wrap(aad len %d): %v", len(aad), err) - } - if len(res.DEK) != 32 || res.Wrapped == "" { - t.Fatalf("Wrap returned DEK len %d, wrapped len %d", len(res.DEK), len(res.Wrapped)) - } - dek, err := c.Unwrap(ctx, testKeyID, res.Wrapped, aad) - if err != nil { - t.Fatalf("Unwrap(aad len %d): %v", len(aad), err) - } - if !bytes.Equal(dek, res.DEK) { - t.Fatalf("Unwrap returned a different DEK") - } - if n := len(m.recorded()); n != 2 { - t.Fatalf("server saw %d requests, want 2", n) - } - } -} - -func TestRequestShape(t *testing.T) { - m := newMockServer(t, "") - c := newTestClient(t, m.URL) - ctx := ctxTimeout(t) - - plain, err := c.Wrap(ctx, testKeyID, nil) - if err != nil { - t.Fatal(err) - } - labeled, err := c.Wrap(ctx, testKeyID, []byte("label")) - if err != nil { - t.Fatal(err) - } - if _, err := c.Unwrap(ctx, testKeyID, plain.Wrapped, nil); err != nil { - t.Fatal(err) - } - if _, err := c.Unwrap(ctx, testKeyID, labeled.Wrapped, []byte("label")); err != nil { - t.Fatal(err) - } - - wrappedJSON := func(s string) string { b, _ := json.Marshal(s); return string(b) } - want := []struct{ path, body string }{ - {"/v1/keys/" + testKeyID + "/wrap", `{}`}, - {"/v1/keys/" + testKeyID + "/wrap", `{"aad":"bGFiZWw="}`}, - {"/v1/keys/" + testKeyID + "/unwrap", `{"wrapped":` + wrappedJSON(plain.Wrapped) + `}`}, - {"/v1/keys/" + testKeyID + "/unwrap", `{"wrapped":` + wrappedJSON(labeled.Wrapped) + `,"aad":"bGFiZWw="}`}, - } - got := m.recorded() - if len(got) != len(want) { - t.Fatalf("server saw %d requests, want %d", len(got), len(want)) - } - for i, w := range want { - r := got[i] - if r.method != http.MethodPost || r.escapedPath != w.path || r.rawQuery != "" { - t.Errorf("request %d: %s %s?%s, want POST %s", i, r.method, r.escapedPath, r.rawQuery, w.path) - } - if string(r.body) != w.body { - t.Errorf("request %d body = %s, want %s", i, r.body, w.body) - } - for name, value := range map[string]string{ - "Authorization": "Bearer " + testToken, - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": testUA, - } { - if got := r.header.Values(name); len(got) != 1 || got[0] != value { - t.Errorf("request %d header %s = %q, want %q", i, name, got, value) - } - } - // The token goes only in the Authorization header. - for name, values := range r.header { - if name == "Authorization" { - continue - } - for _, v := range values { - if strings.Contains(v, testSecret) { - t.Errorf("request %d header %s carries the token", i, name) - } - } - } - if strings.Contains(r.escapedPath+r.rawQuery+string(r.body), testSecret) { - t.Errorf("request %d carries the token outside the Authorization header", i) - } - } -} - -func TestDefaultUserAgent(t *testing.T) { - m := newMockServer(t, "") - c, err := New(Options{BaseURL: m.URL, Token: testToken}) - if err != nil { - t.Fatal(err) - } - if _, err := c.Wrap(ctxTimeout(t), testKeyID, nil); err != nil { - t.Fatal(err) - } - if ua := m.recorded()[0].header.Get("User-Agent"); ua != defaultUserAgent { - t.Errorf("User-Agent = %q, want %q", ua, defaultUserAgent) - } -} - -func TestBasePathPrefix(t *testing.T) { - for _, base := range []string{"/api/rypt", "/api/rypt/"} { - m := newMockServer(t, "/api/rypt") - c := newTestClient(t, m.URL+base) - res, err := c.Wrap(ctxTimeout(t), testKeyID, nil) - if err != nil { - t.Fatalf("base %q: Wrap: %v", base, err) - } - if _, err := c.Unwrap(ctxTimeout(t), testKeyID, res.Wrapped, nil); err != nil { - t.Fatalf("base %q: Unwrap: %v", base, err) - } - got := m.recorded() - if got[0].escapedPath != "/api/rypt/v1/keys/"+testKeyID+"/wrap" || - got[1].escapedPath != "/api/rypt/v1/keys/"+testKeyID+"/unwrap" { - t.Errorf("base %q: paths %q and %q", base, got[0].escapedPath, got[1].escapedPath) - } - } -} - -func TestKeyIDIsPathEscaped(t *testing.T) { - m := newMockServer(t, "") - c := newTestClient(t, m.URL) - _, err := c.Wrap(ctxTimeout(t), "a/b?c#d e", nil) - if e := asError(t, err); e.Kind != KindAPI || e.StatusCode != http.StatusNotFound { - t.Fatalf("got %v, want a 404 KindAPI error", err) - } - got := m.recorded() - if len(got) != 1 || got[0].escapedPath != "/v1/keys/a%2Fb%3Fc%23d%20e/wrap" || got[0].rawQuery != "" { - t.Fatalf("recorded %+v", got) - } -} - -func TestAPIErrorsFromMock(t *testing.T) { - m := newMockServer(t, "") - c := newTestClient(t, m.URL) - ctx := ctxTimeout(t) - - res, err := c.Wrap(ctx, testKeyID, []byte("label")) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - name string - call func() error - status int - code string - }{ - {"aad mismatch", func() error { - _, err := c.Unwrap(ctx, testKeyID, res.Wrapped, []byte("other")) - return err - }, http.StatusBadRequest, "invalid_ciphertext"}, - {"aad missing", func() error { - _, err := c.Unwrap(ctx, testKeyID, res.Wrapped, nil) - return err - }, http.StatusBadRequest, "invalid_ciphertext"}, - {"tampered wrapped", func() error { - // Still valid base64, but a different ciphertext. - b := []byte(res.Wrapped) - if b[20] == 'A' { - b[20] = 'B' - } else { - b[20] = 'A' - } - _, err := c.Unwrap(ctx, testKeyID, string(b), []byte("label")) - return err - }, http.StatusBadRequest, "invalid_ciphertext"}, - {"unknown key", func() error { - _, err := c.Wrap(ctx, "00000000-0000-4000-8000-000000000000", nil) - return err - }, http.StatusNotFound, "not_found"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - before := len(m.recorded()) - err := tt.call() - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != tt.status || e.Code != tt.code || e.Attempts != 1 { - t.Fatalf("got %+v", e) - } - if e.RequestID == "" || !strings.Contains(err.Error(), "(request_id "+e.RequestID+")") { - t.Errorf("Error() = %q, want request_id %q", err, e.RequestID) - } - if !strings.Contains(err.Error(), tt.code) { - t.Errorf("Error() = %q, want the code %q", err, tt.code) - } - if n := len(m.recorded()) - before; n != 1 { - t.Errorf("sent %d requests, want 1", n) - } - assertNoSecrets(t, err, res.Wrapped) - }) - } - - t.Run("wrong token", func(t *testing.T) { - bad, err := New(Options{BaseURL: m.URL, Token: "ry_Wrong000_" + strings.Repeat("x", 32), Backoff: noBackoff}) - if err != nil { - t.Fatal(err) - } - _, err = bad.Wrap(ctx, testKeyID, nil) - if e := asError(t, err); e.Kind != KindAPI || e.StatusCode != http.StatusUnauthorized || e.Code != "unauthorized" { - t.Fatalf("got %+v", e) - } - if strings.Contains(err.Error(), "xxxxxxxx") { - t.Errorf("Error() = %q contains the token", err) - } - }) -} - -func TestDefaultHTTPClient(t *testing.T) { - c, err := New(Options{Token: testToken}) - if err != nil { - t.Fatal(err) - } - if c.base.String() != DefaultBaseURL { - t.Errorf("base = %q, want %q", c.base, DefaultBaseURL) - } - if c.http.Timeout != RequestTimeout { - t.Errorf("Timeout = %v, want %v", c.http.Timeout, RequestTimeout) - } - if c.http.CheckRedirect == nil || !errors.Is(c.http.CheckRedirect(nil, nil), http.ErrUseLastResponse) { - t.Error("CheckRedirect does not stop redirects") - } - tr, ok := c.http.Transport.(*http.Transport) - if !ok || tr == http.DefaultTransport || tr.Proxy == nil { - t.Fatalf("Transport = %T %p; want a clone of http.DefaultTransport with a proxy func", c.http.Transport, c.http.Transport) - } - // Plain http never goes through a proxy, whatever the environment says. - // (The environment is read once per process, so cmd/rypt tests that.) - for _, raw := range []string{"http://localhost:8080/v1/keys/k/wrap", "http://127.0.0.1/x", "http://[::1]/x"} { - req := httptest.NewRequest("POST", raw, nil) - if p, err := tr.Proxy(req); p != nil || err != nil { - t.Errorf("Proxy(%s) = %v, %v; want no proxy", raw, p, err) - } - } - if got := []time.Duration{c.backoff(1), c.backoff(2)}; got[0] != 250*time.Millisecond || got[1] != 750*time.Millisecond { - t.Errorf("default backoff = %v", got) - } - - custom := &http.Client{} - c, err = New(Options{Token: testToken, HTTPClient: custom}) - if err != nil { - t.Fatal(err) - } - if c.http != custom { - t.Error("Options.HTTPClient was not used") - } -} - -// TestHTTPSOnlyProxy: the environment's proxy is used for https and never -// for plain http, where it would see the token in clear text. -func TestHTTPSOnlyProxy(t *testing.T) { - envProxy := &url.URL{Scheme: "http", Host: "proxy.example:3128"} - proxy := httpsOnlyProxy(func(*http.Request) (*url.URL, error) { return envProxy, nil }) - for _, tt := range []struct { - url string - want *url.URL - }{ - {"https://api.rypt.dev/v1/keys/k/wrap", envProxy}, - {"http://localhost:8080/v1/keys/k/wrap", nil}, - {"http://LOCALHOST:8080/v1/keys/k/wrap", nil}, - {"http://127.0.0.1/x", nil}, - } { - got, err := proxy(httptest.NewRequest("POST", tt.url, nil)) - if got != tt.want || err != nil { - t.Errorf("proxy(%s) = %v, %v; want %v", tt.url, got, err, tt.want) - } - } - if got, err := httpsOnlyProxy(nil)(httptest.NewRequest("POST", "https://api.rypt.dev/", nil)); got != nil || err != nil { - t.Errorf("with no proxy function: %v, %v", got, err) - } -} - -func TestRedirectNotFollowed(t *testing.T) { - for _, status := range []int{ - http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, - http.StatusTemporaryRedirect, http.StatusPermanentRedirect, - } { - var elsewhere atomic.Int32 - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - elsewhere.Add(1) - okWrap()(w, r, 1) - })) - t.Cleanup(target.Close) - var hits atomic.Int32 - origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - w.Header().Set("X-Request-Id", "redirect-1") - http.Redirect(w, r, target.URL+r.URL.Path, status) - })) - t.Cleanup(origin.Close) - - // The default HTTP client, as the program uses it. - c, err := New(Options{BaseURL: origin.URL, Token: testToken, Backoff: noBackoff}) - if err != nil { - t.Fatal(err) - } - _, err = c.Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != status || e.Code != "" || e.RequestID != "redirect-1" { - t.Errorf("HTTP %d: got %+v", status, e) - } - if hits.Load() != 1 || elsewhere.Load() != 0 { - t.Errorf("HTTP %d: origin saw %d requests, redirect target %d; want 1 and 0", status, hits.Load(), elsewhere.Load()) - } - assertNoSecrets(t, err) - } -} - -func TestErrorText(t *testing.T) { - tests := []struct { - err *Error - want string - }{ - {&Error{Kind: KindAPI, StatusCode: 429, Code: "ops_cap_reached", Message: "monthly cap reached", RequestID: "abc", Cap: 10000, ResetsAt: "2026-11-01T00:00:00Z"}, - "API error ops_cap_reached (HTTP 429): monthly cap reached; cap 10000, resets at 2026-11-01T00:00:00Z (request_id abc)"}, - {&Error{Kind: KindAPI, StatusCode: 429, Code: "ops_cap_reached", ResetsAt: "2026-11-01T00:00:00Z"}, - "API error ops_cap_reached (HTTP 429); resets at 2026-11-01T00:00:00Z"}, - {&Error{Kind: KindAPI, StatusCode: 502, Message: "unexpected HTTP 502 response", RequestID: "abc"}, - "API error (HTTP 502): unexpected HTTP 502 response (request_id abc)"}, - {&Error{Kind: KindAPI, StatusCode: 404, Code: "not_found", RequestID: "abc"}, - "API error not_found (HTTP 404) (request_id abc)"}, - {&Error{Kind: KindResponse, StatusCode: 200, Message: "dek is not 32 bytes", RequestID: "abc"}, - "malformed API response (HTTP 200): dek is not 32 bytes (request_id abc)"}, - {&Error{Kind: KindTransport, Message: "cannot reach the rypt API", Attempts: 3, Err: errors.New("connection refused")}, - "cannot reach the rypt API after 3 attempts: connection refused"}, - {&Error{Kind: KindTransport, Attempts: 1, Err: errors.New("x509: bad")}, - "cannot reach the rypt API after 1 attempt: x509: bad"}, - {&Error{Kind: KindTransport, Err: errors.New("bad \x1b[2J\u202ething\r\n")}, - "cannot reach the rypt API: bad [2Jthing"}, - } - for _, tt := range tests { - if got := tt.err.Error(); got != tt.want { - t.Errorf("Error() = %q, want %q", got, tt.want) - } - } - inner := errors.New("inner") - if !errors.Is(&Error{Kind: KindTransport, Err: inner}, inner) { - t.Error("Error does not unwrap to Err") - } -} - -func TestWrappedLengthLimit(t *testing.T) { - dek := base64.StdEncoding.EncodeToString(fixedDEK) - s := newScriptServer(t, respond(http.StatusOK, `{"dek":"`+dek+`","wrapped":"`+strings.Repeat("A", maxWrappedLen)+`"}`)) - res, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - if err != nil { - t.Fatalf("a wrapped value of exactly %d bytes: %v", maxWrappedLen, err) - } - if len(res.Wrapped) != maxWrappedLen || !bytes.Equal(res.DEK, fixedDEK) { - t.Fatalf("got wrapped len %d", len(res.Wrapped)) - } -} - -// TestSecretHygiene drives every error path and checks that no error text -// holds the token or a DEK that the server sent. -func TestSecretHygiene(t *testing.T) { - dekB64 := base64.StdEncoding.EncodeToString(fixedDEK) - dek33 := base64.StdEncoding.EncodeToString(append(append([]byte(nil), fixedDEK...), '!')) - closed := httptest.NewServer(http.NotFoundHandler()) - closedURL := closed.URL - closed.Close() - - steps := map[string][]step{ - "401": {respond(401, envelope("unauthorized", "unauthorized", "r"))}, - "429 cap": {respond(429, `{"error":"ops_cap_reached","message":"m","cap":1,"resets_at":"t"}`)}, - "503 x3": {respond(503, "")}, - "html 502": {respond(502, ""+dekB64+"")}, - "hangup": {hangup()}, - "cut body": {cutBody()}, - "short dek": {respond(200, `{"dek":"`+base64.StdEncoding.EncodeToString(fixedDEK[:16])+`","wrapped":"w"}`)}, - "long dek": {respond(200, `{"dek":"`+dek33+`","wrapped":"w"}`)}, - "no wrapped": {respond(200, `{"dek":"`+dekB64+`"}`)}, - "bad json": {respond(200, `{"dek":"`+dekB64+`",`)}, - "error has dek": {respond(400, envelope("invalid_request", "dek "+dekB64, "r"))}, - } - var errs []error - for name, st := range steps { - s := newScriptServer(t, st...) - c := newTestClient(t, s.URL) - if _, err := c.Wrap(ctxTimeout(t), testKeyID, []byte("label")); err != nil { - errs = append(errs, err) - } else { - t.Errorf("%s: Wrap succeeded", name) - } - if _, err := c.Unwrap(ctxTimeout(t), testKeyID, "w", nil); err != nil { - errs = append(errs, err) - } - } - if _, err := newTestClient(t, closedURL).Wrap(ctxTimeout(t), testKeyID, nil); err != nil { - errs = append(errs, err) - } - if _, err := New(Options{Token: testToken + "\n"}); err != nil { - errs = append(errs, err) - } - if len(errs) < len(steps)+2 { - t.Fatalf("only %d errors collected", len(errs)) - } - for _, err := range errs { - // The "error has dek" case is the server's own message; the client - // cannot know it holds a key. Every other path must be clean. - if e, ok := err.(*Error); ok && e.Code == "invalid_request" { - assertNoSecrets(t, err) - continue - } - assertNoSecrets(t, err, dekB64, dek33, string(fixedDEK)) - } -} diff --git a/internal/client/helpers_test.go b/internal/client/helpers_test.go deleted file mode 100644 index e662012..0000000 --- a/internal/client/helpers_test.go +++ /dev/null @@ -1,340 +0,0 @@ -package client - -import ( - "context" - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "mime" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "time" -) - -const ( - // testSecret is the secret part of testToken. Neither may ever appear - // in an error string. - testSecret = "S3cr3tT0k3nMustN3v3rL3akQwErTy12" - testToken = "ry_Tst0Pfx9_" + testSecret - testKeyID = "6f1c2b3a-9d8e-4f70-a1b2-c3d4e5f60718" - testUA = "rypt-test/1.0" - // leakMarker is put into server bodies that must not be echoed. - leakMarker = "LEAKMARKER" -) - -func noBackoff(int) time.Duration { return 0 } - -// newTestClient returns a client for base that never sleeps between retries. -func newTestClient(t *testing.T, base string) *Client { - t.Helper() - c, err := New(Options{BaseURL: base, Token: testToken, UserAgent: testUA, Backoff: noBackoff}) - if err != nil { - t.Fatalf("New: %v", err) - } - return c -} - -// asError returns err as an *Error, failing the test if it is not one. -func asError(t *testing.T, err error) *Error { - t.Helper() - if err == nil { - t.Fatal("got nil error, want *Error") - } - var e *Error - if !errors.As(err, &e) { - t.Fatalf("got %T (%v), want *Error", err, err) - } - return e -} - -// assertNoSecrets fails if err's text holds the token or any of the given -// strings. -func assertNoSecrets(t *testing.T, err error, secrets ...string) { - t.Helper() - if err == nil { - return - } - for _, text := range []string{err.Error(), fmt.Sprintf("%v", err), fmt.Sprintf("%+v", err)} { - for _, s := range append([]string{testToken, testSecret}, secrets...) { - if s != "" && strings.Contains(text, s) { - t.Errorf("error text %q contains a secret (%q)", text, s) - } - } - } -} - -// envelope returns an API error body. -func envelope(code, message, requestID string) string { - b, _ := json.Marshal(map[string]string{"error": code, "message": message, "request_id": requestID}) - return string(b) -} - -// fixedDEK is the DEK the scripted servers return. -var fixedDEK = []byte("0123456789abcdef0123456789abcdef") - -// step answers one request of a scriptServer. n is the 1-based request -// number. -type step func(w http.ResponseWriter, r *http.Request, n int) - -// scriptServer answers request n with steps[n-1]; the last step repeats. -type scriptServer struct { - *httptest.Server - hits atomic.Int32 - arrived chan struct{} // receives once per request, if there is room - - mu sync.Mutex - requests []recorded // every request, in order of arrival -} - -func newScriptServer(t *testing.T, steps ...step) *scriptServer { - t.Helper() - s := &scriptServer{arrived: make(chan struct{}, 16)} - s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - s.mu.Lock() - s.requests = append(s.requests, recorded{ - method: r.Method, - escapedPath: r.URL.EscapedPath(), - rawQuery: r.URL.RawQuery, - header: r.Header.Clone(), - body: body, - }) - s.mu.Unlock() - n := int(s.hits.Add(1)) - select { - case s.arrived <- struct{}{}: - default: - } - steps[min(n, len(steps))-1](w, r, n) - })) - t.Cleanup(s.Close) - return s -} - -func (s *scriptServer) count() int { return int(s.hits.Load()) } - -func (s *scriptServer) recorded() []recorded { - s.mu.Lock() - defer s.mu.Unlock() - return append([]recorded(nil), s.requests...) -} - -// respond writes status and body with a JSON content type and an -// x-request-id header of "hdr-". -func respond(status int, body string) step { - return func(w http.ResponseWriter, r *http.Request, n int) { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("X-Request-Id", fmt.Sprintf("hdr-%d", n)) - w.WriteHeader(status) - _, _ = io.WriteString(w, body) - } -} - -// okWrap answers a wrap or unwrap with fixedDEK. -func okWrap() step { - b, _ := json.Marshal(map[string]any{"dek": fixedDEK, "wrapped": "CiQAwrapped"}) - return respond(http.StatusOK, string(b)) -} - -// hangup closes the connection without answering. -func hangup() step { - return func(w http.ResponseWriter, r *http.Request, n int) { - conn, _, err := w.(http.Hijacker).Hijack() - if err != nil { - panic(err) - } - conn.Close() - } -} - -// cutBody sends a 200 whose Content-Length promises more than is sent, and -// closes the connection. -func cutBody() step { - return func(w http.ResponseWriter, r *http.Request, n int) { - conn, buf, err := w.(http.Hijacker).Hijack() - if err != nil { - panic(err) - } - _, _ = buf.WriteString("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 100\r\n\r\n{\"dek\":\"MDEy") - _ = buf.Flush() - conn.Close() - } -} - -// stall waits until the client goes away. -func stall() step { - return func(w http.ResponseWriter, r *http.Request, n int) { - select { - case <-r.Context().Done(): - case <-time.After(5 * time.Second): - } - } -} - -// recorded is a request seen by mockServer. -type recorded struct { - method string - escapedPath string - rawQuery string - header http.Header - body []byte -} - -// mockServer implements wrap and unwrap with an in-memory AES-256-GCM key, -// following the behavior recorded in docs/api-notes.md. -type mockServer struct { - *httptest.Server - t *testing.T - prefix string - aead cipher.AEAD - - mu sync.Mutex - requests []recorded -} - -func newMockServer(t *testing.T, prefix string) *mockServer { - t.Helper() - key := make([]byte, 32) - if _, err := rand.Read(key); err != nil { - t.Fatal(err) - } - block, err := aes.NewCipher(key) - if err != nil { - t.Fatal(err) - } - aead, err := cipher.NewGCM(block) - if err != nil { - t.Fatal(err) - } - m := &mockServer{t: t, prefix: prefix, aead: aead} - m.Server = httptest.NewServer(http.HandlerFunc(m.serve)) - t.Cleanup(m.Close) - return m -} - -func (m *mockServer) recorded() []recorded { - m.mu.Lock() - defer m.mu.Unlock() - return append([]recorded(nil), m.requests...) -} - -func (m *mockServer) serve(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(io.LimitReader(r.Body, 128<<10+1)) - m.mu.Lock() - m.requests = append(m.requests, recorded{ - method: r.Method, - escapedPath: r.URL.EscapedPath(), - rawQuery: r.URL.RawQuery, - header: r.Header.Clone(), - body: body, - }) - reqID := fmt.Sprintf("mock-%d", len(m.requests)) - m.mu.Unlock() - w.Header().Set("X-Request-Id", reqID) - - fail := func(status int, code, message string) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _, _ = io.WriteString(w, envelope(code, message, reqID)) - } - - rest, ok := strings.CutPrefix(r.URL.Path, m.prefix+"/v1/keys/") - keyID, op, _ := strings.Cut(rest, "/") - if !ok || (op != "wrap" && op != "unwrap") { - fail(http.StatusNotFound, "not_found", "not found") - return - } - if r.Method != http.MethodPost { - w.Header().Set("Allow", "POST") - fail(http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed") - return - } - if r.Header.Get("Authorization") != "Bearer "+testToken { - fail(http.StatusUnauthorized, "unauthorized", "unauthorized") - return - } - if mt, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); mt != "application/json" { - fail(http.StatusUnsupportedMediaType, "unsupported_media_type", "content-type must be application/json") - return - } - if len(body) > 128<<10 { - fail(http.StatusRequestEntityTooLarge, "payload_too_large", "request body too large") - return - } - var fields map[string]json.RawMessage - if json.Unmarshal(body, &fields) != nil { - fail(http.StatusBadRequest, "invalid_request", "invalid JSON") - return - } - allowed := map[string]bool{"aad": true, "wrapped": op == "unwrap"} - for name := range fields { - if !allowed[name] { - // Above all, the client must never send "dek". - m.t.Errorf("%s request has unexpected field %q", op, name) - fail(http.StatusBadRequest, "invalid_request", "unexpected field") - return - } - } - if keyID != testKeyID { - fail(http.StatusNotFound, "not_found", "not found") - return - } - var aad []byte // absent, null and "" all mean no aad - if raw, ok := fields["aad"]; ok && json.Unmarshal(raw, &aad) != nil { - fail(http.StatusBadRequest, "invalid_base64", "aad is not valid base64") - return - } - - w.Header().Set("Content-Type", "application/json") - if op == "wrap" { - dek := make([]byte, 32) - nonce := make([]byte, m.aead.NonceSize()) - _, _ = rand.Read(dek) - _, _ = rand.Read(nonce) - sealed := m.aead.Seal(nonce, nonce, dek, aad) - _ = json.NewEncoder(w).Encode(map[string]any{ - "dek": dek, - "wrapped": base64.StdEncoding.EncodeToString(sealed), - }) - return - } - var wrapped string - if json.Unmarshal(fields["wrapped"], &wrapped) != nil || wrapped == "" { - fail(http.StatusBadRequest, "invalid_request", "wrapped is required") - return - } - sealed, err := base64.StdEncoding.DecodeString(wrapped) - if err != nil { - fail(http.StatusBadRequest, "invalid_base64", "wrapped is not valid base64") - return - } - ns := m.aead.NonceSize() - if len(sealed) < ns { - fail(http.StatusBadRequest, "invalid_ciphertext", "ciphertext or aad invalid") - return - } - dek, err := m.aead.Open(nil, sealed[:ns], sealed[ns:], aad) - if err != nil { - fail(http.StatusBadRequest, "invalid_ciphertext", "ciphertext or aad invalid") - return - } - _ = json.NewEncoder(w).Encode(map[string]any{"dek": dek}) -} - -// ctxTimeout returns a context that fails the test's wait after a while, -// so a bug cannot hang the run. -func ctxTimeout(t *testing.T) context.Context { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - t.Cleanup(cancel) - return ctx -} diff --git a/internal/client/parse_test.go b/internal/client/parse_test.go deleted file mode 100644 index 3fb8746..0000000 --- a/internal/client/parse_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package client - -import ( - "strings" - "testing" -) - -func TestParseBaseURL(t *testing.T) { - ok := []struct{ in, want string }{ - {"", "https://api.rypt.dev"}, - {"https://api.rypt.dev", "https://api.rypt.dev"}, - {"https://api.rypt.dev/", "https://api.rypt.dev"}, - {"HTTPS://api.rypt.dev", "https://api.rypt.dev"}, - {"https://example.com:8443/prefix/", "https://example.com:8443/prefix"}, - {"https://example.com/a/b//", "https://example.com/a/b"}, - {"https://example.com/a%2Fb/", "https://example.com/a%2Fb"}, - {"https://127.0.0.1:9000", "https://127.0.0.1:9000"}, - {"https://[2001:db8::1]:443/x", "https://[2001:db8::1]:443/x"}, - {"http://localhost", "http://localhost"}, - {"http://localhost:8080/", "http://localhost:8080"}, - {"http://LocalHost:8080", "http://localhost:8080"}, - {"http://LOCALHOST", "http://localhost"}, - {"http://127.0.0.1:8080/api", "http://127.0.0.1:8080/api"}, - {"http://[::1]:8080", "http://[::1]:8080"}, - {"http://[::1]", "http://[::1]"}, - } - for _, tt := range ok { - u, err := ParseBaseURL(tt.in) - if err != nil { - t.Errorf("ParseBaseURL(%q): %v", tt.in, err) - continue - } - if got := u.String(); got != tt.want { - t.Errorf("ParseBaseURL(%q) = %q, want %q", tt.in, got, tt.want) - } - if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { - t.Errorf("ParseBaseURL(%q) kept extra parts: %#v", tt.in, u) - } - } - - bad := []string{ - "http://example.com", - "http://api.rypt.dev", - "http://localhost.evil.com", - "http://localhost.", - "http://localhoſt", - "http://LOCALHOſT:8080", - "http://evil.com#@localhost", - "http://127.0.0.2", - "http://127.1", - "http://0.0.0.0:8080", - "http://[::2]", - "http://[::ffff:127.0.0.1]", - "http://[0:0:0:0:0:0:0:1]", - "ftp://api.rypt.dev", - "file:///etc/passwd", - "ws://localhost", - "api.rypt.dev", - "//api.rypt.dev", - "https://", - "https:///path", - "https:api.rypt.dev", - "https://user:hunter2@api.rypt.dev", - "https://user@api.rypt.dev", - "https://:hunter2@api.rypt.dev", - "https://api.rypt.dev?x=1", - "https://api.rypt.dev/?", - "https://api.rypt.dev#frag", - "https://api.rypt.dev/#", - "https://api.rypt.dev:port", - "https://api.rypt.dev\n", - " https://api.rypt.dev", - "https://api rypt.dev", - } - for _, in := range bad { - u, err := ParseBaseURL(in) - if err == nil { - t.Errorf("ParseBaseURL(%q) = %q, want an error", in, u) - continue - } - if strings.Contains(err.Error(), "hunter2") { - t.Errorf("ParseBaseURL(%q) error %q echoes the password", in, err) - } - } -} - -func TestParseBaseURLReturnsCopy(t *testing.T) { - a, _ := ParseBaseURL("") - a.Path = "/changed" - if b, _ := ParseBaseURL(""); b.Path != "" { - t.Error("ParseBaseURL returned shared state") - } -} - -func TestNew(t *testing.T) { - if _, err := New(Options{Token: testToken}); err != nil { - t.Errorf("New with the default base URL: %v", err) - } - if _, err := New(Options{BaseURL: "http://example.com", Token: testToken}); err == nil { - t.Error("New accepted an http base URL that is not loopback") - } - if _, err := New(Options{BaseURL: "http://127.0.0.1:1"}); err == nil { - t.Error("New accepted an empty token") - } - // The token's format is the server's business... - for _, tok := range []string{"x", "not-a-ry-token", strings.Repeat("z", 200), "ry_\u00fcn\u00efcode"} { - if _, err := New(Options{Token: tok}); err != nil { - t.Errorf("New(token %q): %v", tok, err) - } - } - // ...but a control character cannot go in a header. - for _, tok := range []string{testToken + "\n", testToken + "\r", "ry_a\x00b", "ry_a\tb", "ry_\x7f"} { - _, err := New(Options{Token: tok}) - if err == nil { - t.Errorf("New accepted token %q", tok) - continue - } - assertNoSecrets(t, err) - } -} diff --git a/internal/client/request.go b/internal/client/request.go deleted file mode 100644 index 088e8da..0000000 --- a/internal/client/request.go +++ /dev/null @@ -1,361 +0,0 @@ -package client - -import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - "unicode/utf8" -) - -// response is one complete HTTP response. -type response struct { - status int - requestID string // x-request-id header, passed through serverText - body []byte - tooLarge bool // the body exceeded maxResponseBytes and was cut -} - -// reply is the body of a 200 response, for the caller to decode and clear. -type reply struct { - body []byte - requestID string - attempts int -} - -// malformed returns a KindResponse error. msg must be fixed text: it must -// never quote the body, which may hold a DEK. -func (r *reply) malformed(msg string) *Error { - return &Error{ - Kind: KindResponse, - StatusCode: http.StatusOK, - Message: msg, - RequestID: r.requestID, - Attempts: r.attempts, - } -} - -// decode decodes the body into res with decodeDEK and then clears the body. -// It returns the "wrapped" value when wantWrapped, or a KindResponse error. -func (r *reply) decode(wantWrapped bool, res *dekFields) (string, error) { - wrapped, problem := decodeDEK(r.body, wantWrapped, res) - clear(r.body) - if problem != "" { - return "", r.malformed(problem) - } - return wrapped, nil -} - -// call posts payload as JSON to the op endpoint of keyID and returns the -// body of a 200 response. It retries on transport errors and on HTTP 502, -// 503 and 504, up to MaxAttempts requests in total. When ctx is done it -// stops and returns ctx.Err(). -func (c *Client) call(ctx context.Context, keyID, op string, payload any) (*reply, error) { - body, err := json.Marshal(payload) - if err != nil { - // Not reachable with the request types in this package. - return nil, &Error{Kind: KindTransport, Message: "cannot encode the API request"} - } - endpoint := c.base.String() + "/v1/keys/" + url.PathEscape(keyID) + "/" + op - - var last *Error - for attempt := 1; attempt <= MaxAttempts; attempt++ { - if attempt > 1 { - if err := sleep(ctx, c.backoff(attempt-1)); err != nil { - return nil, err - } - } - if err := ctx.Err(); err != nil { - return nil, err - } - - resp, err := c.exchange(ctx, endpoint, body) - if err != nil { - // The caller's ctx decides cancellation. A Client.Timeout error - // can also look like a deadline, but it is a transport failure - // and is retried. - if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr - } - last = &Error{ - Kind: KindTransport, - Message: "cannot reach the rypt API", - Attempts: attempt, - Err: &transportError{err: err, text: c.serverText(err.Error())}, - } - if !retryableErr(err) { - return nil, last - } - continue - } - - if resp.status != http.StatusOK { - last = c.apiError(resp.status, resp.body, resp.requestID) - clear(resp.body) - last.Attempts = attempt - if !retryableStatus(resp.status) { - return nil, last - } - continue - } - if resp.tooLarge { - clear(resp.body) - return nil, (&reply{requestID: resp.requestID, attempts: attempt}).malformed("response too large") - } - return &reply{body: resp.body, requestID: resp.requestID, attempts: attempt}, nil - } - return nil, last -} - -// transportError is the Err of a KindTransport Error. The text of a -// net/http error can quote bytes the server sent, such as a malformed status -// or header line, so it is passed through serverText; errors.Is and -// errors.As still reach err. -type transportError struct { - err error - text string -} - -func (e *transportError) Error() string { return e.text } -func (e *transportError) Unwrap() error { return e.err } - -// exchange sends one request and reads the whole response. A failure to -// read the body is returned as an error, like a failure to connect. -func (c *Client) exchange(ctx context.Context, endpoint string, body []byte) (*response, error) { - // A fresh body reader for every attempt. - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+c.token) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", c.userAgent) - - resp, err := c.http.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - buf, tooLarge, err := readBody(resp.Body, resp.ContentLength) - if err != nil { - clear(buf) - return nil, fmt.Errorf("reading the response: %w", err) - } - return &response{ - status: resp.StatusCode, - requestID: c.serverText(resp.Header.Get("X-Request-Id")), - body: buf, - tooLarge: tooLarge, - }, nil -} - -// readBody reads r to EOF, keeping at most maxResponseBytes+1 bytes. If r -// has more than maxResponseBytes bytes, it stops and reports tooLarge. -// sizeHint is the Content-Length, or -1. The buffer is grown by hand so -// that each outgrown buffer is cleared: the body may hold a DEK. -func readBody(r io.Reader, sizeHint int64) (buf []byte, tooLarge bool, err error) { - size := 4096 - if sizeHint >= 0 && sizeHint < maxResponseBytes { - size = int(sizeHint) + 1 // one spare byte to reach EOF without growing - } - buf = make([]byte, 0, size) - for { - if len(buf) == cap(buf) { - grown := make([]byte, len(buf), min(2*cap(buf), maxResponseBytes+1)) - copy(grown, buf) - clear(buf) - buf = grown - } - n, err := r.Read(buf[len(buf):cap(buf)]) - buf = buf[:len(buf)+n] - if len(buf) > maxResponseBytes { - return buf, true, nil - } - if err == io.EOF { - return buf, false, nil - } - if err != nil { - return buf, false, err - } - } -} - -// sleep waits for d or until ctx is done, whichever comes first. -func sleep(ctx context.Context, d time.Duration) error { - if d <= 0 { - return ctx.Err() - } - t := time.NewTimer(d) - defer t.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-t.C: - return nil - } -} - -// retryableStatus reports whether a response status is worth retrying. -// HTTP 429 is not: neither the monthly cap nor the rate limit clears in the -// time a retry waits. -func retryableStatus(status int) bool { - switch status { - case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: - return true - } - return false -} - -// retryableErr reports whether a failed exchange might succeed if repeated. -// A certificate the client does not trust, or a server that does not speak -// TLS, will fail the same way every time. -func retryableErr(err error) bool { - var ( - verifyErr *tls.CertificateVerificationError - recordErr tls.RecordHeaderError - unknownCA x509.UnknownAuthorityError - hostErr x509.HostnameError - invalidErr x509.CertificateInvalidError - ) - switch { - case errors.As(err, &verifyErr), - errors.As(err, &recordErr), - errors.As(err, &unknownCA), - errors.As(err, &hostErr), - errors.As(err, &invalidErr), - errors.Is(err, http.ErrSchemeMismatch): - return false - } - return true -} - -// dekFields is the 200 body of wrap and unwrap. encoding/json decodes the -// standard base64 "dek" into a []byte, so the key never lives in an -// immutable string. "wrapped" is kept raw so that unwrap can ignore it. -type dekFields struct { - DEK []byte `json:"dek"` - Wrapped json.RawMessage `json:"wrapped"` -} - -// decodeDEK decodes a 200 body into res and checks that it holds a 32-byte -// "dek" and, if wantWrapped, a valid "wrapped" string, which it returns. On -// a problem it zeroes res.DEK and returns a fixed description that never -// quotes the body. -func decodeDEK(body []byte, wantWrapped bool, res *dekFields) (wrapped, problem string) { - err := json.Unmarshal(body, res) - switch { - case err != nil: - problem = jsonProblem(err) - case res.DEK == nil: - problem = "dek is missing" - case len(res.DEK) != dekLen: - problem = "dek is not 32 bytes" - case wantWrapped: - wrapped, problem = decodeWrapped(res.Wrapped) - } - if problem != "" { - // A decode error can come after the DEK was decoded. - clear(res.DEK[:cap(res.DEK)]) - return "", problem - } - return wrapped, "" -} - -// decodeWrapped decodes the "wrapped" value of a wrap response. The value is -// opaque; only its type and length are checked. -func decodeWrapped(raw json.RawMessage) (string, string) { - if len(raw) == 0 { - return "", "wrapped is missing or empty" - } - var s string - if err := json.Unmarshal(raw, &s); err != nil { - return "", "wrapped is not a string" - } - switch { - case s == "": - return "", "wrapped is missing or empty" - case strings.ContainsRune(s, utf8.RuneError): - // encoding/json turns invalid UTF-8 and lone surrogates into - // U+FFFD, so s would not be the value the server sent, and the file - // could never be decrypted. - return "", "wrapped is not valid UTF-8" - case len(s) > maxWrappedLen: - return "", "wrapped is longer than 65536 bytes" - } - return s, "" -} - -// jsonProblem describes a decoding error without its text, which can quote -// bytes of the body. -func jsonProblem(err error) string { - var ( - typeErr *json.UnmarshalTypeError - b64Err base64.CorruptInputError - ) - switch { - case errors.As(err, &b64Err): - return "dek is not valid base64" - case errors.As(err, &typeErr) && typeErr.Field == "dek": - return "dek is not a valid base64 string" - case errors.As(err, &typeErr): - return "response is not a JSON object" - } - return "response is not valid JSON" -} - -// apiError builds the KindAPI error for a non-200 response. It decodes the -// API error envelope when the body is one; otherwise, as for an HTML page -// from a proxy, Code stays empty and Message is fixed text. headerID is the -// sanitized x-request-id header. -func (c *Client) apiError(status int, body []byte, headerID string) *Error { - e := &Error{Kind: KindAPI, StatusCode: status, RequestID: headerID} - - // Each field is decoded on its own, so one of an unexpected type does - // not lose the others. - var env map[string]json.RawMessage - if json.Unmarshal(body, &env) == nil { - e.Code = c.jsonText(env["error"]) - } - if e.Code == "" { - e.Message = fmt.Sprintf("unexpected HTTP %d response", status) - return e - } - e.Message = c.jsonText(env["message"]) - if id := c.jsonText(env["request_id"]); id != "" { - e.RequestID = id - } - e.ResetsAt = c.jsonText(env["resets_at"]) - var limit int64 - if json.Unmarshal(env["cap"], &limit) == nil && limit > 0 { - e.Cap = limit - } - return e -} - -// jsonText returns the value of a JSON string passed through serverText, -// or "" if raw is not a string. -func (c *Client) jsonText(raw json.RawMessage) string { - var s string - if json.Unmarshal(raw, &s) != nil { - return "" - } - return c.serverText(s) -} - -// serverText makes server-supplied text safe to keep in an Error. Besides -// sanitizing it, it redacts the token, in case a server echoes it back; that -// happens before truncation so that no part of the token can survive. -func (c *Client) serverText(s string) string { - return truncate(strings.ReplaceAll(clean(s), c.token, "[redacted]")) -} diff --git a/internal/client/response_test.go b/internal/client/response_test.go deleted file mode 100644 index c943038..0000000 --- a/internal/client/response_test.go +++ /dev/null @@ -1,352 +0,0 @@ -package client - -import ( - "bytes" - "encoding/base64" - "errors" - "io" - "net/http" - "strconv" - "strings" - "testing" - "testing/iotest" -) - -// b64 is the standard padded base64 of n bytes of 0xfb, which encodes with -// '+' and '/' characters. -func b64(n int) string { - return base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xfb}, n)) -} - -func TestMalformedOK(t *testing.T) { - dek := b64(32) - urlDEK := base64.URLEncoding.EncodeToString(bytes.Repeat([]byte{0xfb}, 32)) - rawDEK := base64.RawStdEncoding.EncodeToString(bytes.Repeat([]byte{0xfb}, 32)) - tests := []struct { - name string - wrap bool - body string - prefix string // of the expected Message - }{ - {"not JSON", true, "" + leakMarker + " " + dek + "", "response is not valid JSON"}, - {"empty body", true, "", "response is not valid JSON"}, - {"trailing data", true, `{"dek":"` + dek + `","wrapped":"w"} ` + leakMarker, "response is not valid JSON"}, - {"JSON array", true, `["` + leakMarker + `"]`, "response is not a JSON object"}, - {"JSON string", false, `"` + leakMarker + `"`, "response is not a JSON object"}, - {"missing dek", true, `{"wrapped":"w"}`, "dek is missing"}, - {"null dek", true, `{"dek":null,"wrapped":"w"}`, "dek is missing"}, - {"empty dek", true, `{"dek":"","wrapped":"w"}`, "dek is not 32 bytes"}, - {"16-byte dek", true, `{"dek":"` + b64(16) + `","wrapped":"w"}`, "dek is not 32 bytes"}, - {"33-byte dek", true, `{"dek":"` + b64(33) + `","wrapped":"w"}`, "dek is not 32 bytes"}, - {"invalid base64 dek", true, `{"dek":"` + leakMarker + `!!","wrapped":"w"}`, "dek is not"}, - {"url-safe base64 dek", true, `{"dek":"` + urlDEK + `","wrapped":"w"}`, "dek is not"}, - {"unpadded dek", true, `{"dek":"` + rawDEK + `","wrapped":"w"}`, "dek is not"}, - {"numeric dek", true, `{"dek":12345,"wrapped":"w"}`, "dek is not"}, - {"missing wrapped", true, `{"dek":"` + dek + `"}`, "wrapped is missing or empty"}, - {"empty wrapped", true, `{"dek":"` + dek + `","wrapped":""}`, "wrapped is missing or empty"}, - {"null wrapped", true, `{"dek":"` + dek + `","wrapped":null}`, "wrapped is missing or empty"}, - {"numeric wrapped", true, `{"dek":"` + dek + `","wrapped":12345}`, "wrapped is not a string"}, - // encoding/json would turn both into U+FFFD, changing the value. - {"wrapped invalid UTF-8", true, `{"dek":"` + dek + `","wrapped":"CiQA` + "\xff\xfe" + `AA=="}`, "wrapped is not valid UTF-8"}, - {"wrapped lone surrogate", true, `{"dek":"` + dek + `","wrapped":"CiQA\udc00AA=="}`, "wrapped is not valid UTF-8"}, - {"wrapped too long", true, `{"dek":"` + dek + `","wrapped":"` + strings.Repeat("A", maxWrappedLen+1) + `"}`, "wrapped is longer than 65536 bytes"}, - {"unwrap null", false, `null`, "dek is missing"}, - {"unwrap empty object", false, `{}`, "dek is missing"}, - {"unwrap 31-byte dek", false, `{"dek":"` + b64(31) + `"}`, "dek is not 32 bytes"}, - {"unwrap invalid base64 dek", false, `{"dek":"` + leakMarker + `"}`, "dek is not"}, - {"unwrap too large", false, `{"dek":"` + dek + `","pad":"` + strings.Repeat(" ", maxResponseBytes) + `"}`, "response too large"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := newScriptServer(t, respond(http.StatusOK, tt.body), okWrap()) - c := newTestClient(t, s.URL) - var err error - if tt.wrap { - var res *WrapResult - res, err = c.Wrap(ctxTimeout(t), testKeyID, nil) - if res != nil { - t.Error("got a result with the error") - } - } else { - var got []byte - got, err = c.Unwrap(ctxTimeout(t), testKeyID, "w", nil) - if got != nil { - t.Error("got a DEK with the error") - } - } - e := asError(t, err) - if e.Kind != KindResponse || e.StatusCode != http.StatusOK || e.Attempts != 1 || e.RequestID != "hdr-1" { - t.Errorf("got %+v", e) - } - if !strings.HasPrefix(e.Message, tt.prefix) { - t.Errorf("Message = %q, want prefix %q", e.Message, tt.prefix) - } - if !strings.HasPrefix(err.Error(), "malformed API response (HTTP 200): ") || - !strings.HasSuffix(err.Error(), " (request_id hdr-1)") { - t.Errorf("Error() = %q", err) - } - if s.count() != 1 { - t.Errorf("server saw %d requests, want 1", s.count()) - } - assertNoSecrets(t, err, leakMarker, dek, urlDEK, rawDEK, "illegal", "base64 data", "json") - }) - } -} - -func TestResponseSizeLimit(t *testing.T) { - dek := base64.StdEncoding.EncodeToString(fixedDEK) - head := `{"dek":"` + dek + `","wrapped":"w",` - pad := func(total int) string { - // "x":""} padded to total bytes. - return head + `"x":"` + strings.Repeat(" ", total-len(head)-len(`"x":""}`)) + `"}` - } - exact, over := pad(maxResponseBytes), pad(maxResponseBytes+1) - if len(exact) != maxResponseBytes || len(over) != maxResponseBytes+1 { - t.Fatal("bad padding") - } - - // chunked sends the body without a Content-Length. - chunked := func(body string) step { - return func(w http.ResponseWriter, r *http.Request, n int) { - w.Header().Set("X-Request-Id", "hdr") - w.WriteHeader(http.StatusOK) - for b := []byte(body); len(b) > 0; { - k := min(len(b), 4000) - _, _ = w.Write(b[:k]) - w.(http.Flusher).Flush() - b = b[k:] - } - } - } - withLength := func(body string) step { - return func(w http.ResponseWriter, r *http.Request, n int) { - w.Header().Set("Content-Length", strconv.Itoa(len(body))) - w.WriteHeader(http.StatusOK) - _, _ = io.WriteString(w, body) - } - } - for name, st := range map[string]func(string) step{"chunked": chunked, "content-length": withLength} { - s := newScriptServer(t, st(exact)) - if res, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil); err != nil || !bytes.Equal(res.DEK, fixedDEK) { - t.Errorf("%s: a %d-byte response: %v", name, maxResponseBytes, err) - } - s = newScriptServer(t, st(over)) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - if e := asError(t, err); e.Kind != KindResponse || e.Message != "response too large" { - t.Errorf("%s: a %d-byte response: got %+v", name, maxResponseBytes+1, e) - } - if s.count() != 1 { - t.Errorf("%s: server saw %d requests, want 1", name, s.count()) - } - } -} - -func TestReadBody(t *testing.T) { - data := bytes.Repeat([]byte("0123456789"), 2000) - for _, hint := range []int64{-1, 0, 5, int64(len(data)), int64(len(data)) + 10, maxResponseBytes, 1 << 40} { - buf, tooLarge, err := readBody(iotest.OneByteReader(bytes.NewReader(data)), hint) - if err != nil || tooLarge || !bytes.Equal(buf, data) { - t.Errorf("hint %d: got %d bytes, tooLarge %v, err %v", hint, len(buf), tooLarge, err) - } - } - buf, tooLarge, err := readBody(bytes.NewReader(make([]byte, maxResponseBytes+10)), -1) - if err != nil || !tooLarge || len(buf) != maxResponseBytes+1 { - t.Errorf("oversize: got %d bytes, tooLarge %v, err %v", len(buf), tooLarge, err) - } - boom := errors.New("boom") - if _, _, err := readBody(io.MultiReader(bytes.NewReader(data), iotest.ErrReader(boom)), -1); !errors.Is(err, boom) { - t.Errorf("read error: got %v", err) - } -} - -// recordingReader remembers every slice it filled. -type recordingReader struct { - r io.Reader - reads [][]byte -} - -func (rr *recordingReader) Read(p []byte) (int, error) { - n, err := rr.r.Read(p) - rr.reads = append(rr.reads, p[:n]) - return n, err -} - -func TestReadBodyClearsOutgrownBuffers(t *testing.T) { - data := bytes.Repeat([]byte{0xa5}, 50000) - rr := &recordingReader{r: bytes.NewReader(data)} - buf, _, err := readBody(rr, -1) - if err != nil || !bytes.Equal(buf, data) { - t.Fatalf("readBody: %v", err) - } - // Slices that share the final buffer's backing array end where it ends. - end := &buf[:cap(buf)][cap(buf)-1] - outgrown := 0 - for _, p := range rr.reads { - if len(p) == 0 || &p[:cap(p)][cap(p)-1] == end { - continue - } - outgrown++ - if !bytes.Equal(p, make([]byte, len(p))) { - t.Fatal("an outgrown buffer still holds body bytes") - } - } - if outgrown == 0 { - t.Fatal("the buffer never grew; the test checks nothing") - } -} - -func TestDecodeDEKZeroesOnError(t *testing.T) { - for _, body := range []string{ - `{"dek":"` + b64(32) + `"}`, // wrapped missing - `{"dek":"` + b64(32) + `","wrapped":5}`, // wrapped not a string - `{"dek":"` + b64(32) + `","wrapped":""}`, // wrapped empty - `{"dek":"` + b64(32) + `","dek":7}`, // decode error after the DEK - `{"dek":"` + b64(32) + `","wrapped":"w"} x`, // trailing data - `{"dek":"` + b64(33) + `","wrapped":"w"}`, // wrong length - } { - var res dekFields - wrapped, problem := decodeDEK([]byte(body), true, &res) - if problem == "" || wrapped != "" { - t.Errorf("%.40s...: got %q, %q", body, wrapped, problem) - } - if !bytes.Equal(res.DEK, make([]byte, len(res.DEK))) { - t.Errorf("%.40s...: DEK not zeroed after %q", body, problem) - } - } - - var res dekFields - if _, problem := decodeDEK([]byte(`{"dek":"`+b64(32)+`"}`), false, &res); problem != "" || len(res.DEK) != 32 { - t.Errorf("unwrap shape: %q, %d bytes", problem, len(res.DEK)) - } - // An unwrap response may carry other fields of any type. - res = dekFields{} - if _, problem := decodeDEK([]byte(`{"dek":"`+b64(32)+`","wrapped":5,"extra":[1]}`), false, &res); problem != "" { - t.Errorf("extra fields: %q", problem) - } -} - -func TestReplyDecodeClearsBody(t *testing.T) { - tests := []struct { - body string - wantWrapped bool - problem string - }{ - {`{"dek":"` + b64(32) + `","wrapped":"w"}`, true, ""}, - {`{"dek":"` + b64(32) + `"}`, false, ""}, - {`{"dek":"` + b64(32) + `"}`, true, "wrapped is missing or empty"}, - {`{"dek":"` + b64(16) + `"}`, false, "dek is not 32 bytes"}, - {`not JSON ` + b64(32), false, "response is not valid JSON"}, - } - for _, tt := range tests { - rep := &reply{body: []byte(tt.body), requestID: "r", attempts: 2} - var res dekFields - wrapped, err := rep.decode(tt.wantWrapped, &res) - if !bytes.Equal(rep.body, make([]byte, len(rep.body))) { - t.Errorf("%.30s...: the body was not cleared", tt.body) - } - if tt.problem != "" { - e := asError(t, err) - if e.Kind != KindResponse || e.Message != tt.problem || e.RequestID != "r" || e.Attempts != 2 || wrapped != "" { - t.Errorf("%.30s...: got %q, %+v", tt.body, wrapped, e) - } - continue - } - // The DEK must not share memory with the cleared body. - if err != nil || !bytes.Equal(res.DEK, bytes.Repeat([]byte{0xfb}, 32)) { - t.Errorf("%.30s...: got DEK %x, %v", tt.body, res.DEK, err) - } - if tt.wantWrapped && wrapped != "w" { - t.Errorf("%.30s...: wrapped = %q", tt.body, wrapped) - } - } -} - -func TestOpsCapReached(t *testing.T) { - body := `{"error":"ops_cap_reached","message":"monthly operation cap reached","request_id":"req-cap-1",` + - `"cap":10000,"resets_at":"2026-11-01T00:00:00Z"}` - s := newScriptServer(t, respond(http.StatusTooManyRequests, body), okWrap()) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != 429 || e.Code != "ops_cap_reached" || e.Cap != 10000 || - e.ResetsAt != "2026-11-01T00:00:00Z" || e.RequestID != "req-cap-1" || e.Attempts != 1 { - t.Errorf("got %+v", e) - } - want := "API error ops_cap_reached (HTTP 429): monthly operation cap reached; cap 10000, resets at 2026-11-01T00:00:00Z (request_id req-cap-1)" - if err.Error() != want { - t.Errorf("Error() = %q, want %q", err, want) - } - for _, s := range []string{"ops_cap_reached", "2026-11-01T00:00:00Z", "req-cap-1"} { - if !strings.Contains(err.Error(), s) { - t.Errorf("Error() lacks %q", s) - } - } - if s.count() != 1 { - t.Errorf("server saw %d requests, want 1", s.count()) - } -} - -func TestRateLimitedNotRetried(t *testing.T) { - s := newScriptServer(t, respond(http.StatusTooManyRequests, envelope("rate_limited", "slow down", "req-rl")), okWrap()) - _, err := newTestClient(t, s.URL).Unwrap(ctxTimeout(t), testKeyID, "w", nil) - e := asError(t, err) - if e.Code != "rate_limited" || e.StatusCode != 429 || e.Cap != 0 || e.ResetsAt != "" || s.count() != 1 { - t.Errorf("got %+v after %d requests", e, s.count()) - } - if want := "API error rate_limited (HTTP 429): slow down (request_id req-rl)"; err.Error() != want { - t.Errorf("Error() = %q, want %q", err, want) - } -} - -func TestErrorBodies(t *testing.T) { - tests := []struct { - name string - status int - body string - code string - message string - reqID string - cap int64 - resets string - }{ - {"html", 400, "" + leakMarker + "", "", "unexpected HTTP 400 response", "hdr-1", 0, ""}, - {"empty", 404, "", "", "unexpected HTTP 404 response", "hdr-1", 0, ""}, - {"JSON without error", 400, `{"message":"` + leakMarker + `","request_id":"body-id"}`, "", "unexpected HTTP 400 response", "hdr-1", 0, ""}, - {"error not a string", 400, `{"error":{"code":"x"},"message":"` + leakMarker + `"}`, "", "unexpected HTTP 400 response", "hdr-1", 0, ""}, - {"no request_id in body", 404, `{"error":"not_found","message":"not found"}`, "not_found", "not found", "hdr-1", 0, ""}, - {"no message", 404, `{"error":"not_found","request_id":"body-id"}`, "not_found", "", "body-id", 0, ""}, - {"body request_id wins", 409, envelope("key_pending_deletion", "pending", "body-id"), "key_pending_deletion", "pending", "body-id", 0, ""}, - {"cap of wrong type", 429, `{"error":"ops_cap_reached","message":"m","cap":"lots","resets_at":"2026-11-01T00:00:00Z"}`, - "ops_cap_reached", "m", "hdr-1", 0, "2026-11-01T00:00:00Z"}, - {"message of wrong type", 429, `{"error":"ops_cap_reached","message":5,"cap":7}`, "ops_cap_reached", "", "hdr-1", 7, ""}, - {"too large", 400, `{"error":"invalid_request","message":"` + strings.Repeat("a", maxResponseBytes) + `"}`, - "", "unexpected HTTP 400 response", "hdr-1", 0, ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := newScriptServer(t, respond(tt.status, tt.body)) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != tt.status || e.Code != tt.code || e.Message != tt.message || - e.RequestID != tt.reqID || e.Cap != tt.cap || e.ResetsAt != tt.resets { - t.Errorf("got %+v", e) - } - assertNoSecrets(t, err, leakMarker) - }) - } -} - -func TestNonJSONGatewayErrorAfterRetries(t *testing.T) { - s := newScriptServer(t, respond(http.StatusBadGateway, "502 Bad Gateway "+leakMarker+"")) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != 502 || e.Code != "" || e.Attempts != MaxAttempts || e.RequestID != "hdr-3" { - t.Errorf("got %+v", e) - } - if want := "API error (HTTP 502): unexpected HTTP 502 response (request_id hdr-3)"; err.Error() != want { - t.Errorf("Error() = %q, want %q", err, want) - } - if s.count() != MaxAttempts { - t.Errorf("server saw %d requests, want %d", s.count(), MaxAttempts) - } - assertNoSecrets(t, err, leakMarker, "html") -} diff --git a/internal/client/retry_test.go b/internal/client/retry_test.go deleted file mode 100644 index 0c21b6b..0000000 --- a/internal/client/retry_test.go +++ /dev/null @@ -1,531 +0,0 @@ -package client - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/base64" - "errors" - "fmt" - "io" - "log" - "net" - "net/http" - "net/http/httptest" - "strings" - "sync" - "sync/atomic" - "testing" - "testing/iotest" - "time" -) - -func TestRetryOnGatewayStatus(t *testing.T) { - for _, status := range []int{http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} { - t.Run(fmt.Sprint(status), func(t *testing.T) { - s := newScriptServer(t, respond(status, envelope("unavailable", "busy", "r1")), okWrap()) - res, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - if err != nil { - t.Fatalf("Wrap: %v", err) - } - if !bytes.Equal(res.DEK, fixedDEK) { - t.Error("wrong DEK") - } - if s.count() != 2 { - t.Errorf("server saw %d requests, want 2", s.count()) - } - }) - } -} - -func TestRetryTwiceThenSucceed(t *testing.T) { - s := newScriptServer(t, - respond(http.StatusServiceUnavailable, envelope("unavailable", "busy", "r1")), - respond(http.StatusBadGateway, "bad gateway"), - okWrap()) - var retries []int - c := newTestClient(t, s.URL) - c.backoff = func(n int) time.Duration { retries = append(retries, n); return 0 } - - dek, err := c.Unwrap(ctxTimeout(t), testKeyID, "CiQAwrapped", nil) - if err != nil { - t.Fatalf("Unwrap: %v", err) - } - if !bytes.Equal(dek, fixedDEK) { - t.Error("wrong DEK") - } - if s.count() != 3 { - t.Errorf("server saw %d requests, want 3", s.count()) - } - if fmt.Sprint(retries) != "[1 2]" { - t.Errorf("backoff called with %v, want [1 2]", retries) - } -} - -func TestRetryGivesUp(t *testing.T) { - s := newScriptServer(t, respond(http.StatusServiceUnavailable, envelope("unavailable", "queue full", "req-503"))) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != http.StatusServiceUnavailable || e.Attempts != MaxAttempts || - e.Code != "unavailable" || e.Message != "queue full" || e.RequestID != "req-503" { - t.Errorf("got %+v", e) - } - if s.count() != MaxAttempts { - t.Errorf("server saw %d requests, want %d", s.count(), MaxAttempts) - } - want := "API error unavailable (HTTP 503): queue full (request_id req-503)" - if err.Error() != want { - t.Errorf("Error() = %q, want %q", err, want) - } -} - -func TestNoRetryOnOtherStatus(t *testing.T) { - for _, status := range []int{400, 401, 402, 403, 404, 405, 409, 413, 415, 429, 500, 501, 505, 201, 204, 304} { - t.Run(fmt.Sprint(status), func(t *testing.T) { - body := envelope("some_code", "some message", "req-x") - if status == 204 || status == 304 { - body = "" - } - s := newScriptServer(t, respond(status, body), okWrap()) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindAPI || e.StatusCode != status || e.Attempts != 1 { - t.Errorf("got %+v", e) - } - if s.count() != 1 { - t.Errorf("server saw %d requests, want 1", s.count()) - } - }) - } -} - -func TestRetryStopsAtNonRetryableStatus(t *testing.T) { - s := newScriptServer(t, - respond(http.StatusBadGateway, ""), - respond(http.StatusNotFound, envelope("not_found", "not found", "req-404")), - okWrap()) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.StatusCode != http.StatusNotFound || e.Attempts != 2 || s.count() != 2 { - t.Errorf("got %+v after %d requests", e, s.count()) - } -} - -func TestTransportErrors(t *testing.T) { - tests := []struct { - name string - steps []step - ok bool - hits int - }{ - {"hangup", []step{hangup()}, false, 3}, - {"hangup once", []step{hangup(), okWrap()}, true, 2}, - {"hangup twice", []step{hangup(), hangup(), okWrap()}, true, 3}, - {"body cut short", []step{cutBody()}, false, 3}, - {"body cut short once", []step{cutBody(), okWrap()}, true, 2}, - {"503 then hangups", []step{respond(503, ""), hangup()}, false, 3}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := newScriptServer(t, tt.steps...) - res, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - if s.count() != tt.hits { - t.Errorf("server saw %d requests, want %d", s.count(), tt.hits) - } - if tt.ok { - if err != nil || !bytes.Equal(res.DEK, fixedDEK) { - t.Fatalf("Wrap: %v", err) - } - return - } - e := asError(t, err) - if e.Kind != KindTransport || e.StatusCode != 0 || e.Attempts != MaxAttempts || e.Err == nil { - t.Errorf("got %+v", e) - } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - t.Errorf("transport error %v matches a context error", err) - } - assertNoSecrets(t, err) - }) - } -} - -func TestBodyCutShortIsUnexpectedEOF(t *testing.T) { - s := newScriptServer(t, cutBody()) - _, err := newTestClient(t, s.URL).Unwrap(ctxTimeout(t), testKeyID, "w", nil) - if e := asError(t, err); e.Kind != KindTransport || !errors.Is(err, io.ErrUnexpectedEOF) { - t.Errorf("got %+v (%v), want KindTransport wrapping io.ErrUnexpectedEOF", e, err) - } -} - -func TestConnectionRefused(t *testing.T) { - s := httptest.NewServer(http.NotFoundHandler()) - url := s.URL - s.Close() - - _, err := newTestClient(t, url).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindTransport || e.Attempts != MaxAttempts { - t.Errorf("got %+v", e) - } - var opErr *net.OpError - if !errors.As(err, &opErr) { - t.Errorf("error %v does not wrap a *net.OpError", err) - } - assertNoSecrets(t, err) -} - -func TestClientTimeoutIsRetried(t *testing.T) { - s := newScriptServer(t, stall()) - c, err := New(Options{ - BaseURL: s.URL, - Token: testToken, - HTTPClient: &http.Client{Timeout: 50 * time.Millisecond}, - Backoff: noBackoff, - }) - if err != nil { - t.Fatal(err) - } - _, err = c.Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindTransport || e.Attempts != MaxAttempts { - t.Errorf("got %+v", e) - } - if s.count() != MaxAttempts { - t.Errorf("server saw %d requests, want %d", s.count(), MaxAttempts) - } - // net/http's timeout error unwraps to context.DeadlineExceeded, so err - // matches it through Error.Unwrap. Callers must check for *Error before - // checking for context errors. - if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - t.Errorf("error %v: want a match for context.DeadlineExceeded only", err) - } -} - -func TestContextCanceledDuringBackoff(t *testing.T) { - s := newScriptServer(t, respond(http.StatusServiceUnavailable, "")) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := newTestClient(t, s.URL) - c.backoff = func(int) time.Duration { - cancel() - return time.Hour - } - - start := time.Now() - _, err := c.Wrap(ctx, testKeyID, nil) - if !errors.Is(err, context.Canceled) { - t.Fatalf("got %v, want context.Canceled", err) - } - var e *Error - if errors.As(err, &e) { - t.Errorf("got *Error %+v, want a bare context error", e) - } - if time.Since(start) > 5*time.Second { - t.Error("backoff did not stop when ctx was canceled") - } - if s.count() != 1 { - t.Errorf("server saw %d requests, want 1", s.count()) - } -} - -func TestContextCanceledDuringRequest(t *testing.T) { - s := newScriptServer(t, stall()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { - <-s.arrived - cancel() - }() - - _, err := newTestClient(t, s.URL).Unwrap(ctx, testKeyID, "w", nil) - if !errors.Is(err, context.Canceled) { - t.Fatalf("got %v, want context.Canceled", err) - } - if s.count() != 1 { - t.Errorf("server saw %d requests, want 1", s.count()) - } -} - -func TestContextCanceledDuringLastAttempt(t *testing.T) { - // A failure on the last attempt must still report the cancellation, not - // a transport error. - s := newScriptServer(t, respond(http.StatusServiceUnavailable, ""), respond(http.StatusServiceUnavailable, ""), stall()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { - for range MaxAttempts { - <-s.arrived - } - cancel() - }() - - _, err := newTestClient(t, s.URL).Wrap(ctx, testKeyID, nil) - if !errors.Is(err, context.Canceled) { - t.Fatalf("got %v, want context.Canceled", err) - } - var e *Error - if errors.As(err, &e) { - t.Errorf("got *Error %+v, want a bare context error", e) - } - if s.count() != MaxAttempts { - t.Errorf("server saw %d requests, want %d", s.count(), MaxAttempts) - } -} - -func TestContextAlreadyCanceled(t *testing.T) { - s := newScriptServer(t, okWrap()) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, err := newTestClient(t, s.URL).Wrap(ctx, testKeyID, nil) - if !errors.Is(err, context.Canceled) { - t.Fatalf("got %v, want context.Canceled", err) - } - if s.count() != 0 { - t.Errorf("server saw %d requests, want 0", s.count()) - } -} - -func TestContextDeadline(t *testing.T) { - s := newScriptServer(t, stall()) - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - _, err := newTestClient(t, s.URL).Wrap(ctx, testKeyID, nil) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("got %v, want context.DeadlineExceeded", err) - } - var e *Error - if errors.As(err, &e) { - t.Errorf("got *Error %+v, want a bare context error", e) - } - if s.count() != 1 { - t.Errorf("server saw %d requests, want 1", s.count()) - } -} - -// connCounter counts the connections a server accepts. -type connCounter struct { - mu sync.Mutex - n int -} - -func (c *connCounter) track(_ net.Conn, state http.ConnState) { - if state == http.StateNew { - c.mu.Lock() - c.n++ - c.mu.Unlock() - } -} - -func (c *connCounter) count() int { - c.mu.Lock() - defer c.mu.Unlock() - return c.n -} - -func TestUntrustedCertificateNotRetried(t *testing.T) { - var hits atomic.Int32 - s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - okWrap()(w, r, 1) - })) - var conns connCounter - s.Config.ConnState = conns.track - s.Config.ErrorLog = log.New(io.Discard, "", 0) - s.StartTLS() - t.Cleanup(s.Close) - - // The default client does not trust the test certificate. - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindTransport || e.Attempts != 1 { - t.Errorf("got %+v", e) - } - var verifyErr *tls.CertificateVerificationError - if !errors.As(err, &verifyErr) { - t.Errorf("error %v does not wrap a certificate verification error", err) - } - if conns.count() != 1 || hits.Load() != 0 { - t.Errorf("server saw %d connections and %d requests, want 1 and 0", conns.count(), hits.Load()) - } - assertNoSecrets(t, err) -} - -func TestTrustedTLS(t *testing.T) { - m := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - okWrap()(w, r, 1) - })) - t.Cleanup(m.Close) - c, err := New(Options{BaseURL: m.URL, Token: testToken, HTTPClient: m.Client(), Backoff: noBackoff}) - if err != nil { - t.Fatal(err) - } - if _, err := c.Wrap(ctxTimeout(t), testKeyID, nil); err != nil { - t.Fatalf("Wrap over https: %v", err) - } -} - -func TestHTTPSToPlainServerNotRetried(t *testing.T) { - s := httptest.NewUnstartedServer(http.NotFoundHandler()) - var conns connCounter - s.Config.ConnState = conns.track - s.Config.ErrorLog = log.New(io.Discard, "", 0) - s.Start() - t.Cleanup(s.Close) - - _, err := newTestClient(t, "https://"+s.Listener.Addr().String()).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindTransport || e.Attempts != 1 { - t.Errorf("got %+v (%v)", e, err) - } - if conns.count() != 1 { - t.Errorf("server saw %d connections, want 1", conns.count()) - } -} - -func TestRetryableErr(t *testing.T) { - if !retryableErr(io.ErrUnexpectedEOF) || !retryableErr(&net.OpError{Op: "dial", Err: errors.New("refused")}) { - t.Error("connection errors must be retryable") - } - for _, err := range []error{ - &tls.CertificateVerificationError{Err: errors.New("x")}, - fmt.Errorf("wrapped: %w", http.ErrSchemeMismatch), - tls.RecordHeaderError{Msg: "first record does not look like a TLS handshake"}, - } { - if retryableErr(err) { - t.Errorf("retryableErr(%v) = true", err) - } - } -} - -func TestSleep(t *testing.T) { - if err := sleep(context.Background(), time.Millisecond); err != nil { - t.Errorf("sleep: %v", err) - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - start := time.Now() - if err := sleep(ctx, time.Hour); !errors.Is(err, context.Canceled) { - t.Errorf("sleep on a canceled ctx = %v", err) - } - if err := sleep(ctx, 0); !errors.Is(err, context.Canceled) { - t.Errorf("zero sleep on a canceled ctx = %v", err) - } - if time.Since(start) > time.Second { - t.Error("sleep ignored ctx") - } -} - -// TestRetrySendsSameRequest checks that every attempt carries the whole -// body and the same headers: the body must be rebuilt for each attempt. -func TestRetrySendsSameRequest(t *testing.T) { - tests := []struct { - name string - call func(c *Client) error - path string - body string - }{ - {"wrap", func(c *Client) error { - _, err := c.Wrap(ctxTimeout(t), testKeyID, []byte("label")) - return err - }, "/v1/keys/" + testKeyID + "/wrap", `{"aad":"bGFiZWw="}`}, - {"unwrap", func(c *Client) error { - _, err := c.Unwrap(ctxTimeout(t), testKeyID, "CiQAwrapped", nil) - return err - }, "/v1/keys/" + testKeyID + "/unwrap", `{"wrapped":"CiQAwrapped"}`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := newScriptServer(t, respond(http.StatusServiceUnavailable, ""), hangup(), okWrap()) - if err := tt.call(newTestClient(t, s.URL)); err != nil { - t.Fatal(err) - } - got := s.recorded() - if len(got) != MaxAttempts { - t.Fatalf("server saw %d requests, want %d", len(got), MaxAttempts) - } - for i, r := range got { - if r.method != http.MethodPost || r.escapedPath != tt.path || string(r.body) != tt.body { - t.Errorf("attempt %d: %s %s %q, want POST %s %q", i+1, r.method, r.escapedPath, r.body, tt.path, tt.body) - } - for name, value := range map[string]string{ - "Authorization": "Bearer " + testToken, - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": testUA, - } { - if v := r.header.Get(name); v != value { - t.Errorf("attempt %d: header %s = %q, want %q", i+1, name, v, value) - } - } - } - }) - } -} - -// trackedBody is a response body that records whether it was closed. -type trackedBody struct { - io.Reader - closed atomic.Bool -} - -func (b *trackedBody) Close() error { - b.closed.Store(true) - return nil -} - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } - -func TestResponseBodiesClosed(t *testing.T) { - dek := base64.StdEncoding.EncodeToString(fixedDEK) - text := func(s string) func() io.Reader { - return func() io.Reader { return strings.NewReader(s) } - } - tests := []struct { - name string - status int - body func() io.Reader - hits int - }{ - {"success", 200, text(`{"dek":"` + dek + `","wrapped":"w"}`), 1}, - {"malformed", 200, text(`{}`), 1}, - {"too large", 200, func() io.Reader { return bytes.NewReader(make([]byte, maxResponseBytes+10)) }, 1}, - {"api error", 404, text(envelope("not_found", "not found", "r")), 1}, - {"retried status", 503, text(""), MaxAttempts}, - {"read error", 200, func() io.Reader { - return io.MultiReader(strings.NewReader(`{"dek"`), iotest.ErrReader(io.ErrUnexpectedEOF)) - }, MaxAttempts}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var ( - mu sync.Mutex - bodies []*trackedBody - ) - rt := roundTripFunc(func(r *http.Request) (*http.Response, error) { - b := &trackedBody{Reader: tt.body()} - mu.Lock() - bodies = append(bodies, b) - mu.Unlock() - return &http.Response{StatusCode: tt.status, Header: http.Header{}, Body: b, ContentLength: -1, Request: r}, nil - }) - c, err := New(Options{BaseURL: "http://127.0.0.1:1", Token: testToken, HTTPClient: &http.Client{Transport: rt}, Backoff: noBackoff}) - if err != nil { - t.Fatal(err) - } - _, _ = c.Wrap(ctxTimeout(t), testKeyID, nil) - mu.Lock() - defer mu.Unlock() - if len(bodies) != tt.hits { - t.Fatalf("%d responses, want %d", len(bodies), tt.hits) - } - for i, b := range bodies { - if !b.closed.Load() { - t.Errorf("response %d body was not closed", i+1) - } - } - }) - } -} diff --git a/internal/client/sanitize.go b/internal/client/sanitize.go deleted file mode 100644 index 249d66d..0000000 --- a/internal/client/sanitize.go +++ /dev/null @@ -1,47 +0,0 @@ -package client - -import ( - "strings" - "unicode" - "unicode/utf8" -) - -// maxServerText caps the length in bytes of each piece of server-supplied -// text kept in an Error. -const maxServerText = 512 - -// sanitize makes server-supplied text safe to print on a terminal: it -// applies clean and then truncate. -func sanitize(s string) string { - return truncate(clean(s)) -} - -// clean drops control characters (C0, DEL and C1, so no escape sequences), -// format characters such as U+202E RIGHT-TO-LEFT OVERRIDE and U+200B ZERO -// WIDTH SPACE, and the Unicode line and paragraph separators. Invalid UTF-8 -// becomes U+FFFD. -func clean(s string) string { - var b strings.Builder - for _, r := range s { // invalid UTF-8 arrives as utf8.RuneError (U+FFFD) - if unicode.IsControl(r) || unicode.In(r, unicode.Cf, unicode.Zl, unicode.Zp) { - continue - } - b.WriteRune(r) - } - return b.String() -} - -// truncate trims s and cuts it on a rune boundary to at most maxServerText -// bytes, ending in "..." when cut. -func truncate(s string) string { - const ellipsis = "..." - s = strings.TrimSpace(s) - if len(s) <= maxServerText { - return s - } - cut := maxServerText - len(ellipsis) - for cut > 0 && !utf8.RuneStart(s[cut]) { - cut-- - } - return strings.TrimSpace(s[:cut]) + ellipsis -} diff --git a/internal/client/sanitize_test.go b/internal/client/sanitize_test.go deleted file mode 100644 index dd006f1..0000000 --- a/internal/client/sanitize_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package client - -import ( - "encoding/json" - "net/http" - "strings" - "testing" - "unicode" - "unicode/utf8" -) - -// assertPrintable fails if s holds a rune that sanitize must remove, or -// invalid UTF-8. -func assertPrintable(t *testing.T, what, s string) { - t.Helper() - if !utf8.ValidString(s) { - t.Errorf("%s %q is not valid UTF-8", what, s) - } - for _, r := range s { - if unicode.IsControl(r) || unicode.In(r, unicode.Cf, unicode.Zl, unicode.Zp) { - t.Errorf("%s %q contains %U", what, s, r) - } - } -} - -func TestSanitize(t *testing.T) { - tests := []struct{ in, want string }{ - {"", ""}, - {"plain text", "plain text"}, - {"\x1b[31mred\x1b[0m", "[31mred[0m"}, - {"\x1b]0;title\x07x", "]0;titlex"}, - {"a\r\nb\tc\x00d\x7fe\x08f", "abcdef"}, - {"C1: \u0085\u009b2J\u0090end", "C1: 2Jend"}, - {"rtl \u202egnp.exe\u202c zw\u200bsp\u200c\u200d\u2060 bom\ufeff shy\u00ad", "rtl gnp.exe zwsp bom shy"}, - {"iso \u2066x\u2069 lsep\u2028psep\u2029", "iso x lseppsep"}, - {"bad \xff\xfe utf8", "bad \ufffd\ufffd utf8"}, - {" \t padded \n ", "padded"}, - {"unicode ok: \u00e9 \u00fc \u65e5\u672c \U0001f600", "unicode ok: \u00e9 \u00fc \u65e5\u672c \U0001f600"}, - } - for _, tt := range tests { - if got := sanitize(tt.in); got != tt.want { - t.Errorf("sanitize(%q) = %q, want %q", tt.in, got, tt.want) - } - } -} - -func TestSanitizeTruncates(t *testing.T) { - for _, unit := range []string{"a", "\u00e9", "\u65e5", "\U0001f600", "a\x1b"} { - in := strings.Repeat(unit, 2000) - got := sanitize(in) - if len(got) > maxServerText || !strings.HasSuffix(got, "...") { - t.Errorf("unit %q: got %d bytes, suffix %q", unit, len(got), got[max(0, len(got)-3):]) - } - if len(got) < maxServerText-8 { - t.Errorf("unit %q: cut too short, %d bytes", unit, len(got)) - } - assertPrintable(t, "truncated", got) - } - exact := strings.Repeat("a", maxServerText) - if got := sanitize(exact); got != exact { - t.Errorf("a %d-byte string was changed", maxServerText) - } - // Removed characters do not count toward the cap. - if got := sanitize(strings.Repeat("\x1b", 1000) + exact); got != exact { - t.Error("control characters counted toward the cap") - } -} - -// nasty holds every kind of character that must not reach a terminal. -const nasty = "\x1b[2J\x1b]8;;http://evil\x07\r\n\t\x00\x7f\u0085\u009b\u202e\u200b\u2028\xff" - -func TestServerTextSanitizedEndToEnd(t *testing.T) { - body, _ := json.Marshal(map[string]any{ - "error": "ops_" + nasty + "cap_reached", - "message": "msg" + nasty + strings.Repeat("m", 2000), - "request_id": "id" + nasty + "-1", - "cap": 10, - "resets_at": "2026-11-01" + nasty + "T00:00:00Z", - }) - s := newScriptServer(t, respond(http.StatusTooManyRequests, string(body))) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - // json.Marshal turns the invalid byte into U+FFFD, which is kept. - if e.Code != "ops_[2J]8;;http://evil\ufffdcap_reached" { - t.Errorf("Code = %q", e.Code) - } - if e.RequestID != "id[2J]8;;http://evil\ufffd-1" { - t.Errorf("RequestID = %q", e.RequestID) - } - if e.ResetsAt != "2026-11-01[2J]8;;http://evil\ufffdT00:00:00Z" { - t.Errorf("ResetsAt = %q", e.ResetsAt) - } - if len(e.Message) > maxServerText || !strings.HasPrefix(e.Message, "msg[2J]") { - t.Errorf("Message has %d bytes: %.40q", len(e.Message), e.Message) - } - for what, v := range map[string]string{ - "Code": e.Code, "Message": e.Message, "RequestID": e.RequestID, "ResetsAt": e.ResetsAt, "Error()": err.Error(), - } { - assertPrintable(t, what, v) - } -} - -func TestRequestIDHeaderSanitized(t *testing.T) { - // Of the runes sanitize removes, these are the ones net/http lets - // through in a header value; the rest make the response malformed. - header := "hdr\t\u202e\u200b\u0085\u2028\xff-9" - step := func(w http.ResponseWriter, r *http.Request, n int) { - w.Header()["X-Request-Id"] = []string{header} - w.WriteHeader(http.StatusBadGateway) - } - s := newScriptServer(t, step) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.RequestID != "hdr\ufffd-9" { - t.Errorf("RequestID = %q", e.RequestID) - } - assertPrintable(t, "Error()", err.Error()) - - // The same header on a malformed 200. - step = func(w http.ResponseWriter, r *http.Request, n int) { - w.Header()["X-Request-Id"] = []string{header} - _, _ = w.Write([]byte("{}")) - } - s = newScriptServer(t, step) - _, err = newTestClient(t, s.URL).Unwrap(ctxTimeout(t), testKeyID, "w", nil) - if e := asError(t, err); e.Kind != KindResponse || e.RequestID != "hdr\ufffd-9" { - t.Errorf("got %+v", e) - } -} - -func TestMalformedHeaderTransportError(t *testing.T) { - // net/http refuses an ESC in a header and quotes the line in its error. - step := func(w http.ResponseWriter, r *http.Request, n int) { - conn, buf, _ := w.(http.Hijacker).Hijack() - _, _ = buf.WriteString("HTTP/1.1 200 OK\r\nX-Request-Id: a\x1b[2Jb\u202e\r\nContent-Length: 2\r\n\r\n{}") - _ = buf.Flush() - conn.Close() - } - s := newScriptServer(t, step) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - if e := asError(t, err); e.Kind != KindTransport { - t.Errorf("got %+v", e) - } - assertPrintable(t, "Error()", err.Error()) -} - -func TestServerEchoedTokenRedacted(t *testing.T) { - // A server that echoes the token, even split by control characters, - // must not get it into an error. - split := testToken[:20] + "\x1b\u200b" + testToken[20:] - // Without redaction, the second token would straddle the truncation - // point, 20 bytes of it before the cut. - filler := strings.Repeat("x", maxServerText-len("...")-20-len("bad token ")-len(testToken)-1) - body, _ := json.Marshal(map[string]string{ - "error": "unauthorized", - "message": "bad token " + split + " " + filler + testToken, - "request_id": testToken, - "resets_at": "Bearer " + testToken, - }) - step := func(w http.ResponseWriter, r *http.Request, n int) { - w.Header().Set("X-Request-Id", "id-"+testToken) - w.WriteHeader(http.StatusUnauthorized) - _, _ = w.Write(body) - } - s := newScriptServer(t, step) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if !strings.Contains(e.Message, "[redacted]") || e.RequestID != "[redacted]" { - t.Errorf("got %+v", e) - } - // Every 8-byte piece of the secret, so that a truncated token is caught. - for i := 0; i+8 <= len(testSecret); i++ { - if piece := testSecret[i : i+8]; strings.Contains(err.Error(), piece) { - t.Errorf("Error() %q holds part of the token (%q)", err, piece) - } - } - assertNoSecrets(t, err) - - // A header-only echo, on a non-JSON body. - step = func(w http.ResponseWriter, r *http.Request, n int) { - w.Header().Set("X-Request-Id", testToken) - w.WriteHeader(http.StatusForbidden) - } - s = newScriptServer(t, step) - _, err = newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - if e := asError(t, err); e.RequestID != "[redacted]" { - t.Errorf("got %+v", e) - } - assertNoSecrets(t, err) -} - -func TestTransportErrorRedactsToken(t *testing.T) { - // net/http quotes a malformed status or header line in its error, so a - // server that echoes the token there must not get it into an error. - for name, raw := range map[string]string{ - "header line": "HTTP/1.1 200 OK\r\nX-Echo " + testToken + "\r\nContent-Length: 2\r\n\r\n{}", - "version": testToken + " 200 OK\r\n\r\n", - } { - t.Run(name, func(t *testing.T) { - step := func(w http.ResponseWriter, r *http.Request, n int) { - conn, buf, _ := w.(http.Hijacker).Hijack() - _, _ = buf.WriteString(raw) - _ = buf.Flush() - conn.Close() - } - s := newScriptServer(t, step) - _, err := newTestClient(t, s.URL).Wrap(ctxTimeout(t), testKeyID, nil) - e := asError(t, err) - if e.Kind != KindTransport || !strings.Contains(err.Error(), "[redacted]") { - t.Errorf("got %v", err) - } - for _, text := range []string{err.Error(), e.Err.Error()} { - for i := 0; i+8 <= len(testSecret); i++ { - if piece := testSecret[i : i+8]; strings.Contains(text, piece) { - t.Fatalf("error text %q holds part of the token (%q)", text, piece) - } - } - assertPrintable(t, "error text", text) - } - assertNoSecrets(t, err) - }) - } -} diff --git a/scripts/probe.go b/scripts/probe.go index b95cbcb..df3f9d0 100644 --- a/scripts/probe.go +++ b/scripts/probe.go @@ -28,11 +28,13 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "sort" + "strings" "time" - "github.com/x6c-co/rypt-cli/internal/client" + "github.com/x6c-co/rypt" ) var ( @@ -57,16 +59,26 @@ type result struct { func main() { token = os.Getenv("RYPT_TOKEN") keyID = os.Getenv("RYPT_KEY_ID") - baseURL, err := client.ParseBaseURL(os.Getenv("RYPT_API_URL")) - if err != nil { - fmt.Fprintln(os.Stderr, "probe: RYPT_API_URL:", err) - os.Exit(2) - } - base = baseURL.String() if token == "" || keyID == "" { fmt.Fprintln(os.Stderr, "probe: RYPT_TOKEN and RYPT_KEY_ID must be set") os.Exit(2) } + // rypt.NewClient checks the base URL with the rules rypt uses. The + // probe sends its own requests, to see the raw responses. + base = os.Getenv("RYPT_API_URL") + if _, err := rypt.NewClient(token, rypt.WithBaseURL(base)); err != nil { + fmt.Fprintln(os.Stderr, "probe: RYPT_API_URL:", strings.TrimPrefix(err.Error(), "rypt: ")) + os.Exit(2) + } + if base == "" { + base = rypt.DefaultBaseURL + } + baseURL, _ := url.Parse(base) // valid: NewClient accepted it + if baseURL.Scheme == "http" { + // net/http bypasses a proxy only for the exact name "localhost". + baseURL.Host = strings.ToLower(baseURL.Host) + } + base = strings.TrimRight(baseURL.String(), "/") fmt.Printf("base URL: %s\n\n", base) // Key metadata is not a metered operation.