From e83a5beafd41368e15caff46994901d78a2bc50f Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Wed, 17 Jun 2026 14:07:15 -0400 Subject: [PATCH] =?UTF-8?q?fula-client=200.6.13:=20large=20chunked=20files?= =?UTF-8?q?=20=E2=80=94=20header-safe=20metadata,=20share/collab=20token?= =?UTF-8?q?=20fix,=20index-PUT=20retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large (chunked) uploads failed at the final index PUT. The per-chunk encryption metadata (chunk_nonces + chunk_cids) for a ~480-chunk file is ~70 KB and exceeded the gateway nginx `large_client_header_buffers 4 32k`, so the PUT was rejected (HTTP 400 / connection closed) AFTER every chunk had uploaded — and the whole upload then restarted. Root fix: keep the oversized per-chunk arrays out of the S3 *header* only. The index-object BODY and the forest entry always carry the FULL metadata, and readers recover it from there. - header_safe_enc_metadata(): when the JSON exceeds a 16 KB budget, strip ONLY chunk_nonces + chunk_cids from the `chunked` block for the HTTP-header copy. Applied at all 4 chunked-index writers. Body + forest stay FULL. - resolve_enc_metadata(header, forest, body): on read, return the first COMPLETE source (header -> forest -> body); completeness = chunk_nonces present & non-empty. Applied at all 6 decrypt read sites. Never returns undecryptable metadata silently. - index PUT wrapped in retry_idempotent(4): a transient blip on the finalize PUT retries that one content-addressed PUT instead of deleting every chunk and restarting the whole upload. Share + collaborate (why a recipient could not decrypt a large shared file): - get_object_encryption_metadata_with_fallback() is now completeness-aware: header-if-complete -> forest entry -> index-object BODY (the ~70 KB manifest, not the file content). Share-token creation (fula-flutter create_share_token; FxFiles collaboration_service.dart uses the same call) builds the token's chunked_metadata from this, so without the fix a large-file token shipped WITHOUT chunk_nonces and the recipient/collaborator could not decrypt. Other: - get_object_range: resolve from the in-hand index body (no forest lookup) — the range path runs no forest-backed integrity gate, and for a v7 sharded HAMT the lookup could cost an extra S3 round-trip per seek/thumbnail read. - rewrap_object_dek: guard chunked objects with a clear error. Key rotation of large objects is not yet supported; it already failed on the oversized header, and rewrapping would have mirrored a chunk_nonces-less blob into the forest. Single-object rotation is unaffected; rotate_bucket reports it as a per-object failure, not an abort. Tests: 4 unit tests (header_safe / resolve / completeness); an #[ignore] 120 MB repro (large_index_put_repro_e2e); an #[ignore] share/collab E2E (large_chunked_share_e2e) asserting the stored header is stripped, that with_fallback recovers chunk_nonces, that the share token carries them, and that a recipient decrypts to the exact bytes. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 12 +- Cargo.toml | 2 +- crates/fula-client/src/encryption.rs | 528 ++++++++++++++---- .../tests/large_chunked_share_e2e.rs | 240 ++++++++ .../tests/large_index_put_repro_e2e.rs | 95 ++++ 5 files changed, 759 insertions(+), 118 deletions(-) create mode 100644 crates/fula-client/tests/large_chunked_share_e2e.rs create mode 100644 crates/fula-client/tests/large_index_put_repro_e2e.rs diff --git a/Cargo.lock b/Cargo.lock index 5856089..5ce718c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1736,7 +1736,7 @@ dependencies = [ [[package]] name = "fula-api" -version = "0.6.12" +version = "0.6.13" dependencies = [ "anyhow", "axum", @@ -1765,7 +1765,7 @@ dependencies = [ [[package]] name = "fula-blockstore" -version = "0.6.12" +version = "0.6.13" dependencies = [ "anyhow", "async-trait", @@ -1803,7 +1803,7 @@ dependencies = [ [[package]] name = "fula-cli" -version = "0.6.12" +version = "0.6.13" dependencies = [ "anyhow", "async-trait", @@ -1857,7 +1857,7 @@ dependencies = [ [[package]] name = "fula-client" -version = "0.6.12" +version = "0.6.13" dependencies = [ "anyhow", "async-trait", @@ -1900,7 +1900,7 @@ dependencies = [ [[package]] name = "fula-core" -version = "0.6.12" +version = "0.6.13" dependencies = [ "anyhow", "async-trait", @@ -1935,7 +1935,7 @@ dependencies = [ [[package]] name = "fula-crypto" -version = "0.6.12" +version = "0.6.13" dependencies = [ "aes-gcm", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index bbc0ec2..6bedbde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,7 +79,7 @@ name = "encrypted_upload_test" path = "examples/encrypted_upload_test.rs" [workspace.package] -version = "0.6.12" +version = "0.6.13" edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/functionland/fula-api" diff --git a/crates/fula-client/src/encryption.rs b/crates/fula-client/src/encryption.rs index b0c575d..4c384dc 100644 --- a/crates/fula-client/src/encryption.rs +++ b/crates/fula-client/src/encryption.rs @@ -1062,6 +1062,101 @@ where })? } +/// True iff `v` is a COMPLETE encryption-metadata blob — either a non-chunked +/// object (no `chunked` block) or a chunked object whose `chunked.chunk_nonces` +/// array is present and non-empty. `chunk_nonces` are required to decrypt each +/// chunk and are NOT derivable, so a chunked blob missing them is unusable; the +/// reader must then source the full blob elsewhere (forest entry / index body). +fn enc_metadata_is_complete(v: &serde_json::Value) -> bool { + match v.get("chunked") { + None | Some(serde_json::Value::Null) => true, + Some(chunked) => chunked + .get("chunk_nonces") + .and_then(|n| n.as_array()) + .map(|a| !a.is_empty()) + .unwrap_or(false), + } +} + +/// Resolve the COMPLETE `x-fula-encryption` metadata JSON for an object. +/// +/// For large chunked files the per-chunk arrays (`chunk_nonces` = decrypt; +/// `chunk_cids` = offline hints) no longer ride in the `x-fula-encryption` +/// HTTP header (it would exceed the gateway's `large_client_header_buffers` +/// limit and the PUT would 400). They always live in the index-object BODY and +/// the forest entry. Precedence: +/// 1. the HTTP header, IF complete — legacy objects + small files whose +/// header still carries the arrays (backward compatible); +/// 2. the forest entry's `user_metadata` — owner reads; always full and +/// CID-stamped, and the source used when the master is down (offline); +/// 3. the index-object BODY — share recipients have no forest entry; also the +/// offline-by-CID source. +/// The body is parsed only when header+forest are incomplete; a single object's +/// small header is always complete (no `chunked` block) → resolved at step 1, +/// so its ciphertext body is never mis-parsed as JSON. +fn resolve_enc_metadata( + header_json: Option<&str>, + forest_entry: Option<&ForestFileEntry>, + index_body: Option<&[u8]>, +) -> Result { + if let Some(h) = header_json { + if let Ok(v) = serde_json::from_str::(h) { + if enc_metadata_is_complete(&v) { + return Ok(v); + } + } + } + if let Some(e) = forest_entry { + if let Some(s) = e.user_metadata.get("x-fula-encryption") { + if let Ok(v) = serde_json::from_str::(s) { + if enc_metadata_is_complete(&v) { + return Ok(v); + } + } + } + } + if let Some(b) = index_body { + if let Ok(v) = serde_json::from_slice::(b) { + if enc_metadata_is_complete(&v) { + return Ok(v); + } + } + } + Err(ClientError::Encryption(fula_crypto::CryptoError::Decryption( + "encryption metadata unavailable (incomplete in HTTP header, forest entry, and index body)" + .to_string(), + ))) +} + +/// Build the `x-fula-encryption` HTTP-header value from the full metadata JSON. +/// +/// Returns it verbatim when it fits the gateway header budget; otherwise strips +/// the large per-chunk arrays (`chunk_nonces`, `chunk_cids`) from the `chunked` +/// block so the header stays small. The FULL JSON is ALWAYS written to the +/// index body + forest entry regardless — readers recover the arrays from there +/// via [`resolve_enc_metadata`]. Keying on the serialized length (not chunk +/// count) keeps small files' headers intact, so un-updated readers that only +/// know the header path keep working on small/legacy objects. +fn header_safe_enc_metadata(full_json: &str) -> String { + // Margin below nginx's `large_client_header_buffers 4 32k`: the request also + // carries other headers + the value is a single header line, so 16 KB is a + // safe cap that still lets typical small chunked files keep their full header. + const HEADER_BUDGET: usize = 16 * 1024; + if full_json.len() <= HEADER_BUDGET { + return full_json.to_string(); + } + match serde_json::from_str::(full_json) { + Ok(mut v) => { + if let Some(chunked) = v.get_mut("chunked").and_then(|c| c.as_object_mut()) { + chunked.remove("chunk_nonces"); + chunked.remove("chunk_cids"); + } + v.to_string() + } + Err(_) => full_json.to_string(), + } +} + impl EncryptedClient { /// Create a new encrypted client pub fn new(config: Config, encryption: EncryptionConfig) -> Result { @@ -1338,14 +1433,29 @@ impl EncryptedClient { /// object, with automatic fallback to the forest entry's local copy /// when master is unreachable. /// - /// Behavior: - /// - Tries `head_object` against the configured master first. On - /// success, returns the `x-fula-encryption` HTTP response header. - /// This preserves "live S3 = source of truth" for online flows. + /// Behavior (online — `head_object` succeeds): + /// - Returns the `x-fula-encryption` HTTP header IF it is COMPLETE. + /// Small / legacy objects (and any whose metadata fits the gateway + /// header budget) resolve here with no extra work — "live S3 = + /// source of truth". + /// - For LARGE chunked files the header is stripped of the per-chunk + /// arrays (`header_safe_enc_metadata`), so a present-but-incomplete + /// header (no `chunk_nonces`) is NOT returned directly. It falls + /// through to (2) the forest entry's + /// `user_metadata["x-fula-encryption"]` — written FULL at upload + /// by the main + resumable chunked paths + /// (`register_encrypted_chunked_upload_in_forest` / the stash at the + /// end of `put_object_encrypted`) — then to (3) the index-object + /// BODY (the manifest JSON; one small GET of the index object, NOT + /// the file content, which lives in separate chunk keys). The body + /// is the robust last resort for upload paths that don't populate + /// the forest (e.g. the public `put_object_chunked`). This + /// guarantees share / collab tokens built from the result carry the + /// `chunk_nonces` the recipient needs to decrypt. /// - On a transport-layer failure (master unreachable / DNS / - /// connect / timeout / health-gate-down), falls back to the - /// forest entry's `user_metadata["x-fula-encryption"]`, populated - /// at upload by `put_object_encrypted_*` (see `encryption.rs:5955-5968`). + /// connect / timeout / health-gate-down), falls back to the forest + /// entry's `user_metadata["x-fula-encryption"]` (the index body + /// can't be fetched while master is down). /// - On any other error (404, AccessDenied, parse errors), /// propagates as-is — those are real failure responses, not /// master-down. @@ -1367,16 +1477,77 @@ impl EncryptedClient { storage_key: &str, ) -> Result { match self.inner.head_object(bucket, storage_key).await { - Ok(head) => head - .metadata - .get("x-fula-encryption") - .cloned() - .ok_or_else(|| { - ClientError::Encryption(fula_crypto::CryptoError::Decryption( - "Object is not encrypted or missing encryption metadata" + Ok(head) => { + // 1. HTTP header — source of truth for small / legacy objects + // and any object whose metadata fits the gateway header + // budget. For LARGE chunked files `header_safe_enc_metadata` + // strips the per-chunk arrays (`chunk_nonces` / `chunk_cids`) + // from this header, so a PRESENT header is not sufficient: it + // must be COMPLETE (carry `chunk_nonces`). A share/collab + // token built from a stripped header would omit the nonces + // and the recipient could not decrypt. Small/legacy objects + // resolve here with no extra work (no forest load, no GET). + let header = head.metadata.get("x-fula-encryption").cloned(); + if let Some(ref h) = header { + if serde_json::from_str::(h) + .map(|v| enc_metadata_is_complete(&v)) + .unwrap_or(false) + { + return Ok(h.clone()); + } + } + // 2. Forest entry — the owner (the actor creating a share) holds + // it locally, and the main + resumable chunked-upload paths + // write the FULL blob into `user_metadata`. Cheap: cached for + // monolithic forests, at worst one HAMT shard read for v7. + self.ensure_forest_loaded(bucket).await?; + let forest_entry = self.forest_entry_lookup(bucket, storage_key).await?; + if let Some(ref e) = forest_entry { + if let Some(s) = e.user_metadata.get("x-fula-encryption") { + if serde_json::from_str::(s) + .map(|v| enc_metadata_is_complete(&v)) + .unwrap_or(false) + { + return Ok(s.clone()); + } + } + } + // 3. Index-object BODY — the manifest JSON (NOT the file: chunks + // live under separate keys), always written FULL on upload. + // Robust last resort for upload paths that don't populate the + // forest `user_metadata` (e.g. the public `put_object_chunked`) + // or when the forest cache was evicted. One small GET of the + // index object, taken only when header + forest were both + // incomplete (i.e. a large chunked file with no full forest + // copy) — never on the small/legacy fast path above. + let body = self.inner.get_object(bucket, storage_key).await?; + if serde_json::from_slice::(&body) + .map(|v| enc_metadata_is_complete(&v)) + .unwrap_or(false) + { + return String::from_utf8(body.to_vec()).map_err(|e| { + ClientError::Encryption(fula_crypto::CryptoError::Decryption(format!( + "index body is not valid UTF-8 JSON: {}", + e + ))) + }); + } + // Nothing complete anywhere. A present-but-incomplete header + // means the object IS encrypted but its per-chunk nonces are + // unrecoverable here; surface that distinctly from "not + // encrypted" so callers can tell a genuine gap from a plaintext + // object. + match header { + Some(_) => Err(ClientError::Encryption(fula_crypto::CryptoError::Decryption( + "encryption metadata incomplete in HTTP header, forest entry, and index \ + body (cannot build a share/collab token without per-chunk nonces)" .to_string(), - )) - }), + ))), + None => Err(ClientError::Encryption(fula_crypto::CryptoError::Decryption( + "Object is not encrypted or missing encryption metadata".to_string(), + ))), + } + } Err(err) if Self::is_master_unreachable_error(&err) => { // Master unreachable — fall back to the local forest entry. self.ensure_forest_loaded(bucket).await?; @@ -1696,19 +1867,17 @@ impl EncryptedClient { // a missing entry on BOTH sources means we have no way to // decrypt — surface a clear error rather than silently // returning ciphertext or refusing on the wrong axis. - let enc_metadata_str = get_meta("x-fula-encryption").ok_or_else(|| { - ClientError::Encryption(fula_crypto::CryptoError::Decryption( - "Missing encryption metadata (HTTP headers absent and forest entry has no \ - user_metadata; legacy upload that requires master to be online — re-upload \ - once via the new SDK to enable offline reads for this file)" - .to_string(), - )) - })?; - - let enc_metadata: serde_json::Value = serde_json::from_str(&enc_metadata_str) - .map_err(|e| ClientError::Encryption( - fula_crypto::CryptoError::Decryption(e.to_string()) - ))?; + // Resolve the COMPLETE metadata: HTTP header (legacy/small) → forest + // entry (owner; full + CID-stamped; offline source) → index body. For + // large chunked files the header no longer carries the per-chunk + // arrays, so a present-but-stripped header must NOT short-circuit the + // forest fallback — `resolve_enc_metadata` checks completeness, unlike + // the old `get_meta` (which fell back only on full key-absence). + let enc_metadata = resolve_enc_metadata( + result.metadata.get("x-fula-encryption").map(|s| s.as_str()), + forest_entry.as_ref(), + Some(result.data.as_ref()), + )?; // Unwrap the DEK (common to both chunked and non-chunked) let wrapped_key: EncryptedData = serde_json::from_value( @@ -2195,16 +2364,19 @@ impl EncryptedClient { .map(|v| v == "true") .unwrap_or(false); - let enc_metadata_str = result.metadata - .get("x-fula-encryption") - .ok_or_else(|| ClientError::Encryption( - fula_crypto::CryptoError::Decryption("Missing encryption metadata".to_string()) - ))?; + // H-1 / H-2: look up the forest entry once; shared by metadata + // resolution AND both decrypt branches. + let forest_entry = self.forest_entry_lookup(bucket, storage_key).await?; - let enc_metadata: serde_json::Value = serde_json::from_str(enc_metadata_str) - .map_err(|e| ClientError::Encryption( - fula_crypto::CryptoError::Decryption(e.to_string()) - ))?; + // Resolve COMPLETE metadata: header (legacy/small) → forest (full + + // CID-stamped) → index body. A large chunked file's header keeps + // `wrapped_key` but drops `chunk_nonces`, so the chunked branch needs + // the full blob from forest/body. + let enc_metadata = resolve_enc_metadata( + result.metadata.get("x-fula-encryption").map(|s| s.as_str()), + forest_entry.as_ref(), + Some(result.data.as_ref()), + )?; let wrapped_key: EncryptedData = serde_json::from_value( enc_metadata["wrapped_key"].clone() @@ -2216,9 +2388,6 @@ impl EncryptedClient { let dek = decryptor.decrypt_dek(&wrapped_key) .map_err(ClientError::Encryption)?; - // H-1 / H-2: look up the forest entry once; shared by both branches. - let forest_entry = self.forest_entry_lookup(bucket, storage_key).await?; - if is_chunked { self.get_object_chunked_to_writer(bucket, storage_key, &enc_metadata, &dek, writer, forest_entry.as_ref()).await } else { @@ -2325,16 +2494,19 @@ impl EncryptedClient { .map(|v| v == "true") .unwrap_or(false); - let enc_metadata_str = result.metadata - .get("x-fula-encryption") - .ok_or_else(|| ClientError::Encryption( - fula_crypto::CryptoError::Decryption("Missing encryption metadata".to_string()) - ))?; + // H-1 / H-2: look up the forest entry once; shared by metadata + // resolution AND both decrypt branches. + let forest_entry = self.forest_entry_lookup(bucket, storage_key).await?; - let enc_metadata: serde_json::Value = serde_json::from_str(enc_metadata_str) - .map_err(|e| ClientError::Encryption( - fula_crypto::CryptoError::Decryption(e.to_string()) - ))?; + // Resolve COMPLETE metadata: header (legacy/small) → forest (full + + // CID-stamped) → index body. A large chunked file's header keeps + // `wrapped_key` but drops `chunk_nonces`, so the chunked branch needs + // the full blob from forest/body. + let enc_metadata = resolve_enc_metadata( + result.metadata.get("x-fula-encryption").map(|s| s.as_str()), + forest_entry.as_ref(), + Some(result.data.as_ref()), + )?; let wrapped_key: EncryptedData = serde_json::from_value( enc_metadata["wrapped_key"].clone() @@ -2346,9 +2518,6 @@ impl EncryptedClient { let dek = decryptor.decrypt_dek(&wrapped_key) .map_err(ClientError::Encryption)?; - // H-1 / H-2: look up the forest entry once; shared by both branches. - let forest_entry = self.forest_entry_lookup(bucket, storage_key).await?; - if is_chunked { self.get_object_chunked_buffered_to_writer(bucket, storage_key, &enc_metadata, &dek, writer, forest_entry.as_ref()).await } else { @@ -7304,13 +7473,16 @@ impl EncryptedClient { "chunked": serde_json::to_value(&chunked_metadata).unwrap(), }); - // The index object is small - just metadata, no file content + // Index object: the body is the FULL metadata JSON (incl. per-chunk + // arrays); the HEADER copy is shrunk via `header_safe_enc_metadata` so a + // large chunked file's metadata can't blow past the gateway's header + // limit. Readers recover the full blob from the body / forest entry. let index_body = enc_metadata.to_string(); let metadata = ObjectMetadata::new() .with_content_type("application/json") .with_metadata("x-fula-encrypted", "true") .with_metadata("x-fula-chunked", "true") - .with_metadata("x-fula-encryption", &index_body); + .with_metadata("x-fula-encryption", &header_safe_enc_metadata(&index_body)); // Walkable-v8 (W.9.3): pre-compute `BLAKE3(index_body)` so the // post-PUT self-verify can compare master's etag-attested CID @@ -7326,22 +7498,38 @@ impl EncryptedClient { // Upload index object. If this fails after all chunks were successfully // uploaded, we must compensate by deleting the chunks — otherwise the // upload is non-atomic and leaks storage. - let index_result = if let Some(ref pinning) = self.pinning { - self.inner.put_object_with_metadata_and_pinning( - bucket, - storage_key, - Bytes::from(index_body.clone()), - Some(metadata), - &pinning.endpoint, - &pinning.token, - ).await - } else { - self.inner.put_object_with_metadata( - bucket, - storage_key, - Bytes::from(index_body.clone()), - Some(metadata), - ).await + // Retry the finalize PUT on transient errors — same treatment as the + // content chunks above. The index key is content-addressed (idempotent), + // so a single dropped finalize must NOT trigger the compensating delete + // of every chunk + a whole-upload restart. Per-attempt clones are cheap. + let index_result = { + let client = self.inner.clone(); + let pinning = self.pinning.clone(); + crate::multipart::retry_idempotent(4, || { + let client = client.clone(); + let pinning = pinning.clone(); + let body = Bytes::from(index_body.clone()); + let metadata = metadata.clone(); + async move { + if let Some(ref pin) = pinning { + client + .put_object_with_metadata_and_pinning( + bucket, + storage_key, + body, + Some(metadata), + &pin.endpoint, + &pin.token, + ) + .await + } else { + client + .put_object_with_metadata(bucket, storage_key, body, Some(metadata)) + .await + } + } + }) + .await }; let result = match index_result { @@ -8003,7 +8191,8 @@ impl EncryptedClient { .with_content_type("application/json") .with_metadata("x-fula-encrypted", "true") .with_metadata("x-fula-chunked", "true") - .with_metadata("x-fula-encryption", &index_body); + // Header shrunk for large chunked files; full JSON stays in body + forest. + .with_metadata("x-fula-encryption", &header_safe_enc_metadata(&index_body)); // Walkable-v8 (#82): pre-compute BLAKE3 of the index body so // we can verify against master's etag and stamp the CID into @@ -8459,7 +8648,8 @@ impl EncryptedClient { .with_content_type("application/json") .with_metadata("x-fula-encrypted", "true") .with_metadata("x-fula-chunked", "true") - .with_metadata("x-fula-encryption", index_body); + // Header shrunk for large chunked files; full JSON stays in body + forest. + .with_metadata("x-fula-encryption", &header_safe_enc_metadata(index_body)); // Walkable-v8 (#53 / #82): pre-compute BLAKE3 of the index // body BEFORE the PUT consumes it via Bytes::from. Skipped @@ -10066,17 +10256,16 @@ impl EncryptedClient { .map(|v| v == "true") .unwrap_or(false); - // Parse encryption metadata from S3 headers - let enc_metadata_str = result.metadata - .get("x-fula-encryption") - .ok_or_else(|| ClientError::Encryption( - fula_crypto::CryptoError::Decryption("Missing encryption metadata (not in share token or S3 headers)".to_string()) - ))?; - - let enc_metadata: serde_json::Value = serde_json::from_str(enc_metadata_str) - .map_err(|e| ClientError::Encryption( - fula_crypto::CryptoError::Decryption(e.to_string()) - ))?; + // Share recipients have NO forest entry, so resolve from the HTTP + // header (legacy/small) → index BODY (large chunked files). The body's + // `chunk_nonces` are recovered here; decryption below uses the share's + // own DEK (not the body's wrapped_key), and `chunk_nonces` are never + // changed by key rotation — so the body is a safe source for shares. + let enc_metadata = resolve_enc_metadata( + result.metadata.get("x-fula-encryption").map(|s| s.as_str()), + None, + Some(result.data.as_ref()), + )?; if is_chunked { // CHUNKED DOWNLOAD: Download and decrypt each chunk using the share's DEK @@ -10259,7 +10448,37 @@ impl EncryptedClient { ) -> Result { // Get the object with metadata let result = self.inner.get_object_with_metadata(bucket, storage_key).await?; - + + // GUARD: key rotation of chunked (large) objects is not yet supported. + // For a chunked file the index metadata exceeds the gateway header + // budget, so `header_safe_enc_metadata` strips the per-chunk arrays from + // the `x-fula-encryption` header this function reads. Rewrapping here + // would operate on incomplete metadata AND mirror that stripped blob + // back into the forest entry (via `sync_forest_user_metadata_after_rewrap`), + // dropping the `chunk_nonces` the owner needs to decrypt. A correct + // chunked-rewrap path must resolve the FULL metadata (forest / index + // body), update only `wrapped_key`/`kek_version`, and re-write the header + // via `header_safe_enc_metadata` while keeping body + forest FULL — + // tracked as a follow-up. `rotate_bucket` collects this as a per-object + // failure (it does not abort the run); single-object rotation is + // unaffected. (Pre-0.6.13 this rewrap failed anyway — the re-PUT of the + // full oversized header was rejected by the gateway with HTTP 400.) + if result + .metadata + .get("x-fula-chunked") + .map(|v| v == "true") + .unwrap_or(false) + { + return Err(ClientError::Encryption(fula_crypto::CryptoError::Decryption( + format!( + "key rotation of chunked (large) object '{}' is not yet supported \ + (index metadata exceeds the gateway header budget); single-object \ + rotation is unaffected", + storage_key + ), + ))); + } + let enc_metadata_str = result.metadata .get("x-fula-encryption") .ok_or_else(|| ClientError::Encryption( @@ -10860,7 +11079,8 @@ impl EncryptedClient { .with_content_type("application/json") .with_metadata("x-fula-encrypted", "true") .with_metadata("x-fula-chunked", "true") - .with_metadata("x-fula-encryption", &index_body); + // Header shrunk for large chunked files; full JSON stays in body + forest. + .with_metadata("x-fula-encryption", &header_safe_enc_metadata(&index_body)); // E51 / W.9.3: pre-compute `BLAKE3(index_body)` so the post-PUT // self-verify can compare master's etag-attested CID against a @@ -11017,16 +11237,17 @@ impl EncryptedClient { } // Parse encryption metadata - let enc_metadata_str = index_result.metadata - .get("x-fula-encryption") - .ok_or_else(|| ClientError::Encryption( - fula_crypto::CryptoError::Decryption("Missing encryption metadata".to_string()) - ))?; + // H-1 / H-2: look up forest entry for the owner-read path (used for + // content_hash verification AND metadata resolution below). + let forest_entry = self.forest_entry_lookup(bucket, &storage_key).await?; - let enc_metadata: serde_json::Value = serde_json::from_str(enc_metadata_str) - .map_err(|e| ClientError::Encryption( - fula_crypto::CryptoError::Decryption(e.to_string()) - ))?; + // Resolve COMPLETE metadata: header (legacy/small) → forest (full + + // CID-stamped) → index body. Large chunked headers drop `chunk_nonces`. + let enc_metadata = resolve_enc_metadata( + index_result.metadata.get("x-fula-encryption").map(|s| s.as_str()), + forest_entry.as_ref(), + Some(index_result.data.as_ref()), + )?; // Unwrap DEK let wrapped_dek: EncryptedData = serde_json::from_value(enc_metadata["wrapped_key"].clone()) @@ -11037,11 +11258,6 @@ impl EncryptedClient { let decryptor = Decryptor::new(self.encryption.key_manager.keypair()); let dek = decryptor.decrypt_dek(&wrapped_dek)?; - // H-1 / H-2: look up forest entry for the owner-read path so the - // chunked engine can verify content_hash and reject legacy-format - // blobs pinned to streaming-v2. - let forest_entry = self.forest_entry_lookup(bucket, &storage_key).await?; - // Delegate to the windowed parallel download path shared with the main // get_object_decrypted flow. Handles streaming-v2 and legacy formats // identically — chunk S3 layout and per-chunk nonce/AAD derivation are @@ -11086,17 +11302,20 @@ impl EncryptedClient { return Ok(full.slice(start.min(full.len())..end.min(full.len()))); } - // Parse encryption metadata - let enc_metadata_str = index_result.metadata - .get("x-fula-encryption") - .ok_or_else(|| ClientError::Encryption( - fula_crypto::CryptoError::Decryption("Missing encryption metadata".to_string()) - ))?; - - let enc_metadata: serde_json::Value = serde_json::from_str(enc_metadata_str) - .map_err(|e| ClientError::Encryption( - fula_crypto::CryptoError::Decryption(e.to_string()) - ))?; + // Resolve COMPLETE metadata: header (legacy/small) → in-hand index + // body. Unlike the full-download paths, the range path runs NO + // forest-backed integrity gate (no enforce_min_version / + // enforce_content_hash), and the index body is ALREADY fetched above — + // so we skip the forest lookup here. For a v7 sharded-HAMT forest that + // lookup can trigger an S3 shard read, an avoidable round-trip on every + // large-file range read (video seek / thumbnails). The body is written + // FULL at upload; if it were somehow incomplete, `resolve_enc_metadata` + // errors rather than mis-decrypting. + let enc_metadata = resolve_enc_metadata( + index_result.metadata.get("x-fula-encryption").map(|s| s.as_str()), + None, + Some(index_result.data.as_ref()), + )?; // Unwrap DEK let wrapped_dek: EncryptedData = serde_json::from_value(enc_metadata["wrapped_key"].clone()) @@ -11393,6 +11612,93 @@ impl RotationReport { mod tests { use super::*; + /// A chunked enc-metadata JSON whose serialized form exceeds the 16 KB + /// header budget (the per-chunk arrays dominate), mirroring a large file. + fn big_chunked_meta() -> String { + let chunk_nonces: Vec = + (0..220).map(|i| format!("chunk-nonce-padding-{:04}", i)).collect(); + let chunk_cids: Vec> = (0..220) + .map(|_| (0u16..36).map(|x| (x % 251) as u8).collect()) + .collect(); + serde_json::json!({ + "version": 4, + "algorithm": "AES-256-GCM", + "wrapped_key": {"alg": "HPKE-X25519", "ct": "d3JhcHBlZC1kZWstYmxvYg"}, + "kek_version": 1, + "chunked": { + "format": "streaming-v2", + "num_chunks": 220, + "chunk_size": 262144, + "chunk_nonces": chunk_nonces, + "chunk_cids": chunk_cids, + } + }) + .to_string() + } + + #[test] + fn header_safe_strips_only_per_chunk_arrays_when_oversized() { + let full = big_chunked_meta(); + assert!(full.len() > 16 * 1024, "fixture must exceed the header budget"); + let header = header_safe_enc_metadata(&full); + assert!(header.len() < full.len(), "oversized header must be stripped"); + assert!(header.len() <= 16 * 1024, "stripped header must fit the budget"); + let v: serde_json::Value = serde_json::from_str(&header).unwrap(); + // Per-chunk arrays removed from the HEADER copy... + assert!(v["chunked"].get("chunk_nonces").is_none()); + assert!(v["chunked"].get("chunk_cids").is_none()); + // ...but scalars + wrapped_key (needed to unwrap the DEK) survive. + assert_eq!(v["chunked"]["num_chunks"], 220); + assert!(v.get("wrapped_key").is_some()); + assert_eq!(v["version"], 4); + } + + #[test] + fn header_safe_passes_small_metadata_verbatim() { + // Small files keep their full header (old-client read compatibility). + let small = serde_json::json!({ + "version": 4, "wrapped_key": {"k": "v"}, + "chunked": {"num_chunks": 2, "chunk_nonces": ["a", "b"]} + }) + .to_string(); + assert!(small.len() <= 16 * 1024); + assert_eq!(header_safe_enc_metadata(&small), small); + } + + #[test] + fn enc_metadata_completeness_keys_on_chunk_nonces() { + assert!(enc_metadata_is_complete(&serde_json::json!({"wrapped_key": {}, "nonce": "x"}))); + assert!(enc_metadata_is_complete(&serde_json::json!({"chunked": {"chunk_nonces": ["a"]}}))); + assert!(!enc_metadata_is_complete(&serde_json::json!({"chunked": {"num_chunks": 3}}))); + assert!(!enc_metadata_is_complete(&serde_json::json!({"chunked": {"chunk_nonces": []}}))); + } + + #[test] + fn resolve_enc_metadata_precedence_and_roundtrip() { + let full = big_chunked_meta(); + let stripped = header_safe_enc_metadata(&full); // incomplete: no chunk_nonces + let full_v: serde_json::Value = serde_json::from_str(&full).unwrap(); + let expect_nonces = full_v["chunked"]["chunk_nonces"].clone(); + + // 1. Complete header wins; body is ignored (and never mis-parsed). + let r = resolve_enc_metadata(Some(&full), None, Some(b"not-json-ciphertext")).unwrap(); + assert_eq!(r["chunked"]["chunk_nonces"], expect_nonces); + + // 2. Stripped header + no forest -> recover the FULL blob from the index + // body (the share-recipient / offline-by-CID path). Round-trip: the + // recovered chunk_nonces equal the original (decryption-correctness). + let r = resolve_enc_metadata(Some(&stripped), None, Some(full.as_bytes())).unwrap(); + assert_eq!(r["chunked"]["chunk_nonces"], expect_nonces); + assert_eq!(r["chunked"]["chunk_nonces"].as_array().unwrap().len(), 220); + + // 3. No usable source anywhere -> hard error (never returns incomplete). + assert!(resolve_enc_metadata(Some(&stripped), None, Some(stripped.as_bytes())).is_err()); + + // 4. Absent header (offline empty-metadata) + body present -> body. + let r = resolve_enc_metadata(None, None, Some(full.as_bytes())).unwrap(); + assert_eq!(r["chunked"]["num_chunks"], 220); + } + #[test] fn test_encryption_config() { let config1 = EncryptionConfig::new(); diff --git a/crates/fula-client/tests/large_chunked_share_e2e.rs b/crates/fula-client/tests/large_chunked_share_e2e.rs new file mode 100644 index 0000000..46d7f45 --- /dev/null +++ b/crates/fula-client/tests/large_chunked_share_e2e.rs @@ -0,0 +1,240 @@ +//! E2E (real server): SHARE + COLLABORATE token creation must work for a LARGE +//! chunked file after the 0.6.13 header-stripping fix. +//! +//! Bug context: `header_safe_enc_metadata` strips the per-chunk arrays +//! (`chunk_nonces` / `chunk_cids`) from the `x-fula-encryption` HTTP header for +//! large (chunked) files so the index PUT fits the gateway's +//! `large_client_header_buffers`. Share/collab tokens are built by +//! `get_object_encryption_metadata_with_fallback` -> `ShareBuilder.chunked_metadata` +//! (fula-flutter `create_share_token`; FxFiles `collaboration_service.dart` uses +//! the SAME call), and the recipient CANNOT decrypt unless the token carries the +//! FULL `chunk_nonces`. This test proves fix #1 recovers them from the forest / +//! index body even though the stored HTTP header is stripped. +//! +//! Assertions: +//! 1. NON-VACUOUS: the stored HEAD `x-fula-encryption` header IS stripped (has +//! a `chunked` block but NO `chunk_nonces`). Confirms the file crosses the +//! 16 KB header budget and the strip actually happened — without this the +//! rest of the test could pass on a small file that was never stripped. +//! 2. FIX #1: `get_object_encryption_metadata_with_fallback` returns COMPLETE +//! metadata (non-empty `chunked.chunk_nonces`). +//! 3. SHARE/COLLAB: the built share token's `chunked_metadata` carries +//! non-empty `chunk_nonces`. +//! 4. END-TO-END: a recipient (share-to-self; same account/JWT, no proxy) +//! decrypts via `get_object_with_share` to the EXACT uploaded bytes. +//! +//! `#[ignore]` — needs network + real credentials. Run (PowerShell, env from +//! e2e-credentials.env): +//! cargo test -p fula-client --test large_chunked_share_e2e --release -- --ignored --nocapture +//! +//! Required env: FULA_S3, FULA_JWT, FULA_TEST_PROVIDER, FULA_TEST_OAUTH_SUB, +//! FULA_TEST_EMAIL (the Mode A derivation triple). + +#![cfg(not(target_arch = "wasm32"))] + +use bytes::Bytes; +use fula_client::{Config, EncryptedClient, EncryptionConfig, FulaClient}; +use fula_crypto::{ + hpke::{Decryptor, EncryptedData}, + keys::{PublicKey, SecretKey}, + sharing::{ShareBuilder, ShareToken}, +}; + +fn env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("missing required env {name}")) +} + +/// True iff `v["chunked"]["chunk_nonces"]` is a present, non-empty array. +fn chunked_block_has_nonces(v: &serde_json::Value) -> bool { + v.get("chunked") + .and_then(|c| c.get("chunk_nonces")) + .and_then(|n| n.as_array()) + .map(|a| !a.is_empty()) + .unwrap_or(false) +} + +/// True iff `v["chunk_nonces"]` is a present, non-empty array (the share +/// token's `chunked_metadata` IS the `chunked` block, so nonces are top-level). +fn top_level_has_nonces(v: &serde_json::Value) -> bool { + v.get("chunk_nonces") + .and_then(|n| n.as_array()) + .map(|a| !a.is_empty()) + .unwrap_or(false) +} + +#[tokio::test] +#[ignore = "real-server; needs FULA_S3 + FULA_JWT + Mode A triple"] +async fn large_chunked_share_token_carries_chunk_nonces() { + let s3 = env("FULA_S3"); + let jwt = env("FULA_JWT"); + let input = format!( + "{}:{}:{}", + env("FULA_TEST_PROVIDER"), + env("FULA_TEST_OAUTH_SUB"), + env("FULA_TEST_EMAIL"), + ); + let kek = fula_crypto::hashing::derive_key_argon2id("fula-files-v1", input.as_bytes()); + let secret = SecretKey::from_bytes(&kek).expect("32-byte secret from Argon2id"); + + let mut config = Config::new(&s3).with_token(&jwt); + // Match FxFiles production: stamps chunk_cids into the index metadata, which + // (with ~480 chunks) pushes it well past the gateway header budget. + config.walkable_v8_writer_enabled = true; + let client = EncryptedClient::new(config, EncryptionConfig::from_secret_key(secret)) + .expect("EncryptedClient::new"); + + let epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let bucket = format!("e2e-share-bigchunk-{epoch}-v8"); + eprintln!("[share_e2e] BUCKET={bucket}"); + if let Err(e) = client.create_bucket(&bucket).await { + eprintln!("[share_e2e] create_bucket({bucket}) -> {e} (continuing)"); + } + + // ~120 MB -> ~480 chunks at 256 KB -> ~70 KB index metadata, well over the + // 16 KB header budget, so `header_safe_enc_metadata` MUST strip the header. + let size = 120 * 1024 * 1024; + let mut data = vec![0u8; size]; + for (i, b) in data.iter_mut().enumerate() { + *b = (i % 251) as u8; + } + let key_path = "/big-share.bin"; + + eprintln!( + "[share_e2e] uploading {} bytes (~{} chunks)...", + size, + size / (256 * 1024) + ); + client + .put_object_flat( + &bucket, + key_path, + Bytes::from(data.clone()), + Some("application/octet-stream"), + ) + .await + .expect("upload large chunked file"); + eprintln!("[share_e2e] upload OK"); + + // storage_key (HMAC-derived) for the file — same lookup FxFiles uses. + let listed = client + .list_files_from_forest(&bucket) + .await + .expect("list_files_from_forest"); + let file_meta = listed + .iter() + .find(|f| f.original_key == key_path) + .unwrap_or_else(|| panic!("listing must include {key_path}")); + let storage_key = file_meta.storage_key.clone(); + eprintln!("[share_e2e] storage_key={storage_key}"); + + // ── Assertion 1 (NON-VACUOUS): the stored HTTP header IS stripped. ── + // Read the raw S3 user-metadata header via a plain (non-encrypted) client. + let raw = FulaClient::new(Config::new(&s3).with_token(&jwt)).expect("plain FulaClient::new"); + let head = raw.head_object(&bucket, &storage_key).await.expect("head_object"); + let header_str = head + .metadata + .get("x-fula-encryption") + .expect("encrypted object must carry an x-fula-encryption header"); + let header_json: serde_json::Value = + serde_json::from_str(header_str).expect("parse header JSON"); + assert!( + header_json.get("chunked").is_some(), + "test file must be chunked; header={header_str}" + ); + assert!( + !chunked_block_has_nonces(&header_json), + "VACUOUS TEST GUARD: the stored header still carries chunk_nonces — the \ + metadata did not exceed the 16 KB budget, so header-stripping was never \ + exercised. chunked block: {:?}", + header_json.get("chunked") + ); + eprintln!("[share_e2e] OK #1: stored header is STRIPPED (chunked block present, no chunk_nonces)"); + + // ── Assertion 2 (FIX #1): with_fallback recovers the FULL metadata. ── + let full_str = client + .get_object_encryption_metadata_with_fallback(&bucket, &storage_key) + .await + .expect("get_object_encryption_metadata_with_fallback"); + let full_json: serde_json::Value = + serde_json::from_str(&full_str).expect("parse full metadata JSON"); + assert!( + chunked_block_has_nonces(&full_json), + "FIX #1: with_fallback MUST return metadata WITH chunk_nonces (recovered \ + from forest / index body), got: chunked={:?}", + full_json.get("chunked") + ); + eprintln!("[share_e2e] OK #2: with_fallback recovered chunk_nonces despite stripped header"); + + // ── Assertion 3 (SHARE/COLLAB): the built token carries chunk_nonces. ── + // Mirrors fula-flutter::create_share_token (collab uses the same call). + let wrapped_key: EncryptedData = + serde_json::from_value(full_json["wrapped_key"].clone()).expect("parse wrapped_key"); + let owner_keypair = client.encryption_config().key_manager().keypair(); + let dek = Decryptor::new(owner_keypair) + .decrypt_dek(&wrapped_key) + .expect("owner unwraps own DEK"); + + // Share to self (same account/JWT) so the recipient read needs no proxy. + let recipient_pk = SecretKey::from_bytes(&kek) + .expect("recipient secret") + .public_key(); + let mut builder = ShareBuilder::new(owner_keypair, &recipient_pk, &dek) + .path_scope(&storage_key) + .read_only(); + if let Some(nonce) = full_json["nonce"].as_str() { + builder = builder.nonce(nonce); + } + if let Some(v) = full_json["version"].as_u64() { + builder = builder.encryption_version(v as u8); + } + let chunked_json = + serde_json::to_string(&full_json["chunked"]).expect("serialize chunked metadata"); + builder = builder.chunked_metadata(chunked_json); + let token = builder.build().expect("ShareBuilder.build"); + let token_json = serde_json::to_string(&token).expect("serialize ShareToken"); + + // Parse the token JSON as a generic value (robust to field visibility) and + // verify its embedded chunked_metadata carries non-empty chunk_nonces. + let token_value: serde_json::Value = + serde_json::from_str(&token_json).expect("parse token JSON"); + let token_chunked_str = token_value["chunked_metadata"] + .as_str() + .expect("token must carry a chunked_metadata string"); + let token_chunked_json: serde_json::Value = + serde_json::from_str(token_chunked_str).expect("parse token chunked_metadata"); + assert!( + top_level_has_nonces(&token_chunked_json), + "SHARE/COLLAB: token chunked_metadata MUST carry non-empty chunk_nonces; got {:?}", + token_chunked_json + ); + eprintln!("[share_e2e] OK #3: share/collab token carries chunk_nonces"); + + // ── Assertion 4 (END-TO-END): recipient decrypts to the exact bytes. ── + let parsed: ShareToken = serde_json::from_str(&token_json).expect("recipient parses token"); + let accepted = client.accept_share(&parsed).expect("accept_share (share-to-self)"); + let downloaded = client + .get_object_with_share(&bucket, &storage_key, &storage_key, &accepted) + .await + .expect("recipient get_object_with_share"); + assert_eq!( + downloaded.len(), + data.len(), + "decrypted length must equal the uploaded length" + ); + assert!( + downloaded[..] == data[..], + "recipient must decrypt to the EXACT bytes the owner uploaded" + ); + eprintln!( + "[share_e2e] OK #4: recipient decrypted {} bytes — exact match", + downloaded.len() + ); + + // Cleanup (best-effort). + let _ = client.delete_object_flat(&bucket, key_path).await; + let _ = client.delete_bucket(&bucket).await; + eprintln!("[share_e2e] PASS"); +} diff --git a/crates/fula-client/tests/large_index_put_repro_e2e.rs b/crates/fula-client/tests/large_index_put_repro_e2e.rs new file mode 100644 index 0000000..7b018c0 --- /dev/null +++ b/crates/fula-client/tests/large_index_put_repro_e2e.rs @@ -0,0 +1,95 @@ +//! REPRO PROBE (not a pass/fail gate): upload a LARGE multi-chunk file via the +//! web chunked code path (`put_object_flat` -> `put_object_chunked_internal`) +//! against the LIVE master, to reproduce the "index PUT after all chunks -> +//! ERR_CONNECTION_CLOSED / upstream prematurely closed" failure the user hit on +//! a ~470-chunk (~120 MB) upload. Prints the outcome + the unique bucket name so +//! the gateway logs can be grepped for that exact window. Round-trips on success. +//! +//! `#[ignore]` — needs network + real credentials. Run (PowerShell, env from +//! e2e-credentials.env): +//! cargo test -p fula-client --test large_index_put_repro_e2e --release -- --ignored --nocapture +//! +//! Required env: FULA_S3, FULA_JWT, FULA_TEST_PROVIDER, FULA_TEST_OAUTH_SUB, +//! FULA_TEST_EMAIL (the Mode A derivation triple). + +#![cfg(not(target_arch = "wasm32"))] + +use bytes::Bytes; +use fula_client::{Config, EncryptedClient, EncryptionConfig}; +use fula_crypto::keys::SecretKey; + +fn env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("missing required env {name}")) +} + +#[tokio::test] +#[ignore = "real-server repro; needs FULA_S3 + FULA_JWT + Mode A triple"] +async fn large_chunked_upload_repro_against_real_master() { + let s3 = env("FULA_S3"); + let jwt = env("FULA_JWT"); + let input = format!( + "{}:{}:{}", + env("FULA_TEST_PROVIDER"), + env("FULA_TEST_OAUTH_SUB"), + env("FULA_TEST_EMAIL"), + ); + let key = fula_crypto::hashing::derive_key_argon2id("fula-files-v1", input.as_bytes()); + let secret = SecretKey::from_bytes(&key).expect("32-byte secret from Argon2id"); + + let mut config = Config::new(&s3).with_token(&jwt); + config.walkable_v8_writer_enabled = true; // match FxFiles production (stamps chunk_cids -> big index metadata) + let client = EncryptedClient::new(config, EncryptionConfig::from_secret_key(secret)) + .expect("EncryptedClient::new"); + + let epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let bucket = format!("e2e-bigindex-{epoch}-v8"); + eprintln!("[repro] BUCKET={bucket} (grep gateway logs for this)"); + if let Err(e) = client.create_bucket(&bucket).await { + eprintln!("[repro] create_bucket({bucket}) -> {e} (continuing)"); + } + + // ~120 MB -> ~480 chunks at 256 KB -> ~70 KB chunked-index metadata, + // matching the user's failing upload. + let size = 120 * 1024 * 1024; + let mut data = vec![0u8; size]; + for (i, b) in data.iter_mut().enumerate() { + *b = (i % 251) as u8; + } + let key_path = "/big-repro.bin"; + + eprintln!("[repro] uploading {} bytes (~{} chunks) at {} ...", size, size / (256 * 1024), epoch); + let started = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let result = client + .put_object_flat(&bucket, key_path, Bytes::from(data.clone()), Some("application/octet-stream")) + .await; + let ended = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + + match &result { + Ok(r) => eprintln!("[repro] UPLOAD OK etag={} (started={started} ended={ended})", r.etag), + Err(e) => eprintln!("[repro] UPLOAD FAILED: {e} (started={started} ended={ended})"), + } + + // Round-trip only if it uploaded. + if let Ok(r) = &result { + let _ = r; + match client.get_object_flat(&bucket, key_path).await { + Ok(got) => eprintln!("[repro] download OK, {} bytes, match={}", got.len(), &got[..] == &data[..]), + Err(e) => eprintln!("[repro] download FAILED: {e}"), + } + let _ = client.delete_object_flat(&bucket, key_path).await; + } + let _ = client.delete_bucket(&bucket).await; + + // Surface the outcome as the test result so --nocapture shows it clearly, + // but the eprintln above is the real signal regardless of pass/fail. + result.expect("large chunked upload should succeed (FAILS = repro of the reported bug)"); +}