fix: don't proxy backend ETag by default

docs/Config.md

@@ -37,6 +37,7 @@ Each target may have the following properties:
         * `access_key_path`: Path to the S3 access key (for private buckets)
         * `secret_key_path`: Path to the S3 secret key (for private buckets)
         * `config`: Botocore configuration options (see below)
+        * `proxy_etag`: If true, the backend's `ETag` is proxied through to clients (default: false). Most S3-compatible backends (e.g. VAST) return an `ETag` that is not a true content MD5, which causes the AWS CLI/SDK download integrity check to fail with "Unable to verify integrity of data download". Only set this to true for backends known to return a real content-MD5 `ETag` (e.g. real AWS S3).
     * *file*: Local filesystem targets. Options:
         * `path`: Path to the root
         * `buffer_size`: Size of chunks (in bytes) when streaming file content (default: 8192)

tests/java/pom.xml

@@ -13,6 +13,8 @@
         <maven.compiler.target>17</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <aws.sdk.version>2.25.60</aws.sdk.version>
+        <!-- v1 SDK is what org.janelia.saalfeldlab.n5.s3 (and the Fiji error trace) actually uses -->
+        <aws.sdk.v1.version>1.12.780</aws.sdk.v1.version>
     </properties>
 
     <dependencies>
@@ -21,6 +23,16 @@
             <artifactId>s3</artifactId>
             <version>${aws.sdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>${aws.sdk.v1.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.11.0</version>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

tests/java/src/test/java/org/janelia/x2s3/S3CompatTest.java

@@ -73,7 +73,9 @@ public void testListObjectsV2Basic() {
         for (int i = 0; i < awsObjects.size(); i++) {
             assertEquals("Key[" + i + "]", awsObjects.get(i).key(), proxyObjects.get(i).key());
             assertEquals("Size[" + i + "]", awsObjects.get(i).size(), proxyObjects.get(i).size());
-            assertEquals("ETag[" + i + "]", awsObjects.get(i).eTag(), proxyObjects.get(i).eTag());
+            // ETag is not proxied by default (proxy_etag defaults to false), since many
+            // S3-compatible backends return an ETag that isn't a real content MD5.
+            assertNull("ETag[" + i + "] should not be proxied by default", proxyObjects.get(i).eTag());
             assertEquals("StorageClass[" + i + "]",
                     awsObjects.get(i).storageClassAsString(),
                     proxyObjects.get(i).storageClassAsString());
@@ -212,9 +214,9 @@ public void testGetObject() {
         assertEquals("Content-Length",
                 awsResp.response().contentLength(),
                 proxyResp.response().contentLength());
-        assertEquals("ETag",
-                awsResp.response().eTag(),
-                proxyResp.response().eTag());
+        // ETag is not proxied by default (proxy_etag defaults to false), since many
+        // S3-compatible backends return an ETag that isn't a real content MD5.
+        assertNull("ETag should not be proxied by default", proxyResp.response().eTag());
 
         // Compare body content
         try {
@@ -287,8 +289,9 @@ public void testHeadObject() {
                 awsResp.contentType(), proxyResp.contentType());
         assertEquals("Content-Length",
                 awsResp.contentLength(), proxyResp.contentLength());
-        assertEquals("ETag",
-                awsResp.eTag(), proxyResp.eTag());
+        // ETag is not proxied by default (proxy_etag defaults to false), since many
+        // S3-compatible backends return an ETag that isn't a real content MD5.
+        assertNull("ETag should not be proxied by default", proxyResp.eTag());
         assertEquals("Last-Modified",
                 awsResp.lastModified(), proxyResp.lastModified());
     }

tests/java/src/test/java/org/janelia/x2s3/S3v1IntegrityTest.java

@@ -0,0 +1,137 @@
+package org.janelia.x2s3;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.util.IOUtils;
+import com.google.gson.JsonParser;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+
+/**
+ * Reproduces the "Unable to verify integrity of data download" error reported by
+ * N5/Zarr readers (org.janelia.saalfeldlab.n5.s3), which use the AWS SDK v1 S3 client.
+ * SDK v1 validates GetObject downloads by comparing a locally computed MD5 against the
+ * ETag header -- unless the ETag is absent, looks like a multipart ETag (has a "-N"
+ * suffix), or the object is SSE-C/SSE-KMS encrypted. That's a different code path than
+ * the SDK v2 client used in {@link S3CompatTest}, which does not do this validation.
+ *
+ * Point this at the same x2s3 target/key that failed in Fiji/BigDataViewer to confirm
+ * whether setting `proxy_etag: false` on that target (the default since this fix)
+ * resolves the integrity error, and separately whether the downloaded bytes are
+ * actually intact.
+ *
+ * Configure via env vars:
+ *   PROXY_ENDPOINT (default http://localhost:8000)
+ *   TEST_BUCKET    (default janelia-data-examples)
+ *   TEST_KEY       (default jrc_mus_lung_covid.n5/attributes.json)
+ */
+public class S3v1IntegrityTest {
+
+    private static AmazonS3 client;
+    private static String bucket;
+    private static String key;
+
+    @BeforeClass
+    public static void setup() {
+        String endpoint = System.getenv().getOrDefault("PROXY_ENDPOINT", "http://localhost:8000");
+        bucket = System.getenv().getOrDefault("TEST_BUCKET", "janelia-data-examples");
+        key = System.getenv().getOrDefault("TEST_KEY", "jrc_mus_lung_covid.n5/attributes.json");
+
+        client = AmazonS3ClientBuilder.standard()
+                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "us-east-1"))
+                .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
+                .withPathStyleAccessEnabled(true)
+                .build();
+    }
+
+    @Test
+    public void testGetObjectPassesIntegrityCheck() throws Exception {
+        // This is the exact path that threw SdkClientException in Fiji: reading the
+        // object body fully triggers DigestValidationInputStream's end-of-stream MD5
+        // comparison against the ETag.
+        S3Object obj = client.getObject(bucket, key);
+        byte[] body;
+        try {
+            body = IOUtils.toByteArray(obj.getObjectContent());
+        } finally {
+            obj.close();
+        }
+        assertTrue("Body should not be empty", body.length > 0);
+    }
+
+    @Test
+    public void testDownloadedJsonIsWellFormed() throws Exception {
+        // Independent of the ETag check: verify the bytes we actually got are intact
+        // and not truncated/corrupted, since that's a separate failure mode from the
+        // ETag mismatch (and was seen in the Fiji trace as a MalformedJsonException).
+        S3Object obj = client.getObject(bucket, key);
+        byte[] body;
+        try {
+            body = IOUtils.toByteArray(obj.getObjectContent());
+        } finally {
+            obj.close();
+        }
+        String text = new String(body, StandardCharsets.UTF_8).trim();
+        assertTrue("Response should look like JSON, got: " + text,
+                text.startsWith("{") && text.endsWith("}"));
+        // Throws JsonSyntaxException if malformed, same as the Zarr reader.
+        JsonParser.parseString(text);
+    }
+
+    @Test
+    public void testContentLengthMatchesActualBytes() throws Exception {
+        ObjectMetadata meta = client.getObjectMetadata(bucket, key);
+        S3Object obj = client.getObject(bucket, key);
+        byte[] body;
+        try {
+            body = IOUtils.toByteArray(obj.getObjectContent());
+        } finally {
+            obj.close();
+        }
+        assertEquals("Downloaded byte count should match Content-Length",
+                meta.getContentLength(), body.length);
+    }
+
+    @Test
+    public void testReportedETagVsActualMd5() throws Exception {
+        // Purely diagnostic: disable v1 SDK's own validation so a mismatch here prints
+        // instead of throwing, to confirm whether the backend's ETag is a real content
+        // MD5 at all.
+        System.setProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation", "true");
+        try {
+            ObjectMetadata meta = client.getObjectMetadata(bucket, key);
+            String reportedETag = meta.getETag();
+
+            S3Object obj = client.getObject(bucket, key);
+            byte[] body;
+            try {
+                body = IOUtils.toByteArray(obj.getObjectContent());
+            } finally {
+                obj.close();
+            }
+
+            MessageDigest md5 = MessageDigest.getInstance("MD5");
+            StringBuilder sb = new StringBuilder();
+            for (byte b : md5.digest(body)) {
+                sb.append(String.format("%02x", b & 0xff));
+            }
+            String actualMd5 = sb.toString();
+
+            System.out.println("Reported ETag: " + reportedETag);
+            System.out.println("Actual MD5:    " + actualMd5);
+            System.out.println("Bytes read:    " + body.length);
+        } finally {
+            System.clearProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation");
+        }
+    }
+}

tests/test_boto.py

@@ -19,6 +19,10 @@ def get_settings():
         Target(
             name='janelia-data-examples',
             options={'bucket':'janelia-data-examples'}
+        ),
+        Target(
+            name='janelia-data-examples-with-etag',
+            options={'bucket':'janelia-data-examples', 'proxy_etag': True}
         )
     ]
     return settings
@@ -108,6 +112,15 @@ def test_head_object(app, s3_client):
 def test_get_object(app, s3_client):
     response = s3_client.get_object(Bucket='janelia-data-examples', Key='jrc_mus_lung_covid.n5/attributes.json')
     assert response['ResponseMetadata']['HTTPStatusCode'] == 200
+    assert 'ETag' not in response
+    json_obj = response['Body'].read().decode('utf-8')
+    assert 'n5' in json_obj
+
+
+def test_get_object_with_etag(app, s3_client):
+    response = s3_client.get_object(Bucket='janelia-data-examples-with-etag', Key='jrc_mus_lung_covid.n5/attributes.json')
+    assert response['ResponseMetadata']['HTTPStatusCode'] == 200
+    assert 'ETag' in response
     json_obj = response['Body'].read().decode('utf-8')
     assert 'n5' in json_obj
 

x2s3/client_aioboto.py

@@ -73,6 +73,11 @@ def __init__(self, proxy_kwargs, **kwargs):
         self.bucket_name = kwargs['bucket']
         self.bucket_prefix = kwargs.get('prefix')
 
+        # Some S3-compatible backends (e.g. VAST) return ETags that are not a
+        # true content MD5, which breaks the AWS CLI/SDK download integrity
+        # check. Allow disabling ETag proxying per-target for those backends.
+        self.proxy_etag = kwargs.get('proxy_etag', False)
+
         self.anonymous = True
         access_key,secret_key = '',''
 
@@ -162,12 +167,14 @@ async def head_object(self, key: str):
         try:
             s3_res = await self.client.head_object(Bucket=self.bucket_name, Key=real_key)
             headers = {
-                "ETag": s3_res.get("ETag"),
                 "Accept-Ranges": "bytes",
                 "Content-Length": str(s3_res.get("ContentLength")),
                 "Last-Modified": s3_res.get("LastModified").strftime("%a, %d %b %Y %H:%M:%S GMT"),
             }
 
+            if self.proxy_etag:
+                headers["ETag"] = s3_res.get("ETag")
+
             content_type = guess_content_type(real_key)
             headers['Content-Type'] = content_type
 
@@ -226,7 +233,7 @@ async def open_object(self, key: str, range_header: str = None):
             if "last-modified" in res_headers:
                 headers["Last-Modified"] = res_headers["last-modified"]
 
-            if "etag" in res_headers:
+            if self.proxy_etag and "etag" in res_headers:
                 headers["ETag"] = res_headers["etag"]
 
             return S3ObjectHandle(
@@ -314,7 +321,7 @@ async def list_objects_v2(self,
                 contents.append({
                     'Key': remove_prefix(self.bucket_prefix, obj["Key"]),
                     'LastModified': obj["LastModified"].strftime("%Y-%m-%dT%H:%M:%S.000Z"),
-                    'ETag': obj.get("ETag"),
+                    'ETag': obj.get("ETag") if self.proxy_etag else None,
                     'Size': obj.get("Size"),
                     'StorageClass': obj.get("StorageClass")
                 })