S3 fixes for non AWS instances

doc/release-notes/12454-S3Fixes.md

@@ -0,0 +1,5 @@
+### Support for Backblaze B2 as an S3 store, improved support for storJ
+
+An improvement to the .disable-tagging=true support for S3 stores now allows use of BackBlaze B2 as an S3 implementation (and may help other stores that do not handle tagging).
+
+The /api/datasets/<id>/cleanStorage endpoint will now work for datasets with more than 1000 files when storJ is used as the S3 store.

doc/sphinx-guides/source/installation/config.rst

@@ -1481,9 +1481,14 @@ Reported Working S3-Compatible Storage
 ######################################
 
 `Ceph Object Gateway <https://docs.ceph.com/en/reef/radosgw/#ceph-object-gateway>`_ (added July 2026/Dataverse v6.12)
-Set ``dataverse.files.<id>.disable-multipart-download-for-indirect-download=true`` if not using direct download. 
+Set ``dataverse.files.<id>.disable-multipart-download-for-indirect-download=true`` if not using direct download.
 (This forces the S3 server to handle part reassembly and avoid incompatible headers that cause `412` errors from the Ceph Gateway.)
 
+`BackBlaze B2 <https://www.backblaze.com/cloud-storage>`_
+  (As of 2026-06-11)
+  Set ``dataverse.files.<id>.disable-tagging=true``, as B2 does not support tagging (and will fail without this setting).
+  Tested with ``.path-style-access=true``, ``.download-redirect=true``, and ``.upload-redirect=true``.
+
 `StorJ Object Store <https://www.storj.io>`_
  StorJ is a distributed object store that can be configured with an S3 gateway. Per the S3 Storage instructions above, you'll first set up the StorJ S3 store by defining the id, type, and label. After following the general installation, set the following configuration to use a StorJ object store: ``dataverse.files.<id>.chunked-encoding=false``. For step-by-step instructions see https://docs.storj.io/dcs/how-tos/dataverse-integration-guide/
 

src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java

@@ -77,23 +77,17 @@ public void copyFromS3(Dataset dataset, String s3ImportPath) throws IOException
         ListObjectsV2Request listReq = ListObjectsV2Request.builder()
                 .bucket(dcmBucketName)
                 .prefix(dcmDatasetKey)
+                .maxKeys(1000)
                 .build();
 
-        ListObjectsV2Response listRes;
+        List<S3Object> storedDcmDatasetFilesSummary = new ArrayList<>();
         try {
-            listRes = s3.listObjectsV2(listReq);
+            s3.listObjectsV2Paginator(listReq).stream()
+                .flatMap(r -> r.contents().stream())
+                .forEach(storedDcmDatasetFilesSummary::add);
         } catch (S3Exception se) {
             logger.info("Caught an S3Exception in s3ImportUtil: " + se.getMessage());
-            throw new IOException("S3 listAuxObjects: failed to get a listing for " + dcmDatasetKey);
-        }
-
-        List<S3Object> storedDcmDatasetFilesSummary = new ArrayList<>(listRes.contents());
-
-        while (listRes.isTruncated()) {
-            logger.fine("S3 listAuxObjects: going to next page of list");
-            listReq = listReq.toBuilder().continuationToken(listRes.nextContinuationToken()).build();
-            listRes = s3.listObjectsV2(listReq);
-            storedDcmDatasetFilesSummary.addAll(listRes.contents());
+            throw new IOException("S3 listObjects: failed to get a listing for " + dcmDatasetKey);
         }
 
         for (S3Object item : storedDcmDatasetFilesSummary) {

src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java

@@ -735,63 +735,56 @@ private File createTempFile(Path path, InputStream inputStream) throws IOExcepti
         return targetFile;
     }
 
-    @Override
-    public List<String> listAuxObjects() throws IOException {
-        if (!this.canWrite()) {
-            open();
-        }
-        String prefix = getDestinationKey("");
-
-        List<String> ret = new ArrayList<>();
-        ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
+    private List<S3Object> listObjects(String prefix, String methodName) throws IOException {
+        List<S3Object> objects = new ArrayList<>();
+        ListObjectsV2Request listRequest = ListObjectsV2Request.builder()
+                .bucket(bucketName)
+                .prefix(prefix)
+                .maxKeys(1000) // Required for storJ
                 .build();
 
-        ListObjectsV2Response listObjectsResponse = null;
         try {
-            listObjectsResponse = s3ReadClient.listObjectsV2(listObjectsReqManual).get();
+            ListObjectsV2Response listResponse;
+            String nextToken = null;
+            do {
+                ListObjectsV2Request.Builder reqBuilder = listRequest.toBuilder();
+                if (nextToken != null) {
+                    reqBuilder = reqBuilder.continuationToken(nextToken);
+                }
+                ListObjectsV2Request req = reqBuilder.build();
+                listResponse = s3ReadClient.listObjectsV2(req).get();
+                objects.addAll(listResponse.contents());
+                nextToken = listResponse.nextContinuationToken();
+                if (listResponse.isTruncated() && nextToken == null) {
+                    logger.warning("S3 " + methodName + ": list is truncated but nextContinuationToken is null; stopping to avoid infinite loop");
+                    break;
+                }
+            } while (listResponse.isTruncated());
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
             throw new IOException("S3 listAuxObjects: failed to get a listing for " + prefix, e);
         } catch (ExecutionException e) {
             throw new IOException("S3 listAuxObjects: failed to get a listing for " + prefix, e);
         }
+        return objects;
+    }
 
-        if (listObjectsResponse == null) {
-            return ret;
-        }
-
-        List<S3Object> storedAuxFilesSummary = new ArrayList<>(listObjectsResponse.contents());
-
-        try {
-            String nextContinuationToken = listObjectsResponse.nextContinuationToken();
-            while (nextContinuationToken != null) {
-                logger.fine("S3 listAuxObjects: going to next page of list");
-                ListObjectsV2Request nextReq = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
-                    .continuationToken(nextContinuationToken).build();
-
-                ListObjectsV2Response nextResponse = s3ReadClient.listObjectsV2(nextReq).get();
-                if (nextResponse != null) {
-                    storedAuxFilesSummary.addAll(nextResponse.contents());
-                    nextContinuationToken = nextResponse.nextContinuationToken();
-                } else {
-                    nextContinuationToken = null;
-                }
-            }
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new IOException("S3AccessIO: Failed to get aux objects for listing.", e);
-        }
-        catch (ExecutionException e) {
-            throw new IOException("S3AccessIO: Failed to get aux objects for listing.", e);
+    @Override
+    public List<String> listAuxObjects() throws IOException {
+        if (!this.canWrite()) {
+            open();
         }
+        String prefix = getDestinationKey("");
+        List<S3Object> contents = listObjects(prefix, "listAuxObjects");
 
-        for (S3Object item : storedAuxFilesSummary) {
-            String destinationKey = item.key();
-            String fileName = destinationKey.substring(destinationKey.lastIndexOf(".") + 1);
-            logger.fine("S3 cached aux object fileName: " + fileName);
-            ret.add(fileName);
-        }
-        return ret;
+        return contents.stream()
+                .map(item -> {
+                    String destinationKey = item.key();
+                    String fileName = destinationKey.substring(destinationKey.lastIndexOf(".") + 1);
+                    logger.fine("S3 cached aux object fileName: " + fileName);
+                    return fileName;
+                })
+                .collect(Collectors.toList());
     }
 
     @Override
@@ -823,25 +816,7 @@ public void deleteAllAuxObjects() throws IOException {
         }
 
         String prefix = getDestinationKey("");
-
-        List<S3Object> storedAuxFilesSummary = new ArrayList<>();
-        try {
-            ListObjectsV2Request listRequest = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).build();
-
-            ListObjectsV2Response listResponse;
-            do {
-                listResponse = s3ReadClient.listObjectsV2(listRequest).get();
-                storedAuxFilesSummary.addAll(listResponse.contents());
-
-                listRequest = listRequest.toBuilder().continuationToken(listResponse.nextContinuationToken()).build();
-            } while (listResponse.isTruncated());
-
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new IOException("S3AccessIO: Failed to get aux objects for listing to delete.", e);
-        } catch (ExecutionException e) {
-            throw new IOException("S3AccessIO: Failed to get aux objects for listing to delete.", e);
-        }
+        List<S3Object> storedAuxFilesSummary = listObjects(prefix, "deleteAllAuxObjects");
 
         if (storedAuxFilesSummary.isEmpty()) {
             logger.fine("S3AccessIO: No auxiliary objects to delete.");
@@ -1377,6 +1352,12 @@ else if (s3profile == null) {
     }
 
     public void removeTempTag() throws IOException {
+        final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId)
+                .orElse(false);
+        if (taggingDisabled) {
+            logger.fine("S3 tagging disabled for storage driver " + driverId + "; skipping temp tag removal.");
+            return;
+        }
         if (!(dvObject instanceof DataFile)) {
             logger.warning("Attempt to remove tag from non-file DVObject id: " + dvObject.getId());
             throw new IOException("Attempt to remove temp tag from non-file S3 Object");
@@ -1527,53 +1508,11 @@ private List<String> listAllFiles() throws IOException {
         }
         String prefix = dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/";
 
-        List<String> ret = new ArrayList<>();
-        ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
-                .build();
+        List<S3Object> contents = listObjects(prefix, "listAllFiles");
 
-        ListObjectsV2Response listObjectsResponse = null;
-        try {
-            listObjectsResponse = s3ReadClient.listObjectsV2(listObjectsReqManual).get();
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new IOException("S3 listObjects: failed to get a listing for " + prefix, e);
-        } catch (ExecutionException e) {
-            throw new IOException("S3 listObjects: failed to get a listing for " + prefix, e);
-        }
-
-        if (listObjectsResponse == null) {
-            return ret;
-        }
-
-        List<S3Object> storedFilesSummary = new ArrayList<>(listObjectsResponse.contents());
-
-        try {
-            String nextContinuationToken = listObjectsResponse.nextContinuationToken();
-            while (nextContinuationToken != null) {
-                logger.fine("S3 listObjects: going to next page of list");
-                ListObjectsV2Request nextReq = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
-                        .continuationToken(nextContinuationToken).build();
-
-                ListObjectsV2Response nextResponse = s3ReadClient.listObjectsV2(nextReq).get();
-                if (nextResponse != null) {
-                    storedFilesSummary.addAll(nextResponse.contents());
-                    nextContinuationToken = nextResponse.nextContinuationToken();
-                } else {
-                    nextContinuationToken = null;
-                }
-            }
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new IOException("S3AccessIO: Failed to get objects for listing.", e);
-        } catch (ExecutionException e) {
-            throw new IOException("S3AccessIO: Failed to get objects for listing.", e);
-        }
-
-        for (S3Object item : storedFilesSummary) {
-            String fileName = item.key().substring(prefix.length());
-            ret.add(fileName);
-        }
-        return ret;
+        return contents.stream()
+                .map(item -> item.key().substring(prefix.length()))
+                .collect(Collectors.toList());
     }
 
     private void deleteFile(String fileName) throws IOException {