Brief datacite xml

doc/release-notes/12662-smaller-datacite-xml.md

@@ -0,0 +1,4 @@
+Introduces a new JVM-option `dataverse.datacite.xml.datafile-info` that allows you to configure the number of `size` and `format` elements included in the
+DataCite XML metadata. Valid options are `expanded` (default, the current behavior of including one `size` and one `format` element per datafile), `brief`
+(one `size` element with the sum of all datafile sizes and one `format` element per unique datafile format), and `none` (no `size` or `format` elements included
+in the DataCite XML metadata). See #12662 for details.

doc/sphinx-guides/source/installation/config.rst

@@ -848,6 +848,25 @@ Here are the configuration options for PermaLinks:
 
 You must restart Payara after making changes to these settings.
 
+.. _datacite-settings:
+
+DataCite Settings
+-----------------
+
+.. _dataverse.datacite.xml.datafile-info:
+
+dataverse.datacite.xml.datafile-info
++++++++++++++++++++++++++++++++++++++
+
+``dataverse.datacite.xml.datafile-info`` controls how DataCite XML for datasets represents
+datafile sizes and formats. The default value, ``expanded``, preserves the current behavior and emits
+one ``size`` and one ``format`` element per datafile. Set the value to ``brief`` to emit a single
+``size`` element with the sum of all known datafile sizes and one ``format`` element per distinct
+datafile format. Set the value to ``none`` to omit datafile ``size`` and ``format`` elements entirely.
+This setting applies to both the DataCite metadata export and to DataCite XML sent by the PID provider,
+such as when publishing a new dataset version or explicitly updating PID-provider metadata for a
+published dataset.
+
 .. _auth-modes:
 
 Auth Modes: Local vs. Remote vs. Both

src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java

@@ -138,10 +138,10 @@
                 }
             }
         }
         String protocol = settingsService.getValueForKey(SettingsServiceBean.Key.Protocol);
         String authority = settingsService.getValueForKey(SettingsServiceBean.Key.Authority);
         String shoulder = settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder);
         String provider = settingsService.getValueForKey(SettingsServiceBean.Key.DoiProvider);
 
         if (protocol != null && authority != null && shoulder != null && provider != null) {
             logger.warning("Found legacy settings: " + protocol + " " + authority + " " + shoulder + " " + provider
@@ -153,8 +153,8 @@
                 PidProvider legacy = null;
                 // Try to add a legacy provider
                 String identifierGenerationStyle = settingsService
                         .getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "random");
                 String dataFilePidFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat,
                         "DEPENDENT");
                 switch (protocol) {
                 case "doi":
@@ -174,10 +174,11 @@
                         // Defaults for testing where no account is set up
                         String dcUsername = JvmSettings.LEGACY_DATACITE_USERNAME.lookup();
                         String dcPassword = JvmSettings.LEGACY_DATACITE_PASSWORD.lookup();
+                        String datafileInfoMode = JvmSettings.DATACITE_XML_DATAFILE_INFO.lookupOptional().orElse("expanded");
                         if (mdsUrl != null && restUrl != null && dcUsername != null && dcPassword != null) {
                             legacy = new DataCiteDOIProvider("legacy", "legacy", authority, shoulder,
                                     identifierGenerationStyle, dataFilePidFormat, "", "", mdsUrl, restUrl, dcUsername,
-                                    dcPassword);
+                                    dcPassword, datafileInfoMode);
                         }
                         break;
                     case "FAKE":
@@ -192,8 +193,8 @@
                     String path = JvmSettings.LEGACY_HANDLENET_KEY_PATH.lookup();
                     String passphrase = JvmSettings.LEGACY_HANDLENET_KEY_PASSPHRASE.lookup();
                     boolean independentHandleService = settingsService
                             .isTrueForKey(SettingsServiceBean.Key.IndependentHandleService, false);
                     String handleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle);
 
                     legacy = new HandlePidProvider("legacy", "legacy", authority, shoulder, identifierGenerationStyle,
                             dataFilePidFormat, "", "", index, independentHandleService, handleAuthHandle, path,

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java

@@ -16,6 +16,7 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -58,13 +59,43 @@ public class XmlMetadataTemplate {
     public static final String XML_XSI = "http://www.w3.org/2001/XMLSchema-instance";
     public static final String XML_SCHEMA_VERSION = "4.7";
 
+    public enum DatafileInfoMode {
+        EXPANDED("expanded"),
+        BRIEF("brief"),
+        NONE("none");
+
+        private final String value;
+
+        DatafileInfoMode(String value) {
+            this.value = value;
+        }
+
+        public static DatafileInfoMode from(String value) {
+            if (value != null) {
+                for (DatafileInfoMode mode : values()) {
+                    if (mode.value.equalsIgnoreCase(value.trim())) {
+                        return mode;
+                    }
+                }
+                logger.warning("Unknown DataCite datafile info mode '" + value + "', using expanded");
+            }
+            return EXPANDED;
+        }
+    }
+
     private DoiMetadata doiMetadata;
+    private DatafileInfoMode datafileInfoMode = DatafileInfoMode.EXPANDED;
 
     public XmlMetadataTemplate() {
     }
 
     public XmlMetadataTemplate(DoiMetadata doiMetadata) {
+        this(doiMetadata, DatafileInfoMode.EXPANDED);
+    }
+
+    public XmlMetadataTemplate(DoiMetadata doiMetadata, DatafileInfoMode datafileInfoMode) {
         this.doiMetadata = doiMetadata;
+        this.datafileInfoMode = datafileInfoMode;
     }
 
     public String generateXML(DvObject dvObject) {
@@ -1192,19 +1223,35 @@ private static String getCanonicalPublicationType(String pubIdType) {
     private void writeSize(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException {
         // sizes -> size
         boolean sizesWritten = false;
-        List<DataFile> dataFiles = new ArrayList<DataFile>();
-
-        if (dvObject instanceof Dataset dataset) {
-            dataFiles = dataset.getFiles();
-        } else if (dvObject instanceof DataFile df) {
-            dataFiles.add(df);
+        if (datafileInfoMode == DatafileInfoMode.NONE) {
+            return;
+        }
+        List<DataFile> dataFiles = getDataFiles(dvObject);
+        if (datafileInfoMode == DatafileInfoMode.BRIEF) {
+            long totalSize = 0L;
+            boolean hasKnownSize = false;
+            if (dataFiles != null && !dataFiles.isEmpty()) {
+                for (DataFile dataFile : dataFiles) {
+                    long size = dataFile.getFilesize();
+                    if (size != -1) {
+                        totalSize += size;
+                        hasKnownSize = true;
+                    }
+                }
+            }
+            if (hasKnownSize) {
+                xmlw.writeStartElement("sizes");
+                XmlWriterUtil.writeFullElement(xmlw, "size", Long.toString(totalSize));
+                xmlw.writeEndElement();
+            }
+            return;
         }
         if (dataFiles != null && !dataFiles.isEmpty()) {
             for (DataFile dataFile : dataFiles) {
-                Long size = dataFile.getFilesize();
+                long size = dataFile.getFilesize();
                 if (size != -1) {
                     sizesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "sizes", sizesWritten);
-                    XmlWriterUtil.writeFullElement(xmlw, "size", size.toString());
+                    XmlWriterUtil.writeFullElement(xmlw, "size", Long.toString(size));
                 }
             }
         }
@@ -1217,12 +1264,28 @@ private void writeSize(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStream
     private void writeFormats(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException {
 
         boolean formatsWritten = false;
-        List<DataFile> dataFiles = new ArrayList<DataFile>();
-
-        if (dvObject instanceof Dataset dataset) {
-            dataFiles = dataset.getFiles();
-        } else if (dvObject instanceof DataFile df) {
-            dataFiles.add(df);
+        if (datafileInfoMode == DatafileInfoMode.NONE) {
+            return;
+        }
+        List<DataFile> dataFiles = getDataFiles(dvObject);
+        if (datafileInfoMode == DatafileInfoMode.BRIEF) {
+            Set<String> uniqueFormats = new LinkedHashSet<>();
+            if (dataFiles != null && !dataFiles.isEmpty()) {
+                for (DataFile dataFile : dataFiles) {
+                    String format = dataFile.getContentType();
+                    if (StringUtils.isNotBlank(format)) {
+                        uniqueFormats.add(format);
+                    }
+                }
+            }
+            if (!uniqueFormats.isEmpty()) {
+                xmlw.writeStartElement("formats");
+                for (String format : uniqueFormats) {
+                    XmlWriterUtil.writeFullElement(xmlw, "format", format);
+                }
+                xmlw.writeEndElement();
+            }
+            return;
         }
         if (dataFiles != null && !dataFiles.isEmpty()) {
             for (DataFile dataFile : dataFiles) {
@@ -1245,6 +1308,16 @@ private void writeFormats(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStr
 
     }
 
+    private List<DataFile> getDataFiles(DvObject dvObject) {
+        List<DataFile> dataFiles = new ArrayList<>();
+        if (dvObject instanceof Dataset dataset) {
+            return dataset.getFiles() != null ? dataset.getFiles() : dataFiles;
+        } else if (dvObject instanceof DataFile df) {
+            dataFiles.add(df);
+        }
+        return dataFiles;
+    }
+
     private void writeVersion(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException {
         Dataset d = null;
         if (dvObject instanceof Dataset) {

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java

@@ -23,6 +23,8 @@
 import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider;
 import edu.harvard.iq.dataverse.pidproviders.doi.DoiMetadata;
 import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate;
+import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate.DatafileInfoMode;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 
 import org.xmlunit.builder.DiffBuilder;
 import org.xmlunit.builder.Input;
@@ -58,17 +60,27 @@ public DOIDataCiteRegisterService(String url, String restApiUrl, String username
      * https://support.datacite.org/docs/mds-api-guide#doi-states
      */
     public String reserveIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
+        return reserveIdentifier(identifier, metadata, dvObject, getConfiguredDatafileInfoMode());
+    }
+
+    public String reserveIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject,
+            DatafileInfoMode datafileInfoMode) throws IOException {
         String retString = "";
-        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
+        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject, datafileInfoMode);
 
         retString = client.postMetadata(xmlMetadata);
 
         return retString;
     }
 
     public String registerIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
+        return registerIdentifier(identifier, metadata, dvObject, getConfiguredDatafileInfoMode());
+    }
+
+    public String registerIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject,
+            DatafileInfoMode datafileInfoMode) throws IOException {
         String retString = "";
-        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
+        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject, datafileInfoMode);
         String target = metadata.get("_target");
 
         retString = client.postMetadata(xmlMetadata);
@@ -79,10 +91,15 @@ public String registerIdentifier(String identifier, Map<String, String> metadata
 
 
     public String reRegisterIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
+        return reRegisterIdentifier(identifier, metadata, dvObject, getConfiguredDatafileInfoMode());
+    }
+
+    public String reRegisterIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject,
+            DatafileInfoMode datafileInfoMode) throws IOException {
         String retString = "";
         // "bare identifier" is the canonical pid with the "doi:" prefix stripped
         String bareIdentifier = identifier.substring(identifier.indexOf(":") + 1);
-        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
+        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject, datafileInfoMode);
         String target = metadata.get("_target");
         String currentMetadata = null;
         boolean hasDifferences = false;
@@ -131,8 +148,20 @@ public String deactivateIdentifier(String identifier, Map<String, String> metada
 
         return retString;
     }
-
-        public static String getMetadataFromDvObject(String identifier, Map<String, String> metadata, DvObject dvObject) {
+
+    /**
+     * Backward-compatibility fallback for callers that still use no-arg overloads.
+     */
+    private static DatafileInfoMode getConfiguredDatafileInfoMode() {
+        return DatafileInfoMode.from(JvmSettings.DATACITE_XML_DATAFILE_INFO.lookupOptional().orElse("expanded"));
+    }
+
+    public static String getMetadataFromDvObject(String identifier, Map<String, String> metadata, DvObject dvObject) {
+        return getMetadataFromDvObject(identifier, metadata, dvObject, getConfiguredDatafileInfoMode());
+    }
+
+    public static String getMetadataFromDvObject(String identifier, Map<String, String> metadata, DvObject dvObject,
+            DatafileInfoMode datafileInfoMode) {
 
         Dataset dataset = null;
 
@@ -182,7 +211,7 @@ public static String getMetadataFromDvObject(String identifier, Map<String, Stri
         doiMetadata.setPublisher(producerString);
         doiMetadata.setPublisherYear(metadata.get("datacite.publicationyear"));
 
-        String xmlMetadata = new XmlMetadataTemplate(doiMetadata).generateXML(dvObject);
+        String xmlMetadata = new XmlMetadataTemplate(doiMetadata, datafileInfoMode).generateXML(dvObject);
         logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata);
         return xmlMetadata;
     }
@@ -196,10 +225,8 @@ public static String getMetadataForDeactivateIdentifier(String identifier, Map<S
 
         doiMetadata.setDescription(AbstractPidProvider.UNAVAILABLE);
 
-        String title =metadata.get("datacite.title");
-
-        System.out.print("Map metadata title: "+ metadata.get("datacite.title"));
-
+        String title = metadata.get("datacite.title");
+
         doiMetadata.setAuthors(null);
 
         doiMetadata.setTitle(title);
@@ -215,8 +242,13 @@ public static String getMetadataForDeactivateIdentifier(String identifier, Map<S
 
     public String modifyIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject)
             throws IOException {
+        return modifyIdentifier(identifier, metadata, dvObject, getConfiguredDatafileInfoMode());
+    }
+
+    public String modifyIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject,
+            DatafileInfoMode datafileInfoMode) throws IOException {
 
-        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
+        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject, datafileInfoMode);
 
         logger.fine("XML to send to DataCite: " + xmlMetadata);
 

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java

@@ -15,6 +15,7 @@
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider;
+import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate;
 import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import jakarta.json.JsonObject;
@@ -47,18 +48,20 @@ public class DataCiteDOIProvider extends AbstractDOIProvider {
     private String apiUrl;
     private String username;
     private String password;
+    private XmlMetadataTemplate.DatafileInfoMode datafileInfoMode;
 
     private DOIDataCiteRegisterService doiDataCiteRegisterService;
 
     public DataCiteDOIProvider(String id, String label, String providerAuthority, String providerShoulder,
             String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList,
-            String mdsUrl, String apiUrl, String username, String password) {
+            String mdsUrl, String apiUrl, String username, String password, String datafileInfoMode) {
         super(id, label, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList,
                 excludedList);
         this.mdsUrl = mdsUrl;
         this.apiUrl = apiUrl;
         this.username = username;
         this.password = password;
+        this.datafileInfoMode = XmlMetadataTemplate.DatafileInfoMode.from(datafileInfoMode);
         doiDataCiteRegisterService = new DOIDataCiteRegisterService(mdsUrl, apiUrl, username, password);
     }
 
@@ -95,7 +98,7 @@ public String createIdentifier(DvObject dvObject) throws Exception {
         Map<String, String> metadata = getMetadataForCreateIndicator(dvObject);
         metadata.put("_status", DRAFT);
         try {
-            String retString = doiDataCiteRegisterService.reserveIdentifier(identifier, metadata, dvObject);
+            String retString = doiDataCiteRegisterService.reserveIdentifier(identifier, metadata, dvObject, datafileInfoMode);
             logger.log(Level.FINE, "create DOI identifier retString : " + retString);
             return retString;
         } catch (Exception e) {
@@ -132,7 +135,7 @@ public String modifyIdentifierTargetURL(DvObject dvObject) throws Exception {
         try {
             Map<String, String> metadata = getIdentifierMetadata(dvObject);
             metadata.put("_target", getTargetUrl(dvObject));
-            doiDataCiteRegisterService.modifyIdentifier(identifier, metadata, dvObject);
+            doiDataCiteRegisterService.modifyIdentifier(identifier, metadata, dvObject, datafileInfoMode);
         } catch (Exception e) {
             logger.log(Level.WARNING, "modifyMetadata failed", e);
             throw e;
@@ -219,9 +222,9 @@ public boolean publicizeIdentifier(DvObject dvObject) {
         metadata.put("_target", getTargetUrl(dvObject));
         try {
             if (FeatureFlags.ONLY_UPDATE_DATACITE_WHEN_NEEDED.enabled()) {
-                doiDataCiteRegisterService.reRegisterIdentifier(identifier, metadata, dvObject);
+                doiDataCiteRegisterService.reRegisterIdentifier(identifier, metadata, dvObject, datafileInfoMode);
             } else {
-                doiDataCiteRegisterService.registerIdentifier(identifier, metadata, dvObject);
+                doiDataCiteRegisterService.registerIdentifier(identifier, metadata, dvObject, datafileInfoMode);
             }
             return true;
         } catch (Exception e) {

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteProviderFactory.java

@@ -31,9 +31,10 @@ public PidProvider createPidProvider(String providerId) {
         String apiUrl = JvmSettings.DATACITE_REST_API_URL.lookupOptional(providerId).orElse("https://api.test.datacite.org");
         String username = JvmSettings.DATACITE_USERNAME.lookup(providerId);
         String password = JvmSettings.DATACITE_PASSWORD.lookup(providerId);
+        String datafileInfoMode = JvmSettings.DATACITE_XML_DATAFILE_INFO.lookupOptional().orElse("expanded");
 
         return new DataCiteDOIProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle,
-                datafilePidFormat, managedList, excludedList, mdsUrl, apiUrl, username, password);
+                datafilePidFormat, managedList, excludedList, mdsUrl, apiUrl, username, password, datafileInfoMode);
     }
 
     public String getType() {

src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java

@@ -117,6 +117,10 @@ public enum JvmSettings {
     MDB_SYSTEM_METADATA_KEYS(SCOPE_METADATA, "block-system-metadata-keys"),
     MDB_SYSTEM_KEY_FOR(MDB_SYSTEM_METADATA_KEYS),
 
+    // DATACITE SETTINGS (applies to both DataCite PID provider and DataCite metadata export)
+    SCOPE_DATACITE(PREFIX, "datacite"),
+    DATACITE_XML_DATAFILE_INFO(SCOPE_DATACITE, "xml.datafile-info"),
+
     // PERSISTENT IDENTIFIER SETTINGS
     SCOPE_PID(PREFIX, "pid"),
     PID_PROVIDERS(SCOPE_PID, "providers"),

src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java

@@ -498,7 +498,7 @@ public void testLegacyConfig() throws IOException {
                   if (mdsUrl != null && restUrl != null && dcUsername != null && dcPassword != null) {
                       legacy = new DataCiteDOIProvider("legacy", "legacy", authority, shoulder,
                               identifierGenerationStyle, dataFilePidFormat, "", "", mdsUrl, restUrl, dcUsername,
-                              dcPassword);
+                              dcPassword, "expanded");
                   }
                   break;
               case "FAKE":