fix: include directory paths in file searches

conf/solr/schema.xml

@@ -179,6 +179,7 @@
     <field name="variableNotes" type="text_en" stored="true" indexed="true" multiValued="true"/>
 
     <field name="fileDescription" type="text_en" stored="true" indexed="true" multiValued="false"/>
+    <field name="fileDirectoryLabel" type="text_general" stored="true" indexed="true" multiValued="false"/>
 
     <field name="fileTypeGroupFacet" type="string" stored="true" indexed="true" multiValued="false"/>
     <field name="fileTypeDisplay" type="string" stored="true" indexed="true" multiValued="false"/>
@@ -489,6 +490,7 @@
     <!-- Dataverse 4.0: we want the "filetype_en" and "filename_without_extension_en" field in the "catchall" per https://redmine.hmdc.harvard.edu/issues/3848  -->
     <copyField source="fileType" dest="_text_"/>
     <copyField source="fileNameWithoutExtension" dest="_text_"/>
+    <copyField source="fileDirectoryLabel" dest="_text_"/>
     <!-- <copyField source="manu" dest="_text_"/> -->
     <!-- <copyField source="features" dest="_text_"/> -->
     <!-- <copyField source="includes" dest="_text_"/> -->

doc/release-notes/frontend-1062-directory-search.md

@@ -0,0 +1,34 @@
+## Release Highlights
+
+### Search files by directory name
+
+You can now find files by directory name in dataset searches, site-wide searches,
+and the Search API.
+
+See [IQSS/dataverse-frontend#1062](https://github.com/IQSS/dataverse-frontend/issues/1062)
+and [PR #12685](https://github.com/IQSS/dataverse/pull/12685).
+
+## Upgrade Instructions
+
+1. Add the `fileDirectoryLabel` field and its `copyField` rule from
+   `conf/solr/schema.xml` to the active Solr core's `schema.xml`, keeping any
+   local customizations.
+   See [Directory Name Search](https://dataverse-guide--12685.org.readthedocs.build/en/12685/admin/solr-search-index.html#directory-name-search-index)
+   for details.
+
+2. Reload the Solr core before deploying the updated application. For the default
+   core name:
+
+   ```bash
+   curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"
+   ```
+
+3. After deploying the application, reindex existing files using
+   [Reindex in Place](https://dataverse-guide--12685.org.readthedocs.build/en/12685/admin/solr-search-index.html#reindex-in-place):
+
+   ```bash
+   curl -X DELETE http://localhost:8080/api/admin/index/timestamps
+   curl http://localhost:8080/api/admin/index/continue
+   ```
+
+No PostgreSQL schema migration is required.

doc/sphinx-guides/source/admin/solr-search-index.rst

@@ -6,6 +6,26 @@ A Dataverse installation requires Solr to be operational at all times. If you st
 .. contents:: Contents:
 	:local:
 
+.. _directory-name-search-index:
+
+Directory Name Search
+---------------------
+
+File directory paths are indexed in ``fileDirectoryLabel`` and copied into the ``_text_`` field used by basic search. When upgrading an installation that does not yet index directory paths, add the following definitions from ``conf/solr/schema.xml`` to the active core's ``schema.xml``, preserving any local metadata customizations:
+
+.. code-block:: xml
+
+    <field name="fileDirectoryLabel" type="text_general" stored="true" indexed="true" multiValued="false"/>
+    <copyField source="fileDirectoryLabel" dest="_text_"/>
+
+Reload the Solr core before deploying the application code that indexes directory paths. For the default core name:
+
+.. code-block:: bash
+
+    curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"
+
+After deploying the application, follow `Reindex in Place`_ below to index directory paths for existing files without clearing the search index. Reloading the schema alone does not add directory terms to existing documents; those files become searchable by directory name as they are reindexed. No PostgreSQL schema migration is required for this change.
+
 Full Reindex
 -------------
 

doc/sphinx-guides/source/api/changelog.rst

@@ -28,6 +28,7 @@ v6.12
 
 - Dataset creation API calls may now behave differently when neither a license nor terms are provided, depending on the new :ref:`dataverse.feature.do-not-assume-default-license` feature flag.
 - Whether file extensions are included in the "Content-disposition" header returned when downloading auxiliary files depends on whether the relevant format is one of the content types supported in Tika. A recent update to the version of Tika has added new content types, including "text/markdown", and auxiliary files with these types now have a a file extension included (e.g. ".md" in this case) in the header.
+- File text searches now also match directory paths, which can increase result counts and filtered download sizes. This affects ``searchText`` on dataset version ``files``, ``files/counts``, and ``downloadsize`` endpoints, as well as file results in the Search API. Existing installations must update their Solr schema and reindex existing files for Search API directory matches; see :ref:`directory-name-search-index`. Related request: `dataverse-frontend#1062 <https://github.com/IQSS/dataverse-frontend/issues/1062>`_.
 
 v6.11
 -----

doc/sphinx-guides/source/api/native-api.rst

@@ -2458,7 +2458,7 @@ Usage example:
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?contentType=image/png"
 
-Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the files that contain the text searched in one of such fields.
+Filtering by search text is also optionally supported. The search matches case-insensitive substrings in the labels, descriptions, and directory paths (``directoryLabel``) of the dataset files. Files matching any of these fields are returned, subject to the other requested filters.
 
 Usage example:
 
@@ -2562,7 +2562,7 @@ Usage example:
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?contentType=image/png"
 
-Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return counts only for files that contain the text searched in one of such fields.
+Filtering by search text is also optionally supported. The search matches case-insensitive substrings in the labels, descriptions, and directory paths (``directoryLabel``) of the dataset files. Only files matching the search and the other requested filters are counted.
 
 Usage example:
 
@@ -3621,7 +3621,7 @@ Usage example:
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?contentType=image/png"
 
-Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the size of all files available for download that contain the text searched in one of such fields.
+Filtering by search text is also optionally supported. The search matches case-insensitive substrings in the labels, descriptions, and directory paths (``directoryLabel``) of the dataset files. The returned size includes only files available for download that match the search and the other requested filters.
 
 Usage example:
 

doc/sphinx-guides/source/api/search.rst

@@ -8,6 +8,8 @@ The Search API supports the same searching, sorting, and faceting operations as
 
 To search unpublished content, you must pass in an API token as described in the :doc:`auth` section.
 
+File searches include names within directory paths. For example, ``q=Figure1&type=file`` can return files in both ``Figure1`` and ``results/Figure1``, even when their filenames and descriptions do not contain the query term. Directory names follow the existing search tokenization and case-insensitive matching rules; the results remain files, not folders.
+
 The parameters and JSON response are partly inspired by the `GitHub Search API <https://developer.github.com/v3/search/>`_.
 
 .. note:: |CORS| The search API can be used from scripts running in web browsers, as it allows cross-origin resource sharing (CORS).

doc/sphinx-guides/source/user/find-use-data.rst

@@ -16,6 +16,8 @@ Basic Search
 ------------
 You can search the entire contents of the Dataverse installation, including Dataverse collections, datasets, and files. You can access the search by clicking the "Search" button in the header of every page. The search bar accepts search terms, queries, or exact phrases (in quotations).
 
+File results can also match names in their directory paths. For example, searching for ``Figure1`` can find both ``Figure1/plot.txt`` and ``results/Figure1/nested.txt``, even when the filenames and descriptions do not contain ``Figure1``. The results are files, not separate folder entries.
+
 Sorting and Viewing Search Results
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -78,7 +80,7 @@ Files in a Dataverse installation each have their own landing page that can be r
 File Search within Datasets
 ---------------------------
 
-Datasets containing multiple files offer a file search function. On the Dataset page, under the Files tab, you'll see a search bar you can use to locate an individual file. It searches within the filename and file description. Performing a search will filter the file table to list only files matching your search. After you perform a search, if you'd like to return to the full list of files, just perform an empty search. 
+Datasets containing multiple files offer a file search function. On the Dataset page, under the Files tab, you'll see a search bar you can use to locate an individual file. It searches within the filename, file description, and directory path. Performing a search will filter the file table to list only files matching your search. After you perform a search, if you'd like to return to the full list of files, just perform an empty search.
 
 Under the search bar, you'll see file search facets you can use to filter the dataset's files by file type, access level, and file tags (see the example below). 
 

src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java

@@ -358,9 +358,10 @@ private Predicate createSearchCriteriaPredicate(DatasetVersion datasetVersion,
         }
         String searchText = searchCriteria.getSearchText();
         if (searchText != null && !searchText.isEmpty()) {
-            searchText = searchText.trim().toLowerCase();
-            predicates.add(criteriaBuilder.or(criteriaBuilder.like(criteriaBuilder.lower(fileMetadataRoot.get("label")), "%" + searchText + "%"),
-                    criteriaBuilder.like(criteriaBuilder.lower(fileMetadataRoot.get("description")), "%" + searchText + "%")));
+            String searchPattern = "%" + searchText.trim().toLowerCase() + "%";
+            predicates.add(criteriaBuilder.or(criteriaBuilder.like(criteriaBuilder.lower(fileMetadataRoot.get("label")), searchPattern),
+                    criteriaBuilder.like(criteriaBuilder.lower(fileMetadataRoot.get("description")), searchPattern),
+                    criteriaBuilder.like(criteriaBuilder.lower(fileMetadataRoot.get("directoryLabel")), searchPattern)));
         }
         return criteriaBuilder.and(predicates.toArray(new Predicate[]{}));
     }

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -1633,6 +1633,10 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                             }
                             filenameCompleteFinal = filenameComplete;
                         }
+                        String directoryLabel = fileMetadata.getDirectoryLabel();
+                        if (directoryLabel != null && !directoryLabel.isEmpty()) {
+                            datafileSolrInputDocument.addField(SearchFields.FILE_DIRECTORY_LABEL, directoryLabel);
+                        }
                         for (String tag : fileMetadata.getCategoriesByName()) {
                             datafileSolrInputDocument.addField(SearchFields.FILE_TAG, tag);
                             datafileSolrInputDocument.addField(SearchFields.FILE_TAG_SEARCHABLE, tag);

src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java

@@ -147,6 +147,7 @@ public class SearchFields {
     public static final String AFFILIATION = "affiliation_ss";
     public static final String FILE_NAME = "fileName";
     public static final String FILE_DESCRIPTION = "fileDescription";
+    public static final String FILE_DIRECTORY_LABEL = "fileDirectoryLabel";
     public static final String FILE_PERSISTENT_ID = "filePersistentId";
     /**
      * Can be multivalued and includes both "friendly" and "group" versions:

src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java

@@ -5327,6 +5327,105 @@ public void testDataCiteExport() throws IOException {
         }
     }
 
+    @Test
+    public void testFileDirectorySearch() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        try {
+            Response createDataverse = UtilIT.createRandomDataverse(apiToken);
+            createDataverse.then().assertThat().statusCode(CREATED.getStatusCode());
+            String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse);
+            try {
+                Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+                createDataset.then().assertThat().statusCode(CREATED.getStatusCode());
+                Integer datasetId = createDataset.jsonPath().getInt("data.id");
+                try {
+                    String textPath = "scripts/search/data/replace_test/003.txt";
+                    String imagePath = "src/test/resources/images/coffeeshop.png";
+                    Response uploadTopLevel = UtilIT.uploadFileViaNative(datasetId.toString(), textPath,
+                            JsonUtil.createObjectBuilder().add("directoryLabel", "Figure1").build(), apiToken);
+                    uploadTopLevel.then().assertThat().statusCode(OK.getStatusCode());
+                    int topLevelFileId = uploadTopLevel.jsonPath().getInt("data.files[0].dataFile.id");
+
+                    Response uploadNested = UtilIT.uploadFileViaNative(datasetId.toString(), imagePath,
+                            JsonUtil.createObjectBuilder().add("directoryLabel", "results/Figure1").build(), apiToken);
+                    uploadNested.then().assertThat().statusCode(OK.getStatusCode());
+                    int nestedFileId = uploadNested.jsonPath().getInt("data.files[0].dataFile.id");
+
+                    Response uploadRoot = UtilIT.uploadFileViaNative(datasetId.toString(),
+                            "scripts/search/data/replace_test/004.txt",
+                            JsonUtil.createObjectBuilder().add("description", "Distinct root description control").build(), apiToken);
+                    uploadRoot.then().assertThat().statusCode(OK.getStatusCode());
+                    int rootFileId = uploadRoot.jsonPath().getInt("data.files[0].dataFile.id");
+
+                    // A mixed-case substring matches both directory depths, but not the root file.
+                    String searchText = "iGuRe1";
+                    UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, 1, 0, null, null, null, null,
+                            searchText, null, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.dataFile.id", contains(topLevelFileId))
+                            .body("totalCount", equalTo(2));
+                    UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, 1, 1, null, null, null, null,
+                            searchText, null, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.dataFile.id", contains(nestedFileId))
+                            .body("totalCount", equalTo(2));
+
+                    UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST, null, null, null, null,
+                            searchText, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.total", equalTo(2))
+                            .body("data.perContentType", equalTo(Map.of("text/plain", 1, "image/png", 1)))
+                            .body("data.perAccessStatus", equalTo(Map.of(FileSearchCriteria.FileAccessStatus.Public.toString(), 2)));
+                    Response downloadSize = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST,
+                            null, null, null, null, searchText,
+                            DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), false, apiToken);
+                    downloadSize.then().assertThat().statusCode(OK.getStatusCode());
+                    assertEquals(Files.size(Paths.get(textPath)) + Files.size(Paths.get(imagePath)),
+                            downloadSize.jsonPath().getLong("data.storageSize"));
+
+                    // Content type remains an AND filter, excluding the image and the root text file.
+                    UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, "text/plain", null, null, null,
+                            searchText, null, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.dataFile.id", contains(topLevelFileId))
+                            .body("totalCount", equalTo(1));
+                    UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST, "text/plain", null, null, null,
+                            searchText, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.total", equalTo(1))
+                            .body("data.perContentType", equalTo(Map.of("text/plain", 1)))
+                            .body("data.perAccessStatus", equalTo(Map.of(FileSearchCriteria.FileAccessStatus.Public.toString(), 1)));
+
+                    // A missing directory does not prevent existing filename or description matches.
+                    UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null,
+                            "004", null, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.dataFile.id", contains(rootFileId))
+                            .body("totalCount", equalTo(1));
+                    UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null,
+                            "ROOT DESCRIPTION", null, false, apiToken).then().assertThat()
+                            .statusCode(OK.getStatusCode())
+                            .body("data.dataFile.id", contains(rootFileId))
+                            .body("totalCount", equalTo(1));
+
+                    // Directory matches must not expose files in an unpublished dataset to guests.
+                    UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null,
+                            searchText, null, false, null).then().assertThat()
+                            .statusCode(NOT_FOUND.getStatusCode());
+                } finally {
+                    UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+                }
+            } finally {
+                UtilIT.deleteDataverse(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+            }
+        } finally {
+            UtilIT.deleteUser(username).then().assertThat().statusCode(OK.getStatusCode());
+        }
+    }
+
     @Test
     public void getVersionFiles() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();