diff --git a/spring-batch-notion/README.md b/spring-batch-notion/README.md index 6be1b04..2af0cb5 100644 --- a/spring-batch-notion/README.md +++ b/spring-batch-notion/README.md @@ -32,24 +32,45 @@ implementation("org.springframework.batch.extensions:spring-batch-notion:${sprin The `NotionDatabaseItemReader` is a restartable `ItemReader` that reads entries from a [Notion Database] via a paging technique. +### Basic Usage (Automatic Data Source Discovery) + A minimal configuration of the item reader is as follows: ```java NotionDatabaseItemReader itemReader() { String token = System.getenv("NOTION_TOKEN"); - String databaseId = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"; // UUID + String databaseId = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"; // Database ID (UUID) PropertyMapper propertyMapper = new CustomPropertyMapper(); return new NotionDatabaseItemReader<>(token, databaseId, propertyMapper); } ``` -The following constructor parameters should be provided: +When `dataSourceId` is not provided as shown above, the reader automatically discovers it from the database. The reader will query the database metadata to retrieve the first available data source. + +### Advanced Usage (Manual Data Source Selection) + +If the given Notion database has multiple data sources, you can specify the data source ID directly: + +```java +NotionDatabaseItemReader itemReader() { + String token = System.getenv("NOTION_TOKEN"); + String databaseId = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"; // Database ID (UUID) + String dataSourceId = "YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY"; // Data Source ID (UUID) + PropertyMapper propertyMapper = new CustomPropertyMapper(); + return new NotionDatabaseItemReader<>(token, databaseId, dataSourceId, propertyMapper); +} +``` + +### Constructor Parameters + +The constructor accepts the following parameters: -| Property | Description | -|------------------|---------------------------------------------------------------------------------------------------------------------------| -| `token` | The Notion integration token. | -| `databaseId` | UUID of the database to read from. | -| `propertyMapper` | The `PropertyMapper` responsible for mapping properties of a Notion item into a Java object. | +| Property | Required | Description | +|------------------|----------|----------------------------------------------------------------------------------------------------------------------------| +| `token` | yes | The Notion integration token. | +| `databaseId` | yes | UUID of the database to read from. | +| `dataSourceId` | no | UUID of the data source to query. If not provided, the reader will automatically discover the first available data source. | +| `propertyMapper` | yes | The `PropertyMapper` responsible for mapping properties of a Notion item into a Java object. | and the following configuration options are available: diff --git a/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/DatabaseInfo.java b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/DatabaseInfo.java new file mode 100644 index 0000000..94493e8 --- /dev/null +++ b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/DatabaseInfo.java @@ -0,0 +1,33 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.extensions.notion; + +import tools.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import tools.jackson.databind.annotation.JsonNaming; + +import java.util.List; + +/** + * @author Stefano Cordio + */ +@JsonNaming(SnakeCaseStrategy.class) +record DatabaseInfo(String id, List dataSources) { + + @JsonNaming(SnakeCaseStrategy.class) + record DataSource(String id, String name) { + } + +} diff --git a/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDataSourceService.java b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDataSourceService.java new file mode 100644 index 0000000..17b5f49 --- /dev/null +++ b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDataSourceService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.extensions.notion; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.service.annotation.HttpExchange; +import org.springframework.web.service.annotation.PostExchange; + +@HttpExchange(url = "/data_sources", version = "2025-09-03", accept = MediaType.APPLICATION_JSON_VALUE) +interface NotionDataSourceService { + + @PostExchange("/{dataSourceId}/query") + QueryResult query(@PathVariable String dataSourceId, @RequestBody QueryRequest request); + +} diff --git a/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseItemReader.java b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseItemReader.java index 33916b6..4a01ce0 100644 --- a/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseItemReader.java +++ b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseItemReader.java @@ -71,7 +71,11 @@ public class NotionDatabaseItemReader extends AbstractPaginatedDataItemReader private Sort[] sorts = new Sort[0]; - private @Nullable NotionDatabaseService service; + private @Nullable NotionDatabaseService databaseService; + + private @Nullable NotionDataSourceService dataSourceService; + + private @Nullable String dataSourceId; private boolean hasMore; @@ -79,6 +83,9 @@ public class NotionDatabaseItemReader extends AbstractPaginatedDataItemReader /** * Create a new {@link NotionDatabaseItemReader}. + *

+ * This constructor automatically selects the first available data source from the + * database. * @param token the Notion integration token * @param databaseId UUID of the database to read from * @param propertyMapper the {@link PropertyMapper} responsible for mapping properties @@ -91,6 +98,27 @@ public NotionDatabaseItemReader(String token, String databaseId, PropertyMapper< this.pageSize = DEFAULT_PAGE_SIZE; } + /** + * Create a new {@link NotionDatabaseItemReader} with a specific data source ID. + *

+ * This constructor allows you to specify the data source ID directly, bypassing the + * automatic discovery. This is useful when working with databases that have multiple + * data sources. + * @param token the Notion integration token + * @param databaseId UUID of the database to read from + * @param dataSourceId UUID of the data source to read from + * @param propertyMapper the {@link PropertyMapper} responsible for mapping properties + * of a Notion item into a Java object + */ + public NotionDatabaseItemReader(String token, String databaseId, String dataSourceId, + PropertyMapper propertyMapper) { + this.token = Objects.requireNonNull(token); + this.databaseId = Objects.requireNonNull(databaseId); + this.dataSourceId = Objects.requireNonNull(dataSourceId); + this.propertyMapper = Objects.requireNonNull(propertyMapper); + this.pageSize = DEFAULT_PAGE_SIZE; + } + /** * The base URL of the Notion API. *

@@ -155,7 +183,14 @@ protected void doOpen() { RestClientAdapter adapter = RestClientAdapter.create(restClient); HttpServiceProxyFactory factory = HttpServiceProxyFactory.builderFor(adapter).build(); - service = factory.createClient(NotionDatabaseService.class); + + databaseService = factory.createClient(NotionDatabaseService.class); + dataSourceService = factory.createClient(NotionDataSourceService.class); + + if (dataSourceId == null) { + DatabaseInfo databaseInfo = databaseService.getDatabase(databaseId); + dataSourceId = databaseInfo.dataSources().get(0).id(); + } hasMore = true; } @@ -172,7 +207,7 @@ protected Iterator doPageRead() { QueryRequest request = new QueryRequest(pageSize, nextCursor, filter, sorts); @SuppressWarnings("DataFlowIssue") - QueryResult result = service.query(databaseId, request); + QueryResult result = dataSourceService.query(dataSourceId, request); hasMore = result.hasMore(); nextCursor = result.nextCursor(); diff --git a/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseService.java b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseService.java index e4c5372..ff869be 100644 --- a/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseService.java +++ b/spring-batch-notion/src/main/java/org/springframework/batch/extensions/notion/NotionDatabaseService.java @@ -17,14 +17,13 @@ import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.service.annotation.GetExchange; import org.springframework.web.service.annotation.HttpExchange; -import org.springframework.web.service.annotation.PostExchange; -@HttpExchange(url = "/databases", version = "2022-06-28", accept = MediaType.APPLICATION_JSON_VALUE) +@HttpExchange(url = "/databases", version = "2025-09-03", accept = MediaType.APPLICATION_JSON_VALUE) interface NotionDatabaseService { - @PostExchange("/{databaseId}/query") - QueryResult query(@PathVariable String databaseId, @RequestBody QueryRequest request); + @GetExchange("/{databaseId}") + DatabaseInfo getDatabase(@PathVariable String databaseId); } diff --git a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/RequestHeaders.java b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/RequestHeaders.java index 5cd5dd2..8657cc2 100644 --- a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/RequestHeaders.java +++ b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/RequestHeaders.java @@ -22,6 +22,6 @@ public class RequestHeaders { public static final String NOTION_VERSION = "Notion-Version"; - public static final String NOTION_VERSION_VALUE = "2022-06-28"; + public static final String NOTION_VERSION_VALUE = "2025-09-03"; } diff --git a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/ResponseBodies.java b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/ResponseBodies.java index c62e5d1..4bf9593 100644 --- a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/ResponseBodies.java +++ b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/ResponseBodies.java @@ -30,11 +30,46 @@ */ public class ResponseBodies { - public static String queryResponse(JSONObject... results) { - return queryResponse(null, results); + public static String databaseInfoResponse(UUID databaseId, UUID dataSourceId) { + try { + return new JSONObject() // + .put("object", "database") + .put("id", databaseId.toString()) + .put("data_sources", new JSONArray() // + .put(new JSONObject() // + .put("id", dataSourceId.toString()) + .put("name", "default"))) + .toString(); + } + catch (JSONException e) { + throw new RuntimeException(e); + } + } + + public static String databaseInfoResponse(UUID databaseId, UUID firstDataSourceId, UUID secondDataSourceId) { + try { + return new JSONObject() // + .put("object", "database") + .put("id", databaseId.toString()) + .put("data_sources", new JSONArray() // + .put(new JSONObject() // + .put("id", firstDataSourceId.toString()) + .put("name", "default")) + .put(new JSONObject() // + .put("id", secondDataSourceId.toString()) + .put("name", "secondary"))) + .toString(); + } + catch (JSONException e) { + throw new RuntimeException(e); + } + } + + public static String datasourceQueryResponse(JSONObject... results) { + return datasourceQueryResponse(null, results); } - public static String queryResponse(UUID nextCursor, JSONObject... results) { + public static String datasourceQueryResponse(UUID nextCursor, JSONObject... results) { try { return new JSONObject() // .put("object", "list") @@ -50,7 +85,7 @@ public static String queryResponse(UUID nextCursor, JSONObject... results) { } } - public static JSONObject result(UUID id, UUID databaseId, Map properties) { + public static JSONObject result(UUID id, UUID dataSourceId, Map properties) { try { Instant now = Instant.now(); @@ -62,8 +97,9 @@ public static JSONObject result(UUID id, UUID databaseId, Map properties) .put("created_by", new JSONObject()) .put("last_edited_by", new JSONObject()) .put("parent", new JSONObject() // - .put("type", "database_id") - .put("database_id", databaseId.toString())) + .put("type", "data_source_id") + .put("data_source_id", dataSourceId.toString()) + .put("database_id", randomUUID().toString())) .put("archived", false) .put("properties", new JSONObject(properties)) .put("url", "https://www.notion.so/" + randomUUID().toString().replace("-", "")); diff --git a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/datasource/AutomaticDataSourceDiscoveryTest.java b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/datasource/AutomaticDataSourceDiscoveryTest.java new file mode 100644 index 0000000..4af6ebc --- /dev/null +++ b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/datasource/AutomaticDataSourceDiscoveryTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.extensions.notion.it.datasource; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.springframework.batch.core.job.Job; +import org.springframework.batch.core.job.JobExecution; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.Step; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.extensions.notion.NotionDatabaseItemReader; +import org.springframework.batch.extensions.notion.it.IntegrationTest; +import org.springframework.batch.extensions.notion.it.datasource.AutomaticDataSourceDiscoveryTest.AutomaticDiscoveryJob.Item; +import org.springframework.batch.extensions.notion.mapping.RecordPropertyMapper; +import org.springframework.batch.infrastructure.item.support.ListItemWriter; +import org.springframework.batch.test.JobOperatorTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +import java.util.Map; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.common.ContentTypes.AUTHORIZATION; +import static com.github.tomakehurst.wiremock.common.ContentTypes.CONTENT_TYPE; +import static java.util.UUID.randomUUID; +import static org.assertj.core.api.BDDAssertions.then; +import static org.assertj.core.api.InstanceOfAssertFactories.LIST; +import static org.springframework.batch.core.ExitStatus.COMPLETED; +import static org.springframework.batch.extensions.notion.it.RequestBodies.queryRequest; +import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION; +import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION_VALUE; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.*; + +/** + * Tests for {@link NotionDatabaseItemReader} with automatic data source discovery. + * + * @author Roeniss Moon + */ +@IntegrationTest +class AutomaticDataSourceDiscoveryTest { + + private static final UUID DATABASE_ID = randomUUID(); + + private static final UUID FIRST_DATA_SOURCE_ID = randomUUID(); + + private static final UUID SECOND_DATA_SOURCE_ID = randomUUID(); + + private static final int PAGE_SIZE = 2; + + @Autowired + JobOperatorTestUtils jobOperator; + + @Autowired + ListItemWriter itemWriter; + + @Test + void should_use_firstly_returned_data_source_if_not_specified() throws Exception { + // GIVEN + JSONObject firstResult = result(randomUUID(), FIRST_DATA_SOURCE_ID, + Map.of("Name", title("From first source"), "Value", richText("First value"))); + + // Database API returns 2 data sources (1st: default, 2nd: secondary) + // Discovery should automatically use the 1st data source + givenThat(get("/databases/%s".formatted(DATABASE_ID)) // + .withHeader(AUTHORIZATION, matching("Bearer .+")) + .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) + .willReturn(okJson(databaseInfoResponse(DATABASE_ID, FIRST_DATA_SOURCE_ID, SECOND_DATA_SOURCE_ID)))); + + // Query should go to the 1st data source (discovered automatically) + givenThat(post("/data_sources/%s/query".formatted(FIRST_DATA_SOURCE_ID)) // + .withHeader(AUTHORIZATION, matching("Bearer .+")) + .withHeader(CONTENT_TYPE, containing("application/json")) + .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) + .withRequestBody(equalToJson(queryRequest(PAGE_SIZE))) + .willReturn(okJson(datasourceQueryResponse(firstResult)))); + + // WHEN + JobExecution jobExecution = jobOperator.startJob(); + + // THEN + then(jobExecution.getExitStatus()).isEqualTo(COMPLETED); + + then(itemWriter.getWrittenItems()).asInstanceOf(LIST) + .containsExactly(new Item("From first source", "First value")); + } + + @SpringBootApplication + static class AutomaticDiscoveryJob { + + @Value("${wiremock.server.baseUrl}") + private String wiremockBaseUrl; + + @Bean + Job job(JobRepository jobRepository, Step step) { + return new JobBuilder(jobRepository).start(step).build(); + } + + @Bean + Step step(JobRepository jobRepository) { + return new StepBuilder(jobRepository) // + .chunk(PAGE_SIZE) // + .reader(itemReader()) // + .writer(itemWriter()) // + .build(); + } + + @Bean + NotionDatabaseItemReader itemReader() { + // Use 3-parameter constructor - data source ID will be discovered + // automatically + NotionDatabaseItemReader reader = new NotionDatabaseItemReader<>("token", DATABASE_ID.toString(), + new RecordPropertyMapper<>()); + + reader.setSaveState(false); + reader.setBaseUrl(wiremockBaseUrl); + reader.setPageSize(PAGE_SIZE); + + return reader; + } + + @Bean + ListItemWriter itemWriter() { + return new ListItemWriter<>(); + } + + record Item(String name, String value) { + } + + } + +} diff --git a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/datasource/ManualDataSourceSelectionTests.java b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/datasource/ManualDataSourceSelectionTests.java new file mode 100644 index 0000000..5e00e9c --- /dev/null +++ b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/datasource/ManualDataSourceSelectionTests.java @@ -0,0 +1,154 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.batch.extensions.notion.it.datasource; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.springframework.batch.core.job.Job; +import org.springframework.batch.core.job.JobExecution; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.Step; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.extensions.notion.NotionDatabaseItemReader; +import org.springframework.batch.extensions.notion.it.IntegrationTest; +import org.springframework.batch.extensions.notion.it.datasource.ManualDataSourceSelectionTests.ManualDataSourceJob.Item; +import org.springframework.batch.extensions.notion.mapping.RecordPropertyMapper; +import org.springframework.batch.infrastructure.item.support.ListItemWriter; +import org.springframework.batch.test.JobOperatorTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +import java.util.Map; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.containing; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.givenThat; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.common.ContentTypes.AUTHORIZATION; +import static com.github.tomakehurst.wiremock.common.ContentTypes.CONTENT_TYPE; +import static java.util.UUID.randomUUID; +import static org.assertj.core.api.BDDAssertions.then; +import static org.assertj.core.api.InstanceOfAssertFactories.LIST; +import static org.springframework.batch.core.ExitStatus.COMPLETED; +import static org.springframework.batch.extensions.notion.it.RequestBodies.queryRequest; +import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION; +import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION_VALUE; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.datasourceQueryResponse; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.result; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.richText; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.title; + +/** + * Tests for {@link NotionDatabaseItemReader} with manual data source ID specification. + * + * @author Roeniss Moon + */ +@IntegrationTest +class ManualDataSourceSelectionTests { + + private static final UUID DATABASE_ID = randomUUID(); + + private static final UUID FIRST_DATA_SOURCE_ID = randomUUID(); + + private static final UUID SECOND_DATA_SOURCE_ID = randomUUID(); + + private static final int PAGE_SIZE = 2; + + @Autowired + JobOperatorTestUtils jobOperator; + + @Autowired + ListItemWriter itemWriter; + + @Test + void should_use_manually_specified_data_source_id() throws Exception { + // GIVEN + JSONObject firstResult = result(randomUUID(), SECOND_DATA_SOURCE_ID, + Map.of("Name", title("From second source"), "Value", richText("Second value"))); + + // No GET /databases/{id} stub - discovery should be bypassed when dataSourceId is + // provided. + // If discovery is incorrectly called, the test will fail with 404. + + // Query should go directly to the 2nd data source (bypassing discovery) + givenThat(post("/data_sources/%s/query".formatted(SECOND_DATA_SOURCE_ID)) // + .withHeader(AUTHORIZATION, matching("Bearer .+")) + .withHeader(CONTENT_TYPE, containing("application/json")) + .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) + .withRequestBody(equalToJson(queryRequest(PAGE_SIZE))) + .willReturn(okJson(datasourceQueryResponse(firstResult)))); + + // WHEN + JobExecution jobExecution = jobOperator.startJob(); + + // THEN + then(jobExecution.getExitStatus()).isEqualTo(COMPLETED); + + then(itemWriter.getWrittenItems()).asInstanceOf(LIST) + .containsExactly(new Item("From second source", "Second value")); + } + + @SpringBootApplication + static class ManualDataSourceJob { + + @Value("${wiremock.server.baseUrl}") + private String wiremockBaseUrl; + + @Bean + Job job(JobRepository jobRepository, Step step) { + return new JobBuilder(jobRepository).start(step).build(); + } + + @Bean + Step step(JobRepository jobRepository) { + return new StepBuilder(jobRepository) // + .chunk(PAGE_SIZE) // + .reader(itemReader()) // + .writer(itemWriter()) // + .build(); + } + + @Bean + NotionDatabaseItemReader itemReader() { + // Use 4-parameter constructor with manual data source ID (2nd source) + NotionDatabaseItemReader reader = new NotionDatabaseItemReader<>("token", DATABASE_ID.toString(), + SECOND_DATA_SOURCE_ID.toString(), new RecordPropertyMapper<>()); + + reader.setSaveState(false); + reader.setBaseUrl(wiremockBaseUrl); + reader.setPageSize(PAGE_SIZE); + + return reader; + } + + @Bean + ListItemWriter itemWriter() { + return new ListItemWriter<>(); + } + + record Item(String name, String value) { + } + + } + +} diff --git a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesDescendingTests.java b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesDescendingTests.java index 63ebf64..7df5df2 100644 --- a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesDescendingTests.java +++ b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesDescendingTests.java @@ -41,6 +41,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.containing; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.givenThat; import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; @@ -56,7 +57,8 @@ import static org.springframework.batch.extensions.notion.it.RequestBodies.sortByProperty; import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION; import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION_VALUE; -import static org.springframework.batch.extensions.notion.it.ResponseBodies.queryResponse; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.databaseInfoResponse; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.datasourceQueryResponse; import static org.springframework.batch.extensions.notion.it.ResponseBodies.result; import static org.springframework.batch.extensions.notion.it.ResponseBodies.richText; import static org.springframework.batch.extensions.notion.it.ResponseBodies.title; @@ -69,6 +71,8 @@ class MultiplePagesDescendingTests { private static final UUID DATABASE_ID = randomUUID(); + private static final UUID DATA_SOURCE_ID = randomUUID(); + private static final int PAGE_SIZE = 2; @Autowired @@ -82,26 +86,31 @@ void should_succeed() throws Exception { // GIVEN UUID thirdResultId = randomUUID(); - JSONObject firstResult = result(randomUUID(), DATABASE_ID, + JSONObject firstResult = result(randomUUID(), DATA_SOURCE_ID, Map.of("Name", title("Name string"), "Value", richText("123456"))); - JSONObject secondResult = result(randomUUID(), DATABASE_ID, + JSONObject secondResult = result(randomUUID(), DATA_SOURCE_ID, Map.of("Name", title("Another name string"), "Value", richText("0987654321"))); - JSONObject thirdResult = result(thirdResultId, DATABASE_ID, + JSONObject thirdResult = result(thirdResultId, DATA_SOURCE_ID, Map.of("Name", title(""), "Value", richText("abc-1234"))); - givenThat(post("/databases/%s/query".formatted(DATABASE_ID)) // + givenThat(get("/databases/%s".formatted(DATABASE_ID)) // + .withHeader(AUTHORIZATION, matching("Bearer .+")) + .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) + .willReturn(okJson(databaseInfoResponse(DATABASE_ID, DATA_SOURCE_ID)))); + + givenThat(post("/data_sources/%s/query".formatted(DATA_SOURCE_ID)) // .withHeader(AUTHORIZATION, matching("Bearer .+")) .withHeader(CONTENT_TYPE, containing("application/json")) .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) .withRequestBody(equalToJson(queryRequest(PAGE_SIZE, sortByProperty("Name", DESCENDING)))) - .willReturn(okJson(queryResponse(thirdResultId, firstResult, secondResult)))); + .willReturn(okJson(datasourceQueryResponse(thirdResultId, firstResult, secondResult)))); - givenThat(post("/databases/%s/query".formatted(DATABASE_ID)) // + givenThat(post("/data_sources/%s/query".formatted(DATA_SOURCE_ID)) // .withHeader(AUTHORIZATION, matching("Bearer .+")) .withHeader(CONTENT_TYPE, containing("application/json")) .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) .withRequestBody(equalToJson(queryRequest(thirdResultId, PAGE_SIZE, sortByProperty("Name", DESCENDING)))) - .willReturn(okJson(queryResponse(thirdResult)))); + .willReturn(okJson(datasourceQueryResponse(thirdResult)))); // WHEN JobExecution jobExecution = jobOperator.startJob(); diff --git a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesTests.java b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesTests.java index c381fab..8839564 100644 --- a/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesTests.java +++ b/spring-batch-notion/src/test/java/org/springframework/batch/extensions/notion/it/pagination/MultiplePagesTests.java @@ -16,7 +16,6 @@ package org.springframework.batch.extensions.notion.it.pagination; import org.json.JSONObject; -import org.jspecify.annotations.NonNull; import org.junit.jupiter.api.Test; import org.springframework.batch.core.job.Job; import org.springframework.batch.core.job.JobExecution; @@ -41,6 +40,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.containing; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.givenThat; import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; @@ -54,7 +54,8 @@ import static org.springframework.batch.extensions.notion.it.RequestBodies.queryRequest; import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION; import static org.springframework.batch.extensions.notion.it.RequestHeaders.NOTION_VERSION_VALUE; -import static org.springframework.batch.extensions.notion.it.ResponseBodies.queryResponse; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.databaseInfoResponse; +import static org.springframework.batch.extensions.notion.it.ResponseBodies.datasourceQueryResponse; import static org.springframework.batch.extensions.notion.it.ResponseBodies.result; import static org.springframework.batch.extensions.notion.it.ResponseBodies.richText; import static org.springframework.batch.extensions.notion.it.ResponseBodies.title; @@ -67,6 +68,8 @@ class MultiplePagesTests { private static final UUID DATABASE_ID = randomUUID(); + private static final UUID DATA_SOURCE_ID = randomUUID(); + private static final int PAGE_SIZE = 2; @Autowired @@ -80,26 +83,31 @@ void should_succeed() throws Exception { // GIVEN UUID thirdResultId = randomUUID(); - JSONObject firstResult = result(randomUUID(), DATABASE_ID, + JSONObject firstResult = result(randomUUID(), DATA_SOURCE_ID, Map.of("Name", title("Another name string"), "Value", richText("0987654321"))); - JSONObject secondResult = result(randomUUID(), DATABASE_ID, + JSONObject secondResult = result(randomUUID(), DATA_SOURCE_ID, Map.of("Name", title("Name string"), "Value", richText("123456"))); - JSONObject thirdResult = result(thirdResultId, DATABASE_ID, + JSONObject thirdResult = result(thirdResultId, DATA_SOURCE_ID, Map.of("Name", title(""), "Value", richText("abc-1234"))); - givenThat(post("/databases/%s/query".formatted(DATABASE_ID)) // + givenThat(get("/databases/%s".formatted(DATABASE_ID)) // + .withHeader(AUTHORIZATION, matching("Bearer .+")) + .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) + .willReturn(okJson(databaseInfoResponse(DATABASE_ID, DATA_SOURCE_ID)))); + + givenThat(post("/data_sources/%s/query".formatted(DATA_SOURCE_ID)) // .withHeader(AUTHORIZATION, matching("Bearer .+")) .withHeader(CONTENT_TYPE, containing("application/json")) .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) .withRequestBody(equalToJson(queryRequest(PAGE_SIZE))) - .willReturn(okJson(queryResponse(thirdResultId, firstResult, secondResult)))); + .willReturn(okJson(datasourceQueryResponse(thirdResultId, firstResult, secondResult)))); - givenThat(post("/databases/%s/query".formatted(DATABASE_ID)) // + givenThat(post("/data_sources/%s/query".formatted(DATA_SOURCE_ID)) // .withHeader(AUTHORIZATION, matching("Bearer .+")) .withHeader(CONTENT_TYPE, containing("application/json")) .withHeader(NOTION_VERSION, equalTo(NOTION_VERSION_VALUE)) .withRequestBody(equalToJson(queryRequest(thirdResultId, PAGE_SIZE))) - .willReturn(okJson(queryResponse(thirdResult)))); + .willReturn(okJson(datasourceQueryResponse(thirdResult)))); // WHEN JobExecution jobExecution = jobOperator.startJob();