From af6b086957b676e231711948a940b6d7a26da14b Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 4 Sep 2026 07:44:36 +0000 Subject: [PATCH 1/2] test(spanner): clean up stale backup schedules older than 3 hours ListBackupSchedulesSampleIT and other backup schedule sample tests intermittently fail with FAILED_PRECONDITION: "Reached maximum limit of 4 schedules per database" on the shared 'mysample' database. This occurs when prior builds or aborted runs leak backup schedules that accumulate over time. Because test runs and slow-test suites can take up to ~3 hours, an unconditional deletion of all pre-existing schedules risks deleting schedules currently in use by parallel Kokoro jobs or concurrent Failsafe test forks. Add cleanUpStaleBackupSchedules in SampleTestBaseV2 to inspect each schedule's update_time and delete only schedules that are more than 3 hours old. Wire this into @Before for all backup schedule sample tests to safely reclaim schedule slots from dead runs without disrupting concurrent test executions. --- .../CreateFullBackupScheduleSampleIT.java | 6 ++++ ...eateIncrementalBackupScheduleSampleIT.java | 6 ++++ .../spanner/DeleteBackupScheduleSampleIT.java | 6 ++++ .../spanner/GetBackupScheduleSampleIT.java | 6 ++++ .../spanner/ListBackupSchedulesSampleIT.java | 6 ++++ .../com/example/spanner/SampleTestBaseV2.java | 36 +++++++++++++++++++ .../spanner/UpdateBackupScheduleSampleIT.java | 6 ++++ 7 files changed, 72 insertions(+) diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java index 82b67c4dfe85..6838bd4876dd 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java @@ -20,6 +20,7 @@ import com.google.spanner.admin.database.v1.BackupScheduleName; import java.util.UUID; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +30,11 @@ public class CreateFullBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + @Before + public void cleanUpPreExistingBackupSchedules() { + cleanUpStaleBackupSchedules(instanceId, databaseId); + } + @Test public void testCreateFullBackupScheduleSample() throws Exception { String backupScheduleId = String.format("schedule-%s", UUID.randomUUID()); diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java index ac38150f5203..4353975118bb 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java @@ -20,6 +20,7 @@ import com.google.spanner.admin.database.v1.BackupScheduleName; import java.util.UUID; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +30,11 @@ public class CreateIncrementalBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + @Before + public void cleanUpPreExistingBackupSchedules() { + cleanUpStaleBackupSchedules(multiRegionalInstanceId, databaseId); + } + @Test public void testCreateIncrementalBackupScheduleSample() throws Exception { String backupScheduleId = String.format("schedule-%s", UUID.randomUUID()); diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java index f379c9bfe47d..3efd3b1002be 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java @@ -20,6 +20,7 @@ import com.google.spanner.admin.database.v1.BackupScheduleName; import java.util.UUID; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +30,11 @@ public class DeleteBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + @Before + public void cleanUpPreExistingBackupSchedules() { + cleanUpStaleBackupSchedules(instanceId, databaseId); + } + @Test public void testDeleteBackupScheduleSample() throws Exception { String backupScheduleId = String.format("schedule-%s", UUID.randomUUID()); diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java index 366093cddd5e..d008a848c201 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java @@ -20,6 +20,7 @@ import com.google.spanner.admin.database.v1.BackupScheduleName; import java.util.UUID; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +30,11 @@ public class GetBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + @Before + public void cleanUpPreExistingBackupSchedules() { + cleanUpStaleBackupSchedules(instanceId, databaseId); + } + @Test public void testGetBackupScheduleSample() throws Exception { String backupScheduleId = String.format("schedule-%s", UUID.randomUUID()); diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java index 97feb2d7e9ba..dc59fad50b7f 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java @@ -20,6 +20,7 @@ import com.google.spanner.admin.database.v1.BackupScheduleName; import java.util.UUID; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +30,11 @@ public class ListBackupSchedulesSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + @Before + public void cleanUpPreExistingBackupSchedules() { + cleanUpStaleBackupSchedules(instanceId, databaseId); + } + @Test public void testListBackupSchedulesSample() throws Exception { String backupScheduleId1 = String.format("schedule-%s", UUID.randomUUID()); diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java index 5a43261269d9..08a59969e95c 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java @@ -23,8 +23,10 @@ import com.google.cloud.spanner.admin.database.v1.DatabaseAdminSettings; import com.google.cloud.spanner.admin.instance.v1.InstanceAdminClient; import com.google.cloud.spanner.admin.instance.v1.InstanceAdminSettings; +import com.google.spanner.admin.database.v1.BackupSchedule; import com.google.spanner.admin.database.v1.DatabaseDialect; import java.io.IOException; +import java.time.Instant; import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -148,6 +150,40 @@ public static void afterClass() throws InterruptedException { instanceAdminClient.awaitTermination(AWAIT_TERMINATION_SECONDS, TimeUnit.SECONDS); } + protected static void cleanUpStaleBackupSchedules( + final String targetInstanceId, final String targetDatabaseId) { + try { + final long nowSeconds = Instant.now().getEpochSecond(); + final long staleThresholdSeconds = TimeUnit.HOURS.toSeconds(3); + + for (BackupSchedule schedule : + databaseAdminClient + .listBackupSchedules(getDatabaseName(projectId, targetInstanceId, targetDatabaseId)) + .iterateAll()) { + if (schedule.hasUpdateTime()) { + long ageSeconds = nowSeconds - schedule.getUpdateTime().getSeconds(); + if (ageSeconds > staleThresholdSeconds) { + try { + databaseAdminClient.deleteBackupSchedule(schedule.getName()); + } catch (Exception e) { + System.out.println( + "Failed to delete stale backup schedule " + + schedule.getName() + + " due to " + + e.getMessage() + + ", skipping..."); + } + } + } + } + } catch (Exception e) { + System.out.println( + "Failed to list or clean up stale backup schedules due to " + + e.getMessage() + + ", skipping..."); + } + } + static String getDatabaseName(final String projectId, final String instanceId, final String databaseId) { return String.format( diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java index eff037459f91..d7fb82bf9b26 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java @@ -20,6 +20,7 @@ import com.google.spanner.admin.database.v1.BackupScheduleName; import java.util.UUID; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +30,11 @@ public class UpdateBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + @Before + public void cleanUpPreExistingBackupSchedules() { + cleanUpStaleBackupSchedules(instanceId, databaseId); + } + @Test public void testUpdateBackupScheduleSample() throws Exception { String backupScheduleId = String.format("schedule-%s", UUID.randomUUID()); From 32d801c6936f7f9561d95bb70b214d90275b82ba Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 4 Sep 2026 10:44:38 +0000 Subject: [PATCH 2/2] test(spanner): support per-job sample databases to prevent schedule quota collisions Parameterize spanner.sample.database in pom.xml and SampleTestBaseV2 to allow targeting different databases across parallel Kokoro jobs (e.g., mysample_java8, mysample_java11). Auto-detect the job name from environment variables when running in CI to eliminate concurrent backup schedule limit collisions on the shared 'mysample' database. --- java-spanner/samples/snippets/pom.xml | 5 +- .../CreateFullBackupScheduleSampleIT.java | 2 +- ...eateIncrementalBackupScheduleSampleIT.java | 2 +- .../spanner/DeleteBackupScheduleSampleIT.java | 2 +- .../spanner/GetBackupScheduleSampleIT.java | 2 +- .../spanner/ListBackupSchedulesSampleIT.java | 2 +- .../com/example/spanner/SampleTestBaseV2.java | 37 ++++++++- .../example/spanner/SampleTestBaseV2Test.java | 78 +++++++++++++++++++ .../spanner/UpdateBackupScheduleSampleIT.java | 2 +- 9 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2Test.java diff --git a/java-spanner/samples/snippets/pom.xml b/java-spanner/samples/snippets/pom.xml index 4f41d624282d..e7fcde94aa85 100644 --- a/java-spanner/samples/snippets/pom.xml +++ b/java-spanner/samples/snippets/pom.xml @@ -25,6 +25,7 @@ 1.8 UTF-8 0.31.1 + mysample @@ -137,7 +138,7 @@ us-east1 cmek-test-key-ring cmek-test-key - mysample + ${spanner.sample.database} quick-db @@ -166,7 +167,7 @@ us-east1 cmek-test-key-ring cmek-test-key - mysample + ${spanner.sample.database} quick-db diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java index 6838bd4876dd..4f4a5cfcbb53 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateFullBackupScheduleSampleIT.java @@ -28,7 +28,7 @@ @RunWith(JUnit4.class) public class CreateFullBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. - private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + private static String databaseId = getSampleDatabase(); @Before public void cleanUpPreExistingBackupSchedules() { diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java index 4353975118bb..61af70935f59 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/CreateIncrementalBackupScheduleSampleIT.java @@ -28,7 +28,7 @@ @RunWith(JUnit4.class) public class CreateIncrementalBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. - private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + private static String databaseId = getSampleDatabase(); @Before public void cleanUpPreExistingBackupSchedules() { diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java index 3efd3b1002be..8d89754d7c5e 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/DeleteBackupScheduleSampleIT.java @@ -28,7 +28,7 @@ @RunWith(JUnit4.class) public class DeleteBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. - private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + private static String databaseId = getSampleDatabase(); @Before public void cleanUpPreExistingBackupSchedules() { diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java index d008a848c201..a7c523b8bd5a 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/GetBackupScheduleSampleIT.java @@ -28,7 +28,7 @@ @RunWith(JUnit4.class) public class GetBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. - private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + private static String databaseId = getSampleDatabase(); @Before public void cleanUpPreExistingBackupSchedules() { diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java index dc59fad50b7f..eca11a2ed813 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/ListBackupSchedulesSampleIT.java @@ -28,7 +28,7 @@ @RunWith(JUnit4.class) public class ListBackupSchedulesSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. - private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + private static String databaseId = getSampleDatabase(); @Before public void cleanUpPreExistingBackupSchedules() { diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java index 08a59969e95c..5c9bcddcbeb3 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2.java @@ -39,12 +39,45 @@ public class SampleTestBaseV2 { private static final String BASE_INSTANCE_ID = System.getProperty("spanner.sample.instance", "mysample-instance"); - private static final String BASE_DATABASE_ID = - System.getProperty("spanner.sample.database", "sampledb"); + private static final String BASE_DATABASE_ID = getSampleDatabase(); private static final String BASE_BACKUP_ID = "samplebk"; private static final String BASE_INSTANCE_CONFIG_ID = "sampleconfig"; private static final int AWAIT_TERMINATION_SECONDS = 10; + static String resolveSampleDatabase(String sysProp, String envDb, String kokoroJob) { + if (sysProp != null && !sysProp.trim().isEmpty() && !sysProp.equals("mysample")) { + return sysProp.trim(); + } + if (envDb != null && !envDb.trim().isEmpty()) { + return envDb.trim(); + } + if (kokoroJob != null) { + if (kokoroJob.contains("java8-samples")) { + return "mysample_java8"; + } else if (kokoroJob.contains("java11-samples")) { + return "mysample_java11"; + } + } + if (sysProp != null && !sysProp.trim().isEmpty()) { + return sysProp.trim(); + } + return "mysample"; + } + + public static String getSampleDatabase() { + String kokoroJob = System.getenv("KOKORO_JOB_NAME"); + if (kokoroJob == null || kokoroJob.isEmpty()) { + kokoroJob = System.getenv("JOB_NAME"); + } + if (kokoroJob == null || kokoroJob.isEmpty()) { + kokoroJob = System.getenv("JOB_TYPE"); + } + return resolveSampleDatabase( + System.getProperty("spanner.sample.database"), + System.getenv("SPANNER_SAMPLE_DATABASE"), + kokoroJob); + } + protected static String projectId; protected static final String instanceId = System.getProperty("spanner.test.instance"); protected static DatabaseAdminClient databaseAdminClient; diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2Test.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2Test.java new file mode 100644 index 000000000000..5eab66f50f99 --- /dev/null +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/SampleTestBaseV2Test.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SampleTestBaseV2Test { + + @Test + public void testDefaultFallbackToMySample() { + assertThat(SampleTestBaseV2.resolveSampleDatabase(null, null, null)).isEqualTo("mysample"); + assertThat(SampleTestBaseV2.resolveSampleDatabase("", "", "")).isEqualTo("mysample"); + } + + @Test + public void testDefaultPropertyReturnsMySample() { + assertThat(SampleTestBaseV2.resolveSampleDatabase("mysample", null, null)).isEqualTo("mysample"); + } + + @Test + public void testKokoroJava8SamplesAutoDetection() { + assertThat( + SampleTestBaseV2.resolveSampleDatabase( + "mysample", null, "cloud-devrel/client-libraries/java/java-spanner/nightly/java8-samples")) + .isEqualTo("mysample_java8"); + } + + @Test + public void testKokoroJava11SamplesAutoDetection() { + assertThat( + SampleTestBaseV2.resolveSampleDatabase( + "mysample", null, "cloud-devrel/client-libraries/java/java-spanner/nightly/java11-samples")) + .isEqualTo("mysample_java11"); + } + + @Test + public void testKokoroSamplesJobFallback() { + assertThat( + SampleTestBaseV2.resolveSampleDatabase( + "mysample", null, "cloud-devrel/client-libraries/java/java-spanner/nightly/samples")) + .isEqualTo("mysample"); + } + + @Test + public void testExplicitSystemPropertyOverrideTakesPrecedence() { + assertThat( + SampleTestBaseV2.resolveSampleDatabase( + "custom-db", null, "cloud-devrel/client-libraries/java/java-spanner/nightly/java8-samples")) + .isEqualTo("custom-db"); + } + + @Test + public void testExplicitEnvVarOverrideTakesPrecedence() { + assertThat( + SampleTestBaseV2.resolveSampleDatabase( + "mysample", "env-custom-db", "cloud-devrel/client-libraries/java/java-spanner/nightly/java8-samples")) + .isEqualTo("env-custom-db"); + } +} diff --git a/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java b/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java index d7fb82bf9b26..39bae31a2b0b 100644 --- a/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java +++ b/java-spanner/samples/snippets/src/test/java/com/example/spanner/UpdateBackupScheduleSampleIT.java @@ -28,7 +28,7 @@ @RunWith(JUnit4.class) public class UpdateBackupScheduleSampleIT extends SampleTestBaseV2 { // Default instance and given db should exist for tests to pass. - private static String databaseId = System.getProperty("spanner.sample.database", "mysample"); + private static String databaseId = getSampleDatabase(); @Before public void cleanUpPreExistingBackupSchedules() {